Coverage Report

Created: 2023-05-31 04:38

/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/CodeGen/TargetInfo.cpp
Line
Count
Source (jump to first uncovered line)
1
//===---- TargetInfo.cpp - Encapsulate target details -----------*- C++ -*-===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// These classes wrap the information about a call or function
10
// definition used to handle ABI compliancy.
11
//
12
//===----------------------------------------------------------------------===//
13
14
#include "TargetInfo.h"
15
#include "ABIInfo.h"
16
#include "CGBlocks.h"
17
#include "CGCXXABI.h"
18
#include "CGValue.h"
19
#include "CodeGenFunction.h"
20
#include "clang/AST/Attr.h"
21
#include "clang/AST/RecordLayout.h"
22
#include "clang/Basic/Builtins.h"
23
#include "clang/Basic/CodeGenOptions.h"
24
#include "clang/Basic/DiagnosticFrontend.h"
25
#include "clang/CodeGen/CGFunctionInfo.h"
26
#include "llvm/ADT/SmallBitVector.h"
27
#include "llvm/ADT/StringExtras.h"
28
#include "llvm/ADT/StringSwitch.h"
29
#include "llvm/ADT/Twine.h"
30
#include "llvm/IR/DataLayout.h"
31
#include "llvm/IR/IntrinsicsNVPTX.h"
32
#include "llvm/IR/IntrinsicsS390.h"
33
#include "llvm/IR/Type.h"
34
#include "llvm/Support/MathExtras.h"
35
#include "llvm/Support/raw_ostream.h"
36
#include "llvm/TargetParser/RISCVTargetParser.h"
37
#include "llvm/TargetParser/Triple.h"
38
#include <algorithm>
39
40
using namespace clang;
41
using namespace CodeGen;
42
43
// Helper for coercing an aggregate argument or return value into an integer
44
// array of the same size (including padding) and alignment.  This alternate
45
// coercion happens only for the RenderScript ABI and can be removed after
46
// runtimes that rely on it are no longer supported.
47
//
48
// RenderScript assumes that the size of the argument / return value in the IR
49
// is the same as the size of the corresponding qualified type. This helper
50
// coerces the aggregate type into an array of the same size (including
51
// padding).  This coercion is used in lieu of expansion of struct members or
52
// other canonical coercions that return a coerced-type of larger size.
53
//
54
// Ty          - The argument / return value type
55
// Context     - The associated ASTContext
56
// LLVMContext - The associated LLVMContext
57
static ABIArgInfo coerceToIntArray(QualType Ty,
58
                                   ASTContext &Context,
59
19
                                   llvm::LLVMContext &LLVMContext) {
60
  // Alignment and Size are measured in bits.
61
19
  const uint64_t Size = Context.getTypeSize(Ty);
62
19
  const uint64_t Alignment = Context.getTypeAlign(Ty);
63
19
  llvm::Type *IntType = llvm::Type::getIntNTy(LLVMContext, Alignment);
64
19
  const uint64_t NumElements = (Size + Alignment - 1) / Alignment;
65
19
  return ABIArgInfo::getDirect(llvm::ArrayType::get(IntType, NumElements));
66
19
}
67
68
static void AssignToArrayRange(CodeGen::CGBuilderTy &Builder,
69
                               llvm::Value *Array,
70
                               llvm::Value *Value,
71
                               unsigned FirstIndex,
72
31
                               unsigned LastIndex) {
73
  // Alternatively, we could emit this as a loop in the source.
74
572
  for (unsigned I = FirstIndex; I <= LastIndex; 
++I541
) {
75
541
    llvm::Value *Cell =
76
541
        Builder.CreateConstInBoundsGEP1_32(Builder.getInt8Ty(), Array, I);
77
541
    Builder.CreateAlignedStore(Value, Cell, CharUnits::One());
78
541
  }
79
31
}
80
81
187k
static bool isAggregateTypeForABI(QualType T) {
82
187k
  return !CodeGenFunction::hasScalarEvaluationKind(T) ||
83
187k
         
T->isMemberFunctionPointerType()178k
;
84
187k
}
85
86
ABIArgInfo ABIInfo::getNaturalAlignIndirect(QualType Ty, bool ByVal,
87
                                            bool Realign,
88
4.13k
                                            llvm::Type *Padding) const {
89
4.13k
  return ABIArgInfo::getIndirect(getContext().getTypeAlignInChars(Ty), ByVal,
90
4.13k
                                 Realign, Padding);
91
4.13k
}
92
93
ABIArgInfo
94
25
ABIInfo::getNaturalAlignIndirectInReg(QualType Ty, bool Realign) const {
95
25
  return ABIArgInfo::getIndirectInReg(getContext().getTypeAlignInChars(Ty),
96
25
                                      /*ByVal*/ false, Realign);
97
25
}
98
99
Address ABIInfo::EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr,
100
0
                             QualType Ty) const {
101
0
  return Address::invalid();
102
0
}
103
104
6
static llvm::Type *getVAListElementType(CodeGenFunction &CGF) {
105
6
  return CGF.ConvertTypeForMem(
106
6
      CGF.getContext().getBuiltinVaListType()->getPointeeType());
107
6
}
108
109
264k
bool ABIInfo::isPromotableIntegerTypeForABI(QualType Ty) const {
110
264k
  if (getContext().isPromotableIntegerType(Ty))
111
12.9k
    return true;
112
113
251k
  if (const auto *EIT = Ty->getAs<BitIntType>())
114
339
    if (EIT->getNumBits() < getContext().getTypeSize(getContext().IntTy))
115
132
      return true;
116
117
251k
  return false;
118
251k
}
119
120
33.3k
ABIInfo::~ABIInfo() = default;
121
122
31.6k
SwiftABIInfo::~SwiftABIInfo() = default;
123
124
/// Does the given lowering require more than the given number of
125
/// registers when expanded?
126
///
127
/// This is intended to be the basis of a reasonable basic implementation
128
/// of should{Pass,Return}Indirectly.
129
///
130
/// For most targets, a limit of four total registers is reasonable; this
131
/// limits the amount of code required in order to move around the value
132
/// in case it wasn't produced immediately prior to the call by the caller
133
/// (or wasn't produced in exactly the right registers) or isn't used
134
/// immediately within the callee.  But some targets may need to further
135
/// limit the register count due to an inability to support that many
136
/// return registers.
137
bool SwiftABIInfo::occupiesMoreThan(ArrayRef<llvm::Type *> scalarTypes,
138
1.05k
                                    unsigned maxAllRegisters) const {
139
1.05k
  unsigned intCount = 0, fpCount = 0;
140
4.18k
  for (llvm::Type *type : scalarTypes) {
141
4.18k
    if (type->isPointerTy()) {
142
0
      intCount++;
143
4.18k
    } else if (auto intTy = dyn_cast<llvm::IntegerType>(type)) {
144
2.30k
      auto ptrWidth = CGT.getTarget().getPointerWidth(LangAS::Default);
145
2.30k
      intCount += (intTy->getBitWidth() + ptrWidth - 1) / ptrWidth;
146
2.30k
    } else {
147
1.88k
      assert(type->isVectorTy() || type->isFloatingPointTy());
148
1.88k
      fpCount++;
149
1.88k
    }
150
4.18k
  }
151
152
1.05k
  return (intCount + fpCount > maxAllRegisters);
153
1.05k
}
154
155
bool SwiftABIInfo::shouldPassIndirectly(ArrayRef<llvm::Type *> ComponentTys,
156
1.05k
                                        bool AsReturnValue) const {
157
1.05k
  return occupiesMoreThan(ComponentTys, /*total=*/4);
158
1.05k
}
159
160
bool SwiftABIInfo::isLegalVectorType(CharUnits VectorSize, llvm::Type *EltTy,
161
440
                                     unsigned NumElts) const {
162
  // The default implementation of this assumes that the target guarantees
163
  // 128-bit SIMD support but nothing more.
164
440
  return (VectorSize.getQuantity() > 8 && 
VectorSize.getQuantity() <= 16432
);
165
440
}
166
167
static CGCXXABI::RecordArgABI getRecordArgABI(const RecordType *RT,
168
26.7k
                                              CGCXXABI &CXXABI) {
169
26.7k
  const CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(RT->getDecl());
170
26.7k
  if (!RD) {
171
5.37k
    if (!RT->getDecl()->canPassInRegisters())
172
16
      return CGCXXABI::RAA_Indirect;
173
5.35k
    return CGCXXABI::RAA_Default;
174
5.37k
  }
175
21.3k
  return CXXABI.getRecordArgABI(RD);
176
26.7k
}
177
178
static CGCXXABI::RecordArgABI getRecordArgABI(QualType T,
179
12.9k
                                              CGCXXABI &CXXABI) {
180
12.9k
  const RecordType *RT = T->getAs<RecordType>();
181
12.9k
  if (!RT)
182
4.46k
    return CGCXXABI::RAA_Default;
183
8.44k
  return getRecordArgABI(RT, CXXABI);
184
12.9k
}
185
186
static bool classifyReturnType(const CGCXXABI &CXXABI, CGFunctionInfo &FI,
187
194k
                               const ABIInfo &Info) {
188
194k
  QualType Ty = FI.getReturnType();
189
190
194k
  if (const auto *RT = Ty->getAs<RecordType>())
191
6.94k
    if (!isa<CXXRecordDecl>(RT->getDecl()) &&
192
6.94k
        
!RT->getDecl()->canPassInRegisters()1.26k
) {
193
12
      FI.getReturnInfo() = Info.getNaturalAlignIndirect(Ty);
194
12
      return true;
195
12
    }
196
197
194k
  return CXXABI.classifyReturnType(FI);
198
194k
}
199
200
/// Pass transparent unions as if they were the type of the first element. Sema
201
/// should ensure that all elements of the union have the same "machine type".
202
398k
static QualType useFirstFieldIfTransparentUnion(QualType Ty) {
203
398k
  if (const RecordType *UT = Ty->getAsUnionType()) {
204
191
    const RecordDecl *UD = UT->getDecl();
205
191
    if (UD->hasAttr<TransparentUnionAttr>()) {
206
22
      assert(!UD->field_empty() && "sema created an empty transparent union");
207
22
      return UD->field_begin()->getType();
208
22
    }
209
191
  }
210
398k
  return Ty;
211
398k
}
212
213
251k
CGCXXABI &ABIInfo::getCXXABI() const {
214
251k
  return CGT.getCXXABI();
215
251k
}
216
217
758k
ASTContext &ABIInfo::getContext() const {
218
758k
  return CGT.getContext();
219
758k
}
220
221
18.5k
llvm::LLVMContext &ABIInfo::getVMContext() const {
222
18.5k
  return CGT.getLLVMContext();
223
18.5k
}
224
225
28.7k
const llvm::DataLayout &ABIInfo::getDataLayout() const {
226
28.7k
  return CGT.getDataLayout();
227
28.7k
}
228
229
35.2k
const TargetInfo &ABIInfo::getTarget() const {
230
35.2k
  return CGT.getTarget();
231
35.2k
}
232
233
3.13k
const CodeGenOptions &ABIInfo::getCodeGenOpts() const {
234
3.13k
  return CGT.getCodeGenOpts();
235
3.13k
}
236
237
5.07k
bool ABIInfo::isAndroid() const { return getTarget().getTriple().isAndroid(); }
238
239
49
bool ABIInfo::isOHOSFamily() const {
240
49
  return getTarget().getTriple().isOHOSFamily();
241
49
}
242
243
0
bool ABIInfo::isHomogeneousAggregateBaseType(QualType Ty) const {
244
0
  return false;
245
0
}
246
247
bool ABIInfo::isHomogeneousAggregateSmallEnough(const Type *Base,
248
0
                                                uint64_t Members) const {
249
0
  return false;
250
0
}
251
252
825
bool ABIInfo::isZeroLengthBitfieldPermittedInHomogeneousAggregate() const {
253
  // For compatibility with GCC, ignore empty bitfields in C++ mode.
254
825
  return getContext().getLangOpts().CPlusPlus;
255
825
}
256
257
0
LLVM_DUMP_METHOD void ABIArgInfo::dump() const {
258
0
  raw_ostream &OS = llvm::errs();
259
0
  OS << "(ABIArgInfo Kind=";
260
0
  switch (TheKind) {
261
0
  case Direct:
262
0
    OS << "Direct Type=";
263
0
    if (llvm::Type *Ty = getCoerceToType())
264
0
      Ty->print(OS);
265
0
    else
266
0
      OS << "null";
267
0
    break;
268
0
  case Extend:
269
0
    OS << "Extend";
270
0
    break;
271
0
  case Ignore:
272
0
    OS << "Ignore";
273
0
    break;
274
0
  case InAlloca:
275
0
    OS << "InAlloca Offset=" << getInAllocaFieldIndex();
276
0
    break;
277
0
  case Indirect:
278
0
    OS << "Indirect Align=" << getIndirectAlign().getQuantity()
279
0
       << " ByVal=" << getIndirectByVal()
280
0
       << " Realign=" << getIndirectRealign();
281
0
    break;
282
0
  case IndirectAliased:
283
0
    OS << "Indirect Align=" << getIndirectAlign().getQuantity()
284
0
       << " AadrSpace=" << getIndirectAddrSpace()
285
0
       << " Realign=" << getIndirectRealign();
286
0
    break;
287
0
  case Expand:
288
0
    OS << "Expand";
289
0
    break;
290
0
  case CoerceAndExpand:
291
0
    OS << "CoerceAndExpand Type=";
292
0
    getCoerceAndExpandType()->print(OS);
293
0
    break;
294
0
  }
295
0
  OS << ")\n";
296
0
}
297
298
// Dynamically round a pointer up to a multiple of the given alignment.
299
static llvm::Value *emitRoundPointerUpToAlignment(CodeGenFunction &CGF,
300
                                                  llvm::Value *Ptr,
301
124
                                                  CharUnits Align) {
302
  // OverflowArgArea = (OverflowArgArea + Align - 1) & -Align;
303
124
  llvm::Value *RoundUp = CGF.Builder.CreateConstInBoundsGEP1_32(
304
124
      CGF.Builder.getInt8Ty(), Ptr, Align.getQuantity() - 1);
305
124
  return CGF.Builder.CreateIntrinsic(
306
124
      llvm::Intrinsic::ptrmask, {CGF.AllocaInt8PtrTy, CGF.IntPtrTy},
307
124
      {RoundUp, llvm::ConstantInt::get(CGF.IntPtrTy, -Align.getQuantity())},
308
124
      nullptr, Ptr->getName() + ".aligned");
309
124
}
310
311
/// Emit va_arg for a platform using the common void* representation,
312
/// where arguments are simply emitted in an array of slots on the stack.
313
///
314
/// This version implements the core direct-value passing rules.
315
///
316
/// \param SlotSize - The size and alignment of a stack slot.
317
///   Each argument will be allocated to a multiple of this number of
318
///   slots, and all the slots will be aligned to this value.
319
/// \param AllowHigherAlign - The slot alignment is not a cap;
320
///   an argument type with an alignment greater than the slot size
321
///   will be emitted on a higher-alignment address, potentially
322
///   leaving one or more empty slots behind as padding.  If this
323
///   is false, the returned address might be less-aligned than
324
///   DirectAlign.
325
/// \param ForceRightAdjust - Default is false. On big-endian platform and
326
///   if the argument is smaller than a slot, set this flag will force
327
///   right-adjust the argument in its slot irrespective of the type.
328
static Address emitVoidPtrDirectVAArg(CodeGenFunction &CGF,
329
                                      Address VAListAddr,
330
                                      llvm::Type *DirectTy,
331
                                      CharUnits DirectSize,
332
                                      CharUnits DirectAlign,
333
                                      CharUnits SlotSize,
334
                                      bool AllowHigherAlign,
335
332
                                      bool ForceRightAdjust = false) {
336
  // Cast the element type to i8* if necessary.  Some platforms define
337
  // va_list as a struct containing an i8* instead of just an i8*.
338
332
  if (VAListAddr.getElementType() != CGF.Int8PtrTy)
339
59
    VAListAddr = CGF.Builder.CreateElementBitCast(VAListAddr, CGF.Int8PtrTy);
340
341
332
  llvm::Value *Ptr = CGF.Builder.CreateLoad(VAListAddr, "argp.cur");
342
343
  // If the CC aligns values higher than the slot size, do so if needed.
344
332
  Address Addr = Address::invalid();
345
332
  if (AllowHigherAlign && 
DirectAlign > SlotSize291
) {
346
103
    Addr = Address(emitRoundPointerUpToAlignment(CGF, Ptr, DirectAlign),
347
103
                   CGF.Int8Ty, DirectAlign);
348
229
  } else {
349
229
    Addr = Address(Ptr, CGF.Int8Ty, SlotSize);
350
229
  }
351
352
  // Advance the pointer past the argument, then store that back.
353
332
  CharUnits FullDirectSize = DirectSize.alignTo(SlotSize);
354
332
  Address NextPtr =
355
332
      CGF.Builder.CreateConstInBoundsByteGEP(Addr, FullDirectSize, "argp.next");
356
332
  CGF.Builder.CreateStore(NextPtr.getPointer(), VAListAddr);
357
358
  // If the argument is smaller than a slot, and this is a big-endian
359
  // target, the argument will be right-adjusted in its slot.
360
332
  if (DirectSize < SlotSize && 
CGF.CGM.getDataLayout().isBigEndian()44
&&
361
332
      
(8
!DirectTy->isStructTy()8
||
ForceRightAdjust8
)) {
362
2
    Addr = CGF.Builder.CreateConstInBoundsByteGEP(Addr, SlotSize - DirectSize);
363
2
  }
364
365
332
  Addr = CGF.Builder.CreateElementBitCast(Addr, DirectTy);
366
332
  return Addr;
367
332
}
368
369
/// Emit va_arg for a platform using the common void* representation,
370
/// where arguments are simply emitted in an array of slots on the stack.
371
///
372
/// \param IsIndirect - Values of this type are passed indirectly.
373
/// \param ValueInfo - The size and alignment of this type, generally
374
///   computed with getContext().getTypeInfoInChars(ValueTy).
375
/// \param SlotSizeAndAlign - The size and alignment of a stack slot.
376
///   Each argument will be allocated to a multiple of this number of
377
///   slots, and all the slots will be aligned to this value.
378
/// \param AllowHigherAlign - The slot alignment is not a cap;
379
///   an argument type with an alignment greater than the slot size
380
///   will be emitted on a higher-alignment address, potentially
381
///   leaving one or more empty slots behind as padding.
382
/// \param ForceRightAdjust - Default is false. On big-endian platform and
383
///   if the argument is smaller than a slot, set this flag will force
384
///   right-adjust the argument in its slot irrespective of the type.
385
static Address emitVoidPtrVAArg(CodeGenFunction &CGF, Address VAListAddr,
386
                                QualType ValueTy, bool IsIndirect,
387
                                TypeInfoChars ValueInfo,
388
                                CharUnits SlotSizeAndAlign,
389
                                bool AllowHigherAlign,
390
332
                                bool ForceRightAdjust = false) {
391
  // The size and alignment of the value that was passed directly.
392
332
  CharUnits DirectSize, DirectAlign;
393
332
  if (IsIndirect) {
394
44
    DirectSize = CGF.getPointerSize();
395
44
    DirectAlign = CGF.getPointerAlign();
396
288
  } else {
397
288
    DirectSize = ValueInfo.Width;
398
288
    DirectAlign = ValueInfo.Align;
399
288
  }
400
401
  // Cast the address we've calculated to the right type.
402
332
  llvm::Type *DirectTy = CGF.ConvertTypeForMem(ValueTy), *ElementTy = DirectTy;
403
332
  if (IsIndirect) {
404
44
    unsigned AllocaAS = CGF.CGM.getDataLayout().getAllocaAddrSpace();
405
44
    DirectTy = DirectTy->getPointerTo(AllocaAS);
406
44
  }
407
408
332
  Address Addr = emitVoidPtrDirectVAArg(CGF, VAListAddr, DirectTy, DirectSize,
409
332
                                        DirectAlign, SlotSizeAndAlign,
410
332
                                        AllowHigherAlign, ForceRightAdjust);
411
412
332
  if (IsIndirect) {
413
44
    Addr = Address(CGF.Builder.CreateLoad(Addr), ElementTy, ValueInfo.Align);
414
44
  }
415
416
332
  return Addr;
417
332
}
418
419
static Address complexTempStructure(CodeGenFunction &CGF, Address VAListAddr,
420
                                    QualType Ty, CharUnits SlotSize,
421
0
                                    CharUnits EltSize, const ComplexType *CTy) {
422
0
  Address Addr =
423
0
      emitVoidPtrDirectVAArg(CGF, VAListAddr, CGF.Int8Ty, SlotSize * 2,
424
0
                             SlotSize, SlotSize, /*AllowHigher*/ true);
425
426
0
  Address RealAddr = Addr;
427
0
  Address ImagAddr = RealAddr;
428
0
  if (CGF.CGM.getDataLayout().isBigEndian()) {
429
0
    RealAddr =
430
0
        CGF.Builder.CreateConstInBoundsByteGEP(RealAddr, SlotSize - EltSize);
431
0
    ImagAddr = CGF.Builder.CreateConstInBoundsByteGEP(ImagAddr,
432
0
                                                      2 * SlotSize - EltSize);
433
0
  } else {
434
0
    ImagAddr = CGF.Builder.CreateConstInBoundsByteGEP(RealAddr, SlotSize);
435
0
  }
436
437
0
  llvm::Type *EltTy = CGF.ConvertTypeForMem(CTy->getElementType());
438
0
  RealAddr = CGF.Builder.CreateElementBitCast(RealAddr, EltTy);
439
0
  ImagAddr = CGF.Builder.CreateElementBitCast(ImagAddr, EltTy);
440
0
  llvm::Value *Real = CGF.Builder.CreateLoad(RealAddr, ".vareal");
441
0
  llvm::Value *Imag = CGF.Builder.CreateLoad(ImagAddr, ".vaimag");
442
443
0
  Address Temp = CGF.CreateMemTemp(Ty, "vacplx");
444
0
  CGF.EmitStoreOfComplex({Real, Imag}, CGF.MakeAddrLValue(Temp, Ty),
445
0
                         /*init*/ true);
446
0
  return Temp;
447
0
}
448
449
static Address emitMergePHI(CodeGenFunction &CGF,
450
                            Address Addr1, llvm::BasicBlock *Block1,
451
                            Address Addr2, llvm::BasicBlock *Block2,
452
399
                            const llvm::Twine &Name = "") {
453
399
  assert(Addr1.getType() == Addr2.getType());
454
399
  llvm::PHINode *PHI = CGF.Builder.CreatePHI(Addr1.getType(), 2, Name);
455
399
  PHI->addIncoming(Addr1.getPointer(), Block1);
456
399
  PHI->addIncoming(Addr2.getPointer(), Block2);
457
399
  CharUnits Align = std::min(Addr1.getAlignment(), Addr2.getAlignment());
458
399
  return Address(PHI, Addr1.getElementType(), Align);
459
399
}
460
461
TargetCodeGenInfo::TargetCodeGenInfo(std::unique_ptr<ABIInfo> Info)
462
33.1k
    : Info(std::move(Info)) {}
463
464
33.1k
TargetCodeGenInfo::~TargetCodeGenInfo() = default;
465
466
// If someone can figure out a general rule for this, that would be great.
467
// It's probably just doomed to be platform-dependent, though.
468
5
unsigned TargetCodeGenInfo::getSizeOfUnwindException() const {
469
  // Verified for:
470
  //   x86-64     FreeBSD, Linux, Darwin
471
  //   x86-32     FreeBSD, Linux, Darwin
472
  //   PowerPC    Linux
473
  //   ARM        Darwin (*not* EABI)
474
  //   AArch64    Linux
475
5
  return 32;
476
5
}
477
478
bool TargetCodeGenInfo::isNoProtoCallVariadic(const CallArgList &args,
479
88
                                     const FunctionNoProtoType *fnType) const {
480
  // The following conventions are known to require this to be false:
481
  //   x86_stdcall
482
  //   MIPS
483
  // For everything else, we just prefer false unless we opt out.
484
88
  return false;
485
88
}
486
487
void
488
TargetCodeGenInfo::getDependentLibraryOption(llvm::StringRef Lib,
489
2
                                             llvm::SmallString<24> &Opt) const {
490
  // This assumes the user is passing a library name like "rt" instead of a
491
  // filename like "librt.a/so", and that they don't care whether it's static or
492
  // dynamic.
493
2
  Opt = "-l";
494
2
  Opt += Lib;
495
2
}
496
497
113
unsigned TargetCodeGenInfo::getOpenCLKernelCallingConv() const {
498
  // OpenCL kernels are called via an explicit runtime API with arguments
499
  // set with clSetKernelArg(), not as normal sub-functions.
500
  // Return SPIR_KERNEL by default as the kernel calling convention to
501
  // ensure the fingerprint is fixed such way that each OpenCL argument
502
  // gets one matching argument in the produced kernel function argument
503
  // list to enable feasible implementation of clSetKernelArg() with
504
  // aggregates etc. In case we would use the default C calling conv here,
505
  // clSetKernelArg() might break depending on the target-specific
506
  // conventions; different targets might split structs passed as values
507
  // to multiple function arguments etc.
508
113
  return llvm::CallingConv::SPIR_KERNEL;
509
113
}
510
511
llvm::Constant *TargetCodeGenInfo::getNullPointer(const CodeGen::CodeGenModule &CGM,
512
28.8k
    llvm::PointerType *T, QualType QT) const {
513
28.8k
  return llvm::ConstantPointerNull::get(T);
514
28.8k
}
515
516
LangAS TargetCodeGenInfo::getGlobalVarAddressSpace(CodeGenModule &CGM,
517
67.1k
                                                   const VarDecl *D) const {
518
67.1k
  assert(!CGM.getLangOpts().OpenCL &&
519
67.1k
         !(CGM.getLangOpts().CUDA && CGM.getLangOpts().CUDAIsDevice) &&
520
67.1k
         "Address space agnostic languages only");
521
67.1k
  return D ? 
D->getType().getAddressSpace()65.2k
:
LangAS::Default1.85k
;
522
67.1k
}
523
524
llvm::Value *TargetCodeGenInfo::performAddrSpaceCast(
525
    CodeGen::CodeGenFunction &CGF, llvm::Value *Src, LangAS SrcAddr,
526
12.9k
    LangAS DestAddr, llvm::Type *DestTy, bool isNonNull) const {
527
  // Since target may map different address spaces in AST to the same address
528
  // space, an address space conversion may end up as a bitcast.
529
12.9k
  if (auto *C = dyn_cast<llvm::Constant>(Src))
530
64
    return performAddrSpaceCast(CGF.CGM, C, SrcAddr, DestAddr, DestTy);
531
  // Try to preserve the source's name to make IR more readable.
532
12.9k
  return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
533
12.9k
      Src, DestTy, Src->hasName() ? 
Src->getName() + ".ascast"12.7k
:
""206
);
534
12.9k
}
535
536
llvm::Constant *
537
TargetCodeGenInfo::performAddrSpaceCast(CodeGenModule &CGM, llvm::Constant *Src,
538
                                        LangAS SrcAddr, LangAS DestAddr,
539
466
                                        llvm::Type *DestTy) const {
540
  // Since target may map different address spaces in AST to the same address
541
  // space, an address space conversion may end up as a bitcast.
542
466
  return llvm::ConstantExpr::getPointerCast(Src, DestTy);
543
466
}
544
545
llvm::SyncScope::ID
546
TargetCodeGenInfo::getLLVMSyncScopeID(const LangOptions &LangOpts,
547
                                      SyncScope Scope,
548
                                      llvm::AtomicOrdering Ordering,
549
0
                                      llvm::LLVMContext &Ctx) const {
550
0
  return Ctx.getOrInsertSyncScopeID(""); /* default sync scope */
551
0
}
552
553
static bool isEmptyRecord(ASTContext &Context, QualType T, bool AllowArrays);
554
555
/// isEmptyField - Return true iff a the field is "empty", that is it
556
/// is an unnamed bit-field or an (array of) empty record(s).
557
static bool isEmptyField(ASTContext &Context, const FieldDecl *FD,
558
8.69k
                         bool AllowArrays) {
559
8.69k
  if (FD->isUnnamedBitfield())
560
232
    return true;
561
562
8.45k
  QualType FT = FD->getType();
563
564
  // Constant arrays of empty records count as empty, strip them off.
565
  // Constant arrays of zero length always count as empty.
566
8.45k
  bool WasArray = false;
567
8.45k
  if (AllowArrays)
568
10.7k
    
while (const ConstantArrayType *8.28k
AT = Context.getAsConstantArrayType(FT)) {
569
2.57k
      if (AT->getSize() == 0)
570
75
        return true;
571
2.49k
      FT = AT->getElementType();
572
      // The [[no_unique_address]] special case below does not apply to
573
      // arrays of C++ empty records, so we need to remember this fact.
574
2.49k
      WasArray = true;
575
2.49k
    }
576
577
8.38k
  const RecordType *RT = FT->getAs<RecordType>();
578
8.38k
  if (!RT)
579
6.91k
    return false;
580
581
  // C++ record fields are never empty, at least in the Itanium ABI.
582
  //
583
  // FIXME: We should use a predicate for whether this behavior is true in the
584
  // current ABI.
585
  //
586
  // The exception to the above rule are fields marked with the
587
  // [[no_unique_address]] attribute (since C++20).  Those do count as empty
588
  // according to the Itanium ABI.  The exception applies only to records,
589
  // not arrays of records, so we must also check whether we stripped off an
590
  // array type above.
591
1.47k
  if (isa<CXXRecordDecl>(RT->getDecl()) &&
592
1.47k
      
(320
WasArray320
||
!FD->hasAttr<NoUniqueAddressAttr>()277
))
593
311
    return false;
594
595
1.16k
  return isEmptyRecord(Context, FT, AllowArrays);
596
1.47k
}
597
598
/// isEmptyRecord - Return true iff a structure contains only empty
599
/// fields. Note that a structure with a flexible array member is not
600
/// considered empty.
601
13.3k
static bool isEmptyRecord(ASTContext &Context, QualType T, bool AllowArrays) {
602
13.3k
  const RecordType *RT = T->getAs<RecordType>();
603
13.3k
  if (!RT)
604
5.58k
    return false;
605
7.77k
  const RecordDecl *RD = RT->getDecl();
606
7.77k
  if (RD->hasFlexibleArrayMember())
607
14
    return false;
608
609
  // If this is a C++ record, check the bases first.
610
7.76k
  if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD))
611
1.82k
    for (const auto &I : CXXRD->bases())
612
285
      if (!isEmptyRecord(Context, I.getType(), true))
613
277
        return false;
614
615
7.48k
  for (const auto *I : RD->fields())
616
7.04k
    if (!isEmptyField(Context, I, AllowArrays))
617
6.00k
      return false;
618
1.48k
  return true;
619
7.48k
}
620
621
/// isSingleElementStruct - Determine if a structure is a "single
622
/// element struct", i.e. it has exactly one non-empty field or
623
/// exactly one field which is itself a single element
624
/// struct. Structures with flexible array members are never
625
/// considered single element structs.
626
///
627
/// \return The field declaration for the single non-empty field, if
628
/// it exists.
629
69.4k
static const Type *isSingleElementStruct(QualType T, ASTContext &Context) {
630
69.4k
  const RecordType *RT = T->getAs<RecordType>();
631
69.4k
  if (!RT)
632
68.3k
    return nullptr;
633
634
1.06k
  const RecordDecl *RD = RT->getDecl();
635
1.06k
  if (RD->hasFlexibleArrayMember())
636
4
    return nullptr;
637
638
1.06k
  const Type *Found = nullptr;
639
640
  // If this is a C++ record, check the bases first.
641
1.06k
  if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
642
311
    for (const auto &I : CXXRD->bases()) {
643
      // Ignore empty records.
644
51
      if (isEmptyRecord(Context, I.getType(), true))
645
6
        continue;
646
647
      // If we already found an element then this isn't a single-element struct.
648
45
      if (Found)
649
11
        return nullptr;
650
651
      // If this is non-empty and not a single element struct, the composite
652
      // cannot be a single element struct.
653
34
      Found = isSingleElementStruct(I.getType(), Context);
654
34
      if (!Found)
655
9
        return nullptr;
656
34
    }
657
311
  }
658
659
  // Check for single element.
660
1.52k
  
for (const auto *FD : RD->fields())1.04k
{
661
1.52k
    QualType FT = FD->getType();
662
663
    // Ignore empty fields.
664
1.52k
    if (isEmptyField(Context, FD, true))
665
52
      continue;
666
667
    // If we already found an element then this isn't a single-element
668
    // struct.
669
1.47k
    if (Found)
670
474
      return nullptr;
671
672
    // Treat single element arrays as the element.
673
1.01k
    
while (const ConstantArrayType *1.00k
AT = Context.getAsConstantArrayType(FT)) {
674
229
      if (AT->getSize().getZExtValue() != 1)
675
211
        break;
676
18
      FT = AT->getElementType();
677
18
    }
678
679
1.00k
    if (!isAggregateTypeForABI(FT)) {
680
752
      Found = FT.getTypePtr();
681
752
    } else {
682
249
      Found = isSingleElementStruct(FT, Context);
683
249
      if (!Found)
684
238
        return nullptr;
685
249
    }
686
1.00k
  }
687
688
  // We don't consider a struct a single-element struct if it has
689
  // padding beyond the element type.
690
330
  if (Found && 
Context.getTypeSize(Found) != Context.getTypeSize(T)303
)
691
3
    return nullptr;
692
693
327
  return Found;
694
330
}
695
696
namespace {
697
Address EmitVAArgInstr(CodeGenFunction &CGF, Address VAListAddr, QualType Ty,
698
10
                       const ABIArgInfo &AI) {
699
  // This default implementation defers to the llvm backend's va_arg
700
  // instruction. It can handle only passing arguments directly
701
  // (typically only handled in the backend for primitive types), or
702
  // aggregates passed indirectly by pointer (NOTE: if the "byval"
703
  // flag has ABI impact in the callee, this implementation cannot
704
  // work.)
705
706
  // Only a few cases are covered here at the moment -- those needed
707
  // by the default abi.
708
10
  llvm::Value *Val;
709
710
10
  if (AI.isIndirect()) {
711
1
    assert(!AI.getPaddingType() &&
712
1
           "Unexpected PaddingType seen in arginfo in generic VAArg emitter!");
713
1
    assert(
714
1
        !AI.getIndirectRealign() &&
715
1
        "Unexpected IndirectRealign seen in arginfo in generic VAArg emitter!");
716
717
1
    auto TyInfo = CGF.getContext().getTypeInfoInChars(Ty);
718
1
    CharUnits TyAlignForABI = TyInfo.Align;
719
720
1
    llvm::Type *ElementTy = CGF.ConvertTypeForMem(Ty);
721
1
    llvm::Type *BaseTy = llvm::PointerType::getUnqual(ElementTy);
722
1
    llvm::Value *Addr =
723
1
        CGF.Builder.CreateVAArg(VAListAddr.getPointer(), BaseTy);
724
1
    return Address(Addr, ElementTy, TyAlignForABI);
725
9
  } else {
726
9
    assert((AI.isDirect() || AI.isExtend()) &&
727
9
           "Unexpected ArgInfo Kind in generic VAArg emitter!");
728
729
9
    assert(!AI.getInReg() &&
730
9
           "Unexpected InReg seen in arginfo in generic VAArg emitter!");
731
9
    assert(!AI.getPaddingType() &&
732
9
           "Unexpected PaddingType seen in arginfo in generic VAArg emitter!");
733
9
    assert(!AI.getDirectOffset() &&
734
9
           "Unexpected DirectOffset seen in arginfo in generic VAArg emitter!");
735
9
    assert(!AI.getCoerceToType() &&
736
9
           "Unexpected CoerceToType seen in arginfo in generic VAArg emitter!");
737
738
9
    Address Temp = CGF.CreateMemTemp(Ty, "varet");
739
9
    Val = CGF.Builder.CreateVAArg(VAListAddr.getPointer(),
740
9
                                  CGF.ConvertTypeForMem(Ty));
741
9
    CGF.Builder.CreateStore(Val, Temp);
742
9
    return Temp;
743
9
  }
744
10
}
745
746
/// DefaultABIInfo - The default implementation for ABI specific
747
/// details. This implementation provides information which results in
748
/// self-consistent and sensible LLVM IR generation, but does not
749
/// conform to any particular ABI.
750
class DefaultABIInfo : public ABIInfo {
751
public:
752
1.18k
  DefaultABIInfo(CodeGen::CodeGenTypes &CGT) : ABIInfo(CGT) {}
753
754
  ABIArgInfo classifyReturnType(QualType RetTy) const;
755
  ABIArgInfo classifyArgumentType(QualType RetTy) const;
756
757
1.05k
  void computeInfo(CGFunctionInfo &FI) const override {
758
1.05k
    if (!getCXXABI().classifyReturnType(FI))
759
1.05k
      FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
760
1.05k
    for (auto &I : FI.arguments())
761
1.46k
      I.info = classifyArgumentType(I.type);
762
1.05k
  }
763
764
  Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
765
3
                    QualType Ty) const override {
766
3
    return EmitVAArgInstr(CGF, VAListAddr, Ty, classifyArgumentType(Ty));
767
3
  }
768
};
769
770
class DefaultTargetCodeGenInfo : public TargetCodeGenInfo {
771
public:
772
  DefaultTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT)
773
63
      : TargetCodeGenInfo(std::make_unique<DefaultABIInfo>(CGT)) {}
774
};
775
776
4.28k
ABIArgInfo DefaultABIInfo::classifyArgumentType(QualType Ty) const {
777
4.28k
  Ty = useFirstFieldIfTransparentUnion(Ty);
778
779
4.28k
  if (isAggregateTypeForABI(Ty)) {
780
    // Records with non-trivial destructors/copy-constructors should not be
781
    // passed by value.
782
125
    if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))
783
2
      return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
784
785
123
    return getNaturalAlignIndirect(Ty);
786
125
  }
787
788
  // Treat an enum type as its underlying type.
789
4.16k
  if (const EnumType *EnumTy = Ty->getAs<EnumType>())
790
8
    Ty = EnumTy->getDecl()->getIntegerType();
791
792
4.16k
  ASTContext &Context = getContext();
793
4.16k
  if (const auto *EIT = Ty->getAs<BitIntType>())
794
48
    if (EIT->getNumBits() >
795
48
        Context.getTypeSize(Context.getTargetInfo().hasInt128Type()
796
48
                                ? 
Context.Int128Ty12
797
48
                                : 
Context.LongLongTy36
))
798
12
      return getNaturalAlignIndirect(Ty);
799
800
4.14k
  return (isPromotableIntegerTypeForABI(Ty) ? 
ABIArgInfo::getExtend(Ty)71
801
4.14k
                                            : 
ABIArgInfo::getDirect()4.07k
);
802
4.16k
}
803
804
3.24k
ABIArgInfo DefaultABIInfo::classifyReturnType(QualType RetTy) const {
805
3.24k
  if (RetTy->isVoidType())
806
1.70k
    return ABIArgInfo::getIgnore();
807
808
1.53k
  if (isAggregateTypeForABI(RetTy))
809
63
    return getNaturalAlignIndirect(RetTy);
810
811
  // Treat an enum type as its underlying type.
812
1.47k
  if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
813
0
    RetTy = EnumTy->getDecl()->getIntegerType();
814
815
1.47k
  if (const auto *EIT = RetTy->getAs<BitIntType>())
816
40
    if (EIT->getNumBits() >
817
40
        getContext().getTypeSize(getContext().getTargetInfo().hasInt128Type()
818
40
                                     ? 
getContext().Int128Ty8
819
40
                                     : 
getContext().LongLongTy32
))
820
16
      return getNaturalAlignIndirect(RetTy);
821
822
1.45k
  return (isPromotableIntegerTypeForABI(RetTy) ? 
ABIArgInfo::getExtend(RetTy)82
823
1.45k
                                               : 
ABIArgInfo::getDirect()1.37k
);
824
1.47k
}
825
826
//===----------------------------------------------------------------------===//
827
// WebAssembly ABI Implementation
828
//
829
// This is a very simple ABI that relies a lot on DefaultABIInfo.
830
//===----------------------------------------------------------------------===//
831
832
enum class WebAssemblyABIKind {
833
  MVP = 0,
834
  ExperimentalMV = 1,
835
};
836
837
class WebAssemblyABIInfo final : public ABIInfo {
838
  DefaultABIInfo defaultInfo;
839
  WebAssemblyABIKind Kind;
840
841
public:
842
  explicit WebAssemblyABIInfo(CodeGen::CodeGenTypes &CGT,
843
                              WebAssemblyABIKind Kind)
844
43
      : ABIInfo(CGT), defaultInfo(CGT), Kind(Kind) {}
845
846
private:
847
  ABIArgInfo classifyReturnType(QualType RetTy) const;
848
  ABIArgInfo classifyArgumentType(QualType Ty) const;
849
850
  // DefaultABIInfo's classifyReturnType and classifyArgumentType are
851
  // non-virtual, but computeInfo and EmitVAArg are virtual, so we
852
  // overload them.
853
373
  void computeInfo(CGFunctionInfo &FI) const override {
854
373
    if (!getCXXABI().classifyReturnType(FI))
855
365
      FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
856
373
    for (auto &Arg : FI.arguments())
857
365
      Arg.info = classifyArgumentType(Arg.type);
858
373
  }
859
860
  Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
861
                    QualType Ty) const override;
862
};
863
864
class WebAssemblyTargetCodeGenInfo final : public TargetCodeGenInfo {
865
public:
866
  explicit WebAssemblyTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT,
867
                                        WebAssemblyABIKind K)
868
43
      : TargetCodeGenInfo(std::make_unique<WebAssemblyABIInfo>(CGT, K)) {
869
43
    SwiftInfo =
870
43
        std::make_unique<SwiftABIInfo>(CGT, /*SwiftErrorInRegister=*/false);
871
43
  }
872
873
  void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
874
1.25k
                           CodeGen::CodeGenModule &CGM) const override {
875
1.25k
    TargetCodeGenInfo::setTargetAttributes(D, GV, CGM);
876
1.25k
    if (const auto *FD = dyn_cast_or_null<FunctionDecl>(D)) {
877
1.07k
      if (const auto *Attr = FD->getAttr<WebAssemblyImportModuleAttr>()) {
878
1
        llvm::Function *Fn = cast<llvm::Function>(GV);
879
1
        llvm::AttrBuilder B(GV->getContext());
880
1
        B.addAttribute("wasm-import-module", Attr->getImportModule());
881
1
        Fn->addFnAttrs(B);
882
1
      }
883
1.07k
      if (const auto *Attr = FD->getAttr<WebAssemblyImportNameAttr>()) {
884
1
        llvm::Function *Fn = cast<llvm::Function>(GV);
885
1
        llvm::AttrBuilder B(GV->getContext());
886
1
        B.addAttribute("wasm-import-name", Attr->getImportName());
887
1
        Fn->addFnAttrs(B);
888
1
      }
889
1.07k
      if (const auto *Attr = FD->getAttr<WebAssemblyExportNameAttr>()) {
890
2
        llvm::Function *Fn = cast<llvm::Function>(GV);
891
2
        llvm::AttrBuilder B(GV->getContext());
892
2
        B.addAttribute("wasm-export-name", Attr->getExportName());
893
2
        Fn->addFnAttrs(B);
894
2
      }
895
1.07k
    }
896
897
1.25k
    if (auto *FD = dyn_cast_or_null<FunctionDecl>(D)) {
898
1.07k
      llvm::Function *Fn = cast<llvm::Function>(GV);
899
1.07k
      if (!FD->doesThisDeclarationHaveABody() && 
!FD->hasPrototype()70
)
900
0
        Fn->addFnAttr("no-prototype");
901
1.07k
    }
902
1.25k
  }
903
904
  /// Return the WebAssembly externref reference type.
905
3
  virtual llvm::Type *getWasmExternrefReferenceType() const override {
906
3
    return llvm::Type::getWasm_ExternrefTy(getABIInfo().getVMContext());
907
3
  }
908
  /// Return the WebAssembly funcref reference type.
909
0
  virtual llvm::Type *getWasmFuncrefReferenceType() const override {
910
0
    return llvm::Type::getWasm_FuncrefTy(getABIInfo().getVMContext());
911
0
  }
912
};
913
914
/// Classify argument of given type \p Ty.
915
365
ABIArgInfo WebAssemblyABIInfo::classifyArgumentType(QualType Ty) const {
916
365
  Ty = useFirstFieldIfTransparentUnion(Ty);
917
918
365
  if (isAggregateTypeForABI(Ty)) {
919
    // Records with non-trivial destructors/copy-constructors should not be
920
    // passed by value.
921
42
    if (auto RAA = getRecordArgABI(Ty, getCXXABI()))
922
8
      return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
923
    // Ignore empty structs/unions.
924
34
    if (isEmptyRecord(getContext(), Ty, true))
925
4
      return ABIArgInfo::getIgnore();
926
    // Lower single-element structs to just pass a regular value. TODO: We
927
    // could do reasonable-size multiple-element structs too, using getExpand(),
928
    // though watch out for things like bitfields.
929
30
    if (const Type *SeltTy = isSingleElementStruct(Ty, getContext()))
930
11
      return ABIArgInfo::getDirect(CGT.ConvertType(QualType(SeltTy, 0)));
931
    // For the experimental multivalue ABI, fully expand all other aggregates
932
19
    if (Kind == WebAssemblyABIKind::ExperimentalMV) {
933
3
      const RecordType *RT = Ty->getAs<RecordType>();
934
3
      assert(RT);
935
3
      bool HasBitField = false;
936
5
      for (auto *Field : RT->getDecl()->fields()) {
937
5
        if (Field->isBitField()) {
938
1
          HasBitField = true;
939
1
          break;
940
1
        }
941
5
      }
942
3
      if (!HasBitField)
943
2
        return ABIArgInfo::getExpand();
944
3
    }
945
19
  }
946
947
  // Otherwise just do the default thing.
948
340
  return defaultInfo.classifyArgumentType(Ty);
949
365
}
950
951
365
ABIArgInfo WebAssemblyABIInfo::classifyReturnType(QualType RetTy) const {
952
365
  if (isAggregateTypeForABI(RetTy)) {
953
    // Records with non-trivial destructors/copy-constructors should not be
954
    // returned by value.
955
30
    if (!getRecordArgABI(RetTy, getCXXABI())) {
956
      // Ignore empty structs/unions.
957
30
      if (isEmptyRecord(getContext(), RetTy, true))
958
4
        return ABIArgInfo::getIgnore();
959
      // Lower single-element structs to just return a regular value. TODO: We
960
      // could do reasonable-size multiple-element structs too, using
961
      // ABIArgInfo::getDirect().
962
26
      if (const Type *SeltTy = isSingleElementStruct(RetTy, getContext()))
963
11
        return ABIArgInfo::getDirect(CGT.ConvertType(QualType(SeltTy, 0)));
964
      // For the experimental multivalue ABI, return all other aggregates
965
15
      if (Kind == WebAssemblyABIKind::ExperimentalMV)
966
3
        return ABIArgInfo::getDirect();
967
15
    }
968
30
  }
969
970
  // Otherwise just do the default thing.
971
347
  return defaultInfo.classifyReturnType(RetTy);
972
365
}
973
974
Address WebAssemblyABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
975
5
                                      QualType Ty) const {
976
5
  bool IsIndirect = isAggregateTypeForABI(Ty) &&
977
5
                    
!isEmptyRecord(getContext(), Ty, true)3
&&
978
5
                    
!isSingleElementStruct(Ty, getContext())2
;
979
5
  return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect,
980
5
                          getContext().getTypeInfoInChars(Ty),
981
5
                          CharUnits::fromQuantity(4),
982
5
                          /*AllowHigherAlign=*/true);
983
5
}
984
985
//===----------------------------------------------------------------------===//
986
// le32/PNaCl bitcode ABI Implementation
987
//
988
// This is a simplified version of the x86_32 ABI.  Arguments and return values
989
// are always passed on the stack.
990
//===----------------------------------------------------------------------===//
991
992
class PNaClABIInfo : public ABIInfo {
993
 public:
994
3
  PNaClABIInfo(CodeGen::CodeGenTypes &CGT) : ABIInfo(CGT) {}
995
996
  ABIArgInfo classifyReturnType(QualType RetTy) const;
997
  ABIArgInfo classifyArgumentType(QualType RetTy) const;
998
999
  void computeInfo(CGFunctionInfo &FI) const override;
1000
  Address EmitVAArg(CodeGenFunction &CGF,
1001
                    Address VAListAddr, QualType Ty) const override;
1002
};
1003
1004
class PNaClTargetCodeGenInfo : public TargetCodeGenInfo {
1005
 public:
1006
   PNaClTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT)
1007
3
       : TargetCodeGenInfo(std::make_unique<PNaClABIInfo>(CGT)) {}
1008
};
1009
1010
2
void PNaClABIInfo::computeInfo(CGFunctionInfo &FI) const {
1011
2
  if (!getCXXABI().classifyReturnType(FI))
1012
2
    FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
1013
1014
2
  for (auto &I : FI.arguments())
1015
0
    I.info = classifyArgumentType(I.type);
1016
2
}
1017
1018
Address PNaClABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
1019
0
                                QualType Ty) const {
1020
  // The PNaCL ABI is a bit odd, in that varargs don't use normal
1021
  // function classification. Structs get passed directly for varargs
1022
  // functions, through a rewriting transform in
1023
  // pnacl-llvm/lib/Transforms/NaCl/ExpandVarArgs.cpp, which allows
1024
  // this target to actually support a va_arg instructions with an
1025
  // aggregate type, unlike other targets.
1026
0
  return EmitVAArgInstr(CGF, VAListAddr, Ty, ABIArgInfo::getDirect());
1027
0
}
1028
1029
/// Classify argument of given type \p Ty.
1030
0
ABIArgInfo PNaClABIInfo::classifyArgumentType(QualType Ty) const {
1031
0
  if (isAggregateTypeForABI(Ty)) {
1032
0
    if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))
1033
0
      return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
1034
0
    return getNaturalAlignIndirect(Ty);
1035
0
  } else if (const EnumType *EnumTy = Ty->getAs<EnumType>()) {
1036
    // Treat an enum type as its underlying type.
1037
0
    Ty = EnumTy->getDecl()->getIntegerType();
1038
0
  } else if (Ty->isFloatingType()) {
1039
    // Floating-point types don't go inreg.
1040
0
    return ABIArgInfo::getDirect();
1041
0
  } else if (const auto *EIT = Ty->getAs<BitIntType>()) {
1042
    // Treat bit-precise integers as integers if <= 64, otherwise pass
1043
    // indirectly.
1044
0
    if (EIT->getNumBits() > 64)
1045
0
      return getNaturalAlignIndirect(Ty);
1046
0
    return ABIArgInfo::getDirect();
1047
0
  }
1048
1049
0
  return (isPromotableIntegerTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty)
1050
0
                                            : ABIArgInfo::getDirect());
1051
0
}
1052
1053
2
ABIArgInfo PNaClABIInfo::classifyReturnType(QualType RetTy) const {
1054
2
  if (RetTy->isVoidType())
1055
2
    return ABIArgInfo::getIgnore();
1056
1057
  // In the PNaCl ABI we always return records/structures on the stack.
1058
0
  if (isAggregateTypeForABI(RetTy))
1059
0
    return getNaturalAlignIndirect(RetTy);
1060
1061
  // Treat bit-precise integers as integers if <= 64, otherwise pass indirectly.
1062
0
  if (const auto *EIT = RetTy->getAs<BitIntType>()) {
1063
0
    if (EIT->getNumBits() > 64)
1064
0
      return getNaturalAlignIndirect(RetTy);
1065
0
    return ABIArgInfo::getDirect();
1066
0
  }
1067
1068
  // Treat an enum type as its underlying type.
1069
0
  if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
1070
0
    RetTy = EnumTy->getDecl()->getIntegerType();
1071
1072
0
  return (isPromotableIntegerTypeForABI(RetTy) ? ABIArgInfo::getExtend(RetTy)
1073
0
                                               : ABIArgInfo::getDirect());
1074
0
}
1075
1076
/// IsX86_MMXType - Return true if this is an MMX type.
1077
767
bool IsX86_MMXType(llvm::Type *IRType) {
1078
  // Return true if the type is an MMX type <2 x i32>, <4 x i16>, or <8 x i8>.
1079
767
  return IRType->isVectorTy() && IRType->getPrimitiveSizeInBits() == 64 &&
1080
767
    
cast<llvm::VectorType>(IRType)->getElementType()->isIntegerTy()26
&&
1081
767
    
IRType->getScalarSizeInBits() != 6424
;
1082
767
}
1083
1084
static llvm::Type* X86AdjustInlineAsmType(CodeGen::CodeGenFunction &CGF,
1085
                                          StringRef Constraint,
1086
1.59k
                                          llvm::Type* Ty) {
1087
1.59k
  bool IsMMXCons = llvm::StringSwitch<bool>(Constraint)
1088
1.59k
                     .Cases("y", "&y", "^Ym", true)
1089
1.59k
                     .Default(false);
1090
1.59k
  if (IsMMXCons && 
Ty->isVectorTy()18
) {
1091
14
    if (cast<llvm::VectorType>(Ty)->getPrimitiveSizeInBits().getFixedValue() !=
1092
14
        64) {
1093
      // Invalid MMX constraint
1094
0
      return nullptr;
1095
0
    }
1096
1097
14
    return llvm::Type::getX86_MMXTy(CGF.getLLVMContext());
1098
14
  }
1099
1100
  // No operation needed
1101
1.58k
  return Ty;
1102
1.59k
}
1103
1104
/// Returns true if this type can be passed in SSE registers with the
1105
/// X86_VectorCall calling convention. Shared between x86_32 and x86_64.
1106
831
static bool isX86VectorTypeForVectorCall(ASTContext &Context, QualType Ty) {
1107
831
  if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
1108
531
    if (BT->isFloatingPoint() && 
BT->getKind() != BuiltinType::Half282
) {
1109
282
      if (BT->getKind() == BuiltinType::LongDouble) {
1110
6
        if (&Context.getTargetInfo().getLongDoubleFormat() ==
1111
6
            &llvm::APFloat::x87DoubleExtended())
1112
2
          return false;
1113
6
      }
1114
280
      return true;
1115
282
    }
1116
531
  } else 
if (const VectorType *300
VT300
= Ty->getAs<VectorType>()) {
1117
    // vectorcall can pass XMM, YMM, and ZMM vectors. We don't pass SSE1 MMX
1118
    // registers specially.
1119
267
    unsigned VecSize = Context.getTypeSize(VT);
1120
267
    if (VecSize == 128 || 
VecSize == 2564
||
VecSize == 5124
)
1121
267
      return true;
1122
267
  }
1123
282
  return false;
1124
831
}
1125
1126
/// Returns true if this aggregate is small enough to be passed in SSE registers
1127
/// in the X86_VectorCall calling convention. Shared between x86_32 and x86_64.
1128
741
static bool isX86VectorCallAggregateSmallEnough(uint64_t NumMembers) {
1129
741
  return NumMembers <= 4;
1130
741
}
1131
1132
/// Returns a Homogeneous Vector Aggregate ABIArgInfo, used in X86.
1133
30
static ABIArgInfo getDirectX86Hva(llvm::Type* T = nullptr) {
1134
30
  auto AI = ABIArgInfo::getDirect(T);
1135
30
  AI.setInReg(true);
1136
30
  AI.setCanBeFlattened(false);
1137
30
  return AI;
1138
30
}
1139
1140
//===----------------------------------------------------------------------===//
1141
// X86-32 ABI Implementation
1142
//===----------------------------------------------------------------------===//
1143
1144
/// Similar to llvm::CCState, but for Clang.
1145
struct CCState {
1146
  CCState(CGFunctionInfo &FI)
1147
19.5k
      : IsPreassigned(FI.arg_size()), CC(FI.getCallingConvention()) {}
1148
1149
  llvm::SmallBitVector IsPreassigned;
1150
  unsigned CC = CallingConv::CC_C;
1151
  unsigned FreeRegs = 0;
1152
  unsigned FreeSSERegs = 0;
1153
};
1154
1155
/// X86_32ABIInfo - The X86-32 ABI information.
1156
class X86_32ABIInfo : public ABIInfo {
1157
  enum Class {
1158
    Integer,
1159
    Float
1160
  };
1161
1162
  static const unsigned MinABIStackAlignInBytes = 4;
1163
1164
  bool IsDarwinVectorABI;
1165
  bool IsRetSmallStructInRegABI;
1166
  bool IsWin32StructABI;
1167
  bool IsSoftFloatABI;
1168
  bool IsMCUABI;
1169
  bool IsLinuxABI;
1170
  unsigned DefaultNumRegisterParameters;
1171
1172
534
  static bool isRegisterSize(unsigned Size) {
1173
534
    return (Size == 8 || 
Size == 16503
||
Size == 32484
||
Size == 64234
);
1174
534
  }
1175
1176
436
  bool isHomogeneousAggregateBaseType(QualType Ty) const override {
1177
    // FIXME: Assumes vectorcall is in use.
1178
436
    return isX86VectorTypeForVectorCall(getContext(), Ty);
1179
436
  }
1180
1181
  bool isHomogeneousAggregateSmallEnough(const Type *Ty,
1182
322
                                         uint64_t NumMembers) const override {
1183
    // FIXME: Assumes vectorcall is in use.
1184
322
    return isX86VectorCallAggregateSmallEnough(NumMembers);
1185
322
  }
1186
1187
  bool shouldReturnTypeInRegister(QualType Ty, ASTContext &Context) const;
1188
1189
  /// getIndirectResult - Give a source type \arg Ty, return a suitable result
1190
  /// such that the argument will be passed in memory.
1191
  ABIArgInfo getIndirectResult(QualType Ty, bool ByVal, CCState &State) const;
1192
1193
  ABIArgInfo getIndirectReturnResult(QualType Ty, CCState &State) const;
1194
1195
  /// Return the alignment to use for the given type on the stack.
1196
  unsigned getTypeStackAlignInBytes(QualType Ty, unsigned Align) const;
1197
1198
  Class classify(QualType Ty) const;
1199
  ABIArgInfo classifyReturnType(QualType RetTy, CCState &State) const;
1200
  ABIArgInfo classifyArgumentType(QualType RetTy, CCState &State) const;
1201
1202
  /// Updates the number of available free registers, returns
1203
  /// true if any registers were allocated.
1204
  bool updateFreeRegs(QualType Ty, CCState &State) const;
1205
1206
  bool shouldAggregateUseDirect(QualType Ty, CCState &State, bool &InReg,
1207
                                bool &NeedsPadding) const;
1208
  bool shouldPrimitiveUseInReg(QualType Ty, CCState &State) const;
1209
1210
  bool canExpandIndirectArgument(QualType Ty) const;
1211
1212
  /// Rewrite the function info so that all memory arguments use
1213
  /// inalloca.
1214
  void rewriteWithInAlloca(CGFunctionInfo &FI) const;
1215
1216
  void addFieldToArgStruct(SmallVector<llvm::Type *, 6> &FrameFields,
1217
                           CharUnits &StackOffset, ABIArgInfo &Info,
1218
                           QualType Type) const;
1219
  void runVectorCallFirstPass(CGFunctionInfo &FI, CCState &State) const;
1220
1221
public:
1222
1223
  void computeInfo(CGFunctionInfo &FI) const override;
1224
  Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
1225
                    QualType Ty) const override;
1226
1227
  X86_32ABIInfo(CodeGen::CodeGenTypes &CGT, bool DarwinVectorABI,
1228
                bool RetSmallStructInRegABI, bool Win32StructABI,
1229
                unsigned NumRegisterParameters, bool SoftFloatABI)
1230
      : ABIInfo(CGT), IsDarwinVectorABI(DarwinVectorABI),
1231
        IsRetSmallStructInRegABI(RetSmallStructInRegABI),
1232
        IsWin32StructABI(Win32StructABI), IsSoftFloatABI(SoftFloatABI),
1233
        IsMCUABI(CGT.getTarget().getTriple().isOSIAMCU()),
1234
        IsLinuxABI(CGT.getTarget().getTriple().isOSLinux() ||
1235
                   CGT.getTarget().getTriple().isOSCygMing()),
1236
3.20k
        DefaultNumRegisterParameters(NumRegisterParameters) {}
1237
};
1238
1239
class X86_32SwiftABIInfo : public SwiftABIInfo {
1240
public:
1241
  explicit X86_32SwiftABIInfo(CodeGenTypes &CGT)
1242
3.20k
      : SwiftABIInfo(CGT, /*SwiftErrorInRegister=*/false) {}
1243
1244
  bool shouldPassIndirectly(ArrayRef<llvm::Type *> ComponentTys,
1245
0
                            bool AsReturnValue) const override {
1246
    // LLVM's x86-32 lowering currently only assigns up to three
1247
    // integer registers and three fp registers.  Oddly, it'll use up to
1248
    // four vector registers for vectors, but those can overlap with the
1249
    // scalar registers.
1250
0
    return occupiesMoreThan(ComponentTys, /*total=*/3);
1251
0
  }
1252
};
1253
1254
class X86_32TargetCodeGenInfo : public TargetCodeGenInfo {
1255
public:
1256
  X86_32TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, bool DarwinVectorABI,
1257
                          bool RetSmallStructInRegABI, bool Win32StructABI,
1258
                          unsigned NumRegisterParameters, bool SoftFloatABI)
1259
      : TargetCodeGenInfo(std::make_unique<X86_32ABIInfo>(
1260
            CGT, DarwinVectorABI, RetSmallStructInRegABI, Win32StructABI,
1261
3.20k
            NumRegisterParameters, SoftFloatABI)) {
1262
3.20k
    SwiftInfo = std::make_unique<X86_32SwiftABIInfo>(CGT);
1263
3.20k
  }
1264
1265
  static bool isStructReturnInRegABI(
1266
      const llvm::Triple &Triple, const CodeGenOptions &Opts);
1267
1268
  void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
1269
                           CodeGen::CodeGenModule &CGM) const override;
1270
1271
0
  int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const override {
1272
    // Darwin uses different dwarf register numbers for EH.
1273
0
    if (CGM.getTarget().getTriple().isOSDarwin()) return 5;
1274
0
    return 4;
1275
0
  }
1276
1277
  bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
1278
                               llvm::Value *Address) const override;
1279
1280
  llvm::Type* adjustInlineAsmType(CodeGen::CodeGenFunction &CGF,
1281
                                  StringRef Constraint,
1282
685
                                  llvm::Type* Ty) const override {
1283
685
    return X86AdjustInlineAsmType(CGF, Constraint, Ty);
1284
685
  }
1285
1286
  void addReturnRegisterOutputs(CodeGenFunction &CGF, LValue ReturnValue,
1287
                                std::string &Constraints,
1288
                                std::vector<llvm::Type *> &ResultRegTypes,
1289
                                std::vector<llvm::Type *> &ResultTruncRegTypes,
1290
                                std::vector<LValue> &ResultRegDests,
1291
                                std::string &AsmString,
1292
                                unsigned NumOutputs) const override;
1293
1294
7
  StringRef getARCRetainAutoreleasedReturnValueMarker() const override {
1295
7
    return "movl\t%ebp, %ebp"
1296
7
           "\t\t// marker for objc_retainAutoreleaseReturnValue";
1297
7
  }
1298
};
1299
1300
}
1301
1302
/// Rewrite input constraint references after adding some output constraints.
1303
/// In the case where there is one output and one input and we add one output,
1304
/// we need to replace all operand references greater than or equal to 1:
1305
///     mov $0, $1
1306
///     mov eax, $1
1307
/// The result will be:
1308
///     mov $0, $2
1309
///     mov eax, $2
1310
static void rewriteInputConstraintReferences(unsigned FirstIn,
1311
                                             unsigned NumNewOuts,
1312
27
                                             std::string &AsmString) {
1313
27
  std::string Buf;
1314
27
  llvm::raw_string_ostream OS(Buf);
1315
27
  size_t Pos = 0;
1316
93
  while (Pos < AsmString.size()) {
1317
66
    size_t DollarStart = AsmString.find('$', Pos);
1318
66
    if (DollarStart == std::string::npos)
1319
19
      DollarStart = AsmString.size();
1320
66
    size_t DollarEnd = AsmString.find_first_not_of('$', DollarStart);
1321
66
    if (DollarEnd == std::string::npos)
1322
19
      DollarEnd = AsmString.size();
1323
66
    OS << StringRef(&AsmString[Pos], DollarEnd - Pos);
1324
66
    Pos = DollarEnd;
1325
66
    size_t NumDollars = DollarEnd - DollarStart;
1326
66
    if (NumDollars % 2 != 0 && 
Pos < AsmString.size()24
) {
1327
      // We have an operand reference.
1328
24
      size_t DigitStart = Pos;
1329
24
      if (AsmString[DigitStart] == '{') {
1330
8
        OS << '{';
1331
8
        ++DigitStart;
1332
8
      }
1333
24
      size_t DigitEnd = AsmString.find_first_not_of("0123456789", DigitStart);
1334
24
      if (DigitEnd == std::string::npos)
1335
6
        DigitEnd = AsmString.size();
1336
24
      StringRef OperandStr(&AsmString[DigitStart], DigitEnd - DigitStart);
1337
24
      unsigned OperandIndex;
1338
24
      if (!OperandStr.getAsInteger(10, OperandIndex)) {
1339
24
        if (OperandIndex >= FirstIn)
1340
19
          OperandIndex += NumNewOuts;
1341
24
        OS << OperandIndex;
1342
24
      } else {
1343
0
        OS << OperandStr;
1344
0
      }
1345
24
      Pos = DigitEnd;
1346
24
    }
1347
66
  }
1348
27
  AsmString = std::move(OS.str());
1349
27
}
1350
1351
/// Add output constraints for EAX:EDX because they are return registers.
1352
void X86_32TargetCodeGenInfo::addReturnRegisterOutputs(
1353
    CodeGenFunction &CGF, LValue ReturnSlot, std::string &Constraints,
1354
    std::vector<llvm::Type *> &ResultRegTypes,
1355
    std::vector<llvm::Type *> &ResultTruncRegTypes,
1356
    std::vector<LValue> &ResultRegDests, std::string &AsmString,
1357
27
    unsigned NumOutputs) const {
1358
27
  uint64_t RetWidth = CGF.getContext().getTypeSize(ReturnSlot.getType());
1359
1360
  // Use the EAX constraint if the width is 32 or smaller and EAX:EDX if it is
1361
  // larger.
1362
27
  if (!Constraints.empty())
1363
4
    Constraints += ',';
1364
27
  if (RetWidth <= 32) {
1365
22
    Constraints += "={eax}";
1366
22
    ResultRegTypes.push_back(CGF.Int32Ty);
1367
22
  } else {
1368
    // Use the 'A' constraint for EAX:EDX.
1369
5
    Constraints += "=A";
1370
5
    ResultRegTypes.push_back(CGF.Int64Ty);
1371
5
  }
1372
1373
  // Truncate EAX or EAX:EDX to an integer of the appropriate size.
1374
27
  llvm::Type *CoerceTy = llvm::IntegerType::get(CGF.getLLVMContext(), RetWidth);
1375
27
  ResultTruncRegTypes.push_back(CoerceTy);
1376
1377
  // Coerce the integer by bitcasting the return slot pointer.
1378
27
  ReturnSlot.setAddress(
1379
27
      CGF.Builder.CreateElementBitCast(ReturnSlot.getAddress(CGF), CoerceTy));
1380
27
  ResultRegDests.push_back(ReturnSlot);
1381
1382
27
  rewriteInputConstraintReferences(NumOutputs, 1, AsmString);
1383
27
}
1384
1385
/// shouldReturnTypeInRegister - Determine if the given type should be
1386
/// returned in a register (for the Darwin and MCU ABI).
1387
bool X86_32ABIInfo::shouldReturnTypeInRegister(QualType Ty,
1388
548
                                               ASTContext &Context) const {
1389
548
  uint64_t Size = Context.getTypeSize(Ty);
1390
1391
  // For i386, type must be register sized.
1392
  // For the MCU ABI, it only needs to be <= 8-byte
1393
548
  if ((IsMCUABI && 
Size > 6414
) ||
(546
!IsMCUABI546
&&
!isRegisterSize(Size)534
))
1394
71
   return false;
1395
1396
477
  if (Ty->isVectorType()) {
1397
    // 64- and 128- bit vectors inside structures are not returned in
1398
    // registers.
1399
4
    if (Size == 64 || 
Size == 1281
)
1400
3
      return false;
1401
1402
1
    return true;
1403
4
  }
1404
1405
  // If this is a builtin, pointer, enum, complex type, member pointer, or
1406
  // member function pointer it is ok.
1407
473
  if (Ty->getAs<BuiltinType>() || 
Ty->hasPointerRepresentation()383
||
1408
473
      
Ty->isAnyComplexType()378
||
Ty->isEnumeralType()354
||
1409
473
      
Ty->isBlockPointerType()352
||
Ty->isMemberPointerType()352
)
1410
383
    return true;
1411
1412
  // Arrays are treated like records.
1413
90
  if (const ConstantArrayType *AT = Context.getAsConstantArrayType(Ty))
1414
6
    return shouldReturnTypeInRegister(AT->getElementType(), Context);
1415
1416
  // Otherwise, it must be a record type.
1417
84
  const RecordType *RT = Ty->getAs<RecordType>();
1418
84
  if (!RT) 
return false0
;
1419
1420
  // FIXME: Traverse bases here too.
1421
1422
  // Structure types are passed in register if all fields would be
1423
  // passed in a register.
1424
118
  
for (const auto *FD : RT->getDecl()->fields())84
{
1425
    // Empty fields are ignored.
1426
118
    if (isEmptyField(Context, FD, true))
1427
11
      continue;
1428
1429
    // Check fields recursively.
1430
107
    if (!shouldReturnTypeInRegister(FD->getType(), Context))
1431
4
      return false;
1432
107
  }
1433
80
  return true;
1434
84
}
1435
1436
210
static bool is32Or64BitBasicType(QualType Ty, ASTContext &Context) {
1437
  // Treat complex types as the element type.
1438
210
  if (const ComplexType *CTy = Ty->getAs<ComplexType>())
1439
2
    Ty = CTy->getElementType();
1440
1441
  // Check for a type which we know has a simple scalar argument-passing
1442
  // convention without any padding.  (We're specifically looking for 32
1443
  // and 64-bit integer and integer-equivalents, float, and double.)
1444
210
  if (!Ty->getAs<BuiltinType>() && 
!Ty->hasPointerRepresentation()39
&&
1445
210
      
!Ty->isEnumeralType()32
&&
!Ty->isBlockPointerType()29
)
1446
29
    return false;
1447
1448
181
  uint64_t Size = Context.getTypeSize(Ty);
1449
181
  return Size == 32 || 
Size == 6429
;
1450
210
}
1451
1452
static bool addFieldSizes(ASTContext &Context, const RecordDecl *RD,
1453
250
                          uint64_t &Size) {
1454
250
  for (const auto *FD : RD->fields()) {
1455
    // Scalar arguments on the stack get 4 byte alignment on x86. If the
1456
    // argument is smaller than 32-bits, expanding the struct will create
1457
    // alignment padding.
1458
210
    if (!is32Or64BitBasicType(FD->getType(), Context))
1459
51
      return false;
1460
1461
    // FIXME: Reject bit-fields wholesale; there are two problems, we don't know
1462
    // how to expand them yet, and the predicate for telling if a bitfield still
1463
    // counts as "basic" is more complicated than what we were doing previously.
1464
159
    if (FD->isBitField())
1465
2
      return false;
1466
1467
157
    Size += Context.getTypeSize(FD->getType());
1468
157
  }
1469
197
  return true;
1470
250
}
1471
1472
static bool addBaseAndFieldSizes(ASTContext &Context, const CXXRecordDecl *RD,
1473
148
                                 uint64_t &Size) {
1474
  // Don't do this if there are any non-empty bases.
1475
148
  for (const CXXBaseSpecifier &Base : RD->bases()) {
1476
1
    if (!addBaseAndFieldSizes(Context, Base.getType()->getAsCXXRecordDecl(),
1477
1
                              Size))
1478
0
      return false;
1479
1
  }
1480
148
  if (!addFieldSizes(Context, RD, Size))
1481
11
    return false;
1482
137
  return true;
1483
148
}
1484
1485
/// Test whether an argument type which is to be passed indirectly (on the
1486
/// stack) would have the equivalent layout if it was expanded into separate
1487
/// arguments. If so, we prefer to do the latter to avoid inhibiting
1488
/// optimizations.
1489
340
bool X86_32ABIInfo::canExpandIndirectArgument(QualType Ty) const {
1490
  // We can only expand structure types.
1491
340
  const RecordType *RT = Ty->getAs<RecordType>();
1492
340
  if (!RT)
1493
77
    return false;
1494
263
  const RecordDecl *RD = RT->getDecl();
1495
263
  uint64_t Size = 0;
1496
263
  if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
1497
161
    if (!IsWin32StructABI) {
1498
      // On non-Windows, we have to conservatively match our old bitcode
1499
      // prototypes in order to be ABI-compatible at the bitcode level.
1500
36
      if (!CXXRD->isCLike())
1501
14
        return false;
1502
125
    } else {
1503
      // Don't do this for dynamic classes.
1504
125
      if (CXXRD->isDynamicClass())
1505
0
        return false;
1506
125
    }
1507
147
    if (!addBaseAndFieldSizes(getContext(), CXXRD, Size))
1508
11
      return false;
1509
147
  } else {
1510
102
    if (!addFieldSizes(getContext(), RD, Size))
1511
42
      return false;
1512
102
  }
1513
1514
  // We can do this if there was no alignment padding.
1515
196
  return Size == getContext().getTypeSize(Ty);
1516
263
}
1517
1518
366
ABIArgInfo X86_32ABIInfo::getIndirectReturnResult(QualType RetTy, CCState &State) const {
1519
  // If the return value is indirect, then the hidden argument is consuming one
1520
  // integer register.
1521
366
  if (State.FreeRegs) {
1522
5
    --State.FreeRegs;
1523
5
    if (!IsMCUABI)
1524
3
      return getNaturalAlignIndirectInReg(RetTy);
1525
5
  }
1526
363
  return getNaturalAlignIndirect(RetTy, /*ByVal=*/false);
1527
366
}
1528
1529
ABIArgInfo X86_32ABIInfo::classifyReturnType(QualType RetTy,
1530
19.4k
                                             CCState &State) const {
1531
19.4k
  if (RetTy->isVoidType())
1532
12.8k
    return ABIArgInfo::getIgnore();
1533
1534
6.63k
  const Type *Base = nullptr;
1535
6.63k
  uint64_t NumElts = 0;
1536
6.63k
  if ((State.CC == llvm::CallingConv::X86_VectorCall ||
1537
6.63k
       
State.CC == llvm::CallingConv::X86_RegCall6.62k
) &&
1538
6.63k
      
isHomogeneousAggregate(RetTy, Base, NumElts)26
) {
1539
    // The LLVM struct type for such an aggregate should lower properly.
1540
17
    return ABIArgInfo::getDirect();
1541
17
  }
1542
1543
6.61k
  if (const VectorType *VT = RetTy->getAs<VectorType>()) {
1544
    // On Darwin, some vectors are returned in registers.
1545
539
    if (IsDarwinVectorABI) {
1546
473
      uint64_t Size = getContext().getTypeSize(RetTy);
1547
1548
      // 128-bit vectors are a special case; they are returned in
1549
      // registers and we need to make sure to pick a type the LLVM
1550
      // backend will like.
1551
473
      if (Size == 128)
1552
246
        return ABIArgInfo::getDirect(llvm::FixedVectorType::get(
1553
246
            llvm::Type::getInt64Ty(getVMContext()), 2));
1554
1555
      // Always return in register if it fits in a general purpose
1556
      // register, or if it is 64 bits and has a single element.
1557
227
      if ((Size == 8 || Size == 16 || Size == 32) ||
1558
227
          
(224
Size == 64224
&&
VT->getNumElements() == 19
))
1559
9
        return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(),
1560
9
                                                            Size));
1561
1562
218
      return getIndirectReturnResult(RetTy, State);
1563
227
    }
1564
1565
66
    return ABIArgInfo::getDirect();
1566
539
  }
1567
1568
6.07k
  if (isAggregateTypeForABI(RetTy)) {
1569
532
    if (const RecordType *RT = RetTy->getAs<RecordType>()) {
1570
      // Structures with flexible arrays are always indirect.
1571
198
      if (RT->getDecl()->hasFlexibleArrayMember())
1572
1
        return getIndirectReturnResult(RetTy, State);
1573
198
    }
1574
1575
    // If specified, structs and unions are always indirect.
1576
531
    if (!IsRetSmallStructInRegABI && 
!RetTy->isAnyComplexType()119
)
1577
63
      return getIndirectReturnResult(RetTy, State);
1578
1579
    // Ignore empty structs/unions.
1580
468
    if (isEmptyRecord(getContext(), RetTy, true))
1581
32
      return ABIArgInfo::getIgnore();
1582
1583
    // Return complex of _Float16 as <2 x half> so the backend will use xmm0.
1584
436
    if (const ComplexType *CT = RetTy->getAs<ComplexType>()) {
1585
67
      QualType ET = getContext().getCanonicalType(CT->getElementType());
1586
67
      if (ET->isFloat16Type())
1587
1
        return ABIArgInfo::getDirect(llvm::FixedVectorType::get(
1588
1
            llvm::Type::getHalfTy(getVMContext()), 2));
1589
67
    }
1590
1591
    // Small structures which are register sized are generally returned
1592
    // in a register.
1593
435
    if (shouldReturnTypeInRegister(RetTy, getContext())) {
1594
361
      uint64_t Size = getContext().getTypeSize(RetTy);
1595
1596
      // As a special-case, if the struct is a "single-element" struct, and
1597
      // the field is of type "float" or "double", return it in a
1598
      // floating-point register. (MSVC does not apply this special case.)
1599
      // We apply a similar transformation for pointer types to improve the
1600
      // quality of the generated IR.
1601
361
      if (const Type *SeltTy = isSingleElementStruct(RetTy, getContext()))
1602
51
        if ((!IsWin32StructABI && 
SeltTy->isRealFloatingType()34
)
1603
51
            || 
SeltTy->hasPointerRepresentation()35
)
1604
19
          return ABIArgInfo::getDirect(CGT.ConvertType(QualType(SeltTy, 0)));
1605
1606
      // FIXME: We should be able to narrow this integer in cases with dead
1607
      // padding.
1608
342
      return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(),Size));
1609
361
    }
1610
1611
74
    return getIndirectReturnResult(RetTy, State);
1612
435
  }
1613
1614
  // Treat an enum type as its underlying type.
1615
5.54k
  if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
1616
35
    RetTy = EnumTy->getDecl()->getIntegerType();
1617
1618
5.54k
  if (const auto *EIT = RetTy->getAs<BitIntType>())
1619
28
    if (EIT->getNumBits() > 64)
1620
10
      return getIndirectReturnResult(RetTy, State);
1621
1622
5.53k
  return (isPromotableIntegerTypeForABI(RetTy) ? 
ABIArgInfo::getExtend(RetTy)144
1623
5.53k
                                               : 
ABIArgInfo::getDirect()5.39k
);
1624
5.54k
}
1625
1626
45
static bool isSIMDVectorType(ASTContext &Context, QualType Ty) {
1627
45
  return Ty->getAs<VectorType>() && 
Context.getTypeSize(Ty) == 12812
;
1628
45
}
1629
1630
54
static bool isRecordWithSIMDVectorType(ASTContext &Context, QualType Ty) {
1631
54
  const RecordType *RT = Ty->getAs<RecordType>();
1632
54
  if (!RT)
1633
21
    return false;
1634
33
  const RecordDecl *RD = RT->getDecl();
1635
1636
  // If this is a C++ record, check the bases first.
1637
33
  if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD))
1638
8
    for (const auto &I : CXXRD->bases())
1639
0
      if (!isRecordWithSIMDVectorType(Context, I.getType()))
1640
0
        return false;
1641
1642
33
  for (const auto *i : RD->fields()) {
1643
28
    QualType FT = i->getType();
1644
1645
28
    if (isSIMDVectorType(Context, FT))
1646
7
      return true;
1647
1648
21
    if (isRecordWithSIMDVectorType(Context, FT))
1649
0
      return true;
1650
21
  }
1651
1652
26
  return false;
1653
33
}
1654
1655
unsigned X86_32ABIInfo::getTypeStackAlignInBytes(QualType Ty,
1656
353
                                                 unsigned Align) const {
1657
  // Otherwise, if the alignment is less than or equal to the minimum ABI
1658
  // alignment, just use the default; the backend will handle this.
1659
353
  if (Align <= MinABIStackAlignInBytes)
1660
273
    return 0; // Use default alignment.
1661
1662
80
  if (IsLinuxABI) {
1663
    // Exclude other System V OS (e.g Darwin, PS4 and FreeBSD) since we don't
1664
    // want to spend any effort dealing with the ramifications of ABI breaks.
1665
    //
1666
    // If the vector type is __m128/__m256/__m512, return the default alignment.
1667
34
    if (Ty->isVectorType() && 
(22
Align == 1622
||
Align == 3216
||
Align == 648
))
1668
20
      return Align;
1669
34
  }
1670
  // On non-Darwin, the stack type alignment is always 4.
1671
60
  if (!IsDarwinVectorABI) {
1672
    // Set explicit alignment, since we may need to realign the top.
1673
33
    return MinABIStackAlignInBytes;
1674
33
  }
1675
1676
  // Otherwise, if the type contains an SSE vector type, the alignment is 16.
1677
27
  if (Align >= 16 && 
(17
isSIMDVectorType(getContext(), Ty)17
||
1678
17
                      
isRecordWithSIMDVectorType(getContext(), Ty)16
))
1679
8
    return 16;
1680
1681
19
  return MinABIStackAlignInBytes;
1682
27
}
1683
1684
ABIArgInfo X86_32ABIInfo::getIndirectResult(QualType Ty, bool ByVal,
1685
470
                                            CCState &State) const {
1686
470
  if (!ByVal) {
1687
167
    if (State.FreeRegs) {
1688
22
      --State.FreeRegs; // Non-byval indirects just use one pointer.
1689
22
      if (!IsMCUABI)
1690
22
        return getNaturalAlignIndirectInReg(Ty);
1691
22
    }
1692
145
    return getNaturalAlignIndirect(Ty, false);
1693
167
  }
1694
1695
  // Compute the byval alignment.
1696
303
  unsigned TypeAlign = getContext().getTypeAlign(Ty) / 8;
1697
303
  unsigned StackAlign = getTypeStackAlignInBytes(Ty, TypeAlign);
1698
303
  if (StackAlign == 0)
1699
253
    return ABIArgInfo::getIndirect(CharUnits::fromQuantity(4), /*ByVal=*/true);
1700
1701
  // If the stack alignment is less than the type alignment, realign the
1702
  // argument.
1703
50
  bool Realign = TypeAlign > StackAlign;
1704
50
  return ABIArgInfo::getIndirect(CharUnits::fromQuantity(StackAlign),
1705
50
                                 /*ByVal=*/true, Realign);
1706
303
}
1707
1708
33.2k
X86_32ABIInfo::Class X86_32ABIInfo::classify(QualType Ty) const {
1709
33.2k
  const Type *T = isSingleElementStruct(Ty, getContext());
1710
33.2k
  if (!T)
1711
33.1k
    T = Ty.getTypePtr();
1712
1713
33.2k
  if (const BuiltinType *BT = T->getAs<BuiltinType>()) {
1714
11.5k
    BuiltinType::Kind K = BT->getKind();
1715
11.5k
    if (K == BuiltinType::Float || 
K == BuiltinType::Double11.2k
)
1716
598
      return Float;
1717
11.5k
  }
1718
32.6k
  return Integer;
1719
33.2k
}
1720
1721
33.2k
bool X86_32ABIInfo::updateFreeRegs(QualType Ty, CCState &State) const {
1722
33.2k
  if (!IsSoftFloatABI) {
1723
33.2k
    Class C = classify(Ty);
1724
33.2k
    if (C == Float)
1725
598
      return false;
1726
33.2k
  }
1727
1728
32.6k
  unsigned Size = getContext().getTypeSize(Ty);
1729
32.6k
  unsigned SizeInRegs = (Size + 31) / 32;
1730
1731
32.6k
  if (SizeInRegs == 0)
1732
2
    return false;
1733
1734
32.6k
  if (!IsMCUABI) {
1735
32.6k
    if (SizeInRegs > State.FreeRegs) {
1736
32.3k
      State.FreeRegs = 0;
1737
32.3k
      return false;
1738
32.3k
    }
1739
32.6k
  } else {
1740
    // The MCU psABI allows passing parameters in-reg even if there are
1741
    // earlier parameters that are passed on the stack. Also,
1742
    // it does not allow passing >8-byte structs in-register,
1743
    // even if there are 3 free registers available.
1744
42
    if (SizeInRegs > State.FreeRegs || 
SizeInRegs > 234
)
1745
9
      return false;
1746
42
  }
1747
1748
252
  State.FreeRegs -= SizeInRegs;
1749
252
  return true;
1750
32.6k
}
1751
1752
bool X86_32ABIInfo::shouldAggregateUseDirect(QualType Ty, CCState &State,
1753
                                             bool &InReg,
1754
407
                                             bool &NeedsPadding) const {
1755
  // On Windows, aggregates other than HFAs are never passed in registers, and
1756
  // they do not consume register slots. Homogenous floating-point aggregates
1757
  // (HFAs) have already been dealt with at this point.
1758
407
  if (IsWin32StructABI && 
isAggregateTypeForABI(Ty)176
)
1759
176
    return false;
1760
1761
231
  NeedsPadding = false;
1762
231
  InReg = !IsMCUABI;
1763
1764
231
  if (!updateFreeRegs(Ty, State))
1765
211
    return false;
1766
1767
20
  if (IsMCUABI)
1768
7
    return true;
1769
1770
13
  if (State.CC == llvm::CallingConv::X86_FastCall ||
1771
13
      
State.CC == llvm::CallingConv::X86_VectorCall10
||
1772
13
      
State.CC == llvm::CallingConv::X86_RegCall10
) {
1773
4
    if (getContext().getTypeSize(Ty) <= 32 && State.FreeRegs)
1774
3
      NeedsPadding = true;
1775
1776
4
    return false;
1777
4
  }
1778
1779
9
  return true;
1780
13
}
1781
1782
33.0k
bool X86_32ABIInfo::shouldPrimitiveUseInReg(QualType Ty, CCState &State) const {
1783
33.0k
  bool IsPtrOrInt = (getContext().getTypeSize(Ty) <= 32) &&
1784
33.0k
                    
(32.5k
Ty->isIntegralOrEnumerationType()32.5k
||
Ty->isPointerType()21.7k
||
1785
32.5k
                     
Ty->isReferenceType()6.06k
);
1786
1787
33.0k
  if (!IsPtrOrInt && 
(1.23k
State.CC == llvm::CallingConv::X86_FastCall1.23k
||
1788
1.23k
                      
State.CC == llvm::CallingConv::X86_VectorCall1.22k
))
1789
20
    return false;
1790
1791
33.0k
  if (!updateFreeRegs(Ty, State))
1792
32.7k
    return false;
1793
1794
232
  if (!IsPtrOrInt && 
State.CC == llvm::CallingConv::X86_RegCall12
)
1795
4
    return false;
1796
1797
  // Return true to apply inreg to all legal parameters except for MCU targets.
1798
228
  return !IsMCUABI;
1799
232
}
1800
1801
61
void X86_32ABIInfo::runVectorCallFirstPass(CGFunctionInfo &FI, CCState &State) const {
1802
  // Vectorcall x86 works subtly different than in x64, so the format is
1803
  // a bit different than the x64 version.  First, all vector types (not HVAs)
1804
  // are assigned, with the first 6 ending up in the [XYZ]MM0-5 registers.
1805
  // This differs from the x64 implementation, where the first 6 by INDEX get
1806
  // registers.
1807
  // In the second pass over the arguments, HVAs are passed in the remaining
1808
  // vector registers if possible, or indirectly by address. The address will be
1809
  // passed in ECX/EDX if available. Any other arguments are passed according to
1810
  // the usual fastcall rules.
1811
61
  MutableArrayRef<CGFunctionInfoArgInfo> Args = FI.arguments();
1812
190
  for (int I = 0, E = Args.size(); I < E; 
++I129
) {
1813
129
    const Type *Base = nullptr;
1814
129
    uint64_t NumElts = 0;
1815
129
    const QualType &Ty = Args[I].type;
1816
129
    if ((Ty->isVectorType() || 
Ty->isBuiltinType()108
) &&
1817
129
        
isHomogeneousAggregate(Ty, Base, NumElts)99
) {
1818
42
      if (State.FreeSSERegs >= NumElts) {
1819
38
        State.FreeSSERegs -= NumElts;
1820
38
        Args[I].info = ABIArgInfo::getDirectInReg();
1821
38
        State.IsPreassigned.set(I);
1822
38
      }
1823
42
    }
1824
129
  }
1825
61
}
1826
1827
ABIArgInfo X86_32ABIInfo::classifyArgumentType(QualType Ty,
1828
34.5k
                                               CCState &State) const {
1829
  // FIXME: Set alignment on indirect arguments.
1830
34.5k
  bool IsFastCall = State.CC == llvm::CallingConv::X86_FastCall;
1831
34.5k
  bool IsRegCall = State.CC == llvm::CallingConv::X86_RegCall;
1832
34.5k
  bool IsVectorCall = State.CC == llvm::CallingConv::X86_VectorCall;
1833
1834
34.5k
  Ty = useFirstFieldIfTransparentUnion(Ty);
1835
34.5k
  TypeInfo TI = getContext().getTypeInfo(Ty);
1836
1837
  // Check with the C++ ABI first.
1838
34.5k
  const RecordType *RT = Ty->getAs<RecordType>();
1839
34.5k
  if (RT) {
1840
583
    CGCXXABI::RecordArgABI RAA = getRecordArgABI(RT, getCXXABI());
1841
583
    if (RAA == CGCXXABI::RAA_Indirect) {
1842
105
      return getIndirectResult(Ty, false, State);
1843
478
    } else if (RAA == CGCXXABI::RAA_DirectInMemory) {
1844
      // The field index doesn't matter, we'll fix it up later.
1845
86
      return ABIArgInfo::getInAlloca(/*FieldIndex=*/0);
1846
86
    }
1847
583
  }
1848
1849
  // Regcall uses the concept of a homogenous vector aggregate, similar
1850
  // to other targets.
1851
34.4k
  const Type *Base = nullptr;
1852
34.4k
  uint64_t NumElts = 0;
1853
34.4k
  if ((IsRegCall || 
IsVectorCall34.2k
) &&
1854
34.4k
      
isHomogeneousAggregate(Ty, Base, NumElts)200
) {
1855
79
    if (State.FreeSSERegs >= NumElts) {
1856
57
      State.FreeSSERegs -= NumElts;
1857
1858
      // Vectorcall passes HVAs directly and does not flatten them, but regcall
1859
      // does.
1860
57
      if (IsVectorCall)
1861
12
        return getDirectX86Hva();
1862
1863
45
      if (Ty->isBuiltinType() || 
Ty->isVectorType()35
)
1864
20
        return ABIArgInfo::getDirect();
1865
25
      return ABIArgInfo::getExpand();
1866
45
    }
1867
22
    return getIndirectResult(Ty, /*ByVal=*/false, State);
1868
79
  }
1869
1870
34.3k
  if (isAggregateTypeForABI(Ty)) {
1871
    // Structures with flexible arrays are always indirect.
1872
    // FIXME: This should not be byval!
1873
432
    if (RT && 
RT->getDecl()->hasFlexibleArrayMember()344
)
1874
1
      return getIndirectResult(Ty, true, State);
1875
1876
    // Ignore empty structs/unions on non-Windows.
1877
431
    if (!IsWin32StructABI && 
isEmptyRecord(getContext(), Ty, true)255
)
1878
24
      return ABIArgInfo::getIgnore();
1879
1880
407
    llvm::LLVMContext &LLVMContext = getVMContext();
1881
407
    llvm::IntegerType *Int32 = llvm::Type::getInt32Ty(LLVMContext);
1882
407
    bool NeedsPadding = false;
1883
407
    bool InReg;
1884
407
    if (shouldAggregateUseDirect(Ty, State, InReg, NeedsPadding)) {
1885
16
      unsigned SizeInRegs = (TI.Width + 31) / 32;
1886
16
      SmallVector<llvm::Type*, 3> Elements(SizeInRegs, Int32);
1887
16
      llvm::Type *Result = llvm::StructType::get(LLVMContext, Elements);
1888
16
      if (InReg)
1889
9
        return ABIArgInfo::getDirectInReg(Result);
1890
7
      else
1891
7
        return ABIArgInfo::getDirect(Result);
1892
16
    }
1893
391
    llvm::IntegerType *PaddingType = NeedsPadding ? 
Int323
:
nullptr388
;
1894
1895
    // Pass over-aligned aggregates on Windows indirectly. This behavior was
1896
    // added in MSVC 2015.
1897
391
    if (IsWin32StructABI && 
TI.isAlignRequired()176
&&
TI.Align > 321
)
1898
1
      return getIndirectResult(Ty, /*ByVal=*/false, State);
1899
1900
    // Expand small (<= 128-bit) record types when we know that the stack layout
1901
    // of those arguments will match the struct. This is important because the
1902
    // LLVM backend isn't smart enough to remove byval, which inhibits many
1903
    // optimizations.
1904
    // Don't do this for the MCU if there are still free integer registers
1905
    // (see X86_64 ABI for full explanation).
1906
390
    if (TI.Width <= 4 * 32 && 
(343
!IsMCUABI343
||
State.FreeRegs == 05
) &&
1907
390
        
canExpandIndirectArgument(Ty)340
)
1908
88
      return ABIArgInfo::getExpandWithPadding(
1909
88
          IsFastCall || 
IsVectorCall84
||
IsRegCall83
, PaddingType);
1910
1911
302
    return getIndirectResult(Ty, true, State);
1912
390
  }
1913
1914
33.8k
  if (const VectorType *VT = Ty->getAs<VectorType>()) {
1915
    // On Windows, vectors are passed directly if registers are available, or
1916
    // indirectly if not. This avoids the need to align argument memory. Pass
1917
    // user-defined vector types larger than 512 bits indirectly for simplicity.
1918
849
    if (IsWin32StructABI) {
1919
69
      if (TI.Width <= 512 && 
State.FreeSSERegs > 064
) {
1920
52
        --State.FreeSSERegs;
1921
52
        return ABIArgInfo::getDirectInReg();
1922
52
      }
1923
17
      return getIndirectResult(Ty, /*ByVal=*/false, State);
1924
69
    }
1925
1926
    // On Darwin, some vectors are passed in memory, we handle this by passing
1927
    // it as an i8/i16/i32/i64.
1928
780
    if (IsDarwinVectorABI) {
1929
657
      if ((TI.Width == 8 || TI.Width == 16 || TI.Width == 32) ||
1930
657
          
(655
TI.Width == 64655
&&
VT->getNumElements() == 116
))
1931
13
        return ABIArgInfo::getDirect(
1932
13
            llvm::IntegerType::get(getVMContext(), TI.Width));
1933
657
    }
1934
1935
767
    if (IsX86_MMXType(CGT.ConvertType(Ty)))
1936
16
      return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(), 64));
1937
1938
751
    return ABIArgInfo::getDirect();
1939
767
  }
1940
1941
1942
33.0k
  if (const EnumType *EnumTy = Ty->getAs<EnumType>())
1943
25
    Ty = EnumTy->getDecl()->getIntegerType();
1944
1945
33.0k
  bool InReg = shouldPrimitiveUseInReg(Ty, State);
1946
1947
33.0k
  if (isPromotableIntegerTypeForABI(Ty)) {
1948
543
    if (InReg)
1949
38
      return ABIArgInfo::getExtendInReg(Ty);
1950
505
    return ABIArgInfo::getExtend(Ty);
1951
543
  }
1952
1953
32.4k
  if (const auto *EIT = Ty->getAs<BitIntType>()) {
1954
44
    if (EIT->getNumBits() <= 64) {
1955
22
      if (InReg)
1956
0
        return ABIArgInfo::getDirectInReg();
1957
22
      return ABIArgInfo::getDirect();
1958
22
    }
1959
22
    return getIndirectResult(Ty, /*ByVal=*/false, State);
1960
44
  }
1961
1962
32.4k
  if (InReg)
1963
164
    return ABIArgInfo::getDirectInReg();
1964
32.2k
  return ABIArgInfo::getDirect();
1965
32.4k
}
1966
1967
19.5k
void X86_32ABIInfo::computeInfo(CGFunctionInfo &FI) const {
1968
19.5k
  CCState State(FI);
1969
19.5k
  if (IsMCUABI)
1970
28
    State.FreeRegs = 3;
1971
19.5k
  else if (State.CC == llvm::CallingConv::X86_FastCall) {
1972
75
    State.FreeRegs = 2;
1973
75
    State.FreeSSERegs = 3;
1974
19.4k
  } else if (State.CC == llvm::CallingConv::X86_VectorCall) {
1975
61
    State.FreeRegs = 2;
1976
61
    State.FreeSSERegs = 6;
1977
19.4k
  } else if (FI.getHasRegParm())
1978
31
    State.FreeRegs = FI.getRegParm();
1979
19.3k
  else if (State.CC == llvm::CallingConv::X86_RegCall) {
1980
56
    State.FreeRegs = 5;
1981
56
    State.FreeSSERegs = 8;
1982
19.3k
  } else if (IsWin32StructABI) {
1983
    // Since MSVC 2015, the first three SSE vectors have been passed in
1984
    // registers. The rest are passed indirectly.
1985
4.05k
    State.FreeRegs = DefaultNumRegisterParameters;
1986
4.05k
    State.FreeSSERegs = 3;
1987
4.05k
  } else
1988
15.2k
    State.FreeRegs = DefaultNumRegisterParameters;
1989
1990
19.5k
  if (!::classifyReturnType(getCXXABI(), FI, *this)) {
1991
19.4k
    FI.getReturnInfo() = classifyReturnType(FI.getReturnType(), State);
1992
19.4k
  } else 
if (89
FI.getReturnInfo().isIndirect()89
) {
1993
    // The C++ ABI is not aware of register usage, so we have to check if the
1994
    // return value was sret and put it in a register ourselves if appropriate.
1995
89
    if (State.FreeRegs) {
1996
4
      --State.FreeRegs;  // The sret parameter consumes a register.
1997
4
      if (!IsMCUABI)
1998
4
        FI.getReturnInfo().setInReg(true);
1999
4
    }
2000
89
  }
2001
2002
  // The chain argument effectively gives us another free register.
2003
19.5k
  if (FI.isChainCall())
2004
4
    ++State.FreeRegs;
2005
2006
  // For vectorcall, do a first pass over the arguments, assigning FP and vector
2007
  // arguments to XMM registers as available.
2008
19.5k
  if (State.CC == llvm::CallingConv::X86_VectorCall)
2009
61
    runVectorCallFirstPass(FI, State);
2010
2011
19.5k
  bool UsedInAlloca = false;
2012
19.5k
  MutableArrayRef<CGFunctionInfoArgInfo> Args = FI.arguments();
2013
54.2k
  for (int I = 0, E = Args.size(); I < E; 
++I34.6k
) {
2014
    // Skip arguments that have already been assigned.
2015
34.6k
    if (State.IsPreassigned.test(I))
2016
38
      continue;
2017
2018
34.5k
    Args[I].info = classifyArgumentType(Args[I].type, State);
2019
34.5k
    UsedInAlloca |= (Args[I].info.getKind() == ABIArgInfo::InAlloca);
2020
34.5k
  }
2021
2022
  // If we needed to use inalloca for any argument, do a second pass and rewrite
2023
  // all the memory arguments to use inalloca.
2024
19.5k
  if (UsedInAlloca)
2025
73
    rewriteWithInAlloca(FI);
2026
19.5k
}
2027
2028
void
2029
X86_32ABIInfo::addFieldToArgStruct(SmallVector<llvm::Type *, 6> &FrameFields,
2030
                                   CharUnits &StackOffset, ABIArgInfo &Info,
2031
167
                                   QualType Type) const {
2032
  // Arguments are always 4-byte-aligned.
2033
167
  CharUnits WordSize = CharUnits::fromQuantity(4);
2034
167
  assert(StackOffset.isMultipleOf(WordSize) && "unaligned inalloca struct");
2035
2036
  // sret pointers and indirect things will require an extra pointer
2037
  // indirection, unless they are byval. Most things are byval, and will not
2038
  // require this indirection.
2039
167
  bool IsIndirect = false;
2040
167
  if (Info.isIndirect() && 
!Info.getIndirectByVal()17
)
2041
14
    IsIndirect = true;
2042
167
  Info = ABIArgInfo::getInAlloca(FrameFields.size(), IsIndirect);
2043
167
  llvm::Type *LLTy = CGT.ConvertTypeForMem(Type);
2044
167
  if (IsIndirect)
2045
14
    LLTy = LLTy->getPointerTo(0);
2046
167
  FrameFields.push_back(LLTy);
2047
167
  StackOffset += IsIndirect ? 
WordSize14
:
getContext().getTypeSizeInChars(Type)153
;
2048
2049
  // Insert padding bytes to respect alignment.
2050
167
  CharUnits FieldEnd = StackOffset;
2051
167
  StackOffset = FieldEnd.alignTo(WordSize);
2052
167
  if (StackOffset != FieldEnd) {
2053
40
    CharUnits NumBytes = StackOffset - FieldEnd;
2054
40
    llvm::Type *Ty = llvm::Type::getInt8Ty(getVMContext());
2055
40
    Ty = llvm::ArrayType::get(Ty, NumBytes.getQuantity());
2056
40
    FrameFields.push_back(Ty);
2057
40
  }
2058
167
}
2059
2060
177
static bool isArgInAlloca(const ABIArgInfo &Info) {
2061
  // Leave ignored and inreg arguments alone.
2062
177
  switch (Info.getKind()) {
2063
86
  case ABIArgInfo::InAlloca:
2064
86
    return true;
2065
0
  case ABIArgInfo::Ignore:
2066
0
  case ABIArgInfo::IndirectAliased:
2067
0
    return false;
2068
11
  case ABIArgInfo::Indirect:
2069
84
  case ABIArgInfo::Direct:
2070
90
  case ABIArgInfo::Extend:
2071
90
    return !Info.getInReg();
2072
1
  case ABIArgInfo::Expand:
2073
1
  case ABIArgInfo::CoerceAndExpand:
2074
    // These are aggregate types which are never passed in registers when
2075
    // inalloca is involved.
2076
1
    return true;
2077
177
  }
2078
0
  llvm_unreachable("invalid enum");
2079
0
}
2080
2081
73
void X86_32ABIInfo::rewriteWithInAlloca(CGFunctionInfo &FI) const {
2082
73
  assert(IsWin32StructABI && "inalloca only supported on win32");
2083
2084
  // Build a packed struct type for all of the arguments in memory.
2085
73
  SmallVector<llvm::Type *, 6> FrameFields;
2086
2087
  // The stack alignment is always 4.
2088
73
  CharUnits StackAlign = CharUnits::fromQuantity(4);
2089
2090
73
  CharUnits StackOffset;
2091
73
  CGFunctionInfo::arg_iterator I = FI.arg_begin(), E = FI.arg_end();
2092
2093
  // Put 'this' into the struct before 'sret', if necessary.
2094
73
  bool IsThisCall =
2095
73
      FI.getCallingConvention() == llvm::CallingConv::X86_ThisCall;
2096
73
  ABIArgInfo &Ret = FI.getReturnInfo();
2097
73
  if (Ret.isIndirect() && 
Ret.isSRetAfterThis()9
&&
!IsThisCall7
&&
2098
73
      
isArgInAlloca(I->info)6
) {
2099
5
    addFieldToArgStruct(FrameFields, StackOffset, I->info, I->type);
2100
5
    ++I;
2101
5
  }
2102
2103
  // Put the sret parameter into the inalloca struct if it's in memory.
2104
73
  if (Ret.isIndirect() && 
!Ret.getInReg()9
) {
2105
8
    addFieldToArgStruct(FrameFields, StackOffset, Ret, FI.getReturnType());
2106
    // On Windows, the hidden sret parameter is always returned in eax.
2107
8
    Ret.setInAllocaSRet(IsWin32StructABI);
2108
8
  }
2109
2110
  // Skip the 'this' parameter in ecx.
2111
73
  if (IsThisCall)
2112
17
    ++I;
2113
2114
  // Put arguments passed in memory into the struct.
2115
244
  for (; I != E; 
++I171
) {
2116
171
    if (isArgInAlloca(I->info))
2117
154
      addFieldToArgStruct(FrameFields, StackOffset, I->info, I->type);
2118
171
  }
2119
2120
73
  FI.setArgStruct(llvm::StructType::get(getVMContext(), FrameFields,
2121
73
                                        /*isPacked=*/true),
2122
73
                  StackAlign);
2123
73
}
2124
2125
Address X86_32ABIInfo::EmitVAArg(CodeGenFunction &CGF,
2126
50
                                 Address VAListAddr, QualType Ty) const {
2127
2128
50
  auto TypeInfo = getContext().getTypeInfoInChars(Ty);
2129
2130
  // x86-32 changes the alignment of certain arguments on the stack.
2131
  //
2132
  // Just messing with TypeInfo like this works because we never pass
2133
  // anything indirectly.
2134
50
  TypeInfo.Align = CharUnits::fromQuantity(
2135
50
                getTypeStackAlignInBytes(Ty, TypeInfo.Align.getQuantity()));
2136
2137
50
  return emitVoidPtrVAArg(CGF, VAListAddr, Ty, /*Indirect*/ false,
2138
50
                          TypeInfo, CharUnits::fromQuantity(4),
2139
50
                          /*AllowHigherAlign*/ true);
2140
50
}
2141
2142
bool X86_32TargetCodeGenInfo::isStructReturnInRegABI(
2143
3.20k
    const llvm::Triple &Triple, const CodeGenOptions &Opts) {
2144
3.20k
  assert(Triple.getArch() == llvm::Triple::x86);
2145
2146
3.20k
  switch (Opts.getStructReturnConvention()) {
2147
3.20k
  case CodeGenOptions::SRCK_Default:
2148
3.20k
    break;
2149
3
  case CodeGenOptions::SRCK_OnStack:  // -fpcc-struct-return
2150
3
    return false;
2151
3
  case CodeGenOptions::SRCK_InRegs:  // -freg-struct-return
2152
3
    return true;
2153
3.20k
  }
2154
2155
3.20k
  if (Triple.isOSDarwin() || 
Triple.isOSIAMCU()2.99k
)
2156
208
    return true;
2157
2158
2.99k
  switch (Triple.getOS()) {
2159
1
  case llvm::Triple::DragonFly:
2160
1
  case llvm::Triple::FreeBSD:
2161
2
  case llvm::Triple::OpenBSD:
2162
463
  case llvm::Triple::Win32:
2163
463
    return true;
2164
2.53k
  default:
2165
2.53k
    return false;
2166
2.99k
  }
2167
2.99k
}
2168
2169
static void addX86InterruptAttrs(const FunctionDecl *FD, llvm::GlobalValue *GV,
2170
183k
                                 CodeGen::CodeGenModule &CGM) {
2171
183k
  if (!FD->hasAttr<AnyX86InterruptAttr>())
2172
183k
    return;
2173
2174
25
  llvm::Function *Fn = cast<llvm::Function>(GV);
2175
25
  Fn->setCallingConv(llvm::CallingConv::X86_INTR);
2176
25
  if (FD->getNumParams() == 0)
2177
0
    return;
2178
2179
25
  auto PtrTy = cast<PointerType>(FD->getParamDecl(0)->getType());
2180
25
  llvm::Type *ByValTy = CGM.getTypes().ConvertType(PtrTy->getPointeeType());
2181
25
  llvm::Attribute NewAttr = llvm::Attribute::getWithByValType(
2182
25
    Fn->getContext(), ByValTy);
2183
25
  Fn->addParamAttr(0, NewAttr);
2184
25
}
2185
2186
void X86_32TargetCodeGenInfo::setTargetAttributes(
2187
81.8k
    const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {
2188
81.8k
  if (GV->isDeclaration())
2189
46.9k
    return;
2190
34.8k
  if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) {
2191
25.3k
    if (FD->hasAttr<X86ForceAlignArgPointerAttr>()) {
2192
2
      llvm::Function *Fn = cast<llvm::Function>(GV);
2193
2
      Fn->addFnAttr("stackrealign");
2194
2
    }
2195
2196
25.3k
    addX86InterruptAttrs(FD, GV, CGM);
2197
25.3k
  }
2198
34.8k
}
2199
2200
bool X86_32TargetCodeGenInfo::initDwarfEHRegSizeTable(
2201
                                               CodeGen::CodeGenFunction &CGF,
2202
0
                                               llvm::Value *Address) const {
2203
0
  CodeGen::CGBuilderTy &Builder = CGF.Builder;
2204
2205
0
  llvm::Value *Four8 = llvm::ConstantInt::get(CGF.Int8Ty, 4);
2206
2207
  // 0-7 are the eight integer registers;  the order is different
2208
  //   on Darwin (for EH), but the range is the same.
2209
  // 8 is %eip.
2210
0
  AssignToArrayRange(Builder, Address, Four8, 0, 8);
2211
2212
0
  if (CGF.CGM.getTarget().getTriple().isOSDarwin()) {
2213
    // 12-16 are st(0..4).  Not sure why we stop at 4.
2214
    // These have size 16, which is sizeof(long double) on
2215
    // platforms with 8-byte alignment for that type.
2216
0
    llvm::Value *Sixteen8 = llvm::ConstantInt::get(CGF.Int8Ty, 16);
2217
0
    AssignToArrayRange(Builder, Address, Sixteen8, 12, 16);
2218
2219
0
  } else {
2220
    // 9 is %eflags, which doesn't get a size on Darwin for some
2221
    // reason.
2222
0
    Builder.CreateAlignedStore(
2223
0
        Four8, Builder.CreateConstInBoundsGEP1_32(CGF.Int8Ty, Address, 9),
2224
0
                               CharUnits::One());
2225
2226
    // 11-16 are st(0..5).  Not sure why we stop at 5.
2227
    // These have size 12, which is sizeof(long double) on
2228
    // platforms with 4-byte alignment for that type.
2229
0
    llvm::Value *Twelve8 = llvm::ConstantInt::get(CGF.Int8Ty, 12);
2230
0
    AssignToArrayRange(Builder, Address, Twelve8, 11, 16);
2231
0
  }
2232
2233
0
  return false;
2234
0
}
2235
2236
//===----------------------------------------------------------------------===//
2237
// X86-64 ABI Implementation
2238
//===----------------------------------------------------------------------===//
2239
2240
2241
namespace {
2242
/// The AVX ABI level for X86 targets.
2243
enum class X86AVXABILevel {
2244
  None,
2245
  AVX,
2246
  AVX512
2247
};
2248
2249
/// \p returns the size in bits of the largest (native) vector for \p AVXLevel.
2250
4.11k
static unsigned getNativeVectorSizeForAVXABI(X86AVXABILevel AVXLevel) {
2251
4.11k
  switch (AVXLevel) {
2252
3.11k
  case X86AVXABILevel::AVX512:
2253
3.11k
    return 512;
2254
739
  case X86AVXABILevel::AVX:
2255
739
    return 256;
2256
258
  case X86AVXABILevel::None:
2257
258
    return 128;
2258
4.11k
  }
2259
0
  llvm_unreachable("Unknown AVXLevel");
2260
0
}
2261
2262
/// X86_64ABIInfo - The X86_64 ABI information.
2263
class X86_64ABIInfo : public ABIInfo {
2264
  enum Class {
2265
    Integer = 0,
2266
    SSE,
2267
    SSEUp,
2268
    X87,
2269
    X87Up,
2270
    ComplexX87,
2271
    NoClass,
2272
    Memory
2273
  };
2274
2275
  /// merge - Implement the X86_64 ABI merging algorithm.
2276
  ///
2277
  /// Merge an accumulating classification \arg Accum with a field
2278
  /// classification \arg Field.
2279
  ///
2280
  /// \param Accum - The accumulating classification. This should
2281
  /// always be either NoClass or the result of a previous merge
2282
  /// call. In addition, this should never be Memory (the caller
2283
  /// should just return Memory for the aggregate).
2284
  static Class merge(Class Accum, Class Field);
2285
2286
  /// postMerge - Implement the X86_64 ABI post merging algorithm.
2287
  ///
2288
  /// Post merger cleanup, reduces a malformed Hi and Lo pair to
2289
  /// final MEMORY or SSE classes when necessary.
2290
  ///
2291
  /// \param AggregateSize - The size of the current aggregate in
2292
  /// the classification process.
2293
  ///
2294
  /// \param Lo - The classification for the parts of the type
2295
  /// residing in the low word of the containing object.
2296
  ///
2297
  /// \param Hi - The classification for the parts of the type
2298
  /// residing in the higher words of the containing object.
2299
  ///
2300
  void postMerge(unsigned AggregateSize, Class &Lo, Class &Hi) const;
2301
2302
  /// classify - Determine the x86_64 register classes in which the
2303
  /// given type T should be passed.
2304
  ///
2305
  /// \param Lo - The classification for the parts of the type
2306
  /// residing in the low word of the containing object.
2307
  ///
2308
  /// \param Hi - The classification for the parts of the type
2309
  /// residing in the high word of the containing object.
2310
  ///
2311
  /// \param OffsetBase - The bit offset of this type in the
2312
  /// containing object.  Some parameters are classified different
2313
  /// depending on whether they straddle an eightbyte boundary.
2314
  ///
2315
  /// \param isNamedArg - Whether the argument in question is a "named"
2316
  /// argument, as used in AMD64-ABI 3.5.7.
2317
  ///
2318
  /// \param IsRegCall - Whether the calling conversion is regcall.
2319
  ///
2320
  /// If a word is unused its result will be NoClass; if a type should
2321
  /// be passed in Memory then at least the classification of \arg Lo
2322
  /// will be Memory.
2323
  ///
2324
  /// The \arg Lo class will be NoClass iff the argument is ignored.
2325
  ///
2326
  /// If the \arg Lo class is ComplexX87, then the \arg Hi class will
2327
  /// also be ComplexX87.
2328
  void classify(QualType T, uint64_t OffsetBase, Class &Lo, Class &Hi,
2329
                bool isNamedArg, bool IsRegCall = false) const;
2330
2331
  llvm::Type *GetByteVectorType(QualType Ty) const;
2332
  llvm::Type *GetSSETypeAtOffset(llvm::Type *IRType,
2333
                                 unsigned IROffset, QualType SourceTy,
2334
                                 unsigned SourceOffset) const;
2335
  llvm::Type *GetINTEGERTypeAtOffset(llvm::Type *IRType,
2336
                                     unsigned IROffset, QualType SourceTy,
2337
                                     unsigned SourceOffset) const;
2338
2339
  /// getIndirectResult - Give a source type \arg Ty, return a suitable result
2340
  /// such that the argument will be returned in memory.
2341
  ABIArgInfo getIndirectReturnResult(QualType Ty) const;
2342
2343
  /// getIndirectResult - Give a source type \arg Ty, return a suitable result
2344
  /// such that the argument will be passed in memory.
2345
  ///
2346
  /// \param freeIntRegs - The number of free integer registers remaining
2347
  /// available.
2348
  ABIArgInfo getIndirectResult(QualType Ty, unsigned freeIntRegs) const;
2349
2350
  ABIArgInfo classifyReturnType(QualType RetTy) const;
2351
2352
  ABIArgInfo classifyArgumentType(QualType Ty, unsigned freeIntRegs,
2353
                                  unsigned &neededInt, unsigned &neededSSE,
2354
                                  bool isNamedArg,
2355
                                  bool IsRegCall = false) const;
2356
2357
  ABIArgInfo classifyRegCallStructType(QualType Ty, unsigned &NeededInt,
2358
                                       unsigned &NeededSSE,
2359
                                       unsigned &MaxVectorWidth) const;
2360
2361
  ABIArgInfo classifyRegCallStructTypeImpl(QualType Ty, unsigned &NeededInt,
2362
                                           unsigned &NeededSSE,
2363
                                           unsigned &MaxVectorWidth) const;
2364
2365
  bool IsIllegalVectorType(QualType Ty) const;
2366
2367
  /// The 0.98 ABI revision clarified a lot of ambiguities,
2368
  /// unfortunately in ways that were not always consistent with
2369
  /// certain previous compilers.  In particular, platforms which
2370
  /// required strict binary compatibility with older versions of GCC
2371
  /// may need to exempt themselves.
2372
17
  bool honorsRevision0_98() const {
2373
17
    return !getTarget().getTriple().isOSDarwin();
2374
17
  }
2375
2376
  /// GCC classifies <1 x long long> as SSE but some platform ABIs choose to
2377
  /// classify it as INTEGER (for compatibility with older clang compilers).
2378
213
  bool classifyIntegerMMXAsSSE() const {
2379
    // Clang <= 3.8 did not do this.
2380
213
    if (getContext().getLangOpts().getClangABICompat() <=
2381
213
        LangOptions::ClangABI::Ver3_8)
2382
3
      return false;
2383
2384
210
    const llvm::Triple &Triple = getTarget().getTriple();
2385
210
    if (Triple.isOSDarwin() || 
Triple.isPS()104
||
Triple.isOSFreeBSD()94
)
2386
116
      return false;
2387
94
    return true;
2388
210
  }
2389
2390
  // GCC classifies vectors of __int128 as memory.
2391
12.6k
  bool passInt128VectorsInMem() const {
2392
    // Clang <= 9.0 did not do this.
2393
12.6k
    if (getContext().getLangOpts().getClangABICompat() <=
2394
12.6k
        LangOptions::ClangABI::Ver9)
2395
36
      return false;
2396
2397
12.6k
    const llvm::Triple &T = getTarget().getTriple();
2398
12.6k
    return T.isOSLinux() || 
T.isOSNetBSD()11.6k
;
2399
12.6k
  }
2400
2401
  X86AVXABILevel AVXLevel;
2402
  // Some ABIs (e.g. X32 ABI and Native Client OS) use 32 bit pointers on
2403
  // 64-bit hardware.
2404
  bool Has64BitPointers;
2405
2406
public:
2407
  X86_64ABIInfo(CodeGen::CodeGenTypes &CGT, X86AVXABILevel AVXLevel)
2408
      : ABIInfo(CGT), AVXLevel(AVXLevel),
2409
22.0k
        Has64BitPointers(CGT.getDataLayout().getPointerSize(0) == 8) {}
2410
2411
76
  bool isPassedUsingAVXType(QualType type) const {
2412
76
    unsigned neededInt, neededSSE;
2413
    // The freeIntRegs argument doesn't matter here.
2414
76
    ABIArgInfo info = classifyArgumentType(type, 0, neededInt, neededSSE,
2415
76
                                           /*isNamedArg*/true);
2416
76
    if (info.isDirect()) {
2417
73
      llvm::Type *ty = info.getCoerceToType();
2418
73
      if (llvm::VectorType *vectorTy = dyn_cast_or_null<llvm::VectorType>(ty))
2419
2
        return vectorTy->getPrimitiveSizeInBits().getFixedValue() > 128;
2420
73
    }
2421
74
    return false;
2422
76
  }
2423
2424
  void computeInfo(CGFunctionInfo &FI) const override;
2425
2426
  Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
2427
                    QualType Ty) const override;
2428
  Address EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr,
2429
                      QualType Ty) const override;
2430
2431
0
  bool has64BitPointers() const {
2432
0
    return Has64BitPointers;
2433
0
  }
2434
};
2435
2436
/// WinX86_64ABIInfo - The Windows X86_64 ABI information.
2437
class WinX86_64ABIInfo : public ABIInfo {
2438
public:
2439
  WinX86_64ABIInfo(CodeGen::CodeGenTypes &CGT, X86AVXABILevel AVXLevel)
2440
      : ABIInfo(CGT), AVXLevel(AVXLevel),
2441
451
        IsMingw64(getTarget().getTriple().isWindowsGNUEnvironment()) {}
2442
2443
  void computeInfo(CGFunctionInfo &FI) const override;
2444
2445
  Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
2446
                    QualType Ty) const override;
2447
2448
395
  bool isHomogeneousAggregateBaseType(QualType Ty) const override {
2449
    // FIXME: Assumes vectorcall is in use.
2450
395
    return isX86VectorTypeForVectorCall(getContext(), Ty);
2451
395
  }
2452
2453
  bool isHomogeneousAggregateSmallEnough(const Type *Ty,
2454
419
                                         uint64_t NumMembers) const override {
2455
    // FIXME: Assumes vectorcall is in use.
2456
419
    return isX86VectorCallAggregateSmallEnough(NumMembers);
2457
419
  }
2458
2459
private:
2460
  ABIArgInfo classify(QualType Ty, unsigned &FreeSSERegs, bool IsReturnType,
2461
                      bool IsVectorCall, bool IsRegCall) const;
2462
  ABIArgInfo reclassifyHvaArgForVectorCall(QualType Ty, unsigned &FreeSSERegs,
2463
                                           const ABIArgInfo &current) const;
2464
2465
  X86AVXABILevel AVXLevel;
2466
2467
  bool IsMingw64;
2468
};
2469
2470
class X86_64TargetCodeGenInfo : public TargetCodeGenInfo {
2471
public:
2472
  X86_64TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, X86AVXABILevel AVXLevel)
2473
22.0k
      : TargetCodeGenInfo(std::make_unique<X86_64ABIInfo>(CGT, AVXLevel)) {
2474
22.0k
    SwiftInfo =
2475
22.0k
        std::make_unique<SwiftABIInfo>(CGT, /*SwiftErrorInRegister=*/true);
2476
22.0k
  }
2477
2478
  /// Disable tail call on x86-64. The epilogue code before the tail jump blocks
2479
  /// autoreleaseRV/retainRV and autoreleaseRV/unsafeClaimRV optimizations.
2480
161
  bool markARCOptimizedReturnCallsAsNoTail() const override { return true; }
2481
2482
0
  int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const override {
2483
0
    return 7;
2484
0
  }
2485
2486
  bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
2487
0
                               llvm::Value *Address) const override {
2488
0
    llvm::Value *Eight8 = llvm::ConstantInt::get(CGF.Int8Ty, 8);
2489
2490
    // 0-15 are the 16 integer registers.
2491
    // 16 is %rip.
2492
0
    AssignToArrayRange(CGF.Builder, Address, Eight8, 0, 16);
2493
0
    return false;
2494
0
  }
2495
2496
  llvm::Type* adjustInlineAsmType(CodeGen::CodeGenFunction &CGF,
2497
                                  StringRef Constraint,
2498
914
                                  llvm::Type* Ty) const override {
2499
914
    return X86AdjustInlineAsmType(CGF, Constraint, Ty);
2500
914
  }
2501
2502
  bool isNoProtoCallVariadic(const CallArgList &args,
2503
376
                             const FunctionNoProtoType *fnType) const override {
2504
    // The default CC on x86-64 sets %al to the number of SSA
2505
    // registers used, and GCC sets this when calling an unprototyped
2506
    // function, so we override the default behavior.  However, don't do
2507
    // that when AVX types are involved: the ABI explicitly states it is
2508
    // undefined, and it doesn't work in practice because of how the ABI
2509
    // defines varargs anyway.
2510
376
    if (fnType->getCallConv() == CC_C) {
2511
376
      bool HasAVXType = false;
2512
376
      for (CallArgList::const_iterator
2513
450
             it = args.begin(), ie = args.end(); it != ie; 
++it74
) {
2514
76
        if (getABIInfo<X86_64ABIInfo>().isPassedUsingAVXType(it->Ty)) {
2515
2
          HasAVXType = true;
2516
2
          break;
2517
2
        }
2518
76
      }
2519
2520
376
      if (!HasAVXType)
2521
374
        return true;
2522
376
    }
2523
2524
2
    return TargetCodeGenInfo::isNoProtoCallVariadic(args, fnType);
2525
376
  }
2526
2527
  void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
2528
368k
                           CodeGen::CodeGenModule &CGM) const override {
2529
368k
    if (GV->isDeclaration())
2530
206k
      return;
2531
161k
    if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) {
2532
149k
      if (FD->hasAttr<X86ForceAlignArgPointerAttr>()) {
2533
1
        llvm::Function *Fn = cast<llvm::Function>(GV);
2534
1
        Fn->addFnAttr("stackrealign");
2535
1
      }
2536
2537
149k
      addX86InterruptAttrs(FD, GV, CGM);
2538
149k
    }
2539
161k
  }
2540
2541
  void checkFunctionCallABI(CodeGenModule &CGM, SourceLocation CallLoc,
2542
                            const FunctionDecl *Caller,
2543
                            const FunctionDecl *Callee,
2544
                            const CallArgList &Args) const override;
2545
};
2546
2547
static void initFeatureMaps(const ASTContext &Ctx,
2548
                            llvm::StringMap<bool> &CallerMap,
2549
                            const FunctionDecl *Caller,
2550
                            llvm::StringMap<bool> &CalleeMap,
2551
13.5k
                            const FunctionDecl *Callee) {
2552
13.5k
  if (CalleeMap.empty() && 
CallerMap.empty()6.33k
) {
2553
    // The caller is potentially nullptr in the case where the call isn't in a
2554
    // function.  In this case, the getFunctionFeatureMap ensures we just get
2555
    // the TU level setting (since it cannot be modified by 'target'..
2556
6.33k
    Ctx.getFunctionFeatureMap(CallerMap, Caller);
2557
6.33k
    Ctx.getFunctionFeatureMap(CalleeMap, Callee);
2558
6.33k
  }
2559
13.5k
}
2560
2561
static bool checkAVXParamFeature(DiagnosticsEngine &Diag,
2562
                                 SourceLocation CallLoc,
2563
                                 const llvm::StringMap<bool> &CallerMap,
2564
                                 const llvm::StringMap<bool> &CalleeMap,
2565
                                 QualType Ty, StringRef Feature,
2566
13.5k
                                 bool IsArgument) {
2567
13.5k
  bool CallerHasFeat = CallerMap.lookup(Feature);
2568
13.5k
  bool CalleeHasFeat = CalleeMap.lookup(Feature);
2569
13.5k
  if (!CallerHasFeat && 
!CalleeHasFeat56
)
2570
50
    return Diag.Report(CallLoc, diag::warn_avx_calling_convention)
2571
50
           << IsArgument << Ty << Feature;
2572
2573
  // Mixing calling conventions here is very clearly an error.
2574
13.5k
  if (!CallerHasFeat || 
!CalleeHasFeat13.5k
)
2575
6
    return Diag.Report(CallLoc, diag::err_avx_calling_convention)
2576
6
           << IsArgument << Ty << Feature;
2577
2578
  // Else, both caller and callee have the required feature, so there is no need
2579
  // to diagnose.
2580
13.5k
  return false;
2581
13.5k
}
2582
2583
static bool checkAVXParam(DiagnosticsEngine &Diag, ASTContext &Ctx,
2584
                          SourceLocation CallLoc,
2585
                          const llvm::StringMap<bool> &CallerMap,
2586
                          const llvm::StringMap<bool> &CalleeMap, QualType Ty,
2587
13.5k
                          bool IsArgument) {
2588
13.5k
  uint64_t Size = Ctx.getTypeSize(Ty);
2589
13.5k
  if (Size > 256)
2590
6.37k
    return checkAVXParamFeature(Diag, CallLoc, CallerMap, CalleeMap, Ty,
2591
6.37k
                                "avx512f", IsArgument);
2592
2593
7.18k
  if (Size > 128)
2594
7.18k
    return checkAVXParamFeature(Diag, CallLoc, CallerMap, CalleeMap, Ty, "avx",
2595
7.18k
                                IsArgument);
2596
2597
0
  return false;
2598
7.18k
}
2599
2600
void X86_64TargetCodeGenInfo::checkFunctionCallABI(
2601
    CodeGenModule &CGM, SourceLocation CallLoc, const FunctionDecl *Caller,
2602
253k
    const FunctionDecl *Callee, const CallArgList &Args) const {
2603
253k
  llvm::StringMap<bool> CallerMap;
2604
253k
  llvm::StringMap<bool> CalleeMap;
2605
253k
  unsigned ArgIndex = 0;
2606
2607
  // We need to loop through the actual call arguments rather than the
2608
  // function's parameters, in case this variadic.
2609
427k
  for (const CallArg &Arg : Args) {
2610
    // The "avx" feature changes how vectors >128 in size are passed. "avx512f"
2611
    // additionally changes how vectors >256 in size are passed. Like GCC, we
2612
    // warn when a function is called with an argument where this will change.
2613
    // Unlike GCC, we also error when it is an obvious ABI mismatch, that is,
2614
    // the caller and callee features are mismatched.
2615
    // Unfortunately, we cannot do this diagnostic in SEMA, since the callee can
2616
    // change its ABI with attribute-target after this call.
2617
427k
    if (Arg.getType()->isVectorType() &&
2618
427k
        
CGM.getContext().getTypeSize(Arg.getType()) > 12814.8k
) {
2619
7.83k
      initFeatureMaps(CGM.getContext(), CallerMap, Caller, CalleeMap, Callee);
2620
7.83k
      QualType Ty = Arg.getType();
2621
      // The CallArg seems to have desugared the type already, so for clearer
2622
      // diagnostics, replace it with the type in the FunctionDecl if possible.
2623
7.83k
      if (ArgIndex < Callee->getNumParams())
2624
7.80k
        Ty = Callee->getParamDecl(ArgIndex)->getType();
2625
2626
7.83k
      if (checkAVXParam(CGM.getDiags(), CGM.getContext(), CallLoc, CallerMap,
2627
7.83k
                        CalleeMap, Ty, /*IsArgument*/ true))
2628
48
        return;
2629
7.83k
    }
2630
427k
    ++ArgIndex;
2631
427k
  }
2632
2633
  // Check return always, as we don't have a good way of knowing in codegen
2634
  // whether this value is used, tail-called, etc.
2635
253k
  if (Callee->getReturnType()->isVectorType() &&
2636
253k
      
CGM.getContext().getTypeSize(Callee->getReturnType()) > 12810.1k
) {
2637
5.72k
    initFeatureMaps(CGM.getContext(), CallerMap, Caller, CalleeMap, Callee);
2638
5.72k
    checkAVXParam(CGM.getDiags(), CGM.getContext(), CallLoc, CallerMap,
2639
5.72k
                  CalleeMap, Callee->getReturnType(),
2640
5.72k
                  /*IsArgument*/ false);
2641
5.72k
  }
2642
253k
}
2643
2644
115
static std::string qualifyWindowsLibrary(llvm::StringRef Lib) {
2645
  // If the argument does not end in .lib, automatically add the suffix.
2646
  // If the argument contains a space, enclose it in quotes.
2647
  // This matches the behavior of MSVC.
2648
115
  bool Quote = Lib.contains(' ');
2649
115
  std::string ArgStr = Quote ? 
"\""4
:
""111
;
2650
115
  ArgStr += Lib;
2651
115
  if (!Lib.ends_with_insensitive(".lib") && 
!Lib.ends_with_insensitive(".a")101
)
2652
101
    ArgStr += ".lib";
2653
115
  ArgStr += Quote ? 
"\""4
:
""111
;
2654
115
  return ArgStr;
2655
115
}
2656
2657
class WinX86_32TargetCodeGenInfo : public X86_32TargetCodeGenInfo {
2658
public:
2659
  WinX86_32TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT,
2660
        bool DarwinVectorABI, bool RetSmallStructInRegABI, bool Win32StructABI,
2661
        unsigned NumRegisterParameters)
2662
    : X86_32TargetCodeGenInfo(CGT, DarwinVectorABI, RetSmallStructInRegABI,
2663
463
        Win32StructABI, NumRegisterParameters, false) {}
2664
2665
  void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
2666
                           CodeGen::CodeGenModule &CGM) const override;
2667
2668
  void getDependentLibraryOption(llvm::StringRef Lib,
2669
41
                                 llvm::SmallString<24> &Opt) const override {
2670
41
    Opt = "/DEFAULTLIB:";
2671
41
    Opt += qualifyWindowsLibrary(Lib);
2672
41
  }
2673
2674
  void getDetectMismatchOption(llvm::StringRef Name,
2675
                               llvm::StringRef Value,
2676
2
                               llvm::SmallString<32> &Opt) const override {
2677
2
    Opt = "/FAILIFMISMATCH:\"" + Name.str() + "=" + Value.str() + "\"";
2678
2
  }
2679
};
2680
2681
static void addStackProbeTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
2682
17.9k
                                          CodeGen::CodeGenModule &CGM) {
2683
17.9k
  if (llvm::Function *Fn = dyn_cast_or_null<llvm::Function>(GV)) {
2684
2685
15.4k
    if (CGM.getCodeGenOpts().StackProbeSize != 4096)
2686
1
      Fn->addFnAttr("stack-probe-size",
2687
1
                    llvm::utostr(CGM.getCodeGenOpts().StackProbeSize));
2688
15.4k
    if (CGM.getCodeGenOpts().NoStackArgProbe)
2689
4
      Fn->addFnAttr("no-stack-arg-probe");
2690
15.4k
  }
2691
17.9k
}
2692
2693
void WinX86_32TargetCodeGenInfo::setTargetAttributes(
2694
17.9k
    const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {
2695
17.9k
  X86_32TargetCodeGenInfo::setTargetAttributes(D, GV, CGM);
2696
17.9k
  if (GV->isDeclaration())
2697
10.2k
    return;
2698
7.73k
  addStackProbeTargetAttributes(D, GV, CGM);
2699
7.73k
}
2700
2701
class WinX86_64TargetCodeGenInfo : public TargetCodeGenInfo {
2702
public:
2703
  WinX86_64TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT,
2704
                             X86AVXABILevel AVXLevel)
2705
438
      : TargetCodeGenInfo(std::make_unique<WinX86_64ABIInfo>(CGT, AVXLevel)) {
2706
438
    SwiftInfo =
2707
438
        std::make_unique<SwiftABIInfo>(CGT, /*SwiftErrorInRegister=*/true);
2708
438
  }
2709
2710
  void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
2711
                           CodeGen::CodeGenModule &CGM) const override;
2712
2713
0
  int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const override {
2714
0
    return 7;
2715
0
  }
2716
2717
  bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
2718
0
                               llvm::Value *Address) const override {
2719
0
    llvm::Value *Eight8 = llvm::ConstantInt::get(CGF.Int8Ty, 8);
2720
2721
    // 0-15 are the 16 integer registers.
2722
    // 16 is %rip.
2723
0
    AssignToArrayRange(CGF.Builder, Address, Eight8, 0, 16);
2724
0
    return false;
2725
0
  }
2726
2727
  void getDependentLibraryOption(llvm::StringRef Lib,
2728
51
                                 llvm::SmallString<24> &Opt) const override {
2729
51
    Opt = "/DEFAULTLIB:";
2730
51
    Opt += qualifyWindowsLibrary(Lib);
2731
51
  }
2732
2733
  void getDetectMismatchOption(llvm::StringRef Name,
2734
                               llvm::StringRef Value,
2735
7
                               llvm::SmallString<32> &Opt) const override {
2736
7
    Opt = "/FAILIFMISMATCH:\"" + Name.str() + "=" + Value.str() + "\"";
2737
7
  }
2738
};
2739
2740
void WinX86_64TargetCodeGenInfo::setTargetAttributes(
2741
20.3k
    const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {
2742
20.3k
  TargetCodeGenInfo::setTargetAttributes(D, GV, CGM);
2743
20.3k
  if (GV->isDeclaration())
2744
11.0k
    return;
2745
9.28k
  if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) {
2746
8.16k
    if (FD->hasAttr<X86ForceAlignArgPointerAttr>()) {
2747
0
      llvm::Function *Fn = cast<llvm::Function>(GV);
2748
0
      Fn->addFnAttr("stackrealign");
2749
0
    }
2750
2751
8.16k
    addX86InterruptAttrs(FD, GV, CGM);
2752
8.16k
  }
2753
2754
9.28k
  addStackProbeTargetAttributes(D, GV, CGM);
2755
9.28k
}
2756
}
2757
2758
void X86_64ABIInfo::postMerge(unsigned AggregateSize, Class &Lo,
2759
16.9k
                              Class &Hi) const {
2760
  // AMD64-ABI 3.2.3p2: Rule 5. Then a post merger cleanup is done:
2761
  //
2762
  // (a) If one of the classes is Memory, the whole argument is passed in
2763
  //     memory.
2764
  //
2765
  // (b) If X87UP is not preceded by X87, the whole argument is passed in
2766
  //     memory.
2767
  //
2768
  // (c) If the size of the aggregate exceeds two eightbytes and the first
2769
  //     eightbyte isn't SSE or any other eightbyte isn't SSEUP, the whole
2770
  //     argument is passed in memory. NOTE: This is necessary to keep the
2771
  //     ABI working for processors that don't support the __m256 type.
2772
  //
2773
  // (d) If SSEUP is not preceded by SSE or SSEUP, it is converted to SSE.
2774
  //
2775
  // Some of these are enforced by the merging logic.  Others can arise
2776
  // only with unions; for example:
2777
  //   union { _Complex double; unsigned; }
2778
  //
2779
  // Note that clauses (b) and (c) were added in 0.98.
2780
  //
2781
16.9k
  if (Hi == Memory)
2782
0
    Lo = Memory;
2783
16.9k
  if (Hi == X87Up && 
Lo != X87346
&&
honorsRevision0_98()17
)
2784
15
    Lo = Memory;
2785
16.9k
  if (AggregateSize > 128 && 
(1.05k
Lo != SSE1.05k
||
Hi != SSEUp24
))
2786
1.03k
    Lo = Memory;
2787
16.9k
  if (Hi == SSEUp && 
Lo != SSE33
)
2788
1
    Hi = SSE;
2789
16.9k
}
2790
2791
31.6k
X86_64ABIInfo::Class X86_64ABIInfo::merge(Class Accum, Class Field) {
2792
  // AMD64-ABI 3.2.3p2: Rule 4. Each field of an object is
2793
  // classified recursively so that always two fields are
2794
  // considered. The resulting class is calculated according to
2795
  // the classes of the fields in the eightbyte:
2796
  //
2797
  // (a) If both classes are equal, this is the resulting class.
2798
  //
2799
  // (b) If one of the classes is NO_CLASS, the resulting class is
2800
  // the other class.
2801
  //
2802
  // (c) If one of the classes is MEMORY, the result is the MEMORY
2803
  // class.
2804
  //
2805
  // (d) If one of the classes is INTEGER, the result is the
2806
  // INTEGER.
2807
  //
2808
  // (e) If one of the classes is X87, X87UP, COMPLEX_X87 class,
2809
  // MEMORY is used as class.
2810
  //
2811
  // (f) Otherwise class SSE is used.
2812
2813
  // Accum should never be memory (we should have returned) or
2814
  // ComplexX87 (because this cannot be passed in a structure).
2815
31.6k
  assert((Accum != Memory && Accum != ComplexX87) &&
2816
31.6k
         "Invalid accumulated classification during merge.");
2817
31.6k
  if (Accum == Field || 
Field == NoClass16.5k
)
2818
18.9k
    return Accum;
2819
12.6k
  if (Field == Memory)
2820
18
    return Memory;
2821
12.6k
  if (Accum == NoClass)
2822
12.5k
    return Field;
2823
78
  if (Accum == Integer || 
Field == Integer55
)
2824
75
    return Integer;
2825
3
  if (Field == X87 || Field == X87Up || Field == ComplexX87 ||
2826
3
      Accum == X87 || 
Accum == X87Up0
)
2827
3
    return Memory;
2828
0
  return SSE;
2829
3
}
2830
2831
void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase, Class &Lo,
2832
419k
                             Class &Hi, bool isNamedArg, bool IsRegCall) const {
2833
  // FIXME: This code can be simplified by introducing a simple value class for
2834
  // Class pairs with appropriate constructor methods for the various
2835
  // situations.
2836
2837
  // FIXME: Some of the split computations are wrong; unaligned vectors
2838
  // shouldn't be passed in registers for example, so there is no chance they
2839
  // can straddle an eightbyte. Verify & simplify.
2840
2841
419k
  Lo = Hi = NoClass;
2842
2843
419k
  Class &Current = OffsetBase < 64 ? 
Lo415k
:
Hi3.94k
;
2844
419k
  Current = Memory;
2845
2846
419k
  if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
2847
162k
    BuiltinType::Kind k = BT->getKind();
2848
2849
162k
    if (k == BuiltinType::Void) {
2850
59.9k
      Current = NoClass;
2851
102k
    } else if (k == BuiltinType::Int128 || 
k == BuiltinType::UInt128102k
) {
2852
32
      Lo = Integer;
2853
32
      Hi = Integer;
2854
102k
    } else if (k >= BuiltinType::Bool && 
k <= BuiltinType::LongLong102k
) {
2855
96.7k
      Current = Integer;
2856
96.7k
    } else 
if (5.84k
k == BuiltinType::Float5.84k
||
k == BuiltinType::Double3.51k
||
2857
5.84k
               
k == BuiltinType::Float161.88k
||
k == BuiltinType::BFloat161.47k
) {
2858
4.64k
      Current = SSE;
2859
4.64k
    } else 
if (1.19k
k == BuiltinType::LongDouble1.19k
) {
2860
972
      const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();
2861
972
      if (LDF == &llvm::APFloat::IEEEquad()) {
2862
40
        Lo = SSE;
2863
40
        Hi = SSEUp;
2864
932
      } else if (LDF == &llvm::APFloat::x87DoubleExtended()) {
2865
910
        Lo = X87;
2866
910
        Hi = X87Up;
2867
910
      } else 
if (22
LDF == &llvm::APFloat::IEEEdouble()22
) {
2868
22
        Current = SSE;
2869
22
      } else
2870
0
        llvm_unreachable("unexpected long double representation!");
2871
972
    }
2872
    // FIXME: _Decimal32 and _Decimal64 are SSE.
2873
    // FIXME: _float128 and _Decimal128 are (SSE, SSEUp).
2874
162k
    return;
2875
162k
  }
2876
2877
256k
  if (const EnumType *ET = Ty->getAs<EnumType>()) {
2878
    // Classify the underlying integer type.
2879
990
    classify(ET->getDecl()->getIntegerType(), OffsetBase, Lo, Hi, isNamedArg);
2880
990
    return;
2881
990
  }
2882
2883
255k
  if (Ty->hasPointerRepresentation()) {
2884
230k
    Current = Integer;
2885
230k
    return;
2886
230k
  }
2887
2888
25.5k
  if (Ty->isMemberPointerType()) {
2889
225
    if (Ty->isMemberFunctionPointerType()) {
2890
176
      if (Has64BitPointers) {
2891
        // If Has64BitPointers, this is an {i64, i64}, so classify both
2892
        // Lo and Hi now.
2893
168
        Lo = Hi = Integer;
2894
168
      } else {
2895
        // Otherwise, with 32-bit pointers, this is an {i32, i32}. If that
2896
        // straddles an eightbyte boundary, Hi should be classified as well.
2897
8
        uint64_t EB_FuncPtr = (OffsetBase) / 64;
2898
8
        uint64_t EB_ThisAdj = (OffsetBase + 64 - 1) / 64;
2899
8
        if (EB_FuncPtr != EB_ThisAdj) {
2900
2
          Lo = Hi = Integer;
2901
6
        } else {
2902
6
          Current = Integer;
2903
6
        }
2904
8
      }
2905
176
    } else {
2906
49
      Current = Integer;
2907
49
    }
2908
225
    return;
2909
225
  }
2910
2911
25.3k
  if (const VectorType *VT = Ty->getAs<VectorType>()) {
2912
6.76k
    uint64_t Size = getContext().getTypeSize(VT);
2913
6.76k
    if (Size == 1 || Size == 8 || 
Size == 166.76k
||
Size == 326.76k
) {
2914
      // gcc passes the following as integer:
2915
      // 4 bytes - <4 x char>, <2 x short>, <1 x int>, <1 x float>
2916
      // 2 bytes - <2 x char>, <1 x short>
2917
      // 1 byte  - <1 x char>
2918
26
      Current = Integer;
2919
2920
      // If this type crosses an eightbyte boundary, it should be
2921
      // split.
2922
26
      uint64_t EB_Lo = (OffsetBase) / 64;
2923
26
      uint64_t EB_Hi = (OffsetBase + Size - 1) / 64;
2924
26
      if (EB_Lo != EB_Hi)
2925
0
        Hi = Lo;
2926
6.74k
    } else if (Size == 64) {
2927
213
      QualType ElementType = VT->getElementType();
2928
2929
      // gcc passes <1 x double> in memory. :(
2930
213
      if (ElementType->isSpecificBuiltinType(BuiltinType::Double))
2931
0
        return;
2932
2933
      // gcc passes <1 x long long> as SSE but clang used to unconditionally
2934
      // pass them as integer.  For platforms where clang is the de facto
2935
      // platform compiler, we must continue to use integer.
2936
213
      if (!classifyIntegerMMXAsSSE() &&
2937
213
          
(119
ElementType->isSpecificBuiltinType(BuiltinType::LongLong)119
||
2938
119
           
ElementType->isSpecificBuiltinType(BuiltinType::ULongLong)34
||
2939
119
           
ElementType->isSpecificBuiltinType(BuiltinType::Long)34
||
2940
119
           
ElementType->isSpecificBuiltinType(BuiltinType::ULong)34
))
2941
85
        Current = Integer;
2942
128
      else
2943
128
        Current = SSE;
2944
2945
      // If this type crosses an eightbyte boundary, it should be
2946
      // split.
2947
213
      if (OffsetBase && 
OffsetBase != 640
)
2948
0
        Hi = Lo;
2949
6.52k
    } else if (Size == 128 ||
2950
6.52k
               
(3.88k
isNamedArg3.88k
&&
Size <= getNativeVectorSizeForAVXABI(AVXLevel)3.84k
)) {
2951
6.32k
      QualType ElementType = VT->getElementType();
2952
2953
      // gcc passes 256 and 512 bit <X x __int128> vectors in memory. :(
2954
6.32k
      if (passInt128VectorsInMem() && 
Size != 128533
&&
2955
6.32k
          
(256
ElementType->isSpecificBuiltinType(BuiltinType::Int128)256
||
2956
256
           ElementType->isSpecificBuiltinType(BuiltinType::UInt128)))
2957
6
        return;
2958
2959
      // Arguments of 256-bits are split into four eightbyte chunks. The
2960
      // least significant one belongs to class SSE and all the others to class
2961
      // SSEUP. The original Lo and Hi design considers that types can't be
2962
      // greater than 128-bits, so a 64-bit split in Hi and Lo makes sense.
2963
      // This design isn't correct for 256-bits, but since there're no cases
2964
      // where the upper parts would need to be inspected, avoid adding
2965
      // complexity and just consider Hi to match the 64-256 part.
2966
      //
2967
      // Note that per 3.5.7 of AMD64-ABI, 256-bit args are only passed in
2968
      // registers if they are "named", i.e. not part of the "..." of a
2969
      // variadic function.
2970
      //
2971
      // Similarly, per 3.2.3. of the AVX512 draft, 512-bits ("named") args are
2972
      // split into eight eightbyte chunks, one SSE and seven SSEUP.
2973
6.32k
      Lo = SSE;
2974
6.32k
      Hi = SSEUp;
2975
6.32k
    }
2976
6.76k
    return;
2977
6.76k
  }
2978
2979
18.5k
  if (const ComplexType *CT = Ty->getAs<ComplexType>()) {
2980
544
    QualType ET = getContext().getCanonicalType(CT->getElementType());
2981
2982
544
    uint64_t Size = getContext().getTypeSize(Ty);
2983
544
    if (ET->isIntegralOrEnumerationType()) {
2984
34
      if (Size <= 64)
2985
30
        Current = Integer;
2986
4
      else if (Size <= 128)
2987
4
        Lo = Hi = Integer;
2988
510
    } else if (ET->isFloat16Type() || 
ET == getContext().FloatTy418
||
2989
510
               
ET->isBFloat16Type()252
) {
2990
258
      Current = SSE;
2991
258
    } else 
if (252
ET == getContext().DoubleTy252
) {
2992
143
      Lo = Hi = SSE;
2993
143
    } else 
if (109
ET == getContext().LongDoubleTy109
) {
2994
108
      const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();
2995
108
      if (LDF == &llvm::APFloat::IEEEquad())
2996
8
        Current = Memory;
2997
100
      else if (LDF == &llvm::APFloat::x87DoubleExtended())
2998
92
        Current = ComplexX87;
2999
8
      else if (LDF == &llvm::APFloat::IEEEdouble())
3000
8
        Lo = Hi = SSE;
3001
0
      else
3002
0
        llvm_unreachable("unexpected long double representation!");
3003
108
    }
3004
3005
    // If this complex type crosses an eightbyte boundary then it
3006
    // should be split.
3007
544
    uint64_t EB_Real = (OffsetBase) / 64;
3008
544
    uint64_t EB_Imag = (OffsetBase + getContext().getTypeSize(ET)) / 64;
3009
544
    if (Hi == NoClass && 
EB_Real != EB_Imag389
)
3010
104
      Hi = Lo;
3011
3012
544
    return;
3013
544
  }
3014
3015
18.0k
  if (const auto *EITy = Ty->getAs<BitIntType>()) {
3016
95
    if (EITy->getNumBits() <= 64)
3017
70
      Current = Integer;
3018
25
    else if (EITy->getNumBits() <= 128)
3019
21
      Lo = Hi = Integer;
3020
    // Larger values need to get passed in memory.
3021
95
    return;
3022
95
  }
3023
3024
17.9k
  if (const ConstantArrayType *AT = getContext().getAsConstantArrayType(Ty)) {
3025
    // Arrays are treated like structures.
3026
3027
59
    uint64_t Size = getContext().getTypeSize(Ty);
3028
3029
    // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger
3030
    // than eight eightbytes, ..., it has class MEMORY.
3031
    // regcall ABI doesn't have limitation to an object. The only limitation
3032
    // is the free registers, which will be checked in computeInfo.
3033
59
    if (!IsRegCall && 
Size > 51249
)
3034
0
      return;
3035
3036
    // AMD64-ABI 3.2.3p2: Rule 1. If ..., or it contains unaligned
3037
    // fields, it has class MEMORY.
3038
    //
3039
    // Only need to check alignment of array base.
3040
59
    if (OffsetBase % getContext().getTypeAlign(AT->getElementType()))
3041
0
      return;
3042
3043
    // Otherwise implement simplified merge. We could be smarter about
3044
    // this, but it isn't worth it and would be harder to verify.
3045
59
    Current = NoClass;
3046
59
    uint64_t EltSize = getContext().getTypeSize(AT->getElementType());
3047
59
    uint64_t ArraySize = AT->getSize().getZExtValue();
3048
3049
    // The only case a 256-bit wide vector could be used is when the array
3050
    // contains a single 256-bit element. Since Lo and Hi logic isn't extended
3051
    // to work for sizes wider than 128, early check and fallback to memory.
3052
    //
3053
59
    if (Size > 128 &&
3054
59
        
(19
Size != EltSize19
||
Size > getNativeVectorSizeForAVXABI(AVXLevel)4
))
3055
15
      return;
3056
3057
200
    
for (uint64_t i=0, Offset=OffsetBase; 44
i<ArraySize;
++i, Offset += EltSize156
) {
3058
156
      Class FieldLo, FieldHi;
3059
156
      classify(AT->getElementType(), Offset, FieldLo, FieldHi, isNamedArg);
3060
156
      Lo = merge(Lo, FieldLo);
3061
156
      Hi = merge(Hi, FieldHi);
3062
156
      if (Lo == Memory || Hi == Memory)
3063
0
        break;
3064
156
    }
3065
3066
44
    postMerge(Size, Lo, Hi);
3067
44
    assert((Hi != SSEUp || Lo == SSE) && "Invalid SSEUp array classification.");
3068
44
    return;
3069
44
  }
3070
3071
17.8k
  if (const RecordType *RT = Ty->getAs<RecordType>()) {
3072
17.5k
    uint64_t Size = getContext().getTypeSize(Ty);
3073
3074
    // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger
3075
    // than eight eightbytes, ..., it has class MEMORY.
3076
17.5k
    if (Size > 512)
3077
160
      return;
3078
3079
    // AMD64-ABI 3.2.3p2: Rule 2. If a C++ object has either a non-trivial
3080
    // copy constructor or a non-trivial destructor, it is passed by invisible
3081
    // reference.
3082
17.3k
    if (getRecordArgABI(RT, getCXXABI()))
3083
505
      return;
3084
3085
16.8k
    const RecordDecl *RD = RT->getDecl();
3086
3087
    // Assume variable sized types are passed in memory.
3088
16.8k
    if (RD->hasFlexibleArrayMember())
3089
0
      return;
3090
3091
16.8k
    const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
3092
3093
    // Reset Lo class, this will be recomputed.
3094
16.8k
    Current = NoClass;
3095
3096
    // If this is a C++ record, classify the bases first.
3097
16.8k
    if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
3098
16.3k
      for (const auto &I : CXXRD->bases()) {
3099
3.18k
        assert(!I.isVirtual() && !I.getType()->isDependentType() &&
3100
3.18k
               "Unexpected base class!");
3101
3.18k
        const auto *Base =
3102
3.18k
            cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
3103
3104
        // Classify this field.
3105
        //
3106
        // AMD64-ABI 3.2.3p2: Rule 3. If the size of the aggregate exceeds a
3107
        // single eightbyte, each is classified separately. Each eightbyte gets
3108
        // initialized to class NO_CLASS.
3109
3.18k
        Class FieldLo, FieldHi;
3110
3.18k
        uint64_t Offset =
3111
3.18k
          OffsetBase + getContext().toBits(Layout.getBaseClassOffset(Base));
3112
3.18k
        classify(I.getType(), Offset, FieldLo, FieldHi, isNamedArg);
3113
3.18k
        Lo = merge(Lo, FieldLo);
3114
3.18k
        Hi = merge(Hi, FieldHi);
3115
3.18k
        if (Lo == Memory || 
Hi == Memory3.17k
) {
3116
7
          postMerge(Size, Lo, Hi);
3117
7
          return;
3118
7
        }
3119
3.18k
      }
3120
16.3k
    }
3121
3122
    // Classify the fields one at a time, merging the results.
3123
16.8k
    unsigned idx = 0;
3124
16.8k
    bool UseClang11Compat = getContext().getLangOpts().getClangABICompat() <=
3125
16.8k
                                LangOptions::ClangABI::Ver11 ||
3126
16.8k
                            
getContext().getTargetInfo().getTriple().isPS()16.7k
;
3127
16.8k
    bool IsUnion = RT->isUnionType() && 
!UseClang11Compat80
;
3128
3129
16.8k
    for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
3130
29.3k
           i != e; 
++i, ++idx12.5k
) {
3131
13.5k
      uint64_t Offset = OffsetBase + Layout.getFieldOffset(idx);
3132
13.5k
      bool BitField = i->isBitField();
3133
3134
      // Ignore padding bit-fields.
3135
13.5k
      if (BitField && 
i->isUnnamedBitfield()92
)
3136
29
        continue;
3137
3138
      // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger than
3139
      // eight eightbytes, or it contains unaligned fields, it has class MEMORY.
3140
      //
3141
      // The only case a 256-bit or a 512-bit wide vector could be used is when
3142
      // the struct contains a single 256-bit or 512-bit element. Early check
3143
      // and fallback to memory.
3144
      //
3145
      // FIXME: Extended the Lo and Hi logic properly to work for size wider
3146
      // than 128.
3147
13.5k
      if (Size > 128 &&
3148
13.5k
          
(1.05k
(1.05k
!IsUnion1.05k
&&
Size != getContext().getTypeSize(i->getType())1.04k
) ||
3149
1.05k
           
Size > getNativeVectorSizeForAVXABI(AVXLevel)112
)) {
3150
1.01k
        Lo = Memory;
3151
1.01k
        postMerge(Size, Lo, Hi);
3152
1.01k
        return;
3153
1.01k
      }
3154
      // Note, skip this test for bit-fields, see below.
3155
12.5k
      if (!BitField && 
Offset % getContext().getTypeAlign(i->getType())12.4k
) {
3156
14
        Lo = Memory;
3157
14
        postMerge(Size, Lo, Hi);
3158
14
        return;
3159
14
      }
3160
3161
      // Classify this field.
3162
      //
3163
      // AMD64-ABI 3.2.3p2: Rule 3. If the size of the aggregate
3164
      // exceeds a single eightbyte, each is classified
3165
      // separately. Each eightbyte gets initialized to class
3166
      // NO_CLASS.
3167
12.4k
      Class FieldLo, FieldHi;
3168
3169
      // Bit-fields require special handling, they do not force the
3170
      // structure to be passed in memory even if unaligned, and
3171
      // therefore they can straddle an eightbyte.
3172
12.4k
      if (BitField) {
3173
63
        assert(!i->isUnnamedBitfield());
3174
63
        uint64_t Offset = OffsetBase + Layout.getFieldOffset(idx);
3175
63
        uint64_t Size = i->getBitWidthValue(getContext());
3176
3177
63
        uint64_t EB_Lo = Offset / 64;
3178
63
        uint64_t EB_Hi = (Offset + Size - 1) / 64;
3179
3180
63
        if (EB_Lo) {
3181
1
          assert(EB_Hi == EB_Lo && "Invalid classification, type > 16 bytes.");
3182
1
          FieldLo = NoClass;
3183
1
          FieldHi = Integer;
3184
62
        } else {
3185
62
          FieldLo = Integer;
3186
62
          FieldHi = EB_Hi ? 
Integer0
: NoClass;
3187
62
        }
3188
63
      } else
3189
12.4k
        classify(i->getType(), Offset, FieldLo, FieldHi, isNamedArg);
3190
12.4k
      Lo = merge(Lo, FieldLo);
3191
12.4k
      Hi = merge(Hi, FieldHi);
3192
12.4k
      if (Lo == Memory || 
Hi == Memory12.4k
)
3193
14
        break;
3194
12.4k
    }
3195
3196
15.8k
    postMerge(Size, Lo, Hi);
3197
15.8k
  }
3198
17.8k
}
3199
3200
614
ABIArgInfo X86_64ABIInfo::getIndirectReturnResult(QualType Ty) const {
3201
  // If this is a scalar LLVM value then assume LLVM will pass it in the right
3202
  // place naturally.
3203
614
  if (!isAggregateTypeForABI(Ty)) {
3204
    // Treat an enum type as its underlying type.
3205
156
    if (const EnumType *EnumTy = Ty->getAs<EnumType>())
3206
0
      Ty = EnumTy->getDecl()->getIntegerType();
3207
3208
156
    if (Ty->isBitIntType())
3209
3
      return getNaturalAlignIndirect(Ty);
3210
3211
153
    return (isPromotableIntegerTypeForABI(Ty) ? 
ABIArgInfo::getExtend(Ty)0
3212
153
                                              : ABIArgInfo::getDirect());
3213
156
  }
3214
3215
458
  return getNaturalAlignIndirect(Ty);
3216
614
}
3217
3218
3.17k
bool X86_64ABIInfo::IsIllegalVectorType(QualType Ty) const {
3219
3.17k
  if (const VectorType *VecTy = Ty->getAs<VectorType>()) {
3220
153
    uint64_t Size = getContext().getTypeSize(VecTy);
3221
153
    unsigned LargestVector = getNativeVectorSizeForAVXABI(AVXLevel);
3222
153
    if (Size <= 64 || 
Size > LargestVector147
)
3223
126
      return true;
3224
27
    QualType EltTy = VecTy->getElementType();
3225
27
    if (passInt128VectorsInMem() &&
3226
27
        
(9
EltTy->isSpecificBuiltinType(BuiltinType::Int128)9
||
3227
9
         EltTy->isSpecificBuiltinType(BuiltinType::UInt128)))
3228
6
      return true;
3229
27
  }
3230
3231
3.03k
  return false;
3232
3.17k
}
3233
3234
ABIArgInfo X86_64ABIInfo::getIndirectResult(QualType Ty,
3235
4.83k
                                            unsigned freeIntRegs) const {
3236
  // If this is a scalar LLVM value then assume LLVM will pass it in the right
3237
  // place naturally.
3238
  //
3239
  // This assumption is optimistic, as there could be free registers available
3240
  // when we need to pass this argument in memory, and LLVM could try to pass
3241
  // the argument in the free register. This does not seem to happen currently,
3242
  // but this code would be much safer if we could mark the argument with
3243
  // 'onstack'. See PR12193.
3244
4.83k
  if (!isAggregateTypeForABI(Ty) && 
!IsIllegalVectorType(Ty)3.17k
&&
3245
4.83k
      
!Ty->isBitIntType()3.03k
) {
3246
    // Treat an enum type as its underlying type.
3247
3.03k
    if (const EnumType *EnumTy = Ty->getAs<EnumType>())
3248
0
      Ty = EnumTy->getDecl()->getIntegerType();
3249
3250
3.03k
    return (isPromotableIntegerTypeForABI(Ty) ? 
ABIArgInfo::getExtend(Ty)340
3251
3.03k
                                              : 
ABIArgInfo::getDirect()2.69k
);
3252
3.03k
  }
3253
3254
1.80k
  if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))
3255
515
    return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
3256
3257
  // Compute the byval alignment. We specify the alignment of the byval in all
3258
  // cases so that the mid-level optimizer knows the alignment of the byval.
3259
1.28k
  unsigned Align = std::max(getContext().getTypeAlign(Ty) / 8, 8U);
3260
3261
  // Attempt to avoid passing indirect results using byval when possible. This
3262
  // is important for good codegen.
3263
  //
3264
  // We do this by coercing the value into a scalar type which the backend can
3265
  // handle naturally (i.e., without using byval).
3266
  //
3267
  // For simplicity, we currently only do this when we have exhausted all of the
3268
  // free integer registers. Doing this when there are free integer registers
3269
  // would require more care, as we would have to ensure that the coerced value
3270
  // did not claim the unused register. That would require either reording the
3271
  // arguments to the function (so that any subsequent inreg values came first),
3272
  // or only doing this optimization when there were no following arguments that
3273
  // might be inreg.
3274
  //
3275
  // We currently expect it to be rare (particularly in well written code) for
3276
  // arguments to be passed on the stack when there are still free integer
3277
  // registers available (this would typically imply large structs being passed
3278
  // by value), so this seems like a fair tradeoff for now.
3279
  //
3280
  // We can revisit this if the backend grows support for 'onstack' parameter
3281
  // attributes. See PR12193.
3282
1.28k
  if (freeIntRegs == 0) {
3283
90
    uint64_t Size = getContext().getTypeSize(Ty);
3284
3285
    // If this type fits in an eightbyte, coerce it into the matching integral
3286
    // type, which will end up on the stack (with alignment 8).
3287
90
    if (Align == 8 && 
Size <= 6478
)
3288
8
      return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(),
3289
8
                                                          Size));
3290
90
  }
3291
3292
1.28k
  return ABIArgInfo::getIndirect(CharUnits::fromQuantity(Align));
3293
1.28k
}
3294
3295
/// The ABI specifies that a value should be passed in a full vector XMM/YMM
3296
/// register. Pick an LLVM IR type that will be passed as a vector register.
3297
6.36k
llvm::Type *X86_64ABIInfo::GetByteVectorType(QualType Ty) const {
3298
  // Wrapper structs/arrays that only contain vectors are passed just like
3299
  // vectors; strip them off if present.
3300
6.36k
  if (const Type *InnerTy = isSingleElementStruct(Ty, getContext()))
3301
25
    Ty = QualType(InnerTy, 0);
3302
3303
6.36k
  llvm::Type *IRType = CGT.ConvertType(Ty);
3304
6.36k
  if (isa<llvm::VectorType>(IRType)) {
3305
    // Don't pass vXi128 vectors in their native type, the backend can't
3306
    // legalize them.
3307
6.31k
    if (passInt128VectorsInMem() &&
3308
6.31k
        
cast<llvm::VectorType>(IRType)->getElementType()->isIntegerTy(128)524
) {
3309
      // Use a vXi64 vector.
3310
6
      uint64_t Size = getContext().getTypeSize(Ty);
3311
6
      return llvm::FixedVectorType::get(llvm::Type::getInt64Ty(getVMContext()),
3312
6
                                        Size / 64);
3313
6
    }
3314
3315
6.31k
    return IRType;
3316
6.31k
  }
3317
3318
43
  if (IRType->getTypeID() == llvm::Type::FP128TyID)
3319
40
    return IRType;
3320
3321
  // We couldn't find the preferred IR vector type for 'Ty'.
3322
3
  uint64_t Size = getContext().getTypeSize(Ty);
3323
3
  assert((Size == 128 || Size == 256 || Size == 512) && "Invalid type found!");
3324
3325
3326
  // Return a LLVM IR vector type based on the size of 'Ty'.
3327
3
  return llvm::FixedVectorType::get(llvm::Type::getDoubleTy(getVMContext()),
3328
3
                                    Size / 64);
3329
3
}
3330
3331
/// BitsContainNoUserData - Return true if the specified [start,end) bit range
3332
/// is known to either be off the end of the specified type or being in
3333
/// alignment padding.  The user type specified is known to be at most 128 bits
3334
/// in size, and have passed through X86_64ABIInfo::classify with a successful
3335
/// classification that put one of the two halves in the INTEGER class.
3336
///
3337
/// It is conservatively correct to return false.
3338
static bool BitsContainNoUserData(QualType Ty, unsigned StartBit,
3339
73.4k
                                  unsigned EndBit, ASTContext &Context) {
3340
  // If the bytes being queried are off the end of the type, there is no user
3341
  // data hiding here.  This handles analysis of builtins, vectors and other
3342
  // types that don't contain interesting padding.
3343
73.4k
  unsigned TySize = (unsigned)Context.getTypeSize(Ty);
3344
73.4k
  if (TySize <= StartBit)
3345
72.3k
    return true;
3346
3347
1.08k
  if (const ConstantArrayType *AT = Context.getAsConstantArrayType(Ty)) {
3348
20
    unsigned EltSize = (unsigned)Context.getTypeSize(AT->getElementType());
3349
20
    unsigned NumElts = (unsigned)AT->getSize().getZExtValue();
3350
3351
    // Check each element to see if the element overlaps with the queried range.
3352
72
    for (unsigned i = 0; i != NumElts; 
++i52
) {
3353
      // If the element is after the span we care about, then we're done..
3354
69
      unsigned EltOffset = i*EltSize;
3355
69
      if (EltOffset >= EndBit) 
break0
;
3356
3357
69
      unsigned EltStart = EltOffset < StartBit ? 
StartBit-EltOffset52
:
017
;
3358
69
      if (!BitsContainNoUserData(AT->getElementType(), EltStart,
3359
69
                                 EndBit-EltOffset, Context))
3360
17
        return false;
3361
69
    }
3362
    // If it overlaps no elements, then it is safe to process as padding.
3363
3
    return true;
3364
20
  }
3365
3366
1.06k
  if (const RecordType *RT = Ty->getAs<RecordType>()) {
3367
669
    const RecordDecl *RD = RT->getDecl();
3368
669
    const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
3369
3370
    // If this is a C++ record, check the bases first.
3371
669
    if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
3372
533
      for (const auto &I : CXXRD->bases()) {
3373
17
        assert(!I.isVirtual() && !I.getType()->isDependentType() &&
3374
17
               "Unexpected base class!");
3375
17
        const auto *Base =
3376
17
            cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
3377
3378
        // If the base is after the span we care about, ignore it.
3379
17
        unsigned BaseOffset = Context.toBits(Layout.getBaseClassOffset(Base));
3380
17
        if (BaseOffset >= EndBit) 
continue0
;
3381
3382
17
        unsigned BaseStart = BaseOffset < StartBit ? StartBit-BaseOffset :
00
;
3383
17
        if (!BitsContainNoUserData(I.getType(), BaseStart,
3384
17
                                   EndBit-BaseOffset, Context))
3385
14
          return false;
3386
17
      }
3387
533
    }
3388
3389
    // Verify that no field has data that overlaps the region of interest.  Yes
3390
    // this could be sped up a lot by being smarter about queried fields,
3391
    // however we're only looking at structs up to 16 bytes, so we don't care
3392
    // much.
3393
655
    unsigned idx = 0;
3394
655
    for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
3395
1.45k
         i != e; 
++i, ++idx804
) {
3396
1.29k
      unsigned FieldOffset = (unsigned)Layout.getFieldOffset(idx);
3397
3398
      // If we found a field after the region we care about, then we're done.
3399
1.29k
      if (FieldOffset >= EndBit) 
break95
;
3400
3401
1.20k
      unsigned FieldStart = FieldOffset < StartBit ? 
StartBit-FieldOffset851
:
0352
;
3402
1.20k
      if (!BitsContainNoUserData(i->getType(), FieldStart, EndBit-FieldOffset,
3403
1.20k
                                 Context))
3404
399
        return false;
3405
1.20k
    }
3406
3407
    // If nothing in this record overlapped the area of interest, then we're
3408
    // clean.
3409
256
    return true;
3410
655
  }
3411
3412
394
  return false;
3413
1.06k
}
3414
3415
/// getFPTypeAtOffset - Return a floating point type at the specified offset.
3416
static llvm::Type *getFPTypeAtOffset(llvm::Type *IRType, unsigned IROffset,
3417
13.4k
                                     const llvm::DataLayout &TD) {
3418
13.4k
  if (IROffset == 0 && 
IRType->isFloatingPointTy()12.6k
)
3419
5.43k
    return IRType;
3420
3421
  // If this is a struct, recurse into the field at the specified offset.
3422
8.01k
  if (llvm::StructType *STy = dyn_cast<llvm::StructType>(IRType)) {
3423
1.51k
    if (!STy->getNumContainedTypes())
3424
4
      return nullptr;
3425
3426
1.51k
    const llvm::StructLayout *SL = TD.getStructLayout(STy);
3427
1.51k
    unsigned Elt = SL->getElementContainingOffset(IROffset);
3428
1.51k
    IROffset -= SL->getElementOffset(Elt);
3429
1.51k
    return getFPTypeAtOffset(STy->getElementType(Elt), IROffset, TD);
3430
1.51k
  }
3431
3432
  // If this is an array, recurse into the field at the specified offset.
3433
6.50k
  if (llvm::ArrayType *ATy = dyn_cast<llvm::ArrayType>(IRType)) {
3434
20
    llvm::Type *EltTy = ATy->getElementType();
3435
20
    unsigned EltSize = TD.getTypeAllocSize(EltTy);
3436
20
    IROffset -= IROffset / EltSize * EltSize;
3437
20
    return getFPTypeAtOffset(EltTy, IROffset, TD);
3438
20
  }
3439
3440
6.48k
  return nullptr;
3441
6.50k
}
3442
3443
/// GetSSETypeAtOffset - Return a type that will be passed by the backend in the
3444
/// low 8 bytes of an XMM register, corresponding to the SSE class.
3445
llvm::Type *X86_64ABIInfo::
3446
GetSSETypeAtOffset(llvm::Type *IRType, unsigned IROffset,
3447
11.4k
                   QualType SourceTy, unsigned SourceOffset) const {
3448
11.4k
  const llvm::DataLayout &TD = getDataLayout();
3449
11.4k
  unsigned SourceSize =
3450
11.4k
      (unsigned)getContext().getTypeSize(SourceTy) / 8 - SourceOffset;
3451
11.4k
  llvm::Type *T0 = getFPTypeAtOffset(IRType, IROffset, TD);
3452
11.4k
  if (!T0 || 
T0->isDoubleTy()4.98k
)
3453
8.39k
    return llvm::Type::getDoubleTy(getVMContext());
3454
3455
  // Get the adjacent FP type.
3456
3.04k
  llvm::Type *T1 = nullptr;
3457
3.04k
  unsigned T0Size = TD.getTypeAllocSize(T0);
3458
3.04k
  if (SourceSize > T0Size)
3459
457
      T1 = getFPTypeAtOffset(IRType, IROffset + T0Size, TD);
3460
3.04k
  if (T1 == nullptr) {
3461
    // Check if IRType is a half/bfloat + float. float type will be in IROffset+4 due
3462
    // to its alignment.
3463
2.61k
    if (T0->is16bitFPTy() && 
SourceSize > 4606
)
3464
8
      T1 = getFPTypeAtOffset(IRType, IROffset + 4, TD);
3465
    // If we can't get a second FP type, return a simple half or float.
3466
    // avx512fp16-abi.c:pr51813_2 shows it works to return float for
3467
    // {float, i8} too.
3468
2.61k
    if (T1 == nullptr)
3469
2.60k
      return T0;
3470
2.61k
  }
3471
3472
433
  if (T0->isFloatTy() && 
T1->isFloatTy()314
)
3473
302
    return llvm::FixedVectorType::get(T0, 2);
3474
3475
131
  if (T0->is16bitFPTy() && 
T1->is16bitFPTy()119
) {
3476
113
    llvm::Type *T2 = nullptr;
3477
113
    if (SourceSize > 4)
3478
16
      T2 = getFPTypeAtOffset(IRType, IROffset + 4, TD);
3479
113
    if (T2 == nullptr)
3480
97
      return llvm::FixedVectorType::get(T0, 2);
3481
16
    return llvm::FixedVectorType::get(T0, 4);
3482
113
  }
3483
3484
18
  if (T0->is16bitFPTy() || 
T1->is16bitFPTy()12
)
3485
18
    return llvm::FixedVectorType::get(llvm::Type::getHalfTy(getVMContext()), 4);
3486
3487
0
  return llvm::Type::getDoubleTy(getVMContext());
3488
18
}
3489
3490
3491
/// GetINTEGERTypeAtOffset - The ABI specifies that a value should be passed in
3492
/// an 8-byte GPR.  This means that we either have a scalar or we are talking
3493
/// about the high or low part of an up-to-16-byte struct.  This routine picks
3494
/// the best LLVM IR type to represent this, which may be i64 or may be anything
3495
/// else that the backend will pass in a GPR that works better (e.g. i8, %foo*,
3496
/// etc).
3497
///
3498
/// PrefType is an LLVM IR type that corresponds to (part of) the IR type for
3499
/// the source type.  IROffset is an offset in bytes into the LLVM IR type that
3500
/// the 8-byte value references.  PrefType may be null.
3501
///
3502
/// SourceTy is the source-level type for the entire argument.  SourceOffset is
3503
/// an offset into this that we're processing (which is always either 0 or 8).
3504
///
3505
llvm::Type *X86_64ABIInfo::
3506
GetINTEGERTypeAtOffset(llvm::Type *IRType, unsigned IROffset,
3507
338k
                       QualType SourceTy, unsigned SourceOffset) const {
3508
  // If we're dealing with an un-offset LLVM IR type, then it means that we're
3509
  // returning an 8-byte unit starting with it.  See if we can safely use it.
3510
338k
  if (IROffset == 0) {
3511
    // Pointers and int64's always fill the 8-byte unit.
3512
334k
    if ((isa<llvm::PointerType>(IRType) && 
Has64BitPointers230k
) ||
3513
334k
        
IRType->isIntegerTy(64)104k
)
3514
251k
      return IRType;
3515
3516
    // If we have a 1/2/4-byte integer, we can use it only if the rest of the
3517
    // goodness in the source type is just tail padding.  This is allowed to
3518
    // kick in for struct {double,int} on the int, but not on
3519
    // struct{double,int,int} because we wouldn't return the second int.  We
3520
    // have to do this analysis on the source type because we can't depend on
3521
    // unions being lowered a specific way etc.
3522
83.8k
    if (IRType->isIntegerTy(8) || 
IRType->isIntegerTy(16)80.3k
||
3523
83.8k
        
IRType->isIntegerTy(32)79.5k
||
3524
83.8k
        
(11.7k
isa<llvm::PointerType>(IRType)11.7k
&&
!Has64BitPointers92
)) {
3525
72.1k
      unsigned BitWidth = isa<llvm::PointerType>(IRType) ? 
3292
:
3526
72.1k
          
cast<llvm::IntegerType>(IRType)->getBitWidth()72.0k
;
3527
3528
72.1k
      if (BitsContainNoUserData(SourceTy, SourceOffset*8+BitWidth,
3529
72.1k
                                SourceOffset*8+64, getContext()))
3530
71.7k
        return IRType;
3531
72.1k
    }
3532
83.8k
  }
3533
3534
15.9k
  if (llvm::StructType *STy = dyn_cast<llvm::StructType>(IRType)) {
3535
    // If this is a struct, recurse into the field at the specified offset.
3536
11.6k
    const llvm::StructLayout *SL = getDataLayout().getStructLayout(STy);
3537
11.6k
    if (IROffset < SL->getSizeInBytes()) {
3538
11.6k
      unsigned FieldIdx = SL->getElementContainingOffset(IROffset);
3539
11.6k
      IROffset -= SL->getElementOffset(FieldIdx);
3540
3541
11.6k
      return GetINTEGERTypeAtOffset(STy->getElementType(FieldIdx), IROffset,
3542
11.6k
                                    SourceTy, SourceOffset);
3543
11.6k
    }
3544
11.6k
  }
3545
3546
4.36k
  if (llvm::ArrayType *ATy = dyn_cast<llvm::ArrayType>(IRType)) {
3547
52
    llvm::Type *EltTy = ATy->getElementType();
3548
52
    unsigned EltSize = getDataLayout().getTypeAllocSize(EltTy);
3549
52
    unsigned EltOffset = IROffset/EltSize*EltSize;
3550
52
    return GetINTEGERTypeAtOffset(EltTy, IROffset-EltOffset, SourceTy,
3551
52
                                  SourceOffset);
3552
52
  }
3553
3554
  // Okay, we don't have any better idea of what to pass, so we pass this in an
3555
  // integer register that isn't too big to fit the rest of the struct.
3556
4.31k
  unsigned TySizeInBytes =
3557
4.31k
    (unsigned)getContext().getTypeSizeInChars(SourceTy).getQuantity();
3558
3559
4.31k
  assert(TySizeInBytes != SourceOffset && "Empty field?");
3560
3561
  // It is always safe to classify this as an integer type up to i64 that
3562
  // isn't larger than the structure.
3563
4.31k
  return llvm::IntegerType::get(getVMContext(),
3564
4.31k
                                std::min(TySizeInBytes-SourceOffset, 8U)*8);
3565
4.31k
}
3566
3567
3568
/// GetX86_64ByValArgumentPair - Given a high and low type that can ideally
3569
/// be used as elements of a two register pair to pass or return, return a
3570
/// first class aggregate to represent them.  For example, if the low part of
3571
/// a by-value argument should be passed as i32* and the high part as float,
3572
/// return {i32*, float}.
3573
static llvm::Type *
3574
GetX86_64ByValArgumentPair(llvm::Type *Lo, llvm::Type *Hi,
3575
4.13k
                           const llvm::DataLayout &TD) {
3576
  // In order to correctly satisfy the ABI, we need to the high part to start
3577
  // at offset 8.  If the high and low parts we inferred are both 4-byte types
3578
  // (e.g. i32 and i32) then the resultant struct type ({i32,i32}) won't have
3579
  // the second element at offset 8.  Check for this:
3580
4.13k
  unsigned LoSize = (unsigned)TD.getTypeAllocSize(Lo);
3581
4.13k
  llvm::Align HiAlign = TD.getABITypeAlign(Hi);
3582
4.13k
  unsigned HiStart = llvm::alignTo(LoSize, HiAlign);
3583
4.13k
  assert(HiStart != 0 && HiStart <= 8 && "Invalid x86-64 argument pair!");
3584
3585
  // To handle this, we have to increase the size of the low part so that the
3586
  // second element will start at an 8 byte offset.  We can't increase the size
3587
  // of the second element because it might make us access off the end of the
3588
  // struct.
3589
4.13k
  if (HiStart != 8) {
3590
    // There are usually two sorts of types the ABI generation code can produce
3591
    // for the low part of a pair that aren't 8 bytes in size: half, float or
3592
    // i8/i16/i32.  This can also include pointers when they are 32-bit (X32 and
3593
    // NaCl).
3594
    // Promote these to a larger type.
3595
6
    if (Lo->isHalfTy() || Lo->isFloatTy())
3596
1
      Lo = llvm::Type::getDoubleTy(Lo->getContext());
3597
5
    else {
3598
5
      assert((Lo->isIntegerTy() || Lo->isPointerTy())
3599
5
             && "Invalid/unknown lo type");
3600
5
      Lo = llvm::Type::getInt64Ty(Lo->getContext());
3601
5
    }
3602
6
  }
3603
3604
4.13k
  llvm::StructType *Result = llvm::StructType::get(Lo, Hi);
3605
3606
  // Verify that the second element is at an 8-byte offset.
3607
4.13k
  assert(TD.getStructLayout(Result)->getElementOffset(1) == 8 &&
3608
4.13k
         "Invalid x86-64 argument pair!");
3609
4.13k
  return Result;
3610
4.13k
}
3611
3612
ABIArgInfo X86_64ABIInfo::
3613
143k
classifyReturnType(QualType RetTy) const {
3614
  // AMD64-ABI 3.2.3p4: Rule 1. Classify the return type with the
3615
  // classification algorithm.
3616
143k
  X86_64ABIInfo::Class Lo, Hi;
3617
143k
  classify(RetTy, 0, Lo, Hi, /*isNamedArg*/ true);
3618
3619
  // Check some invariants.
3620
143k
  assert((Hi != Memory || Lo == Memory) && "Invalid memory classification.");
3621
143k
  assert((Hi != SSEUp || Lo == SSE) && "Invalid SSEUp classification.");
3622
3623
143k
  llvm::Type *ResType = nullptr;
3624
143k
  switch (Lo) {
3625
60.3k
  case NoClass:
3626
60.3k
    if (Hi == NoClass)
3627
60.3k
      return ABIArgInfo::getIgnore();
3628
    // If the low part is just padding, it takes no register, leave ResType
3629
    // null.
3630
1
    assert((Hi == SSE || Hi == Integer || Hi == X87Up) &&
3631
1
           "Unknown missing lo part");
3632
1
    break;
3633
3634
1
  case SSEUp:
3635
0
  case X87Up:
3636
0
    llvm_unreachable("Invalid classification for lo word.");
3637
3638
    // AMD64-ABI 3.2.3p4: Rule 2. Types of class memory are returned via
3639
    // hidden argument.
3640
613
  case Memory:
3641
613
    return getIndirectReturnResult(RetTy);
3642
3643
    // AMD64-ABI 3.2.3p4: Rule 3. If the class is INTEGER, the next
3644
    // available register of the sequence %rax, %rdx is used.
3645
78.2k
  case Integer:
3646
78.2k
    ResType = GetINTEGERTypeAtOffset(CGT.ConvertType(RetTy), 0, RetTy, 0);
3647
3648
    // If we have a sign or zero extended integer, make sure to return Extend
3649
    // so that the parameter gets the right LLVM IR attributes.
3650
78.2k
    if (Hi == NoClass && 
isa<llvm::IntegerType>(ResType)76.4k
) {
3651
      // Treat an enum type as its underlying type.
3652
45.5k
      if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
3653
101
        RetTy = EnumTy->getDecl()->getIntegerType();
3654
3655
45.5k
      if (RetTy->isIntegralOrEnumerationType() &&
3656
45.5k
          
isPromotableIntegerTypeForABI(RetTy)44.7k
)
3657
4.40k
        return ABIArgInfo::getExtend(RetTy);
3658
45.5k
    }
3659
73.8k
    break;
3660
3661
    // AMD64-ABI 3.2.3p4: Rule 4. If the class is SSE, the next
3662
    // available SSE register of the sequence %xmm0, %xmm1 is used.
3663
73.8k
  case SSE:
3664
3.70k
    ResType = GetSSETypeAtOffset(CGT.ConvertType(RetTy), 0, RetTy, 0);
3665
3.70k
    break;
3666
3667
    // AMD64-ABI 3.2.3p4: Rule 6. If the class is X87, the value is
3668
    // returned on the X87 stack in %st0 as 80-bit x87 number.
3669
641
  case X87:
3670
641
    ResType = llvm::Type::getX86_FP80Ty(getVMContext());
3671
641
    break;
3672
3673
    // AMD64-ABI 3.2.3p4: Rule 8. If the class is COMPLEX_X87, the real
3674
    // part of the value is returned in %st0 and the imaginary part in
3675
    // %st1.
3676
38
  case ComplexX87:
3677
38
    assert(Hi == ComplexX87 && "Unexpected ComplexX87 classification.");
3678
38
    ResType = llvm::StructType::get(llvm::Type::getX86_FP80Ty(getVMContext()),
3679
38
                                    llvm::Type::getX86_FP80Ty(getVMContext()));
3680
38
    break;
3681
143k
  }
3682
3683
78.2k
  llvm::Type *HighPart = nullptr;
3684
78.2k
  switch (Hi) {
3685
    // Memory was handled previously and X87 should
3686
    // never occur as a hi class.
3687
0
  case Memory:
3688
0
  case X87:
3689
0
    llvm_unreachable("Invalid classification for hi word.");
3690
3691
38
  case ComplexX87: // Previously handled.
3692
73.3k
  case NoClass:
3693
73.3k
    break;
3694
3695
1.72k
  case Integer:
3696
1.72k
    HighPart = GetINTEGERTypeAtOffset(CGT.ConvertType(RetTy), 8, RetTy, 8);
3697
1.72k
    if (Lo == NoClass)  // Return HighPart at offset 8 in memory.
3698
0
      return ABIArgInfo::getDirect(HighPart, 8);
3699
1.72k
    break;
3700
1.72k
  case SSE:
3701
133
    HighPart = GetSSETypeAtOffset(CGT.ConvertType(RetTy), 8, RetTy, 8);
3702
133
    if (Lo == NoClass)  // Return HighPart at offset 8 in memory.
3703
1
      return ABIArgInfo::getDirect(HighPart, 8);
3704
132
    break;
3705
3706
    // AMD64-ABI 3.2.3p4: Rule 5. If the class is SSEUP, the eightbyte
3707
    // is passed in the next available eightbyte chunk if the last used
3708
    // vector register.
3709
    //
3710
    // SSEUP should always be preceded by SSE, just widen.
3711
2.36k
  case SSEUp:
3712
2.36k
    assert(Lo == SSE && "Unexpected SSEUp classification.");
3713
2.36k
    ResType = GetByteVectorType(RetTy);
3714
2.36k
    break;
3715
3716
    // AMD64-ABI 3.2.3p4: Rule 7. If the class is X87UP, the value is
3717
    // returned together with the previous X87 value in %st0.
3718
641
  case X87Up:
3719
    // If X87Up is preceded by X87, we don't need to do
3720
    // anything. However, in some cases with unions it may not be
3721
    // preceded by X87. In such situations we follow gcc and pass the
3722
    // extra bits in an SSE reg.
3723
641
    if (Lo != X87) {
3724
0
      HighPart = GetSSETypeAtOffset(CGT.ConvertType(RetTy), 8, RetTy, 8);
3725
0
      if (Lo == NoClass)  // Return HighPart at offset 8 in memory.
3726
0
        return ABIArgInfo::getDirect(HighPart, 8);
3727
0
    }
3728
641
    break;
3729
78.2k
  }
3730
3731
  // If a high part was specified, merge it together with the low part.  It is
3732
  // known to pass in the high eightbyte of the result.  We do this by forming a
3733
  // first class struct aggregate with the high and low part: {low, high}
3734
78.2k
  if (HighPart)
3735
1.85k
    ResType = GetX86_64ByValArgumentPair(ResType, HighPart, getDataLayout());
3736
3737
78.2k
  return ABIArgInfo::getDirect(ResType);
3738
78.2k
}
3739
3740
ABIArgInfo
3741
X86_64ABIInfo::classifyArgumentType(QualType Ty, unsigned freeIntRegs,
3742
                                    unsigned &neededInt, unsigned &neededSSE,
3743
258k
                                    bool isNamedArg, bool IsRegCall) const {
3744
258k
  Ty = useFirstFieldIfTransparentUnion(Ty);
3745
3746
258k
  X86_64ABIInfo::Class Lo, Hi;
3747
258k
  classify(Ty, 0, Lo, Hi, isNamedArg, IsRegCall);
3748
3749
  // Check some invariants.
3750
  // FIXME: Enforce these by construction.
3751
258k
  assert((Hi != Memory || Lo == Memory) && "Invalid memory classification.");
3752
258k
  assert((Hi != SSEUp || Lo == SSE) && "Invalid SSEUp classification.");
3753
3754
258k
  neededInt = 0;
3755
258k
  neededSSE = 0;
3756
258k
  llvm::Type *ResType = nullptr;
3757
258k
  switch (Lo) {
3758
4.33k
  case NoClass:
3759
4.33k
    if (Hi == NoClass)
3760
4.33k
      return ABIArgInfo::getIgnore();
3761
    // If the low part is just padding, it takes no register, leave ResType
3762
    // null.
3763
2
    assert((Hi == SSE || Hi == Integer || Hi == X87Up) &&
3764
2
           "Unknown missing lo part");
3765
2
    break;
3766
3767
    // AMD64-ABI 3.2.3p3: Rule 1. If the class is MEMORY, pass the argument
3768
    // on the stack.
3769
1.85k
  case Memory:
3770
3771
    // AMD64-ABI 3.2.3p3: Rule 5. If the class is X87, X87UP or
3772
    // COMPLEX_X87, it is passed in memory.
3773
2.11k
  case X87:
3774
2.16k
  case ComplexX87:
3775
2.16k
    if (getRecordArgABI(Ty, getCXXABI()) == CGCXXABI::RAA_Indirect)
3776
515
      ++neededInt;
3777
2.16k
    return getIndirectResult(Ty, freeIntRegs);
3778
3779
0
  case SSEUp:
3780
0
  case X87Up:
3781
0
    llvm_unreachable("Invalid classification for lo word.");
3782
3783
    // AMD64-ABI 3.2.3p3: Rule 2. If the class is INTEGER, the next
3784
    // available register of the sequence %rdi, %rsi, %rdx, %rcx, %r8
3785
    // and %r9 is used.
3786
244k
  case Integer:
3787
244k
    ++neededInt;
3788
3789
    // Pick an 8-byte type based on the preferred type.
3790
244k
    ResType = GetINTEGERTypeAtOffset(CGT.ConvertType(Ty), 0, Ty, 0);
3791
3792
    // If we have a sign or zero extended integer, make sure to return Extend
3793
    // so that the parameter gets the right LLVM IR attributes.
3794
244k
    if (Hi == NoClass && 
isa<llvm::IntegerType>(ResType)242k
) {
3795
      // Treat an enum type as its underlying type.
3796
49.6k
      if (const EnumType *EnumTy = Ty->getAs<EnumType>())
3797
887
        Ty = EnumTy->getDecl()->getIntegerType();
3798
3799
49.6k
      if (Ty->isIntegralOrEnumerationType() &&
3800
49.6k
          
isPromotableIntegerTypeForABI(Ty)49.2k
)
3801
3.18k
        return ABIArgInfo::getExtend(Ty);
3802
49.6k
    }
3803
3804
241k
    break;
3805
3806
    // AMD64-ABI 3.2.3p3: Rule 3. If the class is SSE, the next
3807
    // available SSE register is used, the registers are taken in the
3808
    // order from %xmm0 to %xmm7.
3809
241k
  case SSE: {
3810
7.44k
    llvm::Type *IRType = CGT.ConvertType(Ty);
3811
7.44k
    ResType = GetSSETypeAtOffset(IRType, 0, Ty, 0);
3812
7.44k
    ++neededSSE;
3813
7.44k
    break;
3814
244k
  }
3815
258k
  }
3816
3817
249k
  llvm::Type *HighPart = nullptr;
3818
249k
  switch (Hi) {
3819
    // Memory was handled previously, ComplexX87 and X87 should
3820
    // never occur as hi classes, and X87Up must be preceded by X87,
3821
    // which is passed in memory.
3822
0
  case Memory:
3823
0
  case X87:
3824
0
  case ComplexX87:
3825
0
    llvm_unreachable("Invalid classification for hi word.");
3826
3827
242k
  case NoClass: break;
3828
3829
2.11k
  case Integer:
3830
2.11k
    ++neededInt;
3831
    // Pick an 8-byte type based on the preferred type.
3832
2.11k
    HighPart = GetINTEGERTypeAtOffset(CGT.ConvertType(Ty), 8, Ty, 8);
3833
3834
2.11k
    if (Lo == NoClass)  // Pass HighPart at offset 8 in memory.
3835
2
      return ABIArgInfo::getDirect(HighPart, 8);
3836
2.11k
    break;
3837
3838
    // X87Up generally doesn't occur here (long double is passed in
3839
    // memory), except in situations involving unions.
3840
2.11k
  case X87Up:
3841
162
  case SSE:
3842
162
    HighPart = GetSSETypeAtOffset(CGT.ConvertType(Ty), 8, Ty, 8);
3843
3844
162
    if (Lo == NoClass)  // Pass HighPart at offset 8 in memory.
3845
0
      return ABIArgInfo::getDirect(HighPart, 8);
3846
3847
162
    ++neededSSE;
3848
162
    break;
3849
3850
    // AMD64-ABI 3.2.3p3: Rule 4. If the class is SSEUP, the
3851
    // eightbyte is passed in the upper half of the last used SSE
3852
    // register.  This only happens when 128-bit vectors are passed.
3853
3.99k
  case SSEUp:
3854
3.99k
    assert(Lo == SSE && "Unexpected SSEUp classification");
3855
3.99k
    ResType = GetByteVectorType(Ty);
3856
3.99k
    break;
3857
249k
  }
3858
3859
  // If a high part was specified, merge it together with the low part.  It is
3860
  // known to pass in the high eightbyte of the result.  We do this by forming a
3861
  // first class struct aggregate with the high and low part: {low, high}
3862
249k
  if (HighPart)
3863
2.27k
    ResType = GetX86_64ByValArgumentPair(ResType, HighPart, getDataLayout());
3864
3865
249k
  return ABIArgInfo::getDirect(ResType);
3866
249k
}
3867
3868
ABIArgInfo
3869
X86_64ABIInfo::classifyRegCallStructTypeImpl(QualType Ty, unsigned &NeededInt,
3870
                                             unsigned &NeededSSE,
3871
20
                                             unsigned &MaxVectorWidth) const {
3872
20
  auto RT = Ty->getAs<RecordType>();
3873
20
  assert(RT && "classifyRegCallStructType only valid with struct types");
3874
3875
20
  if (RT->getDecl()->hasFlexibleArrayMember())
3876
0
    return getIndirectReturnResult(Ty);
3877
3878
  // Sum up bases
3879
20
  if (auto CXXRD = dyn_cast<CXXRecordDecl>(RT->getDecl())) {
3880
0
    if (CXXRD->isDynamicClass()) {
3881
0
      NeededInt = NeededSSE = 0;
3882
0
      return getIndirectReturnResult(Ty);
3883
0
    }
3884
3885
0
    for (const auto &I : CXXRD->bases())
3886
0
      if (classifyRegCallStructTypeImpl(I.getType(), NeededInt, NeededSSE,
3887
0
                                        MaxVectorWidth)
3888
0
              .isIndirect()) {
3889
0
        NeededInt = NeededSSE = 0;
3890
0
        return getIndirectReturnResult(Ty);
3891
0
      }
3892
0
  }
3893
3894
  // Sum up members
3895
48
  
for (const auto *FD : RT->getDecl()->fields())20
{
3896
48
    QualType MTy = FD->getType();
3897
48
    if (MTy->isRecordType() && 
!MTy->isUnionType()0
) {
3898
0
      if (classifyRegCallStructTypeImpl(MTy, NeededInt, NeededSSE,
3899
0
                                        MaxVectorWidth)
3900
0
              .isIndirect()) {
3901
0
        NeededInt = NeededSSE = 0;
3902
0
        return getIndirectReturnResult(Ty);
3903
0
      }
3904
48
    } else {
3905
48
      unsigned LocalNeededInt, LocalNeededSSE;
3906
48
      if (classifyArgumentType(MTy, UINT_MAX, LocalNeededInt, LocalNeededSSE,
3907
48
                               true, true)
3908
48
              .isIndirect()) {
3909
0
        NeededInt = NeededSSE = 0;
3910
0
        return getIndirectReturnResult(Ty);
3911
0
      }
3912
48
      if (const auto *AT = getContext().getAsConstantArrayType(MTy))
3913
10
        MTy = AT->getElementType();
3914
48
      if (const auto *VT = MTy->getAs<VectorType>())
3915
25
        if (getContext().getTypeSize(VT) > MaxVectorWidth)
3916
12
          MaxVectorWidth = getContext().getTypeSize(VT);
3917
48
      NeededInt += LocalNeededInt;
3918
48
      NeededSSE += LocalNeededSSE;
3919
48
    }
3920
48
  }
3921
3922
20
  return ABIArgInfo::getDirect();
3923
20
}
3924
3925
ABIArgInfo
3926
X86_64ABIInfo::classifyRegCallStructType(QualType Ty, unsigned &NeededInt,
3927
                                         unsigned &NeededSSE,
3928
20
                                         unsigned &MaxVectorWidth) const {
3929
3930
20
  NeededInt = 0;
3931
20
  NeededSSE = 0;
3932
20
  MaxVectorWidth = 0;
3933
3934
20
  return classifyRegCallStructTypeImpl(Ty, NeededInt, NeededSSE,
3935
20
                                       MaxVectorWidth);
3936
20
}
3937
3938
144k
void X86_64ABIInfo::computeInfo(CGFunctionInfo &FI) const {
3939
3940
144k
  const unsigned CallingConv = FI.getCallingConvention();
3941
  // It is possible to force Win64 calling convention on any x86_64 target by
3942
  // using __attribute__((ms_abi)). In such case to correctly emit Win64
3943
  // compatible code delegate this call to WinX86_64ABIInfo::computeInfo.
3944
144k
  if (CallingConv == llvm::CallingConv::Win64) {
3945
13
    WinX86_64ABIInfo Win64ABIInfo(CGT, AVXLevel);
3946
13
    Win64ABIInfo.computeInfo(FI);
3947
13
    return;
3948
13
  }
3949
3950
144k
  bool IsRegCall = CallingConv == llvm::CallingConv::X86_RegCall;
3951
3952
  // Keep track of the number of assigned registers.
3953
144k
  unsigned FreeIntRegs = IsRegCall ? 
1129
:
6144k
;
3954
144k
  unsigned FreeSSERegs = IsRegCall ? 
1629
:
8144k
;
3955
144k
  unsigned NeededInt = 0, NeededSSE = 0, MaxVectorWidth = 0;
3956
3957
144k
  if (!::classifyReturnType(getCXXABI(), FI, *this)) {
3958
143k
    if (IsRegCall && 
FI.getReturnType()->getTypePtr()->isRecordType()28
&&
3959
143k
        
!FI.getReturnType()->getTypePtr()->isUnionType()3
) {
3960
3
      FI.getReturnInfo() = classifyRegCallStructType(
3961
3
          FI.getReturnType(), NeededInt, NeededSSE, MaxVectorWidth);
3962
3
      if (FreeIntRegs >= NeededInt && FreeSSERegs >= NeededSSE) {
3963
3
        FreeIntRegs -= NeededInt;
3964
3
        FreeSSERegs -= NeededSSE;
3965
3
      } else {
3966
0
        FI.getReturnInfo() = getIndirectReturnResult(FI.getReturnType());
3967
0
      }
3968
143k
    } else if (IsRegCall && 
FI.getReturnType()->getAs<ComplexType>()25
&&
3969
143k
               getContext().getCanonicalType(FI.getReturnType()
3970
2
                                                 ->getAs<ComplexType>()
3971
2
                                                 ->getElementType()) ==
3972
2
                   getContext().LongDoubleTy)
3973
      // Complex Long Double Type is passed in Memory when Regcall
3974
      // calling convention is used.
3975
1
      FI.getReturnInfo() = getIndirectReturnResult(FI.getReturnType());
3976
143k
    else
3977
143k
      FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
3978
143k
  }
3979
3980
  // If the return value is indirect, then the hidden argument is consuming one
3981
  // integer register.
3982
144k
  if (FI.getReturnInfo().isIndirect())
3983
1.42k
    --FreeIntRegs;
3984
143k
  else if (NeededSSE && 
MaxVectorWidth > 01
)
3985
0
    FI.setMaxVectorWidth(MaxVectorWidth);
3986
3987
  // The chain argument effectively gives us another free register.
3988
144k
  if (FI.isChainCall())
3989
4
    ++FreeIntRegs;
3990
3991
144k
  unsigned NumRequiredArgs = FI.getNumRequiredArgs();
3992
  // AMD64-ABI 3.2.3p3: Once arguments are classified, the registers
3993
  // get assigned (in left-to-right order) for passing as follows...
3994
144k
  unsigned ArgNo = 0;
3995
144k
  for (CGFunctionInfo::arg_iterator it = FI.arg_begin(), ie = FI.arg_end();
3996
403k
       it != ie; 
++it, ++ArgNo258k
) {
3997
258k
    bool IsNamedArg = ArgNo < NumRequiredArgs;
3998
3999
258k
    if (IsRegCall && 
it->type->isStructureOrClassType()67
)
4000
17
      it->info = classifyRegCallStructType(it->type, NeededInt, NeededSSE,
4001
17
                                           MaxVectorWidth);
4002
258k
    else
4003
258k
      it->info = classifyArgumentType(it->type, FreeIntRegs, NeededInt,
4004
258k
                                      NeededSSE, IsNamedArg);
4005
4006
    // AMD64-ABI 3.2.3p3: If there are no registers available for any
4007
    // eightbyte of an argument, the whole argument is passed on the
4008
    // stack. If registers have already been assigned for some
4009
    // eightbytes of such an argument, the assignments get reverted.
4010
258k
    if (FreeIntRegs >= NeededInt && 
FreeSSERegs >= NeededSSE256k
) {
4011
256k
      FreeIntRegs -= NeededInt;
4012
256k
      FreeSSERegs -= NeededSSE;
4013
256k
      if (MaxVectorWidth > FI.getMaxVectorWidth())
4014
7
        FI.setMaxVectorWidth(MaxVectorWidth);
4015
256k
    } else {
4016
2.67k
      it->info = getIndirectResult(it->type, FreeIntRegs);
4017
2.67k
    }
4018
258k
  }
4019
144k
}
4020
4021
static Address EmitX86_64VAArgFromMemory(CodeGenFunction &CGF,
4022
84
                                         Address VAListAddr, QualType Ty) {
4023
84
  Address overflow_arg_area_p =
4024
84
      CGF.Builder.CreateStructGEP(VAListAddr, 2, "overflow_arg_area_p");
4025
84
  llvm::Value *overflow_arg_area =
4026
84
    CGF.Builder.CreateLoad(overflow_arg_area_p, "overflow_arg_area");
4027
4028
  // AMD64-ABI 3.5.7p5: Step 7. Align l->overflow_arg_area upwards to a 16
4029
  // byte boundary if alignment needed by type exceeds 8 byte boundary.
4030
  // It isn't stated explicitly in the standard, but in practice we use
4031
  // alignment greater than 16 where necessary.
4032
84
  CharUnits Align = CGF.getContext().getTypeAlignInChars(Ty);
4033
84
  if (Align > CharUnits::fromQuantity(8)) {
4034
20
    overflow_arg_area = emitRoundPointerUpToAlignment(CGF, overflow_arg_area,
4035
20
                                                      Align);
4036
20
  }
4037
4038
  // AMD64-ABI 3.5.7p5: Step 8. Fetch type from l->overflow_arg_area.
4039
84
  llvm::Type *LTy = CGF.ConvertTypeForMem(Ty);
4040
84
  llvm::Value *Res =
4041
84
    CGF.Builder.CreateBitCast(overflow_arg_area,
4042
84
                              llvm::PointerType::getUnqual(LTy));
4043
4044
  // AMD64-ABI 3.5.7p5: Step 9. Set l->overflow_arg_area to:
4045
  // l->overflow_arg_area + sizeof(type).
4046
  // AMD64-ABI 3.5.7p5: Step 10. Align l->overflow_arg_area upwards to
4047
  // an 8 byte boundary.
4048
4049
84
  uint64_t SizeInBytes = (CGF.getContext().getTypeSize(Ty) + 7) / 8;
4050
84
  llvm::Value *Offset =
4051
84
      llvm::ConstantInt::get(CGF.Int32Ty, (SizeInBytes + 7)  & ~7);
4052
84
  overflow_arg_area = CGF.Builder.CreateGEP(CGF.Int8Ty, overflow_arg_area,
4053
84
                                            Offset, "overflow_arg_area.next");
4054
84
  CGF.Builder.CreateStore(overflow_arg_area, overflow_arg_area_p);
4055
4056
  // AMD64-ABI 3.5.7p5: Step 11. Return the fetched type.
4057
84
  return Address(Res, LTy, Align);
4058
84
}
4059
4060
Address X86_64ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
4061
84
                                 QualType Ty) const {
4062
  // Assume that va_list type is correct; should be pointer to LLVM type:
4063
  // struct {
4064
  //   i32 gp_offset;
4065
  //   i32 fp_offset;
4066
  //   i8* overflow_arg_area;
4067
  //   i8* reg_save_area;
4068
  // };
4069
84
  unsigned neededInt, neededSSE;
4070
4071
84
  Ty = getContext().getCanonicalType(Ty);
4072
84
  ABIArgInfo AI = classifyArgumentType(Ty, 0, neededInt, neededSSE,
4073
84
                                       /*isNamedArg*/false);
4074
4075
  // AMD64-ABI 3.5.7p5: Step 1. Determine whether type may be passed
4076
  // in the registers. If not go to step 7.
4077
84
  if (!neededInt && 
!neededSSE35
)
4078
16
    return EmitX86_64VAArgFromMemory(CGF, VAListAddr, Ty);
4079
4080
  // AMD64-ABI 3.5.7p5: Step 2. Compute num_gp to hold the number of
4081
  // general purpose registers needed to pass type and num_fp to hold
4082
  // the number of floating point registers needed.
4083
4084
  // AMD64-ABI 3.5.7p5: Step 3. Verify whether arguments fit into
4085
  // registers. In the case: l->gp_offset > 48 - num_gp * 8 or
4086
  // l->fp_offset > 304 - num_fp * 16 go to step 7.
4087
  //
4088
  // NOTE: 304 is a typo, there are (6 * 8 + 8 * 16) = 176 bytes of
4089
  // register save space).
4090
4091
68
  llvm::Value *InRegs = nullptr;
4092
68
  Address gp_offset_p = Address::invalid(), fp_offset_p = Address::invalid();
4093
68
  llvm::Value *gp_offset = nullptr, *fp_offset = nullptr;
4094
68
  if (neededInt) {
4095
49
    gp_offset_p = CGF.Builder.CreateStructGEP(VAListAddr, 0, "gp_offset_p");
4096
49
    gp_offset = CGF.Builder.CreateLoad(gp_offset_p, "gp_offset");
4097
49
    InRegs = llvm::ConstantInt::get(CGF.Int32Ty, 48 - neededInt * 8);
4098
49
    InRegs = CGF.Builder.CreateICmpULE(gp_offset, InRegs, "fits_in_gp");
4099
49
  }
4100
4101
68
  if (neededSSE) {
4102
23
    fp_offset_p = CGF.Builder.CreateStructGEP(VAListAddr, 1, "fp_offset_p");
4103
23
    fp_offset = CGF.Builder.CreateLoad(fp_offset_p, "fp_offset");
4104
23
    llvm::Value *FitsInFP =
4105
23
      llvm::ConstantInt::get(CGF.Int32Ty, 176 - neededSSE * 16);
4106
23
    FitsInFP = CGF.Builder.CreateICmpULE(fp_offset, FitsInFP, "fits_in_fp");
4107
23
    InRegs = InRegs ? 
CGF.Builder.CreateAnd(InRegs, FitsInFP)4
:
FitsInFP19
;
4108
23
  }
4109
4110
68
  llvm::BasicBlock *InRegBlock = CGF.createBasicBlock("vaarg.in_reg");
4111
68
  llvm::BasicBlock *InMemBlock = CGF.createBasicBlock("vaarg.in_mem");
4112
68
  llvm::BasicBlock *ContBlock = CGF.createBasicBlock("vaarg.end");
4113
68
  CGF.Builder.CreateCondBr(InRegs, InRegBlock, InMemBlock);
4114
4115
  // Emit code to load the value if it was passed in registers.
4116
4117
68
  CGF.EmitBlock(InRegBlock);
4118
4119
  // AMD64-ABI 3.5.7p5: Step 4. Fetch type from l->reg_save_area with
4120
  // an offset of l->gp_offset and/or l->fp_offset. This may require
4121
  // copying to a temporary location in case the parameter is passed
4122
  // in different register classes or requires an alignment greater
4123
  // than 8 for general purpose registers and 16 for XMM registers.
4124
  //
4125
  // FIXME: This really results in shameful code when we end up needing to
4126
  // collect arguments from different places; often what should result in a
4127
  // simple assembling of a structure from scattered addresses has many more
4128
  // loads than necessary. Can we clean this up?
4129
68
  llvm::Type *LTy = CGF.ConvertTypeForMem(Ty);
4130
68
  llvm::Value *RegSaveArea = CGF.Builder.CreateLoad(
4131
68
      CGF.Builder.CreateStructGEP(VAListAddr, 3), "reg_save_area");
4132
4133
68
  Address RegAddr = Address::invalid();
4134
68
  if (neededInt && 
neededSSE49
) {
4135
    // FIXME: Cleanup.
4136
4
    assert(AI.isDirect() && "Unexpected ABI info for mixed regs");
4137
4
    llvm::StructType *ST = cast<llvm::StructType>(AI.getCoerceToType());
4138
4
    Address Tmp = CGF.CreateMemTemp(Ty);
4139
4
    Tmp = CGF.Builder.CreateElementBitCast(Tmp, ST);
4140
4
    assert(ST->getNumElements() == 2 && "Unexpected ABI info for mixed regs");
4141
4
    llvm::Type *TyLo = ST->getElementType(0);
4142
4
    llvm::Type *TyHi = ST->getElementType(1);
4143
4
    assert((TyLo->isFPOrFPVectorTy() ^ TyHi->isFPOrFPVectorTy()) &&
4144
4
           "Unexpected ABI info for mixed regs");
4145
4
    llvm::Type *PTyLo = llvm::PointerType::getUnqual(TyLo);
4146
4
    llvm::Type *PTyHi = llvm::PointerType::getUnqual(TyHi);
4147
4
    llvm::Value *GPAddr =
4148
4
        CGF.Builder.CreateGEP(CGF.Int8Ty, RegSaveArea, gp_offset);
4149
4
    llvm::Value *FPAddr =
4150
4
        CGF.Builder.CreateGEP(CGF.Int8Ty, RegSaveArea, fp_offset);
4151
4
    llvm::Value *RegLoAddr = TyLo->isFPOrFPVectorTy() ? 
FPAddr1
:
GPAddr3
;
4152
4
    llvm::Value *RegHiAddr = TyLo->isFPOrFPVectorTy() ? 
GPAddr1
:
FPAddr3
;
4153
4154
    // Copy the first element.
4155
    // FIXME: Our choice of alignment here and below is probably pessimistic.
4156
4
    llvm::Value *V = CGF.Builder.CreateAlignedLoad(
4157
4
        TyLo, CGF.Builder.CreateBitCast(RegLoAddr, PTyLo),
4158
4
        CharUnits::fromQuantity(getDataLayout().getABITypeAlign(TyLo)));
4159
4
    CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 0));
4160
4161
    // Copy the second element.
4162
4
    V = CGF.Builder.CreateAlignedLoad(
4163
4
        TyHi, CGF.Builder.CreateBitCast(RegHiAddr, PTyHi),
4164
4
        CharUnits::fromQuantity(getDataLayout().getABITypeAlign(TyHi)));
4165
4
    CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 1));
4166
4167
4
    RegAddr = CGF.Builder.CreateElementBitCast(Tmp, LTy);
4168
64
  } else if (neededInt) {
4169
45
    RegAddr = Address(CGF.Builder.CreateGEP(CGF.Int8Ty, RegSaveArea, gp_offset),
4170
45
                      CGF.Int8Ty, CharUnits::fromQuantity(8));
4171
45
    RegAddr = CGF.Builder.CreateElementBitCast(RegAddr, LTy);
4172
4173
    // Copy to a temporary if necessary to ensure the appropriate alignment.
4174
45
    auto TInfo = getContext().getTypeInfoInChars(Ty);
4175
45
    uint64_t TySize = TInfo.Width.getQuantity();
4176
45
    CharUnits TyAlign = TInfo.Align;
4177
4178
    // Copy into a temporary if the type is more aligned than the
4179
    // register save area.
4180
45
    if (TyAlign.getQuantity() > 8) {
4181
3
      Address Tmp = CGF.CreateMemTemp(Ty);
4182
3
      CGF.Builder.CreateMemCpy(Tmp, RegAddr, TySize, false);
4183
3
      RegAddr = Tmp;
4184
3
    }
4185
4186
45
  } else 
if (19
neededSSE == 119
) {
4187
9
    RegAddr = Address(CGF.Builder.CreateGEP(CGF.Int8Ty, RegSaveArea, fp_offset),
4188
9
                      CGF.Int8Ty, CharUnits::fromQuantity(16));
4189
9
    RegAddr = CGF.Builder.CreateElementBitCast(RegAddr, LTy);
4190
10
  } else {
4191
10
    assert(neededSSE == 2 && "Invalid number of needed registers!");
4192
    // SSE registers are spaced 16 bytes apart in the register save
4193
    // area, we need to collect the two eightbytes together.
4194
    // The ABI isn't explicit about this, but it seems reasonable
4195
    // to assume that the slots are 16-byte aligned, since the stack is
4196
    // naturally 16-byte aligned and the prologue is expected to store
4197
    // all the SSE registers to the RSA.
4198
10
    Address RegAddrLo = Address(CGF.Builder.CreateGEP(CGF.Int8Ty, RegSaveArea,
4199
10
                                                      fp_offset),
4200
10
                                CGF.Int8Ty, CharUnits::fromQuantity(16));
4201
10
    Address RegAddrHi =
4202
10
      CGF.Builder.CreateConstInBoundsByteGEP(RegAddrLo,
4203
10
                                             CharUnits::fromQuantity(16));
4204
10
    llvm::Type *ST = AI.canHaveCoerceToType()
4205
10
                         ? AI.getCoerceToType()
4206
10
                         : 
llvm::StructType::get(CGF.DoubleTy, CGF.DoubleTy)0
;
4207
10
    llvm::Value *V;
4208
10
    Address Tmp = CGF.CreateMemTemp(Ty);
4209
10
    Tmp = CGF.Builder.CreateElementBitCast(Tmp, ST);
4210
10
    V = CGF.Builder.CreateLoad(CGF.Builder.CreateElementBitCast(
4211
10
        RegAddrLo, ST->getStructElementType(0)));
4212
10
    CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 0));
4213
10
    V = CGF.Builder.CreateLoad(CGF.Builder.CreateElementBitCast(
4214
10
        RegAddrHi, ST->getStructElementType(1)));
4215
10
    CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 1));
4216
4217
10
    RegAddr = CGF.Builder.CreateElementBitCast(Tmp, LTy);
4218
10
  }
4219
4220
  // AMD64-ABI 3.5.7p5: Step 5. Set:
4221
  // l->gp_offset = l->gp_offset + num_gp * 8
4222
  // l->fp_offset = l->fp_offset + num_fp * 16.
4223
68
  if (neededInt) {
4224
49
    llvm::Value *Offset = llvm::ConstantInt::get(CGF.Int32Ty, neededInt * 8);
4225
49
    CGF.Builder.CreateStore(CGF.Builder.CreateAdd(gp_offset, Offset),
4226
49
                            gp_offset_p);
4227
49
  }
4228
68
  if (neededSSE) {
4229
23
    llvm::Value *Offset = llvm::ConstantInt::get(CGF.Int32Ty, neededSSE * 16);
4230
23
    CGF.Builder.CreateStore(CGF.Builder.CreateAdd(fp_offset, Offset),
4231
23
                            fp_offset_p);
4232
23
  }
4233
68
  CGF.EmitBranch(ContBlock);
4234
4235
  // Emit code to load the value if it was passed in memory.
4236
4237
68
  CGF.EmitBlock(InMemBlock);
4238
68
  Address MemAddr = EmitX86_64VAArgFromMemory(CGF, VAListAddr, Ty);
4239
4240
  // Return the appropriate result.
4241
4242
68
  CGF.EmitBlock(ContBlock);
4243
68
  Address ResAddr = emitMergePHI(CGF, RegAddr, InRegBlock, MemAddr, InMemBlock,
4244
68
                                 "vaarg.addr");
4245
68
  return ResAddr;
4246
68
}
4247
4248
Address X86_64ABIInfo::EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr,
4249
8
                                   QualType Ty) const {
4250
  // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
4251
  // not 1, 2, 4, or 8 bytes, must be passed by reference."
4252
8
  uint64_t Width = getContext().getTypeSize(Ty);
4253
8
  bool IsIndirect = Width > 64 || 
!llvm::isPowerOf2_64(Width)4
;
4254
4255
8
  return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect,
4256
8
                          CGF.getContext().getTypeInfoInChars(Ty),
4257
8
                          CharUnits::fromQuantity(8),
4258
8
                          /*allowHigherAlign*/ false);
4259
8
}
4260
4261
ABIArgInfo WinX86_64ABIInfo::reclassifyHvaArgForVectorCall(
4262
112
    QualType Ty, unsigned &FreeSSERegs, const ABIArgInfo &current) const {
4263
112
  const Type *Base = nullptr;
4264
112
  uint64_t NumElts = 0;
4265
4266
112
  if (!Ty->isBuiltinType() && 
!Ty->isVectorType()49
&&
4267
112
      
isHomogeneousAggregate(Ty, Base, NumElts)36
&&
FreeSSERegs >= NumElts27
) {
4268
18
    FreeSSERegs -= NumElts;
4269
18
    return getDirectX86Hva();
4270
18
  }
4271
94
  return current;
4272
112
}
4273
4274
ABIArgInfo WinX86_64ABIInfo::classify(QualType Ty, unsigned &FreeSSERegs,
4275
                                      bool IsReturnType, bool IsVectorCall,
4276
10.2k
                                      bool IsRegCall) const {
4277
4278
10.2k
  if (Ty->isVoidType())
4279
1.74k
    return ABIArgInfo::getIgnore();
4280
4281
8.49k
  if (const EnumType *EnumTy = Ty->getAs<EnumType>())
4282
20
    Ty = EnumTy->getDecl()->getIntegerType();
4283
4284
8.49k
  TypeInfo Info = getContext().getTypeInfo(Ty);
4285
8.49k
  uint64_t Width = Info.Width;
4286
8.49k
  CharUnits Align = getContext().toCharUnitsFromBits(Info.Align);
4287
4288
8.49k
  const RecordType *RT = Ty->getAs<RecordType>();
4289
8.49k
  if (RT) {
4290
319
    if (!IsReturnType) {
4291
276
      if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(RT, getCXXABI()))
4292
73
        return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
4293
276
    }
4294
4295
246
    if (RT->getDecl()->hasFlexibleArrayMember())
4296
0
      return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
4297
4298
246
  }
4299
4300
8.41k
  const Type *Base = nullptr;
4301
8.41k
  uint64_t NumElts = 0;
4302
  // vectorcall adds the concept of a homogenous vector aggregate, similar to
4303
  // other targets.
4304
8.41k
  if ((IsVectorCall || 
IsRegCall8.29k
) &&
4305
8.41k
      
isHomogeneousAggregate(Ty, Base, NumElts)193
) {
4306
95
    if (IsRegCall) {
4307
32
      if (FreeSSERegs >= NumElts) {
4308
32
        FreeSSERegs -= NumElts;
4309
32
        if (IsReturnType || 
Ty->isBuiltinType()27
||
Ty->isVectorType()21
)
4310
17
          return ABIArgInfo::getDirect();
4311
15
        return ABIArgInfo::getExpand();
4312
32
      }
4313
0
      return ABIArgInfo::getIndirect(Align, /*ByVal=*/false);
4314
63
    } else if (IsVectorCall) {
4315
63
      if (FreeSSERegs >= NumElts &&
4316
63
          
(57
IsReturnType57
||
Ty->isBuiltinType()45
||
Ty->isVectorType()36
)) {
4317
34
        FreeSSERegs -= NumElts;
4318
34
        return ABIArgInfo::getDirect();
4319
34
      } else 
if (29
IsReturnType29
) {
4320
0
        return ABIArgInfo::getExpand();
4321
29
      } else if (!Ty->isBuiltinType() && 
!Ty->isVectorType()27
) {
4322
        // HVAs are delayed and reclassified in the 2nd step.
4323
27
        return ABIArgInfo::getIndirect(Align, /*ByVal=*/false);
4324
27
      }
4325
63
    }
4326
95
  }
4327
4328
8.32k
  if (Ty->isMemberPointerType()) {
4329
    // If the member pointer is represented by an LLVM int or ptr, pass it
4330
    // directly.
4331
160
    llvm::Type *LLTy = CGT.ConvertType(Ty);
4332
160
    if (LLTy->isPointerTy() || 
LLTy->isIntegerTy()77
)
4333
91
      return ABIArgInfo::getDirect();
4334
160
  }
4335
4336
8.23k
  if (RT || 
Ty->isAnyComplexType()8.03k
||
Ty->isMemberPointerType()8.02k
) {
4337
    // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
4338
    // not 1, 2, 4, or 8 bytes, must be passed by reference."
4339
284
    if (Width > 64 || 
!llvm::isPowerOf2_64(Width)152
)
4340
132
      return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
4341
4342
    // Otherwise, coerce it to a small integer.
4343
152
    return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(), Width));
4344
284
  }
4345
4346
7.95k
  if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
4347
2.27k
    switch (BT->getKind()) {
4348
31
    case BuiltinType::Bool:
4349
      // Bool type is always extended to the ABI, other builtin types are not
4350
      // extended.
4351
31
      return ABIArgInfo::getExtend(Ty);
4352
4353
68
    case BuiltinType::LongDouble:
4354
      // Mingw64 GCC uses the old 80 bit extended precision floating point
4355
      // unit. It passes them indirectly through memory.
4356
68
      if (IsMingw64) {
4357
7
        const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();
4358
7
        if (LDF == &llvm::APFloat::x87DoubleExtended())
4359
7
          return ABIArgInfo::getIndirect(Align, /*ByVal=*/false);
4360
7
      }
4361
61
      break;
4362
4363
61
    case BuiltinType::Int128:
4364
10
    case BuiltinType::UInt128:
4365
      // If it's a parameter type, the normal ABI rule is that arguments larger
4366
      // than 8 bytes are passed indirectly. GCC follows it. We follow it too,
4367
      // even though it isn't particularly efficient.
4368
10
      if (!IsReturnType)
4369
4
        return ABIArgInfo::getIndirect(Align, /*ByVal=*/false);
4370
4371
      // Mingw64 GCC returns i128 in XMM0. Coerce to v2i64 to handle that.
4372
      // Clang matches them for compatibility.
4373
6
      return ABIArgInfo::getDirect(llvm::FixedVectorType::get(
4374
6
          llvm::Type::getInt64Ty(getVMContext()), 2));
4375
4376
2.16k
    default:
4377
2.16k
      break;
4378
2.27k
    }
4379
2.27k
  }
4380
4381
7.90k
  if (Ty->isBitIntType()) {
4382
    // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
4383
    // not 1, 2, 4, or 8 bytes, must be passed by reference."
4384
    // However, non-power-of-two bit-precise integers will be passed as 1, 2, 4,
4385
    // or 8 bytes anyway as long is it fits in them, so we don't have to check
4386
    // the power of 2.
4387
55
    if (Width <= 64)
4388
37
      return ABIArgInfo::getDirect();
4389
18
    return ABIArgInfo::getIndirect(Align, /*ByVal=*/false);
4390
55
  }
4391
4392
7.84k
  return ABIArgInfo::getDirect();
4393
7.90k
}
4394
4395
4.25k
void WinX86_64ABIInfo::computeInfo(CGFunctionInfo &FI) const {
4396
4.25k
  const unsigned CC = FI.getCallingConvention();
4397
4.25k
  bool IsVectorCall = CC == llvm::CallingConv::X86_VectorCall;
4398
4.25k
  bool IsRegCall = CC == llvm::CallingConv::X86_RegCall;
4399
4400
  // If __attribute__((sysv_abi)) is in use, use the SysV argument
4401
  // classification rules.
4402
4.25k
  if (CC == llvm::CallingConv::X86_64_SysV) {
4403
14
    X86_64ABIInfo SysVABIInfo(CGT, AVXLevel);
4404
14
    SysVABIInfo.computeInfo(FI);
4405
14
    return;
4406
14
  }
4407
4408
4.23k
  unsigned FreeSSERegs = 0;
4409
4.23k
  if (IsVectorCall) {
4410
    // We can use up to 4 SSE return registers with vectorcall.
4411
53
    FreeSSERegs = 4;
4412
4.18k
  } else if (IsRegCall) {
4413
    // RegCall gives us 16 SSE registers.
4414
27
    FreeSSERegs = 16;
4415
27
  }
4416
4417
4.23k
  if (!getCXXABI().classifyReturnType(FI))
4418
4.19k
    FI.getReturnInfo() = classify(FI.getReturnType(), FreeSSERegs, true,
4419
4.19k
                                  IsVectorCall, IsRegCall);
4420
4421
4.23k
  if (IsVectorCall) {
4422
    // We can use up to 6 SSE register parameters with vectorcall.
4423
53
    FreeSSERegs = 6;
4424
4.18k
  } else if (IsRegCall) {
4425
    // RegCall gives us 16 SSE registers, we can reuse the return registers.
4426
27
    FreeSSERegs = 16;
4427
27
  }
4428
4429
4.23k
  unsigned ArgNum = 0;
4430
4.23k
  unsigned ZeroSSERegs = 0;
4431
6.04k
  for (auto &I : FI.arguments()) {
4432
    // Vectorcall in x64 only permits the first 6 arguments to be passed as
4433
    // XMM/YMM registers. After the sixth argument, pretend no vector
4434
    // registers are left.
4435
6.04k
    unsigned *MaybeFreeSSERegs =
4436
6.04k
        (IsVectorCall && 
ArgNum >= 6112
) ?
&ZeroSSERegs5
:
&FreeSSERegs6.03k
;
4437
6.04k
    I.info =
4438
6.04k
        classify(I.type, *MaybeFreeSSERegs, false, IsVectorCall, IsRegCall);
4439
6.04k
    ++ArgNum;
4440
6.04k
  }
4441
4442
4.23k
  if (IsVectorCall) {
4443
    // For vectorcall, assign aggregate HVAs to any free vector registers in a
4444
    // second pass.
4445
53
    for (auto &I : FI.arguments())
4446
112
      I.info = reclassifyHvaArgForVectorCall(I.type, FreeSSERegs, I.info);
4447
53
  }
4448
4.23k
}
4449
4450
Address WinX86_64ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
4451
23
                                    QualType Ty) const {
4452
  // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
4453
  // not 1, 2, 4, or 8 bytes, must be passed by reference."
4454
23
  uint64_t Width = getContext().getTypeSize(Ty);
4455
23
  bool IsIndirect = Width > 64 || 
!llvm::isPowerOf2_64(Width)10
;
4456
4457
23
  return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect,
4458
23
                          CGF.getContext().getTypeInfoInChars(Ty),
4459
23
                          CharUnits::fromQuantity(8),
4460
23
                          /*allowHigherAlign*/ false);
4461
23
}
4462
4463
static bool PPC_initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
4464
                                        llvm::Value *Address, bool Is64Bit,
4465
4
                                        bool IsAIX) {
4466
  // This is calculated from the LLVM and GCC tables and verified
4467
  // against gcc output.  AFAIK all PPC ABIs use the same encoding.
4468
4469
4
  CodeGen::CGBuilderTy &Builder = CGF.Builder;
4470
4471
4
  llvm::IntegerType *i8 = CGF.Int8Ty;
4472
4
  llvm::Value *Four8 = llvm::ConstantInt::get(i8, 4);
4473
4
  llvm::Value *Eight8 = llvm::ConstantInt::get(i8, 8);
4474
4
  llvm::Value *Sixteen8 = llvm::ConstantInt::get(i8, 16);
4475
4476
  // 0-31: r0-31, the 4-byte or 8-byte general-purpose registers
4477
4
  AssignToArrayRange(Builder, Address, Is64Bit ? 
Eight82
:
Four82
, 0, 31);
4478
4479
  // 32-63: fp0-31, the 8-byte floating-point registers
4480
4
  AssignToArrayRange(Builder, Address, Eight8, 32, 63);
4481
4482
  // 64-67 are various 4-byte or 8-byte special-purpose registers:
4483
  // 64: mq
4484
  // 65: lr
4485
  // 66: ctr
4486
  // 67: ap
4487
4
  AssignToArrayRange(Builder, Address, Is64Bit ? 
Eight82
:
Four82
, 64, 67);
4488
4489
  // 68-76 are various 4-byte special-purpose registers:
4490
  // 68-75 cr0-7
4491
  // 76: xer
4492
4
  AssignToArrayRange(Builder, Address, Four8, 68, 76);
4493
4494
  // 77-108: v0-31, the 16-byte vector registers
4495
4
  AssignToArrayRange(Builder, Address, Sixteen8, 77, 108);
4496
4497
  // 109: vrsave
4498
  // 110: vscr
4499
4
  AssignToArrayRange(Builder, Address, Is64Bit ? 
Eight82
:
Four82
, 109, 110);
4500
4501
  // AIX does not utilize the rest of the registers.
4502
4
  if (IsAIX)
4503
2
    return false;
4504
4505
  // 111: spe_acc
4506
  // 112: spefscr
4507
  // 113: sfp
4508
2
  AssignToArrayRange(Builder, Address, Is64Bit ? 
Eight81
:
Four81
, 111, 113);
4509
4510
2
  if (!Is64Bit)
4511
1
    return false;
4512
4513
  // TODO: Need to verify if these registers are used on 64 bit AIX with Power8
4514
  // or above CPU.
4515
  // 64-bit only registers:
4516
  // 114: tfhar
4517
  // 115: tfiar
4518
  // 116: texasr
4519
1
  AssignToArrayRange(Builder, Address, Eight8, 114, 116);
4520
4521
1
  return false;
4522
2
}
4523
4524
// AIX
4525
namespace {
4526
/// AIXABIInfo - The AIX XCOFF ABI information.
4527
class AIXABIInfo : public ABIInfo {
4528
  const bool Is64Bit;
4529
  const unsigned PtrByteSize;
4530
  CharUnits getParamTypeAlignment(QualType Ty) const;
4531
4532
public:
4533
  AIXABIInfo(CodeGen::CodeGenTypes &CGT, bool Is64Bit)
4534
123
      : ABIInfo(CGT), Is64Bit(Is64Bit), PtrByteSize(Is64Bit ? 8 : 4) {}
4535
4536
  bool isPromotableTypeForABI(QualType Ty) const;
4537
4538
  ABIArgInfo classifyReturnType(QualType RetTy) const;
4539
  ABIArgInfo classifyArgumentType(QualType Ty) const;
4540
4541
340
  void computeInfo(CGFunctionInfo &FI) const override {
4542
340
    if (!getCXXABI().classifyReturnType(FI))
4543
340
      FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
4544
4545
340
    for (auto &I : FI.arguments())
4546
308
      I.info = classifyArgumentType(I.type);
4547
340
  }
4548
4549
  Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
4550
                    QualType Ty) const override;
4551
};
4552
4553
class AIXTargetCodeGenInfo : public TargetCodeGenInfo {
4554
  const bool Is64Bit;
4555
4556
public:
4557
  AIXTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, bool Is64Bit)
4558
      : TargetCodeGenInfo(std::make_unique<AIXABIInfo>(CGT, Is64Bit)),
4559
123
        Is64Bit(Is64Bit) {}
4560
2
  int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
4561
2
    return 1; // r1 is the dedicated stack pointer
4562
2
  }
4563
4564
  bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
4565
                               llvm::Value *Address) const override;
4566
};
4567
} // namespace
4568
4569
// Return true if the ABI requires Ty to be passed sign- or zero-
4570
// extended to 32/64 bits.
4571
416
bool AIXABIInfo::isPromotableTypeForABI(QualType Ty) const {
4572
  // Treat an enum type as its underlying type.
4573
416
  if (const EnumType *EnumTy = Ty->getAs<EnumType>())
4574
2
    Ty = EnumTy->getDecl()->getIntegerType();
4575
4576
  // Promotable integer types are required to be promoted by the ABI.
4577
416
  if (getContext().isPromotableIntegerType(Ty))
4578
5
    return true;
4579
4580
411
  if (!Is64Bit)
4581
215
    return false;
4582
4583
  // For 64 bit mode, in addition to the usual promotable integer types, we also
4584
  // need to extend all 32-bit types, since the ABI requires promotion to 64
4585
  // bits.
4586
196
  if (const BuiltinType *BT = Ty->getAs<BuiltinType>())
4587
103
    switch (BT->getKind()) {
4588
61
    case BuiltinType::Int:
4589
65
    case BuiltinType::UInt:
4590
65
      return true;
4591
38
    default:
4592
38
      break;
4593
103
    }
4594
4595
131
  return false;
4596
196
}
4597
4598
340
ABIArgInfo AIXABIInfo::classifyReturnType(QualType RetTy) const {
4599
340
  if (RetTy->isAnyComplexType())
4600
6
    return ABIArgInfo::getDirect();
4601
4602
334
  if (RetTy->isVectorType())
4603
0
    return ABIArgInfo::getDirect();
4604
4605
334
  if (RetTy->isVoidType())
4606
197
    return ABIArgInfo::getIgnore();
4607
4608
137
  if (isAggregateTypeForABI(RetTy))
4609
6
    return getNaturalAlignIndirect(RetTy);
4610
4611
131
  return (isPromotableTypeForABI(RetTy) ? 
ABIArgInfo::getExtend(RetTy)37
4612
131
                                        : 
ABIArgInfo::getDirect()94
);
4613
137
}
4614
4615
308
ABIArgInfo AIXABIInfo::classifyArgumentType(QualType Ty) const {
4616
308
  Ty = useFirstFieldIfTransparentUnion(Ty);
4617
4618
308
  if (Ty->isAnyComplexType())
4619
6
    return ABIArgInfo::getDirect();
4620
4621
302
  if (Ty->isVectorType())
4622
0
    return ABIArgInfo::getDirect();
4623
4624
302
  if (isAggregateTypeForABI(Ty)) {
4625
    // Records with non-trivial destructors/copy-constructors should not be
4626
    // passed by value.
4627
17
    if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))
4628
0
      return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
4629
4630
17
    CharUnits CCAlign = getParamTypeAlignment(Ty);
4631
17
    CharUnits TyAlign = getContext().getTypeAlignInChars(Ty);
4632
4633
17
    return ABIArgInfo::getIndirect(CCAlign, /*ByVal*/ true,
4634
17
                                   /*Realign*/ TyAlign > CCAlign);
4635
17
  }
4636
4637
285
  return (isPromotableTypeForABI(Ty) ? 
ABIArgInfo::getExtend(Ty)33
4638
285
                                     : 
ABIArgInfo::getDirect()252
);
4639
302
}
4640
4641
17
CharUnits AIXABIInfo::getParamTypeAlignment(QualType Ty) const {
4642
  // Complex types are passed just like their elements.
4643
17
  if (const ComplexType *CTy = Ty->getAs<ComplexType>())
4644
0
    Ty = CTy->getElementType();
4645
4646
17
  if (Ty->isVectorType())
4647
0
    return CharUnits::fromQuantity(16);
4648
4649
  // If the structure contains a vector type, the alignment is 16.
4650
17
  if (isRecordWithSIMDVectorType(getContext(), Ty))
4651
0
    return CharUnits::fromQuantity(16);
4652
4653
17
  return CharUnits::fromQuantity(PtrByteSize);
4654
17
}
4655
4656
Address AIXABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
4657
0
                              QualType Ty) const {
4658
4659
0
  auto TypeInfo = getContext().getTypeInfoInChars(Ty);
4660
0
  TypeInfo.Align = getParamTypeAlignment(Ty);
4661
4662
0
  CharUnits SlotSize = CharUnits::fromQuantity(PtrByteSize);
4663
4664
  // If we have a complex type and the base type is smaller than the register
4665
  // size, the ABI calls for the real and imaginary parts to be right-adjusted
4666
  // in separate words in 32bit mode or doublewords in 64bit mode. However,
4667
  // Clang expects us to produce a pointer to a structure with the two parts
4668
  // packed tightly. So generate loads of the real and imaginary parts relative
4669
  // to the va_list pointer, and store them to a temporary structure. We do the
4670
  // same as the PPC64ABI here.
4671
0
  if (const ComplexType *CTy = Ty->getAs<ComplexType>()) {
4672
0
    CharUnits EltSize = TypeInfo.Width / 2;
4673
0
    if (EltSize < SlotSize)
4674
0
      return complexTempStructure(CGF, VAListAddr, Ty, SlotSize, EltSize, CTy);
4675
0
  }
4676
4677
0
  return emitVoidPtrVAArg(CGF, VAListAddr, Ty, /*Indirect*/ false, TypeInfo,
4678
0
                          SlotSize, /*AllowHigher*/ true);
4679
0
}
4680
4681
bool AIXTargetCodeGenInfo::initDwarfEHRegSizeTable(
4682
2
    CodeGen::CodeGenFunction &CGF, llvm::Value *Address) const {
4683
2
  return PPC_initDwarfEHRegSizeTable(CGF, Address, Is64Bit, /*IsAIX*/ true);
4684
2
}
4685
4686
// PowerPC-32
4687
namespace {
4688
/// PPC32_SVR4_ABIInfo - The 32-bit PowerPC ELF (SVR4) ABI information.
4689
class PPC32_SVR4_ABIInfo : public DefaultABIInfo {
4690
  bool IsSoftFloatABI;
4691
  bool IsRetSmallStructInRegABI;
4692
4693
  CharUnits getParamTypeAlignment(QualType Ty) const;
4694
4695
public:
4696
  PPC32_SVR4_ABIInfo(CodeGen::CodeGenTypes &CGT, bool SoftFloatABI,
4697
                     bool RetSmallStructInRegABI)
4698
      : DefaultABIInfo(CGT), IsSoftFloatABI(SoftFloatABI),
4699
65
        IsRetSmallStructInRegABI(RetSmallStructInRegABI) {}
4700
4701
  ABIArgInfo classifyReturnType(QualType RetTy) const;
4702
4703
167
  void computeInfo(CGFunctionInfo &FI) const override {
4704
167
    if (!getCXXABI().classifyReturnType(FI))
4705
167
      FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
4706
167
    for (auto &I : FI.arguments())
4707
279
      I.info = classifyArgumentType(I.type);
4708
167
  }
4709
4710
  Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
4711
                    QualType Ty) const override;
4712
};
4713
4714
class PPC32TargetCodeGenInfo : public TargetCodeGenInfo {
4715
public:
4716
  PPC32TargetCodeGenInfo(CodeGenTypes &CGT, bool SoftFloatABI,
4717
                         bool RetSmallStructInRegABI)
4718
      : TargetCodeGenInfo(std::make_unique<PPC32_SVR4_ABIInfo>(
4719
65
            CGT, SoftFloatABI, RetSmallStructInRegABI)) {}
4720
4721
  static bool isStructReturnInRegABI(const llvm::Triple &Triple,
4722
                                     const CodeGenOptions &Opts);
4723
4724
1
  int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
4725
    // This is recovered from gcc output.
4726
1
    return 1; // r1 is the dedicated stack pointer
4727
1
  }
4728
4729
  bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
4730
                               llvm::Value *Address) const override;
4731
};
4732
}
4733
4734
0
CharUnits PPC32_SVR4_ABIInfo::getParamTypeAlignment(QualType Ty) const {
4735
  // Complex types are passed just like their elements.
4736
0
  if (const ComplexType *CTy = Ty->getAs<ComplexType>())
4737
0
    Ty = CTy->getElementType();
4738
4739
0
  if (Ty->isVectorType())
4740
0
    return CharUnits::fromQuantity(getContext().getTypeSize(Ty) == 128 ? 16
4741
0
                                                                       : 4);
4742
4743
  // For single-element float/vector structs, we consider the whole type
4744
  // to have the same alignment requirements as its single element.
4745
0
  const Type *AlignTy = nullptr;
4746
0
  if (const Type *EltType = isSingleElementStruct(Ty, getContext())) {
4747
0
    const BuiltinType *BT = EltType->getAs<BuiltinType>();
4748
0
    if ((EltType->isVectorType() && getContext().getTypeSize(EltType) == 128) ||
4749
0
        (BT && BT->isFloatingPoint()))
4750
0
      AlignTy = EltType;
4751
0
  }
4752
4753
0
  if (AlignTy)
4754
0
    return CharUnits::fromQuantity(AlignTy->isVectorType() ? 16 : 4);
4755
0
  return CharUnits::fromQuantity(4);
4756
0
}
4757
4758
167
ABIArgInfo PPC32_SVR4_ABIInfo::classifyReturnType(QualType RetTy) const {
4759
167
  uint64_t Size;
4760
4761
  // -msvr4-struct-return puts small aggregates in GPR3 and GPR4.
4762
167
  if (isAggregateTypeForABI(RetTy) && 
IsRetSmallStructInRegABI22
&&
4763
167
      
(Size = getContext().getTypeSize(RetTy)) <= 6419
) {
4764
    // System V ABI (1995), page 3-22, specified:
4765
    // > A structure or union whose size is less than or equal to 8 bytes
4766
    // > shall be returned in r3 and r4, as if it were first stored in the
4767
    // > 8-byte aligned memory area and then the low addressed word were
4768
    // > loaded into r3 and the high-addressed word into r4.  Bits beyond
4769
    // > the last member of the structure or union are not defined.
4770
    //
4771
    // GCC for big-endian PPC32 inserts the pad before the first member,
4772
    // not "beyond the last member" of the struct.  To stay compatible
4773
    // with GCC, we coerce the struct to an integer of the same size.
4774
    // LLVM will extend it and return i32 in r3, or i64 in r3:r4.
4775
8
    if (Size == 0)
4776
1
      return ABIArgInfo::getIgnore();
4777
7
    else {
4778
7
      llvm::Type *CoerceTy = llvm::Type::getIntNTy(getVMContext(), Size);
4779
7
      return ABIArgInfo::getDirect(CoerceTy);
4780
7
    }
4781
8
  }
4782
4783
159
  return DefaultABIInfo::classifyReturnType(RetTy);
4784
167
}
4785
4786
// TODO: this implementation is now likely redundant with
4787
// DefaultABIInfo::EmitVAArg.
4788
Address PPC32_SVR4_ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAList,
4789
1
                                      QualType Ty) const {
4790
1
  if (getTarget().getTriple().isOSDarwin()) {
4791
0
    auto TI = getContext().getTypeInfoInChars(Ty);
4792
0
    TI.Align = getParamTypeAlignment(Ty);
4793
4794
0
    CharUnits SlotSize = CharUnits::fromQuantity(4);
4795
0
    return emitVoidPtrVAArg(CGF, VAList, Ty,
4796
0
                            classifyArgumentType(Ty).isIndirect(), TI, SlotSize,
4797
0
                            /*AllowHigherAlign=*/true);
4798
0
  }
4799
4800
1
  const unsigned OverflowLimit = 8;
4801
1
  if (const ComplexType *CTy = Ty->getAs<ComplexType>()) {
4802
    // TODO: Implement this. For now ignore.
4803
0
    (void)CTy;
4804
0
    return Address::invalid(); // FIXME?
4805
0
  }
4806
4807
  // struct __va_list_tag {
4808
  //   unsigned char gpr;
4809
  //   unsigned char fpr;
4810
  //   unsigned short reserved;
4811
  //   void *overflow_arg_area;
4812
  //   void *reg_save_area;
4813
  // };
4814
4815
1
  bool isI64 = Ty->isIntegerType() && 
getContext().getTypeSize(Ty) == 640
;
4816
1
  bool isInt = !Ty->isFloatingType();
4817
1
  bool isF64 = Ty->isFloatingType() && getContext().getTypeSize(Ty) == 64;
4818
4819
  // All aggregates are passed indirectly?  That doesn't seem consistent
4820
  // with the argument-lowering code.
4821
1
  bool isIndirect = isAggregateTypeForABI(Ty);
4822
4823
1
  CGBuilderTy &Builder = CGF.Builder;
4824
4825
  // The calling convention either uses 1-2 GPRs or 1 FPR.
4826
1
  Address NumRegsAddr = Address::invalid();
4827
1
  if (isInt || IsSoftFloatABI) {
4828
1
    NumRegsAddr = Builder.CreateStructGEP(VAList, 0, "gpr");
4829
1
  } else {
4830
0
    NumRegsAddr = Builder.CreateStructGEP(VAList, 1, "fpr");
4831
0
  }
4832
4833
1
  llvm::Value *NumRegs = Builder.CreateLoad(NumRegsAddr, "numUsedRegs");
4834
4835
  // "Align" the register count when TY is i64.
4836
1
  if (isI64 || (isF64 && IsSoftFloatABI)) {
4837
1
    NumRegs = Builder.CreateAdd(NumRegs, Builder.getInt8(1));
4838
1
    NumRegs = Builder.CreateAnd(NumRegs, Builder.getInt8((uint8_t) ~1U));
4839
1
  }
4840
4841
1
  llvm::Value *CC =
4842
1
      Builder.CreateICmpULT(NumRegs, Builder.getInt8(OverflowLimit), "cond");
4843
4844
1
  llvm::BasicBlock *UsingRegs = CGF.createBasicBlock("using_regs");
4845
1
  llvm::BasicBlock *UsingOverflow = CGF.createBasicBlock("using_overflow");
4846
1
  llvm::BasicBlock *Cont = CGF.createBasicBlock("cont");
4847
4848
1
  Builder.CreateCondBr(CC, UsingRegs, UsingOverflow);
4849
4850
1
  llvm::Type *DirectTy = CGF.ConvertType(Ty), *ElementTy = DirectTy;
4851
1
  if (isIndirect) 
DirectTy = DirectTy->getPointerTo(0)0
;
4852
4853
  // Case 1: consume registers.
4854
1
  Address RegAddr = Address::invalid();
4855
1
  {
4856
1
    CGF.EmitBlock(UsingRegs);
4857
4858
1
    Address RegSaveAreaPtr = Builder.CreateStructGEP(VAList, 4);
4859
1
    RegAddr = Address(Builder.CreateLoad(RegSaveAreaPtr), CGF.Int8Ty,
4860
1
                      CharUnits::fromQuantity(8));
4861
1
    assert(RegAddr.getElementType() == CGF.Int8Ty);
4862
4863
    // Floating-point registers start after the general-purpose registers.
4864
1
    if (!(isInt || IsSoftFloatABI)) {
4865
0
      RegAddr = Builder.CreateConstInBoundsByteGEP(RegAddr,
4866
0
                                                   CharUnits::fromQuantity(32));
4867
0
    }
4868
4869
    // Get the address of the saved value by scaling the number of
4870
    // registers we've used by the number of
4871
1
    CharUnits RegSize = CharUnits::fromQuantity((isInt || IsSoftFloatABI) ? 4 : 
80
);
4872
1
    llvm::Value *RegOffset =
4873
1
        Builder.CreateMul(NumRegs, Builder.getInt8(RegSize.getQuantity()));
4874
1
    RegAddr = Address(
4875
1
        Builder.CreateInBoundsGEP(CGF.Int8Ty, RegAddr.getPointer(), RegOffset),
4876
1
        CGF.Int8Ty, RegAddr.getAlignment().alignmentOfArrayElement(RegSize));
4877
1
    RegAddr = Builder.CreateElementBitCast(RegAddr, DirectTy);
4878
4879
    // Increase the used-register count.
4880
1
    NumRegs =
4881
1
      Builder.CreateAdd(NumRegs,
4882
1
                        Builder.getInt8((isI64 || (isF64 && IsSoftFloatABI)) ? 2 : 
10
));
4883
1
    Builder.CreateStore(NumRegs, NumRegsAddr);
4884
4885
1
    CGF.EmitBranch(Cont);
4886
1
  }
4887
4888
  // Case 2: consume space in the overflow area.
4889
0
  Address MemAddr = Address::invalid();
4890
1
  {
4891
1
    CGF.EmitBlock(UsingOverflow);
4892
4893
1
    Builder.CreateStore(Builder.getInt8(OverflowLimit), NumRegsAddr);
4894
4895
    // Everything in the overflow area is rounded up to a size of at least 4.
4896
1
    CharUnits OverflowAreaAlign = CharUnits::fromQuantity(4);
4897
4898
1
    CharUnits Size;
4899
1
    if (!isIndirect) {
4900
1
      auto TypeInfo = CGF.getContext().getTypeInfoInChars(Ty);
4901
1
      Size = TypeInfo.Width.alignTo(OverflowAreaAlign);
4902
1
    } else {
4903
0
      Size = CGF.getPointerSize();
4904
0
    }
4905
4906
1
    Address OverflowAreaAddr = Builder.CreateStructGEP(VAList, 3);
4907
1
    Address OverflowArea =
4908
1
        Address(Builder.CreateLoad(OverflowAreaAddr, "argp.cur"), CGF.Int8Ty,
4909
1
                OverflowAreaAlign);
4910
    // Round up address of argument to alignment
4911
1
    CharUnits Align = CGF.getContext().getTypeAlignInChars(Ty);
4912
1
    if (Align > OverflowAreaAlign) {
4913
1
      llvm::Value *Ptr = OverflowArea.getPointer();
4914
1
      OverflowArea = Address(emitRoundPointerUpToAlignment(CGF, Ptr, Align),
4915
1
                             OverflowArea.getElementType(), Align);
4916
1
    }
4917
4918
1
    MemAddr = Builder.CreateElementBitCast(OverflowArea, DirectTy);
4919
4920
    // Increase the overflow area.
4921
1
    OverflowArea = Builder.CreateConstInBoundsByteGEP(OverflowArea, Size);
4922
1
    Builder.CreateStore(OverflowArea.getPointer(), OverflowAreaAddr);
4923
1
    CGF.EmitBranch(Cont);
4924
1
  }
4925
4926
1
  CGF.EmitBlock(Cont);
4927
4928
  // Merge the cases with a phi.
4929
1
  Address Result = emitMergePHI(CGF, RegAddr, UsingRegs, MemAddr, UsingOverflow,
4930
1
                                "vaarg.addr");
4931
4932
  // Load the pointer if the argument was passed indirectly.
4933
1
  if (isIndirect) {
4934
0
    Result = Address(Builder.CreateLoad(Result, "aggr"), ElementTy,
4935
0
                     getContext().getTypeAlignInChars(Ty));
4936
0
  }
4937
4938
1
  return Result;
4939
1
}
4940
4941
bool PPC32TargetCodeGenInfo::isStructReturnInRegABI(
4942
65
    const llvm::Triple &Triple, const CodeGenOptions &Opts) {
4943
65
  assert(Triple.isPPC32());
4944
4945
65
  switch (Opts.getStructReturnConvention()) {
4946
65
  case CodeGenOptions::SRCK_Default:
4947
65
    break;
4948
0
  case CodeGenOptions::SRCK_OnStack: // -maix-struct-return
4949
0
    return false;
4950
0
  case CodeGenOptions::SRCK_InRegs: // -msvr4-struct-return
4951
0
    return true;
4952
65
  }
4953
4954
65
  if (Triple.isOSBinFormatELF() && !Triple.isOSLinux())
4955
56
    return true;
4956
4957
9
  return false;
4958
65
}
4959
4960
bool
4961
PPC32TargetCodeGenInfo::initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
4962
1
                                                llvm::Value *Address) const {
4963
1
  return PPC_initDwarfEHRegSizeTable(CGF, Address, /*Is64Bit*/ false,
4964
1
                                     /*IsAIX*/ false);
4965
1
}
4966
4967
// PowerPC-64
4968
4969
namespace {
4970
enum class PPC64_SVR4_ABIKind {
4971
  ELFv1 = 0,
4972
  ELFv2,
4973
};
4974
4975
/// PPC64_SVR4_ABIInfo - The 64-bit PowerPC ELF (SVR4) ABI information.
4976
class PPC64_SVR4_ABIInfo : public ABIInfo {
4977
  static const unsigned GPRBits = 64;
4978
  PPC64_SVR4_ABIKind Kind;
4979
  bool IsSoftFloatABI;
4980
4981
public:
4982
  PPC64_SVR4_ABIInfo(CodeGen::CodeGenTypes &CGT, PPC64_SVR4_ABIKind Kind,
4983
                     bool SoftFloatABI)
4984
2.59k
      : ABIInfo(CGT), Kind(Kind), IsSoftFloatABI(SoftFloatABI) {}
4985
4986
  bool isPromotableTypeForABI(QualType Ty) const;
4987
  CharUnits getParamTypeAlignment(QualType Ty) const;
4988
4989
  ABIArgInfo classifyReturnType(QualType RetTy) const;
4990
  ABIArgInfo classifyArgumentType(QualType Ty) const;
4991
4992
  bool isHomogeneousAggregateBaseType(QualType Ty) const override;
4993
  bool isHomogeneousAggregateSmallEnough(const Type *Ty,
4994
                                         uint64_t Members) const override;
4995
4996
  // TODO: We can add more logic to computeInfo to improve performance.
4997
  // Example: For aggregate arguments that fit in a register, we could
4998
  // use getDirectInReg (as is done below for structs containing a single
4999
  // floating-point value) to avoid pushing them to memory on function
5000
  // entry.  This would require changing the logic in PPCISelLowering
5001
  // when lowering the parameters in the caller and args in the callee.
5002
14.1k
  void computeInfo(CGFunctionInfo &FI) const override {
5003
14.1k
    if (!getCXXABI().classifyReturnType(FI))
5004
14.1k
      FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
5005
27.8k
    for (auto &I : FI.arguments()) {
5006
      // We rely on the default argument classification for the most part.
5007
      // One exception:  An aggregate containing a single floating-point
5008
      // or vector item must be passed in a register if one is available.
5009
27.8k
      const Type *T = isSingleElementStruct(I.type, getContext());
5010
27.8k
      if (T) {
5011
24
        const BuiltinType *BT = T->getAs<BuiltinType>();
5012
24
        if ((T->isVectorType() && 
getContext().getTypeSize(T) == 1283
) ||
5013
24
            
(22
BT22
&&
BT->isFloatingPoint()19
)) {
5014
19
          QualType QT(T, 0);
5015
19
          I.info = ABIArgInfo::getDirectInReg(CGT.ConvertType(QT));
5016
19
          continue;
5017
19
        }
5018
24
      }
5019
27.8k
      I.info = classifyArgumentType(I.type);
5020
27.8k
    }
5021
14.1k
  }
5022
5023
  Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
5024
                    QualType Ty) const override;
5025
};
5026
5027
class PPC64_SVR4_TargetCodeGenInfo : public TargetCodeGenInfo {
5028
5029
public:
5030
  PPC64_SVR4_TargetCodeGenInfo(CodeGenTypes &CGT, PPC64_SVR4_ABIKind Kind,
5031
                               bool SoftFloatABI)
5032
      : TargetCodeGenInfo(
5033
2.59k
            std::make_unique<PPC64_SVR4_ABIInfo>(CGT, Kind, SoftFloatABI)) {
5034
2.59k
    SwiftInfo =
5035
2.59k
        std::make_unique<SwiftABIInfo>(CGT, /*SwiftErrorInRegister=*/false);
5036
2.59k
  }
5037
5038
1
  int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
5039
    // This is recovered from gcc output.
5040
1
    return 1; // r1 is the dedicated stack pointer
5041
1
  }
5042
5043
  bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
5044
                               llvm::Value *Address) const override;
5045
};
5046
5047
class PPC64TargetCodeGenInfo : public TargetCodeGenInfo {
5048
public:
5049
  PPC64TargetCodeGenInfo(CodeGenTypes &CGT)
5050
0
      : TargetCodeGenInfo(std::make_unique<DefaultABIInfo>(CGT)) {}
5051
5052
0
  int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
5053
    // This is recovered from gcc output.
5054
0
    return 1; // r1 is the dedicated stack pointer
5055
0
  }
5056
5057
  bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
5058
                               llvm::Value *Address) const override;
5059
};
5060
}
5061
5062
// Return true if the ABI requires Ty to be passed sign- or zero-
5063
// extended to 64 bits.
5064
bool
5065
30.6k
PPC64_SVR4_ABIInfo::isPromotableTypeForABI(QualType Ty) const {
5066
  // Treat an enum type as its underlying type.
5067
30.6k
  if (const EnumType *EnumTy = Ty->getAs<EnumType>())
5068
6
    Ty = EnumTy->getDecl()->getIntegerType();
5069
5070
  // Promotable integer types are required to be promoted by the ABI.
5071
30.6k
  if (isPromotableIntegerTypeForABI(Ty))
5072
63
    return true;
5073
5074
  // In addition to the usual promotable integer types, we also need to
5075
  // extend all 32-bit types, since the ABI requires promotion to 64 bits.
5076
30.6k
  if (const BuiltinType *BT = Ty->getAs<BuiltinType>())
5077
13.0k
    switch (BT->getKind()) {
5078
5.46k
    case BuiltinType::Int:
5079
5.48k
    case BuiltinType::UInt:
5080
5.48k
      return true;
5081
7.54k
    default:
5082
7.54k
      break;
5083
13.0k
    }
5084
5085
25.1k
  if (const auto *EIT = Ty->getAs<BitIntType>())
5086
9
    if (EIT->getNumBits() < 64)
5087
2
      return true;
5088
5089
25.1k
  return false;
5090
25.1k
}
5091
5092
/// isAlignedParamType - Determine whether a type requires 16-byte or
5093
/// higher alignment in the parameter area.  Always returns at least 8.
5094
143
CharUnits PPC64_SVR4_ABIInfo::getParamTypeAlignment(QualType Ty) const {
5095
  // Complex types are passed just like their elements.
5096
143
  if (const ComplexType *CTy = Ty->getAs<ComplexType>())
5097
0
    Ty = CTy->getElementType();
5098
5099
180
  auto FloatUsesVector = [this](QualType Ty){
5100
180
    return Ty->isRealFloatingType() && &getContext().getFloatTypeSemantics(
5101
52
                                           Ty) == &llvm::APFloat::IEEEquad();
5102
180
  };
5103
5104
  // Only vector types of size 16 bytes need alignment (larger types are
5105
  // passed via reference, smaller types are not aligned).
5106
143
  if (Ty->isVectorType()) {
5107
0
    return CharUnits::fromQuantity(getContext().getTypeSize(Ty) == 128 ? 16 : 8);
5108
143
  } else if (FloatUsesVector(Ty)) {
5109
    // According to ABI document section 'Optional Save Areas': If extended
5110
    // precision floating-point values in IEEE BINARY 128 QUADRUPLE PRECISION
5111
    // format are supported, map them to a single quadword, quadword aligned.
5112
9
    return CharUnits::fromQuantity(16);
5113
9
  }
5114
5115
  // For single-element float/vector structs, we consider the whole type
5116
  // to have the same alignment requirements as its single element.
5117
134
  const Type *AlignAsType = nullptr;
5118
134
  const Type *EltType = isSingleElementStruct(Ty, getContext());
5119
134
  if (EltType) {
5120
8
    const BuiltinType *BT = EltType->getAs<BuiltinType>();
5121
8
    if ((EltType->isVectorType() && 
getContext().getTypeSize(EltType) == 1282
) ||
5122
8
        
(7
BT7
&&
BT->isFloatingPoint()6
))
5123
5
      AlignAsType = EltType;
5124
8
  }
5125
5126
  // Likewise for ELFv2 homogeneous aggregates.
5127
134
  const Type *Base = nullptr;
5128
134
  uint64_t Members = 0;
5129
134
  if (!AlignAsType && 
Kind == PPC64_SVR4_ABIKind::ELFv2129
&&
5130
134
      
isAggregateTypeForABI(Ty)97
&&
isHomogeneousAggregate(Ty, Base, Members)92
)
5131
51
    AlignAsType = Base;
5132
5133
  // With special case aggregates, only vector base types need alignment.
5134
134
  if (AlignAsType) {
5135
56
    bool UsesVector = AlignAsType->isVectorType() ||
5136
56
                      
FloatUsesVector(QualType(AlignAsType, 0))37
;
5137
56
    return CharUnits::fromQuantity(UsesVector ? 
1631
:
825
);
5138
56
  }
5139
5140
  // Otherwise, we only need alignment for any aggregate type that
5141
  // has an alignment requirement of >= 16 bytes.
5142
78
  if (isAggregateTypeForABI(Ty) && 
getContext().getTypeAlign(Ty) >= 12872
) {
5143
14
    return CharUnits::fromQuantity(16);
5144
14
  }
5145
5146
64
  return CharUnits::fromQuantity(8);
5147
78
}
5148
5149
/// isHomogeneousAggregate - Return true if a type is an ELFv2 homogeneous
5150
/// aggregate.  Base is set to the base element type, and Members is set
5151
/// to the number of base elements.
5152
bool ABIInfo::isHomogeneousAggregate(QualType Ty, const Type *&Base,
5153
6.57k
                                     uint64_t &Members) const {
5154
6.57k
  if (const ConstantArrayType *AT = getContext().getA