Coverage Report

Created: 2022-07-16 07:03

/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/CodeGen/TargetInfo.cpp
Line
Count
Source (jump to first uncovered line)
1
//===---- TargetInfo.cpp - Encapsulate target details -----------*- C++ -*-===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// These classes wrap the information about a call or function
10
// definition used to handle ABI compliancy.
11
//
12
//===----------------------------------------------------------------------===//
13
14
#include "TargetInfo.h"
15
#include "ABIInfo.h"
16
#include "CGBlocks.h"
17
#include "CGCXXABI.h"
18
#include "CGValue.h"
19
#include "CodeGenFunction.h"
20
#include "clang/AST/Attr.h"
21
#include "clang/AST/RecordLayout.h"
22
#include "clang/Basic/Builtins.h"
23
#include "clang/Basic/CodeGenOptions.h"
24
#include "clang/Basic/DiagnosticFrontend.h"
25
#include "clang/CodeGen/CGFunctionInfo.h"
26
#include "clang/CodeGen/SwiftCallingConv.h"
27
#include "llvm/ADT/SmallBitVector.h"
28
#include "llvm/ADT/StringExtras.h"
29
#include "llvm/ADT/StringSwitch.h"
30
#include "llvm/ADT/Triple.h"
31
#include "llvm/ADT/Twine.h"
32
#include "llvm/IR/DataLayout.h"
33
#include "llvm/IR/IntrinsicsNVPTX.h"
34
#include "llvm/IR/IntrinsicsS390.h"
35
#include "llvm/IR/Type.h"
36
#include "llvm/Support/MathExtras.h"
37
#include "llvm/Support/raw_ostream.h"
38
#include <algorithm> // std::sort
39
40
using namespace clang;
41
using namespace CodeGen;
42
43
// Helper for coercing an aggregate argument or return value into an integer
44
// array of the same size (including padding) and alignment.  This alternate
45
// coercion happens only for the RenderScript ABI and can be removed after
46
// runtimes that rely on it are no longer supported.
47
//
48
// RenderScript assumes that the size of the argument / return value in the IR
49
// is the same as the size of the corresponding qualified type. This helper
50
// coerces the aggregate type into an array of the same size (including
51
// padding).  This coercion is used in lieu of expansion of struct members or
52
// other canonical coercions that return a coerced-type of larger size.
53
//
54
// Ty          - The argument / return value type
55
// Context     - The associated ASTContext
56
// LLVMContext - The associated LLVMContext
57
static ABIArgInfo coerceToIntArray(QualType Ty,
58
                                   ASTContext &Context,
59
19
                                   llvm::LLVMContext &LLVMContext) {
60
  // Alignment and Size are measured in bits.
61
19
  const uint64_t Size = Context.getTypeSize(Ty);
62
19
  const uint64_t Alignment = Context.getTypeAlign(Ty);
63
19
  llvm::Type *IntType = llvm::Type::getIntNTy(LLVMContext, Alignment);
64
19
  const uint64_t NumElements = (Size + Alignment - 1) / Alignment;
65
19
  return ABIArgInfo::getDirect(llvm::ArrayType::get(IntType, NumElements));
66
19
}
67
68
static void AssignToArrayRange(CodeGen::CGBuilderTy &Builder,
69
                               llvm::Value *Array,
70
                               llvm::Value *Value,
71
                               unsigned FirstIndex,
72
31
                               unsigned LastIndex) {
73
  // Alternatively, we could emit this as a loop in the source.
74
572
  for (unsigned I = FirstIndex; I <= LastIndex; 
++I541
) {
75
541
    llvm::Value *Cell =
76
541
        Builder.CreateConstInBoundsGEP1_32(Builder.getInt8Ty(), Array, I);
77
541
    Builder.CreateAlignedStore(Value, Cell, CharUnits::One());
78
541
  }
79
31
}
80
81
179k
static bool isAggregateTypeForABI(QualType T) {
82
179k
  return !CodeGenFunction::hasScalarEvaluationKind(T) ||
83
179k
         
T->isMemberFunctionPointerType()172k
;
84
179k
}
85
86
ABIArgInfo ABIInfo::getNaturalAlignIndirect(QualType Ty, bool ByVal,
87
                                            bool Realign,
88
3.70k
                                            llvm::Type *Padding) const {
89
3.70k
  return ABIArgInfo::getIndirect(getContext().getTypeAlignInChars(Ty), ByVal,
90
3.70k
                                 Realign, Padding);
91
3.70k
}
92
93
ABIArgInfo
94
25
ABIInfo::getNaturalAlignIndirectInReg(QualType Ty, bool Realign) const {
95
25
  return ABIArgInfo::getIndirectInReg(getContext().getTypeAlignInChars(Ty),
96
25
                                      /*ByVal*/ false, Realign);
97
25
}
98
99
Address ABIInfo::EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr,
100
0
                             QualType Ty) const {
101
0
  return Address::invalid();
102
0
}
103
104
7
static llvm::Type *getVAListElementType(CodeGenFunction &CGF) {
105
7
  return CGF.ConvertTypeForMem(
106
7
      CGF.getContext().getBuiltinVaListType()->getPointeeType());
107
7
}
108
109
276k
bool ABIInfo::isPromotableIntegerTypeForABI(QualType Ty) const {
110
276k
  if (Ty->isPromotableIntegerType())
111
13.1k
    return true;
112
113
262k
  if (const auto *EIT = Ty->getAs<BitIntType>())
114
339
    if (EIT->getNumBits() < getContext().getTypeSize(getContext().IntTy))
115
134
      return true;
116
117
262k
  return false;
118
262k
}
119
120
36.6k
ABIInfo::~ABIInfo() {}
121
122
/// Does the given lowering require more than the given number of
123
/// registers when expanded?
124
///
125
/// This is intended to be the basis of a reasonable basic implementation
126
/// of should{Pass,Return}IndirectlyForSwift.
127
///
128
/// For most targets, a limit of four total registers is reasonable; this
129
/// limits the amount of code required in order to move around the value
130
/// in case it wasn't produced immediately prior to the call by the caller
131
/// (or wasn't produced in exactly the right registers) or isn't used
132
/// immediately within the callee.  But some targets may need to further
133
/// limit the register count due to an inability to support that many
134
/// return registers.
135
static bool occupiesMoreThan(CodeGenTypes &cgt,
136
                             ArrayRef<llvm::Type*> scalarTypes,
137
1.05k
                             unsigned maxAllRegisters) {
138
1.05k
  unsigned intCount = 0, fpCount = 0;
139
4.18k
  for (llvm::Type *type : scalarTypes) {
140
4.18k
    if (type->isPointerTy()) {
141
0
      intCount++;
142
4.18k
    } else if (auto intTy = dyn_cast<llvm::IntegerType>(type)) {
143
2.30k
      auto ptrWidth = cgt.getTarget().getPointerWidth(0);
144
2.30k
      intCount += (intTy->getBitWidth() + ptrWidth - 1) / ptrWidth;
145
2.30k
    } else {
146
1.88k
      assert(type->isVectorTy() || type->isFloatingPointTy());
147
0
      fpCount++;
148
1.88k
    }
149
4.18k
  }
150
151
1.05k
  return (intCount + fpCount > maxAllRegisters);
152
1.05k
}
153
154
bool SwiftABIInfo::isLegalVectorTypeForSwift(CharUnits vectorSize,
155
                                             llvm::Type *eltTy,
156
440
                                             unsigned numElts) const {
157
  // The default implementation of this assumes that the target guarantees
158
  // 128-bit SIMD support but nothing more.
159
440
  return (vectorSize.getQuantity() > 8 && 
vectorSize.getQuantity() <= 16432
);
160
440
}
161
162
static CGCXXABI::RecordArgABI getRecordArgABI(const RecordType *RT,
163
20.3k
                                              CGCXXABI &CXXABI) {
164
20.3k
  const CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(RT->getDecl());
165
20.3k
  if (!RD) {
166
4.19k
    if (!RT->getDecl()->canPassInRegisters())
167
16
      return CGCXXABI::RAA_Indirect;
168
4.17k
    return CGCXXABI::RAA_Default;
169
4.19k
  }
170
16.1k
  return CXXABI.getRecordArgABI(RD);
171
20.3k
}
172
173
static CGCXXABI::RecordArgABI getRecordArgABI(QualType T,
174
9.30k
                                              CGCXXABI &CXXABI) {
175
9.30k
  const RecordType *RT = T->getAs<RecordType>();
176
9.30k
  if (!RT)
177
3.89k
    return CGCXXABI::RAA_Default;
178
5.41k
  return getRecordArgABI(RT, CXXABI);
179
9.30k
}
180
181
static bool classifyReturnType(const CGCXXABI &CXXABI, CGFunctionInfo &FI,
182
212k
                               const ABIInfo &Info) {
183
212k
  QualType Ty = FI.getReturnType();
184
185
212k
  if (const auto *RT = Ty->getAs<RecordType>())
186
6.19k
    if (!isa<CXXRecordDecl>(RT->getDecl()) &&
187
6.19k
        
!RT->getDecl()->canPassInRegisters()1.27k
) {
188
12
      FI.getReturnInfo() = Info.getNaturalAlignIndirect(Ty);
189
12
      return true;
190
12
    }
191
192
212k
  return CXXABI.classifyReturnType(FI);
193
212k
}
194
195
/// Pass transparent unions as if they were the type of the first element. Sema
196
/// should ensure that all elements of the union have the same "machine type".
197
429k
static QualType useFirstFieldIfTransparentUnion(QualType Ty) {
198
429k
  if (const RecordType *UT = Ty->getAsUnionType()) {
199
161
    const RecordDecl *UD = UT->getDecl();
200
161
    if (UD->hasAttr<TransparentUnionAttr>()) {
201
22
      assert(!UD->field_empty() && "sema created an empty transparent union");
202
0
      return UD->field_begin()->getType();
203
22
    }
204
161
  }
205
429k
  return Ty;
206
429k
}
207
208
260k
CGCXXABI &ABIInfo::getCXXABI() const {
209
260k
  return CGT.getCXXABI();
210
260k
}
211
212
431k
ASTContext &ABIInfo::getContext() const {
213
431k
  return CGT.getContext();
214
431k
}
215
216
17.5k
llvm::LLVMContext &ABIInfo::getVMContext() const {
217
17.5k
  return CGT.getLLVMContext();
218
17.5k
}
219
220
22.6k
const llvm::DataLayout &ABIInfo::getDataLayout() const {
221
22.6k
  return CGT.getDataLayout();
222
22.6k
}
223
224
34.3k
const TargetInfo &ABIInfo::getTarget() const {
225
34.3k
  return CGT.getTarget();
226
34.3k
}
227
228
1.95k
const CodeGenOptions &ABIInfo::getCodeGenOpts() const {
229
1.95k
  return CGT.getCodeGenOpts();
230
1.95k
}
231
232
5.03k
bool ABIInfo::isAndroid() const { return getTarget().getTriple().isAndroid(); }
233
234
0
bool ABIInfo::isHomogeneousAggregateBaseType(QualType Ty) const {
235
0
  return false;
236
0
}
237
238
bool ABIInfo::isHomogeneousAggregateSmallEnough(const Type *Base,
239
0
                                                uint64_t Members) const {
240
0
  return false;
241
0
}
242
243
765
bool ABIInfo::isZeroLengthBitfieldPermittedInHomogeneousAggregate() const {
244
  // For compatibility with GCC, ignore empty bitfields in C++ mode.
245
765
  return getContext().getLangOpts().CPlusPlus;
246
765
}
247
248
0
LLVM_DUMP_METHOD void ABIArgInfo::dump() const {
249
0
  raw_ostream &OS = llvm::errs();
250
0
  OS << "(ABIArgInfo Kind=";
251
0
  switch (TheKind) {
252
0
  case Direct:
253
0
    OS << "Direct Type=";
254
0
    if (llvm::Type *Ty = getCoerceToType())
255
0
      Ty->print(OS);
256
0
    else
257
0
      OS << "null";
258
0
    break;
259
0
  case Extend:
260
0
    OS << "Extend";
261
0
    break;
262
0
  case Ignore:
263
0
    OS << "Ignore";
264
0
    break;
265
0
  case InAlloca:
266
0
    OS << "InAlloca Offset=" << getInAllocaFieldIndex();
267
0
    break;
268
0
  case Indirect:
269
0
    OS << "Indirect Align=" << getIndirectAlign().getQuantity()
270
0
       << " ByVal=" << getIndirectByVal()
271
0
       << " Realign=" << getIndirectRealign();
272
0
    break;
273
0
  case IndirectAliased:
274
0
    OS << "Indirect Align=" << getIndirectAlign().getQuantity()
275
0
       << " AadrSpace=" << getIndirectAddrSpace()
276
0
       << " Realign=" << getIndirectRealign();
277
0
    break;
278
0
  case Expand:
279
0
    OS << "Expand";
280
0
    break;
281
0
  case CoerceAndExpand:
282
0
    OS << "CoerceAndExpand Type=";
283
0
    getCoerceAndExpandType()->print(OS);
284
0
    break;
285
0
  }
286
0
  OS << ")\n";
287
0
}
288
289
// Dynamically round a pointer up to a multiple of the given alignment.
290
static llvm::Value *emitRoundPointerUpToAlignment(CodeGenFunction &CGF,
291
                                                  llvm::Value *Ptr,
292
123
                                                  CharUnits Align) {
293
123
  llvm::Value *PtrAsInt = Ptr;
294
  // OverflowArgArea = (OverflowArgArea + Align - 1) & -Align;
295
123
  PtrAsInt = CGF.Builder.CreatePtrToInt(PtrAsInt, CGF.IntPtrTy);
296
123
  PtrAsInt = CGF.Builder.CreateAdd(PtrAsInt,
297
123
        llvm::ConstantInt::get(CGF.IntPtrTy, Align.getQuantity() - 1));
298
123
  PtrAsInt = CGF.Builder.CreateAnd(PtrAsInt,
299
123
           llvm::ConstantInt::get(CGF.IntPtrTy, -Align.getQuantity()));
300
123
  PtrAsInt = CGF.Builder.CreateIntToPtr(PtrAsInt,
301
123
                                        Ptr->getType(),
302
123
                                        Ptr->getName() + ".aligned");
303
123
  return PtrAsInt;
304
123
}
305
306
/// Emit va_arg for a platform using the common void* representation,
307
/// where arguments are simply emitted in an array of slots on the stack.
308
///
309
/// This version implements the core direct-value passing rules.
310
///
311
/// \param SlotSize - The size and alignment of a stack slot.
312
///   Each argument will be allocated to a multiple of this number of
313
///   slots, and all the slots will be aligned to this value.
314
/// \param AllowHigherAlign - The slot alignment is not a cap;
315
///   an argument type with an alignment greater than the slot size
316
///   will be emitted on a higher-alignment address, potentially
317
///   leaving one or more empty slots behind as padding.  If this
318
///   is false, the returned address might be less-aligned than
319
///   DirectAlign.
320
static Address emitVoidPtrDirectVAArg(CodeGenFunction &CGF,
321
                                      Address VAListAddr,
322
                                      llvm::Type *DirectTy,
323
                                      CharUnits DirectSize,
324
                                      CharUnits DirectAlign,
325
                                      CharUnits SlotSize,
326
327
                                      bool AllowHigherAlign) {
327
  // Cast the element type to i8* if necessary.  Some platforms define
328
  // va_list as a struct containing an i8* instead of just an i8*.
329
327
  if (VAListAddr.getElementType() != CGF.Int8PtrTy)
330
59
    VAListAddr = CGF.Builder.CreateElementBitCast(VAListAddr, CGF.Int8PtrTy);
331
332
327
  llvm::Value *Ptr = CGF.Builder.CreateLoad(VAListAddr, "argp.cur");
333
334
  // If the CC aligns values higher than the slot size, do so if needed.
335
327
  Address Addr = Address::invalid();
336
327
  if (AllowHigherAlign && 
DirectAlign > SlotSize290
) {
337
104
    Addr = Address(emitRoundPointerUpToAlignment(CGF, Ptr, DirectAlign),
338
104
                   CGF.Int8Ty, DirectAlign);
339
223
  } else {
340
223
    Addr = Address(Ptr, CGF.Int8Ty, SlotSize);
341
223
  }
342
343
  // Advance the pointer past the argument, then store that back.
344
327
  CharUnits FullDirectSize = DirectSize.alignTo(SlotSize);
345
327
  Address NextPtr =
346
327
      CGF.Builder.CreateConstInBoundsByteGEP(Addr, FullDirectSize, "argp.next");
347
327
  CGF.Builder.CreateStore(NextPtr.getPointer(), VAListAddr);
348
349
  // If the argument is smaller than a slot, and this is a big-endian
350
  // target, the argument will be right-adjusted in its slot.
351
327
  if (DirectSize < SlotSize && 
CGF.CGM.getDataLayout().isBigEndian()41
&&
352
327
      
!DirectTy->isStructTy()6
) {
353
0
    Addr = CGF.Builder.CreateConstInBoundsByteGEP(Addr, SlotSize - DirectSize);
354
0
  }
355
356
327
  Addr = CGF.Builder.CreateElementBitCast(Addr, DirectTy);
357
327
  return Addr;
358
327
}
359
360
/// Emit va_arg for a platform using the common void* representation,
361
/// where arguments are simply emitted in an array of slots on the stack.
362
///
363
/// \param IsIndirect - Values of this type are passed indirectly.
364
/// \param ValueInfo - The size and alignment of this type, generally
365
///   computed with getContext().getTypeInfoInChars(ValueTy).
366
/// \param SlotSizeAndAlign - The size and alignment of a stack slot.
367
///   Each argument will be allocated to a multiple of this number of
368
///   slots, and all the slots will be aligned to this value.
369
/// \param AllowHigherAlign - The slot alignment is not a cap;
370
///   an argument type with an alignment greater than the slot size
371
///   will be emitted on a higher-alignment address, potentially
372
///   leaving one or more empty slots behind as padding.
373
static Address emitVoidPtrVAArg(CodeGenFunction &CGF, Address VAListAddr,
374
                                QualType ValueTy, bool IsIndirect,
375
                                TypeInfoChars ValueInfo,
376
                                CharUnits SlotSizeAndAlign,
377
327
                                bool AllowHigherAlign) {
378
  // The size and alignment of the value that was passed directly.
379
327
  CharUnits DirectSize, DirectAlign;
380
327
  if (IsIndirect) {
381
44
    DirectSize = CGF.getPointerSize();
382
44
    DirectAlign = CGF.getPointerAlign();
383
283
  } else {
384
283
    DirectSize = ValueInfo.Width;
385
283
    DirectAlign = ValueInfo.Align;
386
283
  }
387
388
  // Cast the address we've calculated to the right type.
389
327
  llvm::Type *DirectTy = CGF.ConvertTypeForMem(ValueTy), *ElementTy = DirectTy;
390
327
  if (IsIndirect)
391
44
    DirectTy = DirectTy->getPointerTo(0);
392
393
327
  Address Addr =
394
327
      emitVoidPtrDirectVAArg(CGF, VAListAddr, DirectTy, DirectSize, DirectAlign,
395
327
                             SlotSizeAndAlign, AllowHigherAlign);
396
397
327
  if (IsIndirect) {
398
44
    Addr = Address(CGF.Builder.CreateLoad(Addr), ElementTy, ValueInfo.Align);
399
44
  }
400
401
327
  return Addr;
402
327
}
403
404
static Address complexTempStructure(CodeGenFunction &CGF, Address VAListAddr,
405
                                    QualType Ty, CharUnits SlotSize,
406
0
                                    CharUnits EltSize, const ComplexType *CTy) {
407
0
  Address Addr =
408
0
      emitVoidPtrDirectVAArg(CGF, VAListAddr, CGF.Int8Ty, SlotSize * 2,
409
0
                             SlotSize, SlotSize, /*AllowHigher*/ true);
410
411
0
  Address RealAddr = Addr;
412
0
  Address ImagAddr = RealAddr;
413
0
  if (CGF.CGM.getDataLayout().isBigEndian()) {
414
0
    RealAddr =
415
0
        CGF.Builder.CreateConstInBoundsByteGEP(RealAddr, SlotSize - EltSize);
416
0
    ImagAddr = CGF.Builder.CreateConstInBoundsByteGEP(ImagAddr,
417
0
                                                      2 * SlotSize - EltSize);
418
0
  } else {
419
0
    ImagAddr = CGF.Builder.CreateConstInBoundsByteGEP(RealAddr, SlotSize);
420
0
  }
421
422
0
  llvm::Type *EltTy = CGF.ConvertTypeForMem(CTy->getElementType());
423
0
  RealAddr = CGF.Builder.CreateElementBitCast(RealAddr, EltTy);
424
0
  ImagAddr = CGF.Builder.CreateElementBitCast(ImagAddr, EltTy);
425
0
  llvm::Value *Real = CGF.Builder.CreateLoad(RealAddr, ".vareal");
426
0
  llvm::Value *Imag = CGF.Builder.CreateLoad(ImagAddr, ".vaimag");
427
428
0
  Address Temp = CGF.CreateMemTemp(Ty, "vacplx");
429
0
  CGF.EmitStoreOfComplex({Real, Imag}, CGF.MakeAddrLValue(Temp, Ty),
430
0
                         /*init*/ true);
431
0
  return Temp;
432
0
}
433
434
static Address emitMergePHI(CodeGenFunction &CGF,
435
                            Address Addr1, llvm::BasicBlock *Block1,
436
                            Address Addr2, llvm::BasicBlock *Block2,
437
409
                            const llvm::Twine &Name = "") {
438
409
  assert(Addr1.getType() == Addr2.getType());
439
0
  llvm::PHINode *PHI = CGF.Builder.CreatePHI(Addr1.getType(), 2, Name);
440
409
  PHI->addIncoming(Addr1.getPointer(), Block1);
441
409
  PHI->addIncoming(Addr2.getPointer(), Block2);
442
409
  CharUnits Align = std::min(Addr1.getAlignment(), Addr2.getAlignment());
443
409
  return Address(PHI, Addr1.getElementType(), Align);
444
409
}
445
446
36.3k
TargetCodeGenInfo::~TargetCodeGenInfo() = default;
447
448
// If someone can figure out a general rule for this, that would be great.
449
// It's probably just doomed to be platform-dependent, though.
450
5
unsigned TargetCodeGenInfo::getSizeOfUnwindException() const {
451
  // Verified for:
452
  //   x86-64     FreeBSD, Linux, Darwin
453
  //   x86-32     FreeBSD, Linux, Darwin
454
  //   PowerPC    Linux, Darwin
455
  //   ARM        Darwin (*not* EABI)
456
  //   AArch64    Linux
457
5
  return 32;
458
5
}
459
460
bool TargetCodeGenInfo::isNoProtoCallVariadic(const CallArgList &args,
461
103
                                     const FunctionNoProtoType *fnType) const {
462
  // The following conventions are known to require this to be false:
463
  //   x86_stdcall
464
  //   MIPS
465
  // For everything else, we just prefer false unless we opt out.
466
103
  return false;
467
103
}
468
469
void
470
TargetCodeGenInfo::getDependentLibraryOption(llvm::StringRef Lib,
471
2
                                             llvm::SmallString<24> &Opt) const {
472
  // This assumes the user is passing a library name like "rt" instead of a
473
  // filename like "librt.a/so", and that they don't care whether it's static or
474
  // dynamic.
475
2
  Opt = "-l";
476
2
  Opt += Lib;
477
2
}
478
479
59
unsigned TargetCodeGenInfo::getOpenCLKernelCallingConv() const {
480
  // OpenCL kernels are called via an explicit runtime API with arguments
481
  // set with clSetKernelArg(), not as normal sub-functions.
482
  // Return SPIR_KERNEL by default as the kernel calling convention to
483
  // ensure the fingerprint is fixed such way that each OpenCL argument
484
  // gets one matching argument in the produced kernel function argument
485
  // list to enable feasible implementation of clSetKernelArg() with
486
  // aggregates etc. In case we would use the default C calling conv here,
487
  // clSetKernelArg() might break depending on the target-specific
488
  // conventions; different targets might split structs passed as values
489
  // to multiple function arguments etc.
490
59
  return llvm::CallingConv::SPIR_KERNEL;
491
59
}
492
493
llvm::Constant *TargetCodeGenInfo::getNullPointer(const CodeGen::CodeGenModule &CGM,
494
35.2k
    llvm::PointerType *T, QualType QT) const {
495
35.2k
  return llvm::ConstantPointerNull::get(T);
496
35.2k
}
497
498
LangAS TargetCodeGenInfo::getGlobalVarAddressSpace(CodeGenModule &CGM,
499
71.4k
                                                   const VarDecl *D) const {
500
71.4k
  assert(!CGM.getLangOpts().OpenCL &&
501
71.4k
         !(CGM.getLangOpts().CUDA && CGM.getLangOpts().CUDAIsDevice) &&
502
71.4k
         "Address space agnostic languages only");
503
71.4k
  return D ? 
D->getType().getAddressSpace()69.4k
:
LangAS::Default1.97k
;
504
71.4k
}
505
506
llvm::Value *TargetCodeGenInfo::performAddrSpaceCast(
507
    CodeGen::CodeGenFunction &CGF, llvm::Value *Src, LangAS SrcAddr,
508
8.56k
    LangAS DestAddr, llvm::Type *DestTy, bool isNonNull) const {
509
  // Since target may map different address spaces in AST to the same address
510
  // space, an address space conversion may end up as a bitcast.
511
8.56k
  if (auto *C = dyn_cast<llvm::Constant>(Src))
512
58
    return performAddrSpaceCast(CGF.CGM, C, SrcAddr, DestAddr, DestTy);
513
  // Try to preserve the source's name to make IR more readable.
514
8.50k
  return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
515
8.50k
      Src, DestTy, Src->hasName() ? 
Src->getName() + ".ascast"8.28k
:
""223
);
516
8.56k
}
517
518
llvm::Constant *
519
TargetCodeGenInfo::performAddrSpaceCast(CodeGenModule &CGM, llvm::Constant *Src,
520
                                        LangAS SrcAddr, LangAS DestAddr,
521
447
                                        llvm::Type *DestTy) const {
522
  // Since target may map different address spaces in AST to the same address
523
  // space, an address space conversion may end up as a bitcast.
524
447
  return llvm::ConstantExpr::getPointerCast(Src, DestTy);
525
447
}
526
527
llvm::SyncScope::ID
528
TargetCodeGenInfo::getLLVMSyncScopeID(const LangOptions &LangOpts,
529
                                      SyncScope Scope,
530
                                      llvm::AtomicOrdering Ordering,
531
0
                                      llvm::LLVMContext &Ctx) const {
532
0
  return Ctx.getOrInsertSyncScopeID(""); /* default sync scope */
533
0
}
534
535
static bool isEmptyRecord(ASTContext &Context, QualType T, bool AllowArrays);
536
537
/// isEmptyField - Return true iff a the field is "empty", that is it
538
/// is an unnamed bit-field or an (array of) empty record(s).
539
static bool isEmptyField(ASTContext &Context, const FieldDecl *FD,
540
6.12k
                         bool AllowArrays) {
541
6.12k
  if (FD->isUnnamedBitfield())
542
147
    return true;
543
544
5.97k
  QualType FT = FD->getType();
545
546
  // Constant arrays of empty records count as empty, strip them off.
547
  // Constant arrays of zero length always count as empty.
548
5.97k
  bool WasArray = false;
549
5.97k
  if (AllowArrays)
550
7.79k
    
while (const ConstantArrayType *5.80k
AT = Context.getAsConstantArrayType(FT)) {
551
1.99k
      if (AT->getSize() == 0)
552
11
        return true;
553
1.98k
      FT = AT->getElementType();
554
      // The [[no_unique_address]] special case below does not apply to
555
      // arrays of C++ empty records, so we need to remember this fact.
556
1.98k
      WasArray = true;
557
1.98k
    }
558
559
5.96k
  const RecordType *RT = FT->getAs<RecordType>();
560
5.96k
  if (!RT)
561
5.45k
    return false;
562
563
  // C++ record fields are never empty, at least in the Itanium ABI.
564
  //
565
  // FIXME: We should use a predicate for whether this behavior is true in the
566
  // current ABI.
567
  //
568
  // The exception to the above rule are fields marked with the
569
  // [[no_unique_address]] attribute (since C++20).  Those do count as empty
570
  // according to the Itanium ABI.  The exception applies only to records,
571
  // not arrays of records, so we must also check whether we stripped off an
572
  // array type above.
573
504
  if (isa<CXXRecordDecl>(RT->getDecl()) &&
574
504
      
(85
WasArray85
||
!FD->hasAttr<NoUniqueAddressAttr>()68
))
575
78
    return false;
576
577
426
  return isEmptyRecord(Context, FT, AllowArrays);
578
504
}
579
580
/// isEmptyRecord - Return true iff a structure contains only empty
581
/// fields. Note that a structure with a flexible array member is not
582
/// considered empty.
583
9.94k
static bool isEmptyRecord(ASTContext &Context, QualType T, bool AllowArrays) {
584
9.94k
  const RecordType *RT = T->getAs<RecordType>();
585
9.94k
  if (!RT)
586
4.91k
    return false;
587
5.03k
  const RecordDecl *RD = RT->getDecl();
588
5.03k
  if (RD->hasFlexibleArrayMember())
589
14
    return false;
590
591
  // If this is a C++ record, check the bases first.
592
5.01k
  if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD))
593
1.02k
    for (const auto &I : CXXRD->bases())
594
115
      if (!isEmptyRecord(Context, I.getType(), true))
595
107
        return false;
596
597
4.91k
  for (const auto *I : RD->fields())
598
4.56k
    if (!isEmptyField(Context, I, AllowArrays))
599
4.24k
      return false;
600
661
  return true;
601
4.91k
}
602
603
/// isSingleElementStruct - Determine if a structure is a "single
604
/// element struct", i.e. it has exactly one non-empty field or
605
/// exactly one field which is itself a single element
606
/// struct. Structures with flexible array members are never
607
/// considered single element structs.
608
///
609
/// \return The field declaration for the single non-empty field, if
610
/// it exists.
611
67.3k
static const Type *isSingleElementStruct(QualType T, ASTContext &Context) {
612
67.3k
  const RecordType *RT = T->getAs<RecordType>();
613
67.3k
  if (!RT)
614
66.3k
    return nullptr;
615
616
1.02k
  const RecordDecl *RD = RT->getDecl();
617
1.02k
  if (RD->hasFlexibleArrayMember())
618
4
    return nullptr;
619
620
1.01k
  const Type *Found = nullptr;
621
622
  // If this is a C++ record, check the bases first.
623
1.01k
  if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
624
281
    for (const auto &I : CXXRD->bases()) {
625
      // Ignore empty records.
626
49
      if (isEmptyRecord(Context, I.getType(), true))
627
6
        continue;
628
629
      // If we already found an element then this isn't a single-element struct.
630
43
      if (Found)
631
11
        return nullptr;
632
633
      // If this is non-empty and not a single element struct, the composite
634
      // cannot be a single element struct.
635
32
      Found = isSingleElementStruct(I.getType(), Context);
636
32
      if (!Found)
637
9
        return nullptr;
638
32
    }
639
281
  }
640
641
  // Check for single element.
642
1.44k
  
for (const auto *FD : RD->fields())997
{
643
1.44k
    QualType FT = FD->getType();
644
645
    // Ignore empty fields.
646
1.44k
    if (isEmptyField(Context, FD, true))
647
43
      continue;
648
649
    // If we already found an element then this isn't a single-element
650
    // struct.
651
1.40k
    if (Found)
652
445
      return nullptr;
653
654
    // Treat single element arrays as the element.
655
976
    
while (const ConstantArrayType *958
AT = Context.getAsConstantArrayType(FT)) {
656
229
      if (AT->getSize().getZExtValue() != 1)
657
211
        break;
658
18
      FT = AT->getElementType();
659
18
    }
660
661
958
    if (!isAggregateTypeForABI(FT)) {
662
715
      Found = FT.getTypePtr();
663
715
    } else {
664
243
      Found = isSingleElementStruct(FT, Context);
665
243
      if (!Found)
666
235
        return nullptr;
667
243
    }
668
958
  }
669
670
  // We don't consider a struct a single-element struct if it has
671
  // padding beyond the element type.
672
317
  if (Found && 
Context.getTypeSize(Found) != Context.getTypeSize(T)290
)
673
3
    return nullptr;
674
675
314
  return Found;
676
317
}
677
678
namespace {
679
Address EmitVAArgInstr(CodeGenFunction &CGF, Address VAListAddr, QualType Ty,
680
10
                       const ABIArgInfo &AI) {
681
  // This default implementation defers to the llvm backend's va_arg
682
  // instruction. It can handle only passing arguments directly
683
  // (typically only handled in the backend for primitive types), or
684
  // aggregates passed indirectly by pointer (NOTE: if the "byval"
685
  // flag has ABI impact in the callee, this implementation cannot
686
  // work.)
687
688
  // Only a few cases are covered here at the moment -- those needed
689
  // by the default abi.
690
10
  llvm::Value *Val;
691
692
10
  if (AI.isIndirect()) {
693
1
    assert(!AI.getPaddingType() &&
694
1
           "Unexpected PaddingType seen in arginfo in generic VAArg emitter!");
695
0
    assert(
696
1
        !AI.getIndirectRealign() &&
697
1
        "Unexpected IndirectRealign seen in arginfo in generic VAArg emitter!");
698
699
0
    auto TyInfo = CGF.getContext().getTypeInfoInChars(Ty);
700
1
    CharUnits TyAlignForABI = TyInfo.Align;
701
702
1
    llvm::Type *ElementTy = CGF.ConvertTypeForMem(Ty);
703
1
    llvm::Type *BaseTy = llvm::PointerType::getUnqual(ElementTy);
704
1
    llvm::Value *Addr =
705
1
        CGF.Builder.CreateVAArg(VAListAddr.getPointer(), BaseTy);
706
1
    return Address(Addr, ElementTy, TyAlignForABI);
707
9
  } else {
708
9
    assert((AI.isDirect() || AI.isExtend()) &&
709
9
           "Unexpected ArgInfo Kind in generic VAArg emitter!");
710
711
0
    assert(!AI.getInReg() &&
712
9
           "Unexpected InReg seen in arginfo in generic VAArg emitter!");
713
0
    assert(!AI.getPaddingType() &&
714
9
           "Unexpected PaddingType seen in arginfo in generic VAArg emitter!");
715
0
    assert(!AI.getDirectOffset() &&
716
9
           "Unexpected DirectOffset seen in arginfo in generic VAArg emitter!");
717
0
    assert(!AI.getCoerceToType() &&
718
9
           "Unexpected CoerceToType seen in arginfo in generic VAArg emitter!");
719
720
0
    Address Temp = CGF.CreateMemTemp(Ty, "varet");
721
9
    Val = CGF.Builder.CreateVAArg(VAListAddr.getPointer(),
722
9
                                  CGF.ConvertTypeForMem(Ty));
723
9
    CGF.Builder.CreateStore(Val, Temp);
724
9
    return Temp;
725
9
  }
726
10
}
727
728
/// DefaultABIInfo - The default implementation for ABI specific
729
/// details. This implementation provides information which results in
730
/// self-consistent and sensible LLVM IR generation, but does not
731
/// conform to any particular ABI.
732
class DefaultABIInfo : public ABIInfo {
733
public:
734
960
  DefaultABIInfo(CodeGen::CodeGenTypes &CGT) : ABIInfo(CGT) {}
735
736
  ABIArgInfo classifyReturnType(QualType RetTy) const;
737
  ABIArgInfo classifyArgumentType(QualType RetTy) const;
738
739
646
  void computeInfo(CGFunctionInfo &FI) const override {
740
646
    if (!getCXXABI().classifyReturnType(FI))
741
645
      FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
742
646
    for (auto &I : FI.arguments())
743
896
      I.info = classifyArgumentType(I.type);
744
646
  }
745
746
  Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
747
3
                    QualType Ty) const override {
748
3
    return EmitVAArgInstr(CGF, VAListAddr, Ty, classifyArgumentType(Ty));
749
3
  }
750
};
751
752
class DefaultTargetCodeGenInfo : public TargetCodeGenInfo {
753
public:
754
  DefaultTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT)
755
25
      : TargetCodeGenInfo(std::make_unique<DefaultABIInfo>(CGT)) {}
756
};
757
758
3.13k
ABIArgInfo DefaultABIInfo::classifyArgumentType(QualType Ty) const {
759
3.13k
  Ty = useFirstFieldIfTransparentUnion(Ty);
760
761
3.13k
  if (isAggregateTypeForABI(Ty)) {
762
    // Records with non-trivial destructors/copy-constructors should not be
763
    // passed by value.
764
122
    if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))
765
2
      return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
766
767
120
    return getNaturalAlignIndirect(Ty);
768
122
  }
769
770
  // Treat an enum type as its underlying type.
771
3.00k
  if (const EnumType *EnumTy = Ty->getAs<EnumType>())
772
8
    Ty = EnumTy->getDecl()->getIntegerType();
773
774
3.00k
  ASTContext &Context = getContext();
775
3.00k
  if (const auto *EIT = Ty->getAs<BitIntType>())
776
48
    if (EIT->getNumBits() >
777
48
        Context.getTypeSize(Context.getTargetInfo().hasInt128Type()
778
48
                                ? 
Context.Int128Ty12
779
48
                                : 
Context.LongLongTy36
))
780
12
      return getNaturalAlignIndirect(Ty);
781
782
2.99k
  return (isPromotableIntegerTypeForABI(Ty) ? 
ABIArgInfo::getExtend(Ty)72
783
2.99k
                                            : 
ABIArgInfo::getDirect()2.92k
);
784
3.00k
}
785
786
2.50k
ABIArgInfo DefaultABIInfo::classifyReturnType(QualType RetTy) const {
787
2.50k
  if (RetTy->isVoidType())
788
1.51k
    return ABIArgInfo::getIgnore();
789
790
987
  if (isAggregateTypeForABI(RetTy))
791
61
    return getNaturalAlignIndirect(RetTy);
792
793
  // Treat an enum type as its underlying type.
794
926
  if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
795
0
    RetTy = EnumTy->getDecl()->getIntegerType();
796
797
926
  if (const auto *EIT = RetTy->getAs<BitIntType>())
798
40
    if (EIT->getNumBits() >
799
40
        getContext().getTypeSize(getContext().getTargetInfo().hasInt128Type()
800
40
                                     ? 
getContext().Int128Ty8
801
40
                                     : 
getContext().LongLongTy32
))
802
16
      return getNaturalAlignIndirect(RetTy);
803
804
910
  return (isPromotableIntegerTypeForABI(RetTy) ? 
ABIArgInfo::getExtend(RetTy)93
805
910
                                               : 
ABIArgInfo::getDirect()817
);
806
926
}
807
808
//===----------------------------------------------------------------------===//
809
// WebAssembly ABI Implementation
810
//
811
// This is a very simple ABI that relies a lot on DefaultABIInfo.
812
//===----------------------------------------------------------------------===//
813
814
class WebAssemblyABIInfo final : public SwiftABIInfo {
815
public:
816
  enum ABIKind {
817
    MVP = 0,
818
    ExperimentalMV = 1,
819
  };
820
821
private:
822
  DefaultABIInfo defaultInfo;
823
  ABIKind Kind;
824
825
public:
826
  explicit WebAssemblyABIInfo(CodeGen::CodeGenTypes &CGT, ABIKind Kind)
827
41
      : SwiftABIInfo(CGT), defaultInfo(CGT), Kind(Kind) {}
828
829
private:
830
  ABIArgInfo classifyReturnType(QualType RetTy) const;
831
  ABIArgInfo classifyArgumentType(QualType Ty) const;
832
833
  // DefaultABIInfo's classifyReturnType and classifyArgumentType are
834
  // non-virtual, but computeInfo and EmitVAArg are virtual, so we
835
  // overload them.
836
357
  void computeInfo(CGFunctionInfo &FI) const override {
837
357
    if (!getCXXABI().classifyReturnType(FI))
838
349
      FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
839
357
    for (auto &Arg : FI.arguments())
840
350
      Arg.info = classifyArgumentType(Arg.type);
841
357
  }
842
843
  Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
844
                    QualType Ty) const override;
845
846
  bool shouldPassIndirectlyForSwift(ArrayRef<llvm::Type*> scalars,
847
0
                                    bool asReturnValue) const override {
848
0
    return occupiesMoreThan(CGT, scalars, /*total*/ 4);
849
0
  }
850
851
0
  bool isSwiftErrorInRegister() const override {
852
0
    return false;
853
0
  }
854
};
855
856
class WebAssemblyTargetCodeGenInfo final : public TargetCodeGenInfo {
857
public:
858
  explicit WebAssemblyTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT,
859
                                        WebAssemblyABIInfo::ABIKind K)
860
41
      : TargetCodeGenInfo(std::make_unique<WebAssemblyABIInfo>(CGT, K)) {}
861
862
  void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
863
1.22k
                           CodeGen::CodeGenModule &CGM) const override {
864
1.22k
    TargetCodeGenInfo::setTargetAttributes(D, GV, CGM);
865
1.22k
    if (const auto *FD = dyn_cast_or_null<FunctionDecl>(D)) {
866
1.05k
      if (const auto *Attr = FD->getAttr<WebAssemblyImportModuleAttr>()) {
867
1
        llvm::Function *Fn = cast<llvm::Function>(GV);
868
1
        llvm::AttrBuilder B(GV->getContext());
869
1
        B.addAttribute("wasm-import-module", Attr->getImportModule());
870
1
        Fn->addFnAttrs(B);
871
1
      }
872
1.05k
      if (const auto *Attr = FD->getAttr<WebAssemblyImportNameAttr>()) {
873
1
        llvm::Function *Fn = cast<llvm::Function>(GV);
874
1
        llvm::AttrBuilder B(GV->getContext());
875
1
        B.addAttribute("wasm-import-name", Attr->getImportName());
876
1
        Fn->addFnAttrs(B);
877
1
      }
878
1.05k
      if (const auto *Attr = FD->getAttr<WebAssemblyExportNameAttr>()) {
879
2
        llvm::Function *Fn = cast<llvm::Function>(GV);
880
2
        llvm::AttrBuilder B(GV->getContext());
881
2
        B.addAttribute("wasm-export-name", Attr->getExportName());
882
2
        Fn->addFnAttrs(B);
883
2
      }
884
1.05k
    }
885
886
1.22k
    if (auto *FD = dyn_cast_or_null<FunctionDecl>(D)) {
887
1.05k
      llvm::Function *Fn = cast<llvm::Function>(GV);
888
1.05k
      if (!FD->doesThisDeclarationHaveABody() && 
!FD->hasPrototype()68
)
889
0
        Fn->addFnAttr("no-prototype");
890
1.05k
    }
891
1.22k
  }
892
};
893
894
/// Classify argument of given type \p Ty.
895
350
ABIArgInfo WebAssemblyABIInfo::classifyArgumentType(QualType Ty) const {
896
350
  Ty = useFirstFieldIfTransparentUnion(Ty);
897
898
350
  if (isAggregateTypeForABI(Ty)) {
899
    // Records with non-trivial destructors/copy-constructors should not be
900
    // passed by value.
901
42
    if (auto RAA = getRecordArgABI(Ty, getCXXABI()))
902
8
      return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
903
    // Ignore empty structs/unions.
904
34
    if (isEmptyRecord(getContext(), Ty, true))
905
4
      return ABIArgInfo::getIgnore();
906
    // Lower single-element structs to just pass a regular value. TODO: We
907
    // could do reasonable-size multiple-element structs too, using getExpand(),
908
    // though watch out for things like bitfields.
909
30
    if (const Type *SeltTy = isSingleElementStruct(Ty, getContext()))
910
11
      return ABIArgInfo::getDirect(CGT.ConvertType(QualType(SeltTy, 0)));
911
    // For the experimental multivalue ABI, fully expand all other aggregates
912
19
    if (Kind == ABIKind::ExperimentalMV) {
913
3
      const RecordType *RT = Ty->getAs<RecordType>();
914
3
      assert(RT);
915
0
      bool HasBitField = false;
916
5
      for (auto *Field : RT->getDecl()->fields()) {
917
5
        if (Field->isBitField()) {
918
1
          HasBitField = true;
919
1
          break;
920
1
        }
921
5
      }
922
3
      if (!HasBitField)
923
2
        return ABIArgInfo::getExpand();
924
3
    }
925
19
  }
926
927
  // Otherwise just do the default thing.
928
325
  return defaultInfo.classifyArgumentType(Ty);
929
350
}
930
931
349
ABIArgInfo WebAssemblyABIInfo::classifyReturnType(QualType RetTy) const {
932
349
  if (isAggregateTypeForABI(RetTy)) {
933
    // Records with non-trivial destructors/copy-constructors should not be
934
    // returned by value.
935
30
    if (!getRecordArgABI(RetTy, getCXXABI())) {
936
      // Ignore empty structs/unions.
937
30
      if (isEmptyRecord(getContext(), RetTy, true))
938
4
        return ABIArgInfo::getIgnore();
939
      // Lower single-element structs to just return a regular value. TODO: We
940
      // could do reasonable-size multiple-element structs too, using
941
      // ABIArgInfo::getDirect().
942
26
      if (const Type *SeltTy = isSingleElementStruct(RetTy, getContext()))
943
11
        return ABIArgInfo::getDirect(CGT.ConvertType(QualType(SeltTy, 0)));
944
      // For the experimental multivalue ABI, return all other aggregates
945
15
      if (Kind == ABIKind::ExperimentalMV)
946
3
        return ABIArgInfo::getDirect();
947
15
    }
948
30
  }
949
950
  // Otherwise just do the default thing.
951
331
  return defaultInfo.classifyReturnType(RetTy);
952
349
}
953
954
Address WebAssemblyABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
955
5
                                      QualType Ty) const {
956
5
  bool IsIndirect = isAggregateTypeForABI(Ty) &&
957
5
                    
!isEmptyRecord(getContext(), Ty, true)3
&&
958
5
                    
!isSingleElementStruct(Ty, getContext())2
;
959
5
  return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect,
960
5
                          getContext().getTypeInfoInChars(Ty),
961
5
                          CharUnits::fromQuantity(4),
962
5
                          /*AllowHigherAlign=*/true);
963
5
}
964
965
//===----------------------------------------------------------------------===//
966
// le32/PNaCl bitcode ABI Implementation
967
//
968
// This is a simplified version of the x86_32 ABI.  Arguments and return values
969
// are always passed on the stack.
970
//===----------------------------------------------------------------------===//
971
972
class PNaClABIInfo : public ABIInfo {
973
 public:
974
3
  PNaClABIInfo(CodeGen::CodeGenTypes &CGT) : ABIInfo(CGT) {}
975
976
  ABIArgInfo classifyReturnType(QualType RetTy) const;
977
  ABIArgInfo classifyArgumentType(QualType RetTy) const;
978
979
  void computeInfo(CGFunctionInfo &FI) const override;
980
  Address EmitVAArg(CodeGenFunction &CGF,
981
                    Address VAListAddr, QualType Ty) const override;
982
};
983
984
class PNaClTargetCodeGenInfo : public TargetCodeGenInfo {
985
 public:
986
   PNaClTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT)
987
3
       : TargetCodeGenInfo(std::make_unique<PNaClABIInfo>(CGT)) {}
988
};
989
990
2
void PNaClABIInfo::computeInfo(CGFunctionInfo &FI) const {
991
2
  if (!getCXXABI().classifyReturnType(FI))
992
2
    FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
993
994
2
  for (auto &I : FI.arguments())
995
0
    I.info = classifyArgumentType(I.type);
996
2
}
997
998
Address PNaClABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
999
0
                                QualType Ty) const {
1000
  // The PNaCL ABI is a bit odd, in that varargs don't use normal
1001
  // function classification. Structs get passed directly for varargs
1002
  // functions, through a rewriting transform in
1003
  // pnacl-llvm/lib/Transforms/NaCl/ExpandVarArgs.cpp, which allows
1004
  // this target to actually support a va_arg instructions with an
1005
  // aggregate type, unlike other targets.
1006
0
  return EmitVAArgInstr(CGF, VAListAddr, Ty, ABIArgInfo::getDirect());
1007
0
}
1008
1009
/// Classify argument of given type \p Ty.
1010
0
ABIArgInfo PNaClABIInfo::classifyArgumentType(QualType Ty) const {
1011
0
  if (isAggregateTypeForABI(Ty)) {
1012
0
    if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))
1013
0
      return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
1014
0
    return getNaturalAlignIndirect(Ty);
1015
0
  } else if (const EnumType *EnumTy = Ty->getAs<EnumType>()) {
1016
    // Treat an enum type as its underlying type.
1017
0
    Ty = EnumTy->getDecl()->getIntegerType();
1018
0
  } else if (Ty->isFloatingType()) {
1019
    // Floating-point types don't go inreg.
1020
0
    return ABIArgInfo::getDirect();
1021
0
  } else if (const auto *EIT = Ty->getAs<BitIntType>()) {
1022
    // Treat bit-precise integers as integers if <= 64, otherwise pass
1023
    // indirectly.
1024
0
    if (EIT->getNumBits() > 64)
1025
0
      return getNaturalAlignIndirect(Ty);
1026
0
    return ABIArgInfo::getDirect();
1027
0
  }
1028
1029
0
  return (isPromotableIntegerTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty)
1030
0
                                            : ABIArgInfo::getDirect());
1031
0
}
1032
1033
2
ABIArgInfo PNaClABIInfo::classifyReturnType(QualType RetTy) const {
1034
2
  if (RetTy->isVoidType())
1035
2
    return ABIArgInfo::getIgnore();
1036
1037
  // In the PNaCl ABI we always return records/structures on the stack.
1038
0
  if (isAggregateTypeForABI(RetTy))
1039
0
    return getNaturalAlignIndirect(RetTy);
1040
1041
  // Treat bit-precise integers as integers if <= 64, otherwise pass indirectly.
1042
0
  if (const auto *EIT = RetTy->getAs<BitIntType>()) {
1043
0
    if (EIT->getNumBits() > 64)
1044
0
      return getNaturalAlignIndirect(RetTy);
1045
0
    return ABIArgInfo::getDirect();
1046
0
  }
1047
1048
  // Treat an enum type as its underlying type.
1049
0
  if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
1050
0
    RetTy = EnumTy->getDecl()->getIntegerType();
1051
1052
0
  return (isPromotableIntegerTypeForABI(RetTy) ? ABIArgInfo::getExtend(RetTy)
1053
0
                                               : ABIArgInfo::getDirect());
1054
0
}
1055
1056
/// IsX86_MMXType - Return true if this is an MMX type.
1057
699
bool IsX86_MMXType(llvm::Type *IRType) {
1058
  // Return true if the type is an MMX type <2 x i32>, <4 x i16>, or <8 x i8>.
1059
699
  return IRType->isVectorTy() && IRType->getPrimitiveSizeInBits() == 64 &&
1060
699
    
cast<llvm::VectorType>(IRType)->getElementType()->isIntegerTy()18
&&
1061
699
    
IRType->getScalarSizeInBits() != 6416
;
1062
699
}
1063
1064
static llvm::Type* X86AdjustInlineAsmType(CodeGen::CodeGenFunction &CGF,
1065
                                          StringRef Constraint,
1066
1.57k
                                          llvm::Type* Ty) {
1067
1.57k
  bool IsMMXCons = llvm::StringSwitch<bool>(Constraint)
1068
1.57k
                     .Cases("y", "&y", "^Ym", true)
1069
1.57k
                     .Default(false);
1070
1.57k
  if (IsMMXCons && 
Ty->isVectorTy()18
) {
1071
14
    if (cast<llvm::VectorType>(Ty)->getPrimitiveSizeInBits().getFixedSize() !=
1072
14
        64) {
1073
      // Invalid MMX constraint
1074
0
      return nullptr;
1075
0
    }
1076
1077
14
    return llvm::Type::getX86_MMXTy(CGF.getLLVMContext());
1078
14
  }
1079
1080
  // No operation needed
1081
1.55k
  return Ty;
1082
1.57k
}
1083
1084
/// Returns true if this type can be passed in SSE registers with the
1085
/// X86_VectorCall calling convention. Shared between x86_32 and x86_64.
1086
777
static bool isX86VectorTypeForVectorCall(ASTContext &Context, QualType Ty) {
1087
777
  if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
1088
477
    if (BT->isFloatingPoint() && 
BT->getKind() != BuiltinType::Half282
) {
1089
282
      if (BT->getKind() == BuiltinType::LongDouble) {
1090
6
        if (&Context.getTargetInfo().getLongDoubleFormat() ==
1091
6
            &llvm::APFloat::x87DoubleExtended())
1092
2
          return false;
1093
6
      }
1094
280
      return true;
1095
282
    }
1096
477
  } else 
if (const VectorType *300
VT300
= Ty->getAs<VectorType>()) {
1097
    // vectorcall can pass XMM, YMM, and ZMM vectors. We don't pass SSE1 MMX
1098
    // registers specially.
1099
267
    unsigned VecSize = Context.getTypeSize(VT);
1100
267
    if (VecSize == 128 || 
VecSize == 2564
||
VecSize == 5124
)
1101
267
      return true;
1102
267
  }
1103
228
  return false;
1104
777
}
1105
1106
/// Returns true if this aggregate is small enough to be passed in SSE registers
1107
/// in the X86_VectorCall calling convention. Shared between x86_32 and x86_64.
1108
741
static bool isX86VectorCallAggregateSmallEnough(uint64_t NumMembers) {
1109
741
  return NumMembers <= 4;
1110
741
}
1111
1112
/// Returns a Homogeneous Vector Aggregate ABIArgInfo, used in X86.
1113
30
static ABIArgInfo getDirectX86Hva(llvm::Type* T = nullptr) {
1114
30
  auto AI = ABIArgInfo::getDirect(T);
1115
30
  AI.setInReg(true);
1116
30
  AI.setCanBeFlattened(false);
1117
30
  return AI;
1118
30
}
1119
1120
//===----------------------------------------------------------------------===//
1121
// X86-32 ABI Implementation
1122
//===----------------------------------------------------------------------===//
1123
1124
/// Similar to llvm::CCState, but for Clang.
1125
struct CCState {
1126
  CCState(CGFunctionInfo &FI)
1127
19.1k
      : IsPreassigned(FI.arg_size()), CC(FI.getCallingConvention()) {}
1128
1129
  llvm::SmallBitVector IsPreassigned;
1130
  unsigned CC = CallingConv::CC_C;
1131
  unsigned FreeRegs = 0;
1132
  unsigned FreeSSERegs = 0;
1133
};
1134
1135
/// X86_32ABIInfo - The X86-32 ABI information.
1136
class X86_32ABIInfo : public SwiftABIInfo {
1137
  enum Class {
1138
    Integer,
1139
    Float
1140
  };
1141
1142
  static const unsigned MinABIStackAlignInBytes = 4;
1143
1144
  bool IsDarwinVectorABI;
1145
  bool IsRetSmallStructInRegABI;
1146
  bool IsWin32StructABI;
1147
  bool IsSoftFloatABI;
1148
  bool IsMCUABI;
1149
  bool IsLinuxABI;
1150
  unsigned DefaultNumRegisterParameters;
1151
1152
521
  static bool isRegisterSize(unsigned Size) {
1153
521
    return (Size == 8 || 
Size == 16490
||
Size == 32471
||
Size == 64231
);
1154
521
  }
1155
1156
400
  bool isHomogeneousAggregateBaseType(QualType Ty) const override {
1157
    // FIXME: Assumes vectorcall is in use.
1158
400
    return isX86VectorTypeForVectorCall(getContext(), Ty);
1159
400
  }
1160
1161
  bool isHomogeneousAggregateSmallEnough(const Type *Ty,
1162
322
                                         uint64_t NumMembers) const override {
1163
    // FIXME: Assumes vectorcall is in use.
1164
322
    return isX86VectorCallAggregateSmallEnough(NumMembers);
1165
322
  }
1166
1167
  bool shouldReturnTypeInRegister(QualType Ty, ASTContext &Context) const;
1168
1169
  /// getIndirectResult - Give a source type \arg Ty, return a suitable result
1170
  /// such that the argument will be passed in memory.
1171
  ABIArgInfo getIndirectResult(QualType Ty, bool ByVal, CCState &State) const;
1172
1173
  ABIArgInfo getIndirectReturnResult(QualType Ty, CCState &State) const;
1174
1175
  /// Return the alignment to use for the given type on the stack.
1176
  unsigned getTypeStackAlignInBytes(QualType Ty, unsigned Align) const;
1177
1178
  Class classify(QualType Ty) const;
1179
  ABIArgInfo classifyReturnType(QualType RetTy, CCState &State) const;
1180
  ABIArgInfo classifyArgumentType(QualType RetTy, CCState &State) const;
1181
1182
  /// Updates the number of available free registers, returns
1183
  /// true if any registers were allocated.
1184
  bool updateFreeRegs(QualType Ty, CCState &State) const;
1185
1186
  bool shouldAggregateUseDirect(QualType Ty, CCState &State, bool &InReg,
1187
                                bool &NeedsPadding) const;
1188
  bool shouldPrimitiveUseInReg(QualType Ty, CCState &State) const;
1189
1190
  bool canExpandIndirectArgument(QualType Ty) const;
1191
1192
  /// Rewrite the function info so that all memory arguments use
1193
  /// inalloca.
1194
  void rewriteWithInAlloca(CGFunctionInfo &FI) const;
1195
1196
  void addFieldToArgStruct(SmallVector<llvm::Type *, 6> &FrameFields,
1197
                           CharUnits &StackOffset, ABIArgInfo &Info,
1198
                           QualType Type) const;
1199
  void runVectorCallFirstPass(CGFunctionInfo &FI, CCState &State) const;
1200
1201
public:
1202
1203
  void computeInfo(CGFunctionInfo &FI) const override;
1204
  Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
1205
                    QualType Ty) const override;
1206
1207
  X86_32ABIInfo(CodeGen::CodeGenTypes &CGT, bool DarwinVectorABI,
1208
                bool RetSmallStructInRegABI, bool Win32StructABI,
1209
                unsigned NumRegisterParameters, bool SoftFloatABI)
1210
    : SwiftABIInfo(CGT), IsDarwinVectorABI(DarwinVectorABI),
1211
      IsRetSmallStructInRegABI(RetSmallStructInRegABI),
1212
      IsWin32StructABI(Win32StructABI), IsSoftFloatABI(SoftFloatABI),
1213
      IsMCUABI(CGT.getTarget().getTriple().isOSIAMCU()),
1214
      IsLinuxABI(CGT.getTarget().getTriple().isOSLinux() ||
1215
                 CGT.getTarget().getTriple().isOSCygMing()),
1216
3.14k
      DefaultNumRegisterParameters(NumRegisterParameters) {}
1217
1218
  bool shouldPassIndirectlyForSwift(ArrayRef<llvm::Type*> scalars,
1219
0
                                    bool asReturnValue) const override {
1220
    // LLVM's x86-32 lowering currently only assigns up to three
1221
    // integer registers and three fp registers.  Oddly, it'll use up to
1222
    // four vector registers for vectors, but those can overlap with the
1223
    // scalar registers.
1224
0
    return occupiesMoreThan(CGT, scalars, /*total*/ 3);
1225
0
  }
1226
1227
0
  bool isSwiftErrorInRegister() const override {
1228
    // x86-32 lowering does not support passing swifterror in a register.
1229
0
    return false;
1230
0
  }
1231
};
1232
1233
class X86_32TargetCodeGenInfo : public TargetCodeGenInfo {
1234
public:
1235
  X86_32TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, bool DarwinVectorABI,
1236
                          bool RetSmallStructInRegABI, bool Win32StructABI,
1237
                          unsigned NumRegisterParameters, bool SoftFloatABI)
1238
      : TargetCodeGenInfo(std::make_unique<X86_32ABIInfo>(
1239
            CGT, DarwinVectorABI, RetSmallStructInRegABI, Win32StructABI,
1240
3.14k
            NumRegisterParameters, SoftFloatABI)) {}
1241
1242
  static bool isStructReturnInRegABI(
1243
      const llvm::Triple &Triple, const CodeGenOptions &Opts);
1244
1245
  void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
1246
                           CodeGen::CodeGenModule &CGM) const override;
1247
1248
0
  int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const override {
1249
    // Darwin uses different dwarf register numbers for EH.
1250
0
    if (CGM.getTarget().getTriple().isOSDarwin()) return 5;
1251
0
    return 4;
1252
0
  }
1253
1254
  bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
1255
                               llvm::Value *Address) const override;
1256
1257
  llvm::Type* adjustInlineAsmType(CodeGen::CodeGenFunction &CGF,
1258
                                  StringRef Constraint,
1259
674
                                  llvm::Type* Ty) const override {
1260
674
    return X86AdjustInlineAsmType(CGF, Constraint, Ty);
1261
674
  }
1262
1263
  void addReturnRegisterOutputs(CodeGenFunction &CGF, LValue ReturnValue,
1264
                                std::string &Constraints,
1265
                                std::vector<llvm::Type *> &ResultRegTypes,
1266
                                std::vector<llvm::Type *> &ResultTruncRegTypes,
1267
                                std::vector<LValue> &ResultRegDests,
1268
                                std::string &AsmString,
1269
                                unsigned NumOutputs) const override;
1270
1271
  llvm::Constant *
1272
37
  getUBSanFunctionSignature(CodeGen::CodeGenModule &CGM) const override {
1273
37
    unsigned Sig = (0xeb << 0) |  // jmp rel8
1274
37
                   (0x06 << 8) |  //           .+0x08
1275
37
                   ('v' << 16) |
1276
37
                   ('2' << 24);
1277
37
    return llvm::ConstantInt::get(CGM.Int32Ty, Sig);
1278
37
  }
1279
1280
7
  StringRef getARCRetainAutoreleasedReturnValueMarker() const override {
1281
7
    return "movl\t%ebp, %ebp"
1282
7
           "\t\t// marker for objc_retainAutoreleaseReturnValue";
1283
7
  }
1284
};
1285
1286
}
1287
1288
/// Rewrite input constraint references after adding some output constraints.
1289
/// In the case where there is one output and one input and we add one output,
1290
/// we need to replace all operand references greater than or equal to 1:
1291
///     mov $0, $1
1292
///     mov eax, $1
1293
/// The result will be:
1294
///     mov $0, $2
1295
///     mov eax, $2
1296
static void rewriteInputConstraintReferences(unsigned FirstIn,
1297
                                             unsigned NumNewOuts,
1298
27
                                             std::string &AsmString) {
1299
27
  std::string Buf;
1300
27
  llvm::raw_string_ostream OS(Buf);
1301
27
  size_t Pos = 0;
1302
88
  while (Pos < AsmString.size()) {
1303
61
    size_t DollarStart = AsmString.find('$', Pos);
1304
61
    if (DollarStart == std::string::npos)
1305
19
      DollarStart = AsmString.size();
1306
61
    size_t DollarEnd = AsmString.find_first_not_of('$', DollarStart);
1307
61
    if (DollarEnd == std::string::npos)
1308
19
      DollarEnd = AsmString.size();
1309
61
    OS << StringRef(&AsmString[Pos], DollarEnd - Pos);
1310
61
    Pos = DollarEnd;
1311
61
    size_t NumDollars = DollarEnd - DollarStart;
1312
61
    if (NumDollars % 2 != 0 && 
Pos < AsmString.size()19
) {
1313
      // We have an operand reference.
1314
19
      size_t DigitStart = Pos;
1315
19
      if (AsmString[DigitStart] == '{') {
1316
3
        OS << '{';
1317
3
        ++DigitStart;
1318
3
      }
1319
19
      size_t DigitEnd = AsmString.find_first_not_of("0123456789", DigitStart);
1320
19
      if (DigitEnd == std::string::npos)
1321
6
        DigitEnd = AsmString.size();
1322
19
      StringRef OperandStr(&AsmString[DigitStart], DigitEnd - DigitStart);
1323
19
      unsigned OperandIndex;
1324
19
      if (!OperandStr.getAsInteger(10, OperandIndex)) {
1325
19
        if (OperandIndex >= FirstIn)
1326
14
          OperandIndex += NumNewOuts;
1327
19
        OS << OperandIndex;
1328
19
      } else {
1329
0
        OS << OperandStr;
1330
0
      }
1331
19
      Pos = DigitEnd;
1332
19
    }
1333
61
  }
1334
27
  AsmString = std::move(OS.str());
1335
27
}
1336
1337
/// Add output constraints for EAX:EDX because they are return registers.
1338
void X86_32TargetCodeGenInfo::addReturnRegisterOutputs(
1339
    CodeGenFunction &CGF, LValue ReturnSlot, std::string &Constraints,
1340
    std::vector<llvm::Type *> &ResultRegTypes,
1341
    std::vector<llvm::Type *> &ResultTruncRegTypes,
1342
    std::vector<LValue> &ResultRegDests, std::string &AsmString,
1343
27
    unsigned NumOutputs) const {
1344
27
  uint64_t RetWidth = CGF.getContext().getTypeSize(ReturnSlot.getType());
1345
1346
  // Use the EAX constraint if the width is 32 or smaller and EAX:EDX if it is
1347
  // larger.
1348
27
  if (!Constraints.empty())
1349
4
    Constraints += ',';
1350
27
  if (RetWidth <= 32) {
1351
22
    Constraints += "={eax}";
1352
22
    ResultRegTypes.push_back(CGF.Int32Ty);
1353
22
  } else {
1354
    // Use the 'A' constraint for EAX:EDX.
1355
5
    Constraints += "=A";
1356
5
    ResultRegTypes.push_back(CGF.Int64Ty);
1357
5
  }
1358
1359
  // Truncate EAX or EAX:EDX to an integer of the appropriate size.
1360
27
  llvm::Type *CoerceTy = llvm::IntegerType::get(CGF.getLLVMContext(), RetWidth);
1361
27
  ResultTruncRegTypes.push_back(CoerceTy);
1362
1363
  // Coerce the integer by bitcasting the return slot pointer.
1364
27
  ReturnSlot.setAddress(
1365
27
      CGF.Builder.CreateElementBitCast(ReturnSlot.getAddress(CGF), CoerceTy));
1366
27
  ResultRegDests.push_back(ReturnSlot);
1367
1368
27
  rewriteInputConstraintReferences(NumOutputs, 1, AsmString);
1369
27
}
1370
1371
/// shouldReturnTypeInRegister - Determine if the given type should be
1372
/// returned in a register (for the Darwin and MCU ABI).
1373
bool X86_32ABIInfo::shouldReturnTypeInRegister(QualType Ty,
1374
535
                                               ASTContext &Context) const {
1375
535
  uint64_t Size = Context.getTypeSize(Ty);
1376
1377
  // For i386, type must be register sized.
1378
  // For the MCU ABI, it only needs to be <= 8-byte
1379
535
  if ((IsMCUABI && 
Size > 6414
) ||
(533
!IsMCUABI533
&&
!isRegisterSize(Size)521
))
1380
69
   return false;
1381
1382
466
  if (Ty->isVectorType()) {
1383
    // 64- and 128- bit vectors inside structures are not returned in
1384
    // registers.
1385
4
    if (Size == 64 || 
Size == 1281
)
1386
3
      return false;
1387
1388
1
    return true;
1389
4
  }
1390
1391
  // If this is a builtin, pointer, enum, complex type, member pointer, or
1392
  // member function pointer it is ok.
1393
462
  if (Ty->getAs<BuiltinType>() || 
Ty->hasPointerRepresentation()376
||
1394
462
      
Ty->isAnyComplexType()371
||
Ty->isEnumeralType()348
||
1395
462
      
Ty->isBlockPointerType()346
||
Ty->isMemberPointerType()346
)
1396
378
    return true;
1397
1398
  // Arrays are treated like records.
1399
84
  if (const ConstantArrayType *AT = Context.getAsConstantArrayType(Ty))
1400
6
    return shouldReturnTypeInRegister(AT->getElementType(), Context);
1401
1402
  // Otherwise, it must be a record type.
1403
78
  const RecordType *RT = Ty->getAs<RecordType>();
1404
78
  if (!RT) 
return false0
;
1405
1406
  // FIXME: Traverse bases here too.
1407
1408
  // Structure types are passed in register if all fields would be
1409
  // passed in a register.
1410
112
  
for (const auto *FD : RT->getDecl()->fields())78
{
1411
    // Empty fields are ignored.
1412
112
    if (isEmptyField(Context, FD, true))
1413
11
      continue;
1414
1415
    // Check fields recursively.
1416
101
    if (!shouldReturnTypeInRegister(FD->getType(), Context))
1417
4
      return false;
1418
101
  }
1419
74
  return true;
1420
78
}
1421
1422
209
static bool is32Or64BitBasicType(QualType Ty, ASTContext &Context) {
1423
  // Treat complex types as the element type.
1424
209
  if (const ComplexType *CTy = Ty->getAs<ComplexType>())
1425
2
    Ty = CTy->getElementType();
1426
1427
  // Check for a type which we know has a simple scalar argument-passing
1428
  // convention without any padding.  (We're specifically looking for 32
1429
  // and 64-bit integer and integer-equivalents, float, and double.)
1430
209
  if (!Ty->getAs<BuiltinType>() && 
!Ty->hasPointerRepresentation()38
&&
1431
209
      
!Ty->isEnumeralType()31
&&
!Ty->isBlockPointerType()28
)
1432
28
    return false;
1433
1434
181
  uint64_t Size = Context.getTypeSize(Ty);
1435
181
  return Size == 32 || 
Size == 6429
;
1436
209
}
1437
1438
static bool addFieldSizes(ASTContext &Context, const RecordDecl *RD,
1439
249
                          uint64_t &Size) {
1440
249
  for (const auto *FD : RD->fields()) {
1441
    // Scalar arguments on the stack get 4 byte alignment on x86. If the
1442
    // argument is smaller than 32-bits, expanding the struct will create
1443
    // alignment padding.
1444
209
    if (!is32Or64BitBasicType(FD->getType(), Context))
1445
50
      return false;
1446
1447
    // FIXME: Reject bit-fields wholesale; there are two problems, we don't know
1448
    // how to expand them yet, and the predicate for telling if a bitfield still
1449
    // counts as "basic" is more complicated than what we were doing previously.
1450
159
    if (FD->isBitField())
1451
2
      return false;
1452
1453
157
    Size += Context.getTypeSize(FD->getType());
1454
157
  }
1455
197
  return true;
1456
249
}
1457
1458
static bool addBaseAndFieldSizes(ASTContext &Context, const CXXRecordDecl *RD,
1459
147
                                 uint64_t &Size) {
1460
  // Don't do this if there are any non-empty bases.
1461
147
  for (const CXXBaseSpecifier &Base : RD->bases()) {
1462
1
    if (!addBaseAndFieldSizes(Context, Base.getType()->getAsCXXRecordDecl(),
1463
1
                              Size))
1464
0
      return false;
1465
1
  }
1466
147
  if (!addFieldSizes(Context, RD, Size))
1467
10
    return false;
1468
137
  return true;
1469
147
}
1470
1471
/// Test whether an argument type which is to be passed indirectly (on the
1472
/// stack) would have the equivalent layout if it was expanded into separate
1473
/// arguments. If so, we prefer to do the latter to avoid inhibiting
1474
/// optimizations.
1475
337
bool X86_32ABIInfo::canExpandIndirectArgument(QualType Ty) const {
1476
  // We can only expand structure types.
1477
337
  const RecordType *RT = Ty->getAs<RecordType>();
1478
337
  if (!RT)
1479
76
    return false;
1480
261
  const RecordDecl *RD = RT->getDecl();
1481
261
  uint64_t Size = 0;
1482
261
  if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
1483
159
    if (!IsWin32StructABI) {
1484
      // On non-Windows, we have to conservatively match our old bitcode
1485
      // prototypes in order to be ABI-compatible at the bitcode level.
1486
35
      if (!CXXRD->isCLike())
1487
13
        return false;
1488
124
    } else {
1489
      // Don't do this for dynamic classes.
1490
124
      if (CXXRD->isDynamicClass())
1491
0
        return false;
1492
124
    }
1493
146
    if (!addBaseAndFieldSizes(getContext(), CXXRD, Size))
1494
10
      return false;
1495
146
  } else {
1496
102
    if (!addFieldSizes(getContext(), RD, Size))
1497
42
      return false;
1498
102
  }
1499
1500
  // We can do this if there was no alignment padding.
1501
196
  return Size == getContext().getTypeSize(Ty);
1502
261
}
1503
1504
353
ABIArgInfo X86_32ABIInfo::getIndirectReturnResult(QualType RetTy, CCState &State) const {
1505
  // If the return value is indirect, then the hidden argument is consuming one
1506
  // integer register.
1507
353
  if (State.FreeRegs) {
1508
5
    --State.FreeRegs;
1509
5
    if (!IsMCUABI)
1510
3
      return getNaturalAlignIndirectInReg(RetTy);
1511
5
  }
1512
350
  return getNaturalAlignIndirect(RetTy, /*ByVal=*/false);
1513
353
}
1514
1515
ABIArgInfo X86_32ABIInfo::classifyReturnType(QualType RetTy,
1516
19.0k
                                             CCState &State) const {
1517
19.0k
  if (RetTy->isVoidType())
1518
12.5k
    return ABIArgInfo::getIgnore();
1519
1520
6.48k
  const Type *Base = nullptr;
1521
6.48k
  uint64_t NumElts = 0;
1522
6.48k
  if ((State.CC == llvm::CallingConv::X86_VectorCall ||
1523
6.48k
       
State.CC == llvm::CallingConv::X86_RegCall6.47k
) &&
1524
6.48k
      
isHomogeneousAggregate(RetTy, Base, NumElts)26
) {
1525
    // The LLVM struct type for such an aggregate should lower properly.
1526
17
    return ABIArgInfo::getDirect();
1527
17
  }
1528
1529
6.46k
  if (const VectorType *VT = RetTy->getAs<VectorType>()) {
1530
    // On Darwin, some vectors are returned in registers.
1531
477
    if (IsDarwinVectorABI) {
1532
463
      uint64_t Size = getContext().getTypeSize(RetTy);
1533
1534
      // 128-bit vectors are a special case; they are returned in
1535
      // registers and we need to make sure to pick a type the LLVM
1536
      // backend will like.
1537
463
      if (Size == 128)
1538
237
        return ABIArgInfo::getDirect(llvm::FixedVectorType::get(
1539
237
            llvm::Type::getInt64Ty(getVMContext()), 2));
1540
1541
      // Always return in register if it fits in a general purpose
1542
      // register, or if it is 64 bits and has a single element.
1543
226
      if ((Size == 8 || Size == 16 || Size == 32) ||
1544
226
          
(223
Size == 64223
&&
VT->getNumElements() == 19
))
1545
9
        return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(),
1546
9
                                                            Size));
1547
1548
217
      return getIndirectReturnResult(RetTy, State);
1549
226
    }
1550
1551
14
    return ABIArgInfo::getDirect();
1552
477
  }
1553
1554
5.99k
  if (isAggregateTypeForABI(RetTy)) {
1555
521
    if (const RecordType *RT = RetTy->getAs<RecordType>()) {
1556
      // Structures with flexible arrays are always indirect.
1557
190
      if (RT->getDecl()->hasFlexibleArrayMember())
1558
1
        return getIndirectReturnResult(RetTy, State);
1559
190
    }
1560
1561
    // If specified, structs and unions are always indirect.
1562
520
    if (!IsRetSmallStructInRegABI && 
!RetTy->isAnyComplexType()112
)
1563
59
      return getIndirectReturnResult(RetTy, State);
1564
1565
    // Ignore empty structs/unions.
1566
461
    if (isEmptyRecord(getContext(), RetTy, true))
1567
32
      return ABIArgInfo::getIgnore();
1568
1569
    // Return complex of _Float16 as <2 x half> so the backend will use xmm0.
1570
429
    if (const ComplexType *CT = RetTy->getAs<ComplexType>()) {
1571
64
      QualType ET = getContext().getCanonicalType(CT->getElementType());
1572
64
      if (ET->isFloat16Type())
1573
1
        return ABIArgInfo::getDirect(llvm::FixedVectorType::get(
1574
1
            llvm::Type::getHalfTy(getVMContext()), 2));
1575
64
    }
1576
1577
    // Small structures which are register sized are generally returned
1578
    // in a register.
1579
428
    if (shouldReturnTypeInRegister(RetTy, getContext())) {
1580
356
      uint64_t Size = getContext().getTypeSize(RetTy);
1581
1582
      // As a special-case, if the struct is a "single-element" struct, and
1583
      // the field is of type "float" or "double", return it in a
1584
      // floating-point register. (MSVC does not apply this special case.)
1585
      // We apply a similar transformation for pointer types to improve the
1586
      // quality of the generated IR.
1587
356
      if (const Type *SeltTy = isSingleElementStruct(RetTy, getContext()))
1588
47
        if ((!IsWin32StructABI && 
SeltTy->isRealFloatingType()34
)
1589
47
            || 
SeltTy->hasPointerRepresentation()31
)
1590
19
          return ABIArgInfo::getDirect(CGT.ConvertType(QualType(SeltTy, 0)));
1591
1592
      // FIXME: We should be able to narrow this integer in cases with dead
1593
      // padding.
1594
337
      return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(),Size));
1595
356
    }
1596
1597
72
    return getIndirectReturnResult(RetTy, State);
1598
428
  }
1599
1600
  // Treat an enum type as its underlying type.
1601
5.47k
  if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
1602
35
    RetTy = EnumTy->getDecl()->getIntegerType();
1603
1604
5.47k
  if (const auto *EIT = RetTy->getAs<BitIntType>())
1605
22
    if (EIT->getNumBits() > 64)
1606
4
      return getIndirectReturnResult(RetTy, State);
1607
1608
5.46k
  return (isPromotableIntegerTypeForABI(RetTy) ? 
ABIArgInfo::getExtend(RetTy)126
1609
5.46k
                                               : 
ABIArgInfo::getDirect()5.34k
);
1610
5.47k
}
1611
1612
39
static bool isSIMDVectorType(ASTContext &Context, QualType Ty) {
1613
39
  return Ty->getAs<VectorType>() && 
Context.getTypeSize(Ty) == 12812
;
1614
39
}
1615
1616
42
static bool isRecordWithSIMDVectorType(ASTContext &Context, QualType Ty) {
1617
42
  const RecordType *RT = Ty->getAs<RecordType>();
1618
42
  if (!RT)
1619
15
    return false;
1620
27
  const RecordDecl *RD = RT->getDecl();
1621
1622
  // If this is a C++ record, check the bases first.
1623
27
  if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD))
1624
8
    for (const auto &I : CXXRD->bases())
1625
0
      if (!isRecordWithSIMDVectorType(Context, I.getType()))
1626
0
        return false;
1627
1628
27
  for (const auto *i : RD->fields()) {
1629
22
    QualType FT = i->getType();
1630
1631
22
    if (isSIMDVectorType(Context, FT))
1632
7
      return true;
1633
1634
15
    if (isRecordWithSIMDVectorType(Context, FT))
1635
0
      return true;
1636
15
  }
1637
1638
20
  return false;
1639
27
}
1640
1641
unsigned X86_32ABIInfo::getTypeStackAlignInBytes(QualType Ty,
1642
342
                                                 unsigned Align) const {
1643
  // Otherwise, if the alignment is less than or equal to the minimum ABI
1644
  // alignment, just use the default; the backend will handle this.
1645
342
  if (Align <= MinABIStackAlignInBytes)
1646
270
    return 0; // Use default alignment.
1647
1648
72
  if (IsLinuxABI) {
1649
    // Exclude other System V OS (e.g Darwin, PS4 and FreeBSD) since we don't
1650
    // want to spend any effort dealing with the ramifications of ABI breaks.
1651
    //
1652
    // If the vector type is __m128/__m256/__m512, return the default alignment.
1653
30
    if (Ty->isVectorType() && 
(18
Align == 1618
||
Align == 3212
||
Align == 646
))
1654
18
      return Align;
1655
30
  }
1656
  // On non-Darwin, the stack type alignment is always 4.
1657
54
  if (!IsDarwinVectorABI) {
1658
    // Set explicit alignment, since we may need to realign the top.
1659
27
    return MinABIStackAlignInBytes;
1660
27
  }
1661
1662
  // Otherwise, if the type contains an SSE vector type, the alignment is 16.
1663
27
  if (Align >= 16 && 
(17
isSIMDVectorType(getContext(), Ty)17
||
1664
17
                      
isRecordWithSIMDVectorType(getContext(), Ty)16
))
1665
8
    return 16;
1666
1667
19
  return MinABIStackAlignInBytes;
1668
27
}
1669
1670
ABIArgInfo X86_32ABIInfo::getIndirectResult(QualType Ty, bool ByVal,
1671
464
                                            CCState &State) const {
1672
464
  if (!ByVal) {
1673
164
    if (State.FreeRegs) {
1674
22
      --State.FreeRegs; // Non-byval indirects just use one pointer.
1675
22
      if (!IsMCUABI)
1676
22
        return getNaturalAlignIndirectInReg(Ty);
1677
22
    }
1678
142
    return getNaturalAlignIndirect(Ty, false);
1679
164
  }
1680
1681
  // Compute the byval alignment.
1682
300
  unsigned TypeAlign = getContext().getTypeAlign(Ty) / 8;
1683
300
  unsigned StackAlign = getTypeStackAlignInBytes(Ty, TypeAlign);
1684
300
  if (StackAlign == 0)
1685
250
    return ABIArgInfo::getIndirect(CharUnits::fromQuantity(4), /*ByVal=*/true);
1686
1687
  // If the stack alignment is less than the type alignment, realign the
1688
  // argument.
1689
50
  bool Realign = TypeAlign > StackAlign;
1690
50
  return ABIArgInfo::getIndirect(CharUnits::fromQuantity(StackAlign),
1691
50
                                 /*ByVal=*/true, Realign);
1692
300
}
1693
1694
32.4k
X86_32ABIInfo::Class X86_32ABIInfo::classify(QualType Ty) const {
1695
32.4k
  const Type *T = isSingleElementStruct(Ty, getContext());
1696
32.4k
  if (!T)
1697
32.3k
    T = Ty.getTypePtr();
1698
1699
32.4k
  if (const BuiltinType *BT = T->getAs<BuiltinType>()) {
1700
11.2k
    BuiltinType::Kind K = BT->getKind();
1701
11.2k
    if (K == BuiltinType::Float || 
K == BuiltinType::Double10.8k
)
1702
595
      return Float;
1703
11.2k
  }
1704
31.8k
  return Integer;
1705
32.4k
}
1706
1707
32.4k
bool X86_32ABIInfo::updateFreeRegs(QualType Ty, CCState &State) const {
1708
32.4k
  if (!IsSoftFloatABI) {
1709
32.4k
    Class C = classify(Ty);
1710
32.4k
    if (C == Float)
1711
595
      return false;
1712
32.4k
  }
1713
1714
31.8k
  unsigned Size = getContext().getTypeSize(Ty);
1715
31.8k
  unsigned SizeInRegs = (Size + 31) / 32;
1716
1717
31.8k
  if (SizeInRegs == 0)
1718
2
    return false;
1719
1720
31.8k
  if (!IsMCUABI) {
1721
31.8k
    if (SizeInRegs > State.FreeRegs) {
1722
31.6k
      State.FreeRegs = 0;
1723
31.6k
      return false;
1724
31.6k
    }
1725
31.8k
  } else {
1726
    // The MCU psABI allows passing parameters in-reg even if there are
1727
    // earlier parameters that are passed on the stack. Also,
1728
    // it does not allow passing >8-byte structs in-register,
1729
    // even if there are 3 free registers available.
1730
42
    if (SizeInRegs > State.FreeRegs || 
SizeInRegs > 234
)
1731
9
      return false;
1732
42
  }
1733
1734
240
  State.FreeRegs -= SizeInRegs;
1735
240
  return true;
1736
31.8k
}
1737
1738
bool X86_32ABIInfo::shouldAggregateUseDirect(QualType Ty, CCState &State,
1739
                                             bool &InReg,
1740
404
                                             bool &NeedsPadding) const {
1741
  // On Windows, aggregates other than HFAs are never passed in registers, and
1742
  // they do not consume register slots. Homogenous floating-point aggregates
1743
  // (HFAs) have already been dealt with at this point.
1744
404
  if (IsWin32StructABI && 
isAggregateTypeForABI(Ty)175
)
1745
175
    return false;
1746
1747
229
  NeedsPadding = false;
1748
229
  InReg = !IsMCUABI;
1749
1750
229
  if (!updateFreeRegs(Ty, State))
1751
209
    return false;
1752
1753
20
  if (IsMCUABI)
1754
7
    return true;
1755
1756
13
  if (State.CC == llvm::CallingConv::X86_FastCall ||
1757
13
      
State.CC == llvm::CallingConv::X86_VectorCall10
||
1758
13
      
State.CC == llvm::CallingConv::X86_RegCall10
) {
1759
4
    if (getContext().getTypeSize(Ty) <= 32 && State.FreeRegs)
1760
3
      NeedsPadding = true;
1761
1762
4
    return false;
1763
4
  }
1764
1765
9
  return true;
1766
13
}
1767
1768
32.2k
bool X86_32ABIInfo::shouldPrimitiveUseInReg(QualType Ty, CCState &State) const {
1769
32.2k
  if (!updateFreeRegs(Ty, State))
1770
32.0k
    return false;
1771
1772
220
  if (IsMCUABI)
1773
26
    return false;
1774
1775
194
  if (State.CC == llvm::CallingConv::X86_FastCall ||
1776
194
      
State.CC == llvm::CallingConv::X86_VectorCall137
||
1777
194
      
State.CC == llvm::CallingConv::X86_RegCall94
) {
1778
143
    if (getContext().getTypeSize(Ty) > 32)
1779
13
      return false;
1780
1781
130
    return (Ty->isIntegralOrEnumerationType() || 
Ty->isPointerType()27
||
1782
130
        
Ty->isReferenceType()7
);
1783
143
  }
1784
1785
51
  return true;
1786
194
}
1787
1788
57
void X86_32ABIInfo::runVectorCallFirstPass(CGFunctionInfo &FI, CCState &State) const {
1789
  // Vectorcall x86 works subtly different than in x64, so the format is
1790
  // a bit different than the x64 version.  First, all vector types (not HVAs)
1791
  // are assigned, with the first 6 ending up in the [XYZ]MM0-5 registers.
1792
  // This differs from the x64 implementation, where the first 6 by INDEX get
1793
  // registers.
1794
  // In the second pass over the arguments, HVAs are passed in the remaining
1795
  // vector registers if possible, or indirectly by address. The address will be
1796
  // passed in ECX/EDX if available. Any other arguments are passed according to
1797
  // the usual fastcall rules.
1798
57
  MutableArrayRef<CGFunctionInfoArgInfo> Args = FI.arguments();
1799
171
  for (int I = 0, E = Args.size(); I < E; 
++I114
) {
1800
114
    const Type *Base = nullptr;
1801
114
    uint64_t NumElts = 0;
1802
114
    const QualType &Ty = Args[I].type;
1803
114
    if ((Ty->isVectorType() || 
Ty->isBuiltinType()93
) &&
1804
114
        
isHomogeneousAggregate(Ty, Base, NumElts)84
) {
1805
42
      if (State.FreeSSERegs >= NumElts) {
1806
38
        State.FreeSSERegs -= NumElts;
1807
38
        Args[I].info = ABIArgInfo::getDirectInReg();
1808
38
        State.IsPreassigned.set(I);
1809
38
      }
1810
42
    }
1811
114
  }
1812
57
}
1813
1814
ABIArgInfo X86_32ABIInfo::classifyArgumentType(QualType Ty,
1815
33.6k
                                               CCState &State) const {
1816
  // FIXME: Set alignment on indirect arguments.
1817
33.6k
  bool IsFastCall = State.CC == llvm::CallingConv::X86_FastCall;
1818
33.6k
  bool IsRegCall = State.CC == llvm::CallingConv::X86_RegCall;
1819
33.6k
  bool IsVectorCall = State.CC == llvm::CallingConv::X86_VectorCall;
1820
1821
33.6k
  Ty = useFirstFieldIfTransparentUnion(Ty);
1822
33.6k
  TypeInfo TI = getContext().getTypeInfo(Ty);
1823
1824
  // Check with the C++ ABI first.
1825
33.6k
  const RecordType *RT = Ty->getAs<RecordType>();
1826
33.6k
  if (RT) {
1827
580
    CGCXXABI::RecordArgABI RAA = getRecordArgABI(RT, getCXXABI());
1828
580
    if (RAA == CGCXXABI::RAA_Indirect) {
1829
104
      return getIndirectResult(Ty, false, State);
1830
476
    } else if (RAA == CGCXXABI::RAA_DirectInMemory) {
1831
      // The field index doesn't matter, we'll fix it up later.
1832
86
      return ABIArgInfo::getInAlloca(/*FieldIndex=*/0);
1833
86
    }
1834
580
  }
1835
1836
  // Regcall uses the concept of a homogenous vector aggregate, similar
1837
  // to other targets.
1838
33.4k
  const Type *Base = nullptr;
1839
33.4k
  uint64_t NumElts = 0;
1840
33.4k
  if ((IsRegCall || 
IsVectorCall33.3k
) &&
1841
33.4k
      
isHomogeneousAggregate(Ty, Base, NumElts)179
) {
1842
79
    if (State.FreeSSERegs >= NumElts) {
1843
57
      State.FreeSSERegs -= NumElts;
1844
1845
      // Vectorcall passes HVAs directly and does not flatten them, but regcall
1846
      // does.
1847
57
      if (IsVectorCall)
1848
12
        return getDirectX86Hva();
1849
1850
45
      if (Ty->isBuiltinType() || 
Ty->isVectorType()35
)
1851
20
        return ABIArgInfo::getDirect();
1852
25
      return ABIArgInfo::getExpand();
1853
45
    }
1854
22
    return getIndirectResult(Ty, /*ByVal=*/false, State);
1855
79
  }
1856
1857
33.4k
  if (isAggregateTypeForABI(Ty)) {
1858
    // Structures with flexible arrays are always indirect.
1859
    // FIXME: This should not be byval!
1860
429
    if (RT && 
RT->getDecl()->hasFlexibleArrayMember()342
)
1861
1
      return getIndirectResult(Ty, true, State);
1862
1863
    // Ignore empty structs/unions on non-Windows.
1864
428
    if (!IsWin32StructABI && 
isEmptyRecord(getContext(), Ty, true)253
)
1865
24
      return ABIArgInfo::getIgnore();
1866
1867
404
    llvm::LLVMContext &LLVMContext = getVMContext();
1868
404
    llvm::IntegerType *Int32 = llvm::Type::getInt32Ty(LLVMContext);
1869
404
    bool NeedsPadding = false;
1870
404
    bool InReg;
1871
404
    if (shouldAggregateUseDirect(Ty, State, InReg, NeedsPadding)) {
1872
16
      unsigned SizeInRegs = (TI.Width + 31) / 32;
1873
16
      SmallVector<llvm::Type*, 3> Elements(SizeInRegs, Int32);
1874
16
      llvm::Type *Result = llvm::StructType::get(LLVMContext, Elements);
1875
16
      if (InReg)
1876
9
        return ABIArgInfo::getDirectInReg(Result);
1877
7
      else
1878
7
        return ABIArgInfo::getDirect(Result);
1879
16
    }
1880
388
    llvm::IntegerType *PaddingType = NeedsPadding ? 
Int323
:
nullptr385
;
1881
1882
    // Pass over-aligned aggregates on Windows indirectly. This behavior was
1883
    // added in MSVC 2015.
1884
388
    if (IsWin32StructABI && 
TI.isAlignRequired()175
&&
TI.Align > 321
)
1885
1
      return getIndirectResult(Ty, /*ByVal=*/false, State);
1886
1887
    // Expand small (<= 128-bit) record types when we know that the stack layout
1888
    // of those arguments will match the struct. This is important because the
1889
    // LLVM backend isn't smart enough to remove byval, which inhibits many
1890
    // optimizations.
1891
    // Don't do this for the MCU if there are still free integer registers
1892
    // (see X86_64 ABI for full explanation).
1893
387
    if (TI.Width <= 4 * 32 && 
(340
!IsMCUABI340
||
State.FreeRegs == 05
) &&
1894
387
        
canExpandIndirectArgument(Ty)337
)
1895
88
      return ABIArgInfo::getExpandWithPadding(
1896
88
          IsFastCall || 
IsVectorCall84
||
IsRegCall83
, PaddingType);
1897
1898
299
    return getIndirectResult(Ty, true, State);
1899
387
  }
1900
1901
32.9k
  if (const VectorType *VT = Ty->getAs<VectorType>()) {
1902
    // On Windows, vectors are passed directly if registers are available, or
1903
    // indirectly if not. This avoids the need to align argument memory. Pass
1904
    // user-defined vector types larger than 512 bits indirectly for simplicity.
1905
757
    if (IsWin32StructABI) {
1906
45
      if (TI.Width <= 512 && 
State.FreeSSERegs > 040
) {
1907
28
        --State.FreeSSERegs;
1908
28
        return ABIArgInfo::getDirectInReg();
1909
28
      }
1910
17
      return getIndirectResult(Ty, /*ByVal=*/false, State);
1911
45
    }
1912
1913
    // On Darwin, some vectors are passed in memory, we handle this by passing
1914
    // it as an i8/i16/i32/i64.
1915
712
    if (IsDarwinVectorABI) {
1916
651
      if ((TI.Width == 8 || TI.Width == 16 || TI.Width == 32) ||
1917
651
          
(649
TI.Width == 64649
&&
VT->getNumElements() == 116
))
1918
13
        return ABIArgInfo::getDirect(
1919
13
            llvm::IntegerType::get(getVMContext(), TI.Width));
1920
651
    }
1921
1922
699
    if (IsX86_MMXType(CGT.ConvertType(Ty)))
1923
8
      return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(), 64));
1924
1925
691
    return ABIArgInfo::getDirect();
1926
699
  }
1927
1928
1929
32.2k
  if (const EnumType *EnumTy = Ty->getAs<EnumType>())
1930
25
    Ty = EnumTy->getDecl()->getIntegerType();
1931
1932
32.2k
  bool InReg = shouldPrimitiveUseInReg(Ty, State);
1933
1934
32.2k
  if (isPromotableIntegerTypeForABI(Ty)) {
1935
444
    if (InReg)
1936
25
      return ABIArgInfo::getExtendInReg(Ty);
1937
419
    return ABIArgInfo::getExtend(Ty);
1938
444
  }
1939
1940
31.7k
  if (const auto *EIT = Ty->getAs<BitIntType>()) {
1941
42
    if (EIT->getNumBits() <= 64) {
1942
22
      if (InReg)
1943
0
        return ABIArgInfo::getDirectInReg();
1944
22
      return ABIArgInfo::getDirect();
1945
22
    }
1946
20
    return getIndirectResult(Ty, /*ByVal=*/false, State);
1947
42
  }
1948
1949
31.7k
  if (InReg)
1950
156
    return ABIArgInfo::getDirectInReg();
1951
31.5k
  return ABIArgInfo::getDirect();
1952
31.7k
}
1953
1954
19.1k
void X86_32ABIInfo::computeInfo(CGFunctionInfo &FI) const {
1955
19.1k
  CCState State(FI);
1956
19.1k
  if (IsMCUABI)
1957
28
    State.FreeRegs = 3;
1958
19.1k
  else if (State.CC == llvm::CallingConv::X86_FastCall) {
1959
74
    State.FreeRegs = 2;
1960
74
    State.FreeSSERegs = 3;
1961
19.0k
  } else if (State.CC == llvm::CallingConv::X86_VectorCall) {
1962
57
    State.FreeRegs = 2;
1963
57
    State.FreeSSERegs = 6;
1964
18.9k
  } else if (FI.getHasRegParm())
1965
31
    State.FreeRegs = FI.getRegParm();
1966
18.9k
  else if (State.CC == llvm::CallingConv::X86_RegCall) {
1967
54
    State.FreeRegs = 5;
1968
54
    State.FreeSSERegs = 8;
1969
18.8k
  } else if (IsWin32StructABI) {
1970
    // Since MSVC 2015, the first three SSE vectors have been passed in
1971
    // registers. The rest are passed indirectly.
1972
4.02k
    State.FreeRegs = DefaultNumRegisterParameters;
1973
4.02k
    State.FreeSSERegs = 3;
1974
4.02k
  } else
1975
14.8k
    State.FreeRegs = DefaultNumRegisterParameters;
1976
1977
19.1k
  if (!::classifyReturnType(getCXXABI(), FI, *this)) {
1978
19.0k
    FI.getReturnInfo() = classifyReturnType(FI.getReturnType(), State);
1979
19.0k
  } else 
if (88
FI.getReturnInfo().isIndirect()88
) {
1980
    // The C++ ABI is not aware of register usage, so we have to check if the
1981
    // return value was sret and put it in a register ourselves if appropriate.
1982
88
    if (State.FreeRegs) {
1983
4
      --State.FreeRegs;  // The sret parameter consumes a register.
1984
4
      if (!IsMCUABI)
1985
4
        FI.getReturnInfo().setInReg(true);
1986
4
    }
1987
88
  }
1988
1989
  // The chain argument effectively gives us another free register.
1990
19.1k
  if (FI.isChainCall())
1991
4
    ++State.FreeRegs;
1992
1993
  // For vectorcall, do a first pass over the arguments, assigning FP and vector
1994
  // arguments to XMM registers as available.
1995
19.1k
  if (State.CC == llvm::CallingConv::X86_VectorCall)
1996
57
    runVectorCallFirstPass(FI, State);
1997
1998
19.1k
  bool UsedInAlloca = false;
1999
19.1k
  MutableArrayRef<CGFunctionInfoArgInfo> Args = FI.arguments();
2000
52.8k
  for (int I = 0, E = Args.size(); I < E; 
++I33.7k
) {
2001
    // Skip arguments that have already been assigned.
2002
33.7k
    if (State.IsPreassigned.test(I))
2003
38
      continue;
2004
2005
33.6k
    Args[I].info = classifyArgumentType(Args[I].type, State);
2006
33.6k
    UsedInAlloca |= (Args[I].info.getKind() == ABIArgInfo::InAlloca);
2007
33.6k
  }
2008
2009
  // If we needed to use inalloca for any argument, do a second pass and rewrite
2010
  // all the memory arguments to use inalloca.
2011
19.1k
  if (UsedInAlloca)
2012
73
    rewriteWithInAlloca(FI);
2013
19.1k
}
2014
2015
void
2016
X86_32ABIInfo::addFieldToArgStruct(SmallVector<llvm::Type *, 6> &FrameFields,
2017
                                   CharUnits &StackOffset, ABIArgInfo &Info,
2018
167
                                   QualType Type) const {
2019
  // Arguments are always 4-byte-aligned.
2020
167
  CharUnits WordSize = CharUnits::fromQuantity(4);
2021
167
  assert(StackOffset.isMultipleOf(WordSize) && "unaligned inalloca struct");
2022
2023
  // sret pointers and indirect things will require an extra pointer
2024
  // indirection, unless they are byval. Most things are byval, and will not
2025
  // require this indirection.
2026
0
  bool IsIndirect = false;
2027
167
  if (Info.isIndirect() && 
!Info.getIndirectByVal()17
)
2028
14
    IsIndirect = true;
2029
167
  Info = ABIArgInfo::getInAlloca(FrameFields.size(), IsIndirect);
2030
167
  llvm::Type *LLTy = CGT.ConvertTypeForMem(Type);
2031
167
  if (IsIndirect)
2032
14
    LLTy = LLTy->getPointerTo(0);
2033
167
  FrameFields.push_back(LLTy);
2034
167
  StackOffset += IsIndirect ? 
WordSize14
:
getContext().getTypeSizeInChars(Type)153
;
2035
2036
  // Insert padding bytes to respect alignment.
2037
167
  CharUnits FieldEnd = StackOffset;
2038
167
  StackOffset = FieldEnd.alignTo(WordSize);
2039
167
  if (StackOffset != FieldEnd) {
2040
40
    CharUnits NumBytes = StackOffset - FieldEnd;
2041
40
    llvm::Type *Ty = llvm::Type::getInt8Ty(getVMContext());
2042
40
    Ty = llvm::ArrayType::get(Ty, NumBytes.getQuantity());
2043
40
    FrameFields.push_back(Ty);
2044
40
  }
2045
167
}
2046
2047
177
static bool isArgInAlloca(const ABIArgInfo &Info) {
2048
  // Leave ignored and inreg arguments alone.
2049
177
  switch (Info.getKind()) {
2050
86
  case ABIArgInfo::InAlloca:
2051
86
    return true;
2052
0
  case ABIArgInfo::Ignore:
2053
0
  case ABIArgInfo::IndirectAliased:
2054
0
    return false;
2055
11
  case ABIArgInfo::Indirect:
2056
84
  case ABIArgInfo::Direct:
2057
90
  case ABIArgInfo::Extend:
2058
90
    return !Info.getInReg();
2059
1
  case ABIArgInfo::Expand:
2060
1
  case ABIArgInfo::CoerceAndExpand:
2061
    // These are aggregate types which are never passed in registers when
2062
    // inalloca is involved.
2063
1
    return true;
2064
177
  }
2065
0
  llvm_unreachable("invalid enum");
2066
0
}
2067
2068
73
void X86_32ABIInfo::rewriteWithInAlloca(CGFunctionInfo &FI) const {
2069
73
  assert(IsWin32StructABI && "inalloca only supported on win32");
2070
2071
  // Build a packed struct type for all of the arguments in memory.
2072
0
  SmallVector<llvm::Type *, 6> FrameFields;
2073
2074
  // The stack alignment is always 4.
2075
73
  CharUnits StackAlign = CharUnits::fromQuantity(4);
2076
2077
73
  CharUnits StackOffset;
2078
73
  CGFunctionInfo::arg_iterator I = FI.arg_begin(), E = FI.arg_end();
2079
2080
  // Put 'this' into the struct before 'sret', if necessary.
2081
73
  bool IsThisCall =
2082
73
      FI.getCallingConvention() == llvm::CallingConv::X86_ThisCall;
2083
73
  ABIArgInfo &Ret = FI.getReturnInfo();
2084
73
  if (Ret.isIndirect() && 
Ret.isSRetAfterThis()9
&&
!IsThisCall7
&&
2085
73
      
isArgInAlloca(I->info)6
) {
2086
5
    addFieldToArgStruct(FrameFields, StackOffset, I->info, I->type);
2087
5
    ++I;
2088
5
  }
2089
2090
  // Put the sret parameter into the inalloca struct if it's in memory.
2091
73
  if (Ret.isIndirect() && 
!Ret.getInReg()9
) {
2092
8
    addFieldToArgStruct(FrameFields, StackOffset, Ret, FI.getReturnType());
2093
    // On Windows, the hidden sret parameter is always returned in eax.
2094
8
    Ret.setInAllocaSRet(IsWin32StructABI);
2095
8
  }
2096
2097
  // Skip the 'this' parameter in ecx.
2098
73
  if (IsThisCall)
2099
17
    ++I;
2100
2101
  // Put arguments passed in memory into the struct.
2102
244
  for (; I != E; 
++I171
) {
2103
171
    if (isArgInAlloca(I->info))
2104
154
      addFieldToArgStruct(FrameFields, StackOffset, I->info, I->type);
2105
171
  }
2106
2107
73
  FI.setArgStruct(llvm::StructType::get(getVMContext(), FrameFields,
2108
73
                                        /*isPacked=*/true),
2109
73
                  StackAlign);
2110
73
}
2111
2112
Address X86_32ABIInfo::EmitVAArg(CodeGenFunction &CGF,
2113
42
                                 Address VAListAddr, QualType Ty) const {
2114
2115
42
  auto TypeInfo = getContext().getTypeInfoInChars(Ty);
2116
2117
  // x86-32 changes the alignment of certain arguments on the stack.
2118
  //
2119
  // Just messing with TypeInfo like this works because we never pass
2120
  // anything indirectly.
2121
42
  TypeInfo.Align = CharUnits::fromQuantity(
2122
42
                getTypeStackAlignInBytes(Ty, TypeInfo.Align.getQuantity()));
2123
2124
42
  return emitVoidPtrVAArg(CGF, VAListAddr, Ty, /*Indirect*/ false,
2125
42
                          TypeInfo, CharUnits::fromQuantity(4),
2126
42
                          /*AllowHigherAlign*/ true);
2127
42
}
2128
2129
bool X86_32TargetCodeGenInfo::isStructReturnInRegABI(
2130
3.14k
    const llvm::Triple &Triple, const CodeGenOptions &Opts) {
2131
3.14k
  assert(Triple.getArch() == llvm::Triple::x86);
2132
2133
0
  switch (Opts.getStructReturnConvention()) {
2134
3.14k
  case CodeGenOptions::SRCK_Default:
2135
3.14k
    break;
2136
3
  case CodeGenOptions::SRCK_OnStack:  // -fpcc-struct-return
2137
3
    return false;
2138
3
  case CodeGenOptions::SRCK_InRegs:  // -freg-struct-return
2139
3
    return true;
2140
3.14k
  }
2141
2142
3.14k
  if (Triple.isOSDarwin() || 
Triple.isOSIAMCU()2.93k
)
2143
207
    return true;
2144
2145
2.93k
  switch (Triple.getOS()) {
2146
1
  case llvm::Triple::DragonFly:
2147
1
  case llvm::Triple::FreeBSD:
2148
2
  case llvm::Triple::OpenBSD:
2149
450
  case llvm::Triple::Win32:
2150
450
    return true;
2151
2.48k
  default:
2152
2.48k
    return false;
2153
2.93k
  }
2154
2.93k
}
2155
2156
static void addX86InterruptAttrs(const FunctionDecl *FD, llvm::GlobalValue *GV,
2157
191k
                                 CodeGen::CodeGenModule &CGM) {
2158
191k
  if (!FD->hasAttr<AnyX86InterruptAttr>())
2159
191k
    return;
2160
2161
25
  llvm::Function *Fn = cast<llvm::Function>(GV);
2162
25
  Fn->setCallingConv(llvm::CallingConv::X86_INTR);
2163
25
  if (FD->getNumParams() == 0)
2164
0
    return;
2165
2166
25
  auto PtrTy = cast<PointerType>(FD->getParamDecl(0)->getType());
2167
25
  llvm::Type *ByValTy = CGM.getTypes().ConvertType(PtrTy->getPointeeType());
2168
25
  llvm::Attribute NewAttr = llvm::Attribute::getWithByValType(
2169
25
    Fn->getContext(), ByValTy);
2170
25
  Fn->addParamAttr(0, NewAttr);
2171
25
}
2172
2173
void X86_32TargetCodeGenInfo::setTargetAttributes(
2174
80.4k
    const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {
2175
80.4k
  if (GV->isDeclaration())
2176
46.2k
    return;
2177
34.1k
  if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) {
2178
24.9k
    if (FD->hasAttr<X86ForceAlignArgPointerAttr>()) {
2179
2
      llvm::Function *Fn = cast<llvm::Function>(GV);
2180
2
      Fn->addFnAttr("stackrealign");
2181
2
    }
2182
2183
24.9k
    addX86InterruptAttrs(FD, GV, CGM);
2184
24.9k
  }
2185
34.1k
}
2186
2187
bool X86_32TargetCodeGenInfo::initDwarfEHRegSizeTable(
2188
                                               CodeGen::CodeGenFunction &CGF,
2189
0
                                               llvm::Value *Address) const {
2190
0
  CodeGen::CGBuilderTy &Builder = CGF.Builder;
2191
2192
0
  llvm::Value *Four8 = llvm::ConstantInt::get(CGF.Int8Ty, 4);
2193
2194
  // 0-7 are the eight integer registers;  the order is different
2195
  //   on Darwin (for EH), but the range is the same.
2196
  // 8 is %eip.
2197
0
  AssignToArrayRange(Builder, Address, Four8, 0, 8);
2198
2199
0
  if (CGF.CGM.getTarget().getTriple().isOSDarwin()) {
2200
    // 12-16 are st(0..4).  Not sure why we stop at 4.
2201
    // These have size 16, which is sizeof(long double) on
2202
    // platforms with 8-byte alignment for that type.
2203
0
    llvm::Value *Sixteen8 = llvm::ConstantInt::get(CGF.Int8Ty, 16);
2204
0
    AssignToArrayRange(Builder, Address, Sixteen8, 12, 16);
2205
2206
0
  } else {
2207
    // 9 is %eflags, which doesn't get a size on Darwin for some
2208
    // reason.
2209
0
    Builder.CreateAlignedStore(
2210
0
        Four8, Builder.CreateConstInBoundsGEP1_32(CGF.Int8Ty, Address, 9),
2211
0
                               CharUnits::One());
2212
2213
    // 11-16 are st(0..5).  Not sure why we stop at 5.
2214
    // These have size 12, which is sizeof(long double) on
2215
    // platforms with 4-byte alignment for that type.
2216
0
    llvm::Value *Twelve8 = llvm::ConstantInt::get(CGF.Int8Ty, 12);
2217
0
    AssignToArrayRange(Builder, Address, Twelve8, 11, 16);
2218
0
  }
2219
2220
0
  return false;
2221
0
}
2222
2223
//===----------------------------------------------------------------------===//
2224
// X86-64 ABI Implementation
2225
//===----------------------------------------------------------------------===//
2226
2227
2228
namespace {
2229
/// The AVX ABI level for X86 targets.
2230
enum class X86AVXABILevel {
2231
  None,
2232
  AVX,
2233
  AVX512
2234
};
2235
2236
/// \p returns the size in bits of the largest (native) vector for \p AVXLevel.
2237
4.01k
static unsigned getNativeVectorSizeForAVXABI(X86AVXABILevel AVXLevel) {
2238
4.01k
  switch (AVXLevel) {
2239
3.06k
  case X86AVXABILevel::AVX512:
2240
3.06k
    return 512;
2241
699
  case X86AVXABILevel::AVX:
2242
699
    return 256;
2243
256
  case X86AVXABILevel::None:
2244
256
    return 128;
2245
4.01k
  }
2246
0
  llvm_unreachable("Unknown AVXLevel");
2247
0
}
2248
2249
/// X86_64ABIInfo - The X86_64 ABI information.
2250
class X86_64ABIInfo : public SwiftABIInfo {
2251
  enum Class {
2252
    Integer = 0,
2253
    SSE,
2254
    SSEUp,
2255
    X87,
2256
    X87Up,
2257
    ComplexX87,
2258
    NoClass,
2259
    Memory
2260
  };
2261
2262
  /// merge - Implement the X86_64 ABI merging algorithm.
2263
  ///
2264
  /// Merge an accumulating classification \arg Accum with a field
2265
  /// classification \arg Field.
2266
  ///
2267
  /// \param Accum - The accumulating classification. This should
2268
  /// always be either NoClass or the result of a previous merge
2269
  /// call. In addition, this should never be Memory (the caller
2270
  /// should just return Memory for the aggregate).
2271
  static Class merge(Class Accum, Class Field);
2272
2273
  /// postMerge - Implement the X86_64 ABI post merging algorithm.
2274
  ///
2275
  /// Post merger cleanup, reduces a malformed Hi and Lo pair to
2276
  /// final MEMORY or SSE classes when necessary.
2277
  ///
2278
  /// \param AggregateSize - The size of the current aggregate in
2279
  /// the classification process.
2280
  ///
2281
  /// \param Lo - The classification for the parts of the type
2282
  /// residing in the low word of the containing object.
2283
  ///
2284
  /// \param Hi - The classification for the parts of the type
2285
  /// residing in the higher words of the containing object.
2286
  ///
2287
  void postMerge(unsigned AggregateSize, Class &Lo, Class &Hi) const;
2288
2289
  /// classify - Determine the x86_64 register classes in which the
2290
  /// given type T should be passed.
2291
  ///
2292
  /// \param Lo - The classification for the parts of the type
2293
  /// residing in the low word of the containing object.
2294
  ///
2295
  /// \param Hi - The classification for the parts of the type
2296
  /// residing in the high word of the containing object.
2297
  ///
2298
  /// \param OffsetBase - The bit offset of this type in the
2299
  /// containing object.  Some parameters are classified different
2300
  /// depending on whether they straddle an eightbyte boundary.
2301
  ///
2302
  /// \param isNamedArg - Whether the argument in question is a "named"
2303
  /// argument, as used in AMD64-ABI 3.5.7.
2304
  ///
2305
  /// \param IsRegCall - Whether the calling conversion is regcall.
2306
  ///
2307
  /// If a word is unused its result will be NoClass; if a type should
2308
  /// be passed in Memory then at least the classification of \arg Lo
2309
  /// will be Memory.
2310
  ///
2311
  /// The \arg Lo class will be NoClass iff the argument is ignored.
2312
  ///
2313
  /// If the \arg Lo class is ComplexX87, then the \arg Hi class will
2314
  /// also be ComplexX87.
2315
  void classify(QualType T, uint64_t OffsetBase, Class &Lo, Class &Hi,
2316
                bool isNamedArg, bool IsRegCall = false) const;
2317
2318
  llvm::Type *GetByteVectorType(QualType Ty) const;
2319
  llvm::Type *GetSSETypeAtOffset(llvm::Type *IRType,
2320
                                 unsigned IROffset, QualType SourceTy,
2321
                                 unsigned SourceOffset) const;
2322
  llvm::Type *GetINTEGERTypeAtOffset(llvm::Type *IRType,
2323
                                     unsigned IROffset, QualType SourceTy,
2324
                                     unsigned SourceOffset) const;
2325
2326
  /// getIndirectResult - Give a source type \arg Ty, return a suitable result
2327
  /// such that the argument will be returned in memory.
2328
  ABIArgInfo getIndirectReturnResult(QualType Ty) const;
2329
2330
  /// getIndirectResult - Give a source type \arg Ty, return a suitable result
2331
  /// such that the argument will be passed in memory.
2332
  ///
2333
  /// \param freeIntRegs - The number of free integer registers remaining
2334
  /// available.
2335
  ABIArgInfo getIndirectResult(QualType Ty, unsigned freeIntRegs) const;
2336
2337
  ABIArgInfo classifyReturnType(QualType RetTy) const;
2338
2339
  ABIArgInfo classifyArgumentType(QualType Ty, unsigned freeIntRegs,
2340
                                  unsigned &neededInt, unsigned &neededSSE,
2341
                                  bool isNamedArg,
2342
                                  bool IsRegCall = false) const;
2343
2344
  ABIArgInfo classifyRegCallStructType(QualType Ty, unsigned &NeededInt,
2345
                                       unsigned &NeededSSE,
2346
                                       unsigned &MaxVectorWidth) const;
2347
2348
  ABIArgInfo classifyRegCallStructTypeImpl(QualType Ty, unsigned &NeededInt,
2349
                                           unsigned &NeededSSE,
2350
                                           unsigned &MaxVectorWidth) const;
2351
2352
  bool IsIllegalVectorType(QualType Ty) const;
2353
2354
  /// The 0.98 ABI revision clarified a lot of ambiguities,
2355
  /// unfortunately in ways that were not always consistent with
2356
  /// certain previous compilers.  In particular, platforms which
2357
  /// required strict binary compatibility with older versions of GCC
2358
  /// may need to exempt themselves.
2359
17
  bool honorsRevision0_98() const {
2360
17
    return !getTarget().getTriple().isOSDarwin();
2361
17
  }
2362
2363
  /// GCC classifies <1 x long long> as SSE but some platform ABIs choose to
2364
  /// classify it as INTEGER (for compatibility with older clang compilers).
2365
192
  bool classifyIntegerMMXAsSSE() const {
2366
    // Clang <= 3.8 did not do this.
2367
192
    if (getContext().getLangOpts().getClangABICompat() <=
2368
192
        LangOptions::ClangABI::Ver3_8)
2369
3
      return false;
2370
2371
189
    const llvm::Triple &Triple = getTarget().getTriple();
2372
189
    if (Triple.isOSDarwin() || 
Triple.isPS()72
)
2373
127
      return false;
2374
62
    if (Triple.isOSFreeBSD() && 
Triple.getOSMajorVersion() >= 100
)
2375
0
      return false;
2376
62
    return true;
2377
62
  }
2378
2379
  // GCC classifies vectors of __int128 as memory.
2380
12.3k
  bool passInt128VectorsInMem() const {
2381
    // Clang <= 9.0 did not do this.
2382
12.3k
    if (getContext().getLangOpts().getClangABICompat() <=
2383
12.3k
        LangOptions::ClangABI::Ver9)
2384
36
      return false;
2385
2386
12.2k
    const llvm::Triple &T = getTarget().getTriple();
2387
12.2k
    return T.isOSLinux() || 
T.isOSNetBSD()11.2k
;
2388
12.3k
  }
2389
2390
  X86AVXABILevel AVXLevel;
2391
  // Some ABIs (e.g. X32 ABI and Native Client OS) use 32 bit pointers on
2392
  // 64-bit hardware.
2393
  bool Has64BitPointers;
2394
2395
public:
2396
  X86_64ABIInfo(CodeGen::CodeGenTypes &CGT, X86AVXABILevel AVXLevel) :
2397
      SwiftABIInfo(CGT), AVXLevel(AVXLevel),
2398
25.8k
      Has64BitPointers(CGT.getDataLayout().getPointerSize(0) == 8) {
2399
25.8k
  }
2400
2401
76
  bool isPassedUsingAVXType(QualType type) const {
2402
76
    unsigned neededInt, neededSSE;
2403
    // The freeIntRegs argument doesn't matter here.
2404
76
    ABIArgInfo info = classifyArgumentType(type, 0, neededInt, neededSSE,
2405
76
                                           /*isNamedArg*/true);
2406
76
    if (info.isDirect()) {
2407
73
      llvm::Type *ty = info.getCoerceToType();
2408
73
      if (llvm::VectorType *vectorTy = dyn_cast_or_null<llvm::VectorType>(ty))
2409
2
        return vectorTy->getPrimitiveSizeInBits().getFixedSize() > 128;
2410
73
    }
2411
74
    return false;
2412
76
  }
2413
2414
  void computeInfo(CGFunctionInfo &FI) const override;
2415
2416
  Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
2417
                    QualType Ty) const override;
2418
  Address EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr,
2419
                      QualType Ty) const override;
2420
2421
0
  bool has64BitPointers() const {
2422
0
    return Has64BitPointers;
2423
0
  }
2424
2425
  bool shouldPassIndirectlyForSwift(ArrayRef<llvm::Type*> scalars,
2426
302
                                    bool asReturnValue) const override {
2427
302
    return occupiesMoreThan(CGT, scalars, /*total*/ 4);
2428
302
  }
2429
0
  bool isSwiftErrorInRegister() const override {
2430
0
    return true;
2431
0
  }
2432
};
2433
2434
/// WinX86_64ABIInfo - The Windows X86_64 ABI information.
2435
class WinX86_64ABIInfo : public SwiftABIInfo {
2436
public:
2437
  WinX86_64ABIInfo(CodeGen::CodeGenTypes &CGT, X86AVXABILevel AVXLevel)
2438
      : SwiftABIInfo(CGT), AVXLevel(AVXLevel),
2439
423
        IsMingw64(getTarget().getTriple().isWindowsGNUEnvironment()) {}
2440
2441
  void computeInfo(CGFunctionInfo &FI) const override;
2442
2443
  Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
2444
                    QualType Ty) const override;
2445
2446
377
  bool isHomogeneousAggregateBaseType(QualType Ty) const override {
2447
    // FIXME: Assumes vectorcall is in use.
2448
377
    return isX86VectorTypeForVectorCall(getContext(), Ty);
2449
377
  }
2450
2451
  bool isHomogeneousAggregateSmallEnough(const Type *Ty,
2452
419
                                         uint64_t NumMembers) const override {
2453
    // FIXME: Assumes vectorcall is in use.
2454
419
    return isX86VectorCallAggregateSmallEnough(NumMembers);
2455
419
  }
2456
2457
  bool shouldPassIndirectlyForSwift(ArrayRef<llvm::Type *> scalars,
2458
25
                                    bool asReturnValue) const override {
2459
25
    return occupiesMoreThan(CGT, scalars, /*total*/ 4);
2460
25
  }
2461
2462
0
  bool isSwiftErrorInRegister() const override {
2463
0
    return true;
2464
0
  }
2465
2466
private:
2467
  ABIArgInfo classify(QualType Ty, unsigned &FreeSSERegs, bool IsReturnType,
2468
                      bool IsVectorCall, bool IsRegCall) const;
2469
  ABIArgInfo reclassifyHvaArgForVectorCall(QualType Ty, unsigned &FreeSSERegs,
2470
                                           const ABIArgInfo &current) const;
2471
2472
  X86AVXABILevel AVXLevel;
2473
2474
  bool IsMingw64;
2475
};
2476
2477
class X86_64TargetCodeGenInfo : public TargetCodeGenInfo {
2478
public:
2479
  X86_64TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, X86AVXABILevel AVXLevel)
2480
25.8k
      : TargetCodeGenInfo(std::make_unique<X86_64ABIInfo>(CGT, AVXLevel)) {}
2481
2482
76
  const X86_64ABIInfo &getABIInfo() const {
2483
76
    return static_cast<const X86_64ABIInfo&>(TargetCodeGenInfo::getABIInfo());
2484
76
  }
2485
2486
  /// Disable tail call on x86-64. The epilogue code before the tail jump blocks
2487
  /// autoreleaseRV/retainRV and autoreleaseRV/unsafeClaimRV optimizations.
2488
161
  bool markARCOptimizedReturnCallsAsNoTail() const override { return true; }
2489
2490
0
  int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const override {
2491
0
    return 7;
2492
0
  }
2493
2494
  bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
2495
0
                               llvm::Value *Address) const override {
2496
0
    llvm::Value *Eight8 = llvm::ConstantInt::get(CGF.Int8Ty, 8);
2497
2498
    // 0-15 are the 16 integer registers.
2499
    // 16 is %rip.
2500
0
    AssignToArrayRange(CGF.Builder, Address, Eight8, 0, 16);
2501
0
    return false;
2502
0
  }
2503
2504
  llvm::Type* adjustInlineAsmType(CodeGen::CodeGenFunction &CGF,
2505
                                  StringRef Constraint,
2506
896
                                  llvm::Type* Ty) const override {
2507
896
    return X86AdjustInlineAsmType(CGF, Constraint, Ty);
2508
896
  }
2509
2510
  bool isNoProtoCallVariadic(const CallArgList &args,
2511
429
                             const FunctionNoProtoType *fnType) const override {
2512
    // The default CC on x86-64 sets %al to the number of SSA
2513
    // registers used, and GCC sets this when calling an unprototyped
2514
    // function, so we override the default behavior.  However, don't do
2515
    // that when AVX types are involved: the ABI explicitly states it is
2516
    // undefined, and it doesn't work in practice because of how the ABI
2517
    // defines varargs anyway.
2518
429
    if (fnType->getCallConv() == CC_C) {
2519
429
      bool HasAVXType = false;
2520
429
      for (CallArgList::const_iterator
2521
503
             it = args.begin(), ie = args.end(); it != ie; 
++it74
) {
2522
76
        if (getABIInfo().isPassedUsingAVXType(it->Ty)) {
2523
2
          HasAVXType = true;
2524
2
          break;
2525
2
        }
2526
76
      }
2527
2528
429
      if (!HasAVXType)
2529
427
        return true;
2530
429
    }
2531
2532
2
    return TargetCodeGenInfo::isNoProtoCallVariadic(args, fnType);
2533
429
  }
2534
2535
  llvm::Constant *
2536
82
  getUBSanFunctionSignature(CodeGen::CodeGenModule &CGM) const override {
2537
82
    unsigned Sig = (0xeb << 0) | // jmp rel8
2538
82
                   (0x06 << 8) | //           .+0x08
2539
82
                   ('v' << 16) |
2540
82
                   ('2' << 24);
2541
82
    return llvm::ConstantInt::get(CGM.Int32Ty, Sig);
2542
82
  }
2543
2544
  void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
2545
393k
                           CodeGen::CodeGenModule &CGM) const override {
2546
393k
    if (GV->isDeclaration())
2547
222k
      return;
2548
171k
    if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) {
2549
159k
      if (FD->hasAttr<X86ForceAlignArgPointerAttr>()) {
2550
1
        llvm::Function *Fn = cast<llvm::Function>(GV);
2551
1
        Fn->addFnAttr("stackrealign");
2552
1
      }
2553
2554
159k
      addX86InterruptAttrs(FD, GV, CGM);
2555
159k
    }
2556
171k
  }
2557
2558
  void checkFunctionCallABI(CodeGenModule &CGM, SourceLocation CallLoc,
2559
                            const FunctionDecl *Caller,
2560
                            const FunctionDecl *Callee,
2561
                            const CallArgList &Args) const override;
2562
};
2563
2564
static void initFeatureMaps(const ASTContext &Ctx,
2565
                            llvm::StringMap<bool> &CallerMap,
2566
                            const FunctionDecl *Caller,
2567
                            llvm::StringMap<bool> &CalleeMap,
2568
13.4k
                            const FunctionDecl *Callee) {
2569
13.4k
  if (CalleeMap.empty() && 
CallerMap.empty()6.25k
) {
2570
    // The caller is potentially nullptr in the case where the call isn't in a
2571
    // function.  In this case, the getFunctionFeatureMap ensures we just get
2572
    // the TU level setting (since it cannot be modified by 'target'..
2573
6.25k
    Ctx.getFunctionFeatureMap(CallerMap, Caller);
2574
6.25k
    Ctx.getFunctionFeatureMap(CalleeMap, Callee);
2575
6.25k
  }
2576
13.4k
}
2577
2578
static bool checkAVXParamFeature(DiagnosticsEngine &Diag,
2579
                                 SourceLocation CallLoc,
2580
                                 const llvm::StringMap<bool> &CallerMap,
2581
                                 const llvm::StringMap<bool> &CalleeMap,
2582
                                 QualType Ty, StringRef Feature,
2583
13.4k
                                 bool IsArgument) {
2584
13.4k
  bool CallerHasFeat = CallerMap.lookup(Feature);
2585
13.4k
  bool CalleeHasFeat = CalleeMap.lookup(Feature);
2586
13.4k
  if (!CallerHasFeat && 
!CalleeHasFeat62
)
2587
56
    return Diag.Report(CallLoc, diag::warn_avx_calling_convention)
2588
56
           << IsArgument << Ty << Feature;
2589
2590
  // Mixing calling conventions here is very clearly an error.
2591
13.3k
  if (!CallerHasFeat || 
!CalleeHasFeat13.3k
)
2592
6
    return Diag.Report(CallLoc, diag::err_avx_calling_convention)
2593
6
           << IsArgument << Ty << Feature;
2594
2595
  // Else, both caller and callee have the required feature, so there is no need
2596
  // to diagnose.
2597
13.3k
  return false;
2598
13.3k
}
2599
2600
static bool checkAVXParam(DiagnosticsEngine &Diag, ASTContext &Ctx,
2601
                          SourceLocation CallLoc,
2602
                          const llvm::StringMap<bool> &CallerMap,
2603
                          const llvm::StringMap<bool> &CalleeMap, QualType Ty,
2604
13.4k
                          bool IsArgument) {
2605
13.4k
  uint64_t Size = Ctx.getTypeSize(Ty);
2606
13.4k
  if (Size > 256)
2607
6.35k
    return checkAVXParamFeature(Diag, CallLoc, CallerMap, CalleeMap, Ty,
2608
6.35k
                                "avx512f", IsArgument);
2609
2610
7.06k
  if (Size > 128)
2611
7.06k
    return checkAVXParamFeature(Diag, CallLoc, CallerMap, CalleeMap, Ty, "avx",
2612
7.06k
                                IsArgument);
2613
2614
0
  return false;
2615
7.06k
}
2616
2617
void X86_64TargetCodeGenInfo::checkFunctionCallABI(
2618
    CodeGenModule &CGM, SourceLocation CallLoc, const FunctionDecl *Caller,
2619
283k
    const FunctionDecl *Callee, const CallArgList &Args) const {
2620
283k
  llvm::StringMap<bool> CallerMap;
2621
283k
  llvm::StringMap<bool> CalleeMap;
2622
283k
  unsigned ArgIndex = 0;
2623
2624
  // We need to loop through the actual call arguments rather than the the
2625
  // function's parameters, in case this variadic.
2626
484k
  for (const CallArg &Arg : Args) {
2627
    // The "avx" feature changes how vectors >128 in size are passed. "avx512f"
2628
    // additionally changes how vectors >256 in size are passed. Like GCC, we
2629
    // warn when a function is called with an argument where this will change.
2630
    // Unlike GCC, we also error when it is an obvious ABI mismatch, that is,
2631
    // the caller and callee features are mismatched.
2632
    // Unfortunately, we cannot do this diagnostic in SEMA, since the callee can
2633
    // change its ABI with attribute-target after this call.
2634
484k
    if (Arg.getType()->isVectorType() &&
2635
484k
        
CGM.getContext().getTypeSize(Arg.getType()) > 12814.6k
) {
2636
7.75k
      initFeatureMaps(CGM.getContext(), CallerMap, Caller, CalleeMap, Callee);
2637
7.75k
      QualType Ty = Arg.getType();
2638
      // The CallArg seems to have desugared the type already, so for clearer
2639
      // diagnostics, replace it with the type in the FunctionDecl if possible.
2640
7.75k
      if (ArgIndex < Callee->getNumParams())
2641
7.72k
        Ty = Callee->getParamDecl(ArgIndex)->getType();
2642
2643
7.75k
      if (checkAVXParam(CGM.getDiags(), CGM.getContext(), CallLoc, CallerMap,
2644
7.75k
                        CalleeMap, Ty, /*IsArgument*/ true))
2645
54
        return;
2646
7.75k
    }
2647
484k
    ++ArgIndex;
2648
484k
  }
2649
2650
  // Check return always, as we don't have a good way of knowing in codegen
2651
  // whether this value is used, tail-called, etc.
2652
283k
  if (Callee->getReturnType()->isVectorType() &&
2653
283k
      
CGM.getContext().getTypeSize(Callee->getReturnType()) > 12810.0k
) {
2654
5.66k
    initFeatureMaps(CGM.getContext(), CallerMap, Caller, CalleeMap, Callee);
2655
5.66k
    checkAVXParam(CGM.getDiags(), CGM.getContext(), CallLoc, CallerMap,
2656
5.66k
                  CalleeMap, Callee->getReturnType(),
2657
5.66k
                  /*IsArgument*/ false);
2658
5.66k
  }
2659
283k
}
2660
2661
107
static std::string qualifyWindowsLibrary(llvm::StringRef Lib) {
2662
  // If the argument does not end in .lib, automatically add the suffix.
2663
  // If the argument contains a space, enclose it in quotes.
2664
  // This matches the behavior of MSVC.
2665
107
  bool Quote = Lib.contains(' ');
2666
107
  std::string ArgStr = Quote ? 
"\""4
:
""103
;
2667
107
  ArgStr += Lib;
2668
107
  if (!Lib.endswith_insensitive(".lib") && 
!Lib.endswith_insensitive(".a")95
)
2669
95
    ArgStr += ".lib";
2670
107
  ArgStr += Quote ? 
"\""4
:
""103
;
2671
107
  return ArgStr;
2672
107
}
2673
2674
class WinX86_32TargetCodeGenInfo : public X86_32TargetCodeGenInfo {
2675
public:
2676
  WinX86_32TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT,
2677
        bool DarwinVectorABI, bool RetSmallStructInRegABI, bool Win32StructABI,
2678
        unsigned NumRegisterParameters)
2679
    : X86_32TargetCodeGenInfo(CGT, DarwinVectorABI, RetSmallStructInRegABI,
2680
450
        Win32StructABI, NumRegisterParameters, false) {}
2681
2682
  void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
2683
                           CodeGen::CodeGenModule &CGM) const override;
2684
2685
  void getDependentLibraryOption(llvm::StringRef Lib,
2686
35
                                 llvm::SmallString<24> &Opt) const override {
2687
35
    Opt = "/DEFAULTLIB:";
2688
35
    Opt += qualifyWindowsLibrary(Lib);
2689
35
  }
2690
2691
  void getDetectMismatchOption(llvm::StringRef Name,
2692
                               llvm::StringRef Value,
2693
2
                               llvm::SmallString<32> &Opt) const override {
2694
2
    Opt = "/FAILIFMISMATCH:\"" + Name.str() + "=" + Value.str() + "\"";
2695
2
  }
2696
};
2697
2698
static void addStackProbeTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
2699
17.6k
                                          CodeGen::CodeGenModule &CGM) {
2700
17.6k
  if (llvm::Function *Fn = dyn_cast_or_null<llvm::Function>(GV)) {
2701
2702
15.1k
    if (CGM.getCodeGenOpts().StackProbeSize != 4096)
2703
1
      Fn->addFnAttr("stack-probe-size",
2704
1
                    llvm::utostr(CGM.getCodeGenOpts().StackProbeSize));
2705
15.1k
    if (CGM.getCodeGenOpts().NoStackArgProbe)
2706
4
      Fn->addFnAttr("no-stack-arg-probe");
2707
15.1k
  }
2708
17.6k
}
2709
2710
void WinX86_32TargetCodeGenInfo::setTargetAttributes(
2711
17.7k
    const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {
2712
17.7k
  X86_32TargetCodeGenInfo::setTargetAttributes(D, GV, CGM);
2713
17.7k
  if (GV->isDeclaration())
2714
10.1k
    return;
2715
7.65k
  addStackProbeTargetAttributes(D, GV, CGM);
2716
7.65k
}
2717
2718
class WinX86_64TargetCodeGenInfo : public TargetCodeGenInfo {
2719
public:
2720
  WinX86_64TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT,
2721
                             X86AVXABILevel AVXLevel)
2722
410
      : TargetCodeGenInfo(std::make_unique<WinX86_64ABIInfo>(CGT, AVXLevel)) {}
2723
2724
  void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
2725
                           CodeGen::CodeGenModule &CGM) const override;
2726
2727
0
  int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const override {
2728
0
    return 7;
2729
0
  }
2730
2731
  bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
2732
0
                               llvm::Value *Address) const override {
2733
0
    llvm::Value *Eight8 = llvm::ConstantInt::get(CGF.Int8Ty, 8);
2734
2735
    // 0-15 are the 16 integer registers.
2736
    // 16 is %rip.
2737
0
    AssignToArrayRange(CGF.Builder, Address, Eight8, 0, 16);
2738
0
    return false;
2739
0
  }
2740
2741
  void getDependentLibraryOption(llvm::StringRef Lib,
2742
49
                                 llvm::SmallString<24> &Opt) const override {
2743
49
    Opt = "/DEFAULTLIB:";
2744
49
    Opt += qualifyWindowsLibrary(Lib);
2745
49
  }
2746
2747
  void getDetectMismatchOption(llvm::StringRef Name,
2748
                               llvm::StringRef Value,
2749
4
                               llvm::SmallString<32> &Opt) const override {
2750
4
    Opt = "/FAILIFMISMATCH:\"" + Name.str() + "=" + Value.str() + "\"";
2751
4
  }
2752
};
2753
2754
void WinX86_64TargetCodeGenInfo::setTargetAttributes(
2755
19.8k
    const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {
2756
19.8k
  TargetCodeGenInfo::setTargetAttributes(D, GV, CGM);
2757
19.8k
  if (GV->isDeclaration())
2758
10.8k
    return;
2759
9.07k
  if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) {
2760
7.96k
    if (FD->hasAttr<X86ForceAlignArgPointerAttr>()) {
2761
0
      llvm::Function *Fn = cast<llvm::Function>(GV);
2762
0
      Fn->addFnAttr("stackrealign");
2763
0
    }
2764
2765
7.96k
    addX86InterruptAttrs(FD, GV, CGM);
2766
7.96k
  }
2767
2768
9.07k
  addStackProbeTargetAttributes(D, GV, CGM);
2769
9.07k
}
2770
}
2771
2772
void X86_64ABIInfo::postMerge(unsigned AggregateSize, Class &Lo,
2773
13.5k
                              Class &Hi) const {
2774
  // AMD64-ABI 3.2.3p2: Rule 5. Then a post merger cleanup is done:
2775
  //
2776
  // (a) If one of the classes is Memory, the whole argument is passed in
2777
  //     memory.
2778
  //
2779
  // (b) If X87UP is not preceded by X87, the whole argument is passed in
2780
  //     memory.
2781
  //
2782
  // (c) If the size of the aggregate exceeds two eightbytes and the first
2783
  //     eightbyte isn't SSE or any other eightbyte isn't SSEUP, the whole
2784
  //     argument is passed in memory. NOTE: This is necessary to keep the
2785
  //     ABI working for processors that don't support the __m256 type.
2786
  //
2787
  // (d) If SSEUP is not preceded by SSE or SSEUP, it is converted to SSE.
2788
  //
2789
  // Some of these are enforced by the merging logic.  Others can arise
2790
  // only with unions; for example:
2791
  //   union { _Complex double; unsigned; }
2792
  //
2793
  // Note that clauses (b) and (c) were added in 0.98.
2794
  //
2795
13.5k
  if (Hi == Memory)
2796
0
    Lo = Memory;
2797
13.5k
  if (Hi == X87Up && 
Lo != X87378
&&
honorsRevision0_98()17
)
2798
15
    Lo = Memory;
2799
13.5k
  if (AggregateSize > 128 && 
(361
Lo != SSE361
||
Hi != SSEUp24
))
2800
337
    Lo = Memory;
2801
13.5k
  if (Hi == SSEUp && 
Lo != SSE33
)
2802
1
    Hi = SSE;
2803
13.5k
}
2804
2805
21.3k
X86_64ABIInfo::Class X86_64ABIInfo::merge(Class Accum, Class Field) {
2806
  // AMD64-ABI 3.2.3p2: Rule 4. Each field of an object is
2807
  // classified recursively so that always two fields are
2808
  // considered. The resulting class is calculated according to
2809
  // the classes of the fields in the eightbyte:
2810
  //
2811
  // (a) If both classes are equal, this is the resulting class.
2812
  //
2813
  // (b) If one of the classes is NO_CLASS, the resulting class is
2814
  // the other class.
2815
  //
2816
  // (c) If one of the classes is MEMORY, the result is the MEMORY
2817
  // class.
2818
  //
2819
  // (d) If one of the classes is INTEGER, the result is the
2820
  // INTEGER.
2821
  //
2822
  // (e) If one of the classes is X87, X87UP, COMPLEX_X87 class,
2823
  // MEMORY is used as class.
2824
  //
2825
  // (f) Otherwise class SSE is used.
2826
2827
  // Accum should never be memory (we should have returned) or
2828
  // ComplexX87 (because this cannot be passed in a structure).
2829
21.3k
  assert((Accum != Memory && Accum != ComplexX87) &&
2830
21.3k
         "Invalid accumulated classification during merge.");
2831
21.3k
  if (Accum == Field || 
Field == NoClass11.6k
)
2832
11.8k
    return Accum;
2833
9.57k
  if (Field == Memory)
2834
17
    return Memory;
2835
9.56k
  if (Accum == NoClass)
2836
9.45k
    return Field;
2837
102
  if (Accum == Integer || 
Field == Integer73
)
2838
99
    return Integer;
2839
3
  if (Field == X87 || Field == X87Up || Field == ComplexX87 ||
2840
3
      Accum == X87 || 
Accum == X87Up0
)
2841
3
    return Memory;
2842
0
  return SSE;
2843
3
}
2844
2845
void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase, Class &Lo,
2846
468k
                             Class &Hi, bool isNamedArg, bool IsRegCall) const {
2847
  // FIXME: This code can be simplified by introducing a simple value class for
2848
  // Class pairs with appropriate constructor methods for the various
2849
  // situations.
2850
2851
  // FIXME: Some of the split computations are wrong; unaligned vectors
2852
  // shouldn't be passed in registers for example, so there is no chance they
2853
  // can straddle an eightbyte. Verify & simplify.
2854
2855
468k
  Lo = Hi = NoClass;
2856
2857
468k
  Class &Current = OffsetBase < 64 ? 
Lo466k
:
Hi2.18k
;
2858
468k
  Current = Memory;
2859
2860
468k
  if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
2861
184k
    BuiltinType::Kind k = BT->getKind();
2862
2863
184k
    if (k == BuiltinType::Void) {
2864
66.2k
      Current = NoClass;
2865
118k
    } else if (k == BuiltinType::Int128 || 
k == BuiltinType::UInt128118k
) {
2866
32
      Lo = Integer;
2867
32
      Hi = Integer;
2868
118k
    } else if (k >= BuiltinType::Bool && 
k <= BuiltinType::LongLong118k
) {
2869
113k
      Current = Integer;
2870
113k
    } else 
if (5.45k
k == BuiltinType::Float5.45k
||
k == BuiltinType::Double3.08k
||
2871
5.45k
               
k == BuiltinType::Float161.43k
) {
2872
4.21k
      Current = SSE;
2873
4.21k
    } else 
if (1.24k
k == BuiltinType::LongDouble1.24k
) {
2874
1.00k
      const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();
2875
1.00k
      if (LDF == &llvm::APFloat::IEEEquad()) {
2876
40
        Lo = SSE;
2877
40
        Hi = SSEUp;
2878
967
      } else if (LDF == &llvm::APFloat::x87DoubleExtended()) {
2879
945
        Lo = X87;
2880
945
        Hi = X87Up;
2881
945
      } else 
if (22
LDF == &llvm::APFloat::IEEEdouble()22
) {
2882
22
        Current = SSE;
2883
22
      } else
2884
0
        llvm_unreachable("unexpected long double representation!");
2885
1.00k
    }
2886
    // FIXME: _Decimal32 and _Decimal64 are SSE.
2887
    // FIXME: _float128 and _Decimal128 are (SSE, SSEUp).
2888
184k
    return;
2889
184k
  }
2890
2891
283k
  if (const EnumType *ET = Ty->getAs<EnumType>()) {
2892
    // Classify the underlying integer type.
2893
979
    classify(ET->getDecl()->getIntegerType(), OffsetBase, Lo, Hi, isNamedArg);
2894
979
    return;
2895
979
  }
2896
2897
282k
  if (Ty->hasPointerRepresentation()) {
2898
260k
    Current = Integer;
2899
260k
    return;
2900
260k
  }
2901
2902
21.7k
  if (Ty->isMemberPointerType()) {
2903
128
    if (Ty->isMemberFunctionPointerType()) {
2904
79
      if (Has64BitPointers) {
2905
        // If Has64BitPointers, this is an {i64, i64}, so classify both
2906
        // Lo and Hi now.
2907
71
        Lo = Hi = Integer;
2908
71
      } else {
2909
        // Otherwise, with 32-bit pointers, this is an {i32, i32}. If that
2910
        // straddles an eightbyte boundary, Hi should be classified as well.
2911
8
        uint64_t EB_FuncPtr = (OffsetBase) / 64;
2912
8
        uint64_t EB_ThisAdj = (OffsetBase + 64 - 1) / 64;
2913
8
        if (EB_FuncPtr != EB_ThisAdj) {
2914
2
          Lo = Hi = Integer;
2915
6
        } else {
2916
6
          Current = Integer;
2917
6
        }
2918
8
      }
2919
79
    } else {
2920
49
      Current = Integer;
2921
49
    }
2922
128
    return;
2923
128
  }
2924
2925
21.6k
  if (const VectorType *VT = Ty->getAs<VectorType>()) {
2926
6.53k
    uint64_t Size = getContext().getTypeSize(VT);
2927
6.53k
    if (Size == 1 || Size == 8 || 
Size == 166.53k
||
Size == 326.53k
) {
2928
      // gcc passes the following as integer:
2929
      // 4 bytes - <4 x char>, <2 x short>, <1 x int>, <1 x float>
2930
      // 2 bytes - <2 x char>, <1 x short>
2931
      // 1 byte  - <1 x char>
2932
5
      Current = Integer;
2933
2934
      // If this type crosses an eightbyte boundary, it should be
2935
      // split.
2936
5
      uint64_t EB_Lo = (OffsetBase) / 64;
2937
5
      uint64_t EB_Hi = (OffsetBase + Size - 1) / 64;
2938
5
      if (EB_Lo != EB_Hi)
2939
0
        Hi = Lo;
2940
6.52k
    } else if (Size == 64) {
2941
192
      QualType ElementType = VT->getElementType();
2942
2943
      // gcc passes <1 x double> in memory. :(
2944
192
      if (ElementType->isSpecificBuiltinType(BuiltinType::Double))
2945
0
        return;
2946
2947
      // gcc passes <1 x long long> as SSE but clang used to unconditionally
2948
      // pass them as integer.  For platforms where clang is the de facto
2949
      // platform compiler, we must continue to use integer.
2950
192
      if (!classifyIntegerMMXAsSSE() &&
2951
192
          
(130
ElementType->isSpecificBuiltinType(BuiltinType::LongLong)130
||
2952
130
           
ElementType->isSpecificBuiltinType(BuiltinType::ULongLong)45
||
2953
130
           
ElementType->isSpecificBuiltinType(BuiltinType::Long)45
||
2954
130
           
ElementType->isSpecificBuiltinType(BuiltinType::ULong)45
))
2955
85
        Current = Integer;
2956
107
      else
2957
107
        Current = SSE;
2958
2959
      // If this type crosses an eightbyte boundary, it should be
2960
      // split.
2961
192
      if (OffsetBase && 
OffsetBase != 640
)
2962
0
        Hi = Lo;
2963
6.33k
    } else if (Size == 128 ||
2964
6.33k
               
(3.78k
isNamedArg3.78k
&&
Size <= getNativeVectorSizeForAVXABI(AVXLevel)3.75k
)) {
2965
6.14k
      QualType ElementType = VT->getElementType();
2966
2967
      // gcc passes 256 and 512 bit <X x __int128> vectors in memory. :(
2968
6.14k
      if (passInt128VectorsInMem() && 
Size != 128519
&&
2969
6.14k
          
(256
ElementType->isSpecificBuiltinType(BuiltinType::Int128)256
||
2970
256
           ElementType->isSpecificBuiltinType(BuiltinType::UInt128)))
2971
6
        return;
2972
2973
      // Arguments of 256-bits are split into four eightbyte chunks. The
2974
      // least significant one belongs to class SSE and all the others to class
2975
      // SSEUP. The original Lo and Hi design considers that types can't be
2976
      // greater than 128-bits, so a 64-bit split in Hi and Lo makes sense.
2977
      // This design isn't correct for 256-bits, but since there're no cases
2978
      // where the upper parts would need to be inspected, avoid adding
2979
      // complexity and just consider Hi to match the 64-256 part.
2980
      //
2981
      // Note that per 3.5.7 of AMD64-ABI, 256-bit args are only passed in
2982
      // registers if they are "named", i.e. not part of the "..." of a
2983
      // variadic function.
2984
      //
2985
      // Similarly, per 3.2.3. of the AVX512 draft, 512-bits ("named") args are
2986
      // split into eight eightbyte chunks, one SSE and seven SSEUP.
2987
6.14k
      Lo = SSE;
2988
6.14k
      Hi = SSEUp;
2989
6.14k
    }
2990
6.52k
    return;
2991
6.53k
  }
2992
2993
15.0k
  if (const ComplexType *CT = Ty->getAs<ComplexType>()) {
2994
418
    QualType ET = getContext().getCanonicalType(CT->getElementType());
2995
2996
418
    uint64_t Size = getContext().getTypeSize(Ty);
2997
418
    if (ET->isIntegralOrEnumerationType()) {
2998
34
      if (Size <= 64)
2999
30
        Current = Integer;
3000
4
      else if (Size <= 128)
3001
4
        Lo = Hi = Integer;
3002
384
    } else if (ET->isFloat16Type() || 
ET == getContext().FloatTy363
) {
3003
162
      Current = SSE;
3004
222
    } else if (ET == getContext().DoubleTy) {
3005
113
      Lo = Hi = SSE;
3006
113
    } else 
if (109
ET == getContext().LongDoubleTy109
) {
3007
108
      const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();
3008
108
      if (LDF == &llvm::APFloat::IEEEquad())
3009
8
        Current = Memory;
3010
100
      else if (LDF == &llvm::APFloat::x87DoubleExtended())
3011
92
        Current = ComplexX87;
3012
8
      else if (LDF == &llvm::APFloat::IEEEdouble())
3013
8
        Lo = Hi = SSE;
3014
0
      else
3015
0
        llvm_unreachable("unexpected long double representation!");
3016
108
    }
3017
3018
    // If this complex type crosses an eightbyte boundary then it
3019
    // should be split.
3020
418
    uint64_t EB_Real = (OffsetBase) / 64;
3021
418
    uint64_t EB_Imag = (OffsetBase + getContext().getTypeSize(ET)) / 64;
3022
418
    if (Hi == NoClass && 
EB_Real != EB_Imag293
)
3023
104
      Hi = Lo;
3024
3025
418
    return;
3026
418
  }
3027
3028
14.6k
  if (const auto *EITy = Ty->getAs<BitIntType>()) {
3029
94
    if (EITy->getNumBits() <= 64)
3030
72
      Current = Integer;
3031
22
    else if (EITy->getNumBits() <= 128)
3032
22
      Lo = Hi = Integer;
3033
    // Larger values need to get passed in memory.
3034
94
    return;
3035
94
  }
3036
3037
14.5k
  if (const ConstantArrayType *AT = getContext().getAsConstantArrayType(Ty)) {
3038
    // Arrays are treated like structures.
3039
3040
59
    uint64_t Size = getContext().getTypeSize(Ty);
3041
3042
    // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger
3043
    // than eight eightbytes, ..., it has class MEMORY.
3044
    // regcall ABI doesn't have limitation to an object. The only limitation
3045
    // is the free registers, which will be checked in computeInfo.
3046
59
    if (!IsRegCall && 
Size > 51249
)
3047
0
      return;
3048
3049
    // AMD64-ABI 3.2.3p2: Rule 1. If ..., or it contains unaligned
3050
    // fields, it has class MEMORY.
3051
    //
3052
    // Only need to check alignment of array base.
3053
59
    if (OffsetBase % getContext().getTypeAlign(AT->getElementType()))
3054
0
      return;
3055
3056
    // Otherwise implement simplified merge. We could be smarter about
3057
    // this, but it isn't worth it and would be harder to verify.
3058
59
    Current = NoClass;
3059
59
    uint64_t EltSize = getContext().getTypeSize(AT->getElementType());
3060
59
    uint64_t ArraySize = AT->getSize().getZExtValue();
3061
3062
    // The only case a 256-bit wide vector could be used is when the array
3063
    // contains a single 256-bit element. Since Lo and Hi logic isn't extended
3064
    // to work for sizes wider than 128, early check and fallback to memory.
3065
    //
3066
59
    if (Size > 128 &&
3067
59
        
(19
Size != EltSize19
||
Size > getNativeVectorSizeForAVXABI(AVXLevel)4
))
3068
15
      return;
3069
3070
200
    
for (uint64_t i=0, Offset=OffsetBase; 44
i<ArraySize;
++i, Offset += EltSize156
) {
3071
156
      Class FieldLo, FieldHi;
3072
156
      classify(AT->getElementType(), Offset, FieldLo, FieldHi, isNamedArg);
3073
156
      Lo = merge(Lo, FieldLo);
3074
156
      Hi = merge(Hi, FieldHi);
3075
156
      if (Lo == Memory || Hi == Memory)
3076
0
        break;
3077
156
    }
3078
3079
44
    postMerge(Size, Lo, Hi);
3080
44
    assert((Hi != SSEUp || Lo == SSE) && "Invalid SSEUp array classification.");
3081
0
    return;
3082
59
  }
3083
3084
14.5k
  if (const RecordType *RT = Ty->getAs<RecordType>()) {
3085
14.2k
    uint64_t Size = getContext().getTypeSize(Ty);
3086
3087
    // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger
3088
    // than eight eightbytes, ..., it has class MEMORY.
3089
14.2k
    if (Size > 512)
3090
161
      return;
3091
3092
    // AMD64-ABI 3.2.3p2: Rule 2. If a C++ object has either a non-trivial
3093
    // copy constructor or a non-trivial destructor, it is passed by invisible
3094
    // reference.
3095
14.0k
    if (getRecordArgABI(RT, getCXXABI()))
3096
520
      return;
3097
3098
13.5k
    const RecordDecl *RD = RT->getDecl();
3099
3100
    // Assume variable sized types are passed in memory.
3101
13.5k
    if (RD->hasFlexibleArrayMember())
3102
0
      return;
3103
3104
13.5k
    const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
3105
3106
    // Reset Lo class, this will be recomputed.
3107
13.5k
    Current = NoClass;
3108
3109
    // If this is a C++ record, classify the bases first.
3110
13.5k
    if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
3111
12.9k
      for (const auto &I : CXXRD->bases()) {
3112
1.14k
        assert(!I.isVirtual() && !I.getType()->isDependentType() &&
3113
1.14k
               "Unexpected base class!");
3114
0
        const auto *Base =
3115
1.14k
            cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
3116
3117
        // Classify this field.
3118
        //
3119
        // AMD64-ABI 3.2.3p2: Rule 3. If the size of the aggregate exceeds a
3120
        // single eightbyte, each is classified separately. Each eightbyte gets
3121
        // initialized to class NO_CLASS.
3122
1.14k
        Class FieldLo, FieldHi;
3123
1.14k
        uint64_t Offset =
3124
1.14k
          OffsetBase + getContext().toBits(Layout.getBaseClassOffset(Base));
3125
1.14k
        classify(I.getType(), Offset, FieldLo, FieldHi, isNamedArg);
3126
1.14k
        Lo = merge(Lo, FieldLo);
3127
1.14k
        Hi = merge(Hi, FieldHi);
3128
1.14k
        if (Lo == Memory || 
Hi == Memory1.14k
) {
3129
6
          postMerge(Size, Lo, Hi);
3130
6
          return;
3131
6
        }
3132
1.14k
      }
3133
12.9k
    }
3134
3135
    // Classify the fields one at a time, merging the results.
3136
13.5k
    unsigned idx = 0;
3137
13.5k
    bool UseClang11Compat = getContext().getLangOpts().getClangABICompat() <=
3138
13.5k
                                LangOptions::ClangABI::Ver11 ||
3139
13.5k
                            
getContext().getTargetInfo().getTriple().isPS()13.4k
;
3140
13.5k
    bool IsUnion = RT->isUnionType() && 
!UseClang11Compat82
;
3141
3142
13.5k
    for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
3143
22.9k
           i != e; 
++i, ++idx9.38k
) {
3144
9.74k
      uint64_t Offset = OffsetBase + Layout.getFieldOffset(idx);
3145
9.74k
      bool BitField = i->isBitField();
3146
3147
      // Ignore padding bit-fields.
3148
9.74k
      if (BitField && 
i->isUnnamedBitfield()51
)
3149
12
        continue;
3150
3151
      // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger than
3152
      // eight eightbytes, or it contains unaligned fields, it has class MEMORY.
3153
      //
3154
      // The only case a 256-bit or a 512-bit wide vector could be used is when
3155
      // the struct contains a single 256-bit or 512-bit element. Early check
3156
      // and fallback to memory.
3157
      //
3158
      // FIXME: Extended the Lo and Hi logic properly to work for size wider
3159
      // than 128.
3160
9.73k
      if (Size > 128 &&
3161
9.73k
          
(358
(358
!IsUnion358
&&
Size != getContext().getTypeSize(i->getType())347
) ||
3162
358
           
Size > getNativeVectorSizeForAVXABI(AVXLevel)112
)) {
3163
325
        Lo = Memory;
3164
325
        postMerge(Size, Lo, Hi);
3165
325
        return;
3166
325
      }
3167
      // Note, skip this test for bit-fields, see below.
3168
9.40k
      if (!BitField && 
Offset % getContext().getTypeAlign(i->getType())9.36k
) {
3169
20
        Lo = Memory;
3170
20
        postMerge(Size, Lo, Hi);
3171
20
        return;
3172
20
      }
3173
3174
      // Classify this field.
3175
      //
3176
      // AMD64-ABI 3.2.3p2: Rule 3. If the size of the aggregate
3177
      // exceeds a single eightbyte, each is classified
3178
      // separately. Each eightbyte gets initialized to class
3179
      // NO_CLASS.
3180
9.38k
      Class FieldLo, FieldHi;
3181
3182
      // Bit-fields require special handling, they do not force the
3183
      // structure to be passed in memory even if unaligned, and
3184
      // therefore they can straddle an eightbyte.
3185
9.38k
      if (BitField) {
3186
39
        assert(!i->isUnnamedBitfield());
3187
0
        uint64_t Offset = OffsetBase + Layout.getFieldOffset(idx);
3188
39
        uint64_t Size = i->getBitWidthValue(getContext());
3189
3190
39
        uint64_t EB_Lo = Offset / 64;
3191
39
        uint64_t EB_Hi = (Offset + Size - 1) / 64;
3192
3193
39
        if (EB_Lo) {
3194
0
          assert(EB_Hi == EB_Lo && "Invalid classification, type > 16 bytes.");
3195
0
          FieldLo = NoClass;
3196
0
          FieldHi = Integer;
3197
39
        } else {
3198
39
          FieldLo = Integer;
3199
39
          FieldHi = EB_Hi ? 
Integer0
: NoClass;
3200
39
        }
3201
39
      } else
3202
9.34k
        classify(i->getType(), Offset, FieldLo, FieldHi, isNamedArg);
3203
0
      Lo = merge(Lo, FieldLo);
3204
9.38k
      Hi = merge(Hi, FieldHi);
3205
9.38k
      if (Lo == Memory || 
Hi == Memory9.37k
)
3206
14
        break;
3207
9.38k
    }
3208
3209
13.1k
    postMerge(Size, Lo, Hi);
3210
13.1k
  }
3211
14.5k
}
3212
3213
397
ABIArgInfo X86_64ABIInfo::getIndirectReturnResult(QualType Ty) const {
3214
  // If this is a scalar LLVM value then assume LLVM will pass it in the right
3215
  // place naturally.
3216
397
  if (!isAggregateTypeForABI(Ty)) {
3217
    // Treat an enum type as its underlying type.
3218
150
    if (const EnumType *EnumTy = Ty->getAs<EnumType>())
3219
0
      Ty = EnumTy->getDecl()->getIntegerType();
3220
3221
150
    if (Ty->isBitIntType())
3222
0
      return getNaturalAlignIndirect(Ty);
3223
3224
150
    return (isPromotableIntegerTypeForABI(Ty) ? 
ABIArgInfo::getExtend(Ty)0
3225
150
                                              : ABIArgInfo::getDirect());
3226
150
  }
3227
3228
247
  return getNaturalAlignIndirect(Ty);
3229
397
}
3230
3231
3.03k
bool X86_64ABIInfo::IsIllegalVectorType(QualType Ty) const {
3232
3.03k
  if (const VectorType *VecTy = Ty->getAs<VectorType>()) {
3233
143
    uint64_t Size = getContext().getTypeSize(VecTy);
3234
143
    unsigned LargestVector = getNativeVectorSizeForAVXABI(AVXLevel);
3235
143
    if (Size <= 64 || 
Size > LargestVector137
)
3236
120
      return true;
3237
23
    QualType EltTy = VecTy->getElementType();
3238
23
    if (passInt128VectorsInMem() &&
3239
23
        
(9
EltTy->isSpecificBuiltinType(BuiltinType::Int128)9
||
3240
9
         EltTy->isSpecificBuiltinType(BuiltinType::UInt128)))
3241
6
      return true;
3242
23
  }
3243
3244
2.91k
  return false;
3245
3.03k
}
3246
3247
ABIArgInfo X86_64ABIInfo::getIndirectResult(QualType Ty,
3248
3.94k
                                            unsigned freeIntRegs) const {
3249
  // If this is a scalar LLVM value then assume LLVM will pass it in the right
3250
  // place naturally.
3251
  //
3252
  // This assumption is optimistic, as there could be free registers available
3253
  // when we need to pass this argument in memory, and LLVM could try to pass
3254
  // the argument in the free register. This does not seem to happen currently,
3255
  // but this code would be much safer if we could mark the argument with
3256
  // 'onstack'. See PR12193.
3257
3.94k
  if (!isAggregateTypeForABI(Ty) && 
!IsIllegalVectorType(Ty)3.03k
&&
3258
3.94k
      
!Ty->isBitIntType()2.91k
) {
3259
    // Treat an enum type as its underlying type.
3260
2.90k
    if (const EnumType *EnumTy = Ty->getAs<EnumType>())
3261
0
      Ty = EnumTy->getDecl()->getIntegerType();
3262
3263
2.90k
    return (isPromotableIntegerTypeForABI(Ty) ? 
ABIArgInfo::getExtend(Ty)290
3264
2.90k
                                              : 
ABIArgInfo::getDirect()2.61k
);
3265
2.90k
  }
3266
3267
1.03k
  if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))
3268
533
    return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
3269
3270
  // Compute the byval alignment. We specify the alignment of the byval in all
3271
  // cases so that the mid-level optimizer knows the alignment of the byval.
3272
503
  unsigned Align = std::max(getContext().getTypeAlign(Ty) / 8, 8U);
3273
3274
  // Attempt to avoid passing indirect results using byval when possible. This
3275
  // is important for good codegen.
3276
  //
3277
  // We do this by coercing the value into a scalar type which the backend can
3278
  // handle naturally (i.e., without using byval).
3279
  //
3280
  // For simplicity, we currently only do this when we have exhausted all of the
3281
  // free integer registers. Doing this when there are free integer registers
3282
  // would require more care, as we would have to ensure that the coerced value
3283
  // did not claim the unused register. That would require either reording the
3284
  // arguments to the function (so that any subsequent inreg values came first),
3285
  // or only doing this optimization when there were no following arguments that
3286
  // might be inreg.
3287
  //
3288
  // We currently expect it to be rare (particularly in well written code) for
3289
  // arguments to be passed on the stack when there are still free integer
3290
  // registers available (this would typically imply large structs being passed
3291
  // by value), so this seems like a fair tradeoff for now.
3292
  //
3293
  // We can revisit this if the backend grows support for 'onstack' parameter
3294
  // attributes. See PR12193.
3295
503
  if (freeIntRegs == 0) {
3296
39
    uint64_t Size = getContext().getTypeSize(Ty);
3297
3298
    // If this type fits in an eightbyte, coerce it into the matching integral
3299
    // type, which will end up on the stack (with alignment 8).
3300
39
    if (Align == 8 && 
Size <= 6429
)
3301
6
      return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(),
3302
6
                                                          Size));
3303
39
  }
3304
3305
497
  return ABIArgInfo::getIndirect(CharUnits::fromQuantity(Align));
3306
503
}
3307
3308
/// The ABI specifies that a value should be passed in a full vector XMM/YMM
3309
/// register. Pick an LLVM IR type that will be passed as a vector register.
3310
6.17k
llvm::Type *X86_64ABIInfo::GetByteVectorType(QualType Ty) const {
3311
  // Wrapper structs/arrays that only contain vectors are passed just like
3312
  // vectors; strip them off if present.
3313
6.17k
  if (const Type *InnerTy = isSingleElementStruct(Ty, getContext()))
3314
25
    Ty = QualType(InnerTy, 0);
3315
3316
6.17k
  llvm::Type *IRType = CGT.ConvertType(Ty);
3317
6.17k
  if (isa<llvm::VectorType>(IRType)) {
3318
    // Don't pass vXi128 vectors in their native type, the backend can't
3319
    // legalize them.
3320
6.13k
    if (passInt128VectorsInMem() &&
3321
6.13k
        
cast<llvm::VectorType>(IRType)->getElementType()->isIntegerTy(128)510
) {
3322
      // Use a vXi64 vector.
3323
6
      uint64_t Size = getContext().getTypeSize(Ty);
3324
6
      return llvm::FixedVectorType::get(llvm::Type::getInt64Ty(getVMContext()),
3325
6
                                        Size / 64);
3326
6
    }
3327
3328
6.13k
    return IRType;
3329
6.13k
  }
3330
3331
43
  if (IRType->getTypeID() == llvm::Type::FP128TyID)
3332
40
    return IRType;
3333
3334
  // We couldn't find the preferred IR vector type for 'Ty'.
3335
3
  uint64_t Size = getContext().getTypeSize(Ty);
3336
3
  assert((Size == 128 || Size == 256 || Size == 512) && "Invalid type found!");
3337
3338
3339
  // Return a LLVM IR vector type based on the size of 'Ty'.
3340
0
  return llvm::FixedVectorType::get(llvm::Type::getDoubleTy(getVMContext()),
3341
3
                                    Size / 64);
3342
43
}
3343
3344
/// BitsContainNoUserData - Return true if the specified [start,end) bit range
3345
/// is known to either be off the end of the specified type or being in
3346
/// alignment padding.  The user type specified is known to be at most 128 bits
3347
/// in size, and have passed through X86_64ABIInfo::classify with a successful
3348
/// classification that put one of the two halves in the INTEGER class.
3349
///
3350
/// It is conservatively correct to return false.
3351
static bool BitsContainNoUserData(QualType Ty, unsigned StartBit,
3352
86.4k
                                  unsigned EndBit, ASTContext &Context) {
3353
  // If the bytes being queried are off the end of the type, there is no user
3354
  // data hiding here.  This handles analysis of builtins, vectors and other
3355
  // types that don't contain interesting padding.
3356
86.4k
  unsigned TySize = (unsigned)Context.getTypeSize(Ty);
3357
86.4k
  if (TySize <= StartBit)
3358
85.1k
    return true;
3359
3360
1.30k
  if (const ConstantArrayType *AT = Context.getAsConstantArrayType(Ty)) {
3361
20
    unsigned EltSize = (unsigned)Context.getTypeSize(AT->getElementType());
3362
20
    unsigned NumElts = (unsigned)AT->getSize().getZExtValue();
3363
3364
    // Check each element to see if the element overlaps with the queried range.
3365
72
    for (unsigned i = 0; i != NumElts; 
++i52
) {
3366
      // If the element is after the span we care about, then we're done..
3367
69
      unsigned EltOffset = i*EltSize;
3368
69
      if (EltOffset >= EndBit) 
break0
;
3369
3370
69
      unsigned EltStart = EltOffset < StartBit ? 
StartBit-EltOffset52
:
017
;
3371
69
      if (!BitsContainNoUserData(AT->getElementType(), EltStart,
3372
69
                                 EndBit-EltOffset, Context))
3373
17
        return false;
3374
69
    }
3375
    // If it overlaps no elements, then it is safe to process as padding.
3376
3
    return true;
3377
20
  }
3378
3379
1.28k
  if (const RecordType *RT = Ty->getAs<RecordType>()) {
3380
822
    const RecordDecl *RD = RT->getDecl();
3381
822
    const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
3382
3383
    // If this is a C++ record, check the bases first.
3384
822
    if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
3385
672
      for (const auto &I : CXXRD->bases()) {
3386
21
        assert(!I.isVirtual() && !I.getType()->isDependentType() &&
3387
21
               "Unexpected base class!");
3388
0
        const auto *Base =
3389
21
            cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
3390
3391
        // If the base is after the span we care about, ignore it.
3392
21
        unsigned BaseOffset = Context.toBits(Layout.getBaseClassOffset(Base));
3393
21
        if (BaseOffset >= EndBit) 
continue0
;
3394
3395
21
        unsigned BaseStart = BaseOffset < StartBit ? StartBit-BaseOffset :
00
;
3396
21
        if (!BitsContainNoUserData(I.getType(), BaseStart,
3397
21
                                   EndBit-BaseOffset, Context))
3398
20
          return false;
3399
21
      }
3400
672
    }
3401
3402
    // Verify that no field has data that overlaps the region of interest.  Yes
3403
    // this could be sped up a lot by being smarter about queried fields,
3404
    // however we're only looking at structs up to 16 bytes, so we don't care
3405
    // much.
3406
802
    unsigned idx = 0;
3407
802
    for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
3408
1.82k
         i != e; 
++i, ++idx1.02k
) {
3409
1.61k
      unsigned FieldOffset = (unsigned)Layout.getFieldOffset(idx);
3410
3411
      // If we found a field after the region we care about, then we're done.
3412
1.61k
      if (FieldOffset >= EndBit) 
break115
;
3413
3414
1.49k
      unsigned FieldStart = FieldOffset < StartBit ? 
StartBit-FieldOffset1.06k
:
0434
;
3415
1.49k
      if (!BitsContainNoUserData(i->getType(), FieldStart, EndBit-FieldOffset,
3416
1.49k
                                 Context))
3417
476
        return false;
3418
1.49k
    }
3419
3420
    // If nothing in this record overlapped the area of interest, then we're
3421
    // clean.
3422
326
    return true;
3423
802
  }
3424
3425
459
  return false;
3426
1.28k
}
3427
3428
/// getFPTypeAtOffset - Return a floating point type at the specified offset.
3429
static llvm::Type *getFPTypeAtOffset(llvm::Type *IRType, unsigned IROffset,
3430
12.3k
                                     const llvm::DataLayout &TD) {
3431
12.3k
  if (IROffset == 0 && 
IRType->isFloatingPointTy()11.6k
)
3432
4.73k
    return IRType;
3433
3434
  // If this is a struct, recurse into the field at the specified offset.
3435
7.65k
  if (llvm::StructType *STy = dyn_cast<llvm::StructType>(IRType)) {
3436
1.35k
    if (!STy->getNumContainedTypes())
3437
4
      return nullptr;
3438
3439
1.34k
    const llvm::StructLayout *SL = TD.getStructLayout(STy);
3440
1.34k
    unsigned Elt = SL->getElementContainingOffset(IROffset);
3441
1.34k
    IROffset -= SL->getElementOffset(Elt);
3442
1.34k
    return getFPTypeAtOffset(STy->getElementType(Elt), IROffset, TD);
3443
1.35k
  }
3444
3445
  // If this is an array, recurse into the field at the specified offset.
3446
6.29k
  if (llvm::ArrayType *ATy = dyn_cast<llvm::ArrayType>(IRType)) {
3447
20
    llvm::Type *EltTy = ATy->getElementType();
3448
20
    unsigned EltSize = TD.getTypeAllocSize(EltTy);
3449
20
    IROffset -= IROffset / EltSize * EltSize;
3450
20
    return getFPTypeAtOffset(EltTy, IROffset, TD);
3451
20
  }
3452
3453
6.27k
  return nullptr;
3454
6.29k
}
3455
3456
/// GetSSETypeAtOffset - Return a type that will be passed by the backend in the
3457
/// low 8 bytes of an XMM register, corresponding to the SSE class.
3458
llvm::Type *X86_64ABIInfo::
3459
GetSSETypeAtOffset(llvm::Type *IRType, unsigned IROffset,
3460
10.6k
                   QualType SourceTy, unsigned SourceOffset) const {
3461
10.6k
  const llvm::DataLayout &TD = getDataLayout();
3462
10.6k
  unsigned SourceSize =
3463
10.6k
      (unsigned)getContext().getTypeSize(SourceTy) / 8 - SourceOffset;
3464
10.6k
  llvm::Type *T0 = getFPTypeAtOffset(IRType, IROffset, TD);
3465
10.6k
  if (!T0 || 
T0->isDoubleTy()4.37k
)
3466
8.14k
    return llvm::Type::getDoubleTy(getVMContext());
3467
3468
  // Get the adjacent FP type.
3469
2.47k
  llvm::Type *T1 = nullptr;
3470
2.47k
  unsigned T0Size = TD.getTypeAllocSize(T0);
3471
2.47k
  if (SourceSize > T0Size)
3472
377
      T1 = getFPTypeAtOffset(IRType, IROffset + T0Size, TD);
3473
2.47k
  if (T1 == nullptr) {
3474
    // Check if IRType is a half + float. float type will be in IROffset+4 due
3475
    // to its alignment.
3476
2.13k
    if (T0->isHalfTy() && 
SourceSize > 4152
)
3477
6
      T1 = getFPTypeAtOffset(IRType, IROffset + 4, TD);
3478
    // If we can't get a second FP type, return a simple half or float.
3479
    // avx512fp16-abi.c:pr51813_2 shows it works to return float for
3480
    // {float, i8} too.
3481
2.13k
    if (T1 == nullptr)
3482
2.12k
      return T0;
3483
2.13k
  }
3484
3485
349
  if (T0->isFloatTy() && 
T1->isFloatTy()313
)
3486
307
    return llvm::FixedVectorType::get(T0, 2);
3487
3488
42
  if (T0->isHalfTy() && 
T1->isHalfTy()36
) {
3489
32
    llvm::Type *T2 = nullptr;
3490
32
    if (SourceSize > 4)
3491
8
      T2 = getFPTypeAtOffset(IRType, IROffset + 4, TD);
3492
32
    if (T2 == nullptr)
3493
24
      return llvm::FixedVectorType::get(T0, 2);
3494
8
    return llvm::FixedVectorType::get(T0, 4);
3495
32
  }
3496
3497
10
  if (T0->isHalfTy() || 
T1->isHalfTy()6
)
3498
10
    return llvm::FixedVectorType::get(llvm::Type::getHalfTy(getVMContext()), 4);
3499
3500
0
  return llvm::Type::getDoubleTy(getVMContext());
3501
10
}
3502
3503
3504
/// GetINTEGERTypeAtOffset - The ABI specifies that a value should be passed in
3505
/// an 8-byte GPR.  This means that we either have a scalar or we are talking
3506
/// about the high or low part of an up-to-16-byte struct.  This routine picks
3507
/// the best LLVM IR type to represent this, which may be i64 or may be anything
3508
/// else that the backend will pass in a GPR that works better (e.g. i8, %foo*,
3509
/// etc).
3510
///
3511
/// PrefType is an LLVM IR type that corresponds to (part of) the IR type for
3512
/// the source type.  IROffset is an offset in bytes into the LLVM IR type that
3513
/// the 8-byte value references.  PrefType may be null.
3514
///
3515
/// SourceTy is the source-level type for the entire argument.  SourceOffset is
3516
/// an offset into this that we're processing (which is always either 0 or 8).
3517
///
3518
llvm::Type *X86_64ABIInfo::
3519
GetINTEGERTypeAtOffset(llvm::Type *IRType, unsigned IROffset,
3520
382k
                       QualType SourceTy, unsigned SourceOffset) const {
3521
  // If we're dealing with an un-offset LLVM IR type, then it means that we're
3522
  // returning an 8-byte unit starting with it.  See if we can safely use it.
3523
382k
  if (IROffset == 0) {
3524
    // Pointers and int64's always fill the 8-byte unit.
3525
380k
    if ((isa<llvm::PointerType>(IRType) && 
Has64BitPointers260k
) ||
3526
380k
        
IRType->isIntegerTy(64)119k
)
3527
285k
      return IRType;
3528
3529
    // If we have a 1/2/4-byte integer, we can use it only if the rest of the
3530
    // goodness in the source type is just tail padding.  This is allowed to
3531
    // kick in for struct {double,int} on the int, but not on
3532
    // struct{double,int,int} because we wouldn't return the second int.  We
3533
    // have to do this analysis on the source type because we can't depend on
3534
    // unions being lowered a specific way etc.
3535
95.0k
    if (IRType->isIntegerTy(8) || 
IRType->isIntegerTy(16)91.3k
||
3536
95.0k
        
IRType->isIntegerTy(32)90.5k
||
3537
95.0k
        
(10.3k
isa<llvm::PointerType>(IRType)10.3k
&&
!Has64BitPointers92
)) {
3538
84.8k
      unsigned BitWidth = isa<llvm::PointerType>(IRType) ? 
3292
:
3539
84.8k
          
cast<llvm::IntegerType>(IRType)->getBitWidth()84.7k
;
3540
3541
84.8k
      if (BitsContainNoUserData(SourceTy, SourceOffset*8+BitWidth,
3542
84.8k
                                SourceOffset*8+64, getContext()))
3543
84.4k
        return IRType;
3544
84.8k
    }
3545
95.0k
  }
3546
3547
12.6k
  if (llvm::StructType *STy = dyn_cast<llvm::StructType>(IRType)) {
3548
    // If this is a struct, recurse into the field at the specified offset.
3549
8.13k
    const llvm::StructLayout *SL = getDataLayout().getStructLayout(STy);
3550
8.13k
    if (IROffset < SL->getSizeInBytes()) {
3551
8.13k
      unsigned FieldIdx = SL->getElementContainingOffset(IROffset);
3552
8.13k
      IROffset -= SL->getElementOffset(FieldIdx);
3553
3554
8.13k
      return GetINTEGERTypeAtOffset(STy->getElementType(FieldIdx), IROffset,
3555
8.13k
                                    SourceTy, SourceOffset);
3556
8.13k
    }
3557
8.13k
  }
3558
3559
4.49k
  if (llvm::ArrayType *ATy = dyn_cast<llvm::ArrayType>(IRType)) {
3560
52
    llvm::Type *EltTy = ATy->getElementType();
3561
52
    unsigned EltSize = getDataLayout().getTypeAllocSize(EltTy);
3562
52
    unsigned EltOffset = IROffset/EltSize*EltSize;
3563
52
    return GetINTEGERTypeAtOffset(EltTy, IROffset-EltOffset, SourceTy,
3564
52
                                  SourceOffset);
3565
52
  }
3566
3567
  // Okay, we don't have any better idea of what to pass, so we pass this in an
3568
  // integer register that isn't too big to fit the rest of the struct.
3569
4.44k
  unsigned TySizeInBytes =
3570
4.44k
    (unsigned)getContext().getTypeSizeInChars(SourceTy).getQuantity();
3571
3572
4.44k
  assert(TySizeInBytes != SourceOffset && "Empty field?");
3573
3574
  // It is always safe to classify this as an integer type up to i64 that
3575
  // isn't larger than the structure.
3576
0
  return llvm::IntegerType::get(getVMContext(),
3577
4.44k
                                std::min(TySizeInBytes-SourceOffset, 8U)*8);
3578
4.49k
}
3579
3580
3581
/// GetX86_64ByValArgumentPair - Given a high and low type that can ideally
3582
/// be used as elements of a two register pair to pass or return, return a
3583
/// first class aggregate to represent them.  For example, if the low part of
3584
/// a by-value argument should be passed as i32* and the high part as float,
3585
/// return {i32*, float}.
3586
static llvm::Type *
3587
GetX86_64ByValArgumentPair(llvm::Type *Lo, llvm::Type *Hi,
3588
2.16k
                           const llvm::DataLayout &TD) {
3589
  // In order to correctly satisfy the ABI, we need to the high part to start
3590
  // at offset 8.  If the high and low parts we inferred are both 4-byte types
3591
  // (e.g. i32 and i32) then the resultant struct type ({i32,i32}) won't have
3592
  // the second element at offset 8.  Check for this:
3593
2.16k
  unsigned LoSize = (unsigned)TD.getTypeAllocSize(Lo);
3594
2.16k
  unsigned HiAlign = TD.getABITypeAlignment(Hi);
3595
2.16k
  unsigned HiStart = llvm::alignTo(LoSize, HiAlign);
3596
2.16k
  assert(HiStart != 0 && HiStart <= 8 && "Invalid x86-64 argument pair!");
3597
3598
  // To handle this, we have to increase the size of the low part so that the
3599
  // second element will start at an 8 byte offset.  We can't increase the size
3600
  // of the second element because it might make us access off the end of the
3601
  // struct.
3602
2.16k
  if (HiStart != 8) {
3603
    // There are usually two sorts of types the ABI generation code can produce
3604
    // for the low part of a pair that aren't 8 bytes in size: half, float or
3605
    // i8/i16/i32.  This can also include pointers when they are 32-bit (X32 and
3606
    // NaCl).
3607
    // Promote these to a larger type.
3608
6
    if (Lo->isHalfTy() || Lo->isFloatTy())
3609
1
      Lo = llvm::Type::getDoubleTy(Lo->getContext());
3610
5
    else {
3611
5
      assert((Lo->isIntegerTy() || Lo->isPointerTy())
3612
5
             && "Invalid/unknown lo type");
3613
0
      Lo = llvm::Type::getInt64Ty(Lo->getContext());
3614
5
    }
3615
6
  }
3616
3617
0
  llvm::StructType *Result = llvm::StructType::get(Lo, Hi);
3618
3619
  // Verify that the second element is at an 8-byte offset.
3620
2.16k
  assert(TD.getStructLayout(Result)->getElementOffset(1) == 8 &&
3621
2.16k
         "Invalid x86-64 argument pair!");
3622
0
  return Result;
3623
2.16k
}
3624
3625
ABIArgInfo X86_64ABIInfo::
3626
162k
classifyReturnType(QualType RetTy) const {
3627
  // AMD64-ABI 3.2.3p4: Rule 1. Classify the return type with the
3628
  // classification algorithm.
3629
162k
  X86_64ABIInfo::Class Lo, Hi;
3630
162k
  classify(RetTy, 0, Lo, Hi, /*isNamedArg*/ true);
3631
3632
  // Check some invariants.
3633
162k
  assert((Hi != Memory || Lo == Memory) && "Invalid memory classification.");
3634
0
  assert((Hi != SSEUp || Lo == SSE) && "Invalid SSEUp classification.");
3635
3636
0
  llvm::Type *ResType = nullptr;
3637
162k
  switch (Lo) {
3638
66.6k
  case NoClass:
3639
66.6k
    if (Hi == NoClass)
3640
66.6k
      return ABIArgInfo::getIgnore();
3641
    // If the low part is just padding, it takes no register, leave ResType
3642
    // null.
3643
1
    assert((Hi == SSE || Hi == Integer || Hi == X87Up) &&
3644
1
           "Unknown missing lo part");
3645
0
    break;
3646
3647
0
  case SSEUp:
3648
0
  case X87Up:
3649
0
    llvm_unreachable("Invalid classification for lo word.");
3650
3651
    // AMD64-ABI 3.2.3p4: Rule 2. Types of class memory are returned via
3652
    // hidden argument.
3653
396
  case Memory:
3654
396
    return getIndirectReturnResult(RetTy);
3655
3656
    // AMD64-ABI 3.2.3p4: Rule 3. If the class is INTEGER, the next
3657
    // available register of the sequence %rax, %rdx is used.
3658
90.8k
  case Integer:
3659
90.8k
    ResType = GetINTEGERTypeAtOffset(CGT.ConvertType(RetTy), 0, RetTy, 0);
3660
3661
    // If we have a sign or zero extended integer, make sure to return Extend
3662
    // so that the parameter gets the right LLVM IR attributes.
3663
90.8k
    if (Hi == NoClass && 
isa<llvm::IntegerType>(ResType)89.7k
) {
3664
      // Treat an enum type as its underlying type.
3665
53.4k
      if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
3666
109
        RetTy = EnumTy->getDecl()->getIntegerType();
3667
3668
53.4k
      if (RetTy->isIntegralOrEnumerationType() &&
3669
53.4k
          
isPromotableIntegerTypeForABI(RetTy)52.5k
)
3670
4.59k
        return ABIArgInfo::getExtend(RetTy);
3671
53.4k
    }
3672
86.2k
    break;
3673
3674
    // AMD64-ABI 3.2.3p4: Rule 4. If the class is SSE, the next
3675
    // available SSE register of the sequence %xmm0, %xmm1 is used.
3676
86.2k
  case SSE:
3677
3.47k
    ResType = GetSSETypeAtOffset(CGT.ConvertType(RetTy), 0, RetTy, 0);
3678
3.47k
    break;
3679
3680
    // AMD64-ABI 3.2.3p4: Rule 6. If the class is X87, the value is
3681
    // returned on the X87 stack in %st0 as 80-bit x87 number.
3682
684
  case X87:
3683
684
    ResType = llvm::Type::getX86_FP80Ty(getVMContext());
3684
684
    break;
3685
3686
    // AMD64-ABI 3.2.3p4: Rule 8. If the class is COMPLEX_X87, the real
3687
    // part of the value is returned in %st0 and the imaginary part in
3688
    // %st1.
3689
38
  case ComplexX87:
3690
38
    assert(Hi == ComplexX87 && "Unexpected ComplexX87 classification.");
3691
0
    ResType = llvm::StructType::get(llvm::Type::getX86_FP80Ty(getVMContext()),
3692
38
                                    llvm::Type::getX86_FP80Ty(getVMContext()));
3693
38
    break;
3694
162k
  }
3695
3696
90.4k
  llvm::Type *HighPart = nullptr;
3697
90.4k
  switch (Hi) {
3698
    // Memory was handled previously and X87 should
3699
    // never occur as a hi class.
3700
0
  case Memory:
3701
0
  case X87:
3702
0
    llvm_unreachable("Invalid classification for hi word.");
3703
3704
38
  case ComplexX87: // Previously handled.
3705
86.2k
  case NoClass:
3706
86.2k
    break;
3707
3708
1.11k
  case Integer:
3709
1.11k
    HighPart = GetINTEGERTypeAtOffset(CGT.ConvertType(RetTy), 8, RetTy, 8);
3710
1.11k
    if (Lo == NoClass)  // Return HighPart at offset 8 in memory.
3711
0
      return ABIArgInfo::getDirect(HighPart, 8);
3712
1.11k
    break;
3713
1.11k
  case SSE:
3714
147
    HighPart = GetSSETypeAtOffset(CGT.ConvertType(RetTy), 8, RetTy, 8);
3715
147
    if (Lo == NoClass)  // Return HighPart at offset 8 in memory.
3716
1
      return ABIArgInfo::getDirect(HighPart, 8);
3717
146
    break;
3718
3719
    // AMD64-ABI 3.2.3p4: Rule 5. If the class is SSEUP, the eightbyte
3720
    // is passed in the next available eightbyte chunk if the last used
3721
    // vector register.
3722
    //
3723
    // SSEUP should always be preceded by SSE, just widen.
3724
2.30k
  case SSEUp:
3725
2.30k
    assert(Lo == SSE && "Unexpected SSEUp classification.");
3726
0
    ResType = GetByteVectorType(RetTy);
3727
2.30k
    break;
3728
3729
    // AMD64-ABI 3.2.3p4: Rule 7. If the class is X87UP, the value is
3730
    // returned together with the previous X87 value in %st0.
3731
684
  case X87Up:
3732
    // If X87Up is preceded by X87, we don't need to do
3733
    // anything. However, in some cases with unions it may not be
3734
    // preceded by X87. In such situations we follow gcc and pass the
3735
    // extra bits in an SSE reg.
3736
684
    if (Lo != X87) {
3737
0
      HighPart = GetSSETypeAtOffset(CGT.ConvertType(RetTy), 8, RetTy, 8);
3738
0
      if (Lo == NoClass)  // Return HighPart at offset 8 in memory.
3739
0
        return ABIArgInfo::getDirect(HighPart, 8);
3740
0
    }
3741
684
    break;
3742
90.4k
  }
3743
3744
  // If a high part was specified, merge it together with the low part.  It is
3745
  // known to pass in the high eightbyte of the result.  We do this by forming a
3746
  // first class struct aggregate with the high and low part: {low, high}
3747
90.4k
  if (HighPart)
3748
1.26k
    ResType = GetX86_64ByValArgumentPair(ResType, HighPart, getDataLayout());
3749
3750
90.4k
  return ABIArgInfo::getDirect(ResType);
3751
90.4k
}
3752
3753
ABIArgInfo
3754
X86_64ABIInfo::classifyArgumentType(QualType Ty, unsigned freeIntRegs,
3755
                                    unsigned &neededInt, unsigned &neededSSE,
3756
294k
                                    bool isNamedArg, bool IsRegCall) const {
3757
294k
  Ty = useFirstFieldIfTransparentUnion(Ty);
3758
3759
294k
  X86_64ABIInfo::Class Lo, Hi;
3760
294k
  classify(Ty, 0, Lo, Hi, isNamedArg, IsRegCall);
3761
3762
  // Check some invariants.
3763
  // FIXME: Enforce these by construction.
3764
294k
  assert((Hi != Memory || Lo == Memory) && "Invalid memory classification.");
3765
0
  assert((Hi != SSEUp || Lo == SSE) && "Invalid SSEUp classification.");
3766
3767
0
  neededInt = 0;
3768
294k
  neededSSE = 0;
3769
294k
  llvm::Type *ResType = nullptr;
3770
294k
  switch (Lo) {
3771
5.09k
  case NoClass:
3772
5.09k
    if (Hi == NoClass)
3773
5.09k
      return ABIArgInfo::getIgnore();
3774
    // If the low part is just padding, it takes no register, leave ResType
3775
    // null.
3776
2
    assert((Hi == SSE || Hi == Integer || Hi == X87Up) &&
3777
2
           "Unknown missing lo part");
3778
0
    break;
3779
3780
    // AMD64-ABI 3.2.3p3: Rule 1. If the class is MEMORY, pass the argument
3781
    // on the stack.
3782
1.38k
  case Memory:
3783
3784
    // AMD64-ABI 3.2.3p3: Rule 5. If the class is X87, X87UP or
3785
    // COMPLEX_X87, it is passed in memory.
3786
1.63k
  case X87:
3787
1.68k
  case ComplexX87:
3788
1.68k
    if (getRecordArgABI(Ty, getCXXABI()) == CGCXXABI::RAA_Indirect)
3789
533
      ++neededInt;
3790
1.68k
    return getIndirectResult(Ty, freeIntRegs);
3791
3792
0
  case SSEUp:
3793
0
  case X87Up:
3794
0
    llvm_unreachable("Invalid classification for lo word.");
3795
3796
    // AMD64-ABI 3.2.3p3: Rule 2. If the class is INTEGER, the next
3797
    // available register of the sequence %rdi, %rsi, %rdx, %rcx, %r8
3798
    // and %r9 is used.
3799
281k
  case Integer:
3800
281k
    ++neededInt;
3801
3802
    // Pick an 8-byte type based on the preferred type.
3803
281k
    ResType = GetINTEGERTypeAtOffset(CGT.ConvertType(Ty), 0, Ty, 0);
3804
3805
    // If we have a sign or zero extended integer, make sure to return Extend
3806
    // so that the parameter gets the right LLVM IR attributes.
3807
281k
    if (Hi == NoClass && 
isa<llvm::IntegerType>(ResType)280k
) {
3808
      // Treat an enum type as its underlying type.
3809
57.7k
      if (const EnumType *EnumTy = Ty->getAs<EnumType>())
3810
868
        Ty = EnumTy->getDecl()->getIntegerType();
3811
3812
57.7k
      if (Ty->isIntegralOrEnumerationType() &&
3813
57.7k
          
isPromotableIntegerTypeForABI(Ty)57.3k
)
3814
3.29k
        return ABIArgInfo::getExtend(Ty);
3815
57.7k
    }
3816
3817
277k
    break;
3818
3819
    // AMD64-ABI 3.2.3p3: Rule 3. If the class is SSE, the next
3820
    // available SSE register is used, the registers are taken in the
3821
    // order from %xmm0 to %xmm7.
3822
277k
  case SSE: {
3823
6.84k
    llvm::Type *IRType = CGT.ConvertType(Ty);
3824
6.84k
    ResType = GetSSETypeAtOffset(IRType, 0, Ty, 0);
3825
6.84k
    ++neededSSE;
3826
6.84k
    break;
3827
281k
  }
3828
294k
  }
3829
3830
284k
  llvm::Type *HighPart = nullptr;
3831
284k
  switch (Hi) {
3832
    // Memory was handled previously, ComplexX87 and X87 should
3833
    // never occur as hi classes, and X87Up must be preceded by X87,
3834
    // which is passed in memory.
3835
0
  case Memory:
3836
0
  case X87:
3837
0
  case ComplexX87:
3838
0
    llvm_unreachable("Invalid classification for hi word.");
3839
3840
279k
  case NoClass: break;
3841
3842
748
  case Integer:
3843
748
    ++neededInt;
3844
    // Pick an 8-byte type based on the preferred type.
3845
748
    HighPart = GetINTEGERTypeAtOffset(CGT.ConvertType(Ty), 8, Ty, 8);
3846
3847
748
    if (Lo == NoClass)  // Pass HighPart at offset 8 in memory.
3848
2
      return ABIArgInfo::getDirect(HighPart, 8);
3849
746
    break;
3850
3851
    // X87Up generally doesn't occur here (long double is passed in
3852
    // memory), except in situations involving unions.
3853
746
  case X87Up:
3854
158
  case SSE:
3855
158
    HighPart = GetSSETypeAtOffset(CGT.ConvertType(Ty), 8, Ty, 8);
3856
3857
158
    if (Lo == NoClass)  // Pass HighPart at offset 8 in memory.
3858
0
      return ABIArgInfo::getDirect(HighPart, 8);
3859
3860
158
    ++neededSSE;
3861
158
    break;
3862
3863
    // AMD64-ABI 3.2.3p3: Rule 4. If the class is SSEUP, the
3864
    // eightbyte is passed in the upper half of the last used SSE
3865
    // register.  This only happens when 128-bit vectors are passed.
3866
3.87k
  case SSEUp:
3867
3.87k
    assert(Lo == SSE && "Unexpected SSEUp classification");
3868
0
    ResType = GetByteVectorType(Ty);
3869
3.87k
    break;
3870
284k
  }
3871
3872
  // If a high part was specified, merge it together with the low part.  It is
3873
  // known to pass in the high eightbyte of the result.  We do this by forming a
3874
  // first class struct aggregate with the high and low part: {low, high}
3875
284k
  if (HighPart)
3876
904
    ResType = GetX86_64ByValArgumentPair(ResType, HighPart, getDataLayout());
3877
3878
284k
  return ABIArgInfo::getDirect(ResType);
3879
284k
}
3880
3881
ABIArgInfo
3882
X86_64ABIInfo::classifyRegCallStructTypeImpl(QualType Ty, unsigned &NeededInt,
3883
                                             unsigned &NeededSSE,
3884
20
                                             unsigned &MaxVectorWidth) const {
3885
20
  auto RT = Ty->getAs<RecordType>();
3886
20
  assert(RT && "classifyRegCallStructType only valid with struct types");
3887
3888
20
  if (RT->getDecl()->hasFlexibleArrayMember())
3889
0
    return getIndirectReturnResult(Ty);
3890
3891
  // Sum up bases
3892
20
  if (auto CXXRD = dyn_cast<CXXRecordDecl>(RT->getDecl())) {
3893
0
    if (CXXRD->isDynamicClass()) {
3894
0
      NeededInt = NeededSSE = 0;
3895
0
      return getIndirectReturnResult(Ty);
3896
0
    }
3897
3898
0
    for (const auto &I : CXXRD->bases())
3899
0
      if (classifyRegCallStructTypeImpl(I.getType(), NeededInt, NeededSSE,
3900
0
                                        MaxVectorWidth)
3901
0
              .isIndirect()) {
3902
0
        NeededInt = NeededSSE = 0;
3903
0
        return getIndirectReturnResult(Ty);
3904
0
      }
3905
0
  }
3906
3907
  // Sum up members
3908
48
  
for (const auto *FD : RT->getDecl()->fields())20
{
3909
48
    QualType MTy = FD->getType();
3910
48
    if (MTy->isRecordType() && 
!MTy->isUnionType()0
) {
3911
0
      if (classifyRegCallStructTypeImpl(MTy, NeededInt, NeededSSE,
3912
0
                                        MaxVectorWidth)
3913
0
              .isIndirect()) {
3914
0
        NeededInt = NeededSSE = 0;
3915
0
        return getIndirectReturnResult(Ty);
3916
0
      }
3917
48
    } else {
3918
48
      unsigned LocalNeededInt, LocalNeededSSE;
3919
48
      if (classifyArgumentType(MTy, UINT_MAX, LocalNeededInt, LocalNeededSSE,
3920
48
                               true, true)
3921
48
              .isIndirect()) {
3922
0
        NeededInt = NeededSSE = 0;
3923
0
        return getIndirectReturnResult(Ty);
3924
0
      }
3925
48
      if (const auto *AT = getContext().getAsConstantArrayType(MTy))
3926
10
        MTy = AT->getElementType();
3927
48
      if (const auto *VT = MTy->getAs<VectorType>())
3928
25
        if (getContext().getTypeSize(VT) > MaxVectorWidth)
3929
12
          MaxVectorWidth = getContext().getTypeSize(VT);
3930
48
      NeededInt += LocalNeededInt;
3931
48
      NeededSSE += LocalNeededSSE;
3932
48
    }
3933
48
  }
3934
3935
20
  return ABIArgInfo::getDirect();
3936
20
}
3937
3938
ABIArgInfo
3939
X86_64ABIInfo::classifyRegCallStructType(QualType Ty, unsigned &NeededInt,
3940
                                         unsigned &NeededSSE,
3941
20
                                         unsigned &MaxVectorWidth) const {
3942
3943
20
  NeededInt = 0;
3944
20
  NeededSSE = 0;
3945
20
  MaxVectorWidth = 0;
3946
3947
20
  return classifyRegCallStructTypeImpl(Ty, NeededInt, NeededSSE,
3948
20
                                       MaxVectorWidth);
3949
20
}
3950
3951
163k
void X86_64ABIInfo::computeInfo(CGFunctionInfo &FI) const {
3952
3953
163k
  const unsigned CallingConv = FI.getCallingConvention();
3954
  // It is possible to force Win64 calling convention on any x86_64 target by
3955
  // using __attribute__((ms_abi)). In such case to correctly emit Win64
3956
  // compatible code delegate this call to WinX86_64ABIInfo::computeInfo.
3957
163k
  if (CallingConv == llvm::CallingConv::Win64) {
3958
13
    WinX86_64ABIInfo Win64ABIInfo(CGT, AVXLevel);
3959
13
    Win64ABIInfo.computeInfo(FI);
3960
13
    return;
3961
13
  }
3962
3963
163k
  bool IsRegCall = CallingConv == llvm::CallingConv::X86_RegCall;
3964
3965
  // Keep track of the number of assigned registers.
3966
163k
  unsigned FreeIntRegs = IsRegCall ? 
1128
:
6162k
;
3967
163k
  unsigned FreeSSERegs = IsRegCall ? 
1628
:
8162k
;
3968
163k
  unsigned NeededInt = 0, NeededSSE = 0, MaxVectorWidth = 0;
3969
3970
163k
  if (!::classifyReturnType(getCXXABI(), FI, *this)) {
3971
162k
    if (IsRegCall && 
FI.getReturnType()->getTypePtr()->isRecordType()27
&&
3972
162k
        
!FI.getReturnType()->getTypePtr()->isUnionType()3
) {
3973
3
      FI.getReturnInfo() = classifyRegCallStructType(
3974
3
          FI.getReturnType(), NeededInt, NeededSSE, MaxVectorWidth);
3975
3
      if (FreeIntRegs >= NeededInt && FreeSSERegs >= NeededSSE) {
3976
3
        FreeIntRegs -= NeededInt;
3977
3
        FreeSSERegs -= NeededSSE;
3978
3
      } else {
3979
0
        FI.getReturnInfo() = getIndirectReturnResult(FI.getReturnType());
3980
0
      }
3981
162k
    } else if (IsRegCall && 
FI.getReturnType()->getAs<ComplexType>()24
&&
3982
162k
               getContext().getCanonicalType(FI.getReturnType()
3983
2
                                                 ->getAs<ComplexType>()
3984
2
                                                 ->getElementType()) ==
3985
2
                   getContext().LongDoubleTy)
3986
      // Complex Long Double Type is passed in Memory when Regcall
3987
      // calling convention is used.
3988
1
      FI.getReturnInfo() = getIndirectReturnResult(FI.getReturnType());
3989
162k
    else
3990
162k
      FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
3991
162k
  }
3992
3993
  // If the return value is indirect, then the hidden argument is consuming one
3994
  // integer register.
3995
163k
  if (FI.getReturnInfo().isIndirect())
3996
1.11k
    --FreeIntRegs;
3997
161k
  else if (NeededSSE && 
MaxVectorWidth > 01
)
3998
0
    FI.setMaxVectorWidth(MaxVectorWidth);
3999
4000
  // The chain argument effectively gives us another free register.
4001
163k
  if (FI.isChainCall())
4002
4
    ++FreeIntRegs;
4003
4004
163k
  unsigned NumRequiredArgs = FI.getNumRequiredArgs();
4005
  // AMD64-ABI 3.2.3p3: Once arguments are classified, the registers
4006
  // get assigned (in left-to-right order) for passing as follows...
4007
163k
  unsigned ArgNo = 0;
4008
163k
  for (CGFunctionInfo::arg_iterator it = FI.arg_begin(), ie = FI.arg_end();
4009
457k
       it != ie; 
++it, ++ArgNo294k
) {
4010
294k
    bool IsNamedArg = ArgNo < NumRequiredArgs;
4011
4012
294k
    if (IsRegCall && 
it->type->isStructureOrClassType()64
)
4013
17
      it->info = classifyRegCallStructType(it->type, NeededInt, NeededSSE,
4014
17
                                           MaxVectorWidth);
4015
294k
    else
4016
294k
      it->info = classifyArgumentType(it->type, FreeIntRegs, NeededInt,
4017
294k
                                      NeededSSE, IsNamedArg);
4018
4019
    // AMD64-ABI 3.2.3p3: If there are no registers available for any
4020
    // eightbyte of an argument, the whole argument is passed on the
4021
    // stack. If registers have already been assigned for some
4022
    // eightbytes of such an argument, the assignments get reverted.
4023
294k
    if (FreeIntRegs >= NeededInt && 
FreeSSERegs >= NeededSSE292k
) {
4024
292k
      FreeIntRegs -= NeededInt;
4025
292k
      FreeSSERegs -= NeededSSE;
4026
292k
      if (MaxVectorWidth > FI.getMaxVectorWidth())
4027
7
        FI.setMaxVectorWidth(MaxVectorWidth);
4028
292k
    } else {
4029
2.25k
      it->info = getIndirectResult(it->type, FreeIntRegs);
4030
2.25k
    }
4031
294k
  }
4032
163k
}
4033
4034
static Address EmitX86_64VAArgFromMemory(CodeGenFunction &CGF,
4035
94
                                         Address VAListAddr, QualType Ty) {
4036
94
  Address overflow_arg_area_p =
4037
94
      CGF.Builder.CreateStructGEP(VAListAddr, 2, "overflow_arg_area_p");
4038
94
  llvm::Value *overflow_arg_area =
4039
94
    CGF.Builder.CreateLoad(overflow_arg_area_p, "overflow_arg_area");
4040
4041
  // AMD64-ABI 3.5.7p5: Step 7. Align l->overflow_arg_area upwards to a 16
4042
  // byte boundary if alignment needed by type exceeds 8 byte boundary.
4043
  // It isn't stated explicitly in the standard, but in practice we use
4044
  // alignment greater than 16 where necessary.
4045
94
  CharUnits Align = CGF.getContext().getTypeAlignInChars(Ty);
4046
94
  if (Align > CharUnits::fromQuantity(8)) {
4047
18
    overflow_arg_area = emitRoundPointerUpToAlignment(CGF, overflow_arg_area,
4048
18
                                                      Align);
4049
18
  }
4050
4051
  // AMD64-ABI 3.5.7p5: Step 8. Fetch type from l->overflow_arg_area.
4052
94
  llvm::Type *LTy = CGF.ConvertTypeForMem(Ty);
4053
94
  llvm::Value *Res =
4054
94
    CGF.Builder.CreateBitCast(overflow_arg_area,
4055
94
                              llvm::PointerType::getUnqual(LTy));
4056
4057
  // AMD64-ABI 3.5.7p5: Step 9. Set l->overflow_arg_area to:
4058
  // l->overflow_arg_area + sizeof(type).
4059
  // AMD64-ABI 3.5.7p5: Step 10. Align l->overflow_arg_area upwards to
4060
  // an 8 byte boundary.
4061
4062
94
  uint64_t SizeInBytes = (CGF.getContext().getTypeSize(Ty) + 7) / 8;
4063
94
  llvm::Value *Offset =
4064
94
      llvm::ConstantInt::get(CGF.Int32Ty, (SizeInBytes + 7)  & ~7);
4065
94
  overflow_arg_area = CGF.Builder.CreateGEP(CGF.Int8Ty, overflow_arg_area,
4066
94
                                            Offset, "overflow_arg_area.next");
4067
94
  CGF.Builder.CreateStore(overflow_arg_area, overflow_arg_area_p);
4068
4069
  // AMD64-ABI 3.5.7p5: Step 11. Return the fetched type.
4070
94
  return Address(Res, LTy, Align);
4071
94
}
4072
4073
Address X86_64ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
4074
94
                                 QualType Ty) const {
4075
  // Assume that va_list type is correct; should be pointer to LLVM type:
4076
  // struct {
4077
  //   i32 gp_offset;
4078
  //   i32 fp_offset;
4079
  //   i8* overflow_arg_area;
4080
  //   i8* reg_save_area;
4081
  // };
4082
94
  unsigned neededInt, neededSSE;
4083
4084
94
  Ty = getContext().getCanonicalType(Ty);
4085
94
  ABIArgInfo AI = classifyArgumentType(Ty, 0, neededInt, neededSSE,
4086
94
                                       /*isNamedArg*/false);
4087
4088
  // AMD64-ABI 3.5.7p5: Step 1. Determine whether type may be passed
4089
  // in the registers. If not go to step 7.
4090
94
  if (!neededInt && 
!neededSSE31
)
4091
14
    return EmitX86_64VAArgFromMemory(CGF, VAListAddr, Ty);
4092
4093
  // AMD64-ABI 3.5.7p5: Step 2. Compute num_gp to hold the number of
4094
  // general purpose registers needed to pass type and num_fp to hold
4095
  // the number of floating point registers needed.
4096
4097
  // AMD64-ABI 3.5.7p5: Step 3. Verify whether arguments fit into
4098
  // registers. In the case: l->gp_offset > 48 - num_gp * 8 or
4099
  // l->fp_offset > 304 - num_fp * 16 go to step 7.
4100
  //
4101
  // NOTE: 304 is a typo, there are (6 * 8 + 8 * 16) = 176 bytes of
4102
  // register save space).
4103
4104
80
  llvm::Value *InRegs = nullptr;
4105
80
  Address gp_offset_p = Address::invalid(), fp_offset_p = Address::invalid();
4106
80
  llvm::Value *gp_offset = nullptr, *fp_offset = nullptr;
4107
80
  if (neededInt) {
4108
63
    gp_offset_p = CGF.Builder.CreateStructGEP(VAListAddr, 0, "gp_offset_p");
4109
63
    gp_offset = CGF.Builder.CreateLoad(gp_offset_p, "gp_offset");
4110
63
    InRegs = llvm::ConstantInt::get(CGF.Int32Ty, 48 - neededInt * 8);
4111
63
    InRegs = CGF.Builder.CreateICmpULE(gp_offset, InRegs, "fits_in_gp");
4112
63
  }
4113
4114
80
  if (neededSSE) {
4115
21
    fp_offset_p = CGF.Builder.CreateStructGEP(VAListAddr, 1, "fp_offset_p");
4116
21
    fp_offset = CGF.Builder.CreateLoad(fp_offset_p, "fp_offset");
4117
21
    llvm::Value *FitsInFP =
4118
21
      llvm::ConstantInt::get(CGF.Int32Ty, 176 - neededSSE * 16);
4119
21
    FitsInFP = CGF.Builder.CreateICmpULE(fp_offset, FitsInFP, "fits_in_fp");
4120
21
    InRegs = InRegs ? 
CGF.Builder.CreateAnd(InRegs, FitsInFP)4
:
FitsInFP17
;
4121
21
  }
4122
4123
80
  llvm::BasicBlock *InRegBlock = CGF.createBasicBlock("vaarg.in_reg");
4124
80
  llvm::BasicBlock *InMemBlock = CGF.createBasicBlock("vaarg.in_mem");
4125
80
  llvm::BasicBlock *ContBlock = CGF.createBasicBlock("vaarg.end");
4126
80
  CGF.Builder.CreateCondBr(InRegs, InRegBlock, InMemBlock);
4127
4128
  // Emit code to load the value if it was passed in registers.
4129
4130
80
  CGF.EmitBlock(InRegBlock);
4131
4132
  // AMD64-ABI 3.5.7p5: Step 4. Fetch type from l->reg_save_area with
4133
  // an offset of l->gp_offset and/or l->fp_offset. This may require
4134
  // copying to a temporary location in case the parameter is passed
4135
  // in different register classes or requires an alignment greater
4136
  // than 8 for general purpose registers and 16 for XMM registers.
4137
  //
4138
  // FIXME: This really results in shameful code when we end up needing to
4139
  // collect arguments from different places; often what should result in a
4140
  // simple assembling of a structure from scattered addresses has many more
4141
  // loads than necessary. Can we clean this up?
4142
80
  llvm::Type *LTy = CGF.ConvertTypeForMem(Ty);
4143
80
  llvm::Value *RegSaveArea = CGF.Builder.CreateLoad(
4144
80
      CGF.Builder.CreateStructGEP(VAListAddr, 3), "reg_save_area");
4145
4146
80
  Address RegAddr = Address::invalid();
4147
80
  if (neededInt && 
neededSSE63
) {
4148
    // FIXME: Cleanup.
4149
4
    assert(AI.isDirect() && "Unexpected ABI info for mixed regs");
4150
0
    llvm::StructType *ST = cast<llvm::StructType>(AI.getCoerceToType());
4151
4
    Address Tmp = CGF.CreateMemTemp(Ty);
4152
4
    Tmp = CGF.Builder.CreateElementBitCast(Tmp, ST);
4153
4
    assert(ST->getNumElements() == 2 && "Unexpected ABI info for mixed regs");
4154
0
    llvm::Type *TyLo = ST->getElementType(0);
4155
4
    llvm::Type *TyHi = ST->getElementType(1);
4156
4
    assert((TyLo->isFPOrFPVectorTy() ^ TyHi->isFPOrFPVectorTy()) &&
4157
4
           "Unexpected ABI info for mixed regs");
4158
0
    llvm::Type *PTyLo = llvm::PointerType::getUnqual(TyLo);
4159
4
    llvm::Type *PTyHi = llvm::PointerType::getUnqual(TyHi);
4160
4
    llvm::Value *GPAddr =
4161
4
        CGF.Builder.CreateGEP(CGF.Int8Ty, RegSaveArea, gp_offset);
4162
4
    llvm::Value *FPAddr =
4163
4
        CGF.Builder.CreateGEP(CGF.Int8Ty, RegSaveArea, fp_offset);
4164
4
    llvm::Value *RegLoAddr = TyLo->isFPOrFPVectorTy() ? 
FPAddr1
:
GPAddr3
;
4165
4
    llvm::Value *RegHiAddr = TyLo->isFPOrFPVectorTy() ? 
GPAddr1
:
FPAddr3
;
4166
4167
    // Copy the first element.
4168
    // FIXME: Our choice of alignment here and below is probably pessimistic.
4169
4
    llvm::Value *V = CGF.Builder.CreateAlignedLoad(
4170
4
        TyLo, CGF.Builder.CreateBitCast(RegLoAddr, PTyLo),
4171
4
        CharUnits::fromQuantity(getDataLayout().getABITypeAlignment(TyLo)));
4172
4
    CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 0));
4173
4174
    // Copy the second element.
4175
4
    V = CGF.Builder.CreateAlignedLoad(
4176
4
        TyHi, CGF.Builder.CreateBitCast(RegHiAddr, PTyHi),
4177
4
        CharUnits::fromQuantity(getDataLayout().getABITypeAlignment(TyHi)));
4178
4
    CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 1));
4179
4180
4
    RegAddr = CGF.Builder.CreateElementBitCast(Tmp, LTy);
4181
76
  } else if (neededInt) {
4182
59
    RegAddr = Address(CGF.Builder.CreateGEP(CGF.Int8Ty, RegSaveArea, gp_offset),
4183
59
                      CGF.Int8Ty, CharUnits::fromQuantity(8));
4184
59
    RegAddr = CGF.Builder.CreateElementBitCast(RegAddr, LTy);
4185
4186
    // Copy to a temporary if necessary to ensure the appropriate alignment.
4187
59
    auto TInfo = getContext().getTypeInfoInChars(Ty);
4188
59
    uint64_t TySize = TInfo.Width.getQuantity();
4189
59
    CharUnits TyAlign = TInfo.Align;
4190
4191
    // Copy into a temporary if the type is more aligned than the
4192
    // register save area.
4193
59
    if (TyAlign.getQuantity() > 8) {
4194
3
      Address Tmp = CGF.CreateMemTemp(Ty);
4195
3
      CGF.Builder.CreateMemCpy(Tmp, RegAddr, TySize, false);
4196
3
      RegAddr = Tmp;
4197
3
    }
4198
4199
59
  } else 
if (17
neededSSE == 117
) {
4200
7
    RegAddr = Address(CGF.Builder.CreateGEP(CGF.Int8Ty, RegSaveArea, fp_offset),
4201
7
                      CGF.Int8Ty, CharUnits::fromQuantity(16));
4202
7
    RegAddr = CGF.Builder.CreateElementBitCast(RegAddr, LTy);
4203
10
  } else {
4204
10
    assert(neededSSE == 2 && "Invalid number of needed registers!");
4205
    // SSE registers are spaced 16 bytes apart in the register save
4206
    // area, we need to collect the two eightbytes together.
4207
    // The ABI isn't explicit about this, but it seems reasonable
4208
    // to assume that the slots are 16-byte aligned, since the stack is
4209
    // naturally 16-byte aligned and the prologue is expected to store
4210
    // all the SSE registers to the RSA.
4211
0
    Address RegAddrLo = Address(CGF.Builder.CreateGEP(CGF.Int8Ty, RegSaveArea,
4212
10
                                                      fp_offset),
4213
10
                                CGF.Int8Ty, CharUnits::fromQuantity(16));
4214
10
    Address RegAddrHi =
4215
10
      CGF.Builder.CreateConstInBoundsByteGEP(RegAddrLo,
4216
10
                                             CharUnits::fromQuantity(16));
4217
10
    llvm::Type *ST = AI.canHaveCoerceToType()
4218
10
                         ? AI.getCoerceToType()
4219
10
                         : 
llvm::StructType::get(CGF.DoubleTy, CGF.DoubleTy)0
;
4220
10
    llvm::Value *V;
4221
10
    Address Tmp = CGF.CreateMemTemp(Ty);
4222
10
    Tmp = CGF.Builder.CreateElementBitCast(Tmp, ST);
4223
10
    V = CGF.Builder.CreateLoad(CGF.Builder.CreateElementBitCast(
4224
10
        RegAddrLo, ST->getStructElementType(0)));
4225
10
    CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 0));
4226
10
    V = CGF.Builder.CreateLoad(CGF.Builder.CreateElementBitCast(
4227
10
        RegAddrHi, ST->getStructElementType(1)));
4228
10
    CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 1));
4229
4230
10
    RegAddr = CGF.Builder.CreateElementBitCast(Tmp, LTy);
4231
10
  }
4232
4233
  // AMD64-ABI 3.5.7p5: Step 5. Set:
4234
  // l->gp_offset = l->gp_offset + num_gp * 8
4235
  // l->fp_offset = l->fp_offset + num_fp * 16.
4236
80
  if (neededInt) {
4237
63
    llvm::Value *Offset = llvm::ConstantInt::get(CGF.Int32Ty, neededInt * 8);
4238
63
    CGF.Builder.CreateStore(CGF.Builder.CreateAdd(gp_offset, Offset),
4239
63
                            gp_offset_p);
4240
63
  }
4241
80
  if (neededSSE) {
4242
21
    llvm::Value *Offset = llvm::ConstantInt::get(CGF.Int32Ty, neededSSE * 16);
4243
21
    CGF.Builder.CreateStore(CGF.Builder.CreateAdd(fp_offset, Offset),
4244
21
                            fp_offset_p);
4245
21
  }
4246
80
  CGF.EmitBranch(ContBlock);
4247
4248
  // Emit code to load the value if it was passed in memory.
4249
4250
80
  CGF.EmitBlock(InMemBlock);
4251
80
  Address MemAddr = EmitX86_64VAArgFromMemory(CGF, VAListAddr, Ty);
4252
4253
  // Return the appropriate result.
4254
4255
80
  CGF.EmitBlock(ContBlock);
4256
80
  Address ResAddr = emitMergePHI(CGF, RegAddr, InRegBlock, MemAddr, InMemBlock,
4257
80
                                 "vaarg.addr");
4258
80
  return ResAddr;
4259
94
}
4260
4261
Address X86_64ABIInfo::EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr,
4262
8
                                   QualType Ty) const {
4263
  // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
4264
  // not 1, 2, 4, or 8 bytes, must be passed by reference."
4265
8
  uint64_t Width = getContext().getTypeSize(Ty);
4266
8
  bool IsIndirect = Width > 64 || 
!llvm::isPowerOf2_64(Width)4
;
4267
4268
8
  return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect,
4269
8
                          CGF.getContext().getTypeInfoInChars(Ty),
4270
8
                          CharUnits::fromQuantity(8),
4271
8
                          /*allowHigherAlign*/ false);
4272
8
}
4273
4274
ABIArgInfo WinX86_64ABIInfo::reclassifyHvaArgForVectorCall(
4275
97
    QualType Ty, unsigned &FreeSSERegs, const ABIArgInfo &current) const {
4276
97
  const Type *Base = nullptr;
4277
97
  uint64_t NumElts = 0;
4278
4279
97
  if (!Ty->isBuiltinType() && 
!Ty->isVectorType()49
&&
4280
97
      
isHomogeneousAggregate(Ty, Base, NumElts)36
&&
FreeSSERegs >= NumElts27
) {
4281
18
    FreeSSERegs -= NumElts;
4282
18
    return getDirectX86Hva();
4283
18
  }
4284
79
  return current;
4285
97
}
4286
4287
ABIArgInfo WinX86_64ABIInfo::classify(QualType Ty, unsigned &FreeSSERegs,
4288
                                      bool IsReturnType, bool IsVectorCall,
4289
10.1k
                                      bool IsRegCall) const {
4290
4291
10.1k
  if (Ty->isVoidType())
4292
1.69k
    return ABIArgInfo::getIgnore();
4293
4294
8.44k
  if (const EnumType *EnumTy = Ty->getAs<EnumType>())
4295
20
    Ty = EnumTy->getDecl()->getIntegerType();
4296
4297
8.44k
  TypeInfo Info = getContext().getTypeInfo(Ty);
4298
8.44k
  uint64_t Width = Info.Width;
4299
8.44k
  CharUnits Align = getContext().toCharUnitsFromBits(Info.Align);
4300
4301
8.44k
  const RecordType *RT = Ty->getAs<RecordType>();
4302
8.44k
  if (RT) {
4303
306
    if (!IsReturnType) {
4304
267
      if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(RT, getCXXABI()))
4305
71
        return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
4306
267
    }
4307
4308
235
    if (RT->getDecl()->hasFlexibleArrayMember())
4309
0
      return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
4310
4311
235
  }
4312
4313
8.37k
  const Type *Base = nullptr;
4314
8.37k
  uint64_t NumElts = 0;
4315
  // vectorcall adds the concept of a homogenous vector aggregate, similar to
4316
  // other targets.
4317
8.37k
  if ((IsVectorCall || 
IsRegCall8.25k
) &&
4318
8.37k
      
isHomogeneousAggregate(Ty, Base, NumElts)175
) {
4319
95
    if (IsRegCall) {
4320
32
      if (FreeSSERegs >= NumElts) {
4321
32
        FreeSSERegs -= NumElts;
4322
32
        if (IsReturnType || 
Ty->isBuiltinType()27
||
Ty->isVectorType()21
)
4323
17
          return ABIArgInfo::getDirect();
4324
15
        return ABIArgInfo::getExpand();
4325
32
      }
4326
0
      return ABIArgInfo::getIndirect(Align, /*ByVal=*/false);
4327
63
    } else if (IsVectorCall) {
4328
63
      if (FreeSSERegs >= NumElts &&
4329
63
          
(57
IsReturnType57
||
Ty->isBuiltinType()45
||
Ty->isVectorType()36
)) {
4330
34
        FreeSSERegs -= NumElts;
4331
34
        return ABIArgInfo::getDirect();
4332
34
      } else 
if (29
IsReturnType29
) {
4333
0
        return ABIArgInfo::getExpand();
4334
29
      } else if (!Ty->isBuiltinType() && 
!Ty->isVectorType()27
) {
4335
        // HVAs are delayed and reclassified in the 2nd step.
4336
27
        return ABIArgInfo::getIndirect(Align, /*ByVal=*/false);
4337
27
      }
4338
63
    }
4339
95
  }
4340
4341
8.27k
  if (Ty->isMemberPointerType()) {
4342
    // If the member pointer is represented by an LLVM int or ptr, pass it
4343
    // directly.
4344
160
    llvm::Type *LLTy = CGT.ConvertType(Ty);
4345
160
    if (LLTy->isPointerTy() || 
LLTy->isIntegerTy()77
)
4346
91
      return ABIArgInfo::getDirect();
4347
160
  }
4348
4349
8.18k
  if (RT || 
Ty->isAnyComplexType()7.99k
||
Ty->isMemberPointerType()7.98k
) {
4350
    // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
4351
    // not 1, 2, 4, or 8 bytes, must be passed by reference."
4352
273
    if (Width > 64 || 
!llvm::isPowerOf2_64(Width)150
)
4353
123
      return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
4354
4355
    // Otherwise, coerce it to a small integer.
4356
150
    return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(), Width));
4357
273
  }
4358
4359
7.91k
  if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
4360
2.22k
    switch (BT->getKind()) {
4361
32
    case BuiltinType::Bool:
4362
      // Bool type is always extended to the ABI, other builtin types are not
4363
      // extended.
4364
32
      return ABIArgInfo::getExtend(Ty);
4365
4366
65
    case BuiltinType::LongDouble:
4367
      // Mingw64 GCC uses the old 80 bit extended precision floating point
4368
      // unit. It passes them indirectly through memory.
4369
65
      if (IsMingw64) {
4370
7
        const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();
4371
7
        if (LDF == &llvm::APFloat::x87DoubleExtended())
4372
7
          return ABIArgInfo::getIndirect(Align, /*ByVal=*/false);
4373
7
      }
4374
58
      break;
4375
4376
58
    case BuiltinType::Int128:
4377
10
    case BuiltinType::UInt128:
4378
      // If it's a parameter type, the normal ABI rule is that arguments larger
4379
      // than 8 bytes are passed indirectly. GCC follows it. We follow it too,
4380
      // even though it isn't particularly efficient.
4381
10
      if (!IsReturnType)
4382
4
        return ABIArgInfo::getIndirect(Align, /*ByVal=*/false);
4383
4384
      // Mingw64 GCC returns i128 in XMM0. Coerce to v2i64 to handle that.
4385
      // Clang matches them for compatibility.
4386
6
      return ABIArgInfo::getDirect(llvm::FixedVectorType::get(
4387
6
          llvm::Type::getInt64Ty(getVMContext()), 2));
4388
4389
2.12k
    default:
4390
2.12k
      break;
4391
2.22k
    }
4392
2.22k
  }
4393
4394
7.86k
  if (Ty->isBitIntType()) {
4395
    // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
4396
    // not 1, 2, 4, or 8 bytes, must be passed by reference."
4397
    // However, non-power-of-two bit-precise integers will be passed as 1, 2, 4,
4398
    // or 8 bytes anyway as long is it fits in them, so we don't have to check
4399
    // the power of 2.
4400
51
    if (Width <= 64)
4401
37
      return ABIArgInfo::getDirect();
4402
14
    return ABIArgInfo::getIndirect(Align, /*ByVal=*/false);
4403
51
  }
4404
4405
7.81k
  return ABIArgInfo::getDirect();
4406
7.86k
}
4407
4408
4.19k
void WinX86_64ABIInfo::computeInfo(CGFunctionInfo &FI) const {
4409
4.19k
  const unsigned CC = FI.getCallingConvention();
4410
4.19k
  bool IsVectorCall = CC == llvm::CallingConv::X86_VectorCall;
4411
4.19k
  bool IsRegCall = CC == llvm::CallingConv::X86_RegCall;
4412
4413
  // If __attribute__((sysv_abi)) is in use, use the SysV argument
4414
  // classification rules.
4415
4.19k
  if (CC == llvm::CallingConv::X86_64_SysV) {
4416
14
    X86_64ABIInfo SysVABIInfo(CGT, AVXLevel);
4417
14
    SysVABIInfo.computeInfo(FI);
4418
14
    return;
4419
14
  }
4420
4421
4.18k
  unsigned FreeSSERegs = 0;
4422
4.18k
  if (IsVectorCall) {
4423
    // We can use up to 4 SSE return registers with vectorcall.
4424
49
    FreeSSERegs = 4;
4425
4.13k
  } else if (IsRegCall) {
4426
    // RegCall gives us 16 SSE registers.
4427
26
    FreeSSERegs = 16;
4428
26
  }
4429
4430
4.18k
  if (!getCXXABI().classifyReturnType(FI))
4431
4.13k
    FI.getReturnInfo() = classify(FI.getReturnType(), FreeSSERegs, true,
4432
4.13k
                                  IsVectorCall, IsRegCall);
4433
4434
4.18k
  if (IsVectorCall) {
4435
    // We can use up to 6 SSE register parameters with vectorcall.
4436
49
    FreeSSERegs = 6;
4437
4.13k
  } else if (IsRegCall) {
4438
    // RegCall gives us 16 SSE registers, we can reuse the return registers.
4439
26
    FreeSSERegs = 16;
4440
26
  }
4441
4442
4.18k
  unsigned ArgNum = 0;
4443
4.18k
  unsigned ZeroSSERegs = 0;
4444
6.00k
  for (auto &I : FI.arguments()) {
4445
    // Vectorcall in x64 only permits the first 6 arguments to be passed as
4446
    // XMM/YMM registers. After the sixth argument, pretend no vector
4447
    // registers are left.
4448
6.00k
    unsigned *MaybeFreeSSERegs =
4449
6.00k
        (IsVectorCall && 
ArgNum >= 697
) ?
&ZeroSSERegs5
:
&FreeSSERegs6.00k
;
4450
6.00k
    I.info =
4451
6.00k
        classify(I.type, *MaybeFreeSSERegs, false, IsVectorCall, IsRegCall);
4452
6.00k
    ++ArgNum;
4453
6.00k
  }
4454
4455
4.18k
  if (IsVectorCall) {
4456
    // For vectorcall, assign aggregate HVAs to any free vector registers in a
4457
    // second pass.
4458
49
    for (auto &I : FI.arguments())
4459
97
      I.info = reclassifyHvaArgForVectorCall(I.type, FreeSSERegs, I.info);
4460
49
  }
4461
4.18k
}
4462
4463
Address WinX86_64ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
4464
19
                                    QualType Ty) const {
4465
  // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
4466
  // not 1, 2, 4, or 8 bytes, must be passed by reference."
4467
19
  uint64_t Width = getContext().getTypeSize(Ty);
4468
19
  bool IsIndirect = Width > 64 || 
!llvm::isPowerOf2_64(Width)8
;
4469
4470
19
  return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect,
4471
19
                          CGF.getContext().getTypeInfoInChars(Ty),
4472
19
                          CharUnits::fromQuantity(8),
4473
19
                          /*allowHigherAlign*/ false);
4474
19
}
4475
4476
static bool PPC_initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
4477
                                        llvm::Value *Address, bool Is64Bit,
4478
4
                                        bool IsAIX) {
4479
  // This is calculated from the LLVM and GCC tables and verified
4480
  // against gcc output.  AFAIK all PPC ABIs use the same encoding.
4481
4482
4
  CodeGen::CGBuilderTy &Builder = CGF.Builder;
4483
4484
4
  llvm::IntegerType *i8 = CGF.Int8Ty;
4485
4
  llvm::Value *Four8 = llvm::ConstantInt::get(i8, 4);
4486
4
  llvm::Value *Eight8 = llvm::ConstantInt::get(i8, 8);
4487
4
  llvm::Value *Sixteen8 = llvm::ConstantInt::get(i8, 16);
4488
4489
  // 0-31: r0-31, the 4-byte or 8-byte general-purpose registers
4490
4
  AssignToArrayRange(Builder, Address, Is64Bit ? 
Eight82
:
Four82
, 0, 31);
4491
4492
  // 32-63: fp0-31, the 8-byte floating-point registers
4493
4
  AssignToArrayRange(Builder, Address, Eight8, 32, 63);
4494
4495
  // 64-67 are various 4-byte or 8-byte special-purpose registers:
4496
  // 64: mq
4497
  // 65: lr
4498
  // 66: ctr
4499
  // 67: ap
4500
4
  AssignToArrayRange(Builder, Address, Is64Bit ? 
Eight82
:
Four82
, 64, 67);
4501
4502
  // 68-76 are various 4-byte special-purpose registers:
4503
  // 68-75 cr0-7
4504
  // 76: xer
4505
4
  AssignToArrayRange(Builder, Address, Four8, 68, 76);
4506
4507
  // 77-108: v0-31, the 16-byte vector registers
4508
4
  AssignToArrayRange(Builder, Address, Sixteen8, 77, 108);
4509
4510
  // 109: vrsave
4511
  // 110: vscr
4512
4
  AssignToArrayRange(Builder, Address, Is64Bit ? 
Eight82
:
Four82
, 109, 110);
4513
4514
  // AIX does not utilize the rest of the registers.
4515
4
  if (IsAIX)
4516
2
    return false;
4517
4518
  // 111: spe_acc
4519
  // 112: spefscr
4520
  // 113: sfp
4521
2
  AssignToArrayRange(Builder, Address, Is64Bit ? 
Eight81
:
Four81
, 111, 113);
4522
4523
2
  if (!Is64Bit)
4524
1
    return false;
4525
4526
  // TODO: Need to verify if these registers are used on 64 bit AIX with Power8
4527
  // or above CPU.
4528
  // 64-bit only registers:
4529
  // 114: tfhar
4530
  // 115: tfiar
4531
  // 116: texasr
4532
1
  AssignToArrayRange(Builder, Address, Eight8, 114, 116);
4533
4534
1
  return false;
4535
2
}
4536
4537
// AIX
4538
namespace {
4539
/// AIXABIInfo - The AIX XCOFF ABI information.
4540
class AIXABIInfo : public ABIInfo {
4541
  const bool Is64Bit;
4542
  const unsigned PtrByteSize;
4543
  CharUnits getParamTypeAlignment(QualType Ty) const;
4544
4545
public:
4546
  AIXABIInfo(CodeGen::CodeGenTypes &CGT, bool Is64Bit)
4547
117
      : ABIInfo(CGT), Is64Bit(Is64Bit), PtrByteSize(Is64Bit ? 8 : 4) {}
4548
4549
  bool isPromotableTypeForABI(QualType Ty) const;
4550
4551
  ABIArgInfo classifyReturnType(QualType RetTy) const;
4552
  ABIArgInfo classifyArgumentType(QualType Ty) const;
4553
4554
311
  void computeInfo(CGFunctionInfo &FI) const override {
4555
311
    if (!getCXXABI().classifyReturnType(FI))
4556
311
      FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
4557
4558
311
    for (auto &I : FI.arguments())
4559
256
      I.info = classifyArgumentType(I.type);
4560
311
  }
4561
4562
  Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
4563
                    QualType Ty) const override;
4564
};
4565
4566
class AIXTargetCodeGenInfo : public TargetCodeGenInfo {
4567
  const bool Is64Bit;
4568
4569
public:
4570
  AIXTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, bool Is64Bit)
4571
      : TargetCodeGenInfo(std::make_unique<AIXABIInfo>(CGT, Is64Bit)),
4572
117
        Is64Bit(Is64Bit) {}
4573
2
  int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
4574
2
    return 1; // r1 is the dedicated stack pointer
4575
2
  }
4576
4577
  bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
4578
                               llvm::Value *Address) const override;
4579
};
4580
} // namespace
4581
4582
// Return true if the ABI requires Ty to be passed sign- or zero-
4583
// extended to 32/64 bits.
4584
355
bool AIXABIInfo::isPromotableTypeForABI(QualType Ty) const {
4585
  // Treat an enum type as its underlying type.
4586
355
  if (const EnumType *EnumTy = Ty->getAs<EnumType>())
4587
2
    Ty = EnumTy->getDecl()->getIntegerType();
4588
4589
  // Promotable integer types are required to be promoted by the ABI.
4590
355
  if (Ty->isPromotableIntegerType())
4591
3
    return true;
4592
4593
352
  if (!Is64Bit)
4594
205
    return false;
4595
4596
  // For 64 bit mode, in addition to the usual promotable integer types, we also
4597
  // need to extend all 32-bit types, since the ABI requires promotion to 64
4598
  // bits.
4599
147
  if (const BuiltinType *BT = Ty->getAs<BuiltinType>())
4600
78
    switch (BT->getKind()) {
4601
50
    case BuiltinType::Int:
4602
54
    case BuiltinType::UInt:
4603
54
      return true;
4604
24
    default:
4605
24
      break;
4606
78
    }
4607
4608
93
  return false;
4609
147
}
4610
4611
311
ABIArgInfo AIXABIInfo::classifyReturnType(QualType RetTy) const {
4612
311
  if (RetTy->isAnyComplexType())
4613
6
    return ABIArgInfo::getDirect();
4614
4615
305
  if (RetTy->isVectorType())
4616
0
    return ABIArgInfo::getDirect();
4617
4618
305
  if (RetTy->isVoidType())
4619
187
    return ABIArgInfo::getIgnore();
4620
4621
118
  if (isAggregateTypeForABI(RetTy))
4622
2
    return getNaturalAlignIndirect(RetTy);
4623
4624
116
  return (isPromotableTypeForABI(RetTy) ? 
ABIArgInfo::getExtend(RetTy)30
4625
116
                                        : 
ABIArgInfo::getDirect()86
);
4626
118
}
4627
4628
256
ABIArgInfo AIXABIInfo::classifyArgumentType(QualType Ty) const {
4629
256
  Ty = useFirstFieldIfTransparentUnion(Ty);
4630
4631
256
  if (Ty->isAnyComplexType())
4632
6
    return ABIArgInfo::getDirect();
4633
4634
250
  if (Ty->isVectorType())
4635
0
    return ABIArgInfo::getDirect();
4636
4637
250
  if (isAggregateTypeForABI(Ty)) {
4638
    // Records with non-trivial destructors/copy-constructors should not be
4639
    // passed by value.
4640
11
    if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))
4641
0
      return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
4642
4643
11
    CharUnits CCAlign = getParamTypeAlignment(Ty);
4644
11
    CharUnits TyAlign = getContext().getTypeAlignInChars(Ty);
4645
4646
11
    return ABIArgInfo::getIndirect(CCAlign, /*ByVal*/ true,
4647
11
                                   /*Realign*/ TyAlign > CCAlign);
4648
11
  }
4649
4650
239
  return (isPromotableTypeForABI(Ty) ? 
ABIArgInfo::getExtend(Ty)27
4651
239
                                     : 
ABIArgInfo::getDirect()212
);
4652
250
}
4653
4654
11
CharUnits AIXABIInfo::getParamTypeAlignment(QualType Ty) const {
4655
  // Complex types are passed just like their elements.
4656
11
  if (const ComplexType *CTy = Ty->getAs<ComplexType>())
4657
0
    Ty = CTy->getElementType();
4658
4659
11
  if (Ty->isVectorType())
4660
0
    return CharUnits::fromQuantity(16);
4661
4662
  // If the structure contains a vector type, the alignment is 16.
4663
11
  if (isRecordWithSIMDVectorType(getContext(), Ty))
4664
0
    return CharUnits::fromQuantity(16);
4665
4666
11
  return CharUnits::fromQuantity(PtrByteSize);
4667
11
}
4668
4669
Address AIXABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
4670
0
                              QualType Ty) const {
4671
4672
0
  auto TypeInfo = getContext().getTypeInfoInChars(Ty);
4673
0
  TypeInfo.Align = getParamTypeAlignment(Ty);
4674
4675
0
  CharUnits SlotSize = CharUnits::fromQuantity(PtrByteSize);
4676
4677
  // If we have a complex type and the base type is smaller than the register
4678
  // size, the ABI calls for the real and imaginary parts to be right-adjusted
4679
  // in separate words in 32bit mode or doublewords in 64bit mode. However,
4680
  // Clang expects us to produce a pointer to a structure with the two parts
4681
  // packed tightly. So generate loads of the real and imaginary parts relative
4682
  // to the va_list pointer, and store them to a temporary structure. We do the
4683
  // same as the PPC64ABI here.
4684
0
  if (const ComplexType *CTy = Ty->getAs<ComplexType>()) {
4685
0
    CharUnits EltSize = TypeInfo.Width / 2;
4686
0
    if (EltSize < SlotSize)
4687
0
      return complexTempStructure(CGF, VAListAddr, Ty, SlotSize, EltSize, CTy);
4688
0
  }
4689
4690
0
  return emitVoidPtrVAArg(CGF, VAListAddr, Ty, /*Indirect*/ false, TypeInfo,
4691
0
                          SlotSize, /*AllowHigher*/ true);
4692
0
}
4693
4694
bool AIXTargetCodeGenInfo::initDwarfEHRegSizeTable(
4695
2
    CodeGen::CodeGenFunction &CGF, llvm::Value *Address) const {
4696
2
  return PPC_initDwarfEHRegSizeTable(CGF, Address, Is64Bit, /*IsAIX*/ true);
4697
2
}
4698
4699
// PowerPC-32
4700
namespace {
4701
/// PPC32_SVR4_ABIInfo - The 32-bit PowerPC ELF (SVR4) ABI information.
4702
class PPC32_SVR4_ABIInfo : public DefaultABIInfo {
4703
  bool IsSoftFloatABI;
4704
  bool IsRetSmallStructInRegABI;
4705
4706
  CharUnits getParamTypeAlignment(QualType Ty) const;
4707
4708
public:
4709
  PPC32_SVR4_ABIInfo(CodeGen::CodeGenTypes &CGT, bool SoftFloatABI,
4710
                     bool RetSmallStructInRegABI)
4711
      : DefaultABIInfo(CGT), IsSoftFloatABI(SoftFloatABI),
4712
65
        IsRetSmallStructInRegABI(RetSmallStructInRegABI) {}
4713
4714
  ABIArgInfo classifyReturnType(QualType RetTy) const;
4715
4716
166
  void computeInfo(CGFunctionInfo &FI) const override {
4717
166
    if (!getCXXABI().classifyReturnType(FI))
4718
166
      FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
4719
166
    for (auto &I : FI.arguments())
4720
277
      I.info = classifyArgumentType(I.type);
4721
166
  }
4722
4723
  Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
4724
                    QualType Ty) const override;
4725
};
4726
4727
class PPC32TargetCodeGenInfo : public TargetCodeGenInfo {
4728
public:
4729
  PPC32TargetCodeGenInfo(CodeGenTypes &CGT, bool SoftFloatABI,
4730
                         bool RetSmallStructInRegABI)
4731
      : TargetCodeGenInfo(std::make_unique<PPC32_SVR4_ABIInfo>(
4732
65
            CGT, SoftFloatABI, RetSmallStructInRegABI)) {}
4733
4734
  static bool isStructReturnInRegABI(const llvm::Triple &Triple,
4735
                                     const CodeGenOptions &Opts);
4736
4737
1
  int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
4738
    // This is recovered from gcc output.
4739
1
    return 1; // r1 is the dedicated stack pointer
4740
1
  }
4741
4742
  bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
4743
                               llvm::Value *Address) const override;
4744
};
4745
}
4746
4747
0
CharUnits PPC32_SVR4_ABIInfo::getParamTypeAlignment(QualType Ty) const {
4748
  // Complex types are passed just like their elements.
4749
0
  if (const ComplexType *CTy = Ty->getAs<ComplexType>())
4750
0
    Ty = CTy->getElementType();
4751
4752
0
  if (Ty->isVectorType())
4753
0
    return CharUnits::fromQuantity(getContext().getTypeSize(Ty) == 128 ? 16
4754
0
                                                                       : 4);
4755
4756
  // For single-element float/vector structs, we consider the whole type
4757
  // to have the same alignment requirements as its single element.
4758
0
  const Type *AlignTy = nullptr;
4759
0
  if (const Type *EltType = isSingleElementStruct(Ty, getContext())) {
4760
0
    const BuiltinType *BT = EltType->getAs<BuiltinType>();
4761
0
    if ((EltType->isVectorType() && getContext().getTypeSize(EltType) == 128) ||
4762
0
        (BT && BT->isFloatingPoint()))
4763
0
      AlignTy = EltType;
4764
0
  }
4765
4766
0
  if (AlignTy)
4767
0
    return CharUnits::fromQuantity(AlignTy->isVectorType() ? 16 : 4);
4768
0
  return CharUnits::fromQuantity(4);
4769
0
}
4770
4771
166
ABIArgInfo PPC32_SVR4_ABIInfo::classifyReturnType(QualType RetTy) const {
4772
166
  uint64_t Size;
4773
4774
  // -msvr4-struct-return puts small aggregates in GPR3 and GPR4.
4775
166
  if (isAggregateTypeForABI(RetTy) && 
IsRetSmallStructInRegABI21
&&
4776
166
      
(Size = getContext().getTypeSize(RetTy)) <= 6418
) {
4777
    // System V ABI (1995), page 3-22, specified:
4778
    // > A structure or union whose size is less than or equal to 8 bytes
4779
    // > shall be returned in r3 and r4, as if it were first stored in the
4780
    // > 8-byte aligned memory area and then the low addressed word were
4781
    // > loaded into r3 and the high-addressed word into r4.  Bits beyond
4782
    // > the last member of the structure or union are not defined.
4783
    //
4784
    // GCC for big-endian PPC32 inserts the pad before the first member,
4785
    // not "beyond the last member" of the struct.  To stay compatible
4786
    // with GCC, we coerce the struct to an integer of the same size.
4787
    // LLVM will extend it and return i32 in r3, or i64 in r3:r4.
4788
7
    if (Size == 0)
4789
1
      return ABIArgInfo::getIgnore();
4790
6
    else {
4791
6
      llvm::Type *CoerceTy = llvm::Type::getIntNTy(getVMContext(), Size);
4792
6
      return ABIArgInfo::getDirect(CoerceTy);
4793
6
    }
4794
7
  }
4795
4796
159
  return DefaultABIInfo::classifyReturnType(RetTy);
4797
166
}
4798
4799
// TODO: this implementation is now likely redundant with
4800
// DefaultABIInfo::EmitVAArg.
4801
Address PPC32_SVR4_ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAList,
4802
1
                                      QualType Ty) const {
4803
1
  if (getTarget().getTriple().isOSDarwin()) {
4804
0
    auto TI = getContext().getTypeInfoInChars(Ty);
4805
0
    TI.Align = getParamTypeAlignment(Ty);
4806
4807
0
    CharUnits SlotSize = CharUnits::fromQuantity(4);
4808
0
    return emitVoidPtrVAArg(CGF, VAList, Ty,
4809
0
                            classifyArgumentType(Ty).isIndirect(), TI, SlotSize,
4810
0
                            /*AllowHigherAlign=*/true);
4811
0
  }
4812
4813
1
  const unsigned OverflowLimit = 8;
4814
1
  if (const ComplexType *CTy = Ty->getAs<ComplexType>()) {
4815
    // TODO: Implement this. For now ignore.
4816
0
    (void)CTy;
4817
0
    return Address::invalid(); // FIXME?
4818
0
  }
4819
4820
  // struct __va_list_tag {
4821
  //   unsigned char gpr;
4822
  //   unsigned char fpr;
4823
  //   unsigned short reserved;
4824
  //   void *overflow_arg_area;
4825
  //   void *reg_save_area;
4826
  // };
4827
4828
1
  bool isI64 = Ty->isIntegerType() && 
getContext().getTypeSize(Ty) == 640
;
4829
1
  bool isInt = !Ty->isFloatingType();
4830
1
  bool isF64 = Ty->isFloatingType() && getContext().getTypeSize(Ty) == 64;
4831
4832
  // All aggregates are passed indirectly?  That doesn't seem consistent
4833
  // with the argument-lowering code.
4834
1
  bool isIndirect = isAggregateTypeForABI(Ty);
4835
4836
1
  CGBuilderTy &Builder = CGF.Builder;
4837
4838
  // The calling convention either uses 1-2 GPRs or 1 FPR.
4839
1
  Address NumRegsAddr = Address::invalid();
4840
1
  if (isInt || IsSoftFloatABI) {
4841
1
    NumRegsAddr = Builder.CreateStructGEP(VAList, 0, "gpr");
4842
1
  } else {
4843
0
    NumRegsAddr = Builder.CreateStructGEP(VAList, 1, "fpr");
4844
0
  }
4845
4846
1
  llvm::Value *NumRegs = Builder.CreateLoad(NumRegsAddr, "numUsedRegs");
4847
4848
  // "Align" the register count when TY is i64.
4849
1
  if (isI64 || (isF64 && IsSoftFloatABI)) {
4850
1
    NumRegs = Builder.CreateAdd(NumRegs, Builder.getInt8(1));
4851
1
    NumRegs = Builder.CreateAnd(NumRegs, Builder.getInt8((uint8_t) ~1U));
4852
1
  }
4853
4854
1
  llvm::Value *CC =
4855
1
      Builder.CreateICmpULT(NumRegs, Builder.getInt8(OverflowLimit), "cond");
4856
4857
1
  llvm::BasicBlock *UsingRegs = CGF.createBasicBlock("using_regs");
4858
1
  llvm::BasicBlock *UsingOverflow = CGF.createBasicBlock("using_overflow");
4859
1
  llvm::BasicBlock *Cont = CGF.createBasicBlock("cont");
4860
4861
1
  Builder.CreateCondBr(CC, UsingRegs, UsingOverflow);
4862
4863
1
  llvm::Type *DirectTy = CGF.ConvertType(Ty), *ElementTy = DirectTy;
4864
1
  if (isIndirect) 
DirectTy = DirectTy->getPointerTo(0)0
;
4865
4866
  // Case 1: consume registers.
4867
1
  Address RegAddr = Address::invalid();
4868
1
  {
4869
1
    CGF.EmitBlock(UsingRegs);
4870
4871
1
    Address RegSaveAreaPtr = Builder.CreateStructGEP(VAList, 4);
4872
1
    RegAddr = Address(Builder.CreateLoad(RegSaveAreaPtr), CGF.Int8Ty,
4873
1
                      CharUnits::fromQuantity(8));
4874
1
    assert(RegAddr.getElementType() == CGF.Int8Ty);
4875
4876
    // Floating-point registers start after the general-purpose registers.
4877
1
    if (!(isInt || IsSoftFloatABI)) {
4878
0
      RegAddr = Builder.CreateConstInBoundsByteGEP(RegAddr,
4879
0
                                                   CharUnits::fromQuantity(32));
4880
0
    }
4881
4882
    // Get the address of the saved value by scaling the number of
4883
    // registers we've used by the number of
4884
1
    CharUnits RegSize = CharUnits::fromQuantity((isInt || IsSoftFloatABI) ? 4 : 
80
);
4885
1
    llvm::Value *RegOffset =
4886
1
        Builder.CreateMul(NumRegs, Builder.getInt8(RegSize.getQuantity()));
4887
1
    RegAddr = Address(
4888
1
        Builder.CreateInBoundsGEP(CGF.Int8Ty, RegAddr.getPointer(), RegOffset),
4889
1
        CGF.Int8Ty, RegAddr.getAlignment().alignmentOfArrayElement(RegSize));
4890
1
    RegAddr = Builder.CreateElementBitCast(RegAddr, DirectTy);
4891
4892
    // Increase the used-register count.
4893
1
    NumRegs =
4894
1
      Builder.CreateAdd(NumRegs,
4895
1
                        Builder.getInt8((isI64 || (isF64 && IsSoftFloatABI)) ? 2 : 
10
));
4896
1
    Builder.CreateStore(NumRegs, NumRegsAddr);
4897
4898
1
    CGF.EmitBranch(Cont);
4899
1
  }
4900
4901
  // Case 2: consume space in the overflow area.
4902
0
  Address MemAddr = Address::invalid();
4903
1
  {
4904
1
    CGF.EmitBlock(UsingOverflow);
4905
4906
1
    Builder.CreateStore(Builder.getInt8(OverflowLimit), NumRegsAddr);
4907
4908
    // Everything in the overflow area is rounded up to a size of at least 4.
4909
1
    CharUnits OverflowAreaAlign = CharUnits::fromQuantity(4);
4910
4911
1
    CharUnits Size;
4912
1
    if (!isIndirect) {
4913
1
      auto TypeInfo = CGF.getContext().getTypeInfoInChars(Ty);
4914
1
      Size = TypeInfo.Width.alignTo(OverflowAreaAlign);
4915
1
    } else {
4916
0
      Size = CGF.getPointerSize();
4917
0
    }
4918
4919
1
    Address OverflowAreaAddr = Builder.CreateStructGEP(VAList, 3);
4920
1
    Address OverflowArea =
4921
1
        Address(Builder.CreateLoad(OverflowAreaAddr, "argp.cur"), CGF.Int8Ty,
4922
1
                OverflowAreaAlign);
4923
    // Round up address of argument to alignment
4924
1
    CharUnits Align = CGF.getContext().getTypeAlignInChars(Ty);
4925
1
    if (Align > OverflowAreaAlign) {
4926
1
      llvm::Value *Ptr = OverflowArea.getPointer();
4927
1
      OverflowArea = Address(emitRoundPointerUpToAlignment(CGF, Ptr, Align),
4928
1
                             OverflowArea.getElementType(), Align);
4929
1
    }
4930
4931
1
    MemAddr = Builder.CreateElementBitCast(OverflowArea, DirectTy);
4932
4933
    // Increase the overflow area.
4934
1
    OverflowArea = Builder.CreateConstInBoundsByteGEP(OverflowArea, Size);
4935
1
    Builder.CreateStore(OverflowArea.getPointer(), OverflowAreaAddr);
4936
1
    CGF.EmitBranch(Cont);
4937
1
  }
4938
4939
1
  CGF.EmitBlock(Cont);
4940
4941
  // Merge the cases with a phi.
4942
1
  Address Result = emitMergePHI(CGF, RegAddr, UsingRegs, MemAddr, UsingOverflow,
4943
1
                                "vaarg.addr");
4944
4945
  // Load the pointer if the argument was passed indirectly.
4946
1
  if (isIndirect) {
4947
0
    Result = Address(Builder.CreateLoad(Result, "aggr"), ElementTy,
4948
0
                     getContext().getTypeAlignInChars(Ty));
4949
0
  }
4950
4951
1
  return Result;
4952
1
}
4953
4954
bool PPC32TargetCodeGenInfo::isStructReturnInRegABI(
4955
65
    const llvm::Triple &Triple, const CodeGenOptions &Opts) {
4956
65
  assert(Triple.isPPC32());
4957
4958
0
  switch (Opts.getStructReturnConvention()) {
4959
65
  case CodeGenOptions::SRCK_Default:
4960
65
    break;
4961
0
  case CodeGenOptions::SRCK_OnStack: // -maix-struct-return
4962
0
    return false;
4963
0
  case CodeGenOptions::SRCK_InRegs: // -msvr4-struct-return
4964
0
    return true;
4965
65
  }
4966
4967
65
  if (Triple.isOSBinFormatELF() && !Triple.isOSLinux())
4968
56
    return true;
4969
4970
9
  return false;
4971
65
}
4972
4973
bool
4974
PPC32TargetCodeGenInfo::initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
4975
1
                                                llvm::Value *Address) const {
4976
1
  return PPC_initDwarfEHRegSizeTable(CGF, Address, /*Is64Bit*/ false,
4977
1
                                     /*IsAIX*/ false);
4978
1
}
4979
4980
// PowerPC-64
4981
4982
namespace {
4983
/// PPC64_SVR4_ABIInfo - The 64-bit PowerPC ELF (SVR4) ABI information.
4984
class PPC64_SVR4_ABIInfo : public SwiftABIInfo {
4985
public:
4986
  enum ABIKind {
4987
    ELFv1 = 0,
4988
    ELFv2
4989
  };
4990
4991
private:
4992
  static const unsigned GPRBits = 64;
4993
  ABIKind Kind;
4994
  bool IsSoftFloatABI;
4995
4996
public:
4997
  PPC64_SVR4_ABIInfo(CodeGen::CodeGenTypes &CGT, ABIKind Kind,
4998
                     bool SoftFloatABI)
4999
2.54k
      : SwiftABIInfo(CGT), Kind(Kind), IsSoftFloatABI(SoftFloatABI) {}
5000
5001
  bool isPromotableTypeForABI(QualType Ty) const;
5002
  CharUnits getParamTypeAlignment(QualType Ty) const;
5003
5004
  ABIArgInfo classifyReturnType(QualType RetTy) const;
5005
  ABIArgInfo classifyArgumentType(QualType Ty) const;
5006
5007
  bool isHomogeneousAggregateBaseType(QualType Ty) const override;
5008
  bool isHomogeneousAggregateSmallEnough(const Type *Ty,
5009
                                         uint64_t Members) const override;
5010
5011
  // TODO: We can add more logic to computeInfo to improve performance.
5012
  // Example: For aggregate arguments that fit in a register, we could
5013
  // use getDirectInReg (as is done below for structs containing a single
5014
  // floating-point value) to avoid pushing them to memory on function
5015
  // entry.  This would require changing the logic in PPCISelLowering
5016
  // when lowering the parameters in the caller and args in the callee.
5017
13.5k
  void computeInfo(CGFunctionInfo &FI) const override {
5018
13.5k
    if (!getCXXABI().classifyReturnType(FI))
5019
13.5k
      FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
5020
26.8k
    for (auto &I : FI.arguments()) {
5021
      // We rely on the default argument classification for the most part.
5022
      // One exception:  An aggregate containing a single floating-point
5023
      // or vector item must be passed in a register if one is available.
5024
26.8k
      const Type *T = isSingleElementStruct(I.type, getContext());
5025
26.8k
      if (T) {
5026
23
        const BuiltinType *BT = T->getAs<BuiltinType>();
5027
23
        if ((T->isVectorType() && 
getContext().getTypeSize(T) == 1283
) ||
5028
23
            
(21
BT21
&&
BT->isFloatingPoint()18
)) {
5029
19
          QualType QT(T, 0);
5030
19
          I.info = ABIArgInfo::getDirectInReg(CGT.ConvertType(QT));
5031
19
          continue;
5032
19
        }
5033
23
      }
5034
26.8k
      I.info = classifyArgumentType(I.type);
5035
26.8k
    }
5036
13.5k
  }
5037
5038
  Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
5039
                    QualType Ty) const override;
5040
5041
  bool shouldPassIndirectlyForSwift(ArrayRef<llvm::Type*> scalars,
5042
0
                                    bool asReturnValue) const override {
5043
0
    return occupiesMoreThan(CGT, scalars, /*total*/ 4);
5044
0
  }
5045
5046
0
  bool isSwiftErrorInRegister() const override {
5047
0
    return false;
5048
0
  }
5049
};
5050
5051
class PPC64_SVR4_TargetCodeGenInfo : public TargetCodeGenInfo {
5052
5053
public:
5054
  PPC64_SVR4_TargetCodeGenInfo(CodeGenTypes &CGT,
5055
                               PPC64_SVR4_ABIInfo::ABIKind Kind,
5056
                               bool SoftFloatABI)
5057
      : TargetCodeGenInfo(
5058
2.54k
            std::make_unique<PPC64_SVR4_ABIInfo>(CGT, Kind, SoftFloatABI)) {}
5059
5060
1
  int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
5061
    // This is recovered from gcc output.
5062
1
    return 1; // r1 is the dedicated stack pointer
5063
1
  }
5064
5065
  bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
5066
                               llvm::Value *Address) const override;
5067
};
5068
5069
class PPC64TargetCodeGenInfo : public DefaultTargetCodeGenInfo {
5070
public:
5071
0
  PPC64TargetCodeGenInfo(CodeGenTypes &CGT) : DefaultTargetCodeGenInfo(CGT) {}
5072
5073
0
  int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
5074
    // This is recovered from gcc output.
5075
0
    return 1; // r1 is the dedicated stack pointer
5076
0
  }
5077
5078
  bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
5079
                               llvm::Value *Address) const override;
5080
};
5081
5082
}
5083
5084
// Return true if the ABI requires Ty to be passed sign- or zero-
5085
// extended to 64 bits.
5086
bool
5087
29.5k
PPC64_SVR4_ABIInfo::isPromotableTypeForABI(QualType Ty) const {
5088
  // Treat an enum type as its underlying type.
5089
29.5k
  if (const EnumType *EnumTy = Ty->getAs<EnumType>())
5090
0
    Ty = EnumTy->getDecl()->getIntegerType();
5091
5092
  // Promotable integer types are required to be promoted by the ABI.
5093
29.5k
  if (isPromotableIntegerTypeForABI(Ty))
5094
63
    return true;
5095
5096
  // In addition to the usual promotable integer types, we also need to
5097
  // extend all 32-bit types, since the ABI requires promotion to 64 bits.
5098
29.5k
  if (const BuiltinType *BT = Ty->getAs<BuiltinType>())
5099
12.5k
    switch (BT->getKind()) {
5100
5.26k
    case BuiltinType::Int:
5101
5.28k
    case BuiltinType::UInt:
5102
5.28k
      return true;
5103
7.28k
    default:
5104
7.28k
      break;
5105
12.5k
    }
5106
5107
24.2k
  if (const auto *EIT = Ty->getAs<BitIntType>())
5108
9
    if (EIT->getNumBits() < 64)
5109
2
      return true;
5110
5111
24.2k
  return false;
5112
24.2k
}
5113
5114
/// isAlignedParamType - Determine whether a type requires 16-byte or
5115
/// higher alignment in the parameter area.  Always returns at least 8.
5116
130
CharUnits PPC64_SVR4_ABIInfo::getParamTypeAlignment(QualType Ty) const {
5117
  // Complex types are passed just like their elements.
5118
130
  if (const ComplexType *CTy = Ty->getAs<ComplexType>())
5119
0
    Ty = CTy->getElementType();
5120
5121
159
  auto FloatUsesVector = [this](QualType Ty){
5122
159
    return Ty->isRealFloatingType() && &getContext().getFloatTypeSemantics(
5123
44
                                           Ty) == &llvm::APFloat::IEEEquad();
5124
159
  };
5125
5126
  // Only vector types of size 16 bytes need alignment (larger types are
5127
  // passed via reference, smaller types are not aligned).
5128
130
  if (Ty->isVectorType()) {
5129
0
    return CharUnits::fromQuantity(getContext().getTypeSize(Ty) == 128 ? 16 : 8);
5130
130
  } else if (FloatUsesVector(Ty)) {
5131
    // According to ABI document section 'Optional Save Areas': If extended
5132
    // precision floating-point values in IEEE BINARY 128 QUADRUPLE PRECISION
5133
    // format are supported, map them to a single quadword, quadword aligned.
5134
9
    return CharUnits::fromQuantity(16);
5135
9
  }
5136
5137
  // For single-element float/vector structs, we consider the whole type
5138
  // to have the same alignment requirements as its single element.
5139
121
  const Type *AlignAsType = nullptr;
5140
121
  const Type *EltType = isSingleElementStruct(Ty, getContext());
5141
121
  if (EltType) {
5142
6
    const BuiltinType *BT = EltType->getAs<BuiltinType>();
5143
6
    if ((EltType->isVectorType() && 
getContext().getTypeSize(EltType) == 1282
) ||
5144
6
        
(5
BT5
&&
BT->isFloatingPoint()4
))
5145
5
      AlignAsType = EltType;
5146
6
  }
5147
5148
  // Likewise for ELFv2 homogeneous aggregates.
5149
121
  const Type *Base = nullptr;
5150
121
  uint64_t Members = 0;
5151
121
  if (!AlignAsType && 
Kind == ELFv2116
&&
5152
121
      
isAggregateTypeForABI(Ty)88
&&
isHomogeneousAggregate(Ty, Base, Members)83
)
5153
43
    AlignAsType = Base;
5154
5155
  // With special case aggregates, only vector base types need alignment.
5156
121
  if (AlignAsType) {
5157
48
    bool UsesVector = AlignAsType->isVectorType() ||
5158
48
                      
FloatUsesVector(QualType(AlignAsType, 0))29
;
5159
48
    return CharUnits::fromQuantity(UsesVector ? 
1631
:
817
);