Coverage Report

Created: 2021-08-24 07:12

/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/CodeGen/TargetInfo.cpp
Line
Count
Source (jump to first uncovered line)
1
//===---- TargetInfo.cpp - Encapsulate target details -----------*- C++ -*-===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// These classes wrap the information about a call or function
10
// definition used to handle ABI compliancy.
11
//
12
//===----------------------------------------------------------------------===//
13
14
#include "TargetInfo.h"
15
#include "ABIInfo.h"
16
#include "CGBlocks.h"
17
#include "CGCXXABI.h"
18
#include "CGValue.h"
19
#include "CodeGenFunction.h"
20
#include "clang/AST/Attr.h"
21
#include "clang/AST/RecordLayout.h"
22
#include "clang/Basic/CodeGenOptions.h"
23
#include "clang/Basic/DiagnosticFrontend.h"
24
#include "clang/Basic/Builtins.h"
25
#include "clang/CodeGen/CGFunctionInfo.h"
26
#include "clang/CodeGen/SwiftCallingConv.h"
27
#include "llvm/ADT/SmallBitVector.h"
28
#include "llvm/ADT/StringExtras.h"
29
#include "llvm/ADT/StringSwitch.h"
30
#include "llvm/ADT/Triple.h"
31
#include "llvm/ADT/Twine.h"
32
#include "llvm/IR/DataLayout.h"
33
#include "llvm/IR/IntrinsicsNVPTX.h"
34
#include "llvm/IR/IntrinsicsS390.h"
35
#include "llvm/IR/Type.h"
36
#include "llvm/Support/raw_ostream.h"
37
#include <algorithm> // std::sort
38
39
using namespace clang;
40
using namespace CodeGen;
41
42
// Helper for coercing an aggregate argument or return value into an integer
43
// array of the same size (including padding) and alignment.  This alternate
44
// coercion happens only for the RenderScript ABI and can be removed after
45
// runtimes that rely on it are no longer supported.
46
//
47
// RenderScript assumes that the size of the argument / return value in the IR
48
// is the same as the size of the corresponding qualified type. This helper
49
// coerces the aggregate type into an array of the same size (including
50
// padding).  This coercion is used in lieu of expansion of struct members or
51
// other canonical coercions that return a coerced-type of larger size.
52
//
53
// Ty          - The argument / return value type
54
// Context     - The associated ASTContext
55
// LLVMContext - The associated LLVMContext
56
static ABIArgInfo coerceToIntArray(QualType Ty,
57
                                   ASTContext &Context,
58
19
                                   llvm::LLVMContext &LLVMContext) {
59
  // Alignment and Size are measured in bits.
60
19
  const uint64_t Size = Context.getTypeSize(Ty);
61
19
  const uint64_t Alignment = Context.getTypeAlign(Ty);
62
19
  llvm::Type *IntType = llvm::Type::getIntNTy(LLVMContext, Alignment);
63
19
  const uint64_t NumElements = (Size + Alignment - 1) / Alignment;
64
19
  return ABIArgInfo::getDirect(llvm::ArrayType::get(IntType, NumElements));
65
19
}
66
67
static void AssignToArrayRange(CodeGen::CGBuilderTy &Builder,
68
                               llvm::Value *Array,
69
                               llvm::Value *Value,
70
                               unsigned FirstIndex,
71
31
                               unsigned LastIndex) {
72
  // Alternatively, we could emit this as a loop in the source.
73
572
  for (unsigned I = FirstIndex; I <= LastIndex; 
++I541
) {
74
541
    llvm::Value *Cell =
75
541
        Builder.CreateConstInBoundsGEP1_32(Builder.getInt8Ty(), Array, I);
76
541
    Builder.CreateAlignedStore(Value, Cell, CharUnits::One());
77
541
  }
78
31
}
79
80
484k
static bool isAggregateTypeForABI(QualType T) {
81
484k
  return !CodeGenFunction::hasScalarEvaluationKind(T) ||
82
484k
         
T->isMemberFunctionPointerType()477k
;
83
484k
}
84
85
ABIArgInfo ABIInfo::getNaturalAlignIndirect(QualType Ty, bool ByVal,
86
                                            bool Realign,
87
3.44k
                                            llvm::Type *Padding) const {
88
3.44k
  return ABIArgInfo::getIndirect(getContext().getTypeAlignInChars(Ty), ByVal,
89
3.44k
                                 Realign, Padding);
90
3.44k
}
91
92
ABIArgInfo
93
28
ABIInfo::getNaturalAlignIndirectInReg(QualType Ty, bool Realign) const {
94
28
  return ABIArgInfo::getIndirectInReg(getContext().getTypeAlignInChars(Ty),
95
28
                                      /*ByVal*/ false, Realign);
96
28
}
97
98
Address ABIInfo::EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr,
99
0
                             QualType Ty) const {
100
0
  return Address::invalid();
101
0
}
102
103
285k
bool ABIInfo::isPromotableIntegerTypeForABI(QualType Ty) const {
104
285k
  if (Ty->isPromotableIntegerType())
105
13.0k
    return true;
106
107
272k
  if (const auto *EIT = Ty->getAs<ExtIntType>())
108
274
    if (EIT->getNumBits() < getContext().getTypeSize(getContext().IntTy))
109
96
      return true;
110
111
272k
  return false;
112
272k
}
113
114
45.8k
ABIInfo::~ABIInfo() {}
115
116
/// Does the given lowering require more than the given number of
117
/// registers when expanded?
118
///
119
/// This is intended to be the basis of a reasonable basic implementation
120
/// of should{Pass,Return}IndirectlyForSwift.
121
///
122
/// For most targets, a limit of four total registers is reasonable; this
123
/// limits the amount of code required in order to move around the value
124
/// in case it wasn't produced immediately prior to the call by the caller
125
/// (or wasn't produced in exactly the right registers) or isn't used
126
/// immediately within the callee.  But some targets may need to further
127
/// limit the register count due to an inability to support that many
128
/// return registers.
129
static bool occupiesMoreThan(CodeGenTypes &cgt,
130
                             ArrayRef<llvm::Type*> scalarTypes,
131
1.05k
                             unsigned maxAllRegisters) {
132
1.05k
  unsigned intCount = 0, fpCount = 0;
133
4.18k
  for (llvm::Type *type : scalarTypes) {
134
4.18k
    if (type->isPointerTy()) {
135
0
      intCount++;
136
4.18k
    } else if (auto intTy = dyn_cast<llvm::IntegerType>(type)) {
137
2.30k
      auto ptrWidth = cgt.getTarget().getPointerWidth(0);
138
2.30k
      intCount += (intTy->getBitWidth() + ptrWidth - 1) / ptrWidth;
139
2.30k
    } else {
140
1.88k
      assert(type->isVectorTy() || type->isFloatingPointTy());
141
0
      fpCount++;
142
1.88k
    }
143
4.18k
  }
144
145
1.05k
  return (intCount + fpCount > maxAllRegisters);
146
1.05k
}
147
148
bool SwiftABIInfo::isLegalVectorTypeForSwift(CharUnits vectorSize,
149
                                             llvm::Type *eltTy,
150
440
                                             unsigned numElts) const {
151
  // The default implementation of this assumes that the target guarantees
152
  // 128-bit SIMD support but nothing more.
153
440
  return (vectorSize.getQuantity() > 8 && 
vectorSize.getQuantity() <= 16432
);
154
440
}
155
156
static CGCXXABI::RecordArgABI getRecordArgABI(const RecordType *RT,
157
19.3k
                                              CGCXXABI &CXXABI) {
158
19.3k
  const CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(RT->getDecl());
159
19.3k
  if (!RD) {
160
4.00k
    if (!RT->getDecl()->canPassInRegisters())
161
16
      return CGCXXABI::RAA_Indirect;
162
3.99k
    return CGCXXABI::RAA_Default;
163
4.00k
  }
164
15.3k
  return CXXABI.getRecordArgABI(RD);
165
19.3k
}
166
167
static CGCXXABI::RecordArgABI getRecordArgABI(QualType T,
168
306k
                                              CGCXXABI &CXXABI) {
169
306k
  const RecordType *RT = T->getAs<RecordType>();
170
306k
  if (!RT)
171
301k
    return CGCXXABI::RAA_Default;
172
4.97k
  return getRecordArgABI(RT, CXXABI);
173
306k
}
174
175
static bool classifyReturnType(const CGCXXABI &CXXABI, CGFunctionInfo &FI,
176
213k
                               const ABIInfo &Info) {
177
213k
  QualType Ty = FI.getReturnType();
178
179
213k
  if (const auto *RT = Ty->getAs<RecordType>())
180
5.62k
    if (!isa<CXXRecordDecl>(RT->getDecl()) &&
181
5.62k
        
!RT->getDecl()->canPassInRegisters()1.29k
) {
182
12
      FI.getReturnInfo() = Info.getNaturalAlignIndirect(Ty);
183
12
      return true;
184
12
    }
185
186
213k
  return CXXABI.classifyReturnType(FI);
187
213k
}
188
189
/// Pass transparent unions as if they were the type of the first element. Sema
190
/// should ensure that all elements of the union have the same "machine type".
191
729k
static QualType useFirstFieldIfTransparentUnion(QualType Ty) {
192
729k
  if (const RecordType *UT = Ty->getAsUnionType()) {
193
137
    const RecordDecl *UD = UT->getDecl();
194
137
    if (UD->hasAttr<TransparentUnionAttr>()) {
195
22
      assert(!UD->field_empty() && "sema created an empty transparent union");
196
0
      return UD->field_begin()->getType();
197
22
    }
198
137
  }
199
729k
  return Ty;
200
729k
}
201
202
608k
CGCXXABI &ABIInfo::getCXXABI() const {
203
608k
  return CGT.getCXXABI();
204
608k
}
205
206
1.33M
ASTContext &ABIInfo::getContext() const {
207
1.33M
  return CGT.getContext();
208
1.33M
}
209
210
19.0k
llvm::LLVMContext &ABIInfo::getVMContext() const {
211
19.0k
  return CGT.getLLVMContext();
212
19.0k
}
213
214
45.8k
const llvm::DataLayout &ABIInfo::getDataLayout() const {
215
45.8k
  return CGT.getDataLayout();
216
45.8k
}
217
218
33.9k
const TargetInfo &ABIInfo::getTarget() const {
219
33.9k
  return CGT.getTarget();
220
33.9k
}
221
222
1.80k
const CodeGenOptions &ABIInfo::getCodeGenOpts() const {
223
1.80k
  return CGT.getCodeGenOpts();
224
1.80k
}
225
226
5.04k
bool ABIInfo::isAndroid() const { return getTarget().getTriple().isAndroid(); }
227
228
0
bool ABIInfo::isHomogeneousAggregateBaseType(QualType Ty) const {
229
0
  return false;
230
0
}
231
232
bool ABIInfo::isHomogeneousAggregateSmallEnough(const Type *Base,
233
0
                                                uint64_t Members) const {
234
0
  return false;
235
0
}
236
237
0
LLVM_DUMP_METHOD void ABIArgInfo::dump() const {
238
0
  raw_ostream &OS = llvm::errs();
239
0
  OS << "(ABIArgInfo Kind=";
240
0
  switch (TheKind) {
241
0
  case Direct:
242
0
    OS << "Direct Type=";
243
0
    if (llvm::Type *Ty = getCoerceToType())
244
0
      Ty->print(OS);
245
0
    else
246
0
      OS << "null";
247
0
    break;
248
0
  case Extend:
249
0
    OS << "Extend";
250
0
    break;
251
0
  case Ignore:
252
0
    OS << "Ignore";
253
0
    break;
254
0
  case InAlloca:
255
0
    OS << "InAlloca Offset=" << getInAllocaFieldIndex();
256
0
    break;
257
0
  case Indirect:
258
0
    OS << "Indirect Align=" << getIndirectAlign().getQuantity()
259
0
       << " ByVal=" << getIndirectByVal()
260
0
       << " Realign=" << getIndirectRealign();
261
0
    break;
262
0
  case IndirectAliased:
263
0
    OS << "Indirect Align=" << getIndirectAlign().getQuantity()
264
0
       << " AadrSpace=" << getIndirectAddrSpace()
265
0
       << " Realign=" << getIndirectRealign();
266
0
    break;
267
0
  case Expand:
268
0
    OS << "Expand";
269
0
    break;
270
0
  case CoerceAndExpand:
271
0
    OS << "CoerceAndExpand Type=";
272
0
    getCoerceAndExpandType()->print(OS);
273
0
    break;
274
0
  }
275
0
  OS << ")\n";
276
0
}
277
278
// Dynamically round a pointer up to a multiple of the given alignment.
279
static llvm::Value *emitRoundPointerUpToAlignment(CodeGenFunction &CGF,
280
                                                  llvm::Value *Ptr,
281
115
                                                  CharUnits Align) {
282
115
  llvm::Value *PtrAsInt = Ptr;
283
  // OverflowArgArea = (OverflowArgArea + Align - 1) & -Align;
284
115
  PtrAsInt = CGF.Builder.CreatePtrToInt(PtrAsInt, CGF.IntPtrTy);
285
115
  PtrAsInt = CGF.Builder.CreateAdd(PtrAsInt,
286
115
        llvm::ConstantInt::get(CGF.IntPtrTy, Align.getQuantity() - 1));
287
115
  PtrAsInt = CGF.Builder.CreateAnd(PtrAsInt,
288
115
           llvm::ConstantInt::get(CGF.IntPtrTy, -Align.getQuantity()));
289
115
  PtrAsInt = CGF.Builder.CreateIntToPtr(PtrAsInt,
290
115
                                        Ptr->getType(),
291
115
                                        Ptr->getName() + ".aligned");
292
115
  return PtrAsInt;
293
115
}
294
295
/// Emit va_arg for a platform using the common void* representation,
296
/// where arguments are simply emitted in an array of slots on the stack.
297
///
298
/// This version implements the core direct-value passing rules.
299
///
300
/// \param SlotSize - The size and alignment of a stack slot.
301
///   Each argument will be allocated to a multiple of this number of
302
///   slots, and all the slots will be aligned to this value.
303
/// \param AllowHigherAlign - The slot alignment is not a cap;
304
///   an argument type with an alignment greater than the slot size
305
///   will be emitted on a higher-alignment address, potentially
306
///   leaving one or more empty slots behind as padding.  If this
307
///   is false, the returned address might be less-aligned than
308
///   DirectAlign.
309
static Address emitVoidPtrDirectVAArg(CodeGenFunction &CGF,
310
                                      Address VAListAddr,
311
                                      llvm::Type *DirectTy,
312
                                      CharUnits DirectSize,
313
                                      CharUnits DirectAlign,
314
                                      CharUnits SlotSize,
315
304
                                      bool AllowHigherAlign) {
316
  // Cast the element type to i8* if necessary.  Some platforms define
317
  // va_list as a struct containing an i8* instead of just an i8*.
318
304
  if (VAListAddr.getElementType() != CGF.Int8PtrTy)
319
59
    VAListAddr = CGF.Builder.CreateElementBitCast(VAListAddr, CGF.Int8PtrTy);
320
321
304
  llvm::Value *Ptr = CGF.Builder.CreateLoad(VAListAddr, "argp.cur");
322
323
  // If the CC aligns values higher than the slot size, do so if needed.
324
304
  Address Addr = Address::invalid();
325
304
  if (AllowHigherAlign && 
DirectAlign > SlotSize263
) {
326
95
    Addr = Address(emitRoundPointerUpToAlignment(CGF, Ptr, DirectAlign),
327
95
                                                 DirectAlign);
328
209
  } else {
329
209
    Addr = Address(Ptr, SlotSize);
330
209
  }
331
332
  // Advance the pointer past the argument, then store that back.
333
304
  CharUnits FullDirectSize = DirectSize.alignTo(SlotSize);
334
304
  Address NextPtr =
335
304
      CGF.Builder.CreateConstInBoundsByteGEP(Addr, FullDirectSize, "argp.next");
336
304
  CGF.Builder.CreateStore(NextPtr.getPointer(), VAListAddr);
337
338
  // If the argument is smaller than a slot, and this is a big-endian
339
  // target, the argument will be right-adjusted in its slot.
340
304
  if (DirectSize < SlotSize && 
CGF.CGM.getDataLayout().isBigEndian()39
&&
341
304
      
!DirectTy->isStructTy()8
) {
342
2
    Addr = CGF.Builder.CreateConstInBoundsByteGEP(Addr, SlotSize - DirectSize);
343
2
  }
344
345
304
  Addr = CGF.Builder.CreateElementBitCast(Addr, DirectTy);
346
304
  return Addr;
347
304
}
348
349
/// Emit va_arg for a platform using the common void* representation,
350
/// where arguments are simply emitted in an array of slots on the stack.
351
///
352
/// \param IsIndirect - Values of this type are passed indirectly.
353
/// \param ValueInfo - The size and alignment of this type, generally
354
///   computed with getContext().getTypeInfoInChars(ValueTy).
355
/// \param SlotSizeAndAlign - The size and alignment of a stack slot.
356
///   Each argument will be allocated to a multiple of this number of
357
///   slots, and all the slots will be aligned to this value.
358
/// \param AllowHigherAlign - The slot alignment is not a cap;
359
///   an argument type with an alignment greater than the slot size
360
///   will be emitted on a higher-alignment address, potentially
361
///   leaving one or more empty slots behind as padding.
362
static Address emitVoidPtrVAArg(CodeGenFunction &CGF, Address VAListAddr,
363
                                QualType ValueTy, bool IsIndirect,
364
                                TypeInfoChars ValueInfo,
365
                                CharUnits SlotSizeAndAlign,
366
296
                                bool AllowHigherAlign) {
367
  // The size and alignment of the value that was passed directly.
368
296
  CharUnits DirectSize, DirectAlign;
369
296
  if (IsIndirect) {
370
48
    DirectSize = CGF.getPointerSize();
371
48
    DirectAlign = CGF.getPointerAlign();
372
248
  } else {
373
248
    DirectSize = ValueInfo.Width;
374
248
    DirectAlign = ValueInfo.Align;
375
248
  }
376
377
  // Cast the address we've calculated to the right type.
378
296
  llvm::Type *DirectTy = CGF.ConvertTypeForMem(ValueTy);
379
296
  if (IsIndirect)
380
48
    DirectTy = DirectTy->getPointerTo(0);
381
382
296
  Address Addr = emitVoidPtrDirectVAArg(CGF, VAListAddr, DirectTy,
383
296
                                        DirectSize, DirectAlign,
384
296
                                        SlotSizeAndAlign,
385
296
                                        AllowHigherAlign);
386
387
296
  if (IsIndirect) {
388
48
    Addr = Address(CGF.Builder.CreateLoad(Addr), ValueInfo.Align);
389
48
  }
390
391
296
  return Addr;
392
393
296
}
394
395
static Address emitMergePHI(CodeGenFunction &CGF,
396
                            Address Addr1, llvm::BasicBlock *Block1,
397
                            Address Addr2, llvm::BasicBlock *Block2,
398
392
                            const llvm::Twine &Name = "") {
399
392
  assert(Addr1.getType() == Addr2.getType());
400
0
  llvm::PHINode *PHI = CGF.Builder.CreatePHI(Addr1.getType(), 2, Name);
401
392
  PHI->addIncoming(Addr1.getPointer(), Block1);
402
392
  PHI->addIncoming(Addr2.getPointer(), Block2);
403
392
  CharUnits Align = std::min(Addr1.getAlignment(), Addr2.getAlignment());
404
392
  return Address(PHI, Align);
405
392
}
406
407
45.6k
TargetCodeGenInfo::~TargetCodeGenInfo() = default;
408
409
// If someone can figure out a general rule for this, that would be great.
410
// It's probably just doomed to be platform-dependent, though.
411
5
unsigned TargetCodeGenInfo::getSizeOfUnwindException() const {
412
  // Verified for:
413
  //   x86-64     FreeBSD, Linux, Darwin
414
  //   x86-32     FreeBSD, Linux, Darwin
415
  //   PowerPC    Linux, Darwin
416
  //   ARM        Darwin (*not* EABI)
417
  //   AArch64    Linux
418
5
  return 32;
419
5
}
420
421
bool TargetCodeGenInfo::isNoProtoCallVariadic(const CallArgList &args,
422
289
                                     const FunctionNoProtoType *fnType) const {
423
  // The following conventions are known to require this to be false:
424
  //   x86_stdcall
425
  //   MIPS
426
  // For everything else, we just prefer false unless we opt out.
427
289
  return false;
428
289
}
429
430
void
431
TargetCodeGenInfo::getDependentLibraryOption(llvm::StringRef Lib,
432
2
                                             llvm::SmallString<24> &Opt) const {
433
  // This assumes the user is passing a library name like "rt" instead of a
434
  // filename like "librt.a/so", and that they don't care whether it's static or
435
  // dynamic.
436
2
  Opt = "-l";
437
2
  Opt += Lib;
438
2
}
439
440
54
unsigned TargetCodeGenInfo::getOpenCLKernelCallingConv() const {
441
  // OpenCL kernels are called via an explicit runtime API with arguments
442
  // set with clSetKernelArg(), not as normal sub-functions.
443
  // Return SPIR_KERNEL by default as the kernel calling convention to
444
  // ensure the fingerprint is fixed such way that each OpenCL argument
445
  // gets one matching argument in the produced kernel function argument
446
  // list to enable feasible implementation of clSetKernelArg() with
447
  // aggregates etc. In case we would use the default C calling conv here,
448
  // clSetKernelArg() might break depending on the target-specific
449
  // conventions; different targets might split structs passed as values
450
  // to multiple function arguments etc.
451
54
  return llvm::CallingConv::SPIR_KERNEL;
452
54
}
453
454
llvm::Constant *TargetCodeGenInfo::getNullPointer(const CodeGen::CodeGenModule &CGM,
455
34.8k
    llvm::PointerType *T, QualType QT) const {
456
34.8k
  return llvm::ConstantPointerNull::get(T);
457
34.8k
}
458
459
LangAS TargetCodeGenInfo::getGlobalVarAddressSpace(CodeGenModule &CGM,
460
74.4k
                                                   const VarDecl *D) const {
461
74.4k
  assert(!CGM.getLangOpts().OpenCL &&
462
74.4k
         !(CGM.getLangOpts().CUDA && CGM.getLangOpts().CUDAIsDevice) &&
463
74.4k
         "Address space agnostic languages only");
464
74.4k
  return D ? 
D->getType().getAddressSpace()72.4k
:
LangAS::Default1.95k
;
465
74.4k
}
466
467
llvm::Value *TargetCodeGenInfo::performAddrSpaceCast(
468
    CodeGen::CodeGenFunction &CGF, llvm::Value *Src, LangAS SrcAddr,
469
7.17k
    LangAS DestAddr, llvm::Type *DestTy, bool isNonNull) const {
470
  // Since target may map different address spaces in AST to the same address
471
  // space, an address space conversion may end up as a bitcast.
472
7.17k
  if (auto *C = dyn_cast<llvm::Constant>(Src))
473
51
    return performAddrSpaceCast(CGF.CGM, C, SrcAddr, DestAddr, DestTy);
474
  // Try to preserve the source's name to make IR more readable.
475
7.12k
  return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
476
7.12k
      Src, DestTy, Src->hasName() ? 
Src->getName() + ".ascast"6.92k
:
""206
);
477
7.17k
}
478
479
llvm::Constant *
480
TargetCodeGenInfo::performAddrSpaceCast(CodeGenModule &CGM, llvm::Constant *Src,
481
                                        LangAS SrcAddr, LangAS DestAddr,
482
518
                                        llvm::Type *DestTy) const {
483
  // Since target may map different address spaces in AST to the same address
484
  // space, an address space conversion may end up as a bitcast.
485
518
  return llvm::ConstantExpr::getPointerCast(Src, DestTy);
486
518
}
487
488
llvm::SyncScope::ID
489
TargetCodeGenInfo::getLLVMSyncScopeID(const LangOptions &LangOpts,
490
                                      SyncScope Scope,
491
                                      llvm::AtomicOrdering Ordering,
492
0
                                      llvm::LLVMContext &Ctx) const {
493
0
  return Ctx.getOrInsertSyncScopeID(""); /* default sync scope */
494
0
}
495
496
static bool isEmptyRecord(ASTContext &Context, QualType T, bool AllowArrays);
497
498
/// isEmptyField - Return true iff a the field is "empty", that is it
499
/// is an unnamed bit-field or an (array of) empty record(s).
500
static bool isEmptyField(ASTContext &Context, const FieldDecl *FD,
501
5.61k
                         bool AllowArrays) {
502
5.61k
  if (FD->isUnnamedBitfield())
503
127
    return true;
504
505
5.49k
  QualType FT = FD->getType();
506
507
  // Constant arrays of empty records count as empty, strip them off.
508
  // Constant arrays of zero length always count as empty.
509
5.49k
  bool WasArray = false;
510
5.49k
  if (AllowArrays)
511
7.21k
    
while (const ConstantArrayType *5.31k
AT = Context.getAsConstantArrayType(FT)) {
512
1.90k
      if (AT->getSize() == 0)
513
11
        return true;
514
1.89k
      FT = AT->getElementType();
515
      // The [[no_unique_address]] special case below does not apply to
516
      // arrays of C++ empty records, so we need to remember this fact.
517
1.89k
      WasArray = true;
518
1.89k
    }
519
520
5.47k
  const RecordType *RT = FT->getAs<RecordType>();
521
5.47k
  if (!RT)
522
5.03k
    return false;
523
524
  // C++ record fields are never empty, at least in the Itanium ABI.
525
  //
526
  // FIXME: We should use a predicate for whether this behavior is true in the
527
  // current ABI.
528
  //
529
  // The exception to the above rule are fields marked with the
530
  // [[no_unique_address]] attribute (since C++20).  Those do count as empty
531
  // according to the Itanium ABI.  The exception applies only to records,
532
  // not arrays of records, so we must also check whether we stripped off an
533
  // array type above.
534
443
  if (isa<CXXRecordDecl>(RT->getDecl()) &&
535
443
      
(85
WasArray85
||
!FD->hasAttr<NoUniqueAddressAttr>()68
))
536
78
    return false;
537
538
365
  return isEmptyRecord(Context, FT, AllowArrays);
539
443
}
540
541
/// isEmptyRecord - Return true iff a structure contains only empty
542
/// fields. Note that a structure with a flexible array member is not
543
/// considered empty.
544
304k
static bool isEmptyRecord(ASTContext &Context, QualType T, bool AllowArrays) {
545
304k
  const RecordType *RT = T->getAs<RecordType>();
546
304k
  if (!RT)
547
300k
    return false;
548
4.78k
  const RecordDecl *RD = RT->getDecl();
549
4.78k
  if (RD->hasFlexibleArrayMember())
550
14
    return false;
551
552
  // If this is a C++ record, check the bases first.
553
4.77k
  if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD))
554
1.00k
    for (const auto &I : CXXRD->bases())
555
115
      if (!isEmptyRecord(Context, I.getType(), true))
556
107
        return false;
557
558
4.66k
  for (const auto *I : RD->fields())
559
4.32k
    if (!isEmptyField(Context, I, AllowArrays))
560
4.04k
      return false;
561
623
  return true;
562
4.66k
}
563
564
/// isSingleElementStruct - Determine if a structure is a "single
565
/// element struct", i.e. it has exactly one non-empty field or
566
/// exactly one field which is itself a single element
567
/// struct. Structures with flexible array members are never
568
/// considered single element structs.
569
///
570
/// \return The field declaration for the single non-empty field, if
571
/// it exists.
572
71.2k
static const Type *isSingleElementStruct(QualType T, ASTContext &Context) {
573
71.2k
  const RecordType *RT = T->getAs<RecordType>();
574
71.2k
  if (!RT)
575
70.3k
    return nullptr;
576
577
863
  const RecordDecl *RD = RT->getDecl();
578
863
  if (RD->hasFlexibleArrayMember())
579
4
    return nullptr;
580
581
859
  const Type *Found = nullptr;
582
583
  // If this is a C++ record, check the bases first.
584
859
  if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
585
273
    for (const auto &I : CXXRD->bases()) {
586
      // Ignore empty records.
587
49
      if (isEmptyRecord(Context, I.getType(), true))
588
6
        continue;
589
590
      // If we already found an element then this isn't a single-element struct.
591
43
      if (Found)
592
11
        return nullptr;
593
594
      // If this is non-empty and not a single element struct, the composite
595
      // cannot be a single element struct.
596
32
      Found = isSingleElementStruct(I.getType(), Context);
597
32
      if (!Found)
598
9
        return nullptr;
599
32
    }
600
273
  }
601
602
  // Check for single element.
603
1.17k
  
for (const auto *FD : RD->fields())839
{
604
1.17k
    QualType FT = FD->getType();
605
606
    // Ignore empty fields.
607
1.17k
    if (isEmptyField(Context, FD, true))
608
19
      continue;
609
610
    // If we already found an element then this isn't a single-element
611
    // struct.
612
1.15k
    if (Found)
613
355
      return nullptr;
614
615
    // Treat single element arrays as the element.
616
816
    
while (const ConstantArrayType *802
AT = Context.getAsConstantArrayType(FT)) {
617
192
      if (AT->getSize().getZExtValue() != 1)
618
178
        break;
619
14
      FT = AT->getElementType();
620
14
    }
621
622
802
    if (!isAggregateTypeForABI(FT)) {
623
594
      Found = FT.getTypePtr();
624
594
    } else {
625
208
      Found = isSingleElementStruct(FT, Context);
626
208
      if (!Found)
627
198
        return nullptr;
628
208
    }
629
802
  }
630
631
  // We don't consider a struct a single-element struct if it has
632
  // padding beyond the element type.
633
286
  if (Found && 
Context.getTypeSize(Found) != Context.getTypeSize(T)261
)
634
3
    return nullptr;
635
636
283
  return Found;
637
286
}
638
639
namespace {
640
Address EmitVAArgInstr(CodeGenFunction &CGF, Address VAListAddr, QualType Ty,
641
8
                       const ABIArgInfo &AI) {
642
  // This default implementation defers to the llvm backend's va_arg
643
  // instruction. It can handle only passing arguments directly
644
  // (typically only handled in the backend for primitive types), or
645
  // aggregates passed indirectly by pointer (NOTE: if the "byval"
646
  // flag has ABI impact in the callee, this implementation cannot
647
  // work.)
648
649
  // Only a few cases are covered here at the moment -- those needed
650
  // by the default abi.
651
8
  llvm::Value *Val;
652
653
8
  if (AI.isIndirect()) {
654
1
    assert(!AI.getPaddingType() &&
655
1
           "Unexpected PaddingType seen in arginfo in generic VAArg emitter!");
656
0
    assert(
657
1
        !AI.getIndirectRealign() &&
658
1
        "Unexpected IndirectRealign seen in arginfo in generic VAArg emitter!");
659
660
0
    auto TyInfo = CGF.getContext().getTypeInfoInChars(Ty);
661
1
    CharUnits TyAlignForABI = TyInfo.Align;
662
663
1
    llvm::Type *BaseTy =
664
1
        llvm::PointerType::getUnqual(CGF.ConvertTypeForMem(Ty));
665
1
    llvm::Value *Addr =
666
1
        CGF.Builder.CreateVAArg(VAListAddr.getPointer(), BaseTy);
667
1
    return Address(Addr, TyAlignForABI);
668
7
  } else {
669
7
    assert((AI.isDirect() || AI.isExtend()) &&
670
7
           "Unexpected ArgInfo Kind in generic VAArg emitter!");
671
672
0
    assert(!AI.getInReg() &&
673
7
           "Unexpected InReg seen in arginfo in generic VAArg emitter!");
674
0
    assert(!AI.getPaddingType() &&
675
7
           "Unexpected PaddingType seen in arginfo in generic VAArg emitter!");
676
0
    assert(!AI.getDirectOffset() &&
677
7
           "Unexpected DirectOffset seen in arginfo in generic VAArg emitter!");
678
0
    assert(!AI.getCoerceToType() &&
679
7
           "Unexpected CoerceToType seen in arginfo in generic VAArg emitter!");
680
681
0
    Address Temp = CGF.CreateMemTemp(Ty, "varet");
682
7
    Val = CGF.Builder.CreateVAArg(VAListAddr.getPointer(), CGF.ConvertType(Ty));
683
7
    CGF.Builder.CreateStore(Val, Temp);
684
7
    return Temp;
685
7
  }
686
8
}
687
688
/// DefaultABIInfo - The default implementation for ABI specific
689
/// details. This implementation provides information which results in
690
/// self-consistent and sensible LLVM IR generation, but does not
691
/// conform to any particular ABI.
692
class DefaultABIInfo : public ABIInfo {
693
public:
694
1.25k
  DefaultABIInfo(CodeGen::CodeGenTypes &CGT) : ABIInfo(CGT) {}
695
696
  ABIArgInfo classifyReturnType(QualType RetTy) const;
697
  ABIArgInfo classifyArgumentType(QualType RetTy) const;
698
699
614
  void computeInfo(CGFunctionInfo &FI) const override {
700
614
    if (!getCXXABI().classifyReturnType(FI))
701
613
      FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
702
614
    for (auto &I : FI.arguments())
703
820
      I.info = classifyArgumentType(I.type);
704
614
  }
705
706
  Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
707
3
                    QualType Ty) const override {
708
3
    return EmitVAArgInstr(CGF, VAListAddr, Ty, classifyArgumentType(Ty));
709
3
  }
710
};
711
712
class DefaultTargetCodeGenInfo : public TargetCodeGenInfo {
713
public:
714
  DefaultTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT)
715
30
      : TargetCodeGenInfo(std::make_unique<DefaultABIInfo>(CGT)) {}
716
};
717
718
4.74k
ABIArgInfo DefaultABIInfo::classifyArgumentType(QualType Ty) const {
719
4.74k
  Ty = useFirstFieldIfTransparentUnion(Ty);
720
721
4.74k
  if (isAggregateTypeForABI(Ty)) {
722
    // Records with non-trivial destructors/copy-constructors should not be
723
    // passed by value.
724
134
    if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))
725
2
      return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
726
727
132
    return getNaturalAlignIndirect(Ty);
728
134
  }
729
730
  // Treat an enum type as its underlying type.
731
4.61k
  if (const EnumType *EnumTy = Ty->getAs<EnumType>())
732
9
    Ty = EnumTy->getDecl()->getIntegerType();
733
734
4.61k
  ASTContext &Context = getContext();
735
4.61k
  if (const auto *EIT = Ty->getAs<ExtIntType>())
736
64
    if (EIT->getNumBits() >
737
64
        Context.getTypeSize(Context.getTargetInfo().hasInt128Type()
738
64
                                ? 
Context.Int128Ty16
739
64
                                : 
Context.LongLongTy48
))
740
28
      return getNaturalAlignIndirect(Ty);
741
742
4.58k
  return (isPromotableIntegerTypeForABI(Ty) ? 
ABIArgInfo::getExtend(Ty)218
743
4.58k
                                            : 
ABIArgInfo::getDirect()4.36k
);
744
4.61k
}
745
746
3.29k
ABIArgInfo DefaultABIInfo::classifyReturnType(QualType RetTy) const {
747
3.29k
  if (RetTy->isVoidType())
748
1.59k
    return ABIArgInfo::getIgnore();
749
750
1.69k
  if (isAggregateTypeForABI(RetTy))
751
123
    return getNaturalAlignIndirect(RetTy);
752
753
  // Treat an enum type as its underlying type.
754
1.57k
  if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
755
0
    RetTy = EnumTy->getDecl()->getIntegerType();
756
757
1.57k
  if (const auto *EIT = RetTy->getAs<ExtIntType>())
758
67
    if (EIT->getNumBits() >
759
67
        getContext().getTypeSize(getContext().getTargetInfo().hasInt128Type()
760
67
                                     ? 
getContext().Int128Ty14
761
67
                                     : 
getContext().LongLongTy53
))
762
43
      return getNaturalAlignIndirect(RetTy);
763
764
1.53k
  return (isPromotableIntegerTypeForABI(RetTy) ? 
ABIArgInfo::getExtend(RetTy)97
765
1.53k
                                               : 
ABIArgInfo::getDirect()1.43k
);
766
1.57k
}
767
768
//===----------------------------------------------------------------------===//
769
// WebAssembly ABI Implementation
770
//
771
// This is a very simple ABI that relies a lot on DefaultABIInfo.
772
//===----------------------------------------------------------------------===//
773
774
class WebAssemblyABIInfo final : public SwiftABIInfo {
775
public:
776
  enum ABIKind {
777
    MVP = 0,
778
    ExperimentalMV = 1,
779
  };
780
781
private:
782
  DefaultABIInfo defaultInfo;
783
  ABIKind Kind;
784
785
public:
786
  explicit WebAssemblyABIInfo(CodeGen::CodeGenTypes &CGT, ABIKind Kind)
787
44
      : SwiftABIInfo(CGT), defaultInfo(CGT), Kind(Kind) {}
788
789
private:
790
  ABIArgInfo classifyReturnType(QualType RetTy) const;
791
  ABIArgInfo classifyArgumentType(QualType Ty) const;
792
793
  // DefaultABIInfo's classifyReturnType and classifyArgumentType are
794
  // non-virtual, but computeInfo and EmitVAArg are virtual, so we
795
  // overload them.
796
426
  void computeInfo(CGFunctionInfo &FI) const override {
797
426
    if (!getCXXABI().classifyReturnType(FI))
798
418
      FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
799
426
    for (auto &Arg : FI.arguments())
800
500
      Arg.info = classifyArgumentType(Arg.type);
801
426
  }
802
803
  Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
804
                    QualType Ty) const override;
805
806
  bool shouldPassIndirectlyForSwift(ArrayRef<llvm::Type*> scalars,
807
0
                                    bool asReturnValue) const override {
808
0
    return occupiesMoreThan(CGT, scalars, /*total*/ 4);
809
0
  }
810
811
0
  bool isSwiftErrorInRegister() const override {
812
0
    return false;
813
0
  }
814
};
815
816
class WebAssemblyTargetCodeGenInfo final : public TargetCodeGenInfo {
817
public:
818
  explicit WebAssemblyTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT,
819
                                        WebAssemblyABIInfo::ABIKind K)
820
44
      : TargetCodeGenInfo(std::make_unique<WebAssemblyABIInfo>(CGT, K)) {}
821
822
  void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
823
2.41k
                           CodeGen::CodeGenModule &CGM) const override {
824
2.41k
    TargetCodeGenInfo::setTargetAttributes(D, GV, CGM);
825
2.41k
    if (const auto *FD = dyn_cast_or_null<FunctionDecl>(D)) {
826
2.26k
      if (const auto *Attr = FD->getAttr<WebAssemblyImportModuleAttr>()) {
827
1
        llvm::Function *Fn = cast<llvm::Function>(GV);
828
1
        llvm::AttrBuilder B;
829
1
        B.addAttribute("wasm-import-module", Attr->getImportModule());
830
1
        Fn->addFnAttrs(B);
831
1
      }
832
2.26k
      if (const auto *Attr = FD->getAttr<WebAssemblyImportNameAttr>()) {
833
1
        llvm::Function *Fn = cast<llvm::Function>(GV);
834
1
        llvm::AttrBuilder B;
835
1
        B.addAttribute("wasm-import-name", Attr->getImportName());
836
1
        Fn->addFnAttrs(B);
837
1
      }
838
2.26k
      if (const auto *Attr = FD->getAttr<WebAssemblyExportNameAttr>()) {
839
2
        llvm::Function *Fn = cast<llvm::Function>(GV);
840
2
        llvm::AttrBuilder B;
841
2
        B.addAttribute("wasm-export-name", Attr->getExportName());
842
2
        Fn->addFnAttrs(B);
843
2
      }
844
2.26k
    }
845
846
2.41k
    if (auto *FD = dyn_cast_or_null<FunctionDecl>(D)) {
847
2.26k
      llvm::Function *Fn = cast<llvm::Function>(GV);
848
2.26k
      if (!FD->doesThisDeclarationHaveABody() && 
!FD->hasPrototype()75
)
849
1
        Fn->addFnAttr("no-prototype");
850
2.26k
    }
851
2.41k
  }
852
};
853
854
/// Classify argument of given type \p Ty.
855
500
ABIArgInfo WebAssemblyABIInfo::classifyArgumentType(QualType Ty) const {
856
500
  Ty = useFirstFieldIfTransparentUnion(Ty);
857
858
500
  if (isAggregateTypeForABI(Ty)) {
859
    // Records with non-trivial destructors/copy-constructors should not be
860
    // passed by value.
861
42
    if (auto RAA = getRecordArgABI(Ty, getCXXABI()))
862
8
      return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
863
    // Ignore empty structs/unions.
864
34
    if (isEmptyRecord(getContext(), Ty, true))
865
4
      return ABIArgInfo::getIgnore();
866
    // Lower single-element structs to just pass a regular value. TODO: We
867
    // could do reasonable-size multiple-element structs too, using getExpand(),
868
    // though watch out for things like bitfields.
869
30
    if (const Type *SeltTy = isSingleElementStruct(Ty, getContext()))
870
11
      return ABIArgInfo::getDirect(CGT.ConvertType(QualType(SeltTy, 0)));
871
    // For the experimental multivalue ABI, fully expand all other aggregates
872
19
    if (Kind == ABIKind::ExperimentalMV) {
873
3
      const RecordType *RT = Ty->getAs<RecordType>();
874
3
      assert(RT);
875
0
      bool HasBitField = false;
876
5
      for (auto *Field : RT->getDecl()->fields()) {
877
5
        if (Field->isBitField()) {
878
1
          HasBitField = true;
879
1
          break;
880
1
        }
881
5
      }
882
3
      if (!HasBitField)
883
2
        return ABIArgInfo::getExpand();
884
3
    }
885
19
  }
886
887
  // Otherwise just do the default thing.
888
475
  return defaultInfo.classifyArgumentType(Ty);
889
500
}
890
891
418
ABIArgInfo WebAssemblyABIInfo::classifyReturnType(QualType RetTy) const {
892
418
  if (isAggregateTypeForABI(RetTy)) {
893
    // Records with non-trivial destructors/copy-constructors should not be
894
    // returned by value.
895
30
    if (!getRecordArgABI(RetTy, getCXXABI())) {
896
      // Ignore empty structs/unions.
897
30
      if (isEmptyRecord(getContext(), RetTy, true))
898
4
        return ABIArgInfo::getIgnore();
899
      // Lower single-element structs to just return a regular value. TODO: We
900
      // could do reasonable-size multiple-element structs too, using
901
      // ABIArgInfo::getDirect().
902
26
      if (const Type *SeltTy = isSingleElementStruct(RetTy, getContext()))
903
11
        return ABIArgInfo::getDirect(CGT.ConvertType(QualType(SeltTy, 0)));
904
      // For the experimental multivalue ABI, return all other aggregates
905
15
      if (Kind == ABIKind::ExperimentalMV)
906
3
        return ABIArgInfo::getDirect();
907
15
    }
908
30
  }
909
910
  // Otherwise just do the default thing.
911
400
  return defaultInfo.classifyReturnType(RetTy);
912
418
}
913
914
Address WebAssemblyABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
915
5
                                      QualType Ty) const {
916
5
  bool IsIndirect = isAggregateTypeForABI(Ty) &&
917
5
                    
!isEmptyRecord(getContext(), Ty, true)3
&&
918
5
                    
!isSingleElementStruct(Ty, getContext())2
;
919
5
  return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect,
920
5
                          getContext().getTypeInfoInChars(Ty),
921
5
                          CharUnits::fromQuantity(4),
922
5
                          /*AllowHigherAlign=*/true);
923
5
}
924
925
//===----------------------------------------------------------------------===//
926
// le32/PNaCl bitcode ABI Implementation
927
//
928
// This is a simplified version of the x86_32 ABI.  Arguments and return values
929
// are always passed on the stack.
930
//===----------------------------------------------------------------------===//
931
932
class PNaClABIInfo : public ABIInfo {
933
 public:
934
3
  PNaClABIInfo(CodeGen::CodeGenTypes &CGT) : ABIInfo(CGT) {}
935
936
  ABIArgInfo classifyReturnType(QualType RetTy) const;
937
  ABIArgInfo classifyArgumentType(QualType RetTy) const;
938
939
  void computeInfo(CGFunctionInfo &FI) const override;
940
  Address EmitVAArg(CodeGenFunction &CGF,
941
                    Address VAListAddr, QualType Ty) const override;
942
};
943
944
class PNaClTargetCodeGenInfo : public TargetCodeGenInfo {
945
 public:
946
   PNaClTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT)
947
3
       : TargetCodeGenInfo(std::make_unique<PNaClABIInfo>(CGT)) {}
948
};
949
950
2
void PNaClABIInfo::computeInfo(CGFunctionInfo &FI) const {
951
2
  if (!getCXXABI().classifyReturnType(FI))
952
2
    FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
953
954
2
  for (auto &I : FI.arguments())
955
0
    I.info = classifyArgumentType(I.type);
956
2
}
957
958
Address PNaClABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
959
0
                                QualType Ty) const {
960
  // The PNaCL ABI is a bit odd, in that varargs don't use normal
961
  // function classification. Structs get passed directly for varargs
962
  // functions, through a rewriting transform in
963
  // pnacl-llvm/lib/Transforms/NaCl/ExpandVarArgs.cpp, which allows
964
  // this target to actually support a va_arg instructions with an
965
  // aggregate type, unlike other targets.
966
0
  return EmitVAArgInstr(CGF, VAListAddr, Ty, ABIArgInfo::getDirect());
967
0
}
968
969
/// Classify argument of given type \p Ty.
970
0
ABIArgInfo PNaClABIInfo::classifyArgumentType(QualType Ty) const {
971
0
  if (isAggregateTypeForABI(Ty)) {
972
0
    if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))
973
0
      return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
974
0
    return getNaturalAlignIndirect(Ty);
975
0
  } else if (const EnumType *EnumTy = Ty->getAs<EnumType>()) {
976
    // Treat an enum type as its underlying type.
977
0
    Ty = EnumTy->getDecl()->getIntegerType();
978
0
  } else if (Ty->isFloatingType()) {
979
    // Floating-point types don't go inreg.
980
0
    return ABIArgInfo::getDirect();
981
0
  } else if (const auto *EIT = Ty->getAs<ExtIntType>()) {
982
    // Treat extended integers as integers if <=64, otherwise pass indirectly.
983
0
    if (EIT->getNumBits() > 64)
984
0
      return getNaturalAlignIndirect(Ty);
985
0
    return ABIArgInfo::getDirect();
986
0
  }
987
988
0
  return (isPromotableIntegerTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty)
989
0
                                            : ABIArgInfo::getDirect());
990
0
}
991
992
2
ABIArgInfo PNaClABIInfo::classifyReturnType(QualType RetTy) const {
993
2
  if (RetTy->isVoidType())
994
2
    return ABIArgInfo::getIgnore();
995
996
  // In the PNaCl ABI we always return records/structures on the stack.
997
0
  if (isAggregateTypeForABI(RetTy))
998
0
    return getNaturalAlignIndirect(RetTy);
999
1000
  // Treat extended integers as integers if <=64, otherwise pass indirectly.
1001
0
  if (const auto *EIT = RetTy->getAs<ExtIntType>()) {
1002
0
    if (EIT->getNumBits() > 64)
1003
0
      return getNaturalAlignIndirect(RetTy);
1004
0
    return ABIArgInfo::getDirect();
1005
0
  }
1006
1007
  // Treat an enum type as its underlying type.
1008
0
  if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
1009
0
    RetTy = EnumTy->getDecl()->getIntegerType();
1010
1011
0
  return (isPromotableIntegerTypeForABI(RetTy) ? ABIArgInfo::getExtend(RetTy)
1012
0
                                               : ABIArgInfo::getDirect());
1013
0
}
1014
1015
/// IsX86_MMXType - Return true if this is an MMX type.
1016
67
bool IsX86_MMXType(llvm::Type *IRType) {
1017
  // Return true if the type is an MMX type <2 x i32>, <4 x i16>, or <8 x i8>.
1018
67
  return IRType->isVectorTy() && IRType->getPrimitiveSizeInBits() == 64 &&
1019
67
    
cast<llvm::VectorType>(IRType)->getElementType()->isIntegerTy()18
&&
1020
67
    
IRType->getScalarSizeInBits() != 6416
;
1021
67
}
1022
1023
static llvm::Type* X86AdjustInlineAsmType(CodeGen::CodeGenFunction &CGF,
1024
                                          StringRef Constraint,
1025
1.39k
                                          llvm::Type* Ty) {
1026
1.39k
  bool IsMMXCons = llvm::StringSwitch<bool>(Constraint)
1027
1.39k
                     .Cases("y", "&y", "^Ym", true)
1028
1.39k
                     .Default(false);
1029
1.39k
  if (IsMMXCons && 
Ty->isVectorTy()18
) {
1030
14
    if (cast<llvm::VectorType>(Ty)->getPrimitiveSizeInBits().getFixedSize() !=
1031
14
        64) {
1032
      // Invalid MMX constraint
1033
0
      return nullptr;
1034
0
    }
1035
1036
14
    return llvm::Type::getX86_MMXTy(CGF.getLLVMContext());
1037
14
  }
1038
1039
  // No operation needed
1040
1.38k
  return Ty;
1041
1.39k
}
1042
1043
/// Returns true if this type can be passed in SSE registers with the
1044
/// X86_VectorCall calling convention. Shared between x86_32 and x86_64.
1045
771
static bool isX86VectorTypeForVectorCall(ASTContext &Context, QualType Ty) {
1046
771
  if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
1047
475
    if (BT->isFloatingPoint() && 
BT->getKind() != BuiltinType::Half281
) {
1048
281
      if (BT->getKind() == BuiltinType::LongDouble) {
1049
6
        if (&Context.getTargetInfo().getLongDoubleFormat() ==
1050
6
            &llvm::APFloat::x87DoubleExtended())
1051
2
          return false;
1052
6
      }
1053
279
      return true;
1054
281
    }
1055
475
  } else 
if (const VectorType *296
VT296
= Ty->getAs<VectorType>()) {
1056
    // vectorcall can pass XMM, YMM, and ZMM vectors. We don't pass SSE1 MMX
1057
    // registers specially.
1058
263
    unsigned VecSize = Context.getTypeSize(VT);
1059
263
    if (VecSize == 128 || 
VecSize == 2560
||
VecSize == 5120
)
1060
263
      return true;
1061
263
  }
1062
227
  return false;
1063
771
}
1064
1065
/// Returns true if this aggregate is small enough to be passed in SSE registers
1066
/// in the X86_VectorCall calling convention. Shared between x86_32 and x86_64.
1067
730
static bool isX86VectorCallAggregateSmallEnough(uint64_t NumMembers) {
1068
730
  return NumMembers <= 4;
1069
730
}
1070
1071
/// Returns a Homogeneous Vector Aggregate ABIArgInfo, used in X86.
1072
30
static ABIArgInfo getDirectX86Hva(llvm::Type* T = nullptr) {
1073
30
  auto AI = ABIArgInfo::getDirect(T);
1074
30
  AI.setInReg(true);
1075
30
  AI.setCanBeFlattened(false);
1076
30
  return AI;
1077
30
}
1078
1079
//===----------------------------------------------------------------------===//
1080
// X86-32 ABI Implementation
1081
//===----------------------------------------------------------------------===//
1082
1083
/// Similar to llvm::CCState, but for Clang.
1084
struct CCState {
1085
  CCState(CGFunctionInfo &FI)
1086
18.5k
      : IsPreassigned(FI.arg_size()), CC(FI.getCallingConvention()) {}
1087
1088
  llvm::SmallBitVector IsPreassigned;
1089
  unsigned CC = CallingConv::CC_C;
1090
  unsigned FreeRegs = 0;
1091
  unsigned FreeSSERegs = 0;
1092
};
1093
1094
/// X86_32ABIInfo - The X86-32 ABI information.
1095
class X86_32ABIInfo : public SwiftABIInfo {
1096
  enum Class {
1097
    Integer,
1098
    Float
1099
  };
1100
1101
  static const unsigned MinABIStackAlignInBytes = 4;
1102
1103
  bool IsDarwinVectorABI;
1104
  bool IsRetSmallStructInRegABI;
1105
  bool IsWin32StructABI;
1106
  bool IsSoftFloatABI;
1107
  bool IsMCUABI;
1108
  bool IsLinuxABI;
1109
  unsigned DefaultNumRegisterParameters;
1110
1111
519
  static bool isRegisterSize(unsigned Size) {
1112
519
    return (Size == 8 || 
Size == 16488
||
Size == 32469
||
Size == 64231
);
1113
519
  }
1114
1115
400
  bool isHomogeneousAggregateBaseType(QualType Ty) const override {
1116
    // FIXME: Assumes vectorcall is in use.
1117
400
    return isX86VectorTypeForVectorCall(getContext(), Ty);
1118
400
  }
1119
1120
  bool isHomogeneousAggregateSmallEnough(const Type *Ty,
1121
322
                                         uint64_t NumMembers) const override {
1122
    // FIXME: Assumes vectorcall is in use.
1123
322
    return isX86VectorCallAggregateSmallEnough(NumMembers);
1124
322
  }
1125
1126
  bool shouldReturnTypeInRegister(QualType Ty, ASTContext &Context) const;
1127
1128
  /// getIndirectResult - Give a source type \arg Ty, return a suitable result
1129
  /// such that the argument will be passed in memory.
1130
  ABIArgInfo getIndirectResult(QualType Ty, bool ByVal, CCState &State) const;
1131
1132
  ABIArgInfo getIndirectReturnResult(QualType Ty, CCState &State) const;
1133
1134
  /// Return the alignment to use for the given type on the stack.
1135
  unsigned getTypeStackAlignInBytes(QualType Ty, unsigned Align) const;
1136
1137
  Class classify(QualType Ty) const;
1138
  ABIArgInfo classifyReturnType(QualType RetTy, CCState &State) const;
1139
  ABIArgInfo classifyArgumentType(QualType RetTy, CCState &State) const;
1140
1141
  /// Updates the number of available free registers, returns
1142
  /// true if any registers were allocated.
1143
  bool updateFreeRegs(QualType Ty, CCState &State) const;
1144
1145
  bool shouldAggregateUseDirect(QualType Ty, CCState &State, bool &InReg,
1146
                                bool &NeedsPadding) const;
1147
  bool shouldPrimitiveUseInReg(QualType Ty, CCState &State) const;
1148
1149
  bool canExpandIndirectArgument(QualType Ty) const;
1150
1151
  /// Rewrite the function info so that all memory arguments use
1152
  /// inalloca.
1153
  void rewriteWithInAlloca(CGFunctionInfo &FI) const;
1154
1155
  void addFieldToArgStruct(SmallVector<llvm::Type *, 6> &FrameFields,
1156
                           CharUnits &StackOffset, ABIArgInfo &Info,
1157
                           QualType Type) const;
1158
  void runVectorCallFirstPass(CGFunctionInfo &FI, CCState &State) const;
1159
1160
public:
1161
1162
  void computeInfo(CGFunctionInfo &FI) const override;
1163
  Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
1164
                    QualType Ty) const override;
1165
1166
  X86_32ABIInfo(CodeGen::CodeGenTypes &CGT, bool DarwinVectorABI,
1167
                bool RetSmallStructInRegABI, bool Win32StructABI,
1168
                unsigned NumRegisterParameters, bool SoftFloatABI)
1169
    : SwiftABIInfo(CGT), IsDarwinVectorABI(DarwinVectorABI),
1170
      IsRetSmallStructInRegABI(RetSmallStructInRegABI),
1171
      IsWin32StructABI(Win32StructABI), IsSoftFloatABI(SoftFloatABI),
1172
      IsMCUABI(CGT.getTarget().getTriple().isOSIAMCU()),
1173
      IsLinuxABI(CGT.getTarget().getTriple().isOSLinux()),
1174
3.06k
      DefaultNumRegisterParameters(NumRegisterParameters) {}
1175
1176
  bool shouldPassIndirectlyForSwift(ArrayRef<llvm::Type*> scalars,
1177
0
                                    bool asReturnValue) const override {
1178
    // LLVM's x86-32 lowering currently only assigns up to three
1179
    // integer registers and three fp registers.  Oddly, it'll use up to
1180
    // four vector registers for vectors, but those can overlap with the
1181
    // scalar registers.
1182
0
    return occupiesMoreThan(CGT, scalars, /*total*/ 3);
1183
0
  }
1184
1185
0
  bool isSwiftErrorInRegister() const override {
1186
    // x86-32 lowering does not support passing swifterror in a register.
1187
0
    return false;
1188
0
  }
1189
};
1190
1191
class X86_32TargetCodeGenInfo : public TargetCodeGenInfo {
1192
public:
1193
  X86_32TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, bool DarwinVectorABI,
1194
                          bool RetSmallStructInRegABI, bool Win32StructABI,
1195
                          unsigned NumRegisterParameters, bool SoftFloatABI)
1196
      : TargetCodeGenInfo(std::make_unique<X86_32ABIInfo>(
1197
            CGT, DarwinVectorABI, RetSmallStructInRegABI, Win32StructABI,
1198
3.06k
            NumRegisterParameters, SoftFloatABI)) {}
1199
1200
  static bool isStructReturnInRegABI(
1201
      const llvm::Triple &Triple, const CodeGenOptions &Opts);
1202
1203
  void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
1204
                           CodeGen::CodeGenModule &CGM) const override;
1205
1206
0
  int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const override {
1207
    // Darwin uses different dwarf register numbers for EH.
1208
0
    if (CGM.getTarget().getTriple().isOSDarwin()) return 5;
1209
0
    return 4;
1210
0
  }
1211
1212
  bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
1213
                               llvm::Value *Address) const override;
1214
1215
  llvm::Type* adjustInlineAsmType(CodeGen::CodeGenFunction &CGF,
1216
                                  StringRef Constraint,
1217
566
                                  llvm::Type* Ty) const override {
1218
566
    return X86AdjustInlineAsmType(CGF, Constraint, Ty);
1219
566
  }
1220
1221
  void addReturnRegisterOutputs(CodeGenFunction &CGF, LValue ReturnValue,
1222
                                std::string &Constraints,
1223
                                std::vector<llvm::Type *> &ResultRegTypes,
1224
                                std::vector<llvm::Type *> &ResultTruncRegTypes,
1225
                                std::vector<LValue> &ResultRegDests,
1226
                                std::string &AsmString,
1227
                                unsigned NumOutputs) const override;
1228
1229
  llvm::Constant *
1230
37
  getUBSanFunctionSignature(CodeGen::CodeGenModule &CGM) const override {
1231
37
    unsigned Sig = (0xeb << 0) |  // jmp rel8
1232
37
                   (0x06 << 8) |  //           .+0x08
1233
37
                   ('v' << 16) |
1234
37
                   ('2' << 24);
1235
37
    return llvm::ConstantInt::get(CGM.Int32Ty, Sig);
1236
37
  }
1237
1238
7
  StringRef getARCRetainAutoreleasedReturnValueMarker() const override {
1239
7
    return "movl\t%ebp, %ebp"
1240
7
           "\t\t// marker for objc_retainAutoreleaseReturnValue";
1241
7
  }
1242
};
1243
1244
}
1245
1246
/// Rewrite input constraint references after adding some output constraints.
1247
/// In the case where there is one output and one input and we add one output,
1248
/// we need to replace all operand references greater than or equal to 1:
1249
///     mov $0, $1
1250
///     mov eax, $1
1251
/// The result will be:
1252
///     mov $0, $2
1253
///     mov eax, $2
1254
static void rewriteInputConstraintReferences(unsigned FirstIn,
1255
                                             unsigned NumNewOuts,
1256
26
                                             std::string &AsmString) {
1257
26
  std::string Buf;
1258
26
  llvm::raw_string_ostream OS(Buf);
1259
26
  size_t Pos = 0;
1260
86
  while (Pos < AsmString.size()) {
1261
60
    size_t DollarStart = AsmString.find('$', Pos);
1262
60
    if (DollarStart == std::string::npos)
1263
19
      DollarStart = AsmString.size();
1264
60
    size_t DollarEnd = AsmString.find_first_not_of('$', DollarStart);
1265
60
    if (DollarEnd == std::string::npos)
1266
19
      DollarEnd = AsmString.size();
1267
60
    OS << StringRef(&AsmString[Pos], DollarEnd - Pos);
1268
60
    Pos = DollarEnd;
1269
60
    size_t NumDollars = DollarEnd - DollarStart;
1270
60
    if (NumDollars % 2 != 0 && 
Pos < AsmString.size()18
) {
1271
      // We have an operand reference.
1272
18
      size_t DigitStart = Pos;
1273
18
      if (AsmString[DigitStart] == '{') {
1274
3
        OS << '{';
1275
3
        ++DigitStart;
1276
3
      }
1277
18
      size_t DigitEnd = AsmString.find_first_not_of("0123456789", DigitStart);
1278
18
      if (DigitEnd == std::string::npos)
1279
5
        DigitEnd = AsmString.size();
1280
18
      StringRef OperandStr(&AsmString[DigitStart], DigitEnd - DigitStart);
1281
18
      unsigned OperandIndex;
1282
18
      if (!OperandStr.getAsInteger(10, OperandIndex)) {
1283
18
        if (OperandIndex >= FirstIn)
1284
14
          OperandIndex += NumNewOuts;
1285
18
        OS << OperandIndex;
1286
18
      } else {
1287
0
        OS << OperandStr;
1288
0
      }
1289
18
      Pos = DigitEnd;
1290
18
    }
1291
60
  }
1292
26
  AsmString = std::move(OS.str());
1293
26
}
1294
1295
/// Add output constraints for EAX:EDX because they are return registers.
1296
void X86_32TargetCodeGenInfo::addReturnRegisterOutputs(
1297
    CodeGenFunction &CGF, LValue ReturnSlot, std::string &Constraints,
1298
    std::vector<llvm::Type *> &ResultRegTypes,
1299
    std::vector<llvm::Type *> &ResultTruncRegTypes,
1300
    std::vector<LValue> &ResultRegDests, std::string &AsmString,
1301
26
    unsigned NumOutputs) const {
1302
26
  uint64_t RetWidth = CGF.getContext().getTypeSize(ReturnSlot.getType());
1303
1304
  // Use the EAX constraint if the width is 32 or smaller and EAX:EDX if it is
1305
  // larger.
1306
26
  if (!Constraints.empty())
1307
3
    Constraints += ',';
1308
26
  if (RetWidth <= 32) {
1309
22
    Constraints += "={eax}";
1310
22
    ResultRegTypes.push_back(CGF.Int32Ty);
1311
22
  } else {
1312
    // Use the 'A' constraint for EAX:EDX.
1313
4
    Constraints += "=A";
1314
4
    ResultRegTypes.push_back(CGF.Int64Ty);
1315
4
  }
1316
1317
  // Truncate EAX or EAX:EDX to an integer of the appropriate size.
1318
26
  llvm::Type *CoerceTy = llvm::IntegerType::get(CGF.getLLVMContext(), RetWidth);
1319
26
  ResultTruncRegTypes.push_back(CoerceTy);
1320
1321
  // Coerce the integer by bitcasting the return slot pointer.
1322
26
  ReturnSlot.setAddress(CGF.Builder.CreateBitCast(ReturnSlot.getAddress(CGF),
1323
26
                                                  CoerceTy->getPointerTo()));
1324
26
  ResultRegDests.push_back(ReturnSlot);
1325
1326
26
  rewriteInputConstraintReferences(NumOutputs, 1, AsmString);
1327
26
}
1328
1329
/// shouldReturnTypeInRegister - Determine if the given type should be
1330
/// returned in a register (for the Darwin and MCU ABI).
1331
bool X86_32ABIInfo::shouldReturnTypeInRegister(QualType Ty,
1332
538
                                               ASTContext &Context) const {
1333
538
  uint64_t Size = Context.getTypeSize(Ty);
1334
1335
  // For i386, type must be register sized.
1336
  // For the MCU ABI, it only needs to be <= 8-byte
1337
538
  if ((IsMCUABI && 
Size > 6419
) ||
(535
!IsMCUABI535
&&
!isRegisterSize(Size)519
))
1338
70
   return false;
1339
1340
468
  if (Ty->isVectorType()) {
1341
    // 64- and 128- bit vectors inside structures are not returned in
1342
    // registers.
1343
4
    if (Size == 64 || 
Size == 1281
)
1344
3
      return false;
1345
1346
1
    return true;
1347
4
  }
1348
1349
  // If this is a builtin, pointer, enum, complex type, member pointer, or
1350
  // member function pointer it is ok.
1351
464
  if (Ty->getAs<BuiltinType>() || 
Ty->hasPointerRepresentation()376
||
1352
464
      
Ty->isAnyComplexType()372
||
Ty->isEnumeralType()349
||
1353
464
      
Ty->isBlockPointerType()347
||
Ty->isMemberPointerType()347
)
1354
379
    return true;
1355
1356
  // Arrays are treated like records.
1357
85
  if (const ConstantArrayType *AT = Context.getAsConstantArrayType(Ty))
1358
6
    return shouldReturnTypeInRegister(AT->getElementType(), Context);
1359
1360
  // Otherwise, it must be a record type.
1361
79
  const RecordType *RT = Ty->getAs<RecordType>();
1362
79
  if (!RT) 
return false0
;
1363
1364
  // FIXME: Traverse bases here too.
1365
1366
  // Structure types are passed in register if all fields would be
1367
  // passed in a register.
1368
113
  
for (const auto *FD : RT->getDecl()->fields())79
{
1369
    // Empty fields are ignored.
1370
113
    if (isEmptyField(Context, FD, true))
1371
11
      continue;
1372
1373
    // Check fields recursively.
1374
102
    if (!shouldReturnTypeInRegister(FD->getType(), Context))
1375
4
      return false;
1376
102
  }
1377
75
  return true;
1378
79
}
1379
1380
207
static bool is32Or64BitBasicType(QualType Ty, ASTContext &Context) {
1381
  // Treat complex types as the element type.
1382
207
  if (const ComplexType *CTy = Ty->getAs<ComplexType>())
1383
2
    Ty = CTy->getElementType();
1384
1385
  // Check for a type which we know has a simple scalar argument-passing
1386
  // convention without any padding.  (We're specifically looking for 32
1387
  // and 64-bit integer and integer-equivalents, float, and double.)
1388
207
  if (!Ty->getAs<BuiltinType>() && 
!Ty->hasPointerRepresentation()36
&&
1389
207
      
!Ty->isEnumeralType()31
&&
!Ty->isBlockPointerType()28
)
1390
28
    return false;
1391
1392
179
  uint64_t Size = Context.getTypeSize(Ty);
1393
179
  return Size == 32 || 
Size == 6429
;
1394
207
}
1395
1396
static bool addFieldSizes(ASTContext &Context, const RecordDecl *RD,
1397
246
                          uint64_t &Size) {
1398
246
  for (const auto *FD : RD->fields()) {
1399
    // Scalar arguments on the stack get 4 byte alignment on x86. If the
1400
    // argument is smaller than 32-bits, expanding the struct will create
1401
    // alignment padding.
1402
207
    if (!is32Or64BitBasicType(FD->getType(), Context))
1403
50
      return false;
1404
1405
    // FIXME: Reject bit-fields wholesale; there are two problems, we don't know
1406
    // how to expand them yet, and the predicate for telling if a bitfield still
1407
    // counts as "basic" is more complicated than what we were doing previously.
1408
157
    if (FD->isBitField())
1409
2
      return false;
1410
1411
155
    Size += Context.getTypeSize(FD->getType());
1412
155
  }
1413
194
  return true;
1414
246
}
1415
1416
static bool addBaseAndFieldSizes(ASTContext &Context, const CXXRecordDecl *RD,
1417
144
                                 uint64_t &Size) {
1418
  // Don't do this if there are any non-empty bases.
1419
144
  for (const CXXBaseSpecifier &Base : RD->bases()) {
1420
1
    if (!addBaseAndFieldSizes(Context, Base.getType()->getAsCXXRecordDecl(),
1421
1
                              Size))
1422
0
      return false;
1423
1
  }
1424
144
  if (!addFieldSizes(Context, RD, Size))
1425
10
    return false;
1426
134
  return true;
1427
144
}
1428
1429
/// Test whether an argument type which is to be passed indirectly (on the
1430
/// stack) would have the equivalent layout if it was expanded into separate
1431
/// arguments. If so, we prefer to do the latter to avoid inhibiting
1432
/// optimizations.
1433
326
bool X86_32ABIInfo::canExpandIndirectArgument(QualType Ty) const {
1434
  // We can only expand structure types.
1435
326
  const RecordType *RT = Ty->getAs<RecordType>();
1436
326
  if (!RT)
1437
68
    return false;
1438
258
  const RecordDecl *RD = RT->getDecl();
1439
258
  uint64_t Size = 0;
1440
258
  if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
1441
156
    if (!IsWin32StructABI) {
1442
      // On non-Windows, we have to conservatively match our old bitcode
1443
      // prototypes in order to be ABI-compatible at the bitcode level.
1444
35
      if (!CXXRD->isCLike())
1445
13
        return false;
1446
121
    } else {
1447
      // Don't do this for dynamic classes.
1448
121
      if (CXXRD->isDynamicClass())
1449
0
        return false;
1450
121
    }
1451
143
    if (!addBaseAndFieldSizes(getContext(), CXXRD, Size))
1452
10
      return false;
1453
143
  } else {
1454
102
    if (!addFieldSizes(getContext(), RD, Size))
1455
42
      return false;
1456
102
  }
1457
1458
  // We can do this if there was no alignment padding.
1459
193
  return Size == getContext().getTypeSize(Ty);
1460
258
}
1461
1462
149
ABIArgInfo X86_32ABIInfo::getIndirectReturnResult(QualType RetTy, CCState &State) const {
1463
  // If the return value is indirect, then the hidden argument is consuming one
1464
  // integer register.
1465
149
  if (State.FreeRegs) {
1466
6
    --State.FreeRegs;
1467
6
    if (!IsMCUABI)
1468
3
      return getNaturalAlignIndirectInReg(RetTy);
1469
6
  }
1470
146
  return getNaturalAlignIndirect(RetTy, /*ByVal=*/false);
1471
149
}
1472
1473
ABIArgInfo X86_32ABIInfo::classifyReturnType(QualType RetTy,
1474
18.3k
                                             CCState &State) const {
1475
18.3k
  if (RetTy->isVoidType())
1476
12.4k
    return ABIArgInfo::getIgnore();
1477
1478
5.96k
  const Type *Base = nullptr;
1479
5.96k
  uint64_t NumElts = 0;
1480
5.96k
  if ((State.CC == llvm::CallingConv::X86_VectorCall ||
1481
5.96k
       
State.CC == llvm::CallingConv::X86_RegCall5.95k
) &&
1482
5.96k
      
isHomogeneousAggregate(RetTy, Base, NumElts)26
) {
1483
    // The LLVM struct type for such an aggregate should lower properly.
1484
17
    return ABIArgInfo::getDirect();
1485
17
  }
1486
1487
5.94k
  if (const VectorType *VT = RetTy->getAs<VectorType>()) {
1488
    // On Darwin, some vectors are returned in registers.
1489
31
    if (IsDarwinVectorABI) {
1490
15
      uint64_t Size = getContext().getTypeSize(RetTy);
1491
1492
      // 128-bit vectors are a special case; they are returned in
1493
      // registers and we need to make sure to pick a type the LLVM
1494
      // backend will like.
1495
15
      if (Size == 128)
1496
5
        return ABIArgInfo::getDirect(llvm::FixedVectorType::get(
1497
5
            llvm::Type::getInt64Ty(getVMContext()), 2));
1498
1499
      // Always return in register if it fits in a general purpose
1500
      // register, or if it is 64 bits and has a single element.
1501
10
      if ((Size == 8 || Size == 16 || Size == 32) ||
1502
10
          
(7
Size == 647
&&
VT->getNumElements() == 17
))
1503
7
        return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(),
1504
7
                                                            Size));
1505
1506
3
      return getIndirectReturnResult(RetTy, State);
1507
10
    }
1508
1509
16
    return ABIArgInfo::getDirect();
1510
31
  }
1511
1512
5.91k
  if (isAggregateTypeForABI(RetTy)) {
1513
528
    if (const RecordType *RT = RetTy->getAs<RecordType>()) {
1514
      // Structures with flexible arrays are always indirect.
1515
197
      if (RT->getDecl()->hasFlexibleArrayMember())
1516
1
        return getIndirectReturnResult(RetTy, State);
1517
197
    }
1518
1519
    // If specified, structs and unions are always indirect.
1520
527
    if (!IsRetSmallStructInRegABI && 
!RetTy->isAnyComplexType()115
)
1521
62
      return getIndirectReturnResult(RetTy, State);
1522
1523
    // Ignore empty structs/unions.
1524
465
    if (isEmptyRecord(getContext(), RetTy, true))
1525
34
      return ABIArgInfo::getIgnore();
1526
1527
    // Return complex of _Float16 as <2 x half> so the backend will use xmm0.
1528
431
    if (const ComplexType *CT = RetTy->getAs<ComplexType>()) {
1529
64
      QualType ET = getContext().getCanonicalType(CT->getElementType());
1530
64
      if (ET->isFloat16Type())
1531
1
        return ABIArgInfo::getDirect(llvm::FixedVectorType::get(
1532
1
            llvm::Type::getHalfTy(getVMContext()), 2));
1533
64
    }
1534
1535
    // Small structures which are register sized are generally returned
1536
    // in a register.
1537
430
    if (shouldReturnTypeInRegister(RetTy, getContext())) {
1538
357
      uint64_t Size = getContext().getTypeSize(RetTy);
1539
1540
      // As a special-case, if the struct is a "single-element" struct, and
1541
      // the field is of type "float" or "double", return it in a
1542
      // floating-point register. (MSVC does not apply this special case.)
1543
      // We apply a similar transformation for pointer types to improve the
1544
      // quality of the generated IR.
1545
357
      if (const Type *SeltTy = isSingleElementStruct(RetTy, getContext()))
1546
48
        if ((!IsWin32StructABI && 
SeltTy->isRealFloatingType()36
)
1547
48
            || 
SeltTy->hasPointerRepresentation()32
)
1548
18
          return ABIArgInfo::getDirect(CGT.ConvertType(QualType(SeltTy, 0)));
1549
1550
      // FIXME: We should be able to narrow this integer in cases with dead
1551
      // padding.
1552
339
      return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(),Size));
1553
357
    }
1554
1555
73
    return getIndirectReturnResult(RetTy, State);
1556
430
  }
1557
1558
  // Treat an enum type as its underlying type.
1559
5.38k
  if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
1560
35
    RetTy = EnumTy->getDecl()->getIntegerType();
1561
1562
5.38k
  if (const auto *EIT = RetTy->getAs<ExtIntType>())
1563
14
    if (EIT->getNumBits() > 64)
1564
10
      return getIndirectReturnResult(RetTy, State);
1565
1566
5.37k
  return (isPromotableIntegerTypeForABI(RetTy) ? 
ABIArgInfo::getExtend(RetTy)120
1567
5.37k
                                               : 
ABIArgInfo::getDirect()5.25k
);
1568
5.38k
}
1569
1570
66
static bool isSIMDVectorType(ASTContext &Context, QualType Ty) {
1571
66
  return Ty->getAs<VectorType>() && 
Context.getTypeSize(Ty) == 12814
;
1572
66
}
1573
1574
84
static bool isRecordWithSIMDVectorType(ASTContext &Context, QualType Ty) {
1575
84
  const RecordType *RT = Ty->getAs<RecordType>();
1576
84
  if (!RT)
1577
42
    return 0;
1578
42
  const RecordDecl *RD = RT->getDecl();
1579
1580
  // If this is a C++ record, check the bases first.
1581
42
  if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD))
1582
0
    for (const auto &I : CXXRD->bases())
1583
0
      if (!isRecordWithSIMDVectorType(Context, I.getType()))
1584
0
        return false;
1585
1586
49
  
for (const auto *i : RD->fields())42
{
1587
49
    QualType FT = i->getType();
1588
1589
49
    if (isSIMDVectorType(Context, FT))
1590
9
      return true;
1591
1592
40
    if (isRecordWithSIMDVectorType(Context, FT))
1593
0
      return true;
1594
40
  }
1595
1596
33
  return false;
1597
42
}
1598
1599
unsigned X86_32ABIInfo::getTypeStackAlignInBytes(QualType Ty,
1600
314
                                                 unsigned Align) const {
1601
  // Otherwise, if the alignment is less than or equal to the minimum ABI
1602
  // alignment, just use the default; the backend will handle this.
1603
314
  if (Align <= MinABIStackAlignInBytes)
1604
255
    return 0; // Use default alignment.
1605
1606
59
  if (IsLinuxABI) {
1607
    // Exclude other System V OS (e.g Darwin, PS4 and FreeBSD) since we don't
1608
    // want to spend any effort dealing with the ramifications of ABI breaks.
1609
    //
1610
    // If the vector type is __m128/__m256/__m512, return the default alignment.
1611
21
    if (Ty->isVectorType() && 
(9
Align == 169
||
Align == 326
||
Align == 643
))
1612
9
      return Align;
1613
21
  }
1614
  // On non-Darwin, the stack type alignment is always 4.
1615
50
  if (!IsDarwinVectorABI) {
1616
    // Set explicit alignment, since we may need to realign the top.
1617
23
    return MinABIStackAlignInBytes;
1618
23
  }
1619
1620
  // Otherwise, if the type contains an SSE vector type, the alignment is 16.
1621
27
  if (Align >= 16 && 
(17
isSIMDVectorType(getContext(), Ty)17
||
1622
17
                      
isRecordWithSIMDVectorType(getContext(), Ty)16
))
1623
8
    return 16;
1624
1625
19
  return MinABIStackAlignInBytes;
1626
27
}
1627
1628
ABIArgInfo X86_32ABIInfo::getIndirectResult(QualType Ty, bool ByVal,
1629
449
                                            CCState &State) const {
1630
449
  if (!ByVal) {
1631
158
    if (State.FreeRegs) {
1632
22
      --State.FreeRegs; // Non-byval indirects just use one pointer.
1633
22
      if (!IsMCUABI)
1634
22
        return getNaturalAlignIndirectInReg(Ty);
1635
22
    }
1636
136
    return getNaturalAlignIndirect(Ty, false);
1637
158
  }
1638
1639
  // Compute the byval alignment.
1640
291
  unsigned TypeAlign = getContext().getTypeAlign(Ty) / 8;
1641
291
  unsigned StackAlign = getTypeStackAlignInBytes(Ty, TypeAlign);
1642
291
  if (StackAlign == 0)
1643
243
    return ABIArgInfo::getIndirect(CharUnits::fromQuantity(4), /*ByVal=*/true);
1644
1645
  // If the stack alignment is less than the type alignment, realign the
1646
  // argument.
1647
48
  bool Realign = TypeAlign > StackAlign;
1648
48
  return ABIArgInfo::getIndirect(CharUnits::fromQuantity(StackAlign),
1649
48
                                 /*ByVal=*/true, Realign);
1650
291
}
1651
1652
31.6k
X86_32ABIInfo::Class X86_32ABIInfo::classify(QualType Ty) const {
1653
31.6k
  const Type *T = isSingleElementStruct(Ty, getContext());
1654
31.6k
  if (!T)
1655
31.6k
    T = Ty.getTypePtr();
1656
1657
31.6k
  if (const BuiltinType *BT = T->getAs<BuiltinType>()) {
1658
10.8k
    BuiltinType::Kind K = BT->getKind();
1659
10.8k
    if (K == BuiltinType::Float || 
K == BuiltinType::Double10.4k
)
1660
517
      return Float;
1661
10.8k
  }
1662
31.1k
  return Integer;
1663
31.6k
}
1664
1665
31.7k
bool X86_32ABIInfo::updateFreeRegs(QualType Ty, CCState &State) const {
1666
31.7k
  if (!IsSoftFloatABI) {
1667
31.6k
    Class C = classify(Ty);
1668
31.6k
    if (C == Float)
1669
517
      return false;
1670
31.6k
  }
1671
1672
31.2k
  unsigned Size = getContext().getTypeSize(Ty);
1673
31.2k
  unsigned SizeInRegs = (Size + 31) / 32;
1674
1675
31.2k
  if (SizeInRegs == 0)
1676
2
    return false;
1677
1678
31.2k
  if (!IsMCUABI) {
1679
31.1k
    if (SizeInRegs > State.FreeRegs) {
1680
30.9k
      State.FreeRegs = 0;
1681
30.9k
      return false;
1682
30.9k
    }
1683
31.1k
  } else {
1684
    // The MCU psABI allows passing parameters in-reg even if there are
1685
    // earlier parameters that are passed on the stack. Also,
1686
    // it does not allow passing >8-byte structs in-register,
1687
    // even if there are 3 free registers available.
1688
42
    if (SizeInRegs > State.FreeRegs || 
SizeInRegs > 234
)
1689
9
      return false;
1690
42
  }
1691
1692
243
  State.FreeRegs -= SizeInRegs;
1693
243
  return true;
1694
31.2k
}
1695
1696
bool X86_32ABIInfo::shouldAggregateUseDirect(QualType Ty, CCState &State,
1697
                                             bool &InReg,
1698
393
                                             bool &NeedsPadding) const {
1699
  // On Windows, aggregates other than HFAs are never passed in registers, and
1700
  // they do not consume register slots. Homogenous floating-point aggregates
1701
  // (HFAs) have already been dealt with at this point.
1702
393
  if (IsWin32StructABI && 
isAggregateTypeForABI(Ty)168
)
1703
168
    return false;
1704
1705
225
  NeedsPadding = false;
1706
225
  InReg = !IsMCUABI;
1707
1708
225
  if (!updateFreeRegs(Ty, State))
1709
205
    return false;
1710
1711
20
  if (IsMCUABI)
1712
7
    return true;
1713
1714
13
  if (State.CC == llvm::CallingConv::X86_FastCall ||
1715
13
      
State.CC == llvm::CallingConv::X86_VectorCall10
||
1716
13
      
State.CC == llvm::CallingConv::X86_RegCall10
) {
1717
4
    if (getContext().getTypeSize(Ty) <= 32 && State.FreeRegs)
1718
3
      NeedsPadding = true;
1719
1720
4
    return false;
1721
4
  }
1722
1723
9
  return true;
1724
13
}
1725
1726
31.5k
bool X86_32ABIInfo::shouldPrimitiveUseInReg(QualType Ty, CCState &State) const {
1727
31.5k
  if (!updateFreeRegs(Ty, State))
1728
31.2k
    return false;
1729
1730
223
  if (IsMCUABI)
1731
26
    return false;
1732
1733
197
  if (State.CC == llvm::CallingConv::X86_FastCall ||
1734
197
      
State.CC == llvm::CallingConv::X86_VectorCall137
||
1735
197
      
State.CC == llvm::CallingConv::X86_RegCall94
) {
1736
146
    if (getContext().getTypeSize(Ty) > 32)
1737
13
      return false;
1738
1739
133
    return (Ty->isIntegralOrEnumerationType() || 
Ty->isPointerType()27
||
1740
133
        
Ty->isReferenceType()7
);
1741
146
  }
1742
1743
51
  return true;
1744
197
}
1745
1746
57
void X86_32ABIInfo::runVectorCallFirstPass(CGFunctionInfo &FI, CCState &State) const {
1747
  // Vectorcall x86 works subtly different than in x64, so the format is
1748
  // a bit different than the x64 version.  First, all vector types (not HVAs)
1749
  // are assigned, with the first 6 ending up in the [XYZ]MM0-5 registers.
1750
  // This differs from the x64 implementation, where the first 6 by INDEX get
1751
  // registers.
1752
  // In the second pass over the arguments, HVAs are passed in the remaining
1753
  // vector registers if possible, or indirectly by address. The address will be
1754
  // passed in ECX/EDX if available. Any other arguments are passed according to
1755
  // the usual fastcall rules.
1756
57
  MutableArrayRef<CGFunctionInfoArgInfo> Args = FI.arguments();
1757
171
  for (int I = 0, E = Args.size(); I < E; 
++I114
) {
1758
114
    const Type *Base = nullptr;
1759
114
    uint64_t NumElts = 0;
1760
114
    const QualType &Ty = Args[I].type;
1761
114
    if ((Ty->isVectorType() || 
Ty->isBuiltinType()93
) &&
1762
114
        
isHomogeneousAggregate(Ty, Base, NumElts)84
) {
1763
42
      if (State.FreeSSERegs >= NumElts) {
1764
38
        State.FreeSSERegs -= NumElts;
1765
38
        Args[I].info = ABIArgInfo::getDirectInReg();
1766
38
        State.IsPreassigned.set(I);
1767
38
      }
1768
42
    }
1769
114
  }
1770
57
}
1771
1772
ABIArgInfo X86_32ABIInfo::classifyArgumentType(QualType Ty,
1773
32.3k
                                               CCState &State) const {
1774
  // FIXME: Set alignment on indirect arguments.
1775
32.3k
  bool IsFastCall = State.CC == llvm::CallingConv::X86_FastCall;
1776
32.3k
  bool IsRegCall = State.CC == llvm::CallingConv::X86_RegCall;
1777
32.3k
  bool IsVectorCall = State.CC == llvm::CallingConv::X86_VectorCall;
1778
1779
32.3k
  Ty = useFirstFieldIfTransparentUnion(Ty);
1780
32.3k
  TypeInfo TI = getContext().getTypeInfo(Ty);
1781
1782
  // Check with the C++ ABI first.
1783
32.3k
  const RecordType *RT = Ty->getAs<RecordType>();
1784
32.3k
  if (RT) {
1785
575
    CGCXXABI::RecordArgABI RAA = getRecordArgABI(RT, getCXXABI());
1786
575
    if (RAA == CGCXXABI::RAA_Indirect) {
1787
102
      return getIndirectResult(Ty, false, State);
1788
473
    } else if (RAA == CGCXXABI::RAA_DirectInMemory) {
1789
      // The field index doesn't matter, we'll fix it up later.
1790
86
      return ABIArgInfo::getInAlloca(/*FieldIndex=*/0);
1791
86
    }
1792
575
  }
1793
1794
  // Regcall uses the concept of a homogenous vector aggregate, similar
1795
  // to other targets.
1796
32.1k
  const Type *Base = nullptr;
1797
32.1k
  uint64_t NumElts = 0;
1798
32.1k
  if ((IsRegCall || 
IsVectorCall32.0k
) &&
1799
32.1k
      
isHomogeneousAggregate(Ty, Base, NumElts)179
) {
1800
79
    if (State.FreeSSERegs >= NumElts) {
1801
57
      State.FreeSSERegs -= NumElts;
1802
1803
      // Vectorcall passes HVAs directly and does not flatten them, but regcall
1804
      // does.
1805
57
      if (IsVectorCall)
1806
12
        return getDirectX86Hva();
1807
1808
45
      if (Ty->isBuiltinType() || 
Ty->isVectorType()35
)
1809
20
        return ABIArgInfo::getDirect();
1810
25
      return ABIArgInfo::getExpand();
1811
45
    }
1812
22
    return getIndirectResult(Ty, /*ByVal=*/false, State);
1813
79
  }
1814
1815
32.0k
  if (isAggregateTypeForABI(Ty)) {
1816
    // Structures with flexible arrays are always indirect.
1817
    // FIXME: This should not be byval!
1818
418
    if (RT && 
RT->getDecl()->hasFlexibleArrayMember()339
)
1819
1
      return getIndirectResult(Ty, true, State);
1820
1821
    // Ignore empty structs/unions on non-Windows.
1822
417
    if (!IsWin32StructABI && 
isEmptyRecord(getContext(), Ty, true)249
)
1823
24
      return ABIArgInfo::getIgnore();
1824
1825
393
    llvm::LLVMContext &LLVMContext = getVMContext();
1826
393
    llvm::IntegerType *Int32 = llvm::Type::getInt32Ty(LLVMContext);
1827
393
    bool NeedsPadding = false;
1828
393
    bool InReg;
1829
393
    if (shouldAggregateUseDirect(Ty, State, InReg, NeedsPadding)) {
1830
16
      unsigned SizeInRegs = (TI.Width + 31) / 32;
1831
16
      SmallVector<llvm::Type*, 3> Elements(SizeInRegs, Int32);
1832
16
      llvm::Type *Result = llvm::StructType::get(LLVMContext, Elements);
1833
16
      if (InReg)
1834
9
        return ABIArgInfo::getDirectInReg(Result);
1835
7
      else
1836
7
        return ABIArgInfo::getDirect(Result);
1837
16
    }
1838
377
    llvm::IntegerType *PaddingType = NeedsPadding ? 
Int323
:
nullptr374
;
1839
1840
    // Pass over-aligned aggregates on Windows indirectly. This behavior was
1841
    // added in MSVC 2015.
1842
377
    if (IsWin32StructABI && 
TI.AlignIsRequired168
&&
TI.Align > 321
)
1843
1
      return getIndirectResult(Ty, /*ByVal=*/false, State);
1844
1845
    // Expand small (<= 128-bit) record types when we know that the stack layout
1846
    // of those arguments will match the struct. This is important because the
1847
    // LLVM backend isn't smart enough to remove byval, which inhibits many
1848
    // optimizations.
1849
    // Don't do this for the MCU if there are still free integer registers
1850
    // (see X86_64 ABI for full explanation).
1851
376
    if (TI.Width <= 4 * 32 && 
(329
!IsMCUABI329
||
State.FreeRegs == 05
) &&
1852
376
        
canExpandIndirectArgument(Ty)326
)
1853
86
      return ABIArgInfo::getExpandWithPadding(
1854
86
          IsFastCall || 
IsVectorCall82
||
IsRegCall81
, PaddingType);
1855
1856
290
    return getIndirectResult(Ty, true, State);
1857
376
  }
1858
1859
31.6k
  if (const VectorType *VT = Ty->getAs<VectorType>()) {
1860
    // On Windows, vectors are passed directly if registers are available, or
1861
    // indirectly if not. This avoids the need to align argument memory. Pass
1862
    // user-defined vector types larger than 512 bits indirectly for simplicity.
1863
119
    if (IsWin32StructABI) {
1864
45
      if (TI.Width <= 512 && 
State.FreeSSERegs > 040
) {
1865
28
        --State.FreeSSERegs;
1866
28
        return ABIArgInfo::getDirectInReg();
1867
28
      }
1868
17
      return getIndirectResult(Ty, /*ByVal=*/false, State);
1869
45
    }
1870
1871
    // On Darwin, some vectors are passed in memory, we handle this by passing
1872
    // it as an i8/i16/i32/i64.
1873
74
    if (IsDarwinVectorABI) {
1874
24
      if ((TI.Width == 8 || TI.Width == 16 || TI.Width == 32) ||
1875
24
          
(22
TI.Width == 6422
&&
VT->getNumElements() == 110
))
1876
7
        return ABIArgInfo::getDirect(
1877
7
            llvm::IntegerType::get(getVMContext(), TI.Width));
1878
24
    }
1879
1880
67
    if (IsX86_MMXType(CGT.ConvertType(Ty)))
1881
8
      return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(), 64));
1882
1883
59
    return ABIArgInfo::getDirect();
1884
67
  }
1885
1886
1887
31.5k
  if (const EnumType *EnumTy = Ty->getAs<EnumType>())
1888
21
    Ty = EnumTy->getDecl()->getIntegerType();
1889
1890
31.5k
  bool InReg = shouldPrimitiveUseInReg(Ty, State);
1891
1892
31.5k
  if (isPromotableIntegerTypeForABI(Ty)) {
1893
244
    if (InReg)
1894
25
      return ABIArgInfo::getExtendInReg(Ty);
1895
219
    return ABIArgInfo::getExtend(Ty);
1896
244
  }
1897
1898
31.2k
  if (const auto * EIT = Ty->getAs<ExtIntType>()) {
1899
20
    if (EIT->getNumBits() <= 64) {
1900
4
      if (InReg)
1901
0
        return ABIArgInfo::getDirectInReg();
1902
4
      return ABIArgInfo::getDirect();
1903
4
    }
1904
16
    return getIndirectResult(Ty, /*ByVal=*/false, State);
1905
20
  }
1906
1907
31.2k
  if (InReg)
1908
159
    return ABIArgInfo::getDirectInReg();
1909
31.0k
  return ABIArgInfo::getDirect();
1910
31.2k
}
1911
1912
18.4k
void X86_32ABIInfo::computeInfo(CGFunctionInfo &FI) const {
1913
18.4k
  CCState State(FI);
1914
18.4k
  if (IsMCUABI)
1915
33
    State.FreeRegs = 3;
1916
18.4k
  else if (State.CC == llvm::CallingConv::X86_FastCall) {
1917
77
    State.FreeRegs = 2;
1918
77
    State.FreeSSERegs = 3;
1919
18.3k
  } else if (State.CC == llvm::CallingConv::X86_VectorCall) {
1920
57
    State.FreeRegs = 2;
1921
57
    State.FreeSSERegs = 6;
1922
18.3k
  } else if (FI.getHasRegParm())
1923
31
    State.FreeRegs = FI.getRegParm();
1924
18.2k
  else if (State.CC == llvm::CallingConv::X86_RegCall) {
1925
54
    State.FreeRegs = 5;
1926
54
    State.FreeSSERegs = 8;
1927
18.2k
  } else if (IsWin32StructABI) {
1928
    // Since MSVC 2015, the first three SSE vectors have been passed in
1929
    // registers. The rest are passed indirectly.
1930
3.99k
    State.FreeRegs = DefaultNumRegisterParameters;
1931
3.99k
    State.FreeSSERegs = 3;
1932
3.99k
  } else
1933
14.2k
    State.FreeRegs = DefaultNumRegisterParameters;
1934
1935
18.4k
  if (!::classifyReturnType(getCXXABI(), FI, *this)) {
1936
18.3k
    FI.getReturnInfo() = classifyReturnType(FI.getReturnType(), State);
1937
18.3k
  } else 
if (82
FI.getReturnInfo().isIndirect()82
) {
1938
    // The C++ ABI is not aware of register usage, so we have to check if the
1939
    // return value was sret and put it in a register ourselves if appropriate.
1940
82
    if (State.FreeRegs) {
1941
4
      --State.FreeRegs;  // The sret parameter consumes a register.
1942
4
      if (!IsMCUABI)
1943
4
        FI.getReturnInfo().setInReg(true);
1944
4
    }
1945
82
  }
1946
1947
  // The chain argument effectively gives us another free register.
1948
18.4k
  if (FI.isChainCall())
1949
4
    ++State.FreeRegs;
1950
1951
  // For vectorcall, do a first pass over the arguments, assigning FP and vector
1952
  // arguments to XMM registers as available.
1953
18.4k
  if (State.CC == llvm::CallingConv::X86_VectorCall)
1954
57
    runVectorCallFirstPass(FI, State);
1955
1956
18.4k
  bool UsedInAlloca = false;
1957
18.4k
  MutableArrayRef<CGFunctionInfoArgInfo> Args = FI.arguments();
1958
50.8k
  for (int I = 0, E = Args.size(); I < E; 
++I32.3k
) {
1959
    // Skip arguments that have already been assigned.
1960
32.3k
    if (State.IsPreassigned.test(I))
1961
38
      continue;
1962
1963
32.3k
    Args[I].info = classifyArgumentType(Args[I].type, State);
1964
32.3k
    UsedInAlloca |= (Args[I].info.getKind() == ABIArgInfo::InAlloca);
1965
32.3k
  }
1966
1967
  // If we needed to use inalloca for any argument, do a second pass and rewrite
1968
  // all the memory arguments to use inalloca.
1969
18.4k
  if (UsedInAlloca)
1970
73
    rewriteWithInAlloca(FI);
1971
18.4k
}
1972
1973
void
1974
X86_32ABIInfo::addFieldToArgStruct(SmallVector<llvm::Type *, 6> &FrameFields,
1975
                                   CharUnits &StackOffset, ABIArgInfo &Info,
1976
167
                                   QualType Type) const {
1977
  // Arguments are always 4-byte-aligned.
1978
167
  CharUnits WordSize = CharUnits::fromQuantity(4);
1979
167
  assert(StackOffset.isMultipleOf(WordSize) && "unaligned inalloca struct");
1980
1981
  // sret pointers and indirect things will require an extra pointer
1982
  // indirection, unless they are byval. Most things are byval, and will not
1983
  // require this indirection.
1984
0
  bool IsIndirect = false;
1985
167
  if (Info.isIndirect() && 
!Info.getIndirectByVal()17
)
1986
14
    IsIndirect = true;
1987
167
  Info = ABIArgInfo::getInAlloca(FrameFields.size(), IsIndirect);
1988
167
  llvm::Type *LLTy = CGT.ConvertTypeForMem(Type);
1989
167
  if (IsIndirect)
1990
14
    LLTy = LLTy->getPointerTo(0);
1991
167
  FrameFields.push_back(LLTy);
1992
167
  StackOffset += IsIndirect ? 
WordSize14
:
getContext().getTypeSizeInChars(Type)153
;
1993
1994
  // Insert padding bytes to respect alignment.
1995
167
  CharUnits FieldEnd = StackOffset;
1996
167
  StackOffset = FieldEnd.alignTo(WordSize);
1997
167
  if (StackOffset != FieldEnd) {
1998
40
    CharUnits NumBytes = StackOffset - FieldEnd;
1999
40
    llvm::Type *Ty = llvm::Type::getInt8Ty(getVMContext());
2000
40
    Ty = llvm::ArrayType::get(Ty, NumBytes.getQuantity());
2001
40
    FrameFields.push_back(Ty);
2002
40
  }
2003
167
}
2004
2005
177
static bool isArgInAlloca(const ABIArgInfo &Info) {
2006
  // Leave ignored and inreg arguments alone.
2007
177
  switch (Info.getKind()) {
2008
86
  case ABIArgInfo::InAlloca:
2009
86
    return true;
2010
0
  case ABIArgInfo::Ignore:
2011
0
  case ABIArgInfo::IndirectAliased:
2012
0
    return false;
2013
11
  case ABIArgInfo::Indirect:
2014
84
  case ABIArgInfo::Direct:
2015
90
  case ABIArgInfo::Extend:
2016
90
    return !Info.getInReg();
2017
1
  case ABIArgInfo::Expand:
2018
1
  case ABIArgInfo::CoerceAndExpand:
2019
    // These are aggregate types which are never passed in registers when
2020
    // inalloca is involved.
2021
1
    return true;
2022
177
  }
2023
0
  llvm_unreachable("invalid enum");
2024
0
}
2025
2026
73
void X86_32ABIInfo::rewriteWithInAlloca(CGFunctionInfo &FI) const {
2027
73
  assert(IsWin32StructABI && "inalloca only supported on win32");
2028
2029
  // Build a packed struct type for all of the arguments in memory.
2030
0
  SmallVector<llvm::Type *, 6> FrameFields;
2031
2032
  // The stack alignment is always 4.
2033
73
  CharUnits StackAlign = CharUnits::fromQuantity(4);
2034
2035
73
  CharUnits StackOffset;
2036
73
  CGFunctionInfo::arg_iterator I = FI.arg_begin(), E = FI.arg_end();
2037
2038
  // Put 'this' into the struct before 'sret', if necessary.
2039
73
  bool IsThisCall =
2040
73
      FI.getCallingConvention() == llvm::CallingConv::X86_ThisCall;
2041
73
  ABIArgInfo &Ret = FI.getReturnInfo();
2042
73
  if (Ret.isIndirect() && 
Ret.isSRetAfterThis()9
&&
!IsThisCall7
&&
2043
73
      
isArgInAlloca(I->info)6
) {
2044
5
    addFieldToArgStruct(FrameFields, StackOffset, I->info, I->type);
2045
5
    ++I;
2046
5
  }
2047
2048
  // Put the sret parameter into the inalloca struct if it's in memory.
2049
73
  if (Ret.isIndirect() && 
!Ret.getInReg()9
) {
2050
8
    addFieldToArgStruct(FrameFields, StackOffset, Ret, FI.getReturnType());
2051
    // On Windows, the hidden sret parameter is always returned in eax.
2052
8
    Ret.setInAllocaSRet(IsWin32StructABI);
2053
8
  }
2054
2055
  // Skip the 'this' parameter in ecx.
2056
73
  if (IsThisCall)
2057
17
    ++I;
2058
2059
  // Put arguments passed in memory into the struct.
2060
244
  for (; I != E; 
++I171
) {
2061
171
    if (isArgInAlloca(I->info))
2062
154
      addFieldToArgStruct(FrameFields, StackOffset, I->info, I->type);
2063
171
  }
2064
2065
73
  FI.setArgStruct(llvm::StructType::get(getVMContext(), FrameFields,
2066
73
                                        /*isPacked=*/true),
2067
73
                  StackAlign);
2068
73
}
2069
2070
Address X86_32ABIInfo::EmitVAArg(CodeGenFunction &CGF,
2071
23
                                 Address VAListAddr, QualType Ty) const {
2072
2073
23
  auto TypeInfo = getContext().getTypeInfoInChars(Ty);
2074
2075
  // x86-32 changes the alignment of certain arguments on the stack.
2076
  //
2077
  // Just messing with TypeInfo like this works because we never pass
2078
  // anything indirectly.
2079
23
  TypeInfo.Align = CharUnits::fromQuantity(
2080
23
                getTypeStackAlignInBytes(Ty, TypeInfo.Align.getQuantity()));
2081
2082
23
  return emitVoidPtrVAArg(CGF, VAListAddr, Ty, /*Indirect*/ false,
2083
23
                          TypeInfo, CharUnits::fromQuantity(4),
2084
23
                          /*AllowHigherAlign*/ true);
2085
23
}
2086
2087
bool X86_32TargetCodeGenInfo::isStructReturnInRegABI(
2088
3.06k
    const llvm::Triple &Triple, const CodeGenOptions &Opts) {
2089
3.06k
  assert(Triple.getArch() == llvm::Triple::x86);
2090
2091
0
  switch (Opts.getStructReturnConvention()) {
2092
3.06k
  case CodeGenOptions::SRCK_Default:
2093
3.06k
    break;
2094
3
  case CodeGenOptions::SRCK_OnStack:  // -fpcc-struct-return
2095
3
    return false;
2096
3
  case CodeGenOptions::SRCK_InRegs:  // -freg-struct-return
2097
3
    return true;
2098
3.06k
  }
2099
2100
3.06k
  if (Triple.isOSDarwin() || 
Triple.isOSIAMCU()2.88k
)
2101
177
    return true;
2102
2103
2.88k
  switch (Triple.getOS()) {
2104
1
  case llvm::Triple::DragonFly:
2105
1
  case llvm::Triple::FreeBSD:
2106
2
  case llvm::Triple::OpenBSD:
2107
432
  case llvm::Triple::Win32:
2108
432
    return true;
2109
2.45k
  default:
2110
2.45k
    return false;
2111
2.88k
  }
2112
2.88k
}
2113
2114
static void addX86InterruptAttrs(const FunctionDecl *FD, llvm::GlobalValue *GV,
2115
191k
                                 CodeGen::CodeGenModule &CGM) {
2116
191k
  if (!FD->hasAttr<AnyX86InterruptAttr>())
2117
191k
    return;
2118
2119
25
  llvm::Function *Fn = cast<llvm::Function>(GV);
2120
25
  Fn->setCallingConv(llvm::CallingConv::X86_INTR);
2121
25
  if (FD->getNumParams() == 0)
2122
0
    return;
2123
2124
25
  auto PtrTy = cast<PointerType>(FD->getParamDecl(0)->getType());
2125
25
  llvm::Type *ByValTy = CGM.getTypes().ConvertType(PtrTy->getPointeeType());
2126
25
  llvm::Attribute NewAttr = llvm::Attribute::getWithByValType(
2127
25
    Fn->getContext(), ByValTy);
2128
25
  Fn->addParamAttr(0, NewAttr);
2129
25
}
2130
2131
void X86_32TargetCodeGenInfo::setTargetAttributes(
2132
67.9k
    const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {
2133
67.9k
  if (GV->isDeclaration())
2134
42.4k
    return;
2135
25.5k
  if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) {
2136
21.1k
    if (FD->hasAttr<X86ForceAlignArgPointerAttr>()) {
2137
2
      llvm::Function *Fn = cast<llvm::Function>(GV);
2138
2
      Fn->addFnAttr("stackrealign");
2139
2
    }
2140
2141
21.1k
    addX86InterruptAttrs(FD, GV, CGM);
2142
21.1k
  }
2143
25.5k
}
2144
2145
bool X86_32TargetCodeGenInfo::initDwarfEHRegSizeTable(
2146
                                               CodeGen::CodeGenFunction &CGF,
2147
0
                                               llvm::Value *Address) const {
2148
0
  CodeGen::CGBuilderTy &Builder = CGF.Builder;
2149
2150
0
  llvm::Value *Four8 = llvm::ConstantInt::get(CGF.Int8Ty, 4);
2151
2152
  // 0-7 are the eight integer registers;  the order is different
2153
  //   on Darwin (for EH), but the range is the same.
2154
  // 8 is %eip.
2155
0
  AssignToArrayRange(Builder, Address, Four8, 0, 8);
2156
2157
0
  if (CGF.CGM.getTarget().getTriple().isOSDarwin()) {
2158
    // 12-16 are st(0..4).  Not sure why we stop at 4.
2159
    // These have size 16, which is sizeof(long double) on
2160
    // platforms with 8-byte alignment for that type.
2161
0
    llvm::Value *Sixteen8 = llvm::ConstantInt::get(CGF.Int8Ty, 16);
2162
0
    AssignToArrayRange(Builder, Address, Sixteen8, 12, 16);
2163
2164
0
  } else {
2165
    // 9 is %eflags, which doesn't get a size on Darwin for some
2166
    // reason.
2167
0
    Builder.CreateAlignedStore(
2168
0
        Four8, Builder.CreateConstInBoundsGEP1_32(CGF.Int8Ty, Address, 9),
2169
0
                               CharUnits::One());
2170
2171
    // 11-16 are st(0..5).  Not sure why we stop at 5.
2172
    // These have size 12, which is sizeof(long double) on
2173
    // platforms with 4-byte alignment for that type.
2174
0
    llvm::Value *Twelve8 = llvm::ConstantInt::get(CGF.Int8Ty, 12);
2175
0
    AssignToArrayRange(Builder, Address, Twelve8, 11, 16);
2176
0
  }
2177
2178
0
  return false;
2179
0
}
2180
2181
//===----------------------------------------------------------------------===//
2182
// X86-64 ABI Implementation
2183
//===----------------------------------------------------------------------===//
2184
2185
2186
namespace {
2187
/// The AVX ABI level for X86 targets.
2188
enum class X86AVXABILevel {
2189
  None,
2190
  AVX,
2191
  AVX512
2192
};
2193
2194
/// \p returns the size in bits of the largest (native) vector for \p AVXLevel.
2195
3.94k
static unsigned getNativeVectorSizeForAVXABI(X86AVXABILevel AVXLevel) {
2196
3.94k
  switch (AVXLevel) {
2197
3.01k
  case X86AVXABILevel::AVX512:
2198
3.01k
    return 512;
2199
696
  case X86AVXABILevel::AVX:
2200
696
    return 256;
2201
226
  case X86AVXABILevel::None:
2202
226
    return 128;
2203
3.94k
  }
2204
0
  llvm_unreachable("Unknown AVXLevel");
2205
0
}
2206
2207
/// X86_64ABIInfo - The X86_64 ABI information.
2208
class X86_64ABIInfo : public SwiftABIInfo {
2209
  enum Class {
2210
    Integer = 0,
2211
    SSE,
2212
    SSEUp,
2213
    X87,
2214
    X87Up,
2215
    ComplexX87,
2216
    NoClass,
2217
    Memory
2218
  };
2219
2220
  /// merge - Implement the X86_64 ABI merging algorithm.
2221
  ///
2222
  /// Merge an accumulating classification \arg Accum with a field
2223
  /// classification \arg Field.
2224
  ///
2225
  /// \param Accum - The accumulating classification. This should
2226
  /// always be either NoClass or the result of a previous merge
2227
  /// call. In addition, this should never be Memory (the caller
2228
  /// should just return Memory for the aggregate).
2229
  static Class merge(Class Accum, Class Field);
2230
2231
  /// postMerge - Implement the X86_64 ABI post merging algorithm.
2232
  ///
2233
  /// Post merger cleanup, reduces a malformed Hi and Lo pair to
2234
  /// final MEMORY or SSE classes when necessary.
2235
  ///
2236
  /// \param AggregateSize - The size of the current aggregate in
2237
  /// the classification process.
2238
  ///
2239
  /// \param Lo - The classification for the parts of the type
2240
  /// residing in the low word of the containing object.
2241
  ///
2242
  /// \param Hi - The classification for the parts of the type
2243
  /// residing in the higher words of the containing object.
2244
  ///
2245
  void postMerge(unsigned AggregateSize, Class &Lo, Class &Hi) const;
2246
2247
  /// classify - Determine the x86_64 register classes in which the
2248
  /// given type T should be passed.
2249
  ///
2250
  /// \param Lo - The classification for the parts of the type
2251
  /// residing in the low word of the containing object.
2252
  ///
2253
  /// \param Hi - The classification for the parts of the type
2254
  /// residing in the high word of the containing object.
2255
  ///
2256
  /// \param OffsetBase - The bit offset of this type in the
2257
  /// containing object.  Some parameters are classified different
2258
  /// depending on whether they straddle an eightbyte boundary.
2259
  ///
2260
  /// \param isNamedArg - Whether the argument in question is a "named"
2261
  /// argument, as used in AMD64-ABI 3.5.7.
2262
  ///
2263
  /// If a word is unused its result will be NoClass; if a type should
2264
  /// be passed in Memory then at least the classification of \arg Lo
2265
  /// will be Memory.
2266
  ///
2267
  /// The \arg Lo class will be NoClass iff the argument is ignored.
2268
  ///
2269
  /// If the \arg Lo class is ComplexX87, then the \arg Hi class will
2270
  /// also be ComplexX87.
2271
  void classify(QualType T, uint64_t OffsetBase, Class &Lo, Class &Hi,
2272
                bool isNamedArg) const;
2273
2274
  llvm::Type *GetByteVectorType(QualType Ty) const;
2275
  llvm::Type *GetSSETypeAtOffset(llvm::Type *IRType,
2276
                                 unsigned IROffset, QualType SourceTy,
2277
                                 unsigned SourceOffset) const;
2278
  llvm::Type *GetINTEGERTypeAtOffset(llvm::Type *IRType,
2279
                                     unsigned IROffset, QualType SourceTy,
2280
                                     unsigned SourceOffset) const;
2281
2282
  /// getIndirectResult - Give a source type \arg Ty, return a suitable result
2283
  /// such that the argument will be returned in memory.
2284
  ABIArgInfo getIndirectReturnResult(QualType Ty) const;
2285
2286
  /// getIndirectResult - Give a source type \arg Ty, return a suitable result
2287
  /// such that the argument will be passed in memory.
2288
  ///
2289
  /// \param freeIntRegs - The number of free integer registers remaining
2290
  /// available.
2291
  ABIArgInfo getIndirectResult(QualType Ty, unsigned freeIntRegs) const;
2292
2293
  ABIArgInfo classifyReturnType(QualType RetTy) const;
2294
2295
  ABIArgInfo classifyArgumentType(QualType Ty, unsigned freeIntRegs,
2296
                                  unsigned &neededInt, unsigned &neededSSE,
2297
                                  bool isNamedArg) const;
2298
2299
  ABIArgInfo classifyRegCallStructType(QualType Ty, unsigned &NeededInt,
2300
                                       unsigned &NeededSSE) const;
2301
2302
  ABIArgInfo classifyRegCallStructTypeImpl(QualType Ty, unsigned &NeededInt,
2303
                                           unsigned &NeededSSE) const;
2304
2305
  bool IsIllegalVectorType(QualType Ty) const;
2306
2307
  /// The 0.98 ABI revision clarified a lot of ambiguities,
2308
  /// unfortunately in ways that were not always consistent with
2309
  /// certain previous compilers.  In particular, platforms which
2310
  /// required strict binary compatibility with older versions of GCC
2311
  /// may need to exempt themselves.
2312
17
  bool honorsRevision0_98() const {
2313
17
    return !getTarget().getTriple().isOSDarwin();
2314
17
  }
2315
2316
  /// GCC classifies <1 x long long> as SSE but some platform ABIs choose to
2317
  /// classify it as INTEGER (for compatibility with older clang compilers).
2318
186
  bool classifyIntegerMMXAsSSE() const {
2319
    // Clang <= 3.8 did not do this.
2320
186
    if (getContext().getLangOpts().getClangABICompat() <=
2321
186
        LangOptions::ClangABI::Ver3_8)
2322
3
      return false;
2323
2324
183
    const llvm::Triple &Triple = getTarget().getTriple();
2325
183
    if (Triple.isOSDarwin() || 
Triple.getOS() == llvm::Triple::PS466
)
2326
122
      return false;
2327
61
    if (Triple.isOSFreeBSD() && 
Triple.getOSMajorVersion() >= 100
)
2328
0
      return false;
2329
61
    return true;
2330
61
  }
2331
2332
  // GCC classifies vectors of __int128 as memory.
2333
12.1k
  bool passInt128VectorsInMem() const {
2334
    // Clang <= 9.0 did not do this.
2335
12.1k
    if (getContext().getLangOpts().getClangABICompat() <=
2336
12.1k
        LangOptions::ClangABI::Ver9)
2337
36
      return false;
2338
2339
12.1k
    const llvm::Triple &T = getTarget().getTriple();
2340
12.1k
    return T.isOSLinux() || 
T.isOSNetBSD()11.1k
;
2341
12.1k
  }
2342
2343
  X86AVXABILevel AVXLevel;
2344
  // Some ABIs (e.g. X32 ABI and Native Client OS) use 32 bit pointers on
2345
  // 64-bit hardware.
2346
  bool Has64BitPointers;
2347
2348
public:
2349
  X86_64ABIInfo(CodeGen::CodeGenTypes &CGT, X86AVXABILevel AVXLevel) :
2350
      SwiftABIInfo(CGT), AVXLevel(AVXLevel),
2351
34.7k
      Has64BitPointers(CGT.getDataLayout().getPointerSize(0) == 8) {
2352
34.7k
  }
2353
2354
124
  bool isPassedUsingAVXType(QualType type) const {
2355
124
    unsigned neededInt, neededSSE;
2356
    // The freeIntRegs argument doesn't matter here.
2357
124
    ABIArgInfo info = classifyArgumentType(type, 0, neededInt, neededSSE,
2358
124
                                           /*isNamedArg*/true);
2359
124
    if (info.isDirect()) {
2360
121
      llvm::Type *ty = info.getCoerceToType();
2361
121
      if (llvm::VectorType *vectorTy = dyn_cast_or_null<llvm::VectorType>(ty))
2362
2
        return vectorTy->getPrimitiveSizeInBits().getFixedSize() > 128;
2363
121
    }
2364
122
    return false;
2365
124
  }
2366
2367
  void computeInfo(CGFunctionInfo &FI) const override;
2368
2369
  Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
2370
                    QualType Ty) const override;
2371
  Address EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr,
2372
                      QualType Ty) const override;
2373
2374
0
  bool has64BitPointers() const {
2375
0
    return Has64BitPointers;
2376
0
  }
2377
2378
  bool shouldPassIndirectlyForSwift(ArrayRef<llvm::Type*> scalars,
2379
302
                                    bool asReturnValue) const override {
2380
302
    return occupiesMoreThan(CGT, scalars, /*total*/ 4);
2381
302
  }
2382
0
  bool isSwiftErrorInRegister() const override {
2383
0
    return true;
2384
0
  }
2385
};
2386
2387
/// WinX86_64ABIInfo - The Windows X86_64 ABI information.
2388
class WinX86_64ABIInfo : public SwiftABIInfo {
2389
public:
2390
  WinX86_64ABIInfo(CodeGen::CodeGenTypes &CGT, X86AVXABILevel AVXLevel)
2391
      : SwiftABIInfo(CGT), AVXLevel(AVXLevel),
2392
389
        IsMingw64(getTarget().getTriple().isWindowsGNUEnvironment()) {}
2393
2394
  void computeInfo(CGFunctionInfo &FI) const override;
2395
2396
  Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
2397
                    QualType Ty) const override;
2398
2399
371
  bool isHomogeneousAggregateBaseType(QualType Ty) const override {
2400
    // FIXME: Assumes vectorcall is in use.
2401
371
    return isX86VectorTypeForVectorCall(getContext(), Ty);
2402
371
  }
2403
2404
  bool isHomogeneousAggregateSmallEnough(const Type *Ty,
2405
408
                                         uint64_t NumMembers) const override {
2406
    // FIXME: Assumes vectorcall is in use.
2407
408
    return isX86VectorCallAggregateSmallEnough(NumMembers);
2408
408
  }
2409
2410
  bool shouldPassIndirectlyForSwift(ArrayRef<llvm::Type *> scalars,
2411
25
                                    bool asReturnValue) const override {
2412
25
    return occupiesMoreThan(CGT, scalars, /*total*/ 4);
2413
25
  }
2414
2415
0
  bool isSwiftErrorInRegister() const override {
2416
0
    return true;
2417
0
  }
2418
2419
private:
2420
  ABIArgInfo classify(QualType Ty, unsigned &FreeSSERegs, bool IsReturnType,
2421
                      bool IsVectorCall, bool IsRegCall) const;
2422
  ABIArgInfo reclassifyHvaArgForVectorCall(QualType Ty, unsigned &FreeSSERegs,
2423
                                           const ABIArgInfo &current) const;
2424
2425
  X86AVXABILevel AVXLevel;
2426
2427
  bool IsMingw64;
2428
};
2429
2430
class X86_64TargetCodeGenInfo : public TargetCodeGenInfo {
2431
public:
2432
  X86_64TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, X86AVXABILevel AVXLevel)
2433
34.7k
      : TargetCodeGenInfo(std::make_unique<X86_64ABIInfo>(CGT, AVXLevel)) {}
2434
2435
124
  const X86_64ABIInfo &getABIInfo() const {
2436
124
    return static_cast<const X86_64ABIInfo&>(TargetCodeGenInfo::getABIInfo());
2437
124
  }
2438
2439
  /// Disable tail call on x86-64. The epilogue code before the tail jump blocks
2440
  /// autoreleaseRV/retainRV and autoreleaseRV/unsafeClaimRV optimizations.
2441
248
  bool markARCOptimizedReturnCallsAsNoTail() const override { return true; }
2442
2443
0
  int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const override {
2444
0
    return 7;
2445
0
  }
2446
2447
  bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
2448
0
                               llvm::Value *Address) const override {
2449
0
    llvm::Value *Eight8 = llvm::ConstantInt::get(CGF.Int8Ty, 8);
2450
2451
    // 0-15 are the 16 integer registers.
2452
    // 16 is %rip.
2453
0
    AssignToArrayRange(CGF.Builder, Address, Eight8, 0, 16);
2454
0
    return false;
2455
0
  }
2456
2457
  llvm::Type* adjustInlineAsmType(CodeGen::CodeGenFunction &CGF,
2458
                                  StringRef Constraint,
2459
829
                                  llvm::Type* Ty) const override {
2460
829
    return X86AdjustInlineAsmType(CGF, Constraint, Ty);
2461
829
  }
2462
2463
  bool isNoProtoCallVariadic(const CallArgList &args,
2464
733
                             const FunctionNoProtoType *fnType) const override {
2465
    // The default CC on x86-64 sets %al to the number of SSA
2466
    // registers used, and GCC sets this when calling an unprototyped
2467
    // function, so we override the default behavior.  However, don't do
2468
    // that when AVX types are involved: the ABI explicitly states it is
2469
    // undefined, and it doesn't work in practice because of how the ABI
2470
    // defines varargs anyway.
2471
733
    if (fnType->getCallConv() == CC_C) {
2472
733
      bool HasAVXType = false;
2473
733
      for (CallArgList::const_iterator
2474
855
             it = args.begin(), ie = args.end(); it != ie; 
++it122
) {
2475
124
        if (getABIInfo().isPassedUsingAVXType(it->Ty)) {
2476
2
          HasAVXType = true;
2477
2
          break;
2478
2
        }
2479
124
      }
2480
2481
733
      if (!HasAVXType)
2482
731
        return true;
2483
733
    }
2484
2485
2
    return TargetCodeGenInfo::isNoProtoCallVariadic(args, fnType);
2486
733
  }
2487
2488
  llvm::Constant *
2489
82
  getUBSanFunctionSignature(CodeGen::CodeGenModule &CGM) const override {
2490
82
    unsigned Sig = (0xeb << 0) | // jmp rel8
2491
82
                   (0x06 << 8) | //           .+0x08
2492
82
                   ('v' << 16) |
2493
82
                   ('2' << 24);
2494
82
    return llvm::ConstantInt::get(CGM.Int32Ty, Sig);
2495
82
  }
2496
2497
  void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
2498
397k
                           CodeGen::CodeGenModule &CGM) const override {
2499
397k
    if (GV->isDeclaration())
2500
223k
      return;
2501
173k
    if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) {
2502
162k
      if (FD->hasAttr<X86ForceAlignArgPointerAttr>()) {
2503
1
        llvm::Function *Fn = cast<llvm::Function>(GV);
2504
1
        Fn->addFnAttr("stackrealign");
2505
1
      }
2506
2507
162k
      addX86InterruptAttrs(FD, GV, CGM);
2508
162k
    }
2509
173k
  }
2510
2511
  void checkFunctionCallABI(CodeGenModule &CGM, SourceLocation CallLoc,
2512
                            const FunctionDecl *Caller,
2513
                            const FunctionDecl *Callee,
2514
                            const CallArgList &Args) const override;
2515
};
2516
2517
static void initFeatureMaps(const ASTContext &Ctx,
2518
                            llvm::StringMap<bool> &CallerMap,
2519
                            const FunctionDecl *Caller,
2520
                            llvm::StringMap<bool> &CalleeMap,
2521
13.2k
                            const FunctionDecl *Callee) {
2522
13.2k
  if (CalleeMap.empty() && 
CallerMap.empty()6.17k
) {
2523
    // The caller is potentially nullptr in the case where the call isn't in a
2524
    // function.  In this case, the getFunctionFeatureMap ensures we just get
2525
    // the TU level setting (since it cannot be modified by 'target'..
2526
6.17k
    Ctx.getFunctionFeatureMap(CallerMap, Caller);
2527
6.17k
    Ctx.getFunctionFeatureMap(CalleeMap, Callee);
2528
6.17k
  }
2529
13.2k
}
2530
2531
static bool checkAVXParamFeature(DiagnosticsEngine &Diag,
2532
                                 SourceLocation CallLoc,
2533
                                 const llvm::StringMap<bool> &CallerMap,
2534
                                 const llvm::StringMap<bool> &CalleeMap,
2535
                                 QualType Ty, StringRef Feature,
2536
13.2k
                                 bool IsArgument) {
2537
13.2k
  bool CallerHasFeat = CallerMap.lookup(Feature);
2538
13.2k
  bool CalleeHasFeat = CalleeMap.lookup(Feature);
2539
13.2k
  if (!CallerHasFeat && 
!CalleeHasFeat62
)
2540
56
    return Diag.Report(CallLoc, diag::warn_avx_calling_convention)
2541
56
           << IsArgument << Ty << Feature;
2542
2543
  // Mixing calling conventions here is very clearly an error.
2544
13.1k
  if (!CallerHasFeat || 
!CalleeHasFeat13.1k
)
2545
6
    return Diag.Report(CallLoc, diag::err_avx_calling_convention)
2546
6
           << IsArgument << Ty << Feature;
2547
2548
  // Else, both caller and callee have the required feature, so there is no need
2549
  // to diagnose.
2550
13.1k
  return false;
2551
13.1k
}
2552
2553
static bool checkAVXParam(DiagnosticsEngine &Diag, ASTContext &Ctx,
2554
                          SourceLocation CallLoc,
2555
                          const llvm::StringMap<bool> &CallerMap,
2556
                          const llvm::StringMap<bool> &CalleeMap, QualType Ty,
2557
13.2k
                          bool IsArgument) {
2558
13.2k
  uint64_t Size = Ctx.getTypeSize(Ty);
2559
13.2k
  if (Size > 256)
2560
6.25k
    return checkAVXParamFeature(Diag, CallLoc, CallerMap, CalleeMap, Ty,
2561
6.25k
                                "avx512f", IsArgument);
2562
2563
6.96k
  if (Size > 128)
2564
6.96k
    return checkAVXParamFeature(Diag, CallLoc, CallerMap, CalleeMap, Ty, "avx",
2565
6.96k
                                IsArgument);
2566
2567
0
  return false;
2568
6.96k
}
2569
2570
void X86_64TargetCodeGenInfo::checkFunctionCallABI(
2571
    CodeGenModule &CGM, SourceLocation CallLoc, const FunctionDecl *Caller,
2572
290k
    const FunctionDecl *Callee, const CallArgList &Args) const {
2573
290k
  llvm::StringMap<bool> CallerMap;
2574
290k
  llvm::StringMap<bool> CalleeMap;
2575
290k
  unsigned ArgIndex = 0;
2576
2577
  // We need to loop through the actual call arguments rather than the the
2578
  // function's parameters, in case this variadic.
2579
488k
  for (const CallArg &Arg : Args) {
2580
    // The "avx" feature changes how vectors >128 in size are passed. "avx512f"
2581
    // additionally changes how vectors >256 in size are passed. Like GCC, we
2582
    // warn when a function is called with an argument where this will change.
2583
    // Unlike GCC, we also error when it is an obvious ABI mismatch, that is,
2584
    // the caller and callee features are mismatched.
2585
    // Unfortunately, we cannot do this diagnostic in SEMA, since the callee can
2586
    // change its ABI with attribute-target after this call.
2587
488k
    if (Arg.getType()->isVectorType() &&
2588
488k
        
CGM.getContext().getTypeSize(Arg.getType()) > 12814.4k
) {
2589
7.63k
      initFeatureMaps(CGM.getContext(), CallerMap, Caller, CalleeMap, Callee);
2590
7.63k
      QualType Ty = Arg.getType();
2591
      // The CallArg seems to have desugared the type already, so for clearer
2592
      // diagnostics, replace it with the type in the FunctionDecl if possible.
2593
7.63k
      if (ArgIndex < Callee->getNumParams())
2594
7.60k
        Ty = Callee->getParamDecl(ArgIndex)->getType();
2595
2596
7.63k
      if (checkAVXParam(CGM.getDiags(), CGM.getContext(), CallLoc, CallerMap,
2597
7.63k
                        CalleeMap, Ty, /*IsArgument*/ true))
2598
54
        return;
2599
7.63k
    }
2600
488k
    ++ArgIndex;
2601
488k
  }
2602
2603
  // Check return always, as we don't have a good way of knowing in codegen
2604
  // whether this value is used, tail-called, etc.
2605
290k
  if (Callee->getReturnType()->isVectorType() &&
2606
290k
      
CGM.getContext().getTypeSize(Callee->getReturnType()) > 1289.87k
) {
2607
5.58k
    initFeatureMaps(CGM.getContext(), CallerMap, Caller, CalleeMap, Callee);
2608
5.58k
    checkAVXParam(CGM.getDiags(), CGM.getContext(), CallLoc, CallerMap,
2609
5.58k
                  CalleeMap, Callee->getReturnType(),
2610
5.58k
                  /*IsArgument*/ false);
2611
5.58k
  }
2612
290k
}
2613
2614
73
static std::string qualifyWindowsLibrary(llvm::StringRef Lib) {
2615
  // If the argument does not end in .lib, automatically add the suffix.
2616
  // If the argument contains a space, enclose it in quotes.
2617
  // This matches the behavior of MSVC.
2618
73
  bool Quote = (Lib.find(' ') != StringRef::npos);
2619
73
  std::string ArgStr = Quote ? 
"\""4
:
""69
;
2620
73
  ArgStr += Lib;
2621
73
  if (!Lib.endswith_insensitive(".lib") && 
!Lib.endswith_insensitive(".a")61
)
2622
61
    ArgStr += ".lib";
2623
73
  ArgStr += Quote ? 
"\""4
:
""69
;
2624
73
  return ArgStr;
2625
73
}
2626
2627
class WinX86_32TargetCodeGenInfo : public X86_32TargetCodeGenInfo {
2628
public:
2629
  WinX86_32TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT,
2630
        bool DarwinVectorABI, bool RetSmallStructInRegABI, bool Win32StructABI,
2631
        unsigned NumRegisterParameters)
2632
    : X86_32TargetCodeGenInfo(CGT, DarwinVectorABI, RetSmallStructInRegABI,
2633
432
        Win32StructABI, NumRegisterParameters, false) {}
2634
2635
  void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
2636
                           CodeGen::CodeGenModule &CGM) const override;
2637
2638
  void getDependentLibraryOption(llvm::StringRef Lib,
2639
31
                                 llvm::SmallString<24> &Opt) const override {
2640
31
    Opt = "/DEFAULTLIB:";
2641
31
    Opt += qualifyWindowsLibrary(Lib);
2642
31
  }
2643
2644
  void getDetectMismatchOption(llvm::StringRef Name,
2645
                               llvm::StringRef Value,
2646
2
                               llvm::SmallString<32> &Opt) const override {
2647
2
    Opt = "/FAILIFMISMATCH:\"" + Name.str() + "=" + Value.str() + "\"";
2648
2
  }
2649
};
2650
2651
static void addStackProbeTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
2652
17.3k
                                          CodeGen::CodeGenModule &CGM) {
2653
17.3k
  if (llvm::Function *Fn = dyn_cast_or_null<llvm::Function>(GV)) {
2654
2655
14.9k
    if (CGM.getCodeGenOpts().StackProbeSize != 4096)
2656
1
      Fn->addFnAttr("stack-probe-size",
2657
1
                    llvm::utostr(CGM.getCodeGenOpts().StackProbeSize));
2658
14.9k
    if (CGM.getCodeGenOpts().NoStackArgProbe)
2659
4
      Fn->addFnAttr("no-stack-arg-probe");
2660
14.9k
  }
2661
17.3k
}
2662
2663
void WinX86_32TargetCodeGenInfo::setTargetAttributes(
2664
17.7k
    const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {
2665
17.7k
  X86_32TargetCodeGenInfo::setTargetAttributes(D, GV, CGM);
2666
17.7k
  if (GV->isDeclaration())
2667
10.0k
    return;
2668
7.62k
  addStackProbeTargetAttributes(D, GV, CGM);
2669
7.62k
}
2670
2671
class WinX86_64TargetCodeGenInfo : public TargetCodeGenInfo {
2672
public:
2673
  WinX86_64TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT,
2674
                             X86AVXABILevel AVXLevel)
2675
378
      : TargetCodeGenInfo(std::make_unique<WinX86_64ABIInfo>(CGT, AVXLevel)) {}
2676
2677
  void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
2678
                           CodeGen::CodeGenModule &CGM) const override;
2679
2680
0
  int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const override {
2681
0
    return 7;
2682
0
  }
2683
2684
  bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
2685
0
                               llvm::Value *Address) const override {
2686
0
    llvm::Value *Eight8 = llvm::ConstantInt::get(CGF.Int8Ty, 8);
2687
2688
    // 0-15 are the 16 integer registers.
2689
    // 16 is %rip.
2690
0
    AssignToArrayRange(CGF.Builder, Address, Eight8, 0, 16);
2691
0
    return false;
2692
0
  }
2693
2694
  void getDependentLibraryOption(llvm::StringRef Lib,
2695
31
                                 llvm::SmallString<24> &Opt) const override {
2696
31
    Opt = "/DEFAULTLIB:";
2697
31
    Opt += qualifyWindowsLibrary(Lib);
2698
31
  }
2699
2700
  void getDetectMismatchOption(llvm::StringRef Name,
2701
                               llvm::StringRef Value,
2702
4
                               llvm::SmallString<32> &Opt) const override {
2703
4
    Opt = "/FAILIFMISMATCH:\"" + Name.str() + "=" + Value.str() + "\"";
2704
4
  }
2705
};
2706
2707
void WinX86_64TargetCodeGenInfo::setTargetAttributes(
2708
19.5k
    const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {
2709
19.5k
  TargetCodeGenInfo::setTargetAttributes(D, GV, CGM);
2710
19.5k
  if (GV->isDeclaration())
2711
10.6k
    return;
2712
8.91k
  if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) {
2713
7.82k
    if (FD->hasAttr<X86ForceAlignArgPointerAttr>()) {
2714
0
      llvm::Function *Fn = cast<llvm::Function>(GV);
2715
0
      Fn->addFnAttr("stackrealign");
2716
0
    }
2717
2718
7.82k
    addX86InterruptAttrs(FD, GV, CGM);
2719
7.82k
  }
2720
2721
8.91k
  addStackProbeTargetAttributes(D, GV, CGM);
2722
8.91k
}
2723
}
2724
2725
void X86_64ABIInfo::postMerge(unsigned AggregateSize, Class &Lo,
2726
13.1k
                              Class &Hi) const {
2727
  // AMD64-ABI 3.2.3p2: Rule 5. Then a post merger cleanup is done:
2728
  //
2729
  // (a) If one of the classes is Memory, the whole argument is passed in
2730
  //     memory.
2731
  //
2732
  // (b) If X87UP is not preceded by X87, the whole argument is passed in
2733
  //     memory.
2734
  //
2735
  // (c) If the size of the aggregate exceeds two eightbytes and the first
2736
  //     eightbyte isn't SSE or any other eightbyte isn't SSEUP, the whole
2737
  //     argument is passed in memory. NOTE: This is necessary to keep the
2738
  //     ABI working for processors that don't support the __m256 type.
2739
  //
2740
  // (d) If SSEUP is not preceded by SSE or SSEUP, it is converted to SSE.
2741
  //
2742
  // Some of these are enforced by the merging logic.  Others can arise
2743
  // only with unions; for example:
2744
  //   union { _Complex double; unsigned; }
2745
  //
2746
  // Note that clauses (b) and (c) were added in 0.98.
2747
  //
2748
13.1k
  if (Hi == Memory)
2749
0
    Lo = Memory;
2750
13.1k
  if (Hi == X87Up && 
Lo != X87366
&&
honorsRevision0_98()17
)
2751
15
    Lo = Memory;
2752
13.1k
  if (AggregateSize > 128 && 
(328
Lo != SSE328
||
Hi != SSEUp24
))
2753
304
    Lo = Memory;
2754
13.1k
  if (Hi == SSEUp && 
Lo != SSE33
)
2755
1
    Hi = SSE;
2756
13.1k
}
2757
2758
19.7k
X86_64ABIInfo::Class X86_64ABIInfo::merge(Class Accum, Class Field) {
2759
  // AMD64-ABI 3.2.3p2: Rule 4. Each field of an object is
2760
  // classified recursively so that always two fields are
2761
  // considered. The resulting class is calculated according to
2762
  // the classes of the fields in the eightbyte:
2763
  //
2764
  // (a) If both classes are equal, this is the resulting class.
2765
  //
2766
  // (b) If one of the classes is NO_CLASS, the resulting class is
2767
  // the other class.
2768
  //
2769
  // (c) If one of the classes is MEMORY, the result is the MEMORY
2770
  // class.
2771
  //
2772
  // (d) If one of the classes is INTEGER, the result is the
2773
  // INTEGER.
2774
  //
2775
  // (e) If one of the classes is X87, X87UP, COMPLEX_X87 class,
2776
  // MEMORY is used as class.
2777
  //
2778
  // (f) Otherwise class SSE is used.
2779
2780
  // Accum should never be memory (we should have returned) or
2781
  // ComplexX87 (because this cannot be passed in a structure).
2782
19.7k
  assert((Accum != Memory && Accum != ComplexX87) &&
2783
19.7k
         "Invalid accumulated classification during merge.");
2784
19.7k
  if (Accum == Field || 
Field == NoClass10.3k
)
2785
10.9k
    return Accum;
2786
8.81k
  if (Field == Memory)
2787
15
    return Memory;
2788
8.79k
  if (Accum == NoClass)
2789
8.69k
    return Field;
2790
101
  if (Accum == Integer || 
Field == Integer73
)
2791
98
    return Integer;
2792
3
  if (Field == X87 || Field == X87Up || Field == ComplexX87 ||
2793
3
      Accum == X87 || 
Accum == X87Up0
)
2794
3
    return Memory;
2795
0
  return SSE;
2796
3
}
2797
2798
void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase,
2799
470k
                             Class &Lo, Class &Hi, bool isNamedArg) const {
2800
  // FIXME: This code can be simplified by introducing a simple value class for
2801
  // Class pairs with appropriate constructor methods for the various
2802
  // situations.
2803
2804
  // FIXME: Some of the split computations are wrong; unaligned vectors
2805
  // shouldn't be passed in registers for example, so there is no chance they
2806
  // can straddle an eightbyte. Verify & simplify.
2807
2808
470k
  Lo = Hi = NoClass;
2809
2810
470k
  Class &Current = OffsetBase < 64 ? 
Lo468k
:
Hi1.64k
;
2811
470k
  Current = Memory;
2812
2813
470k
  if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
2814
181k
    BuiltinType::Kind k = BT->getKind();
2815
2816
181k
    if (k == BuiltinType::Void) {
2817
65.5k
      Current = NoClass;
2818
115k
    } else if (k == BuiltinType::Int128 || 
k == BuiltinType::UInt128115k
) {
2819
32
      Lo = Integer;
2820
32
      Hi = Integer;
2821
115k
    } else if (k >= BuiltinType::Bool && 
k <= BuiltinType::LongLong115k
) {
2822
110k
      Current = Integer;
2823
110k
    } else 
if (5.29k
k == BuiltinType::Float5.29k
||
k == BuiltinType::Double3.11k
||
2824
5.29k
               
k == BuiltinType::Float161.45k
) {
2825
3.95k
      Current = SSE;
2826
3.95k
    } else 
if (1.33k
k == BuiltinType::LongDouble1.33k
) {
2827
1.01k
      const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();
2828
1.01k
      if (LDF == &llvm::APFloat::IEEEquad()) {
2829
40
        Lo = SSE;
2830
40
        Hi = SSEUp;
2831
971
      } else if (LDF == &llvm::APFloat::x87DoubleExtended()) {
2832
949
        Lo = X87;
2833
949
        Hi = X87Up;
2834
949
      } else 
if (22
LDF == &llvm::APFloat::IEEEdouble()22
) {
2835
22
        Current = SSE;
2836
22
      } else
2837
0
        llvm_unreachable("unexpected long double representation!");
2838
1.01k
    }
2839
    // FIXME: _Decimal32 and _Decimal64 are SSE.
2840
    // FIXME: _float128 and _Decimal128 are (SSE, SSEUp).
2841
181k
    return;
2842
181k
  }
2843
2844
288k
  if (const EnumType *ET = Ty->getAs<EnumType>()) {
2845
    // Classify the underlying integer type.
2846
952
    classify(ET->getDecl()->getIntegerType(), OffsetBase, Lo, Hi, isNamedArg);
2847
952
    return;
2848
952
  }
2849
2850
287k
  if (Ty->hasPointerRepresentation()) {
2851
266k
    Current = Integer;
2852
266k
    return;
2853
266k
  }
2854
2855
21.0k
  if (Ty->isMemberPointerType()) {
2856
121
    if (Ty->isMemberFunctionPointerType()) {
2857
75
      if (Has64BitPointers) {
2858
        // If Has64BitPointers, this is an {i64, i64}, so classify both
2859
        // Lo and Hi now.
2860
67
        Lo = Hi = Integer;
2861
67
      } else {
2862
        // Otherwise, with 32-bit pointers, this is an {i32, i32}. If that
2863
        // straddles an eightbyte boundary, Hi should be classified as well.
2864
8
        uint64_t EB_FuncPtr = (OffsetBase) / 64;
2865
8
        uint64_t EB_ThisAdj = (OffsetBase + 64 - 1) / 64;
2866
8
        if (EB_FuncPtr != EB_ThisAdj) {
2867
2
          Lo = Hi = Integer;
2868
6
        } else {
2869
6
          Current = Integer;
2870
6
        }
2871
8
      }
2872
75
    } else {
2873
46
      Current = Integer;
2874
46
    }
2875
121
    return;
2876
121
  }
2877
2878
20.9k
  if (const VectorType *VT = Ty->getAs<VectorType>()) {
2879
6.44k
    uint64_t Size = getContext().getTypeSize(VT);
2880
6.44k
    if (Size == 1 || Size == 8 || 
Size == 166.44k
||
Size == 326.44k
) {
2881
      // gcc passes the following as integer:
2882
      // 4 bytes - <4 x char>, <2 x short>, <1 x int>, <1 x float>
2883
      // 2 bytes - <2 x char>, <1 x short>
2884
      // 1 byte  - <1 x char>
2885
5
      Current = Integer;
2886
2887
      // If this type crosses an eightbyte boundary, it should be
2888
      // split.
2889
5
      uint64_t EB_Lo = (OffsetBase) / 64;
2890
5
      uint64_t EB_Hi = (OffsetBase + Size - 1) / 64;
2891
5
      if (EB_Lo != EB_Hi)
2892
0
        Hi = Lo;
2893
6.44k
    } else if (Size == 64) {
2894
186
      QualType ElementType = VT->getElementType();
2895
2896
      // gcc passes <1 x double> in memory. :(
2897
186
      if (ElementType->isSpecificBuiltinType(BuiltinType::Double))
2898
0
        return;
2899
2900
      // gcc passes <1 x long long> as SSE but clang used to unconditionally
2901
      // pass them as integer.  For platforms where clang is the de facto
2902
      // platform compiler, we must continue to use integer.
2903
186
      if (!classifyIntegerMMXAsSSE() &&
2904
186
          
(125
ElementType->isSpecificBuiltinType(BuiltinType::LongLong)125
||
2905
125
           
ElementType->isSpecificBuiltinType(BuiltinType::ULongLong)45
||
2906
125
           
ElementType->isSpecificBuiltinType(BuiltinType::Long)45
||
2907
125
           
ElementType->isSpecificBuiltinType(BuiltinType::ULong)45
))
2908
80
        Current = Integer;
2909
106
      else
2910
106
        Current = SSE;
2911
2912
      // If this type crosses an eightbyte boundary, it should be
2913
      // split.
2914
186
      if (OffsetBase && 
OffsetBase != 640
)
2915
0
        Hi = Lo;
2916
6.25k
    } else if (Size == 128 ||
2917
6.25k
               
(3.74k
isNamedArg3.74k
&&
Size <= getNativeVectorSizeForAVXABI(AVXLevel)3.71k
)) {
2918
6.07k
      QualType ElementType = VT->getElementType();
2919
2920
      // gcc passes 256 and 512 bit <X x __int128> vectors in memory. :(
2921
6.07k
      if (passInt128VectorsInMem() && 
Size != 128514
&&
2922
6.07k
          
(255
ElementType->isSpecificBuiltinType(BuiltinType::Int128)255
||
2923
255
           ElementType->isSpecificBuiltinType(BuiltinType::UInt128)))
2924
6
        return;
2925
2926
      // Arguments of 256-bits are split into four eightbyte chunks. The
2927
      // least significant one belongs to class SSE and all the others to class
2928
      // SSEUP. The original Lo and Hi design considers that types can't be
2929
      // greater than 128-bits, so a 64-bit split in Hi and Lo makes sense.
2930
      // This design isn't correct for 256-bits, but since there're no cases
2931
      // where the upper parts would need to be inspected, avoid adding
2932
      // complexity and just consider Hi to match the 64-256 part.
2933
      //
2934
      // Note that per 3.5.7 of AMD64-ABI, 256-bit args are only passed in
2935
      // registers if they are "named", i.e. not part of the "..." of a
2936
      // variadic function.
2937
      //
2938
      // Similarly, per 3.2.3. of the AVX512 draft, 512-bits ("named") args are
2939
      // split into eight eightbyte chunks, one SSE and seven SSEUP.
2940
6.06k
      Lo = SSE;
2941
6.06k
      Hi = SSEUp;
2942
6.06k
    }
2943
6.44k
    return;
2944
6.44k
  }
2945
2946
14.5k
  if (const ComplexType *CT = Ty->getAs<ComplexType>()) {
2947
404
    QualType ET = getContext().getCanonicalType(CT->getElementType());
2948
2949
404
    uint64_t Size = getContext().getTypeSize(Ty);
2950
404
    if (ET->isIntegralOrEnumerationType()) {
2951
34
      if (Size <= 64)
2952
30
        Current = Integer;
2953
4
      else if (Size <= 128)
2954
4
        Lo = Hi = Integer;
2955
370
    } else if (ET->isFloat16Type() || 
ET == getContext().FloatTy361
) {
2956
148
      Current = SSE;
2957
222
    } else if (ET == getContext().DoubleTy) {
2958
113
      Lo = Hi = SSE;
2959
113
    } else 
if (109
ET == getContext().LongDoubleTy109
) {
2960
108
      const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();
2961
108
      if (LDF == &llvm::APFloat::IEEEquad())
2962
8
        Current = Memory;
2963
100
      else if (LDF == &llvm::APFloat::x87DoubleExtended())
2964
92
        Current = ComplexX87;
2965
8
      else if (LDF == &llvm::APFloat::IEEEdouble())
2966
8
        Lo = Hi = SSE;
2967
0
      else
2968
0
        llvm_unreachable("unexpected long double representation!");
2969
108
    }
2970
2971
    // If this complex type crosses an eightbyte boundary then it
2972
    // should be split.
2973
404
    uint64_t EB_Real = (OffsetBase) / 64;
2974
404
    uint64_t EB_Imag = (OffsetBase + getContext().getTypeSize(ET)) / 64;
2975
404
    if (Hi == NoClass && 
EB_Real != EB_Imag279
)
2976
104
      Hi = Lo;
2977
2978
404
    return;
2979
404
  }
2980
2981
14.0k
  if (const auto *EITy = Ty->getAs<ExtIntType>()) {
2982
86
    if (EITy->getNumBits() <= 64)
2983
58
      Current = Integer;
2984
28
    else if (EITy->getNumBits() <= 128)
2985
18
      Lo = Hi = Integer;
2986
    // Larger values need to get passed in memory.
2987
86
    return;
2988
86
  }
2989
2990
14.0k
  if (const ConstantArrayType *AT = getContext().getAsConstantArrayType(Ty)) {
2991
    // Arrays are treated like structures.
2992
2993
54
    uint64_t Size = getContext().getTypeSize(Ty);
2994
2995
    // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger
2996
    // than eight eightbytes, ..., it has class MEMORY.
2997
54
    if (Size > 512)
2998
0
      return;
2999
3000
    // AMD64-ABI 3.2.3p2: Rule 1. If ..., or it contains unaligned
3001
    // fields, it has class MEMORY.
3002
    //
3003
    // Only need to check alignment of array base.
3004
54
    if (OffsetBase % getContext().getTypeAlign(AT->getElementType()))
3005
0
      return;
3006
3007
    // Otherwise implement simplified merge. We could be smarter about
3008
    // this, but it isn't worth it and would be harder to verify.
3009
54
    Current = NoClass;
3010
54
    uint64_t EltSize = getContext().getTypeSize(AT->getElementType());
3011
54
    uint64_t ArraySize = AT->getSize().getZExtValue();
3012
3013
    // The only case a 256-bit wide vector could be used is when the array
3014
    // contains a single 256-bit element. Since Lo and Hi logic isn't extended
3015
    // to work for sizes wider than 128, early check and fallback to memory.
3016
    //
3017
54
    if (Size > 128 &&
3018
54
        
(15
Size != EltSize15
||
Size > getNativeVectorSizeForAVXABI(AVXLevel)4
))
3019
11
      return;
3020
3021
196
    
for (uint64_t i=0, Offset=OffsetBase; 43
i<ArraySize;
++i, Offset += EltSize153
) {
3022
153
      Class FieldLo, FieldHi;
3023
153
      classify(AT->getElementType(), Offset, FieldLo, FieldHi, isNamedArg);
3024
153
      Lo = merge(Lo, FieldLo);
3025
153
      Hi = merge(Hi, FieldHi);
3026
153
      if (Lo == Memory || Hi == Memory)
3027
0
        break;
3028
153
    }
3029
3030
43
    postMerge(Size, Lo, Hi);
3031
43
    assert((Hi != SSEUp || Lo == SSE) && "Invalid SSEUp array classification.");
3032
0
    return;
3033
54
  }
3034
3035
13.9k
  if (const RecordType *RT = Ty->getAs<RecordType>()) {
3036
13.7k
    uint64_t Size = getContext().getTypeSize(Ty);
3037
3038
    // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger
3039
    // than eight eightbytes, ..., it has class MEMORY.
3040
13.7k
    if (Size > 512)
3041
138
      return;
3042
3043
    // AMD64-ABI 3.2.3p2: Rule 2. If a C++ object has either a non-trivial
3044
    // copy constructor or a non-trivial destructor, it is passed by invisible
3045
    // reference.
3046
13.5k
    if (getRecordArgABI(RT, getCXXABI()))
3047
437
      return;
3048
3049
13.1k
    const RecordDecl *RD = RT->getDecl();
3050
3051
    // Assume variable sized types are passed in memory.
3052
13.1k
    if (RD->hasFlexibleArrayMember())
3053
0
      return;
3054
3055
13.1k
    const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
3056
3057
    // Reset Lo class, this will be recomputed.
3058
13.1k
    Current = NoClass;
3059
3060
    // If this is a C++ record, classify the bases first.
3061
13.1k
    if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
3062
12.5k
      for (const auto &I : CXXRD->bases()) {
3063
1.21k
        assert(!I.isVirtual() && !I.getType()->isDependentType() &&
3064
1.21k
               "Unexpected base class!");
3065
0
        const auto *Base =
3066
1.21k
            cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
3067
3068
        // Classify this field.
3069
        //
3070
        // AMD64-ABI 3.2.3p2: Rule 3. If the size of the aggregate exceeds a
3071
        // single eightbyte, each is classified separately. Each eightbyte gets
3072
        // initialized to class NO_CLASS.
3073
1.21k
        Class FieldLo, FieldHi;
3074
1.21k
        uint64_t Offset =
3075
1.21k
          OffsetBase + getContext().toBits(Layout.getBaseClassOffset(Base));
3076
1.21k
        classify(I.getType(), Offset, FieldLo, FieldHi, isNamedArg);
3077
1.21k
        Lo = merge(Lo, FieldLo);
3078
1.21k
        Hi = merge(Hi, FieldHi);
3079
1.21k
        if (Lo == Memory || 
Hi == Memory1.21k
) {
3080
6
          postMerge(Size, Lo, Hi);
3081
6
          return;
3082
6
        }
3083
1.21k
      }
3084
12.5k
    }
3085
3086
    // Classify the fields one at a time, merging the results.
3087
13.1k
    unsigned idx = 0;
3088
13.1k
    bool UseClang11Compat = getContext().getLangOpts().getClangABICompat() <=
3089
13.1k
                                LangOptions::ClangABI::Ver11 ||
3090
13.1k
                            
getContext().getTargetInfo().getTriple().isPS4()13.0k
;
3091
13.1k
    bool IsUnion = RT->isUnionType() && 
!UseClang11Compat67
;
3092
3093
13.1k
    for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
3094
21.6k
           i != e; 
++i, ++idx8.49k
) {
3095
8.81k
      uint64_t Offset = OffsetBase + Layout.getFieldOffset(idx);
3096
8.81k
      bool BitField = i->isBitField();
3097
3098
      // Ignore padding bit-fields.
3099
8.81k
      if (BitField && 
i->isUnnamedBitfield()42
)
3100
12
        continue;
3101
3102
      // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger than
3103
      // eight eightbytes, or it contains unaligned fields, it has class MEMORY.
3104
      //
3105
      // The only case a 256-bit or a 512-bit wide vector could be used is when
3106
      // the struct contains a single 256-bit or 512-bit element. Early check
3107
      // and fallback to memory.
3108
      //
3109
      // FIXME: Extended the Lo and Hi logic properly to work for size wider
3110
      // than 128.
3111
8.80k
      if (Size > 128 &&
3112
8.80k
          
(325
(325
!IsUnion325
&&
Size != getContext().getTypeSize(i->getType())314
) ||
3113
325
           
Size > getNativeVectorSizeForAVXABI(AVXLevel)86
)) {
3114
292
        Lo = Memory;
3115
292
        postMerge(Size, Lo, Hi);
3116
292
        return;
3117
292
      }
3118
      // Note, skip this test for bit-fields, see below.
3119
8.51k
      if (!BitField && 
Offset % getContext().getTypeAlign(i->getType())8.48k
) {
3120
20
        Lo = Memory;
3121
20
        postMerge(Size, Lo, Hi);
3122
20
        return;
3123
20
      }
3124
3125
      // Classify this field.
3126
      //
3127
      // AMD64-ABI 3.2.3p2: Rule 3. If the size of the aggregate
3128
      // exceeds a single eightbyte, each is classified
3129
      // separately. Each eightbyte gets initialized to class
3130
      // NO_CLASS.
3131
8.49k
      Class FieldLo, FieldHi;
3132
3133
      // Bit-fields require special handling, they do not force the
3134
      // structure to be passed in memory even if unaligned, and
3135
      // therefore they can straddle an eightbyte.
3136
8.49k
      if (BitField) {
3137
30
        assert(!i->isUnnamedBitfield());
3138
0
        uint64_t Offset = OffsetBase + Layout.getFieldOffset(idx);
3139
30
        uint64_t Size = i->getBitWidthValue(getContext());
3140
3141
30
        uint64_t EB_Lo = Offset / 64;
3142
30
        uint64_t EB_Hi = (Offset + Size - 1) / 64;
3143
3144
30
        if (EB_Lo) {
3145
0
          assert(EB_Hi == EB_Lo && "Invalid classification, type > 16 bytes.");
3146
0
          FieldLo = NoClass;
3147
0
          FieldHi = Integer;
3148
30
        } else {
3149
30
          FieldLo = Integer;
3150
30
          FieldHi = EB_Hi ? 
Integer0
: NoClass;
3151
30
        }
3152
30
      } else
3153
8.46k
        classify(i->getType(), Offset, FieldLo, FieldHi, isNamedArg);
3154
0
      Lo = merge(Lo, FieldLo);
3155
8.49k
      Hi = merge(Hi, FieldHi);
3156
8.49k
      if (Lo == Memory || 
Hi == Memory8.48k
)
3157
12
        break;
3158
8.49k
    }
3159
3160
12.8k
    postMerge(Size, Lo, Hi);
3161
12.8k
  }
3162
13.9k
}
3163
3164
456
ABIArgInfo X86_64ABIInfo::getIndirectReturnResult(QualType Ty) const {
3165
  // If this is a scalar LLVM value then assume LLVM will pass it in the right
3166
  // place naturally.
3167
456
  if (!isAggregateTypeForABI(Ty)) {
3168
    // Treat an enum type as its underlying type.
3169
235
    if (const EnumType *EnumTy = Ty->getAs<EnumType>())
3170
0
      Ty = EnumTy->getDecl()->getIntegerType();
3171
3172
235
    if (Ty->isExtIntType())
3173
3
      return getNaturalAlignIndirect(Ty);
3174
3175
232
    return (isPromotableIntegerTypeForABI(Ty) ? 
ABIArgInfo::getExtend(Ty)0
3176
232
                                              : ABIArgInfo::getDirect());
3177
235
  }
3178
3179
221
  return getNaturalAlignIndirect(Ty);
3180
456
}
3181
3182
2.94k
bool X86_64ABIInfo::IsIllegalVectorType(QualType Ty) const {
3183
2.94k
  if (const VectorType *VecTy = Ty->getAs<VectorType>()) {
3184
138
    uint64_t Size = getContext().getTypeSize(VecTy);
3185
138
    unsigned LargestVector = getNativeVectorSizeForAVXABI(AVXLevel);
3186
138
    if (Size <= 64 || 
Size > LargestVector132
)
3187
117
      return true;
3188
21
    QualType EltTy = VecTy->getElementType();
3189
21
    if (passInt128VectorsInMem() &&
3190
21
        
(9
EltTy->isSpecificBuiltinType(BuiltinType::Int128)9
||
3191
9
         EltTy->isSpecificBuiltinType(BuiltinType::UInt128)))
3192
6
      return true;
3193
21
  }
3194
3195
2.81k
  return false;
3196
2.94k
}
3197
3198
ABIArgInfo X86_64ABIInfo::getIndirectResult(QualType Ty,
3199
3.72k
                                            unsigned freeIntRegs) const {
3200
  // If this is a scalar LLVM value then assume LLVM will pass it in the right
3201
  // place naturally.
3202
  //
3203
  // This assumption is optimistic, as there could be free registers available
3204
  // when we need to pass this argument in memory, and LLVM could try to pass
3205
  // the argument in the free register. This does not seem to happen currently,
3206
  // but this code would be much safer if we could mark the argument with
3207
  // 'onstack'. See PR12193.
3208
3.72k
  if (!isAggregateTypeForABI(Ty) && 
!IsIllegalVectorType(Ty)2.94k
&&
3209
3.72k
      
!Ty->isExtIntType()2.81k
) {
3210
    // Treat an enum type as its underlying type.
3211
2.81k
    if (const EnumType *EnumTy = Ty->getAs<EnumType>())
3212
0
      Ty = EnumTy->getDecl()->getIntegerType();
3213
3214
2.81k
    return (isPromotableIntegerTypeForABI(Ty) ? 
ABIArgInfo::getExtend(Ty)286
3215
2.81k
                                              : 
ABIArgInfo::getDirect()2.52k
);
3216
2.81k
  }
3217
3218
916
  if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))
3219
447
    return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
3220
3221
  // Compute the byval alignment. We specify the alignment of the byval in all
3222
  // cases so that the mid-level optimizer knows the alignment of the byval.
3223
469
  unsigned Align = std::max(getContext().getTypeAlign(Ty) / 8, 8U);
3224
3225
  // Attempt to avoid passing indirect results using byval when possible. This
3226
  // is important for good codegen.
3227
  //
3228
  // We do this by coercing the value into a scalar type which the backend can
3229
  // handle naturally (i.e., without using byval).
3230
  //
3231
  // For simplicity, we currently only do this when we have exhausted all of the
3232
  // free integer registers. Doing this when there are free integer registers
3233
  // would require more care, as we would have to ensure that the coerced value
3234
  // did not claim the unused register. That would require either reording the
3235
  // arguments to the function (so that any subsequent inreg values came first),
3236
  // or only doing this optimization when there were no following arguments that
3237
  // might be inreg.
3238
  //
3239
  // We currently expect it to be rare (particularly in well written code) for
3240
  // arguments to be passed on the stack when there are still free integer
3241
  // registers available (this would typically imply large structs being passed
3242
  // by value), so this seems like a fair tradeoff for now.
3243
  //
3244
  // We can revisit this if the backend grows support for 'onstack' parameter
3245
  // attributes. See PR12193.
3246
469
  if (freeIntRegs == 0) {
3247
32
    uint64_t Size = getContext().getTypeSize(Ty);
3248
3249
    // If this type fits in an eightbyte, coerce it into the matching integral
3250
    // type, which will end up on the stack (with alignment 8).
3251
32
    if (Align == 8 && 
Size <= 6422
)
3252
4
      return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(),
3253
4
                                                          Size));
3254
32
  }
3255
3256
465
  return ABIArgInfo::getIndirect(CharUnits::fromQuantity(Align));
3257
469
}
3258
3259
/// The ABI specifies that a value should be passed in a full vector XMM/YMM
3260
/// register. Pick an LLVM IR type that will be passed as a vector register.
3261
6.10k
llvm::Type *X86_64ABIInfo::GetByteVectorType(QualType Ty) const {
3262
  // Wrapper structs/arrays that only contain vectors are passed just like
3263
  // vectors; strip them off if present.
3264
6.10k
  if (const Type *InnerTy = isSingleElementStruct(Ty, getContext()))
3265
25
    Ty = QualType(InnerTy, 0);
3266
3267
6.10k
  llvm::Type *IRType = CGT.ConvertType(Ty);
3268
6.10k
  if (isa<llvm::VectorType>(IRType)) {
3269
    // Don't pass vXi128 vectors in their native type, the backend can't
3270
    // legalize them.
3271
6.06k
    if (passInt128VectorsInMem() &&
3272
6.06k
        
cast<llvm::VectorType>(IRType)->getElementType()->isIntegerTy(128)505
) {
3273
      // Use a vXi64 vector.
3274
6
      uint64_t Size = getContext().getTypeSize(Ty);
3275
6
      return llvm::FixedVectorType::get(llvm::Type::getInt64Ty(getVMContext()),
3276
6
                                        Size / 64);
3277
6
    }
3278
3279
6.05k
    return IRType;
3280
6.06k
  }
3281
3282
43
  if (IRType->getTypeID() == llvm::Type::FP128TyID)
3283
40
    return IRType;
3284
3285
  // We couldn't find the preferred IR vector type for 'Ty'.
3286
3
  uint64_t Size = getContext().getTypeSize(Ty);
3287
3
  assert((Size == 128 || Size == 256 || Size == 512) && "Invalid type found!");
3288
3289
3290
  // Return a LLVM IR vector type based on the size of 'Ty'.
3291
0
  return llvm::FixedVectorType::get(llvm::Type::getDoubleTy(getVMContext()),
3292
3
                                    Size / 64);
3293
43
}
3294
3295
/// BitsContainNoUserData - Return true if the specified [start,end) bit range
3296
/// is known to either be off the end of the specified type or being in
3297
/// alignment padding.  The user type specified is known to be at most 128 bits
3298
/// in size, and have passed through X86_64ABIInfo::classify with a successful
3299
/// classification that put one of the two halves in the INTEGER class.
3300
///
3301
/// It is conservatively correct to return false.
3302
static bool BitsContainNoUserData(QualType Ty, unsigned StartBit,
3303
95.7k
                                  unsigned EndBit, ASTContext &Context) {
3304
  // If the bytes being queried are off the end of the type, there is no user
3305
  // data hiding here.  This handles analysis of builtins, vectors and other
3306
  // types that don't contain interesting padding.
3307
95.7k
  unsigned TySize = (unsigned)Context.getTypeSize(Ty);
3308
95.7k
  if (TySize <= StartBit)
3309
85.5k
    return true;
3310
3311
10.1k
  if (const ConstantArrayType *AT = Context.getAsConstantArrayType(Ty)) {
3312
32
    unsigned EltSize = (unsigned)Context.getTypeSize(AT->getElementType());
3313
32
    unsigned NumElts = (unsigned)AT->getSize().getZExtValue();
3314
3315
    // Check each element to see if the element overlaps with the queried range.
3316
89
    for (unsigned i = 0; i != NumElts; 
++i57
) {
3317
      // If the element is after the span we care about, then we're done..
3318
86
      unsigned EltOffset = i*EltSize;
3319
86
      if (EltOffset >= EndBit) 
break0
;
3320
3321
86
      unsigned EltStart = EltOffset < StartBit ? 
StartBit-EltOffset64
:
022
;
3322
86
      if (!BitsContainNoUserData(AT->getElementType(), EltStart,
3323
86
                                 EndBit-EltOffset, Context))
3324
29
        return false;
3325
86
    }
3326
    // If it overlaps no elements, then it is safe to process as padding.
3327
3
    return true;
3328
32
  }
3329
3330
10.1k
  if (const RecordType *RT = Ty->getAs<RecordType>()) {
3331
1.24k
    const RecordDecl *RD = RT->getDecl();
3332
1.24k
    const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
3333
3334
    // If this is a C++ record, check the bases first.
3335
1.24k
    if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
3336
955
      for (const auto &I : CXXRD->bases()) {
3337
29
        assert(!I.isVirtual() && !I.getType()->isDependentType() &&
3338
29
               "Unexpected base class!");
3339
0
        const auto *Base =
3340
29
            cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
3341
3342
        // If the base is after the span we care about, ignore it.
3343
29
        unsigned BaseOffset = Context.toBits(Layout.getBaseClassOffset(Base));
3344
29
        if (BaseOffset >= EndBit) 
continue0
;
3345
3346
29
        unsigned BaseStart = BaseOffset < StartBit ? 
StartBit-BaseOffset26
:
03
;
3347
29
        if (!BitsContainNoUserData(I.getType(), BaseStart,
3348
29
                                   EndBit-BaseOffset, Context))
3349
24
          return false;
3350
29
      }
3351
955
    }
3352
3353
    // Verify that no field has data that overlaps the region of interest.  Yes
3354
    // this could be sped up a lot by being smarter about queried fields,
3355
    // however we're only looking at structs up to 16 bytes, so we don't care
3356
    // much.
3357
1.22k
    unsigned idx = 0;
3358
1.22k
    for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
3359
2.67k
         i != e; 
++i, ++idx1.45k
) {
3360
2.47k
      unsigned FieldOffset = (unsigned)Layout.getFieldOffset(idx);
3361
3362
      // If we found a field after the region we care about, then we're done.
3363
2.47k
      if (FieldOffset >= EndBit) 
break140
;
3364
3365
2.33k
      unsigned FieldStart = FieldOffset < StartBit ? 
StartBit-FieldOffset1.70k
:
0628
;
3366
2.33k
      if (!BitsContainNoUserData(i->getType(), FieldStart, EndBit-FieldOffset,
3367
2.33k
                                 Context))
3368
878
        return false;
3369
2.33k
    }
3370
3371
    // If nothing in this record overlapped the area of interest, then we're
3372
    // clean.
3373
346
    return true;
3374
1.22k
  }
3375
3376
8.87k
  return false;
3377
10.1k
}
3378
3379
/// ContainsFloatAtOffset - Return true if the specified LLVM IR type has a
3380
/// float member at the specified offset.  For example, {int,{float}} has a
3381
/// float at offset 4.  It is conservatively correct for this routine to return
3382
/// false.
3383
static bool ContainsFloatAtOffset(llvm::Type *IRType, unsigned IROffset,
3384
28.9k
                                  const llvm::DataLayout &TD) {
3385
  // Base case if we find a float.
3386
28.9k
  if (IROffset == 0 && 
IRType->isFloatTy()19.5k
)
3387
2.37k
    return true;
3388
3389
  // If this is a struct, recurse into the field at the specified offset.
3390
26.5k
  if (llvm::StructType *STy = dyn_cast<llvm::StructType>(IRType)) {
3391
2.04k
    const llvm::StructLayout *SL = TD.getStructLayout(STy);
3392
2.04k
    unsigned Elt = SL->getElementContainingOffset(IROffset);
3393
2.04k
    IROffset -= SL->getElementOffset(Elt);
3394
2.04k
    return ContainsFloatAtOffset(STy->getElementType(Elt), IROffset, TD);
3395
2.04k
  }
3396
3397
  // If this is an array, recurse into the field at the specified offset.
3398
24.5k
  if (llvm::ArrayType *ATy = dyn_cast<llvm::ArrayType>(IRType)) {
3399
31
    llvm::Type *EltTy = ATy->getElementType();
3400
31
    unsigned EltSize = TD.getTypeAllocSize(EltTy);
3401
31
    IROffset -= IROffset/EltSize*EltSize;
3402
31
    return ContainsFloatAtOffset(EltTy, IROffset, TD);
3403
31
  }
3404
3405
24.4k
  return false;
3406
24.5k
}
3407
3408
/// ContainsHalfAtOffset - Return true if the specified LLVM IR type has a
3409
/// half member at the specified offset.  For example, {int,{half}} has a
3410
/// half at offset 4.  It is conservatively correct for this routine to return
3411
/// false.
3412
/// FIXME: Merge with ContainsFloatAtOffset
3413
static bool ContainsHalfAtOffset(llvm::Type *IRType, unsigned IROffset,
3414
8.70k
                                 const llvm::DataLayout &TD) {
3415
  // Base case if we find a float.
3416
8.70k
  if (IROffset == 0 && 
IRType->isHalfTy()8.38k
)
3417
22
    return true;
3418
3419
  // If this is a struct, recurse into the field at the specified offset.
3420
8.68k
  if (llvm::StructType *STy = dyn_cast<llvm::StructType>(IRType)) {
3421
456
    const llvm::StructLayout *SL = TD.getStructLayout(STy);
3422
456
    unsigned Elt = SL->getElementContainingOffset(IROffset);
3423
456
    IROffset -= SL->getElementOffset(Elt);
3424
456
    return ContainsHalfAtOffset(STy->getElementType(Elt), IROffset, TD);
3425
456
  }
3426
3427
  // If this is an array, recurse into the field at the specified offset.
3428
8.22k
  if (llvm::ArrayType *ATy = dyn_cast<llvm::ArrayType>(IRType)) {
3429
7
    llvm::Type *EltTy = ATy->getElementType();
3430
7
    unsigned EltSize = TD.getTypeAllocSize(EltTy);
3431
7
    IROffset -= IROffset / EltSize * EltSize;
3432
7
    return ContainsHalfAtOffset(EltTy, IROffset, TD);
3433
7
  }
3434
3435
8.22k
  return false;
3436
8.22k
}
3437
3438
/// GetSSETypeAtOffset - Return a type that will be passed by the backend in the
3439
/// low 8 bytes of an XMM register, corresponding to the SSE class.
3440
llvm::Type *X86_64ABIInfo::
3441
GetSSETypeAtOffset(llvm::Type *IRType, unsigned IROffset,
3442
10.3k
                   QualType SourceTy, unsigned SourceOffset) const {
3443
  // If the high 32 bits are not used, we have three choices. Single half,
3444
  // single float or two halfs.
3445
10.3k
  if (BitsContainNoUserData(SourceTy, SourceOffset * 8 + 32,
3446
10.3k
                            SourceOffset * 8 + 64, getContext())) {
3447
1.88k
    if (ContainsFloatAtOffset(IRType, IROffset, getDataLayout()))
3448
1.77k
      return llvm::Type::getFloatTy(getVMContext());
3449
107
    if (ContainsHalfAtOffset(IRType, IROffset + 2, getDataLayout()))
3450
10
      return llvm::FixedVectorType::get(llvm::Type::getHalfTy(getVMContext()),
3451
10
                                        2);
3452
3453
97
    return llvm::Type::getHalfTy(getVMContext());
3454
107
  }
3455
3456
  // We want to pass as <2 x float> if the LLVM IR type contains a float at
3457
  // offset+0 and offset+4. Walk the LLVM IR type to find out if this is the
3458
  // case.
3459
8.42k
  if (ContainsFloatAtOffset(IRType, IROffset, getDataLayout()) &&
3460
8.42k
      
ContainsFloatAtOffset(IRType, IROffset + 4, getDataLayout())300
)
3461
297
    return llvm::FixedVectorType::get(llvm::Type::getFloatTy(getVMContext()),
3462
297
                                      2);
3463
3464
  // We want to pass as <4 x half> if the LLVM IR type contains a half at
3465
  // offset+0, +2, +4. Walk the LLVM IR type to find out if this is the case.
3466
8.12k
  if (ContainsHalfAtOffset(IRType, IROffset, getDataLayout()) &&
3467
8.12k
      
ContainsHalfAtOffset(IRType, IROffset + 2, getDataLayout())5
&&
3468
8.12k
      
ContainsHalfAtOffset(IRType, IROffset + 4, getDataLayout())4
)
3469
3
    return llvm::FixedVectorType::get(llvm::Type::getHalfTy(getVMContext()), 4);
3470
3471
  // We want to pass as <4 x half> if the LLVM IR type contains a mix of float
3472
  // and half.
3473
  // FIXME: Do we have a better representation for the mixed type?
3474
8.12k
  if (ContainsFloatAtOffset(IRType, IROffset, getDataLayout()) ||
3475
8.12k
      
ContainsFloatAtOffset(IRType, IROffset + 4, getDataLayout())8.12k
)
3476
5
    return llvm::FixedVectorType::get(llvm::Type::getHalfTy(getVMContext()), 4);
3477
3478
8.12k
  return llvm::Type::getDoubleTy(getVMContext());
3479
8.12k
}
3480
3481
3482
/// GetINTEGERTypeAtOffset - The ABI specifies that a value should be passed in
3483
/// an 8-byte GPR.  This means that we either have a scalar or we are talking
3484
/// about the high or low part of an up-to-16-byte struct.  This routine picks
3485
/// the best LLVM IR type to represent this, which may be i64 or may be anything
3486
/// else that the backend will pass in a GPR that works better (e.g. i8, %foo*,
3487
/// etc).
3488
///
3489
/// PrefType is an LLVM IR type that corresponds to (part of) the IR type for
3490
/// the source type.  IROffset is an offset in bytes into the LLVM IR type that
3491
/// the 8-byte value references.  PrefType may be null.
3492
///
3493
/// SourceTy is the source-level type for the entire argument.  SourceOffset is
3494
/// an offset into this that we're processing (which is always either 0 or 8).
3495
///
3496
llvm::Type *X86_64ABIInfo::
3497
GetINTEGERTypeAtOffset(llvm::Type *IRType, unsigned IROffset,
3498
384k
                       QualType SourceTy, unsigned SourceOffset) const {
3499
  // If we're dealing with an un-offset LLVM IR type, then it means that we're
3500
  // returning an 8-byte unit starting with it.  See if we can safely use it.
3501
384k
  if (IROffset == 0) {
3502
    // Pointers and int64's always fill the 8-byte unit.
3503
383k
    if ((isa<llvm::PointerType>(IRType) && 
Has64BitPointers266k
) ||
3504
383k
        
IRType->isIntegerTy(64)116k
)
3505
290k
      return IRType;
3506
3507
    // If we have a 1/2/4-byte integer, we can use it only if the rest of the
3508
    // goodness in the source type is just tail padding.  This is allowed to
3509
    // kick in for struct {double,int} on the int, but not on
3510
    // struct{double,int,int} because we wouldn't return the second int.  We
3511
    // have to do this analysis on the source type because we can't depend on
3512
    // unions being lowered a specific way etc.
3513
92.8k
    if (IRType->isIntegerTy(8) || 
IRType->isIntegerTy(16)89.1k
||
3514
92.8k
        
IRType->isIntegerTy(32)88.3k
||
3515
92.8k
        
(9.92k
isa<llvm::PointerType>(IRType)9.92k
&&
!Has64BitPointers92
)) {
3516
82.9k
      unsigned BitWidth = isa<llvm::PointerType>(IRType) ? 
3292
:
3517
82.9k
          
cast<llvm::IntegerType>(IRType)->getBitWidth()82.8k
;
3518
3519
82.9k
      if (BitsContainNoUserData(SourceTy, SourceOffset*8+BitWidth,
3520
82.9k
                                SourceOffset*8+64, getContext()))
3521
82.5k
        return IRType;
3522
82.9k
    }
3523
92.8k
  }
3524
3525
11.7k
  if (llvm::StructType *STy = dyn_cast<llvm::StructType>(IRType)) {
3526
    // If this is a struct, recurse into the field at the specified offset.
3527
7.43k
    const llvm::StructLayout *SL = getDataLayout().getStructLayout(STy);
3528
7.43k
    if (IROffset < SL->getSizeInBytes()) {
3529
7.43k
      unsigned FieldIdx = SL->getElementContainingOffset(IROffset);
3530
7.43k
      IROffset -= SL->getElementOffset(FieldIdx);
3531
3532
7.43k
      return GetINTEGERTypeAtOffset(STy->getElementType(FieldIdx), IROffset,
3533
7.43k
                                    SourceTy, SourceOffset);
3534
7.43k
    }
3535
7.43k
  }
3536
3537
4.27k
  if (llvm::ArrayType *ATy = dyn_cast<llvm::ArrayType>(IRType)) {
3538
52
    llvm::Type *EltTy = ATy->getElementType();
3539
52
    unsigned EltSize = getDataLayout().getTypeAllocSize(EltTy);
3540
52
    unsigned EltOffset = IROffset/EltSize*EltSize;
3541
52
    return GetINTEGERTypeAtOffset(EltTy, IROffset-EltOffset, SourceTy,
3542
52
                                  SourceOffset);
3543
52
  }
3544
3545
  // Okay, we don't have any better idea of what to pass, so we pass this in an
3546
  // integer register that isn't too big to fit the rest of the struct.
3547
4.21k
  unsigned TySizeInBytes =
3548
4.21k
    (unsigned)getContext().getTypeSizeInChars(SourceTy).getQuantity();
3549
3550
4.21k
  assert(TySizeInBytes != SourceOffset && "Empty field?");
3551
3552
  // It is always safe to classify this as an integer type up to i64 that
3553
  // isn't larger than the structure.
3554
0
  return llvm::IntegerType::get(getVMContext(),
3555
4.21k
                                std::min(TySizeInBytes-SourceOffset, 8U)*8);
3556
4.27k
}
3557
3558
3559
/// GetX86_64ByValArgumentPair - Given a high and low type that can ideally
3560
/// be used as elements of a two register pair to pass or return, return a
3561
/// first class aggregate to represent them.  For example, if the low part of
3562
/// a by-value argument should be passed as i32* and the high part as float,
3563
/// return {i32*, float}.
3564
static llvm::Type *
3565
GetX86_64ByValArgumentPair(llvm::Type *Lo, llvm::Type *Hi,
3566
1.62k
                           const llvm::DataLayout &TD) {
3567
  // In order to correctly satisfy the ABI, we need to the high part to start
3568
  // at offset 8.  If the high and low parts we inferred are both 4-byte types
3569
  // (e.g. i32 and i32) then the resultant struct type ({i32,i32}) won't have
3570
  // the second element at offset 8.  Check for this:
3571
1.62k
  unsigned LoSize = (unsigned)TD.getTypeAllocSize(Lo);
3572
1.62k
  unsigned HiAlign = TD.getABITypeAlignment(Hi);
3573
1.62k
  unsigned HiStart = llvm::alignTo(LoSize, HiAlign);
3574
1.62k
  assert(HiStart != 0 && HiStart <= 8 && "Invalid x86-64 argument pair!");
3575
3576
  // To handle this, we have to increase the size of the low part so that the
3577
  // second element will start at an 8 byte offset.  We can't increase the size
3578
  // of the second element because it might make us access off the end of the
3579
  // struct.
3580
1.62k
  if (HiStart != 8) {
3581
    // There are usually two sorts of types the ABI generation code can produce
3582
    // for the low part of a pair that aren't 8 bytes in size: half, float or
3583
    // i8/i16/i32.  This can also include pointers when they are 32-bit (X32 and
3584
    // NaCl).
3585
    // Promote these to a larger type.
3586
5
    if (Lo->isHalfTy() || Lo->isFloatTy())
3587
0
      Lo = llvm::Type::getDoubleTy(Lo->getContext());
3588
5
    else {
3589
5
      assert((Lo->isIntegerTy() || Lo->isPointerTy())
3590
5
             && "Invalid/unknown lo type");
3591
0
      Lo = llvm::Type::getInt64Ty(Lo->getContext());
3592
5
    }
3593
5
  }
3594
3595
0
  llvm::StructType *Result = llvm::StructType::get(Lo, Hi);
3596
3597
  // Verify that the second element is at an 8-byte offset.
3598
1.62k
  assert(TD.getStructLayout(Result)->getElementOffset(1) == 8 &&
3599
1.62k
         "Invalid x86-64 argument pair!");
3600
0
  return Result;
3601
1.62k
}
3602
3603
ABIArgInfo X86_64ABIInfo::
3604
164k
classifyReturnType(QualType RetTy) const {
3605
  // AMD64-ABI 3.2.3p4: Rule 1. Classify the return type with the
3606
  // classification algorithm.
3607
164k
  X86_64ABIInfo::Class Lo, Hi;
3608
164k
  classify(RetTy, 0, Lo, Hi, /*isNamedArg*/ true);
3609
3610
  // Check some invariants.
3611
164k
  assert((Hi != Memory || Lo == Memory) && "Invalid memory classification.");
3612
0
  assert((Hi != SSEUp || Lo == SSE) && "Invalid SSEUp classification.");
3613
3614
0
  llvm::Type *ResType = nullptr;
3615
164k
  switch (Lo) {
3616
65.8k
  case NoClass:
3617
65.8k
    if (Hi == NoClass)
3618
65.8k
      return ABIArgInfo::getIgnore();
3619
    // If the low part is just padding, it takes no register, leave ResType
3620
    // null.
3621
1
    assert((Hi == SSE || Hi == Integer || Hi == X87Up) &&
3622
1
           "Unknown missing lo part");
3623
0
    break;
3624
3625
0
  case SSEUp:
3626
0
  case X87Up:
3627
0
    llvm_unreachable("Invalid classification for lo word.");
3628
3629
    // AMD64-ABI 3.2.3p4: Rule 2. Types of class memory are returned via
3630
    // hidden argument.
3631
455
  case Memory:
3632
455
    return getIndirectReturnResult(RetTy);
3633
3634
    // AMD64-ABI 3.2.3p4: Rule 3. If the class is INTEGER, the next
3635
    // available register of the sequence %rax, %rdx is used.
3636
93.7k
  case Integer:
3637
93.7k
    ResType = GetINTEGERTypeAtOffset(CGT.ConvertType(RetTy), 0, RetTy, 0);
3638
3639
    // If we have a sign or zero extended integer, make sure to return Extend
3640
    // so that the parameter gets the right LLVM IR attributes.
3641
93.7k
    if (Hi == NoClass && 
isa<llvm::IntegerType>(ResType)93.0k
) {
3642
      // Treat an enum type as its underlying type.
3643
52.5k
      if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
3644
101
        RetTy = EnumTy->getDecl()->getIntegerType();
3645
3646
52.5k
      if (RetTy->isIntegralOrEnumerationType() &&
3647
52.5k
          
isPromotableIntegerTypeForABI(RetTy)51.6k
)
3648
4.40k
        return ABIArgInfo::getExtend(RetTy);
3649
52.5k
    }
3650
89.3k
    break;
3651
3652
    // AMD64-ABI 3.2.3p4: Rule 4. If the class is SSE, the next
3653
    // available SSE register of the sequence %xmm0, %xmm1 is used.
3654
89.3k
  case SSE:
3655
3.38k
    ResType = GetSSETypeAtOffset(CGT.ConvertType(RetTy), 0, RetTy, 0);
3656
3.38k
    break;
3657
3658
    // AMD64-ABI 3.2.3p4: Rule 6. If the class is X87, the value is
3659
    // returned on the X87 stack in %st0 as 80-bit x87 number.
3660
675
  case X87:
3661
675
    ResType = llvm::Type::getX86_FP80Ty(getVMContext());
3662
675
    break;
3663
3664
    // AMD64-ABI 3.2.3p4: Rule 8. If the class is COMPLEX_X87, the real
3665
    // part of the value is returned in %st0 and the imaginary part in
3666
    // %st1.
3667
38
  case ComplexX87:
3668
38
    assert(Hi == ComplexX87 && "Unexpected ComplexX87 classification.");
3669
0
    ResType = llvm::StructType::get(llvm::Type::getX86_FP80Ty(getVMContext()),
3670
38
                                    llvm::Type::getX86_FP80Ty(getVMContext()));
3671
38
    break;
3672
164k
  }
3673
3674
93.4k
  llvm::Type *HighPart = nullptr;
3675
93.4k
  switch (Hi) {
3676
    // Memory was handled previously and X87 should
3677
    // never occur as a hi class.
3678
0
  case Memory:
3679
0
  case X87:
3680
0
    llvm_unreachable("Invalid classification for hi word.");
3681
3682
38
  case ComplexX87: // Previously handled.
3683
89.6k
  case NoClass:
3684
89.6k
    break;
3685
3686
664
  case Integer:
3687
664
    HighPart = GetINTEGERTypeAtOffset(CGT.ConvertType(RetTy), 8, RetTy, 8);
3688
664
    if (Lo == NoClass)  // Return HighPart at offset 8 in memory.
3689
0
      return ABIArgInfo::getDirect(HighPart, 8);
3690
664
    break;
3691
664
  case SSE:
3692
137
    HighPart = GetSSETypeAtOffset(CGT.ConvertType(RetTy), 8, RetTy, 8);
3693
137
    if (Lo == NoClass)  // Return HighPart at offset 8 in memory.
3694
1
      return ABIArgInfo::getDirect(HighPart, 8);
3695
136
    break;
3696
3697
    // AMD64-ABI 3.2.3p4: Rule 5. If the class is SSEUP, the eightbyte
3698
    // is passed in the next available eightbyte chunk if the last used
3699
    // vector register.
3700
    //
3701
    // SSEUP should always be preceded by SSE, just widen.
3702
2.27k
  case SSEUp:
3703
2.27k
    assert(Lo == SSE && "Unexpected SSEUp classification.");
3704
0
    ResType = GetByteVectorType(RetTy);
3705
2.27k
    break;
3706
3707
    // AMD64-ABI 3.2.3p4: Rule 7. If the class is X87UP, the value is
3708
    // returned together with the previous X87 value in %st0.
3709
675
  case X87Up:
3710
    // If X87Up is preceded by X87, we don't need to do
3711
    // anything. However, in some cases with unions it may not be
3712
    // preceded by X87. In such situations we follow gcc and pass the
3713
    // extra bits in an SSE reg.
3714
675
    if (Lo != X87) {
3715
0
      HighPart = GetSSETypeAtOffset(CGT.ConvertType(RetTy), 8, RetTy, 8);
3716
0
      if (Lo == NoClass)  // Return HighPart at offset 8 in memory.
3717
0
        return ABIArgInfo::getDirect(HighPart, 8);
3718
0
    }
3719
675
    break;
3720
93.4k
  }
3721
3722
  // If a high part was specified, merge it together with the low part.  It is
3723
  // known to pass in the high eightbyte of the result.  We do this by forming a
3724
  // first class struct aggregate with the high and low part: {low, high}
3725
93.4k
  if (HighPart)
3726
800
    ResType = GetX86_64ByValArgumentPair(ResType, HighPart, getDataLayout());
3727
3728
93.4k
  return ABIArgInfo::getDirect(ResType);
3729
93.4k
}
3730
3731
ABIArgInfo X86_64ABIInfo::classifyArgumentType(
3732
  QualType Ty, unsigned freeIntRegs, unsigned &neededInt, unsigned &neededSSE,
3733
  bool isNamedArg)
3734
  const
3735
295k
{
3736
295k
  Ty = useFirstFieldIfTransparentUnion(Ty);
3737
3738
295k
  X86_64ABIInfo::Class Lo, Hi;
3739
295k
  classify(Ty, 0, Lo, Hi, isNamedArg);
3740
3741
  // Check some invariants.
3742
  // FIXME: Enforce these by construction.
3743
295k
  assert((Hi != Memory || Lo == Memory) && "Invalid memory classification.");
3744
0
  assert((Hi != SSEUp || Lo == SSE) && "Invalid SSEUp classification.");
3745
3746
0
  neededInt = 0;
3747
295k
  neededSSE = 0;
3748
295k
  llvm::Type *ResType = nullptr;
3749
295k
  switch (Lo) {
3750
5.10k
  case NoClass:
3751
5.10k
    if (Hi == NoClass)
3752
5.10k
      return ABIArgInfo::getIgnore();
3753
    // If the low part is just padding, it takes no register, leave ResType
3754
    // null.
3755
2
    assert((Hi == SSE || Hi == Integer || Hi == X87Up) &&
3756
2
           "Unknown missing lo part");
3757
0
    break;
3758
3759
    // AMD64-ABI 3.2.3p3: Rule 1. If the class is MEMORY, pass the argument
3760
    // on the stack.
3761
1.23k
  case Memory:
3762
3763
    // AMD64-ABI 3.2.3p3: Rule 5. If the class is X87, X87UP or
3764
    // COMPLEX_X87, it is passed in memory.
3765
1.49k
  case X87:
3766
1.54k
  case ComplexX87:
3767
1.54k
    if (getRecordArgABI(Ty, getCXXABI()) == CGCXXABI::RAA_Indirect)
3768
447
      ++neededInt;
3769
1.54k
    return getIndirectResult(Ty, freeIntRegs);
3770
3771
0
  case SSEUp:
3772
0
  case X87Up:
3773
0
    llvm_unreachable("Invalid classification for lo word.");
3774
3775
    // AMD64-ABI 3.2.3p3: Rule 2. If the class is INTEGER, the next
3776
    // available register of the sequence %rdi, %rsi, %rdx, %rcx, %r8
3777
    // and %r9 is used.
3778
282k
  case Integer:
3779
282k
    ++neededInt;
3780
3781
    // Pick an 8-byte type based on the preferred type.
3782
282k
    ResType = GetINTEGERTypeAtOffset(CGT.ConvertType(Ty), 0, Ty, 0);
3783
3784
    // If we have a sign or zero extended integer, make sure to return Extend
3785
    // so that the parameter gets the right LLVM IR attributes.
3786
282k
    if (Hi == NoClass && 
isa<llvm::IntegerType>(ResType)281k
) {
3787
      // Treat an enum type as its underlying type.
3788
56.3k
      if (const EnumType *EnumTy = Ty->getAs<EnumType>())
3789
849
        Ty = EnumTy->getDecl()->getIntegerType();
3790
3791
56.3k
      if (Ty->isIntegralOrEnumerationType() &&
3792
56.3k
          
isPromotableIntegerTypeForABI(Ty)55.9k
)
3793
3.15k
        return ABIArgInfo::getExtend(Ty);
3794
56.3k
    }
3795
3796
278k
    break;
3797
3798
    // AMD64-ABI 3.2.3p3: Rule 3. If the class is SSE, the next
3799
    // available SSE register is used, the registers are taken in the
3800
    // order from %xmm0 to %xmm7.
3801
278k
  case SSE: {
3802
6.63k
    llvm::Type *IRType = CGT.ConvertType(Ty);
3803
6.63k
    ResType = GetSSETypeAtOffset(IRType, 0, Ty, 0);
3804
6.63k
    ++neededSSE;
3805
6.63k
    break;
3806
282k
  }
3807
295k
  }
3808
3809
285k
  llvm::Type *HighPart = nullptr;
3810
285k
  switch (Hi) {
3811
    // Memory was handled previously, ComplexX87 and X87 should
3812
    // never occur as hi classes, and X87Up must be preceded by X87,
3813
    // which is passed in memory.
3814
0
  case Memory:
3815
0
  case X87:
3816
0
  case ComplexX87:
3817
0
    llvm_unreachable("Invalid classification for hi word.");
3818
3819
280k
  case NoClass: break;
3820
3821
670
  case Integer:
3822
670
    ++neededInt;
3823
    // Pick an 8-byte type based on the preferred type.
3824
670
    HighPart = GetINTEGERTypeAtOffset(CGT.ConvertType(Ty), 8, Ty, 8);
3825
3826
670
    if (Lo == NoClass)  // Pass HighPart at offset 8 in memory.
3827
2
      return ABIArgInfo::getDirect(HighPart, 8);
3828
668
    break;
3829
3830
    // X87Up generally doesn't occur here (long double is passed in
3831
    // memory), except in situations involving unions.
3832
668
  case X87Up:
3833
153
  case SSE:
3834
153
    HighPart = GetSSETypeAtOffset(CGT.ConvertType(Ty), 8, Ty, 8);
3835
3836
153
    if (Lo == NoClass)  // Pass HighPart at offset 8 in memory.
3837
0
      return ABIArgInfo::getDirect(HighPart, 8);
3838
3839
153
    ++neededSSE;
3840
153
    break;
3841
3842
    // AMD64-ABI 3.2.3p3: Rule 4. If the class is SSEUP, the
3843
    // eightbyte is passed in the upper half of the last used SSE
3844
    // register.  This only happens when 128-bit vectors are passed.
3845
3.82k
  case SSEUp:
3846
3.82k
    assert(Lo == SSE && "Unexpected SSEUp classification");
3847
0
    ResType = GetByteVectorType(Ty);
3848
3.82k
    break;
3849
285k
  }
3850
3851
  // If a high part was specified, merge it together with the low part.  It is
3852
  // known to pass in the high eightbyte of the result.  We do this by forming a
3853
  // first class struct aggregate with the high and low part: {low, high}
3854
285k
  if (HighPart)
3855
821
    ResType = GetX86_64ByValArgumentPair(ResType, HighPart, getDataLayout());
3856
3857
285k
  return ABIArgInfo::getDirect(ResType);
3858
285k
}
3859
3860
ABIArgInfo
3861
X86_64ABIInfo::classifyRegCallStructTypeImpl(QualType Ty, unsigned &NeededInt,
3862
18
                                             unsigned &NeededSSE) const {
3863
18
  auto RT = Ty->getAs<RecordType>();
3864
18
  assert(RT && "classifyRegCallStructType only valid with struct types");
3865
3866
18
  if (RT->getDecl()->hasFlexibleArrayMember())
3867
0
    return getIndirectReturnResult(Ty);
3868
3869
  // Sum up bases
3870
18
  if (auto CXXRD = dyn_cast<CXXRecordDecl>(RT->getDecl())) {
3871
0
    if (CXXRD->isDynamicClass()) {
3872
0
      NeededInt = NeededSSE = 0;
3873
0
      return getIndirectReturnResult(Ty);
3874
0
    }
3875
3876
0
    for (const auto &I : CXXRD->bases())
3877
0
      if (classifyRegCallStructTypeImpl(I.getType(), NeededInt, NeededSSE)
3878
0
              .isIndirect()) {
3879
0
        NeededInt = NeededSSE = 0;
3880
0
        return getIndirectReturnResult(Ty);
3881
0
      }
3882
0
  }
3883
3884
  // Sum up members
3885
44
  
for (const auto *FD : RT->getDecl()->fields())18
{
3886
44
    if (FD->getType()->isRecordType() && 
!FD->getType()->isUnionType()0
) {
3887
0
      if (classifyRegCallStructTypeImpl(FD->getType(), NeededInt, NeededSSE)
3888
0
              .isIndirect()) {
3889
0
        NeededInt = NeededSSE = 0;
3890
0
        return getIndirectReturnResult(Ty);
3891
0
      }
3892
44
    } else {
3893
44
      unsigned LocalNeededInt, LocalNeededSSE;
3894
44
      if (classifyArgumentType(FD->getType(), UINT_MAX, LocalNeededInt,
3895
44
                               LocalNeededSSE, true)
3896
44
              .isIndirect()) {
3897
0
        NeededInt = NeededSSE = 0;
3898
0
        return getIndirectReturnResult(Ty);
3899
0
      }
3900
44
      NeededInt += LocalNeededInt;
3901
44
      NeededSSE += LocalNeededSSE;
3902
44
    }
3903
44
  }
3904
3905
18
  return ABIArgInfo::getDirect();
3906
18
}
3907
3908
ABIArgInfo X86_64ABIInfo::classifyRegCallStructType(QualType Ty,
3909
                                                    unsigned &NeededInt,
3910
18
                                                    unsigned &NeededSSE) const {
3911
3912
18
  NeededInt = 0;
3913
18
  NeededSSE = 0;
3914
3915
18
  return classifyRegCallStructTypeImpl(Ty, NeededInt, NeededSSE);
3916
18
}
3917
3918
164k
void X86_64ABIInfo::computeInfo(CGFunctionInfo &FI) const {
3919
3920
164k
  const unsigned CallingConv = FI.getCallingConvention();
3921
  // It is possible to force Win64 calling convention on any x86_64 target by
3922
  // using __attribute__((ms_abi)). In such case to correctly emit Win64
3923
  // compatible code delegate this call to WinX86_64ABIInfo::computeInfo.
3924
164k
  if (CallingConv == llvm::CallingConv::Win64) {
3925
11
    WinX86_64ABIInfo Win64ABIInfo(CGT, AVXLevel);
3926
11
    Win64ABIInfo.computeInfo(FI);
3927
11
    return;
3928
11
  }
3929
3930
164k
  bool IsRegCall = CallingConv == llvm::CallingConv::X86_RegCall;
3931
3932
  // Keep track of the number of assigned registers.
3933
164k
  unsigned FreeIntRegs = IsRegCall ? 
1124
:
6164k
;
3934
164k
  unsigned FreeSSERegs = IsRegCall ? 
1624
:
8164k
;
3935
164k
  unsigned NeededInt, NeededSSE;
3936
3937
164k
  if (!::classifyReturnType(getCXXABI(), FI, *this)) {
3938
164k
    if (IsRegCall && 
FI.getReturnType()->getTypePtr()->isRecordType()23
&&
3939
164k
        
!FI.getReturnType()->getTypePtr()->isUnionType()2
) {
3940
2
      FI.getReturnInfo() =
3941
2
          classifyRegCallStructType(FI.getReturnType(), NeededInt, NeededSSE);
3942
2
      if (FreeIntRegs >= NeededInt && FreeSSERegs >= NeededSSE) {
3943
2
        FreeIntRegs -= NeededInt;
3944
2
        FreeSSERegs -= NeededSSE;
3945
2
      } else {
3946
0
        FI.getReturnInfo() = getIndirectReturnResult(FI.getReturnType());
3947
0
      }
3948
164k
    } else if (IsRegCall && 
FI.getReturnType()->getAs<ComplexType>()21
&&
3949
164k
               getContext().getCanonicalType(FI.getReturnType()
3950
2
                                                 ->getAs<ComplexType>()
3951
2
                                                 ->getElementType()) ==
3952
2
                   getContext().LongDoubleTy)
3953
      // Complex Long Double Type is passed in Memory when Regcall
3954
      // calling convention is used.
3955
1
      FI.getReturnInfo() = getIndirectReturnResult(FI.getReturnType());
3956
164k
    else
3957
164k
      FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
3958
164k
  }
3959
3960
  // If the return value is indirect, then the hidden argument is consuming one
3961
  // integer register.
3962
164k
  if (FI.getReturnInfo().isIndirect())
3963
1.03k
    --FreeIntRegs;
3964
3965
  // The chain argument effectively gives us another free register.
3966
164k
  if (FI.isChainCall())
3967
4
    ++FreeIntRegs;
3968
3969
164k
  unsigned NumRequiredArgs = FI.getNumRequiredArgs();
3970
  // AMD64-ABI 3.2.3p3: Once arguments are classified, the registers
3971
  // get assigned (in left-to-right order) for passing as follows...
3972
164k
  unsigned ArgNo = 0;
3973
164k
  for (CGFunctionInfo::arg_iterator it = FI.arg_begin(), ie = FI.arg_end();
3974
460k
       it != ie; 
++it, ++ArgNo295k
) {
3975
295k
    bool IsNamedArg = ArgNo < NumRequiredArgs;
3976
3977
295k
    if (IsRegCall && 
it->type->isStructureOrClassType()54
)
3978
16
      it->info = classifyRegCallStructType(it->type, NeededInt, NeededSSE);
3979
295k
    else
3980
295k
      it->info = classifyArgumentType(it->type, FreeIntRegs, NeededInt,
3981
295k
                                      NeededSSE, IsNamedArg);
3982
3983
    // AMD64-ABI 3.2.3p3: If there are no registers available for any
3984
    // eightbyte of an argument, the whole argument is passed on the
3985
    // stack. If registers have already been assigned for some
3986
    // eightbytes of such an argument, the assignments get reverted.
3987
295k
    if (FreeIntRegs >= NeededInt && 
FreeSSERegs >= NeededSSE292k
) {
3988
292k
      FreeIntRegs -= NeededInt;
3989
292k
      FreeSSERegs -= NeededSSE;
3990
292k
    } else {
3991
2.17k
      it->info = getIndirectResult(it->type, FreeIntRegs);
3992
2.17k
    }
3993
295k
  }
3994
164k
}
3995
3996
static Address EmitX86_64VAArgFromMemory(CodeGenFunction &CGF,
3997
97
                                         Address VAListAddr, QualType Ty) {
3998
97
  Address overflow_arg_area_p =
3999
97
      CGF.Builder.CreateStructGEP(VAListAddr, 2, "overflow_arg_area_p");
4000
97
  llvm::Value *overflow_arg_area =
4001
97
    CGF.Builder.CreateLoad(overflow_arg_area_p, "overflow_arg_area");
4002
4003
  // AMD64-ABI 3.5.7p5: Step 7. Align l->overflow_arg_area upwards to a 16
4004
  // byte boundary if alignment needed by type exceeds 8 byte boundary.
4005
  // It isn't stated explicitly in the standard, but in practice we use
4006
  // alignment greater than 16 where necessary.
4007
97
  CharUnits Align = CGF.getContext().getTypeAlignInChars(Ty);
4008
97
  if (Align > CharUnits::fromQuantity(8)) {
4009
18
    overflow_arg_area = emitRoundPointerUpToAlignment(CGF, overflow_arg_area,
4010
18
                                                      Align);
4011
18
  }
4012
4013
  // AMD64-ABI 3.5.7p5: Step 8. Fetch type from l->overflow_arg_area.
4014
97
  llvm::Type *LTy = CGF.ConvertTypeForMem(Ty);
4015
97
  llvm::Value *Res =
4016
97
    CGF.Builder.CreateBitCast(overflow_arg_area,
4017
97
                              llvm::PointerType::getUnqual(LTy));
4018
4019
  // AMD64-ABI 3.5.7p5: Step 9. Set l->overflow_arg_area to:
4020
  // l->overflow_arg_area + sizeof(type).
4021
  // AMD64-ABI 3.5.7p5: Step 10. Align l->overflow_arg_area upwards to
4022
  // an 8 byte boundary.
4023
4024
97
  uint64_t SizeInBytes = (CGF.getContext().getTypeSize(Ty) + 7) / 8;
4025
97
  llvm::Value *Offset =
4026
97
      llvm::ConstantInt::get(CGF.Int32Ty, (SizeInBytes + 7)  & ~7);
4027
97
  overflow_arg_area = CGF.Builder.CreateGEP(CGF.Int8Ty, overflow_arg_area,
4028
97
                                            Offset, "overflow_arg_area.next");
4029
97
  CGF.Builder.CreateStore(overflow_arg_area, overflow_arg_area_p);
4030
4031
  // AMD64-ABI 3.5.7p5: Step 11. Return the fetched type.
4032
97
  return Address(Res, Align);
4033
97
}
4034
4035
Address X86_64ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
4036
97
                                 QualType Ty) const {
4037
  // Assume that va_list type is correct; should be pointer to LLVM type:
4038
  // struct {
4039
  //   i32 gp_offset;
4040
  //   i32 fp_offset;
4041
  //   i8* overflow_arg_area;
4042
  //   i8* reg_save_area;
4043
  // };
4044
97
  unsigned neededInt, neededSSE;
4045
4046
97
  Ty = getContext().getCanonicalType(Ty);
4047
97
  ABIArgInfo AI = classifyArgumentType(Ty, 0, neededInt, neededSSE,
4048
97
                                       /*isNamedArg*/false);
4049
4050
  // AMD64-ABI 3.5.7p5: Step 1. Determine whether type may be passed
4051
  // in the registers. If not go to step 7.
4052
97
  if (!neededInt && 
!neededSSE34
)
4053
18
    return EmitX86_64VAArgFromMemory(CGF, VAListAddr, Ty);
4054
4055
  // AMD64-ABI 3.5.7p5: Step 2. Compute num_gp to hold the number of
4056
  // general purpose registers needed to pass type and num_fp to hold
4057
  // the number of floating point registers needed.
4058
4059
  // AMD64-ABI 3.5.7p5: Step 3. Verify whether arguments fit into
4060
  // registers. In the case: l->gp_offset > 48 - num_gp * 8 or
4061
  // l->fp_offset > 304 - num_fp * 16 go to step 7.
4062
  //
4063
  // NOTE: 304 is a typo, there are (6 * 8 + 8 * 16) = 176 bytes of
4064
  // register save space).
4065
4066
79
  llvm::Value *InRegs = nullptr;
4067
79
  Address gp_offset_p = Address::invalid(), fp_offset_p = Address::invalid();
4068
79
  llvm::Value *gp_offset = nullptr, *fp_offset = nullptr;
4069
79
  if (neededInt) {
4070
63
    gp_offset_p = CGF.Builder.CreateStructGEP(VAListAddr, 0, "gp_offset_p");
4071
63
    gp_offset = CGF.Builder.CreateLoad(gp_offset_p, "gp_offset");
4072
63
    InRegs = llvm::ConstantInt::get(CGF.Int32Ty, 48 - neededInt * 8);
4073
63
    InRegs = CGF.Builder.CreateICmpULE(gp_offset, InRegs, "fits_in_gp");
4074
63
  }
4075
4076
79
  if (neededSSE) {
4077
20
    fp_offset_p = CGF.Builder.CreateStructGEP(VAListAddr, 1, "fp_offset_p");
4078
20
    fp_offset = CGF.Builder.CreateLoad(fp_offset_p, "fp_offset");
4079
20
    llvm::Value *FitsInFP =
4080
20
      llvm::ConstantInt::get(CGF.Int32Ty, 176 - neededSSE * 16);
4081
20
    FitsInFP = CGF.Builder.CreateICmpULE(fp_offset, FitsInFP, "fits_in_fp");
4082
20
    InRegs = InRegs ? 
CGF.Builder.CreateAnd(InRegs, FitsInFP)4
:
FitsInFP16
;
4083
20
  }
4084
4085
79
  llvm::BasicBlock *InRegBlock = CGF.createBasicBlock("vaarg.in_reg");
4086
79
  llvm::BasicBlock *InMemBlock = CGF.createBasicBlock("vaarg.in_mem");
4087
79
  llvm::BasicBlock *ContBlock = CGF.createBasicBlock("vaarg.end");
4088
79
  CGF.Builder.CreateCondBr(InRegs, InRegBlock, InMemBlock);
4089
4090
  // Emit code to load the value if it was passed in registers.
4091
4092
79
  CGF.EmitBlock(InRegBlock);
4093
4094
  // AMD64-ABI 3.5.7p5: Step 4. Fetch type from l->reg_save_area with
4095
  // an offset of l->gp_offset and/or l->fp_offset. This may require
4096
  // copying to a temporary location in case the parameter is passed
4097
  // in different register classes or requires an alignment greater
4098
  // than 8 for general purpose registers and 16 for XMM registers.
4099
  //
4100
  // FIXME: This really results in shameful code when we end up needing to
4101
  // collect arguments from different places; often what should result in a
4102
  // simple assembling of a structure from scattered addresses has many more
4103
  // loads than necessary. Can we clean this up?
4104
79
  llvm::Type *LTy = CGF.ConvertTypeForMem(Ty);
4105
79
  llvm::Value *RegSaveArea = CGF.Builder.CreateLoad(
4106
79
      CGF.Builder.CreateStructGEP(VAListAddr, 3), "reg_save_area");
4107
4108
79
  Address RegAddr = Address::invalid();
4109
79
  if (neededInt && 
neededSSE63
) {
4110
    // FIXME: Cleanup.
4111
4
    assert(AI.isDirect() && "Unexpected ABI info for mixed regs");
4112
0
    llvm::StructType *ST = cast<llvm::StructType>(AI.getCoerceToType());
4113
4
    Address Tmp = CGF.CreateMemTemp(Ty);
4114
4
    Tmp = CGF.Builder.CreateElementBitCast(Tmp, ST);
4115
4
    assert(ST->getNumElements() == 2 && "Unexpected ABI info for mixed regs");
4116
0
    llvm::Type *TyLo = ST->getElementType(0);
4117
4
    llvm::Type *TyHi = ST->getElementType(1);
4118
4
    assert((TyLo->isFPOrFPVectorTy() ^ TyHi->isFPOrFPVectorTy()) &&
4119
4
           "Unexpected ABI info for mixed regs");
4120
0
    llvm::Type *PTyLo = llvm::PointerType::getUnqual(TyLo);
4121
4
    llvm::Type *PTyHi = llvm::PointerType::getUnqual(TyHi);
4122
4
    llvm::Value *GPAddr =
4123
4
        CGF.Builder.CreateGEP(CGF.Int8Ty, RegSaveArea, gp_offset);
4124
4
    llvm::Value *FPAddr =
4125
4
        CGF.Builder.CreateGEP(CGF.Int8Ty, RegSaveArea, fp_offset);
4126
4
    llvm::Value *RegLoAddr = TyLo->isFPOrFPVectorTy() ? 
FPAddr1
:
GPAddr3
;
4127
4
    llvm::Value *RegHiAddr = TyLo->isFPOrFPVectorTy() ? 
GPAddr1
:
FPAddr3
;
4128
4129
    // Copy the first element.
4130
    // FIXME: Our choice of alignment here and below is probably pessimistic.
4131
4
    llvm::Value *V = CGF.Builder.CreateAlignedLoad(
4132
4
        TyLo, CGF.Builder.CreateBitCast(RegLoAddr, PTyLo),
4133
4
        CharUnits::fromQuantity(getDataLayout().getABITypeAlignment(TyLo)));
4134
4
    CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 0));
4135
4136
    // Copy the second element.
4137
4
    V = CGF.Builder.CreateAlignedLoad(
4138
4
        TyHi, CGF.Builder.CreateBitCast(RegHiAddr, PTyHi),
4139
4
        CharUnits::fromQuantity(getDataLayout().getABITypeAlignment(TyHi)));
4140
4
    CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 1));
4141
4142
4
    RegAddr = CGF.Builder.CreateElementBitCast(Tmp, LTy);
4143
75
  } else if (neededInt) {
4144
59
    RegAddr = Address(CGF.Builder.CreateGEP(CGF.Int8Ty, RegSaveArea, gp_offset),
4145
59
                      CharUnits::fromQuantity(8));
4146
59
    RegAddr = CGF.Builder.CreateElementBitCast(RegAddr, LTy);
4147
4148
    // Copy to a temporary if necessary to ensure the appropriate alignment.
4149
59
    auto TInfo = getContext().getTypeInfoInChars(Ty);
4150
59
    uint64_t TySize = TInfo.Width.getQuantity();
4151
59
    CharUnits TyAlign = TInfo.Align;
4152
4153
    // Copy into a temporary if the type is more aligned than the
4154
    // register save area.
4155
59
    if (TyAlign.getQuantity() > 8) {
4156
3
      Address Tmp = CGF.CreateMemTemp(Ty);
4157
3
      CGF.Builder.CreateMemCpy(Tmp, RegAddr, TySize, false);
4158
3
      RegAddr = Tmp;
4159
3
    }
4160
4161
59
  } else 
if (16
neededSSE == 116
) {
4162
7
    RegAddr = Address(CGF.Builder.CreateGEP(CGF.Int8Ty, RegSaveArea, fp_offset),
4163
7
                      CharUnits::fromQuantity(16));
4164
7
    RegAddr = CGF.Builder.CreateElementBitCast(RegAddr, LTy);
4165
9
  } else {
4166
9
    assert(neededSSE == 2 && "Invalid number of needed registers!");
4167
    // SSE registers are spaced 16 bytes apart in the register save
4168
    // area, we need to collect the two eightbytes together.
4169
    // The ABI isn't explicit about this, but it seems reasonable
4170
    // to assume that the slots are 16-byte aligned, since the stack is
4171
    // naturally 16-byte aligned and the prologue is expected to store
4172
    // all the SSE registers to the RSA.
4173
0
    Address RegAddrLo = Address(CGF.Builder.CreateGEP(CGF.Int8Ty, RegSaveArea,
4174
9
                                                      fp_offset),
4175
9
                                CharUnits::fromQuantity(16));
4176
9
    Address RegAddrHi =
4177
9
      CGF.Builder.CreateConstInBoundsByteGEP(RegAddrLo,
4178
9
                                             CharUnits::fromQuantity(16));
4179
9
    llvm::Type *ST = AI.canHaveCoerceToType()
4180
9
                         ? AI.getCoerceToType()
4181
9
                         : 
llvm::StructType::get(CGF.DoubleTy, CGF.DoubleTy)0
;
4182
9
    llvm::Value *V;
4183
9
    Address Tmp = CGF.CreateMemTemp(Ty);
4184
9
    Tmp = CGF.Builder.CreateElementBitCast(Tmp, ST);
4185
9
    V = CGF.Builder.CreateLoad(CGF.Builder.CreateElementBitCast(
4186
9
        RegAddrLo, ST->getStructElementType(0)));
4187
9
    CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 0));
4188
9
    V = CGF.Builder.CreateLoad(CGF.Builder.CreateElementBitCast(
4189
9
        RegAddrHi, ST->getStructElementType(1)));
4190
9
    CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 1));
4191
4192
9
    RegAddr = CGF.Builder.CreateElementBitCast(Tmp, LTy);
4193
9
  }
4194
4195
  // AMD64-ABI 3.5.7p5: Step 5. Set:
4196
  // l->gp_offset = l->gp_offset + num_gp * 8
4197
  // l->fp_offset = l->fp_offset + num_fp * 16.
4198
79
  if (neededInt) {
4199
63
    llvm::Value *Offset = llvm::ConstantInt::get(CGF.Int32Ty, neededInt * 8);
4200
63
    CGF.Builder.CreateStore(CGF.Builder.CreateAdd(gp_offset, Offset),
4201
63
                            gp_offset_p);
4202
63
  }
4203
79
  if (neededSSE) {
4204
20
    llvm::Value *Offset = llvm::ConstantInt::get(CGF.Int32Ty, neededSSE * 16);
4205
20
    CGF.Builder.CreateStore(CGF.Builder.CreateAdd(fp_offset, Offset),
4206
20
                            fp_offset_p);
4207
20
  }
4208
79
  CGF.EmitBranch(ContBlock);
4209
4210
  // Emit code to load the value if it was passed in memory.
4211
4212
79
  CGF.EmitBlock(InMemBlock);
4213
79
  Address MemAddr = EmitX86_64VAArgFromMemory(CGF, VAListAddr, Ty);
4214
4215
  // Return the appropriate result.
4216
4217
79
  CGF.EmitBlock(ContBlock);
4218
79
  Address ResAddr = emitMergePHI(CGF, RegAddr, InRegBlock, MemAddr, InMemBlock,
4219
79
                                 "vaarg.addr");
4220
79
  return ResAddr;
4221
97
}
4222
4223
Address X86_64ABIInfo::EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr,
4224
8
                                   QualType Ty) const {
4225
  // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
4226
  // not 1, 2, 4, or 8 bytes, must be passed by reference."
4227
8
  uint64_t Width = getContext().getTypeSize(Ty);
4228
8
  bool IsIndirect = Width > 64 || 
!llvm::isPowerOf2_64(Width)4
;
4229
4230
8
  return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect,
4231
8
                          CGF.getContext().getTypeInfoInChars(Ty),
4232
8
                          CharUnits::fromQuantity(8),
4233
8
                          /*allowHigherAlign*/ false);
4234
8
}
4235
4236
ABIArgInfo WinX86_64ABIInfo::reclassifyHvaArgForVectorCall(
4237
97
    QualType Ty, unsigned &FreeSSERegs, const ABIArgInfo &current) const {
4238
97
  const Type *Base = nullptr;
4239
97
  uint64_t NumElts = 0;
4240
4241
97
  if (!Ty->isBuiltinType() && 
!Ty->isVectorType()49
&&
4242
97
      
isHomogeneousAggregate(Ty, Base, NumElts)36
&&
FreeSSERegs >= NumElts27
) {
4243
18
    FreeSSERegs -= NumElts;
4244
18
    return getDirectX86Hva();
4245
18
  }
4246
79
  return current;
4247
97
}
4248
4249
ABIArgInfo WinX86_64ABIInfo::classify(QualType Ty, unsigned &FreeSSERegs,
4250
                                      bool IsReturnType, bool IsVectorCall,
4251
9.89k
                                      bool IsRegCall) const {
4252
4253
9.89k
  if (Ty->isVoidType())
4254
1.65k
    return ABIArgInfo::getIgnore();
4255
4256
8.23k
  if (const EnumType *EnumTy = Ty->getAs<EnumType>())
4257
20
    Ty = EnumTy->getDecl()->getIntegerType();
4258
4259
8.23k
  TypeInfo Info = getContext().getTypeInfo(Ty);
4260
8.23k
  uint64_t Width = Info.Width;
4261
8.23k
  CharUnits Align = getContext().toCharUnitsFromBits(Info.Align);
4262
4263
8.23k
  const RecordType *RT = Ty->getAs<RecordType>();
4264
8.23k
  if (RT) {
4265
287
    if (!IsReturnType) {
4266
252
      if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(RT, getCXXABI()))
4267
71
        return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
4268
252
    }
4269
4270
216
    if (RT->getDecl()->hasFlexibleArrayMember())
4271
0
      return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
4272
4273
216
  }
4274
4275
8.16k
  const Type *Base = nullptr;
4276
8.16k
  uint64_t NumElts = 0;
4277
  // vectorcall adds the concept of a homogenous vector aggregate, similar to
4278
  // other targets.
4279
8.16k
  if ((IsVectorCall || 
IsRegCall8.05k
) &&
4280
8.16k
      
isHomogeneousAggregate(Ty, Base, NumElts)171
) {
4281
94
    if (IsRegCall) {
4282
31
      if (FreeSSERegs >= NumElts) {
4283
31
        FreeSSERegs -= NumElts;
4284
31
        if (IsReturnType || 
Ty->isBuiltinType()27
||
Ty->isVectorType()21
)
4285
16
          return ABIArgInfo::getDirect();
4286
15
        return ABIArgInfo::getExpand();
4287
31
      }
4288
0
      return ABIArgInfo::getIndirect(Align, /*ByVal=*/false);
4289
63
    } else if (IsVectorCall) {
4290
63
      if (FreeSSERegs >= NumElts &&
4291
63
          
(57
IsReturnType57
||
Ty->isBuiltinType()45
||
Ty->isVectorType()36
)) {
4292
34
        FreeSSERegs -= NumElts;
4293
34
        return ABIArgInfo::getDirect();
4294
34
      } else 
if (29
IsReturnType29
) {
4295
0
        return ABIArgInfo::getExpand();
4296
29
      } else if (!Ty->isBuiltinType() && 
!Ty->isVectorType()27
) {
4297
        // HVAs are delayed and reclassified in the 2nd step.
4298
27
        return ABIArgInfo::getIndirect(Align, /*ByVal=*/false);
4299
27
      }
4300
63
    }
4301
94
  }
4302
4303
8.07k
  if (Ty->isMemberPointerType()) {
4304
    // If the member pointer is represented by an LLVM int or ptr, pass it
4305
    // directly.
4306
159
    llvm::Type *LLTy = CGT.ConvertType(Ty);
4307
159
    if (LLTy->isPointerTy() || 
LLTy->isIntegerTy()76
)
4308
91
      return ABIArgInfo::getDirect();
4309
159
  }
4310
4311
7.98k
  if (RT || 
Ty->isAnyComplexType()7.81k
||
Ty->isMemberPointerType()7.79k
) {
4312
    // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
4313
    // not 1, 2, 4, or 8 bytes, must be passed by reference."
4314
253
    if (Width > 64 || 
!llvm::isPowerOf2_64(Width)143
)
4315
110
      return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
4316
4317
    // Otherwise, coerce it to a small integer.
4318
143
    return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(), Width));
4319
253
  }
4320
4321
7.73k
  if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
4322
2.13k
    switch (BT->getKind()) {
4323
29
    case BuiltinType::Bool:
4324
      // Bool type is always extended to the ABI, other builtin types are not
4325
      // extended.
4326
29
      return ABIArgInfo::getExtend(Ty);
4327
4328
56
    case BuiltinType::LongDouble:
4329
      // Mingw64 GCC uses the old 80 bit extended precision floating point
4330
      // unit. It passes them indirectly through memory.
4331
56
      if (IsMingw64) {
4332
6
        const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();
4333
6
        if (LDF == &llvm::APFloat::x87DoubleExtended())
4334
6
          return ABIArgInfo::getIndirect(Align, /*ByVal=*/false);
4335
6
      }
4336
50
      break;
4337
4338
50
    case BuiltinType::Int128:
4339
10
    case BuiltinType::UInt128:
4340
      // If it's a parameter type, the normal ABI rule is that arguments larger
4341
      // than 8 bytes are passed indirectly. GCC follows it. We follow it too,
4342
      // even though it isn't particularly efficient.
4343
10
      if (!IsReturnType)
4344
4
        return ABIArgInfo::getIndirect(Align, /*ByVal=*/false);
4345
4346
      // Mingw64 GCC returns i128 in XMM0. Coerce to v2i64 to handle that.
4347
      // Clang matches them for compatibility.
4348
6
      return ABIArgInfo::getDirect(llvm::FixedVectorType::get(
4349
6
          llvm::Type::getInt64Ty(getVMContext()), 2));
4350
4351
2.03k
    default:
4352
2.03k
      break;
4353
2.13k
    }
4354
2.13k
  }
4355
4356
7.68k
  if (Ty->isExtIntType()) {
4357
    // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
4358
    // not 1, 2, 4, or 8 bytes, must be passed by reference."
4359
    // However, non-power-of-two _ExtInts will be passed as 1,2,4 or 8 bytes
4360
    // anyway as long is it fits in them, so we don't have to check the power of
4361
    // 2.
4362
54
    if (Width <= 64)
4363
35
      return ABIArgInfo::getDirect();
4364
19
    return ABIArgInfo::getIndirect(Align, /*ByVal=*/false);
4365
54
  }
4366
4367
7.63k
  return ABIArgInfo::getDirect();
4368
7.68k
}
4369
4370
4.08k
void WinX86_64ABIInfo::computeInfo(CGFunctionInfo &FI) const {
4371
4.08k
  const unsigned CC = FI.getCallingConvention();
4372
4.08k
  bool IsVectorCall = CC == llvm::CallingConv::X86_VectorCall;
4373
4.08k
  bool IsRegCall = CC == llvm::CallingConv::X86_RegCall;
4374
4375
  // If __attribute__((sysv_abi)) is in use, use the SysV argument
4376
  // classification rules.
4377
4.08k
  if (CC == llvm::CallingConv::X86_64_SysV) {
4378
14
    X86_64ABIInfo SysVABIInfo(CGT, AVXLevel);
4379
14
    SysVABIInfo.computeInfo(FI);
4380
14
    return;
4381
14
  }
4382
4383
4.06k
  unsigned FreeSSERegs = 0;
4384
4.06k
  if (IsVectorCall) {
4385
    // We can use up to 4 SSE return registers with vectorcall.
4386
49
    FreeSSERegs = 4;
4387
4.01k
  } else if (IsRegCall) {
4388
    // RegCall gives us 16 SSE registers.
4389
24
    FreeSSERegs = 16;
4390
24
  }
4391
4392
4.06k
  if (!getCXXABI().classifyReturnType(FI))
4393
4.02k
    FI.getReturnInfo() = classify(FI.getReturnType(), FreeSSERegs, true,
4394
4.02k
                                  IsVectorCall, IsRegCall);
4395
4396
4.06k
  if (IsVectorCall) {
4397
    // We can use up to 6 SSE register parameters with vectorcall.
4398
49
    FreeSSERegs = 6;
4399
4.01k
  } else if (IsRegCall) {
4400
    // RegCall gives us 16 SSE registers, we can reuse the return registers.
4401
24
    FreeSSERegs = 16;
4402
24
  }
4403
4404
4.06k
  unsigned ArgNum = 0;
4405
4.06k
  unsigned ZeroSSERegs = 0;
4406
5.86k
  for (auto &I : FI.arguments()) {
4407
    // Vectorcall in x64 only permits the first 6 arguments to be passed as
4408
    // XMM/YMM registers. After the sixth argument, pretend no vector
4409
    // registers are left.
4410
5.86k
    unsigned *MaybeFreeSSERegs =
4411
5.86k
        (IsVectorCall && 
ArgNum >= 697
) ?
&ZeroSSERegs5
:
&FreeSSERegs5.85k
;
4412
5.86k
    I.info =
4413
5.86k
        classify(I.type, *MaybeFreeSSERegs, false, IsVectorCall, IsRegCall);
4414
5.86k
    ++ArgNum;
4415
5.86k
  }
4416
4417
4.06k
  if (IsVectorCall) {
4418
    // For vectorcall, assign aggregate HVAs to any free vector registers in a
4419
    // second pass.
4420
49
    for (auto &I : FI.arguments())
4421
97
      I.info = reclassifyHvaArgForVectorCall(I.type, FreeSSERegs, I.info);
4422
49
  }
4423
4.06k
}
4424
4425
Address WinX86_64ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
4426
23
                                    QualType Ty) const {
4427
  // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
4428
  // not 1, 2, 4, or 8 bytes, must be passed by reference."
4429
23
  uint64_t Width = getContext().getTypeSize(Ty);
4430
23
  bool IsIndirect = Width > 64 || 
!llvm::isPowerOf2_64(Width)8
;
4431
4432
23
  return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect,
4433
23
                          CGF.getContext().getTypeInfoInChars(Ty),
4434
23
                          CharUnits::fromQuantity(8),
4435
23
                          /*allowHigherAlign*/ false);
4436
23
}
4437
4438
static bool PPC_initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
4439
                                        llvm::Value *Address, bool Is64Bit,
4440
4
                                        bool IsAIX) {
4441
  // This is calculated from the LLVM and GCC tables and verified
4442
  // against gcc output.  AFAIK all PPC ABIs use the same encoding.
4443
4444
4
  CodeGen::CGBuilderTy &Builder = CGF.Builder;
4445
4446
4
  llvm::IntegerType *i8 = CGF.Int8Ty;
4447
4
  llvm::Value *Four8 = llvm::ConstantInt::get(i8, 4);
4448
4
  llvm::Value *Eight8 = llvm::ConstantInt::get(i8, 8);
4449
4
  llvm::Value *Sixteen8 = llvm::ConstantInt::get(i8, 16);
4450
4451
  // 0-31: r0-31, the 4-byte or 8-byte general-purpose registers
4452
4
  AssignToArrayRange(Builder, Address, Is64Bit ? 
Eight82
:
Four82
, 0, 31);
4453
4454
  // 32-63: fp0-31, the 8-byte floating-point registers
4455
4
  AssignToArrayRange(Builder, Address, Eight8, 32, 63);
4456
4457
  // 64-67 are various 4-byte or 8-byte special-purpose registers:
4458
  // 64: mq
4459
  // 65: lr
4460
  // 66: ctr
4461
  // 67: ap
4462
4
  AssignToArrayRange(Builder, Address, Is64Bit ? 
Eight82
:
Four82
, 64, 67);
4463
4464
  // 68-76 are various 4-byte special-purpose registers:
4465
  // 68-75 cr0-7
4466
  // 76: xer
4467
4
  AssignToArrayRange(Builder, Address, Four8, 68, 76);
4468
4469
  // 77-108: v0-31, the 16-byte vector registers
4470
4
  AssignToArrayRange(Builder, Address, Sixteen8, 77, 108);
4471
4472
  // 109: vrsave
4473
  // 110: vscr
4474
4
  AssignToArrayRange(Builder, Address, Is64Bit ? 
Eight82
:
Four82
, 109, 110);
4475
4476
  // AIX does not utilize the rest of the registers.
4477
4
  if (IsAIX)
4478
2
    return false;
4479
4480
  // 111: spe_acc
4481
  // 112: spefscr
4482
  // 113: sfp
4483
2
  AssignToArrayRange(Builder, Address, Is64Bit ? 
Eight81
:
Four81
, 111, 113);
4484
4485
2
  if (!Is64Bit)
4486
1
    return false;
4487
4488
  // TODO: Need to verify if these registers are used on 64 bit AIX with Power8
4489
  // or above CPU.
4490
  // 64-bit only registers:
4491
  // 114: tfhar
4492
  // 115: tfiar
4493
  // 116: texasr
4494
1
  AssignToArrayRange(Builder, Address, Eight8, 114, 116);
4495
4496
1
  return false;
4497
2
}
4498
4499
// AIX
4500
namespace {
4501
/// AIXABIInfo - The AIX XCOFF ABI information.
4502
class AIXABIInfo : public ABIInfo {
4503
  const bool Is64Bit;
4504
  const unsigned PtrByteSize;
4505
  CharUnits getParamTypeAlignment(QualType Ty) const;
4506
4507
public:
4508
  AIXABIInfo(CodeGen::CodeGenTypes &CGT, bool Is64Bit)
4509
150
      : ABIInfo(CGT), Is64Bit(Is64Bit), PtrByteSize(Is64Bit ? 8 : 4) {}
4510
4511
  bool isPromotableTypeForABI(QualType Ty) const;
4512
4513
  ABIArgInfo classifyReturnType(QualType RetTy) const;
4514
  ABIArgInfo classifyArgumentType(QualType Ty) const;
4515
4516
408
  void computeInfo(CGFunctionInfo &FI) const override {
4517
408
    if (!getCXXABI().classifyReturnType(FI))
4518
406
      FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
4519
4520
408
    for (auto &I : FI.arguments())
4521
311
      I.info = classifyArgumentType(I.type);
4522
408
  }
4523
4524
  Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
4525
                    QualType Ty) const override;
4526
};
4527
4528
class AIXTargetCodeGenInfo : public TargetCodeGenInfo {
4529
  const bool Is64Bit;
4530
4531
public:
4532
  AIXTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, bool Is64Bit)
4533
      : TargetCodeGenInfo(std::make_unique<AIXABIInfo>(CGT, Is64Bit)),
4534
150
        Is64Bit(Is64Bit) {}
4535
2
  int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
4536
2
    return 1; // r1 is the dedicated stack pointer
4537
2
  }
4538
4539
  bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
4540
                               llvm::Value *Address) const override;
4541
};
4542
} // namespace
4543
4544
// Return true if the ABI requires Ty to be passed sign- or zero-
4545
// extended to 32/64 bits.
4546
432
bool AIXABIInfo::isPromotableTypeForABI(QualType Ty) const {
4547
  // Treat an enum type as its underlying type.
4548
432
  if (const EnumType *EnumTy = Ty->getAs<EnumType>())
4549
8
    Ty = EnumTy->getDecl()->getIntegerType();
4550
4551
  // Promotable integer types are required to be promoted by the ABI.
4552
432
  if (Ty->isPromotableIntegerType())
4553
18
    return true;
4554
4555
414
  if (!Is64Bit)
4556
226
    return false;
4557
4558
  // For 64 bit mode, in addition to the usual promotable integer types, we also
4559
  // need to extend all 32-bit types, since the ABI requires promotion to 64
4560
  // bits.
4561
188
  if (const BuiltinType *BT = Ty->getAs<BuiltinType>())
4562
141
    switch (BT->getKind()) {
4563
53
    case BuiltinType::Int:
4564
68
    case BuiltinType::UInt:
4565
68
      return true;
4566
73
    default:
4567
73
      break;
4568
141
    }
4569
4570
120
  return false;
4571
188
}
4572
4573
406
ABIArgInfo AIXABIInfo::classifyReturnType(QualType RetTy) const {
4574
406
  if (RetTy->isAnyComplexType())
4575
12
    return ABIArgInfo::getDirect();
4576
4577
394
  if (RetTy->isVectorType())
4578
33
    return ABIArgInfo::getDirect();
4579
4580
361
  if (RetTy->isVoidType())
4581
141
    return ABIArgInfo::getIgnore();
4582
4583
220
  if (isAggregateTypeForABI(RetTy))
4584
22
    return getNaturalAlignIndirect(RetTy);
4585
4586
198
  return (isPromotableTypeForABI(RetTy) ? 
ABIArgInfo::getExtend(RetTy)55
4587
198
                                        : 
ABIArgInfo::getDirect()143
);
4588
220
}
4589
4590
311
ABIArgInfo AIXABIInfo::classifyArgumentType(QualType Ty) const {
4591
311
  Ty = useFirstFieldIfTransparentUnion(Ty);
4592
4593
311
  if (Ty->isAnyComplexType())
4594
6
    return ABIArgInfo::getDirect();
4595
4596
305
  if (Ty->isVectorType())
4597
45
    return ABIArgInfo::getDirect();
4598
4599
260
  if (isAggregateTypeForABI(Ty)) {
4600
    // Records with non-trivial destructors/copy-constructors should not be
4601
    // passed by value.
4602
26
    if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))
4603
2
      return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
4604
4605
24
    CharUnits CCAlign = getParamTypeAlignment(Ty);
4606
24
    CharUnits TyAlign = getContext().getTypeAlignInChars(Ty);
4607
4608
24
    return ABIArgInfo::getIndirect(CCAlign, /*ByVal*/ true,
4609
24
                                   /*Realign*/ TyAlign > CCAlign);
4610
26
  }
4611
4612
234
  return (isPromotableTypeForABI(Ty) ? 
ABIArgInfo::getExtend(Ty)31
4613
234
                                     : 
ABIArgInfo::getDirect()203
);
4614
260
}
4615
4616
30
CharUnits AIXABIInfo::getParamTypeAlignment(QualType Ty) const {
4617
  // Complex types are passed just like their elements.
4618
30
  if (const ComplexType *CTy = Ty->getAs<ComplexType>())
4619
0
    Ty = CTy->getElementType();
4620
4621
30
  if (Ty->isVectorType())
4622
2
    return CharUnits::fromQuantity(16);
4623
4624
  // If the structure contains a vector type, the alignment is 16.
4625
28
  if (isRecordWithSIMDVectorType(getContext(), Ty))
4626
2
    return CharUnits::fromQuantity(16);
4627
4628
26
  return CharUnits::fromQuantity(PtrByteSize);
4629
28
}
4630
4631
Address AIXABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
4632
6
                              QualType Ty) const {
4633
6
  if (Ty->isAnyComplexType())
4634
0
    llvm::report_fatal_error("complex type is not supported on AIX yet");
4635
4636
6
  auto TypeInfo = getContext().getTypeInfoInChars(Ty);
4637
6
  TypeInfo.Align = getParamTypeAlignment(Ty);
4638
4639
6
  CharUnits SlotSize = CharUnits::fromQuantity(PtrByteSize);
4640
4641
6
  return emitVoidPtrVAArg(CGF, VAListAddr, Ty, /*Indirect*/ false, TypeInfo,
4642
6
                          SlotSize, /*AllowHigher*/ true);
4643
6
}
4644
4645
bool AIXTargetCodeGenInfo::initDwarfEHRegSizeTable(
4646
2
    CodeGen::CodeGenFunction &CGF, llvm::Value *Address) const {
4647
2
  return PPC_initDwarfEHRegSizeTable(CGF, Address, Is64Bit, /*IsAIX*/ true);
4648
2
}
4649
4650
// PowerPC-32
4651
namespace {
4652
/// PPC32_SVR4_ABIInfo - The 32-bit PowerPC ELF (SVR4) ABI information.
4653
class PPC32_SVR4_ABIInfo : public DefaultABIInfo {
4654
  bool IsSoftFloatABI;
4655
  bool IsRetSmallStructInRegABI;
4656
4657
  CharUnits getParamTypeAlignment(QualType Ty) const;
4658
4659
public:
4660
  PPC32_SVR4_ABIInfo(CodeGen::CodeGenTypes &CGT, bool SoftFloatABI,
4661
                     bool RetSmallStructInRegABI)
4662
      : DefaultABIInfo(CGT), IsSoftFloatABI(SoftFloatABI),
4663
99
        IsRetSmallStructInRegABI(RetSmallStructInRegABI) {}
4664
4665
  ABIArgInfo classifyReturnType(QualType RetTy) const;
4666
4667
1.05k
  void computeInfo(CGFunctionInfo &FI) const override {
4668
1.05k
    if (!getCXXABI().classifyReturnType(FI))
4669
1.05k
      FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
4670
1.05k
    for (auto &I : FI.arguments())
4671
1.93k
      I.info = classifyArgumentType(I.type);
4672
1.05k
  }
4673
4674
  Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
4675
                    QualType Ty) const override;
4676
};
4677
4678
class PPC32TargetCodeGenInfo : public TargetCodeGenInfo {
4679
public:
4680
  PPC32TargetCodeGenInfo(CodeGenTypes &CGT, bool SoftFloatABI,
4681
                         bool RetSmallStructInRegABI)
4682
      : TargetCodeGenInfo(std::make_unique<PPC32_SVR4_ABIInfo>(
4683
99
            CGT, SoftFloatABI, RetSmallStructInRegABI)) {}
4684
4685
  static bool isStructReturnInRegABI(const llvm::Triple &Triple,
4686
                                     const CodeGenOptions &Opts);
4687
4688
1
  int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
4689
    // This is recovered from gcc output.
4690
1
    return 1; // r1 is the dedicated stack pointer
4691
1
  }
4692
4693
  bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
4694
                               llvm::Value *Address) const override;
4695
};
4696
}
4697
4698
0
CharUnits PPC32_SVR4_ABIInfo::getParamTypeAlignment(QualType Ty) const {
4699
  // Complex types are passed just like their elements.
4700
0
  if (const ComplexType *CTy = Ty->getAs<ComplexType>())
4701
0
    Ty = CTy->getElementType();
4702
4703
0
  if (Ty->isVectorType())
4704
0
    return CharUnits::fromQuantity(getContext().getTypeSize(Ty) == 128 ? 16
4705
0
                                                                       : 4);
4706
4707
  // For single-element float/vector structs, we consider the whole type
4708
  // to have the same alignment requirements as its single element.
4709
0
  const Type *AlignTy = nullptr;
4710
0
  if (const Type *EltType = isSingleElementStruct(Ty, getContext())) {
4711
0
    const BuiltinType *BT = EltType->getAs<BuiltinType>();
4712
0
    if ((EltType->isVectorType() && getContext().getTypeSize(EltType) == 128) ||
4713
0
        (BT && BT->isFloatingPoint()))
4714
0
      AlignTy = EltType;
4715
0
  }
4716
4717
0
  if (AlignTy)
4718
0
    return CharUnits::fromQuantity(AlignTy->isVectorType() ? 16 : 4);
4719
0
  return CharUnits::fromQuantity(4);
4720
0
}
4721
4722
1.05k
ABIArgInfo PPC32_SVR4_ABIInfo::classifyReturnType(QualType RetTy) const {
4723
1.05k
  uint64_t Size;
4724
4725
  // -msvr4-struct-return puts small aggregates in GPR3 and GPR4.
4726
1.05k
  if (isAggregateTypeForABI(RetTy) && 
IsRetSmallStructInRegABI148
&&
4727
1.05k
      
(Size = getContext().getTypeSize(RetTy)) <= 6495
) {
4728
    // System V ABI (1995), page 3-22, specified:
4729
    // > A structure or union whose size is less than or equal to 8 bytes
4730
    // > shall be returned in r3 and r4, as if it were first stored in the
4731
    // > 8-byte aligned memory area and then the low addressed word were
4732
    // > loaded into r3 and the high-addressed word into r4.  Bits beyond
4733
    // > the last member of the structure or union are not defined.
4734
    //
4735
    // GCC for big-endian PPC32 inserts the pad before the first member,
4736
    // not "beyond the last member" of the struct.  To stay compatible
4737
    // with GCC, we coerce the struct to an integer of the same size.
4738
    // LLVM will extend it and return i32 in r3, or i64 in r3:r4.
4739
73
    if (Size == 0)
4740
8
      return ABIArgInfo::getIgnore();
4741
65
    else {
4742
65
      llvm::Type *CoerceTy = llvm::Type::getIntNTy(getVMContext(), Size);
4743
65
      return ABIArgInfo::getDirect(CoerceTy);
4744
65
    }
4745
73
  }
4746
4747
984
  return DefaultABIInfo::classifyReturnType(RetTy);
4748
1.05k
}
4749
4750
// TODO: this implementation is now likely redundant with
4751
// DefaultABIInfo::EmitVAArg.
4752
Address PPC32_SVR4_ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAList,
4753
6
                                      QualType Ty) const {
4754
6
  if (getTarget().getTriple().isOSDarwin()) {
4755
0
    auto TI = getContext().getTypeInfoInChars(Ty);
4756
0
    TI.Align = getParamTypeAlignment(Ty);
4757
4758
0
    CharUnits SlotSize = CharUnits::fromQuantity(4);
4759
0
    return emitVoidPtrVAArg(CGF, VAList, Ty,
4760
0
                            classifyArgumentType(Ty).isIndirect(), TI, SlotSize,
4761
0
                            /*AllowHigherAlign=*/true);
4762
0
  }
4763
4764
6
  const unsigned OverflowLimit = 8;
4765
6
  if (const ComplexType *CTy = Ty->getAs<ComplexType>()) {
4766
    // TODO: Implement this. For now ignore.
4767
0
    (void)CTy;
4768
0
    return Address::invalid(); // FIXME?
4769
0
  }
4770
4771
  // struct __va_list_tag {
4772
  //   unsigned char gpr;
4773
  //   unsigned char fpr;
4774
  //   unsigned short reserved;
4775
  //   void *overflow_arg_area;
4776
  //   void *reg_save_area;
4777
  // };
4778
4779
6
  bool isI64 = Ty->isIntegerType() && 
getContext().getTypeSize(Ty) == 641
;
4780
6
  bool isInt = !Ty->isFloatingType();
4781
6
  bool isF64 = Ty->isFloatingType() && 
getContext().getTypeSize(Ty) == 641
;
4782
4783
  // All aggregates are passed indirectly?  That doesn't seem consistent
4784
  // with the argument-lowering code.
4785
6
  bool isIndirect = isAggregateTypeForABI(Ty);
4786
4787
6
  CGBuilderTy &Builder = CGF.Builder;
4788
4789
  // The calling convention either uses 1-2 GPRs or 1 FPR.
4790
6
  Address NumRegsAddr = Address::invalid();
4791
6
  if (isInt || 
IsSoftFloatABI1
) {
4792
6
    NumRegsAddr = Builder.CreateStructGEP(VAList, 0, "gpr");
4793
6
  } else {
4794
0
    NumRegsAddr = Builder.CreateStructGEP(VAList, 1, "fpr");
4795
0
  }
4796
4797
6
  llvm::Value *NumRegs = Builder.CreateLoad(NumRegsAddr, "numUsedRegs");
4798
4799
  // "Align" the register count when TY is i64.
4800
6
  if (isI64 || (isF64 && 
IsSoftFloatABI1
)) {
4801
1
    NumRegs = Builder.CreateAdd(NumRegs, Builder.getInt8(1));
4802
1
    NumRegs = Builder.CreateAnd(NumRegs, Builder.getInt8((uint8_t) ~1U));
4803
1
  }
4804
4805
6
  llvm::Value *CC =
4806
6
      Builder.CreateICmpULT(NumRegs, Builder.getInt8(OverflowLimit), "cond");
4807
4808
6
  llvm::BasicBlock *UsingRegs = CGF.createBasicBlock("using_regs");
4809
6
  llvm::BasicBlock *UsingOverflow = CGF.createBasicBlock("using_overflow");
4810
6
  llvm::BasicBlock *Cont = CGF.createBasicBlock("cont");
4811
4812
6
  Builder.CreateCondBr(CC, UsingRegs, UsingOverflow);
4813
4814
6
  llvm::Type *DirectTy = CGF.ConvertType(Ty);
4815
6
  if (isIndirect) 
DirectTy = DirectTy->getPointerTo(0)2
;
4816
4817
  // Case 1: consume registers.
4818
6
  Address RegAddr = Address::invalid();
4819
6
  {
4820
6
    CGF.EmitBlock(UsingRegs);
4821
4822
6
    Address RegSaveAreaPtr = Builder.CreateStructGEP(VAList, 4);
4823
6
    RegAddr = Address(Builder.CreateLoad(RegSaveAreaPtr),
4824
6
                      CharUnits::fromQuantity(8));
4825
6
    assert(RegAddr.getElementType() == CGF.Int8Ty);
4826
4827
    // Floating-point registers start after the general-purpose registers.
4828
6
    if (!(isInt || 
IsSoftFloatABI1
)) {
4829
0
      RegAddr = Builder.CreateConstInBoundsByteGEP(RegAddr,
4830
0
                                                   CharUnits::fromQuantity(32));
4831
0
    }
4832
4833
    // Get the address of the saved value by scaling the number of
4834
    // registers we've used by the number of
4835
6
    CharUnits RegSize = CharUnits::fromQuantity((isInt || 
IsSoftFloatABI1
) ? 4 :
80
);
4836
6
    llvm::Value *RegOffset =
4837
6
      Builder.CreateMul(NumRegs, Builder.getInt8(RegSize.getQuantity()));
4838
6
    RegAddr = Address(Builder.CreateInBoundsGEP(CGF.Int8Ty,
4839
6
                                            RegAddr.getPointer(), RegOffset),
4840
6
                      RegAddr.getAlignment().alignmentOfArrayElement(RegSize));
4841
6
    RegAddr = Builder.CreateElementBitCast(RegAddr, DirectTy);
4842
4843
    // Increase the used-register count.
4844
6
    NumRegs =
4845
6
      Builder.CreateAdd(NumRegs,
4846
6
                        Builder.getInt8((isI64 || (isF64 && 
IsSoftFloatABI1
)) ?
21
:
15
));
4847
6
    Builder.CreateStore(NumRegs, NumRegsAddr);
4848
4849
6
    CGF.EmitBranch(Cont);
4850
6
  }
4851
4852
  // Case 2: consume space in the overflow area.
4853
0
  Address MemAddr = Address::invalid();
4854
6
  {
4855
6
    CGF.EmitBlock(UsingOverflow);
4856
4857
6
    Builder.CreateStore(Builder.getInt8(OverflowLimit), NumRegsAddr);
4858
4859
    // Everything in the overflow area is rounded up to a size of at least 4.
4860
6
    CharUnits OverflowAreaAlign = CharUnits::fromQuantity(4);
4861
4862
6
    CharUnits Size;
4863
6
    if (!isIndirect) {
4864
4
      auto TypeInfo = CGF.getContext().getTypeInfoInChars(Ty);
4865
4
      Size = TypeInfo.Width.alignTo(OverflowAreaAlign);
4866
4
    } else {
4867
2
      Size = CGF.getPointerSize();
4868
2
    }
4869
4870
6
    Address OverflowAreaAddr = Builder.CreateStructGEP(VAList, 3);
4871
6
    Address OverflowArea(Builder.CreateLoad(OverflowAreaAddr, "argp.cur"),
4872
6
                         OverflowAreaAlign);
4873
    // Round up address of argument to alignment
4874
6
    CharUnits Align = CGF.getContext().getTypeAlignInChars(Ty);
4875
6
    if (Align > OverflowAreaAlign) {
4876
2
      llvm::Value *Ptr = OverflowArea.getPointer();
4877
2
      OverflowArea = Address(emitRoundPointerUpToAlignment(CGF, Ptr, Align),
4878
2
                                                           Align);
4879
2
    }
4880
4881
6
    MemAddr = Builder.CreateElementBitCast(OverflowArea, DirectTy);
4882
4883
    // Increase the overflow area.
4884
6
    OverflowArea = Builder.CreateConstInBoundsByteGEP(OverflowArea, Size);
4885
6
    Builder.CreateStore(OverflowArea.getPointer(), OverflowAreaAddr);
4886
6
    CGF.EmitBranch(Cont);
4887
6
  }
4888
4889
6
  CGF.EmitBlock(Cont);
4890
4891
  // Merge the cases with a phi.
4892
6
  Address Result = emitMergePHI(CGF, RegAddr, UsingRegs, MemAddr, UsingOverflow,
4893
6
                                "vaarg.addr");
4894
4895
  // Load the pointer if the argument was passed indirectly.
4896
6
  if (isIndirect) {
4897
2
    Result = Address(Builder.CreateLoad(Result, "aggr"),
4898
2
                     getContext().getTypeAlignInChars(Ty));
4899
2
  }
4900
4901
6
  return Result;
4902
6
}
4903
4904
bool PPC32TargetCodeGenInfo::isStructReturnInRegABI(
4905
99
    const llvm::Triple &Triple, const CodeGenOptions &Opts) {
4906
99
  assert(Triple.isPPC32());
4907
4908
0
  switch (Opts.getStructReturnConvention()) {
4909
93
  case CodeGenOptions::SRCK_Default:
4910
93
    break;
4911
3
  case CodeGenOptions::SRCK_OnStack: // -maix-struct-return
4912
3
    return false;
4913
3
  case CodeGenOptions::SRCK_InRegs: // -msvr4-struct-return
4914
3
    return true;
4915
99
  }
4916
4917
93
  if (Triple.isOSBinFormatELF() && !Triple.isOSLinux())
4918
77
    return true;
4919
4920
16
  return false;
4921
93
}
4922
4923
bool
4924
PPC32TargetCodeGenInfo::initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
4925
1
                                                llvm::Value *Address) const {
4926
1
  return PPC_initDwarfEHRegSizeTable(CGF, Address, /*Is64Bit*/ false,
4927
1
                                     /*IsAIX*/ false);
4928
1
}
4929
4930
// PowerPC-64
4931
4932
namespace {
4933
/// PPC64_SVR4_ABIInfo - The 64-bit PowerPC ELF (SVR4) ABI information.
4934
class PPC64_SVR4_ABIInfo : public SwiftABIInfo {
4935
public:
4936
  enum ABIKind {
4937
    ELFv1 = 0,
4938
    ELFv2
4939
  };
4940
4941
private:
4942
  static const unsigned GPRBits = 64;
4943
  ABIKind Kind;
4944
  bool IsSoftFloatABI;
4945
4946
public:
4947
  PPC64_SVR4_ABIInfo(CodeGen::CodeGenTypes &CGT, ABIKind Kind,
4948
                     bool SoftFloatABI)
4949
2.63k
      : SwiftABIInfo(CGT), Kind(Kind), IsSoftFloatABI(SoftFloatABI) {}
4950
4951
  bool isPromotableTypeForABI(QualType Ty) const;
4952
  CharUnits getParamTypeAlignment(QualType Ty) const;
4953
4954
  ABIArgInfo classifyReturnType(QualType RetTy) const;
4955
  ABIArgInfo classifyArgumentType(QualType Ty) const;
4956
4957
  bool isHomogeneousAggregateBaseType(QualType Ty) const override;
4958
  bool isHomogeneousAggregateSmallEnough(const Type *Ty,
4959
                                         uint64_t Members) const override;
4960
4961
  // TODO: We can add more logic to computeInfo to improve performance.
4962
  // Example: For aggregate arguments that fit in a register, we could
4963
  // use getDirectInReg (as is done below for structs containing a single
4964
  // floating-point value) to avoid pushing them to memory on function
4965
  // entry.  This would require changing the logic in PPCISelLowering
4966
  // when lowering the parameters in the caller and args in the callee.
4967
16.3k
  void computeInfo(CGFunctionInfo &FI) const override {
4968
16.3k
    if (!getCXXABI().classifyReturnType(FI))
4969
16.2k
      FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
4970
32.1k
    for (auto &I : FI.arguments()) {
4971
      // We rely on the default argument classification for the most part.
4972
      // One exception:  An aggregate containing a single floating-point
4973
      // or vector item must be passed in a register if one is available.
4974
32.1k
      const Type *T = isSingleElementStruct(I.type, getContext());
4975
32.1k
      if (T) {
4976
25
        const BuiltinType *BT = T->getAs<BuiltinType>();
4977
25
        if ((T->isVectorType() && 
getContext().getTypeSize(T) == 1284
) ||
4978
25
            
(22
BT22
&&
BT->isFloatingPoint()19
)) {
4979
21
          QualType QT(T, 0);
4980
21
          I.info = ABIArgInfo::getDirectInReg(CGT.ConvertType(QT));
4981
21
          continue;
4982
21
        }
4983
25
      }
4984
32.0k
      I.info = classifyArgumentType(I.type);
4985
32.0k
    }
4986
16.3k
  }
4987
4988
  Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
4989
                    QualType Ty) const override;
4990
4991
  bool shouldPassIndirectlyForSwift(ArrayRef<llvm::Type*> scalars,
4992
0
                                    bool asReturnValue) const override {
4993
0
    return occupiesMoreThan(CGT, scalars, /*total*/ 4);
4994
0
  }
4995
4996
0
  bool isSwiftErrorInRegister() const override {
4997
0
    return false;
4998
0
  }
4999
};
5000
5001
class PPC64_SVR4_TargetCodeGenInfo : public TargetCodeGenInfo {
5002
5003
public:
5004
  PPC64_SVR4_TargetCodeGenInfo(CodeGenTypes &CGT,
5005
                               PPC64_SVR4_ABIInfo::ABIKind Kind,
5006
                               bool SoftFloatABI)
5007
      : TargetCodeGenInfo(
5008
2.63k
            std::make_unique<PPC64_SVR4_ABIInfo>(CGT, Kind, SoftFloatABI)) {}
5009
5010
1
  int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
5011
    // This is recovered from gcc output.
5012
1
    return 1; // r1 is the dedicated stack pointer
5013
1
  }
5014
5015
  bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
5016
                               llvm::Value *Address) const override;
5017
};
5018
5019
class PPC64TargetCodeGenInfo : public DefaultTargetCodeGenInfo {
5020
public:
5021
0
  PPC64TargetCodeGenInfo(CodeGenTypes &CGT) : DefaultTargetCodeGenInfo(CGT) {}
5022
5023
0
  int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
5024
    // This is recovered from gcc output.
5025
0
    return 1; // r1 is the dedicated stack pointer
5026
0
  }
5027
5028
  bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
5029
                               llvm::Value *Address) const override;
5030
};
5031
5032
}
5033
5034
// Return true if the ABI requires Ty to be passed sign- or zero-
5035
// extended to 64 bits.
5036
bool
5037
37.0k
PPC64_SVR4_ABIInfo::isPromotableTypeForABI(QualType Ty) const {
5038
  // Treat an enum type as its underlying type.
5039
37.0k
  if (const EnumType *EnumTy = Ty->getAs<EnumType>())
5040
2
    Ty = EnumTy->getDecl()->getIntegerType();
5041
5042
  // Promotable integer types are required to be promoted by the ABI.
5043
37.0k
  if (isPromotableIntegerTypeForABI(Ty))
5044
302
    return true;
5045
5046
  // In addition to the usual promotable integer types, we also need to
5047
  // extend all 32-bit types, since the ABI requires promotion to 64 bits.
5048
36.7k
  if (const BuiltinType *BT = Ty->getAs<BuiltinType>())
5049
14.1k
    switch (BT->getKind()) {
5050
5.73k
    case BuiltinType::Int:
5051
5.99k
    case BuiltinType::UInt:
5052
5.99k
      return true;
5053
8.14k
    default:
5054
8.14k
      break;
5055
14.1k
    }
5056
5057
30.7k
  if (const auto *EIT = Ty->getAs<ExtIntType>())
5058
9
    if (EIT->getNumBits() < 64)
5059
2
      return true;
5060
5061
30.7k
  return false;
5062
30.7k
}
5063
5064
/// isAlignedParamType - Determine whether a type requires 16-byte or
5065
/// higher alignment in the parameter area.  Always returns at least 8.
5066
140
CharUnits PPC64_SVR4_ABIInfo::getParamTypeAlignment(QualType Ty) const {
5067
  // Complex types are passed just like their elements.
5068
140
  if (const ComplexType *CTy = Ty->getAs<ComplexType>())
5069
8
    Ty = CTy->getElementType();
5070
5071
  // Only vector types of size 16 bytes need alignment (larger types are
5072
  // passed via reference, smaller types are not aligned).
5073
140
  if (Ty->isVectorType()) {
5074
0
    return CharUnits::fromQuantity(getContext().getTypeSize(Ty) == 128 ? 16 : 8);
5075
140
  } else if (Ty->isRealFloatingType() &&
5076
140
             &getContext().getFloatTypeSemantics(Ty) ==
5077
15
                 &llvm::APFloat::IEEEquad()) {
5078
    // According to ABI document section 'Optional Save Areas': If extended
5079
    // precision floating-point values in IEEE BINARY 128 QUADRUPLE PRECISION
5080
    // format are supported, map them to a single quadword, quadword aligned.
5081
9
    return CharUnits::fromQuantity(16);
5082
9
  }
5083
5084
  // For single-element float/vector structs, we consider the whole type
5085
  // to have the same alignment requirements as its single element.
5086
131
  const Type *AlignAsType = nullptr;
5087
131
  const Type *EltType = isSingleElementStruct(Ty, getContext());
5088
131
  if (EltType) {
5089
3
    const BuiltinType *BT = EltType->getAs<BuiltinType>();
5090
3
    if ((EltType->isVectorType() && 
getContext().getTypeSize(EltType) == 1282
) ||
5091
3
        
(2
BT2
&&
BT->isFloatingPoint()1
))
5092
2
      AlignAsType = EltType;
5093
3
  }
5094
5095
  // Likewise for ELFv2 homogeneous aggregates.
5096
131
  const Type *Base = nullptr;
5097
131
  uint64_t Members = 0;
5098
131
  if (!AlignAsType && 
Kind == ELFv2129
&&
5099
131
      
isAggregateTypeForABI(Ty)92
&&
isHomogeneousAggregate(Ty, Base, Members)84
)
5100
50
    AlignAsType = Base;
5101
5102
  // With special case aggregates, only vector base types need alignment.
5103
131
  if (AlignAsType) {
5104
52
    return CharUnits::fromQuantity(AlignAsType->isVectorType() ? 
1619
:
833
);
5105
52
  }
5106
5107
  // Otherwise, we only need alignment for any aggregate type that
5108
  // has an alignment requirement of >= 16 bytes.
5109
79
  if (isAggregateTypeForABI(Ty) && 
getContext().getTypeAlign(Ty) >= 12865
) {
5110
14
    return CharUnits::fromQuantity(16);
5111
14
  }
5112
5113
65
  return CharUnits::fromQuantity(8);
5114
79
}
5115
5116
/// isHomogeneousAggregate - Return true if a type is an ELFv2 homogeneous
5117
/// aggregate.  Base is set to the base element type, and Members is set
5118
/// to the number of base elements.
5119
bool ABIInfo::isHomogeneousAggregate(QualType Ty, const Type *&Base,
5120
6.23k
                                     uint64_t &Members) const {
5121
6.23k
  if (const ConstantArrayType *AT = getContext().getAsConstantArrayType(Ty)) {
5122
1.13k
    uint64_t NElements = AT->getSize().getZExtValue();
5123
1.13k
    if (NElements == 0)
5124
0
      return false;
5125
1.13k
    if (!isHomogeneousAggregate(AT->getElementType(), Base, Members))
5126
155
      return false;
5127
984
    Members *= NElements;
5128
5.09k
  } else if (const RecordType *RT = Ty->getAs<RecordType>()) {
5129
2.09k
    const RecordDecl *RD = RT->getDecl();
5130
2.09k
    if (RD->hasFlexibleArrayMember())
5131
2
      return false;
5132
5133
2.09k
    Members = 0;
5134
5135
    // If this is a C++ record, check the properties of the record such as
5136
    // bases and ABI specific restrictions
5137
2.09k
    if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
5138
507
      if (!getCXXABI().isPermittedToBeHomogeneousAggregate(CXXRD))
5139
24
        return false;
5140
5141
483
      for (const auto &I : CXXRD->bases()) {
5142
        // Ignore empty records.
5143
174
        if (isEmptyRecord(getContext(), I.getType(), true))
5144
10
          continue;
5145
5146
164
        uint64_t FldMembers;
5147
164
        if (!isHomogeneousAggregate(I.getType(), Base, FldMembers))
5148
27
          return false;
5149
5150
137
        Members += FldMembers;
5151
137
      }
5152
483
    }
5153
5154
2.56k
    
for (const auto *FD : RD->fields())2.04k
{
5155
      // Ignore (non-zero arrays of) empty records.
5156
2.56k
      QualType FT = FD->getType();
5157
3.71k
      while (const ConstantArrayType *AT =
5158
2.56k
             getContext().getAsConstantArrayType(FT)) {
5159
1.14k
        if (AT->getSize().getZExtValue() == 0)
5160
0
          return false;
5161
1.14k
        FT = AT->getElementType();
5162
1.14k
      }
5163
2.56k
      if (isEmptyRecord(getContext(), FT, true))
5164
25
        continue;
5165
5166
      // For compatibility with GCC, ignore empty bitfields in C++ mode.
5167
2.54k
      if (getContext().getLangOpts().CPlusPlus &&
5168
2.54k
          
FD->isZeroLengthBitField(getContext())439
)
5169
7