Coverage Report

Created: 2021-01-19 06:58

/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/CodeGen/TargetInfo.cpp
Line
Count
Source (jump to first uncovered line)
1
//===---- TargetInfo.cpp - Encapsulate target details -----------*- C++ -*-===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// These classes wrap the information about a call or function
10
// definition used to handle ABI compliancy.
11
//
12
//===----------------------------------------------------------------------===//
13
14
#include "TargetInfo.h"
15
#include "ABIInfo.h"
16
#include "CGBlocks.h"
17
#include "CGCXXABI.h"
18
#include "CGValue.h"
19
#include "CodeGenFunction.h"
20
#include "clang/AST/Attr.h"
21
#include "clang/AST/RecordLayout.h"
22
#include "clang/Basic/CodeGenOptions.h"
23
#include "clang/Basic/DiagnosticFrontend.h"
24
#include "clang/CodeGen/CGFunctionInfo.h"
25
#include "clang/CodeGen/SwiftCallingConv.h"
26
#include "llvm/ADT/SmallBitVector.h"
27
#include "llvm/ADT/StringExtras.h"
28
#include "llvm/ADT/StringSwitch.h"
29
#include "llvm/ADT/Triple.h"
30
#include "llvm/ADT/Twine.h"
31
#include "llvm/IR/DataLayout.h"
32
#include "llvm/IR/IntrinsicsNVPTX.h"
33
#include "llvm/IR/Type.h"
34
#include "llvm/Support/raw_ostream.h"
35
#include <algorithm> // std::sort
36
37
using namespace clang;
38
using namespace CodeGen;
39
40
// Helper for coercing an aggregate argument or return value into an integer
41
// array of the same size (including padding) and alignment.  This alternate
42
// coercion happens only for the RenderScript ABI and can be removed after
43
// runtimes that rely on it are no longer supported.
44
//
45
// RenderScript assumes that the size of the argument / return value in the IR
46
// is the same as the size of the corresponding qualified type. This helper
47
// coerces the aggregate type into an array of the same size (including
48
// padding).  This coercion is used in lieu of expansion of struct members or
49
// other canonical coercions that return a coerced-type of larger size.
50
//
51
// Ty          - The argument / return value type
52
// Context     - The associated ASTContext
53
// LLVMContext - The associated LLVMContext
54
static ABIArgInfo coerceToIntArray(QualType Ty,
55
                                   ASTContext &Context,
56
19
                                   llvm::LLVMContext &LLVMContext) {
57
  // Alignment and Size are measured in bits.
58
19
  const uint64_t Size = Context.getTypeSize(Ty);
59
19
  const uint64_t Alignment = Context.getTypeAlign(Ty);
60
19
  llvm::Type *IntType = llvm::Type::getIntNTy(LLVMContext, Alignment);
61
19
  const uint64_t NumElements = (Size + Alignment - 1) / Alignment;
62
19
  return ABIArgInfo::getDirect(llvm::ArrayType::get(IntType, NumElements));
63
19
}
64
65
static void AssignToArrayRange(CodeGen::CGBuilderTy &Builder,
66
                               llvm::Value *Array,
67
                               llvm::Value *Value,
68
                               unsigned FirstIndex,
69
31
                               unsigned LastIndex) {
70
  // Alternatively, we could emit this as a loop in the source.
71
572
  for (unsigned I = FirstIndex; I <= LastIndex; 
++I541
) {
72
541
    llvm::Value *Cell =
73
541
        Builder.CreateConstInBoundsGEP1_32(Builder.getInt8Ty(), Array, I);
74
541
    Builder.CreateAlignedStore(Value, Cell, CharUnits::One());
75
541
  }
76
31
}
77
78
158k
static bool isAggregateTypeForABI(QualType T) {
79
158k
  return !CodeGenFunction::hasScalarEvaluationKind(T) ||
80
151k
         T->isMemberFunctionPointerType();
81
158k
}
82
83
ABIArgInfo ABIInfo::getNaturalAlignIndirect(QualType Ty, bool ByVal,
84
                                            bool Realign,
85
3.21k
                                            llvm::Type *Padding) const {
86
3.21k
  return ABIArgInfo::getIndirect(getContext().getTypeAlignInChars(Ty), ByVal,
87
3.21k
                                 Realign, Padding);
88
3.21k
}
89
90
ABIArgInfo
91
28
ABIInfo::getNaturalAlignIndirectInReg(QualType Ty, bool Realign) const {
92
28
  return ABIArgInfo::getIndirectInReg(getContext().getTypeAlignInChars(Ty),
93
28
                                      /*ByVal*/ false, Realign);
94
28
}
95
96
Address ABIInfo::EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr,
97
0
                             QualType Ty) const {
98
0
  return Address::invalid();
99
0
}
100
101
234k
bool ABIInfo::isPromotableIntegerTypeForABI(QualType Ty) const {
102
234k
  if (Ty->isPromotableIntegerType())
103
11.4k
    return true;
104
105
222k
  if (const auto *EIT = Ty->getAs<ExtIntType>())
106
262
    if (EIT->getNumBits() < getContext().getTypeSize(getContext().IntTy))
107
86
      return true;
108
109
222k
  return false;
110
222k
}
111
112
31.5k
ABIInfo::~ABIInfo() {}
113
114
/// Does the given lowering require more than the given number of
115
/// registers when expanded?
116
///
117
/// This is intended to be the basis of a reasonable basic implementation
118
/// of should{Pass,Return}IndirectlyForSwift.
119
///
120
/// For most targets, a limit of four total registers is reasonable; this
121
/// limits the amount of code required in order to move around the value
122
/// in case it wasn't produced immediately prior to the call by the caller
123
/// (or wasn't produced in exactly the right registers) or isn't used
124
/// immediately within the callee.  But some targets may need to further
125
/// limit the register count due to an inability to support that many
126
/// return registers.
127
static bool occupiesMoreThan(CodeGenTypes &cgt,
128
                             ArrayRef<llvm::Type*> scalarTypes,
129
1.05k
                             unsigned maxAllRegisters) {
130
1.05k
  unsigned intCount = 0, fpCount = 0;
131
4.18k
  for (llvm::Type *type : scalarTypes) {
132
4.18k
    if (type->isPointerTy()) {
133
0
      intCount++;
134
4.18k
    } else if (auto intTy = dyn_cast<llvm::IntegerType>(type)) {
135
2.30k
      auto ptrWidth = cgt.getTarget().getPointerWidth(0);
136
2.30k
      intCount += (intTy->getBitWidth() + ptrWidth - 1) / ptrWidth;
137
1.88k
    } else {
138
1.88k
      assert(type->isVectorTy() || type->isFloatingPointTy());
139
1.88k
      fpCount++;
140
1.88k
    }
141
4.18k
  }
142
143
1.05k
  return (intCount + fpCount > maxAllRegisters);
144
1.05k
}
145
146
bool SwiftABIInfo::isLegalVectorTypeForSwift(CharUnits vectorSize,
147
                                             llvm::Type *eltTy,
148
440
                                             unsigned numElts) const {
149
  // The default implementation of this assumes that the target guarantees
150
  // 128-bit SIMD support but nothing more.
151
440
  return (vectorSize.getQuantity() > 8 && 
vectorSize.getQuantity() <= 16432
);
152
440
}
153
154
static CGCXXABI::RecordArgABI getRecordArgABI(const RecordType *RT,
155
18.2k
                                              CGCXXABI &CXXABI) {
156
18.2k
  const CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(RT->getDecl());
157
18.2k
  if (!RD) {
158
3.82k
    if (!RT->getDecl()->canPassInRegisters())
159
16
      return CGCXXABI::RAA_Indirect;
160
3.81k
    return CGCXXABI::RAA_Default;
161
3.81k
  }
162
14.4k
  return CXXABI.getRecordArgABI(RD);
163
14.4k
}
164
165
static CGCXXABI::RecordArgABI getRecordArgABI(QualType T,
166
10.6k
                                              CGCXXABI &CXXABI) {
167
10.6k
  const RecordType *RT = T->getAs<RecordType>();
168
10.6k
  if (!RT)
169
5.95k
    return CGCXXABI::RAA_Default;
170
4.68k
  return getRecordArgABI(RT, CXXABI);
171
4.68k
}
172
173
static bool classifyReturnType(const CGCXXABI &CXXABI, CGFunctionInfo &FI,
174
179k
                               const ABIInfo &Info) {
175
179k
  QualType Ty = FI.getReturnType();
176
177
179k
  if (const auto *RT = Ty->getAs<RecordType>())
178
5.29k
    if (!isa<CXXRecordDecl>(RT->getDecl()) &&
179
1.26k
        !RT->getDecl()->canPassInRegisters()) {
180
12
      FI.getReturnInfo() = Info.getNaturalAlignIndirect(Ty);
181
12
      return true;
182
12
    }
183
184
179k
  return CXXABI.classifyReturnType(FI);
185
179k
}
186
187
/// Pass transparent unions as if they were the type of the first element. Sema
188
/// should ensure that all elements of the union have the same "machine type".
189
359k
static QualType useFirstFieldIfTransparentUnion(QualType Ty) {
190
359k
  if (const RecordType *UT = Ty->getAsUnionType()) {
191
129
    const RecordDecl *UD = UT->getDecl();
192
129
    if (UD->hasAttr<TransparentUnionAttr>()) {
193
22
      assert(!UD->field_empty() && "sema created an empty transparent union");
194
22
      return UD->field_begin()->getType();
195
22
    }
196
359k
  }
197
359k
  return Ty;
198
359k
}
199
200
231k
CGCXXABI &ABIInfo::getCXXABI() const {
201
231k
  return CGT.getCXXABI();
202
231k
}
203
204
407k
ASTContext &ABIInfo::getContext() const {
205
407k
  return CGT.getContext();
206
407k
}
207
208
17.6k
llvm::LLVMContext &ABIInfo::getVMContext() const {
209
17.6k
  return CGT.getLLVMContext();
210
17.6k
}
211
212
18.2k
const llvm::DataLayout &ABIInfo::getDataLayout() const {
213
18.2k
  return CGT.getDataLayout();
214
18.2k
}
215
216
32.4k
const TargetInfo &ABIInfo::getTarget() const {
217
32.4k
  return CGT.getTarget();
218
32.4k
}
219
220
0
const CodeGenOptions &ABIInfo::getCodeGenOpts() const {
221
0
  return CGT.getCodeGenOpts();
222
0
}
223
224
5.03k
bool ABIInfo::isAndroid() const { return getTarget().getTriple().isAndroid(); }
225
226
0
bool ABIInfo::isHomogeneousAggregateBaseType(QualType Ty) const {
227
0
  return false;
228
0
}
229
230
bool ABIInfo::isHomogeneousAggregateSmallEnough(const Type *Base,
231
0
                                                uint64_t Members) const {
232
0
  return false;
233
0
}
234
235
0
LLVM_DUMP_METHOD void ABIArgInfo::dump() const {
236
0
  raw_ostream &OS = llvm::errs();
237
0
  OS << "(ABIArgInfo Kind=";
238
0
  switch (TheKind) {
239
0
  case Direct:
240
0
    OS << "Direct Type=";
241
0
    if (llvm::Type *Ty = getCoerceToType())
242
0
      Ty->print(OS);
243
0
    else
244
0
      OS << "null";
245
0
    break;
246
0
  case Extend:
247
0
    OS << "Extend";
248
0
    break;
249
0
  case Ignore:
250
0
    OS << "Ignore";
251
0
    break;
252
0
  case InAlloca:
253
0
    OS << "InAlloca Offset=" << getInAllocaFieldIndex();
254
0
    break;
255
0
  case Indirect:
256
0
    OS << "Indirect Align=" << getIndirectAlign().getQuantity()
257
0
       << " ByVal=" << getIndirectByVal()
258
0
       << " Realign=" << getIndirectRealign();
259
0
    break;
260
0
  case IndirectAliased:
261
0
    OS << "Indirect Align=" << getIndirectAlign().getQuantity()
262
0
       << " AadrSpace=" << getIndirectAddrSpace()
263
0
       << " Realign=" << getIndirectRealign();
264
0
    break;
265
0
  case Expand:
266
0
    OS << "Expand";
267
0
    break;
268
0
  case CoerceAndExpand:
269
0
    OS << "CoerceAndExpand Type=";
270
0
    getCoerceAndExpandType()->print(OS);
271
0
    break;
272
0
  }
273
0
  OS << ")\n";
274
0
}
275
276
// Dynamically round a pointer up to a multiple of the given alignment.
277
static llvm::Value *emitRoundPointerUpToAlignment(CodeGenFunction &CGF,
278
                                                  llvm::Value *Ptr,
279
102
                                                  CharUnits Align) {
280
102
  llvm::Value *PtrAsInt = Ptr;
281
  // OverflowArgArea = (OverflowArgArea + Align - 1) & -Align;
282
102
  PtrAsInt = CGF.Builder.CreatePtrToInt(PtrAsInt, CGF.IntPtrTy);
283
102
  PtrAsInt = CGF.Builder.CreateAdd(PtrAsInt,
284
102
        llvm::ConstantInt::get(CGF.IntPtrTy, Align.getQuantity() - 1));
285
102
  PtrAsInt = CGF.Builder.CreateAnd(PtrAsInt,
286
102
           llvm::ConstantInt::get(CGF.IntPtrTy, -Align.getQuantity()));
287
102
  PtrAsInt = CGF.Builder.CreateIntToPtr(PtrAsInt,
288
102
                                        Ptr->getType(),
289
102
                                        Ptr->getName() + ".aligned");
290
102
  return PtrAsInt;
291
102
}
292
293
/// Emit va_arg for a platform using the common void* representation,
294
/// where arguments are simply emitted in an array of slots on the stack.
295
///
296
/// This version implements the core direct-value passing rules.
297
///
298
/// \param SlotSize - The size and alignment of a stack slot.
299
///   Each argument will be allocated to a multiple of this number of
300
///   slots, and all the slots will be aligned to this value.
301
/// \param AllowHigherAlign - The slot alignment is not a cap;
302
///   an argument type with an alignment greater than the slot size
303
///   will be emitted on a higher-alignment address, potentially
304
///   leaving one or more empty slots behind as padding.  If this
305
///   is false, the returned address might be less-aligned than
306
///   DirectAlign.
307
static Address emitVoidPtrDirectVAArg(CodeGenFunction &CGF,
308
                                      Address VAListAddr,
309
                                      llvm::Type *DirectTy,
310
                                      CharUnits DirectSize,
311
                                      CharUnits DirectAlign,
312
                                      CharUnits SlotSize,
313
276
                                      bool AllowHigherAlign) {
314
  // Cast the element type to i8* if necessary.  Some platforms define
315
  // va_list as a struct containing an i8* instead of just an i8*.
316
276
  if (VAListAddr.getElementType() != CGF.Int8PtrTy)
317
54
    VAListAddr = CGF.Builder.CreateElementBitCast(VAListAddr, CGF.Int8PtrTy);
318
319
276
  llvm::Value *Ptr = CGF.Builder.CreateLoad(VAListAddr, "argp.cur");
320
321
  // If the CC aligns values higher than the slot size, do so if needed.
322
276
  Address Addr = Address::invalid();
323
276
  if (AllowHigherAlign && 
DirectAlign > SlotSize245
) {
324
82
    Addr = Address(emitRoundPointerUpToAlignment(CGF, Ptr, DirectAlign),
325
82
                                                 DirectAlign);
326
194
  } else {
327
194
    Addr = Address(Ptr, SlotSize);
328
194
  }
329
330
  // Advance the pointer past the argument, then store that back.
331
276
  CharUnits FullDirectSize = DirectSize.alignTo(SlotSize);
332
276
  Address NextPtr =
333
276
      CGF.Builder.CreateConstInBoundsByteGEP(Addr, FullDirectSize, "argp.next");
334
276
  CGF.Builder.CreateStore(NextPtr.getPointer(), VAListAddr);
335
336
  // If the argument is smaller than a slot, and this is a big-endian
337
  // target, the argument will be right-adjusted in its slot.
338
276
  if (DirectSize < SlotSize && 
CGF.CGM.getDataLayout().isBigEndian()37
&&
339
8
      !DirectTy->isStructTy()) {
340
2
    Addr = CGF.Builder.CreateConstInBoundsByteGEP(Addr, SlotSize - DirectSize);
341
2
  }
342
343
276
  Addr = CGF.Builder.CreateElementBitCast(Addr, DirectTy);
344
276
  return Addr;
345
276
}
346
347
/// Emit va_arg for a platform using the common void* representation,
348
/// where arguments are simply emitted in an array of slots on the stack.
349
///
350
/// \param IsIndirect - Values of this type are passed indirectly.
351
/// \param ValueInfo - The size and alignment of this type, generally
352
///   computed with getContext().getTypeInfoInChars(ValueTy).
353
/// \param SlotSizeAndAlign - The size and alignment of a stack slot.
354
///   Each argument will be allocated to a multiple of this number of
355
///   slots, and all the slots will be aligned to this value.
356
/// \param AllowHigherAlign - The slot alignment is not a cap;
357
///   an argument type with an alignment greater than the slot size
358
///   will be emitted on a higher-alignment address, potentially
359
///   leaving one or more empty slots behind as padding.
360
static Address emitVoidPtrVAArg(CodeGenFunction &CGF, Address VAListAddr,
361
                                QualType ValueTy, bool IsIndirect,
362
                                TypeInfoChars ValueInfo,
363
                                CharUnits SlotSizeAndAlign,
364
268
                                bool AllowHigherAlign) {
365
  // The size and alignment of the value that was passed directly.
366
268
  CharUnits DirectSize, DirectAlign;
367
268
  if (IsIndirect) {
368
31
    DirectSize = CGF.getPointerSize();
369
31
    DirectAlign = CGF.getPointerAlign();
370
237
  } else {
371
237
    DirectSize = ValueInfo.Width;
372
237
    DirectAlign = ValueInfo.Align;
373
237
  }
374
375
  // Cast the address we've calculated to the right type.
376
268
  llvm::Type *DirectTy = CGF.ConvertTypeForMem(ValueTy);
377
268
  if (IsIndirect)
378
31
    DirectTy = DirectTy->getPointerTo(0);
379
380
268
  Address Addr = emitVoidPtrDirectVAArg(CGF, VAListAddr, DirectTy,
381
268
                                        DirectSize, DirectAlign,
382
268
                                        SlotSizeAndAlign,
383
268
                                        AllowHigherAlign);
384
385
268
  if (IsIndirect) {
386
31
    Addr = Address(CGF.Builder.CreateLoad(Addr), ValueInfo.Align);
387
31
  }
388
389
268
  return Addr;
390
391
268
}
392
393
static Address emitMergePHI(CodeGenFunction &CGF,
394
                            Address Addr1, llvm::BasicBlock *Block1,
395
                            Address Addr2, llvm::BasicBlock *Block2,
396
349
                            const llvm::Twine &Name = "") {
397
349
  assert(Addr1.getType() == Addr2.getType());
398
349
  llvm::PHINode *PHI = CGF.Builder.CreatePHI(Addr1.getType(), 2, Name);
399
349
  PHI->addIncoming(Addr1.getPointer(), Block1);
400
349
  PHI->addIncoming(Addr2.getPointer(), Block2);
401
349
  CharUnits Align = std::min(Addr1.getAlignment(), Addr2.getAlignment());
402
349
  return Address(PHI, Align);
403
349
}
404
405
31.3k
TargetCodeGenInfo::~TargetCodeGenInfo() = default;
406
407
// If someone can figure out a general rule for this, that would be great.
408
// It's probably just doomed to be platform-dependent, though.
409
5
unsigned TargetCodeGenInfo::getSizeOfUnwindException() const {
410
  // Verified for:
411
  //   x86-64     FreeBSD, Linux, Darwin
412
  //   x86-32     FreeBSD, Linux, Darwin
413
  //   PowerPC    Linux, Darwin
414
  //   ARM        Darwin (*not* EABI)
415
  //   AArch64    Linux
416
5
  return 32;
417
5
}
418
419
bool TargetCodeGenInfo::isNoProtoCallVariadic(const CallArgList &args,
420
187
                                     const FunctionNoProtoType *fnType) const {
421
  // The following conventions are known to require this to be false:
422
  //   x86_stdcall
423
  //   MIPS
424
  // For everything else, we just prefer false unless we opt out.
425
187
  return false;
426
187
}
427
428
void
429
TargetCodeGenInfo::getDependentLibraryOption(llvm::StringRef Lib,
430
2
                                             llvm::SmallString<24> &Opt) const {
431
  // This assumes the user is passing a library name like "rt" instead of a
432
  // filename like "librt.a/so", and that they don't care whether it's static or
433
  // dynamic.
434
2
  Opt = "-l";
435
2
  Opt += Lib;
436
2
}
437
438
44
unsigned TargetCodeGenInfo::getOpenCLKernelCallingConv() const {
439
  // OpenCL kernels are called via an explicit runtime API with arguments
440
  // set with clSetKernelArg(), not as normal sub-functions.
441
  // Return SPIR_KERNEL by default as the kernel calling convention to
442
  // ensure the fingerprint is fixed such way that each OpenCL argument
443
  // gets one matching argument in the produced kernel function argument
444
  // list to enable feasible implementation of clSetKernelArg() with
445
  // aggregates etc. In case we would use the default C calling conv here,
446
  // clSetKernelArg() might break depending on the target-specific
447
  // conventions; different targets might split structs passed as values
448
  // to multiple function arguments etc.
449
44
  return llvm::CallingConv::SPIR_KERNEL;
450
44
}
451
452
llvm::Constant *TargetCodeGenInfo::getNullPointer(const CodeGen::CodeGenModule &CGM,
453
29.7k
    llvm::PointerType *T, QualType QT) const {
454
29.7k
  return llvm::ConstantPointerNull::get(T);
455
29.7k
}
456
457
LangAS TargetCodeGenInfo::getGlobalVarAddressSpace(CodeGenModule &CGM,
458
67.6k
                                                   const VarDecl *D) const {
459
67.6k
  assert(!CGM.getLangOpts().OpenCL &&
460
67.6k
         !(CGM.getLangOpts().CUDA && CGM.getLangOpts().CUDAIsDevice) &&
461
67.6k
         "Address space agnostic languages only");
462
65.7k
  return D ? D->getType().getAddressSpace() : 
LangAS::Default1.94k
;
463
67.6k
}
464
465
llvm::Value *TargetCodeGenInfo::performAddrSpaceCast(
466
    CodeGen::CodeGenFunction &CGF, llvm::Value *Src, LangAS SrcAddr,
467
4.71k
    LangAS DestAddr, llvm::Type *DestTy, bool isNonNull) const {
468
  // Since target may map different address spaces in AST to the same address
469
  // space, an address space conversion may end up as a bitcast.
470
4.71k
  if (auto *C = dyn_cast<llvm::Constant>(Src))
471
47
    return performAddrSpaceCast(CGF.CGM, C, SrcAddr, DestAddr, DestTy);
472
  // Try to preserve the source's name to make IR more readable.
473
4.66k
  return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
474
4.52k
      Src, DestTy, Src->hasName() ? Src->getName() + ".ascast" : 
""149
);
475
4.66k
}
476
477
llvm::Constant *
478
TargetCodeGenInfo::performAddrSpaceCast(CodeGenModule &CGM, llvm::Constant *Src,
479
                                        LangAS SrcAddr, LangAS DestAddr,
480
370
                                        llvm::Type *DestTy) const {
481
  // Since target may map different address spaces in AST to the same address
482
  // space, an address space conversion may end up as a bitcast.
483
370
  return llvm::ConstantExpr::getPointerCast(Src, DestTy);
484
370
}
485
486
llvm::SyncScope::ID
487
TargetCodeGenInfo::getLLVMSyncScopeID(const LangOptions &LangOpts,
488
                                      SyncScope Scope,
489
                                      llvm::AtomicOrdering Ordering,
490
0
                                      llvm::LLVMContext &Ctx) const {
491
0
  return Ctx.getOrInsertSyncScopeID(""); /* default sync scope */
492
0
}
493
494
static bool isEmptyRecord(ASTContext &Context, QualType T, bool AllowArrays);
495
496
/// isEmptyField - Return true iff a the field is "empty", that is it
497
/// is an unnamed bit-field or an (array of) empty record(s).
498
static bool isEmptyField(ASTContext &Context, const FieldDecl *FD,
499
5.44k
                         bool AllowArrays) {
500
5.44k
  if (FD->isUnnamedBitfield())
501
120
    return true;
502
503
5.32k
  QualType FT = FD->getType();
504
505
  // Constant arrays of empty records count as empty, strip them off.
506
  // Constant arrays of zero length always count as empty.
507
5.32k
  bool WasArray = false;
508
5.32k
  if (AllowArrays)
509
6.98k
    
while (const ConstantArrayType *5.15k
AT = Context.getAsConstantArrayType(FT)) {
510
1.83k
      if (AT->getSize() == 0)
511
11
        return true;
512
1.82k
      FT = AT->getElementType();
513
      // The [[no_unique_address]] special case below does not apply to
514
      // arrays of C++ empty records, so we need to remember this fact.
515
1.82k
      WasArray = true;
516
1.82k
    }
517
518
5.31k
  const RecordType *RT = FT->getAs<RecordType>();
519
5.31k
  if (!RT)
520
4.88k
    return false;
521
522
  // C++ record fields are never empty, at least in the Itanium ABI.
523
  //
524
  // FIXME: We should use a predicate for whether this behavior is true in the
525
  // current ABI.
526
  //
527
  // The exception to the above rule are fields marked with the
528
  // [[no_unique_address]] attribute (since C++20).  Those do count as empty
529
  // according to the Itanium ABI.  The exception applies only to records,
530
  // not arrays of records, so we must also check whether we stripped off an
531
  // array type above.
532
433
  if (isa<CXXRecordDecl>(RT->getDecl()) &&
533
85
      (WasArray || 
!FD->hasAttr<NoUniqueAddressAttr>()68
))
534
78
    return false;
535
536
355
  return isEmptyRecord(Context, FT, AllowArrays);
537
355
}
538
539
/// isEmptyRecord - Return true iff a structure contains only empty
540
/// fields. Note that a structure with a flexible array member is not
541
/// considered empty.
542
9.00k
static bool isEmptyRecord(ASTContext &Context, QualType T, bool AllowArrays) {
543
9.00k
  const RecordType *RT = T->getAs<RecordType>();
544
9.00k
  if (!RT)
545
4.35k
    return false;
546
4.65k
  const RecordDecl *RD = RT->getDecl();
547
4.65k
  if (RD->hasFlexibleArrayMember())
548
14
    return false;
549
550
  // If this is a C++ record, check the bases first.
551
4.64k
  if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD))
552
1.01k
    for (const auto &I : CXXRD->bases())
553
115
      if (!isEmptyRecord(Context, I.getType(), true))
554
107
        return false;
555
556
4.53k
  for (const auto *I : RD->fields())
557
4.18k
    if (!isEmptyField(Context, I, AllowArrays))
558
3.91k
      return false;
559
618
  return true;
560
4.53k
}
561
562
/// isSingleElementStruct - Determine if a structure is a "single
563
/// element struct", i.e. it has exactly one non-empty field or
564
/// exactly one field which is itself a single element
565
/// struct. Structures with flexible array members are never
566
/// considered single element structs.
567
///
568
/// \return The field declaration for the single non-empty field, if
569
/// it exists.
570
69.8k
static const Type *isSingleElementStruct(QualType T, ASTContext &Context) {
571
69.8k
  const RecordType *RT = T->getAs<RecordType>();
572
69.8k
  if (!RT)
573
69.0k
    return nullptr;
574
575
836
  const RecordDecl *RD = RT->getDecl();
576
836
  if (RD->hasFlexibleArrayMember())
577
4
    return nullptr;
578
579
832
  const Type *Found = nullptr;
580
581
  // If this is a C++ record, check the bases first.
582
832
  if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
583
49
    for (const auto &I : CXXRD->bases()) {
584
      // Ignore empty records.
585
49
      if (isEmptyRecord(Context, I.getType(), true))
586
6
        continue;
587
588
      // If we already found an element then this isn't a single-element struct.
589
43
      if (Found)
590
11
        return nullptr;
591
592
      // If this is non-empty and not a single element struct, the composite
593
      // cannot be a single element struct.
594
32
      Found = isSingleElementStruct(I.getType(), Context);
595
32
      if (!Found)
596
9
        return nullptr;
597
32
    }
598
270
  }
599
600
  // Check for single element.
601
1.14k
  
for (const auto *FD : RD->fields())812
{
602
1.14k
    QualType FT = FD->getType();
603
604
    // Ignore empty fields.
605
1.14k
    if (isEmptyField(Context, FD, true))
606
19
      continue;
607
608
    // If we already found an element then this isn't a single-element
609
    // struct.
610
1.12k
    if (Found)
611
345
      return nullptr;
612
613
    // Treat single element arrays as the element.
614
792
    
while (const ConstantArrayType *778
AT = Context.getAsConstantArrayType(FT)) {
615
176
      if (AT->getSize().getZExtValue() != 1)
616
162
        break;
617
14
      FT = AT->getElementType();
618
14
    }
619
620
778
    if (!isAggregateTypeForABI(FT)) {
621
586
      Found = FT.getTypePtr();
622
192
    } else {
623
192
      Found = isSingleElementStruct(FT, Context);
624
192
      if (!Found)
625
182
        return nullptr;
626
192
    }
627
778
  }
628
629
  // We don't consider a struct a single-element struct if it has
630
  // padding beyond the element type.
631
285
  if (Found && 
Context.getTypeSize(Found) != Context.getTypeSize(T)263
)
632
3
    return nullptr;
633
634
282
  return Found;
635
282
}
636
637
namespace {
638
Address EmitVAArgInstr(CodeGenFunction &CGF, Address VAListAddr, QualType Ty,
639
9
                       const ABIArgInfo &AI) {
640
  // This default implementation defers to the llvm backend's va_arg
641
  // instruction. It can handle only passing arguments directly
642
  // (typically only handled in the backend for primitive types), or
643
  // aggregates passed indirectly by pointer (NOTE: if the "byval"
644
  // flag has ABI impact in the callee, this implementation cannot
645
  // work.)
646
647
  // Only a few cases are covered here at the moment -- those needed
648
  // by the default abi.
649
9
  llvm::Value *Val;
650
651
9
  if (AI.isIndirect()) {
652
1
    assert(!AI.getPaddingType() &&
653
1
           "Unexpected PaddingType seen in arginfo in generic VAArg emitter!");
654
1
    assert(
655
1
        !AI.getIndirectRealign() &&
656
1
        "Unexpected IndirectRealign seen in arginfo in generic VAArg emitter!");
657
658
1
    auto TyInfo = CGF.getContext().getTypeInfoInChars(Ty);
659
1
    CharUnits TyAlignForABI = TyInfo.Align;
660
661
1
    llvm::Type *BaseTy =
662
1
        llvm::PointerType::getUnqual(CGF.ConvertTypeForMem(Ty));
663
1
    llvm::Value *Addr =
664
1
        CGF.Builder.CreateVAArg(VAListAddr.getPointer(), BaseTy);
665
1
    return Address(Addr, TyAlignForABI);
666
8
  } else {
667
8
    assert((AI.isDirect() || AI.isExtend()) &&
668
8
           "Unexpected ArgInfo Kind in generic VAArg emitter!");
669
670
8
    assert(!AI.getInReg() &&
671
8
           "Unexpected InReg seen in arginfo in generic VAArg emitter!");
672
8
    assert(!AI.getPaddingType() &&
673
8
           "Unexpected PaddingType seen in arginfo in generic VAArg emitter!");
674
8
    assert(!AI.getDirectOffset() &&
675
8
           "Unexpected DirectOffset seen in arginfo in generic VAArg emitter!");
676
8
    assert(!AI.getCoerceToType() &&
677
8
           "Unexpected CoerceToType seen in arginfo in generic VAArg emitter!");
678
679
8
    Address Temp = CGF.CreateMemTemp(Ty, "varet");
680
8
    Val = CGF.Builder.CreateVAArg(VAListAddr.getPointer(), CGF.ConvertType(Ty));
681
8
    CGF.Builder.CreateStore(Val, Temp);
682
8
    return Temp;
683
8
  }
684
9
}
685
686
/// DefaultABIInfo - The default implementation for ABI specific
687
/// details. This implementation provides information which results in
688
/// self-consistent and sensible LLVM IR generation, but does not
689
/// conform to any particular ABI.
690
class DefaultABIInfo : public ABIInfo {
691
public:
692
727
  DefaultABIInfo(CodeGen::CodeGenTypes &CGT) : ABIInfo(CGT) {}
693
694
  ABIArgInfo classifyReturnType(QualType RetTy) const;
695
  ABIArgInfo classifyArgumentType(QualType RetTy) const;
696
697
533
  void computeInfo(CGFunctionInfo &FI) const override {
698
533
    if (!getCXXABI().classifyReturnType(FI))
699
532
      FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
700
533
    for (auto &I : FI.arguments())
701
698
      I.info = classifyArgumentType(I.type);
702
533
  }
703
704
  Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
705
3
                    QualType Ty) const override {
706
3
    return EmitVAArgInstr(CGF, VAListAddr, Ty, classifyArgumentType(Ty));
707
3
  }
708
};
709
710
class DefaultTargetCodeGenInfo : public TargetCodeGenInfo {
711
public:
712
  DefaultTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT)
713
28
      : TargetCodeGenInfo(std::make_unique<DefaultABIInfo>(CGT)) {}
714
};
715
716
3.65k
ABIArgInfo DefaultABIInfo::classifyArgumentType(QualType Ty) const {
717
3.65k
  Ty = useFirstFieldIfTransparentUnion(Ty);
718
719
3.65k
  if (isAggregateTypeForABI(Ty)) {
720
    // Records with non-trivial destructors/copy-constructors should not be
721
    // passed by value.
722
123
    if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))
723
2
      return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
724
725
121
    return getNaturalAlignIndirect(Ty);
726
121
  }
727
728
  // Treat an enum type as its underlying type.
729
3.53k
  if (const EnumType *EnumTy = Ty->getAs<EnumType>())
730
9
    Ty = EnumTy->getDecl()->getIntegerType();
731
732
3.53k
  ASTContext &Context = getContext();
733
3.53k
  if (const auto *EIT = Ty->getAs<ExtIntType>())
734
64
    if (EIT->getNumBits() >
735
64
        Context.getTypeSize(Context.getTargetInfo().hasInt128Type()
736
16
                                ? Context.Int128Ty
737
48
                                : Context.LongLongTy))
738
28
      return getNaturalAlignIndirect(Ty);
739
740
3.50k
  return (isPromotableIntegerTypeForABI(Ty) ? 
ABIArgInfo::getExtend(Ty)142
741
3.36k
                                            : ABIArgInfo::getDirect());
742
3.50k
}
743
744
2.60k
ABIArgInfo DefaultABIInfo::classifyReturnType(QualType RetTy) const {
745
2.60k
  if (RetTy->isVoidType())
746
1.23k
    return ABIArgInfo::getIgnore();
747
748
1.36k
  if (isAggregateTypeForABI(RetTy))
749
116
    return getNaturalAlignIndirect(RetTy);
750
751
  // Treat an enum type as its underlying type.
752
1.25k
  if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
753
0
    RetTy = EnumTy->getDecl()->getIntegerType();
754
755
1.25k
  if (const auto *EIT = RetTy->getAs<ExtIntType>())
756
67
    if (EIT->getNumBits() >
757
67
        getContext().getTypeSize(getContext().getTargetInfo().hasInt128Type()
758
14
                                     ? getContext().Int128Ty
759
53
                                     : getContext().LongLongTy))
760
43
      return getNaturalAlignIndirect(RetTy);
761
762
1.20k
  return (isPromotableIntegerTypeForABI(RetTy) ? 
ABIArgInfo::getExtend(RetTy)79
763
1.12k
                                               : ABIArgInfo::getDirect());
764
1.20k
}
765
766
//===----------------------------------------------------------------------===//
767
// WebAssembly ABI Implementation
768
//
769
// This is a very simple ABI that relies a lot on DefaultABIInfo.
770
//===----------------------------------------------------------------------===//
771
772
class WebAssemblyABIInfo final : public SwiftABIInfo {
773
public:
774
  enum ABIKind {
775
    MVP = 0,
776
    ExperimentalMV = 1,
777
  };
778
779
private:
780
  DefaultABIInfo defaultInfo;
781
  ABIKind Kind;
782
783
public:
784
  explicit WebAssemblyABIInfo(CodeGen::CodeGenTypes &CGT, ABIKind Kind)
785
42
      : SwiftABIInfo(CGT), defaultInfo(CGT), Kind(Kind) {}
786
787
private:
788
  ABIArgInfo classifyReturnType(QualType RetTy) const;
789
  ABIArgInfo classifyArgumentType(QualType Ty) const;
790
791
  // DefaultABIInfo's classifyReturnType and classifyArgumentType are
792
  // non-virtual, but computeInfo and EmitVAArg are virtual, so we
793
  // overload them.
794
409
  void computeInfo(CGFunctionInfo &FI) const override {
795
409
    if (!getCXXABI().classifyReturnType(FI))
796
401
      FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
797
409
    for (auto &Arg : FI.arguments())
798
432
      Arg.info = classifyArgumentType(Arg.type);
799
409
  }
800
801
  Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
802
                    QualType Ty) const override;
803
804
  bool shouldPassIndirectlyForSwift(ArrayRef<llvm::Type*> scalars,
805
0
                                    bool asReturnValue) const override {
806
0
    return occupiesMoreThan(CGT, scalars, /*total*/ 4);
807
0
  }
808
809
0
  bool isSwiftErrorInRegister() const override {
810
0
    return false;
811
0
  }
812
};
813
814
class WebAssemblyTargetCodeGenInfo final : public TargetCodeGenInfo {
815
public:
816
  explicit WebAssemblyTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT,
817
                                        WebAssemblyABIInfo::ABIKind K)
818
42
      : TargetCodeGenInfo(std::make_unique<WebAssemblyABIInfo>(CGT, K)) {}
819
820
  void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
821
1.48k
                           CodeGen::CodeGenModule &CGM) const override {
822
1.48k
    TargetCodeGenInfo::setTargetAttributes(D, GV, CGM);
823
1.48k
    if (const auto *FD = dyn_cast_or_null<FunctionDecl>(D)) {
824
1.33k
      if (const auto *Attr = FD->getAttr<WebAssemblyImportModuleAttr>()) {
825
1
        llvm::Function *Fn = cast<llvm::Function>(GV);
826
1
        llvm::AttrBuilder B;
827
1
        B.addAttribute("wasm-import-module", Attr->getImportModule());
828
1
        Fn->addAttributes(llvm::AttributeList::FunctionIndex, B);
829
1
      }
830
1.33k
      if (const auto *Attr = FD->getAttr<WebAssemblyImportNameAttr>()) {
831
1
        llvm::Function *Fn = cast<llvm::Function>(GV);
832
1
        llvm::AttrBuilder B;
833
1
        B.addAttribute("wasm-import-name", Attr->getImportName());
834
1
        Fn->addAttributes(llvm::AttributeList::FunctionIndex, B);
835
1
      }
836
1.33k
      if (const auto *Attr = FD->getAttr<WebAssemblyExportNameAttr>()) {
837
2
        llvm::Function *Fn = cast<llvm::Function>(GV);
838
2
        llvm::AttrBuilder B;
839
2
        B.addAttribute("wasm-export-name", Attr->getExportName());
840
2
        Fn->addAttributes(llvm::AttributeList::FunctionIndex, B);
841
2
      }
842
1.33k
    }
843
844
1.48k
    if (auto *FD = dyn_cast_or_null<FunctionDecl>(D)) {
845
1.33k
      llvm::Function *Fn = cast<llvm::Function>(GV);
846
1.33k
      if (!FD->doesThisDeclarationHaveABody() && 
!FD->hasPrototype()73
)
847
1
        Fn->addFnAttr("no-prototype");
848
1.33k
    }
849
1.48k
  }
850
};
851
852
/// Classify argument of given type \p Ty.
853
432
ABIArgInfo WebAssemblyABIInfo::classifyArgumentType(QualType Ty) const {
854
432
  Ty = useFirstFieldIfTransparentUnion(Ty);
855
856
432
  if (isAggregateTypeForABI(Ty)) {
857
    // Records with non-trivial destructors/copy-constructors should not be
858
    // passed by value.
859
42
    if (auto RAA = getRecordArgABI(Ty, getCXXABI()))
860
8
      return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
861
    // Ignore empty structs/unions.
862
34
    if (isEmptyRecord(getContext(), Ty, true))
863
4
      return ABIArgInfo::getIgnore();
864
    // Lower single-element structs to just pass a regular value. TODO: We
865
    // could do reasonable-size multiple-element structs too, using getExpand(),
866
    // though watch out for things like bitfields.
867
30
    if (const Type *SeltTy = isSingleElementStruct(Ty, getContext()))
868
11
      return ABIArgInfo::getDirect(CGT.ConvertType(QualType(SeltTy, 0)));
869
    // For the experimental multivalue ABI, fully expand all other aggregates
870
19
    if (Kind == ABIKind::ExperimentalMV) {
871
3
      const RecordType *RT = Ty->getAs<RecordType>();
872
3
      assert(RT);
873
3
      bool HasBitField = false;
874
5
      for (auto *Field : RT->getDecl()->fields()) {
875
5
        if (Field->isBitField()) {
876
1
          HasBitField = true;
877
1
          break;
878
1
        }
879
5
      }
880
3
      if (!HasBitField)
881
2
        return ABIArgInfo::getExpand();
882
407
    }
883
19
  }
884
885
  // Otherwise just do the default thing.
886
407
  return defaultInfo.classifyArgumentType(Ty);
887
407
}
888
889
401
ABIArgInfo WebAssemblyABIInfo::classifyReturnType(QualType RetTy) const {
890
401
  if (isAggregateTypeForABI(RetTy)) {
891
    // Records with non-trivial destructors/copy-constructors should not be
892
    // returned by value.
893
30
    if (!getRecordArgABI(RetTy, getCXXABI())) {
894
      // Ignore empty structs/unions.
895
30
      if (isEmptyRecord(getContext(), RetTy, true))
896
4
        return ABIArgInfo::getIgnore();
897
      // Lower single-element structs to just return a regular value. TODO: We
898
      // could do reasonable-size multiple-element structs too, using
899
      // ABIArgInfo::getDirect().
900
26
      if (const Type *SeltTy = isSingleElementStruct(RetTy, getContext()))
901
11
        return ABIArgInfo::getDirect(CGT.ConvertType(QualType(SeltTy, 0)));
902
      // For the experimental multivalue ABI, return all other aggregates
903
15
      if (Kind == ABIKind::ExperimentalMV)
904
3
        return ABIArgInfo::getDirect();
905
383
    }
906
30
  }
907
908
  // Otherwise just do the default thing.
909
383
  return defaultInfo.classifyReturnType(RetTy);
910
383
}
911
912
Address WebAssemblyABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
913
5
                                      QualType Ty) const {
914
5
  bool IsIndirect = isAggregateTypeForABI(Ty) &&
915
3
                    !isEmptyRecord(getContext(), Ty, true) &&
916
2
                    !isSingleElementStruct(Ty, getContext());
917
5
  return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect,
918
5
                          getContext().getTypeInfoInChars(Ty),
919
5
                          CharUnits::fromQuantity(4),
920
5
                          /*AllowHigherAlign=*/true);
921
5
}
922
923
//===----------------------------------------------------------------------===//
924
// le32/PNaCl bitcode ABI Implementation
925
//
926
// This is a simplified version of the x86_32 ABI.  Arguments and return values
927
// are always passed on the stack.
928
//===----------------------------------------------------------------------===//
929
930
class PNaClABIInfo : public ABIInfo {
931
 public:
932
17
  PNaClABIInfo(CodeGen::CodeGenTypes &CGT) : ABIInfo(CGT) {}
933
934
  ABIArgInfo classifyReturnType(QualType RetTy) const;
935
  ABIArgInfo classifyArgumentType(QualType RetTy) const;
936
937
  void computeInfo(CGFunctionInfo &FI) const override;
938
  Address EmitVAArg(CodeGenFunction &CGF,
939
                    Address VAListAddr, QualType Ty) const override;
940
};
941
942
class PNaClTargetCodeGenInfo : public TargetCodeGenInfo {
943
 public:
944
   PNaClTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT)
945
17
       : TargetCodeGenInfo(std::make_unique<PNaClABIInfo>(CGT)) {}
946
};
947
948
101
void PNaClABIInfo::computeInfo(CGFunctionInfo &FI) const {
949
101
  if (!getCXXABI().classifyReturnType(FI))
950
101
    FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
951
952
101
  for (auto &I : FI.arguments())
953
70
    I.info = classifyArgumentType(I.type);
954
101
}
955
956
Address PNaClABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
957
3
                                QualType Ty) const {
958
  // The PNaCL ABI is a bit odd, in that varargs don't use normal
959
  // function classification. Structs get passed directly for varargs
960
  // functions, through a rewriting transform in
961
  // pnacl-llvm/lib/Transforms/NaCl/ExpandVarArgs.cpp, which allows
962
  // this target to actually support a va_arg instructions with an
963
  // aggregate type, unlike other targets.
964
3
  return EmitVAArgInstr(CGF, VAListAddr, Ty, ABIArgInfo::getDirect());
965
3
}
966
967
/// Classify argument of given type \p Ty.
968
70
ABIArgInfo PNaClABIInfo::classifyArgumentType(QualType Ty) const {
969
70
  if (isAggregateTypeForABI(Ty)) {
970
9
    if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))
971
0
      return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
972
9
    return getNaturalAlignIndirect(Ty);
973
61
  } else if (const EnumType *EnumTy = Ty->getAs<EnumType>()) {
974
    // Treat an enum type as its underlying type.
975
1
    Ty = EnumTy->getDecl()->getIntegerType();
976
60
  } else if (Ty->isFloatingType()) {
977
    // Floating-point types don't go inreg.
978
19
    return ABIArgInfo::getDirect();
979
41
  } else if (const auto *EIT = Ty->getAs<ExtIntType>()) {
980
    // Treat extended integers as integers if <=64, otherwise pass indirectly.
981
8
    if (EIT->getNumBits() > 64)
982
4
      return getNaturalAlignIndirect(Ty);
983
4
    return ABIArgInfo::getDirect();
984
4
  }
985
986
34
  return (isPromotableIntegerTypeForABI(Ty) ? 
ABIArgInfo::getExtend(Ty)5
987
29
                                            : ABIArgInfo::getDirect());
988
34
}
989
990
101
ABIArgInfo PNaClABIInfo::classifyReturnType(QualType RetTy) const {
991
101
  if (RetTy->isVoidType())
992
38
    return ABIArgInfo::getIgnore();
993
994
  // In the PNaCl ABI we always return records/structures on the stack.
995
63
  if (isAggregateTypeForABI(RetTy))
996
2
    return getNaturalAlignIndirect(RetTy);
997
998
  // Treat extended integers as integers if <=64, otherwise pass indirectly.
999
61
  if (const auto *EIT = RetTy->getAs<ExtIntType>()) {
1000
7
    if (EIT->getNumBits() > 64)
1001
5
      return getNaturalAlignIndirect(RetTy);
1002
2
    return ABIArgInfo::getDirect();
1003
2
  }
1004
1005
  // Treat an enum type as its underlying type.
1006
54
  if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
1007
0
    RetTy = EnumTy->getDecl()->getIntegerType();
1008
1009
18
  return (isPromotableIntegerTypeForABI(RetTy) ? ABIArgInfo::getExtend(RetTy)
1010
36
                                               : ABIArgInfo::getDirect());
1011
54
}
1012
1013
/// IsX86_MMXType - Return true if this is an MMX type.
1014
65
bool IsX86_MMXType(llvm::Type *IRType) {
1015
  // Return true if the type is an MMX type <2 x i32>, <4 x i16>, or <8 x i8>.
1016
65
  return IRType->isVectorTy() && IRType->getPrimitiveSizeInBits() == 64 &&
1017
18
    cast<llvm::VectorType>(IRType)->getElementType()->isIntegerTy() &&
1018
16
    IRType->getScalarSizeInBits() != 64;
1019
65
}
1020
1021
static llvm::Type* X86AdjustInlineAsmType(CodeGen::CodeGenFunction &CGF,
1022
                                          StringRef Constraint,
1023
1.38k
                                          llvm::Type* Ty) {
1024
1.38k
  bool IsMMXCons = llvm::StringSwitch<bool>(Constraint)
1025
1.38k
                     .Cases("y", "&y", "^Ym", true)
1026
1.38k
                     .Default(false);
1027
1.38k
  if (IsMMXCons && 
Ty->isVectorTy()18
) {
1028
14
    if (cast<llvm::VectorType>(Ty)->getPrimitiveSizeInBits().getFixedSize() !=
1029
0
        64) {
1030
      // Invalid MMX constraint
1031
0
      return nullptr;
1032
0
    }
1033
1034
14
    return llvm::Type::getX86_MMXTy(CGF.getLLVMContext());
1035
14
  }
1036
1037
  // No operation needed
1038
1.36k
  return Ty;
1039
1.36k
}
1040
1041
/// Returns true if this type can be passed in SSE registers with the
1042
/// X86_VectorCall calling convention. Shared between x86_32 and x86_64.
1043
771
static bool isX86VectorTypeForVectorCall(ASTContext &Context, QualType Ty) {
1044
771
  if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
1045
475
    if (BT->isFloatingPoint() && 
BT->getKind() != BuiltinType::Half281
) {
1046
281
      if (BT->getKind() == BuiltinType::LongDouble) {
1047
6
        if (&Context.getTargetInfo().getLongDoubleFormat() ==
1048
6
            &llvm::APFloat::x87DoubleExtended())
1049
2
          return false;
1050
279
      }
1051
279
      return true;
1052
279
    }
1053
296
  } else if (const VectorType *VT = Ty->getAs<VectorType>()) {
1054
    // vectorcall can pass XMM, YMM, and ZMM vectors. We don't pass SSE1 MMX
1055
    // registers specially.
1056
263
    unsigned VecSize = Context.getTypeSize(VT);
1057
263
    if (VecSize == 128 || 
VecSize == 2560
||
VecSize == 5120
)
1058
263
      return true;
1059
227
  }
1060
227
  return false;
1061
227
}
1062
1063
/// Returns true if this aggregate is small enough to be passed in SSE registers
1064
/// in the X86_VectorCall calling convention. Shared between x86_32 and x86_64.
1065
730
static bool isX86VectorCallAggregateSmallEnough(uint64_t NumMembers) {
1066
730
  return NumMembers <= 4;
1067
730
}
1068
1069
/// Returns a Homogeneous Vector Aggregate ABIArgInfo, used in X86.
1070
30
static ABIArgInfo getDirectX86Hva(llvm::Type* T = nullptr) {
1071
30
  auto AI = ABIArgInfo::getDirect(T);
1072
30
  AI.setInReg(true);
1073
30
  AI.setCanBeFlattened(false);
1074
30
  return AI;
1075
30
}
1076
1077
//===----------------------------------------------------------------------===//
1078
// X86-32 ABI Implementation
1079
//===----------------------------------------------------------------------===//
1080
1081
/// Similar to llvm::CCState, but for Clang.
1082
struct CCState {
1083
  CCState(CGFunctionInfo &FI)
1084
18.5k
      : IsPreassigned(FI.arg_size()), CC(FI.getCallingConvention()) {}
1085
1086
  llvm::SmallBitVector IsPreassigned;
1087
  unsigned CC = CallingConv::CC_C;
1088
  unsigned FreeRegs = 0;
1089
  unsigned FreeSSERegs = 0;
1090
};
1091
1092
/// X86_32ABIInfo - The X86-32 ABI information.
1093
class X86_32ABIInfo : public SwiftABIInfo {
1094
  enum Class {
1095
    Integer,
1096
    Float
1097
  };
1098
1099
  static const unsigned MinABIStackAlignInBytes = 4;
1100
1101
  bool IsDarwinVectorABI;
1102
  bool IsRetSmallStructInRegABI;
1103
  bool IsWin32StructABI;
1104
  bool IsSoftFloatABI;
1105
  bool IsMCUABI;
1106
  unsigned DefaultNumRegisterParameters;
1107
1108
533
  static bool isRegisterSize(unsigned Size) {
1109
533
    return (Size == 8 || 
Size == 16502
||
Size == 32483
||
Size == 64241
);
1110
533
  }
1111
1112
400
  bool isHomogeneousAggregateBaseType(QualType Ty) const override {
1113
    // FIXME: Assumes vectorcall is in use.
1114
400
    return isX86VectorTypeForVectorCall(getContext(), Ty);
1115
400
  }
1116
1117
  bool isHomogeneousAggregateSmallEnough(const Type *Ty,
1118
322
                                         uint64_t NumMembers) const override {
1119
    // FIXME: Assumes vectorcall is in use.
1120
322
    return isX86VectorCallAggregateSmallEnough(NumMembers);
1121
322
  }
1122
1123
  bool shouldReturnTypeInRegister(QualType Ty, ASTContext &Context) const;
1124
1125
  /// getIndirectResult - Give a source type \arg Ty, return a suitable result
1126
  /// such that the argument will be passed in memory.
1127
  ABIArgInfo getIndirectResult(QualType Ty, bool ByVal, CCState &State) const;
1128
1129
  ABIArgInfo getIndirectReturnResult(QualType Ty, CCState &State) const;
1130
1131
  /// Return the alignment to use for the given type on the stack.
1132
  unsigned getTypeStackAlignInBytes(QualType Ty, unsigned Align) const;
1133
1134
  Class classify(QualType Ty) const;
1135
  ABIArgInfo classifyReturnType(QualType RetTy, CCState &State) const;
1136
  ABIArgInfo classifyArgumentType(QualType RetTy, CCState &State) const;
1137
1138
  /// Updates the number of available free registers, returns
1139
  /// true if any registers were allocated.
1140
  bool updateFreeRegs(QualType Ty, CCState &State) const;
1141
1142
  bool shouldAggregateUseDirect(QualType Ty, CCState &State, bool &InReg,
1143
                                bool &NeedsPadding) const;
1144
  bool shouldPrimitiveUseInReg(QualType Ty, CCState &State) const;
1145
1146
  bool canExpandIndirectArgument(QualType Ty) const;
1147
1148
  /// Rewrite the function info so that all memory arguments use
1149
  /// inalloca.
1150
  void rewriteWithInAlloca(CGFunctionInfo &FI) const;
1151
1152
  void addFieldToArgStruct(SmallVector<llvm::Type *, 6> &FrameFields,
1153
                           CharUnits &StackOffset, ABIArgInfo &Info,
1154
                           QualType Type) const;
1155
  void runVectorCallFirstPass(CGFunctionInfo &FI, CCState &State) const;
1156
1157
public:
1158
1159
  void computeInfo(CGFunctionInfo &FI) const override;
1160
  Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
1161
                    QualType Ty) const override;
1162
1163
  X86_32ABIInfo(CodeGen::CodeGenTypes &CGT, bool DarwinVectorABI,
1164
                bool RetSmallStructInRegABI, bool Win32StructABI,
1165
                unsigned NumRegisterParameters, bool SoftFloatABI)
1166
    : SwiftABIInfo(CGT), IsDarwinVectorABI(DarwinVectorABI),
1167
      IsRetSmallStructInRegABI(RetSmallStructInRegABI),
1168
      IsWin32StructABI(Win32StructABI),
1169
      IsSoftFloatABI(SoftFloatABI),
1170
      IsMCUABI(CGT.getTarget().getTriple().isOSIAMCU()),
1171
2.99k
      DefaultNumRegisterParameters(NumRegisterParameters) {}
1172
1173
  bool shouldPassIndirectlyForSwift(ArrayRef<llvm::Type*> scalars,
1174
0
                                    bool asReturnValue) const override {
1175
    // LLVM's x86-32 lowering currently only assigns up to three
1176
    // integer registers and three fp registers.  Oddly, it'll use up to
1177
    // four vector registers for vectors, but those can overlap with the
1178
    // scalar registers.
1179
0
    return occupiesMoreThan(CGT, scalars, /*total*/ 3);
1180
0
  }
1181
1182
0
  bool isSwiftErrorInRegister() const override {
1183
    // x86-32 lowering does not support passing swifterror in a register.
1184
0
    return false;
1185
0
  }
1186
};
1187
1188
class X86_32TargetCodeGenInfo : public TargetCodeGenInfo {
1189
public:
1190
  X86_32TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, bool DarwinVectorABI,
1191
                          bool RetSmallStructInRegABI, bool Win32StructABI,
1192
                          unsigned NumRegisterParameters, bool SoftFloatABI)
1193
      : TargetCodeGenInfo(std::make_unique<X86_32ABIInfo>(
1194
            CGT, DarwinVectorABI, RetSmallStructInRegABI, Win32StructABI,
1195
2.99k
            NumRegisterParameters, SoftFloatABI)) {}
1196
1197
  static bool isStructReturnInRegABI(
1198
      const llvm::Triple &Triple, const CodeGenOptions &Opts);
1199
1200
  void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
1201
                           CodeGen::CodeGenModule &CGM) const override;
1202
1203
0
  int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const override {
1204
    // Darwin uses different dwarf register numbers for EH.
1205
0
    if (CGM.getTarget().getTriple().isOSDarwin()) return 5;
1206
0
    return 4;
1207
0
  }
1208
1209
  bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
1210
                               llvm::Value *Address) const override;
1211
1212
  llvm::Type* adjustInlineAsmType(CodeGen::CodeGenFunction &CGF,
1213
                                  StringRef Constraint,
1214
570
                                  llvm::Type* Ty) const override {
1215
570
    return X86AdjustInlineAsmType(CGF, Constraint, Ty);
1216
570
  }
1217
1218
  void addReturnRegisterOutputs(CodeGenFunction &CGF, LValue ReturnValue,
1219
                                std::string &Constraints,
1220
                                std::vector<llvm::Type *> &ResultRegTypes,
1221
                                std::vector<llvm::Type *> &ResultTruncRegTypes,
1222
                                std::vector<LValue> &ResultRegDests,
1223
                                std::string &AsmString,
1224
                                unsigned NumOutputs) const override;
1225
1226
  llvm::Constant *
1227
37
  getUBSanFunctionSignature(CodeGen::CodeGenModule &CGM) const override {
1228
37
    unsigned Sig = (0xeb << 0) |  // jmp rel8
1229
37
                   (0x06 << 8) |  //           .+0x08
1230
37
                   ('v' << 16) |
1231
37
                   ('2' << 24);
1232
37
    return llvm::ConstantInt::get(CGM.Int32Ty, Sig);
1233
37
  }
1234
1235
7
  StringRef getARCRetainAutoreleasedReturnValueMarker() const override {
1236
7
    return "movl\t%ebp, %ebp"
1237
7
           "\t\t// marker for objc_retainAutoreleaseReturnValue";
1238
7
  }
1239
};
1240
1241
}
1242
1243
/// Rewrite input constraint references after adding some output constraints.
1244
/// In the case where there is one output and one input and we add one output,
1245
/// we need to replace all operand references greater than or equal to 1:
1246
///     mov $0, $1
1247
///     mov eax, $1
1248
/// The result will be:
1249
///     mov $0, $2
1250
///     mov eax, $2
1251
static void rewriteInputConstraintReferences(unsigned FirstIn,
1252
                                             unsigned NumNewOuts,
1253
23
                                             std::string &AsmString) {
1254
23
  std::string Buf;
1255
23
  llvm::raw_string_ostream OS(Buf);
1256
23
  size_t Pos = 0;
1257
75
  while (Pos < AsmString.size()) {
1258
52
    size_t DollarStart = AsmString.find('$', Pos);
1259
52
    if (DollarStart == std::string::npos)
1260
16
      DollarStart = AsmString.size();
1261
52
    size_t DollarEnd = AsmString.find_first_not_of('$', DollarStart);
1262
52
    if (DollarEnd == std::string::npos)
1263
16
      DollarEnd = AsmString.size();
1264
52
    OS << StringRef(&AsmString[Pos], DollarEnd - Pos);
1265
52
    Pos = DollarEnd;
1266
52
    size_t NumDollars = DollarEnd - DollarStart;
1267
52
    if (NumDollars % 2 != 0 && 
Pos < AsmString.size()18
) {
1268
      // We have an operand reference.
1269
18
      size_t DigitStart = Pos;
1270
18
      if (AsmString[DigitStart] == '{') {
1271
3
        OS << '{';
1272
3
        ++DigitStart;
1273
3
      }
1274
18
      size_t DigitEnd = AsmString.find_first_not_of("0123456789", DigitStart);
1275
18
      if (DigitEnd == std::string::npos)
1276
5
        DigitEnd = AsmString.size();
1277
18
      StringRef OperandStr(&AsmString[DigitStart], DigitEnd - DigitStart);
1278
18
      unsigned OperandIndex;
1279
18
      if (!OperandStr.getAsInteger(10, OperandIndex)) {
1280
18
        if (OperandIndex >= FirstIn)
1281
14
          OperandIndex += NumNewOuts;
1282
18
        OS << OperandIndex;
1283
0
      } else {
1284
0
        OS << OperandStr;
1285
0
      }
1286
18
      Pos = DigitEnd;
1287
18
    }
1288
52
  }
1289
23
  AsmString = std::move(OS.str());
1290
23
}
1291
1292
/// Add output constraints for EAX:EDX because they are return registers.
1293
void X86_32TargetCodeGenInfo::addReturnRegisterOutputs(
1294
    CodeGenFunction &CGF, LValue ReturnSlot, std::string &Constraints,
1295
    std::vector<llvm::Type *> &ResultRegTypes,
1296
    std::vector<llvm::Type *> &ResultTruncRegTypes,
1297
    std::vector<LValue> &ResultRegDests, std::string &AsmString,
1298
23
    unsigned NumOutputs) const {
1299
23
  uint64_t RetWidth = CGF.getContext().getTypeSize(ReturnSlot.getType());
1300
1301
  // Use the EAX constraint if the width is 32 or smaller and EAX:EDX if it is
1302
  // larger.
1303
23
  if (!Constraints.empty())
1304
3
    Constraints += ',';
1305
23
  if (RetWidth <= 32) {
1306
20
    Constraints += "={eax}";
1307
20
    ResultRegTypes.push_back(CGF.Int32Ty);
1308
3
  } else {
1309
    // Use the 'A' constraint for EAX:EDX.
1310
3
    Constraints += "=A";
1311
3
    ResultRegTypes.push_back(CGF.Int64Ty);
1312
3
  }
1313
1314
  // Truncate EAX or EAX:EDX to an integer of the appropriate size.
1315
23
  llvm::Type *CoerceTy = llvm::IntegerType::get(CGF.getLLVMContext(), RetWidth);
1316
23
  ResultTruncRegTypes.push_back(CoerceTy);
1317
1318
  // Coerce the integer by bitcasting the return slot pointer.
1319
23
  ReturnSlot.setAddress(CGF.Builder.CreateBitCast(ReturnSlot.getAddress(CGF),
1320
23
                                                  CoerceTy->getPointerTo()));
1321
23
  ResultRegDests.push_back(ReturnSlot);
1322
1323
23
  rewriteInputConstraintReferences(NumOutputs, 1, AsmString);
1324
23
}
1325
1326
/// shouldReturnTypeInRegister - Determine if the given type should be
1327
/// returned in a register (for the Darwin and MCU ABI).
1328
bool X86_32ABIInfo::shouldReturnTypeInRegister(QualType Ty,
1329
552
                                               ASTContext &Context) const {
1330
552
  uint64_t Size = Context.getTypeSize(Ty);
1331
1332
  // For i386, type must be register sized.
1333
  // For the MCU ABI, it only needs to be <= 8-byte
1334
552
  if ((IsMCUABI && 
Size > 6419
) ||
(549
!IsMCUABI549
&&
!isRegisterSize(Size)533
))
1335
72
   return false;
1336
1337
480
  if (Ty->isVectorType()) {
1338
    // 64- and 128- bit vectors inside structures are not returned in
1339
    // registers.
1340
4
    if (Size == 64 || 
Size == 1281
)
1341
3
      return false;
1342
1343
1
    return true;
1344
1
  }
1345
1346
  // If this is a builtin, pointer, enum, complex type, member pointer, or
1347
  // member function pointer it is ok.
1348
476
  if (Ty->getAs<BuiltinType>() || 
Ty->hasPointerRepresentation()384
||
1349
378
      Ty->isAnyComplexType() || 
Ty->isEnumeralType()355
||
1350
353
      Ty->isBlockPointerType() || Ty->isMemberPointerType())
1351
385
    return true;
1352
1353
  // Arrays are treated like records.
1354
91
  if (const ConstantArrayType *AT = Context.getAsConstantArrayType(Ty))
1355
6
    return shouldReturnTypeInRegister(AT->getElementType(), Context);
1356
1357
  // Otherwise, it must be a record type.
1358
85
  const RecordType *RT = Ty->getAs<RecordType>();
1359
85
  if (!RT) 
return false0
;
1360
1361
  // FIXME: Traverse bases here too.
1362
1363
  // Structure types are passed in register if all fields would be
1364
  // passed in a register.
1365
119
  
for (const auto *FD : RT->getDecl()->fields())85
{
1366
    // Empty fields are ignored.
1367
119
    if (isEmptyField(Context, FD, true))
1368
11
      continue;
1369
1370
    // Check fields recursively.
1371
108
    if (!shouldReturnTypeInRegister(FD->getType(), Context))
1372
4
      return false;
1373
108
  }
1374
81
  return true;
1375
85
}
1376
1377
204
static bool is32Or64BitBasicType(QualType Ty, ASTContext &Context) {
1378
  // Treat complex types as the element type.
1379
204
  if (const ComplexType *CTy = Ty->getAs<ComplexType>())
1380
2
    Ty = CTy->getElementType();
1381
1382
  // Check for a type which we know has a simple scalar argument-passing
1383
  // convention without any padding.  (We're specifically looking for 32
1384
  // and 64-bit integer and integer-equivalents, float, and double.)
1385
204
  if (!Ty->getAs<BuiltinType>() && 
!Ty->hasPointerRepresentation()36
&&
1386
31
      !Ty->isEnumeralType() && 
!Ty->isBlockPointerType()28
)
1387
28
    return false;
1388
1389
176
  uint64_t Size = Context.getTypeSize(Ty);
1390
176
  return Size == 32 || 
Size == 6428
;
1391
176
}
1392
1393
static bool addFieldSizes(ASTContext &Context, const RecordDecl *RD,
1394
244
                          uint64_t &Size) {
1395
204
  for (const auto *FD : RD->fields()) {
1396
    // Scalar arguments on the stack get 4 byte alignment on x86. If the
1397
    // argument is smaller than 32-bits, expanding the struct will create
1398
    // alignment padding.
1399
204
    if (!is32Or64BitBasicType(FD->getType(), Context))
1400
49
      return false;
1401
1402
    // FIXME: Reject bit-fields wholesale; there are two problems, we don't know
1403
    // how to expand them yet, and the predicate for telling if a bitfield still
1404
    // counts as "basic" is more complicated than what we were doing previously.
1405
155
    if (FD->isBitField())
1406
2
      return false;
1407
1408
153
    Size += Context.getTypeSize(FD->getType());
1409
153
  }
1410
193
  return true;
1411
244
}
1412
1413
static bool addBaseAndFieldSizes(ASTContext &Context, const CXXRecordDecl *RD,
1414
142
                                 uint64_t &Size) {
1415
  // Don't do this if there are any non-empty bases.
1416
1
  for (const CXXBaseSpecifier &Base : RD->bases()) {
1417
1
    if (!addBaseAndFieldSizes(Context, Base.getType()->getAsCXXRecordDecl(),
1418
1
                              Size))
1419
0
      return false;
1420
1
  }
1421
142
  if (!addFieldSizes(Context, RD, Size))
1422
9
    return false;
1423
133
  return true;
1424
133
}
1425
1426
/// Test whether an argument type which is to be passed indirectly (on the
1427
/// stack) would have the equivalent layout if it was expanded into separate
1428
/// arguments. If so, we prefer to do the latter to avoid inhibiting
1429
/// optimizations.
1430
322
bool X86_32ABIInfo::canExpandIndirectArgument(QualType Ty) const {
1431
  // We can only expand structure types.
1432
322
  const RecordType *RT = Ty->getAs<RecordType>();
1433
322
  if (!RT)
1434
66
    return false;
1435
256
  const RecordDecl *RD = RT->getDecl();
1436
256
  uint64_t Size = 0;
1437
256
  if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
1438
154
    if (!IsWin32StructABI) {
1439
      // On non-Windows, we have to conservatively match our old bitcode
1440
      // prototypes in order to be ABI-compatible at the bitcode level.
1441
33
      if (!CXXRD->isCLike())
1442
13
        return false;
1443
121
    } else {
1444
      // Don't do this for dynamic classes.
1445
121
      if (CXXRD->isDynamicClass())
1446
0
        return false;
1447
141
    }
1448
141
    if (!addBaseAndFieldSizes(getContext(), CXXRD, Size))
1449
9
      return false;
1450
102
  } else {
1451
102
    if (!addFieldSizes(getContext(), RD, Size))
1452
42
      return false;
1453
192
  }
1454
1455
  // We can do this if there was no alignment padding.
1456
192
  return Size == getContext().getTypeSize(Ty);
1457
192
}
1458
1459
153
ABIArgInfo X86_32ABIInfo::getIndirectReturnResult(QualType RetTy, CCState &State) const {
1460
  // If the return value is indirect, then the hidden argument is consuming one
1461
  // integer register.
1462
153
  if (State.FreeRegs) {
1463
6
    --State.FreeRegs;
1464
6
    if (!IsMCUABI)
1465
3
      return getNaturalAlignIndirectInReg(RetTy);
1466
150
  }
1467
150
  return getNaturalAlignIndirect(RetTy, /*ByVal=*/false);
1468
150
}
1469
1470
ABIArgInfo X86_32ABIInfo::classifyReturnType(QualType RetTy,
1471
18.4k
                                             CCState &State) const {
1472
18.4k
  if (RetTy->isVoidType())
1473
12.3k
    return ABIArgInfo::getIgnore();
1474
1475
6.02k
  const Type *Base = nullptr;
1476
6.02k
  uint64_t NumElts = 0;
1477
6.02k
  if ((State.CC == llvm::CallingConv::X86_VectorCall ||
1478
6.01k
       State.CC == llvm::CallingConv::X86_RegCall) &&
1479
26
      isHomogeneousAggregate(RetTy, Base, NumElts)) {
1480
    // The LLVM struct type for such an aggregate should lower properly.
1481
17
    return ABIArgInfo::getDirect();
1482
17
  }
1483
1484
6.01k
  if (const VectorType *VT = RetTy->getAs<VectorType>()) {
1485
    // On Darwin, some vectors are returned in registers.
1486
31
    if (IsDarwinVectorABI) {
1487
15
      uint64_t Size = getContext().getTypeSize(RetTy);
1488
1489
      // 128-bit vectors are a special case; they are returned in
1490
      // registers and we need to make sure to pick a type the LLVM
1491
      // backend will like.
1492
15
      if (Size == 128)
1493
5
        return ABIArgInfo::getDirect(llvm::FixedVectorType::get(
1494
5
            llvm::Type::getInt64Ty(getVMContext()), 2));
1495
1496
      // Always return in register if it fits in a general purpose
1497
      // register, or if it is 64 bits and has a single element.
1498
10
      if ((Size == 8 || Size == 16 || Size == 32) ||
1499
7
          (Size == 64 && VT->getNumElements() == 1))
1500
7
        return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(),
1501
7
                                                            Size));
1502
1503
3
      return getIndirectReturnResult(RetTy, State);
1504
3
    }
1505
1506
16
    return ABIArgInfo::getDirect();
1507
16
  }
1508
1509
5.97k
  if (isAggregateTypeForABI(RetTy)) {
1510
537
    if (const RecordType *RT = RetTy->getAs<RecordType>()) {
1511
      // Structures with flexible arrays are always indirect.
1512
207
      if (RT->getDecl()->hasFlexibleArrayMember())
1513
1
        return getIndirectReturnResult(RetTy, State);
1514
536
    }
1515
1516
    // If specified, structs and unions are always indirect.
1517
536
    if (!IsRetSmallStructInRegABI && 
!RetTy->isAnyComplexType()116
)
1518
64
      return getIndirectReturnResult(RetTy, State);
1519
1520
    // Ignore empty structs/unions.
1521
472
    if (isEmptyRecord(getContext(), RetTy, true))
1522
34
      return ABIArgInfo::getIgnore();
1523
1524
    // Small structures which are register sized are generally returned
1525
    // in a register.
1526
438
    if (shouldReturnTypeInRegister(RetTy, getContext())) {
1527
363
      uint64_t Size = getContext().getTypeSize(RetTy);
1528
1529
      // As a special-case, if the struct is a "single-element" struct, and
1530
      // the field is of type "float" or "double", return it in a
1531
      // floating-point register. (MSVC does not apply this special case.)
1532
      // We apply a similar transformation for pointer types to improve the
1533
      // quality of the generated IR.
1534
363
      if (const Type *SeltTy = isSingleElementStruct(RetTy, getContext()))
1535
54
        if ((!IsWin32StructABI && 
SeltTy->isRealFloatingType()42
)
1536
38
            || SeltTy->hasPointerRepresentation())
1537
20
          return ABIArgInfo::getDirect(CGT.ConvertType(QualType(SeltTy, 0)));
1538
1539
      // FIXME: We should be able to narrow this integer in cases with dead
1540
      // padding.
1541
343
      return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(),Size));
1542
343
    }
1543
1544
75
    return getIndirectReturnResult(RetTy, State);
1545
75
  }
1546
1547
  // Treat an enum type as its underlying type.
1548
5.44k
  if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
1549
35
    RetTy = EnumTy->getDecl()->getIntegerType();
1550
1551
5.44k
  if (const auto *EIT = RetTy->getAs<ExtIntType>())
1552
14
    if (EIT->getNumBits() > 64)
1553
10
      return getIndirectReturnResult(RetTy, State);
1554
1555
5.43k
  return (isPromotableIntegerTypeForABI(RetTy) ? 
ABIArgInfo::getExtend(RetTy)130
1556
5.30k
                                               : ABIArgInfo::getDirect());
1557
5.43k
}
1558
1559
66
static bool isSIMDVectorType(ASTContext &Context, QualType Ty) {
1560
66
  return Ty->getAs<VectorType>() && 
Context.getTypeSize(Ty) == 12814
;
1561
66
}
1562
1563
84
static bool isRecordWithSIMDVectorType(ASTContext &Context, QualType Ty) {
1564
84
  const RecordType *RT = Ty->getAs<RecordType>();
1565
84
  if (!RT)
1566
42
    return 0;
1567
42
  const RecordDecl *RD = RT->getDecl();
1568
1569
  // If this is a C++ record, check the bases first.
1570
42
  if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD))
1571
0
    for (const auto &I : CXXRD->bases())
1572
0
      if (!isRecordWithSIMDVectorType(Context, I.getType()))
1573
0
        return false;
1574
1575
49
  
for (const auto *i : RD->fields())42
{
1576
49
    QualType FT = i->getType();
1577
1578
49
    if (isSIMDVectorType(Context, FT))
1579
9
      return true;
1580
1581
40
    if (isRecordWithSIMDVectorType(Context, FT))
1582
0
      return true;
1583
40
  }
1584
1585
33
  return false;
1586
42
}
1587
1588
unsigned X86_32ABIInfo::getTypeStackAlignInBytes(QualType Ty,
1589
299
                                                 unsigned Align) const {
1590
  // Otherwise, if the alignment is less than or equal to the minimum ABI
1591
  // alignment, just use the default; the backend will handle this.
1592
299
  if (Align <= MinABIStackAlignInBytes)
1593
249
    return 0; // Use default alignment.
1594
1595
  // On non-Darwin, the stack type alignment is always 4.
1596
50
  if (!IsDarwinVectorABI) {
1597
    // Set explicit alignment, since we may need to realign the top.
1598
23
    return MinABIStackAlignInBytes;
1599
23
  }
1600
1601
  // Otherwise, if the type contains an SSE vector type, the alignment is 16.
1602
27
  if (Align >= 16 && 
(17
isSIMDVectorType(getContext(), Ty)17
||
1603
16
                      isRecordWithSIMDVectorType(getContext(), Ty)))
1604
8
    return 16;
1605
1606
19
  return MinABIStackAlignInBytes;
1607
19
}
1608
1609
ABIArgInfo X86_32ABIInfo::getIndirectResult(QualType Ty, bool ByVal,
1610
445
                                            CCState &State) const {
1611
445
  if (!ByVal) {
1612
158
    if (State.FreeRegs) {
1613
22
      --State.FreeRegs; // Non-byval indirects just use one pointer.
1614
22
      if (!IsMCUABI)
1615
22
        return getNaturalAlignIndirectInReg(Ty);
1616
136
    }
1617
136
    return getNaturalAlignIndirect(Ty, false);
1618
136
  }
1619
1620
  // Compute the byval alignment.
1621
287
  unsigned TypeAlign = getContext().getTypeAlign(Ty) / 8;
1622
287
  unsigned StackAlign = getTypeStackAlignInBytes(Ty, TypeAlign);
1623
287
  if (StackAlign == 0)
1624
239
    return ABIArgInfo::getIndirect(CharUnits::fromQuantity(4), /*ByVal=*/true);
1625
1626
  // If the stack alignment is less than the type alignment, realign the
1627
  // argument.
1628
48
  bool Realign = TypeAlign > StackAlign;
1629
48
  return ABIArgInfo::getIndirect(CharUnits::fromQuantity(StackAlign),
1630
48
                                 /*ByVal=*/true, Realign);
1631
48
}
1632
1633
31.8k
X86_32ABIInfo::Class X86_32ABIInfo::classify(QualType Ty) const {
1634
31.8k
  const Type *T = isSingleElementStruct(Ty, getContext());
1635
31.8k
  if (!T)
1636
31.7k
    T = Ty.getTypePtr();
1637
1638
31.8k
  if (const BuiltinType *BT = T->getAs<BuiltinType>()) {
1639
10.7k
    BuiltinType::Kind K = BT->getKind();
1640
10.7k
    if (K == BuiltinType::Float || 
K == BuiltinType::Double10.4k
)
1641
468
      return Float;
1642
31.4k
  }
1643
31.4k
  return Integer;
1644
31.4k
}
1645
1646
31.9k
bool X86_32ABIInfo::updateFreeRegs(QualType Ty, CCState &State) const {
1647
31.9k
  if (!IsSoftFloatABI) {
1648
31.8k
    Class C = classify(Ty);
1649
31.8k
    if (C == Float)
1650
468
      return false;
1651
31.4k
  }
1652
1653
31.4k
  unsigned Size = getContext().getTypeSize(Ty);
1654
31.4k
  unsigned SizeInRegs = (Size + 31) / 32;
1655
1656
31.4k
  if (SizeInRegs == 0)
1657
2
    return false;
1658
1659
31.4k
  if (!IsMCUABI) {
1660
31.3k
    if (SizeInRegs > State.FreeRegs) {
1661
31.1k
      State.FreeRegs = 0;
1662
31.1k
      return false;
1663
31.1k
    }
1664
42
  } else {
1665
    // The MCU psABI allows passing parameters in-reg even if there are
1666
    // earlier parameters that are passed on the stack. Also,
1667
    // it does not allow passing >8-byte structs in-register,
1668
    // even if there are 3 free registers available.
1669
42
    if (SizeInRegs > State.FreeRegs || 
SizeInRegs > 234
)
1670
9
      return false;
1671
243
  }
1672
1673
243
  State.FreeRegs -= SizeInRegs;
1674
243
  return true;
1675
243
}
1676
1677
bool X86_32ABIInfo::shouldAggregateUseDirect(QualType Ty, CCState &State,
1678
                                             bool &InReg,
1679
388
                                             bool &NeedsPadding) const {
1680
  // On Windows, aggregates other than HFAs are never passed in registers, and
1681
  // they do not consume register slots. Homogenous floating-point aggregates
1682
  // (HFAs) have already been dealt with at this point.
1683
388
  if (IsWin32StructABI && 
isAggregateTypeForABI(Ty)168
)
1684
168
    return false;
1685
1686
220
  NeedsPadding = false;
1687
220
  InReg = !IsMCUABI;
1688
1689
220
  if (!updateFreeRegs(Ty, State))
1690
200
    return false;
1691
1692
20
  if (IsMCUABI)
1693
7
    return true;
1694
1695
13
  if (State.CC == llvm::CallingConv::X86_FastCall ||
1696
10
      State.CC == llvm::CallingConv::X86_VectorCall ||
1697
10
      State.CC == llvm::CallingConv::X86_RegCall) {
1698
4
    if (getContext().getTypeSize(Ty) <= 32 && State.FreeRegs)
1699
3
      NeedsPadding = true;
1700
1701
4
    return false;
1702
4
  }
1703
1704
9
  return true;
1705
9
}
1706
1707
31.6k
bool X86_32ABIInfo::shouldPrimitiveUseInReg(QualType Ty, CCState &State) const {
1708
31.6k
  if (!updateFreeRegs(Ty, State))
1709
31.4k
    return false;
1710
1711
223
  if (IsMCUABI)
1712
26
    return false;
1713
1714
197
  if (State.CC == llvm::CallingConv::X86_FastCall ||
1715
137
      State.CC == llvm::CallingConv::X86_VectorCall ||
1716
146
      
State.CC == llvm::CallingConv::X86_RegCall94
) {
1717
146
    if (getContext().getTypeSize(Ty) > 32)
1718
13
      return false;
1719
1720
133
    return (Ty->isIntegralOrEnumerationType() || 
Ty->isPointerType()27
||
1721
7
        Ty->isReferenceType());
1722
133
  }
1723
1724
51
  return true;
1725
51
}
1726
1727
55
void X86_32ABIInfo::runVectorCallFirstPass(CGFunctionInfo &FI, CCState &State) const {
1728
  // Vectorcall x86 works subtly different than in x64, so the format is
1729
  // a bit different than the x64 version.  First, all vector types (not HVAs)
1730
  // are assigned, with the first 6 ending up in the [XYZ]MM0-5 registers.
1731
  // This differs from the x64 implementation, where the first 6 by INDEX get
1732
  // registers.
1733
  // In the second pass over the arguments, HVAs are passed in the remaining
1734
  // vector registers if possible, or indirectly by address. The address will be
1735
  // passed in ECX/EDX if available. Any other arguments are passed according to
1736
  // the usual fastcall rules.
1737
55
  MutableArrayRef<CGFunctionInfoArgInfo> Args = FI.arguments();
1738
169
  for (int I = 0, E = Args.size(); I < E; 
++I114
) {
1739
114
    const Type *Base = nullptr;
1740
114
    uint64_t NumElts = 0;
1741
114
    const QualType &Ty = Args[I].type;
1742
114
    if ((Ty->isVectorType() || 
Ty->isBuiltinType()93
) &&
1743
84
        isHomogeneousAggregate(Ty, Base, NumElts)) {
1744
42
      if (State.FreeSSERegs >= NumElts) {
1745
38
        State.FreeSSERegs -= NumElts;
1746
38
        Args[I].info = ABIArgInfo::getDirectInReg();
1747
38
        State.IsPreassigned.set(I);
1748
38
      }
1749
42
    }
1750
114
  }
1751
55
}
1752
1753
ABIArgInfo X86_32ABIInfo::classifyArgumentType(QualType Ty,
1754
32.4k
                                               CCState &State) const {
1755
  // FIXME: Set alignment on indirect arguments.
1756
32.4k
  bool IsFastCall = State.CC == llvm::CallingConv::X86_FastCall;
1757
32.4k
  bool IsRegCall = State.CC == llvm::CallingConv::X86_RegCall;
1758
32.4k
  bool IsVectorCall = State.CC == llvm::CallingConv::X86_VectorCall;
1759
1760
32.4k
  Ty = useFirstFieldIfTransparentUnion(Ty);
1761
32.4k
  TypeInfo TI = getContext().getTypeInfo(Ty);
1762
1763
  // Check with the C++ ABI first.
1764
32.4k
  const RecordType *RT = Ty->getAs<RecordType>();
1765
32.4k
  if (RT) {
1766
582
    CGCXXABI::RecordArgABI RAA = getRecordArgABI(RT, getCXXABI());
1767
582
    if (RAA == CGCXXABI::RAA_Indirect) {
1768
102
      return getIndirectResult(Ty, false, State);
1769
480
    } else if (RAA == CGCXXABI::RAA_DirectInMemory) {
1770
      // The field index doesn't matter, we'll fix it up later.
1771
84
      return ABIArgInfo::getInAlloca(/*FieldIndex=*/0);
1772
84
    }
1773
32.3k
  }
1774
1775
  // Regcall uses the concept of a homogenous vector aggregate, similar
1776
  // to other targets.
1777
32.3k
  const Type *Base = nullptr;
1778
32.3k
  uint64_t NumElts = 0;
1779
32.3k
  if ((IsRegCall || 
IsVectorCall32.2k
) &&
1780
179
      isHomogeneousAggregate(Ty, Base, NumElts)) {
1781
79
    if (State.FreeSSERegs >= NumElts) {
1782
57
      State.FreeSSERegs -= NumElts;
1783
1784
      // Vectorcall passes HVAs directly and does not flatten them, but regcall
1785
      // does.
1786
57
      if (IsVectorCall)
1787
12
        return getDirectX86Hva();
1788
1789
45
      if (Ty->isBuiltinType() || 
Ty->isVectorType()35
)
1790
20
        return ABIArgInfo::getDirect();
1791
25
      return ABIArgInfo::getExpand();
1792
25
    }
1793
22
    return getIndirectResult(Ty, /*ByVal=*/false, State);
1794
22
  }
1795
1796
32.2k
  if (isAggregateTypeForABI(Ty)) {
1797
    // Structures with flexible arrays are always indirect.
1798
    // FIXME: This should not be byval!
1799
424
    if (RT && 
RT->getDecl()->hasFlexibleArrayMember()348
)
1800
1
      return getIndirectResult(Ty, true, State);
1801
1802
    // Ignore empty structs/unions on non-Windows.
1803
423
    if (!IsWin32StructABI && 
isEmptyRecord(getContext(), Ty, true)255
)
1804
35
      return ABIArgInfo::getIgnore();
1805
1806
388
    llvm::LLVMContext &LLVMContext = getVMContext();
1807
388
    llvm::IntegerType *Int32 = llvm::Type::getInt32Ty(LLVMContext);
1808
388
    bool NeedsPadding = false;
1809
388
    bool InReg;
1810
388
    if (shouldAggregateUseDirect(Ty, State, InReg, NeedsPadding)) {
1811
16
      unsigned SizeInRegs = (TI.Width + 31) / 32;
1812
16
      SmallVector<llvm::Type*, 3> Elements(SizeInRegs, Int32);
1813
16
      llvm::Type *Result = llvm::StructType::get(LLVMContext, Elements);
1814
16
      if (InReg)
1815
9
        return ABIArgInfo::getDirectInReg(Result);
1816
7
      else
1817
7
        return ABIArgInfo::getDirect(Result);
1818
372
    }
1819
372
    llvm::IntegerType *PaddingType = NeedsPadding ? 
Int323
:
nullptr369
;
1820
1821
    // Pass over-aligned aggregates on Windows indirectly. This behavior was
1822
    // added in MSVC 2015.
1823
372
    if (IsWin32StructABI && 
TI.AlignIsRequired168
&&
TI.Align > 321
)
1824
1
      return getIndirectResult(Ty, /*ByVal=*/false, State);
1825
1826
    // Expand small (<= 128-bit) record types when we know that the stack layout
1827
    // of those arguments will match the struct. This is important because the
1828
    // LLVM backend isn't smart enough to remove byval, which inhibits many
1829
    // optimizations.
1830
    // Don't do this for the MCU if there are still free integer registers
1831
    // (see X86_64 ABI for full explanation).
1832
371
    if (TI.Width <= 4 * 32 && 
(325
!IsMCUABI325
||
State.FreeRegs == 05
) &&
1833
322
        canExpandIndirectArgument(Ty))
1834
85
      return ABIArgInfo::getExpandWithPadding(
1835
85
          IsFastCall || 
IsVectorCall81
||
IsRegCall80
, PaddingType);
1836
1837
286
    return getIndirectResult(Ty, true, State);
1838
286
  }
1839
1840
31.8k
  if (const VectorType *VT = Ty->getAs<VectorType>()) {
1841
    // On Windows, vectors are passed directly if registers are available, or
1842
    // indirectly if not. This avoids the need to align argument memory. Pass
1843
    // user-defined vector types larger than 512 bits indirectly for simplicity.
1844
117
    if (IsWin32StructABI) {
1845
45
      if (TI.Width <= 512 && 
State.FreeSSERegs > 040
) {
1846
28
        --State.FreeSSERegs;
1847
28
        return ABIArgInfo::getDirectInReg();
1848
28
      }
1849
17
      return getIndirectResult(Ty, /*ByVal=*/false, State);
1850
17
    }
1851
1852
    // On Darwin, some vectors are passed in memory, we handle this by passing
1853
    // it as an i8/i16/i32/i64.
1854
72
    if (IsDarwinVectorABI) {
1855
24
      if ((TI.Width == 8 || TI.Width == 16 || TI.Width == 32) ||
1856
22
          (TI.Width == 64 && 
VT->getNumElements() == 110
))
1857
7
        return ABIArgInfo::getDirect(
1858
7
            llvm::IntegerType::get(getVMContext(), TI.Width));
1859
65
    }
1860
1861
65
    if (IsX86_MMXType(CGT.ConvertType(Ty)))
1862
8
      return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(), 64));
1863
1864
57
    return ABIArgInfo::getDirect();
1865
57
  }
1866
1867
1868
31.6k
  if (const EnumType *EnumTy = Ty->getAs<EnumType>())
1869
23
    Ty = EnumTy->getDecl()->getIntegerType();
1870
1871
31.6k
  bool InReg = shouldPrimitiveUseInReg(Ty, State);
1872
1873
31.6k
  if (isPromotableIntegerTypeForABI(Ty)) {
1874
242
    if (InReg)
1875
25
      return ABIArgInfo::getExtendInReg(Ty);
1876
217
    return ABIArgInfo::getExtend(Ty);
1877
217
  }
1878
1879
31.4k
  if (const auto * EIT = Ty->getAs<ExtIntType>()) {
1880
20
    if (EIT->getNumBits() <= 64) {
1881
4
      if (InReg)
1882
0
        return ABIArgInfo::getDirectInReg();
1883
4
      return ABIArgInfo::getDirect();
1884
4
    }
1885
16
    return getIndirectResult(Ty, /*ByVal=*/false, State);
1886
16
  }
1887
1888
31.4k
  if (InReg)
1889
159
    return ABIArgInfo::getDirectInReg();
1890
31.2k
  return ABIArgInfo::getDirect();
1891
31.2k
}
1892
1893
18.5k
void X86_32ABIInfo::computeInfo(CGFunctionInfo &FI) const {
1894
18.5k
  CCState State(FI);
1895
18.5k
  if (IsMCUABI)
1896
33
    State.FreeRegs = 3;
1897
18.4k
  else if (State.CC == llvm::CallingConv::X86_FastCall) {
1898
74
    State.FreeRegs = 2;
1899
74
    State.FreeSSERegs = 3;
1900
18.3k
  } else if (State.CC == llvm::CallingConv::X86_VectorCall) {
1901
55
    State.FreeRegs = 2;
1902
55
    State.FreeSSERegs = 6;
1903
18.3k
  } else if (FI.getHasRegParm())
1904
31
    State.FreeRegs = FI.getRegParm();
1905
18.3k
  else if (State.CC == llvm::CallingConv::X86_RegCall) {
1906
52
    State.FreeRegs = 5;
1907
52
    State.FreeSSERegs = 8;
1908
18.2k
  } else if (IsWin32StructABI) {
1909
    // Since MSVC 2015, the first three SSE vectors have been passed in
1910
    // registers. The rest are passed indirectly.
1911
3.96k
    State.FreeRegs = DefaultNumRegisterParameters;
1912
3.96k
    State.FreeSSERegs = 3;
1913
3.96k
  } else
1914
14.2k
    State.FreeRegs = DefaultNumRegisterParameters;
1915
1916
18.5k
  if (!::classifyReturnType(getCXXABI(), FI, *this)) {
1917
18.4k
    FI.getReturnInfo() = classifyReturnType(FI.getReturnType(), State);
1918
82
  } else if (FI.getReturnInfo().isIndirect()) {
1919
    // The C++ ABI is not aware of register usage, so we have to check if the
1920
    // return value was sret and put it in a register ourselves if appropriate.
1921
82
    if (State.FreeRegs) {
1922
4
      --State.FreeRegs;  // The sret parameter consumes a register.
1923
4
      if (!IsMCUABI)
1924
4
        FI.getReturnInfo().setInReg(true);
1925
4
    }
1926
82
  }
1927
1928
  // The chain argument effectively gives us another free register.
1929
18.5k
  if (FI.isChainCall())
1930
4
    ++State.FreeRegs;
1931
1932
  // For vectorcall, do a first pass over the arguments, assigning FP and vector
1933
  // arguments to XMM registers as available.
1934
18.5k
  if (State.CC == llvm::CallingConv::X86_VectorCall)
1935
55
    runVectorCallFirstPass(FI, State);
1936
1937
18.5k
  bool UsedInAlloca = false;
1938
18.5k
  MutableArrayRef<CGFunctionInfoArgInfo> Args = FI.arguments();
1939
51.0k
  for (int I = 0, E = Args.size(); I < E; 
++I32.5k
) {
1940
    // Skip arguments that have already been assigned.
1941
32.5k
    if (State.IsPreassigned.test(I))
1942
38
      continue;
1943
1944
32.4k
    Args[I].info = classifyArgumentType(Args[I].type, State);
1945
32.4k
    UsedInAlloca |= (Args[I].info.getKind() == ABIArgInfo::InAlloca);
1946
32.4k
  }
1947
1948
  // If we needed to use inalloca for any argument, do a second pass and rewrite
1949
  // all the memory arguments to use inalloca.
1950
18.5k
  if (UsedInAlloca)
1951
72
    rewriteWithInAlloca(FI);
1952
18.5k
}
1953
1954
void
1955
X86_32ABIInfo::addFieldToArgStruct(SmallVector<llvm::Type *, 6> &FrameFields,
1956
                                   CharUnits &StackOffset, ABIArgInfo &Info,
1957
165
                                   QualType Type) const {
1958
  // Arguments are always 4-byte-aligned.
1959
165
  CharUnits WordSize = CharUnits::fromQuantity(4);
1960
165
  assert(StackOffset.isMultipleOf(WordSize) && "unaligned inalloca struct");
1961
1962
  // sret pointers and indirect things will require an extra pointer
1963
  // indirection, unless they are byval. Most things are byval, and will not
1964
  // require this indirection.
1965
165
  bool IsIndirect = false;
1966
165
  if (Info.isIndirect() && 
!Info.getIndirectByVal()17
)
1967
14
    IsIndirect = true;
1968
165
  Info = ABIArgInfo::getInAlloca(FrameFields.size(), IsIndirect);
1969
165
  llvm::Type *LLTy = CGT.ConvertTypeForMem(Type);
1970
165
  if (IsIndirect)
1971
14
    LLTy = LLTy->getPointerTo(0);
1972
165
  FrameFields.push_back(LLTy);
1973
151
  StackOffset += IsIndirect ? 
WordSize14
: getContext().getTypeSizeInChars(Type);
1974
1975
  // Insert padding bytes to respect alignment.
1976
165
  CharUnits FieldEnd = StackOffset;
1977
165
  StackOffset = FieldEnd.alignTo(WordSize);
1978
165
  if (StackOffset != FieldEnd) {
1979
40
    CharUnits NumBytes = StackOffset - FieldEnd;
1980
40
    llvm::Type *Ty = llvm::Type::getInt8Ty(getVMContext());
1981
40
    Ty = llvm::ArrayType::get(Ty, NumBytes.getQuantity());
1982
40
    FrameFields.push_back(Ty);
1983
40
  }
1984
165
}
1985
1986
175
static bool isArgInAlloca(const ABIArgInfo &Info) {
1987
  // Leave ignored and inreg arguments alone.
1988
175
  switch (Info.getKind()) {
1989
84
  case ABIArgInfo::InAlloca:
1990
84
    return true;
1991
0
  case ABIArgInfo::Ignore:
1992
0
  case ABIArgInfo::IndirectAliased:
1993
0
    return false;
1994
11
  case ABIArgInfo::Indirect:
1995
84
  case ABIArgInfo::Direct:
1996
90
  case ABIArgInfo::Extend:
1997
90
    return !Info.getInReg();
1998
1
  case ABIArgInfo::Expand:
1999
1
  case ABIArgInfo::CoerceAndExpand:
2000
    // These are aggregate types which are never passed in registers when
2001
    // inalloca is involved.
2002
1
    return true;
2003
0
  }
2004
0
  llvm_unreachable("invalid enum");
2005
0
}
2006
2007
72
void X86_32ABIInfo::rewriteWithInAlloca(CGFunctionInfo &FI) const {
2008
72
  assert(IsWin32StructABI && "inalloca only supported on win32");
2009
2010
  // Build a packed struct type for all of the arguments in memory.
2011
72
  SmallVector<llvm::Type *, 6> FrameFields;
2012
2013
  // The stack alignment is always 4.
2014
72
  CharUnits StackAlign = CharUnits::fromQuantity(4);
2015
2016
72
  CharUnits StackOffset;
2017
72
  CGFunctionInfo::arg_iterator I = FI.arg_begin(), E = FI.arg_end();
2018
2019
  // Put 'this' into the struct before 'sret', if necessary.
2020
72
  bool IsThisCall =
2021
72
      FI.getCallingConvention() == llvm::CallingConv::X86_ThisCall;
2022
72
  ABIArgInfo &Ret = FI.getReturnInfo();
2023
72
  if (Ret.isIndirect() && 
Ret.isSRetAfterThis()9
&&
!IsThisCall7
&&
2024
6
      isArgInAlloca(I->info)) {
2025
5
    addFieldToArgStruct(FrameFields, StackOffset, I->info, I->type);
2026
5
    ++I;
2027
5
  }
2028
2029
  // Put the sret parameter into the inalloca struct if it's in memory.
2030
72
  if (Ret.isIndirect() && 
!Ret.getInReg()9
) {
2031
8
    addFieldToArgStruct(FrameFields, StackOffset, Ret, FI.getReturnType());
2032
    // On Windows, the hidden sret parameter is always returned in eax.
2033
8
    Ret.setInAllocaSRet(IsWin32StructABI);
2034
8
  }
2035
2036
  // Skip the 'this' parameter in ecx.
2037
72
  if (IsThisCall)
2038
17
    ++I;
2039
2040
  // Put arguments passed in memory into the struct.
2041
241
  for (; I != E; 
++I169
) {
2042
169
    if (isArgInAlloca(I->info))
2043
152
      addFieldToArgStruct(FrameFields, StackOffset, I->info, I->type);
2044
169
  }
2045
2046
72
  FI.setArgStruct(llvm::StructType::get(getVMContext(), FrameFields,
2047
72
                                        /*isPacked=*/true),
2048
72
                  StackAlign);
2049
72
}
2050
2051
Address X86_32ABIInfo::EmitVAArg(CodeGenFunction &CGF,
2052
12
                                 Address VAListAddr, QualType Ty) const {
2053
2054
12
  auto TypeInfo = getContext().getTypeInfoInChars(Ty);
2055
2056
  // x86-32 changes the alignment of certain arguments on the stack.
2057
  //
2058
  // Just messing with TypeInfo like this works because we never pass
2059
  // anything indirectly.
2060
12
  TypeInfo.Align = CharUnits::fromQuantity(
2061
12
                getTypeStackAlignInBytes(Ty, TypeInfo.Align.getQuantity()));
2062
2063
12
  return emitVoidPtrVAArg(CGF, VAListAddr, Ty, /*Indirect*/ false,
2064
12
                          TypeInfo, CharUnits::fromQuantity(4),
2065
12
                          /*AllowHigherAlign*/ true);
2066
12
}
2067
2068
bool X86_32TargetCodeGenInfo::isStructReturnInRegABI(
2069
2.99k
    const llvm::Triple &Triple, const CodeGenOptions &Opts) {
2070
2.99k
  assert(Triple.getArch() == llvm::Triple::x86);
2071
2072
2.99k
  switch (Opts.getStructReturnConvention()) {
2073
2.99k
  case CodeGenOptions::SRCK_Default:
2074
2.99k
    break;
2075
3
  case CodeGenOptions::SRCK_OnStack:  // -fpcc-struct-return
2076
3
    return false;
2077
3
  case CodeGenOptions::SRCK_InRegs:  // -freg-struct-return
2078
3
    return true;
2079
2.99k
  }
2080
2081
2.99k
  if (Triple.isOSDarwin() || 
Triple.isOSIAMCU()2.81k
)
2082
176
    return true;
2083
2084
2.81k
  switch (Triple.getOS()) {
2085
1
  case llvm::Triple::DragonFly:
2086
1
  case llvm::Triple::FreeBSD:
2087
2
  case llvm::Triple::OpenBSD:
2088
428
  case llvm::Triple::Win32:
2089
428
    return true;
2090
2.38k
  default:
2091
2.38k
    return false;
2092
2.81k
  }
2093
2.81k
}
2094
2095
static void addX86InterruptAttrs(const FunctionDecl *FD, llvm::GlobalValue *GV,
2096
174k
                                 CodeGen::CodeGenModule &CGM) {
2097
174k
  if (!FD->hasAttr<AnyX86InterruptAttr>())
2098
174k
    return;
2099
2100
25
  llvm::Function *Fn = cast<llvm::Function>(GV);
2101
25
  Fn->setCallingConv(llvm::CallingConv::X86_INTR);
2102
25
  if (FD->getNumParams() == 0)
2103
0
    return;
2104
2105
25
  auto PtrTy = cast<PointerType>(FD->getParamDecl(0)->getType());
2106
25
  llvm::Type *ByValTy = CGM.getTypes().ConvertType(PtrTy->getPointeeType());
2107
25
  llvm::Attribute NewAttr = llvm::Attribute::getWithByValType(
2108
25
    Fn->getContext(), ByValTy);
2109
25
  Fn->addParamAttr(0, NewAttr);
2110
25
}
2111
2112
void X86_32TargetCodeGenInfo::setTargetAttributes(
2113
67.3k
    const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {
2114
67.3k
  if (GV->isDeclaration())
2115
41.9k
    return;
2116
25.3k
  if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) {
2117
21.0k
    if (FD->hasAttr<X86ForceAlignArgPointerAttr>()) {
2118
2
      llvm::Function *Fn = cast<llvm::Function>(GV);
2119
2
      Fn->addFnAttr("stackrealign");
2120
2
    }
2121
2122
21.0k
    addX86InterruptAttrs(FD, GV, CGM);
2123
21.0k
  }
2124
25.3k
}
2125
2126
bool X86_32TargetCodeGenInfo::initDwarfEHRegSizeTable(
2127
                                               CodeGen::CodeGenFunction &CGF,
2128
0
                                               llvm::Value *Address) const {
2129
0
  CodeGen::CGBuilderTy &Builder = CGF.Builder;
2130
2131
0
  llvm::Value *Four8 = llvm::ConstantInt::get(CGF.Int8Ty, 4);
2132
2133
  // 0-7 are the eight integer registers;  the order is different
2134
  //   on Darwin (for EH), but the range is the same.
2135
  // 8 is %eip.
2136
0
  AssignToArrayRange(Builder, Address, Four8, 0, 8);
2137
2138
0
  if (CGF.CGM.getTarget().getTriple().isOSDarwin()) {
2139
    // 12-16 are st(0..4).  Not sure why we stop at 4.
2140
    // These have size 16, which is sizeof(long double) on
2141
    // platforms with 8-byte alignment for that type.
2142
0
    llvm::Value *Sixteen8 = llvm::ConstantInt::get(CGF.Int8Ty, 16);
2143
0
    AssignToArrayRange(Builder, Address, Sixteen8, 12, 16);
2144
2145
0
  } else {
2146
    // 9 is %eflags, which doesn't get a size on Darwin for some
2147
    // reason.
2148
0
    Builder.CreateAlignedStore(
2149
0
        Four8, Builder.CreateConstInBoundsGEP1_32(CGF.Int8Ty, Address, 9),
2150
0
                               CharUnits::One());
2151
2152
    // 11-16 are st(0..5).  Not sure why we stop at 5.
2153
    // These have size 12, which is sizeof(long double) on
2154
    // platforms with 4-byte alignment for that type.
2155
0
    llvm::Value *Twelve8 = llvm::ConstantInt::get(CGF.Int8Ty, 12);
2156
0
    AssignToArrayRange(Builder, Address, Twelve8, 11, 16);
2157
0
  }
2158
2159
0
  return false;
2160
0
}
2161
2162
//===----------------------------------------------------------------------===//
2163
// X86-64 ABI Implementation
2164
//===----------------------------------------------------------------------===//
2165
2166
2167
namespace {
2168
/// The AVX ABI level for X86 targets.
2169
enum class X86AVXABILevel {
2170
  None,
2171
  AVX,
2172
  AVX512
2173
};
2174
2175
/// \p returns the size in bits of the largest (native) vector for \p AVXLevel.
2176
3.62k
static unsigned getNativeVectorSizeForAVXABI(X86AVXABILevel AVXLevel) {
2177
3.62k
  switch (AVXLevel) {
2178
2.70k
  case X86AVXABILevel::AVX512:
2179
2.70k
    return 512;
2180
696
  case X86AVXABILevel::AVX:
2181
696
    return 256;
2182
224
  case X86AVXABILevel::None:
2183
224
    return 128;
2184
0
  }
2185
0
  llvm_unreachable("Unknown AVXLevel");
2186
0
}
2187
2188
/// X86_64ABIInfo - The X86_64 ABI information.
2189
class X86_64ABIInfo : public SwiftABIInfo {
2190
  enum Class {
2191
    Integer = 0,
2192
    SSE,
2193
    SSEUp,
2194
    X87,
2195
    X87Up,
2196
    ComplexX87,
2197
    NoClass,
2198
    Memory
2199
  };
2200
2201
  /// merge - Implement the X86_64 ABI merging algorithm.
2202
  ///
2203
  /// Merge an accumulating classification \arg Accum with a field
2204
  /// classification \arg Field.
2205
  ///
2206
  /// \param Accum - The accumulating classification. This should
2207
  /// always be either NoClass or the result of a previous merge
2208
  /// call. In addition, this should never be Memory (the caller
2209
  /// should just return Memory for the aggregate).
2210
  static Class merge(Class Accum, Class Field);
2211
2212
  /// postMerge - Implement the X86_64 ABI post merging algorithm.
2213
  ///
2214
  /// Post merger cleanup, reduces a malformed Hi and Lo pair to
2215
  /// final MEMORY or SSE classes when necessary.
2216
  ///
2217
  /// \param AggregateSize - The size of the current aggregate in
2218
  /// the classification process.
2219
  ///
2220
  /// \param Lo - The classification for the parts of the type
2221
  /// residing in the low word of the containing object.
2222
  ///
2223
  /// \param Hi - The classification for the parts of the type
2224
  /// residing in the higher words of the containing object.
2225
  ///
2226
  void postMerge(unsigned AggregateSize, Class &Lo, Class &Hi) const;
2227
2228
  /// classify - Determine the x86_64 register classes in which the
2229
  /// given type T should be passed.
2230
  ///
2231
  /// \param Lo - The classification for the parts of the type
2232
  /// residing in the low word of the containing object.
2233
  ///
2234
  /// \param Hi - The classification for the parts of the type
2235
  /// residing in the high word of the containing object.
2236
  ///
2237
  /// \param OffsetBase - The bit offset of this type in the
2238
  /// containing object.  Some parameters are classified different
2239
  /// depending on whether they straddle an eightbyte boundary.
2240
  ///
2241
  /// \param isNamedArg - Whether the argument in question is a "named"
2242
  /// argument, as used in AMD64-ABI 3.5.7.
2243
  ///
2244
  /// If a word is unused its result will be NoClass; if a type should
2245
  /// be passed in Memory then at least the classification of \arg Lo
2246
  /// will be Memory.
2247
  ///
2248
  /// The \arg Lo class will be NoClass iff the argument is ignored.
2249
  ///
2250
  /// If the \arg Lo class is ComplexX87, then the \arg Hi class will
2251
  /// also be ComplexX87.
2252
  void classify(QualType T, uint64_t OffsetBase, Class &Lo, Class &Hi,
2253
                bool isNamedArg) const;
2254
2255
  llvm::Type *GetByteVectorType(QualType Ty) const;
2256
  llvm::Type *GetSSETypeAtOffset(llvm::Type *IRType,
2257
                                 unsigned IROffset, QualType SourceTy,
2258
                                 unsigned SourceOffset) const;
2259
  llvm::Type *GetINTEGERTypeAtOffset(llvm::Type *IRType,
2260
                                     unsigned IROffset, QualType SourceTy,
2261
                                     unsigned SourceOffset) const;
2262
2263
  /// getIndirectResult - Give a source type \arg Ty, return a suitable result
2264
  /// such that the argument will be returned in memory.
2265
  ABIArgInfo getIndirectReturnResult(QualType Ty) const;
2266
2267
  /// getIndirectResult - Give a source type \arg Ty, return a suitable result
2268
  /// such that the argument will be passed in memory.
2269
  ///
2270
  /// \param freeIntRegs - The number of free integer registers remaining
2271
  /// available.
2272
  ABIArgInfo getIndirectResult(QualType Ty, unsigned freeIntRegs) const;
2273
2274
  ABIArgInfo classifyReturnType(QualType RetTy) const;
2275
2276
  ABIArgInfo classifyArgumentType(QualType Ty, unsigned freeIntRegs,
2277
                                  unsigned &neededInt, unsigned &neededSSE,
2278
                                  bool isNamedArg) const;
2279
2280
  ABIArgInfo classifyRegCallStructType(QualType Ty, unsigned &NeededInt,
2281
                                       unsigned &NeededSSE) const;
2282
2283
  ABIArgInfo classifyRegCallStructTypeImpl(QualType Ty, unsigned &NeededInt,
2284
                                           unsigned &NeededSSE) const;
2285
2286
  bool IsIllegalVectorType(QualType Ty) const;
2287
2288
  /// The 0.98 ABI revision clarified a lot of ambiguities,
2289
  /// unfortunately in ways that were not always consistent with
2290
  /// certain previous compilers.  In particular, platforms which
2291
  /// required strict binary compatibility with older versions of GCC
2292
  /// may need to exempt themselves.
2293
17
  bool honorsRevision0_98() const {
2294
17
    return !getTarget().getTriple().isOSDarwin();
2295
17
  }
2296
2297
  /// GCC classifies <1 x long long> as SSE but some platform ABIs choose to
2298
  /// classify it as INTEGER (for compatibility with older clang compilers).
2299
184
  bool classifyIntegerMMXAsSSE() const {
2300
    // Clang <= 3.8 did not do this.
2301
184
    if (getContext().getLangOpts().getClangABICompat() <=
2302
184
        LangOptions::ClangABI::Ver3_8)
2303
3
      return false;
2304
2305
181
    const llvm::Triple &Triple = getTarget().getTriple();
2306
181
    if (Triple.isOSDarwin() || 
Triple.getOS() == llvm::Triple::PS464
)
2307
122
      return false;
2308
59
    if (Triple.isOSFreeBSD() && 
Triple.getOSMajorVersion() >= 100
)
2309
0
      return false;
2310
59
    return true;
2311
59
  }
2312
2313
  // GCC classifies vectors of __int128 as memory.
2314
11.0k
  bool passInt128VectorsInMem() const {
2315
    // Clang <= 9.0 did not do this.
2316
11.0k
    if (getContext().getLangOpts().getClangABICompat() <=
2317
11.0k
        LangOptions::ClangABI::Ver9)
2318
36
      return false;
2319
2320
10.9k
    const llvm::Triple &T = getTarget().getTriple();
2321
10.9k
    return T.isOSLinux() || 
T.isOSNetBSD()9.97k
;
2322
10.9k
  }
2323
2324
  X86AVXABILevel AVXLevel;
2325
  // Some ABIs (e.g. X32 ABI and Native Client OS) use 32 bit pointers on
2326
  // 64-bit hardware.
2327
  bool Has64BitPointers;
2328
2329
public:
2330
  X86_64ABIInfo(CodeGen::CodeGenTypes &CGT, X86AVXABILevel AVXLevel) :
2331
      SwiftABIInfo(CGT), AVXLevel(AVXLevel),
2332
22.1k
      Has64BitPointers(CGT.getDataLayout().getPointerSize(0) == 8) {
2333
22.1k
  }
2334
2335
89
  bool isPassedUsingAVXType(QualType type) const {
2336
89
    unsigned neededInt, neededSSE;
2337
    // The freeIntRegs argument doesn't matter here.
2338
89
    ABIArgInfo info = classifyArgumentType(type, 0, neededInt, neededSSE,
2339
89
                                           /*isNamedArg*/true);
2340
89
    if (info.isDirect()) {
2341
88
      llvm::Type *ty = info.getCoerceToType();
2342
88
      if (llvm::VectorType *vectorTy = dyn_cast_or_null<llvm::VectorType>(ty))
2343
2
        return vectorTy->getPrimitiveSizeInBits().getFixedSize() > 128;
2344
87
    }
2345
87
    return false;
2346
87
  }
2347
2348
  void computeInfo(CGFunctionInfo &FI) const override;
2349
2350
  Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
2351
                    QualType Ty) const override;
2352
  Address EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr,
2353
                      QualType Ty) const override;
2354
2355
0
  bool has64BitPointers() const {
2356
0
    return Has64BitPointers;
2357
0
  }
2358
2359
  bool shouldPassIndirectlyForSwift(ArrayRef<llvm::Type*> scalars,
2360
302
                                    bool asReturnValue) const override {
2361
302
    return occupiesMoreThan(CGT, scalars, /*total*/ 4);
2362
302
  }
2363
0
  bool isSwiftErrorInRegister() const override {
2364
0
    return true;
2365
0
  }
2366
};
2367
2368
/// WinX86_64ABIInfo - The Windows X86_64 ABI information.
2369
class WinX86_64ABIInfo : public SwiftABIInfo {
2370
public:
2371
  WinX86_64ABIInfo(CodeGen::CodeGenTypes &CGT, X86AVXABILevel AVXLevel)
2372
      : SwiftABIInfo(CGT), AVXLevel(AVXLevel),
2373
369
        IsMingw64(getTarget().getTriple().isWindowsGNUEnvironment()) {}
2374
2375
  void computeInfo(CGFunctionInfo &FI) const override;
2376
2377
  Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
2378
                    QualType Ty) const override;
2379
2380
371
  bool isHomogeneousAggregateBaseType(QualType Ty) const override {
2381
    // FIXME: Assumes vectorcall is in use.
2382
371
    return isX86VectorTypeForVectorCall(getContext(), Ty);
2383
371
  }
2384
2385
  bool isHomogeneousAggregateSmallEnough(const Type *Ty,
2386
408
                                         uint64_t NumMembers) const override {
2387
    // FIXME: Assumes vectorcall is in use.
2388
408
    return isX86VectorCallAggregateSmallEnough(NumMembers);
2389
408
  }
2390
2391
  bool shouldPassIndirectlyForSwift(ArrayRef<llvm::Type *> scalars,
2392
25
                                    bool asReturnValue) const override {
2393
25
    return occupiesMoreThan(CGT, scalars, /*total*/ 4);
2394
25
  }
2395
2396
0
  bool isSwiftErrorInRegister() const override {
2397
0
    return true;
2398
0
  }
2399
2400
private:
2401
  ABIArgInfo classify(QualType Ty, unsigned &FreeSSERegs, bool IsReturnType,
2402
                      bool IsVectorCall, bool IsRegCall) const;
2403
  ABIArgInfo reclassifyHvaArgForVectorCall(QualType Ty, unsigned &FreeSSERegs,
2404
                                           const ABIArgInfo &current) const;
2405
2406
  X86AVXABILevel AVXLevel;
2407
2408
  bool IsMingw64;
2409
};
2410
2411
class X86_64TargetCodeGenInfo : public TargetCodeGenInfo {
2412
public:
2413
  X86_64TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, X86AVXABILevel AVXLevel)
2414
22.1k
      : TargetCodeGenInfo(std::make_unique<X86_64ABIInfo>(CGT, AVXLevel)) {}
2415
2416
89
  const X86_64ABIInfo &getABIInfo() const {
2417
89
    return static_cast<const X86_64ABIInfo&>(TargetCodeGenInfo::getABIInfo());
2418
89
  }
2419
2420
  /// Disable tail call on x86-64. The epilogue code before the tail jump blocks
2421
  /// autoreleaseRV/retainRV and autoreleaseRV/unsafeClaimRV optimizations.
2422
245
  bool markARCOptimizedReturnCallsAsNoTail() const override { return true; }
2423
2424
0
  int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const override {
2425
0
    return 7;
2426
0
  }
2427
2428
  bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
2429
0
                               llvm::Value *Address) const override {
2430
0
    llvm::Value *Eight8 = llvm::ConstantInt::get(CGF.Int8Ty, 8);
2431
2432
    // 0-15 are the 16 integer registers.
2433
    // 16 is %rip.
2434
0
    AssignToArrayRange(CGF.Builder, Address, Eight8, 0, 16);
2435
0
    return false;
2436
0
  }
2437
2438
  llvm::Type* adjustInlineAsmType(CodeGen::CodeGenFunction &CGF,
2439
                                  StringRef Constraint,
2440
812
                                  llvm::Type* Ty) const override {
2441
812
    return X86AdjustInlineAsmType(CGF, Constraint, Ty);
2442
812
  }
2443
2444
  bool isNoProtoCallVariadic(const CallArgList &args,
2445
681
                             const FunctionNoProtoType *fnType) const override {
2446
    // The default CC on x86-64 sets %al to the number of SSA
2447
    // registers used, and GCC sets this when calling an unprototyped
2448
    // function, so we override the default behavior.  However, don't do
2449
    // that when AVX types are involved: the ABI explicitly states it is
2450
    // undefined, and it doesn't work in practice because of how the ABI
2451
    // defines varargs anyway.
2452
681
    if (fnType->getCallConv() == CC_C) {
2453
681
      bool HasAVXType = false;
2454
681
      for (CallArgList::const_iterator
2455
768
             it = args.begin(), ie = args.end(); it != ie; 
++it87
) {
2456
89
        if (getABIInfo().isPassedUsingAVXType(it->Ty)) {
2457
2
          HasAVXType = true;
2458
2
          break;
2459
2
        }
2460
89
      }
2461
2462
681
      if (!HasAVXType)
2463
679
        return true;
2464
2
    }
2465
2466
2
    return TargetCodeGenInfo::isNoProtoCallVariadic(args, fnType);
2467
2
  }
2468
2469
  llvm::Constant *
2470
84
  getUBSanFunctionSignature(CodeGen::CodeGenModule &CGM) const override {
2471
84
    unsigned Sig = (0xeb << 0) | // jmp rel8
2472
84
                   (0x06 << 8) | //           .+0x08
2473
84
                   ('v' << 16) |
2474
84
                   ('2' << 24);
2475
84
    return llvm::ConstantInt::get(CGM.Int32Ty, Sig);
2476
84
  }
2477
2478
  void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
2479
355k
                           CodeGen::CodeGenModule &CGM) const override {
2480
355k
    if (GV->isDeclaration())
2481
198k
      return;
2482
156k
    if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) {
2483
145k
      if (FD->hasAttr<X86ForceAlignArgPointerAttr>()) {
2484
1
        llvm::Function *Fn = cast<llvm::Function>(GV);
2485
1
        Fn->addFnAttr("stackrealign");
2486
1
      }
2487
2488
145k
      addX86InterruptAttrs(FD, GV, CGM);
2489
145k
    }
2490
156k
  }
2491
2492
  void checkFunctionCallABI(CodeGenModule &CGM, SourceLocation CallLoc,
2493
                            const FunctionDecl *Caller,
2494
                            const FunctionDecl *Callee,
2495
                            const CallArgList &Args) const override;
2496
};
2497
2498
static void initFeatureMaps(const ASTContext &Ctx,
2499
                            llvm::StringMap<bool> &CallerMap,
2500
                            const FunctionDecl *Caller,
2501
                            llvm::StringMap<bool> &CalleeMap,
2502
12.0k
                            const FunctionDecl *Callee) {
2503
12.0k
  if (CalleeMap.empty() && 
CallerMap.empty()5.55k
) {
2504
    // The caller is potentially nullptr in the case where the call isn't in a
2505
    // function.  In this case, the getFunctionFeatureMap ensures we just get
2506
    // the TU level setting (since it cannot be modified by 'target'..
2507
5.55k
    Ctx.getFunctionFeatureMap(CallerMap, Caller);
2508
5.55k
    Ctx.getFunctionFeatureMap(CalleeMap, Callee);
2509
5.55k
  }
2510
12.0k
}
2511
2512
static bool checkAVXParamFeature(DiagnosticsEngine &Diag,
2513
                                 SourceLocation CallLoc,
2514
                                 const llvm::StringMap<bool> &CallerMap,
2515
                                 const llvm::StringMap<bool> &CalleeMap,
2516
                                 QualType Ty, StringRef Feature,
2517
12.0k
                                 bool IsArgument) {
2518
12.0k
  bool CallerHasFeat = CallerMap.lookup(Feature);
2519
12.0k
  bool CalleeHasFeat = CalleeMap.lookup(Feature);
2520
12.0k
  if (!CallerHasFeat && 
!CalleeHasFeat62
)
2521
56
    return Diag.Report(CallLoc, diag::warn_avx_calling_convention)
2522
56
           << IsArgument << Ty << Feature;
2523
2524
  // Mixing calling conventions here is very clearly an error.
2525
12.0k
  if (!CallerHasFeat || 
!CalleeHasFeat12.0k
)
2526
6
    return Diag.Report(CallLoc, diag::err_avx_calling_convention)
2527
6
           << IsArgument << Ty << Feature;
2528
2529
  // Else, both caller and callee have the required feature, so there is no need
2530
  // to diagnose.
2531
12.0k
  return false;
2532
12.0k
}
2533
2534
static bool checkAVXParam(DiagnosticsEngine &Diag, ASTContext &Ctx,
2535
                          SourceLocation CallLoc,
2536
                          const llvm::StringMap<bool> &CallerMap,
2537
                          const llvm::StringMap<bool> &CalleeMap, QualType Ty,
2538
12.0k
                          bool IsArgument) {
2539
12.0k
  uint64_t Size = Ctx.getTypeSize(Ty);
2540
12.0k
  if (Size > 256)
2541
5.61k
    return checkAVXParamFeature(Diag, CallLoc, CallerMap, CalleeMap, Ty,
2542
5.61k
                                "avx512f", IsArgument);
2543
2544
6.45k
  if (Size > 128)
2545
6.45k
    return checkAVXParamFeature(Diag, CallLoc, CallerMap, CalleeMap, Ty, "avx",
2546
6.45k
                                IsArgument);
2547
2548
0
  return false;
2549
0
}
2550
2551
void X86_64TargetCodeGenInfo::checkFunctionCallABI(
2552
    CodeGenModule &CGM, SourceLocation CallLoc, const FunctionDecl *Caller,
2553
237k
    const FunctionDecl *Callee, const CallArgList &Args) const {
2554
237k
  llvm::StringMap<bool> CallerMap;
2555
237k
  llvm::StringMap<bool> CalleeMap;
2556
237k
  unsigned ArgIndex = 0;
2557
2558
  // We need to loop through the actual call arguments rather than the the
2559
  // function's parameters, in case this variadic.
2560
390k
  for (const CallArg &Arg : Args) {
2561
    // The "avx" feature changes how vectors >128 in size are passed. "avx512f"
2562
    // additionally changes how vectors >256 in size are passed. Like GCC, we
2563
    // warn when a function is called with an argument where this will change.
2564
    // Unlike GCC, we also error when it is an obvious ABI mismatch, that is,
2565
    // the caller and callee features are mismatched.
2566
    // Unfortunately, we cannot do this diagnostic in SEMA, since the callee can
2567
    // change its ABI with attribute-target after this call.
2568
390k
    if (Arg.getType()->isVectorType() &&
2569
13.3k
        CGM.getContext().getTypeSize(Arg.getType()) > 128) {
2570
7.05k
      initFeatureMaps(CGM.getContext(), CallerMap, Caller, CalleeMap, Callee);
2571
7.05k
      QualType Ty = Arg.getType();
2572
      // The CallArg seems to have desugared the type already, so for clearer
2573
      // diagnostics, replace it with the type in the FunctionDecl if possible.
2574
7.05k
      if (ArgIndex < Callee->getNumParams())
2575
7.02k
        Ty = Callee->getParamDecl(ArgIndex)->getType();
2576
2577
7.05k
      if (checkAVXParam(CGM.getDiags(), CGM.getContext(), CallLoc, CallerMap,
2578
7.05k
                        CalleeMap, Ty, /*IsArgument*/ true))
2579
54
        return;
2580
390k
    }
2581
390k
    ++ArgIndex;
2582
390k
  }
2583
2584
  // Check return always, as we don't have a good way of knowing in codegen
2585
  // whether this value is used, tail-called, etc.
2586
237k
  if (Callee->getReturnType()->isVectorType() &&
2587
8.88k
      CGM.getContext().getTypeSize(Callee->getReturnType()) > 128) {
2588
5.01k
    initFeatureMaps(CGM.getContext(), CallerMap, Caller, CalleeMap, Callee);
2589
5.01k
    checkAVXParam(CGM.getDiags(), CGM.getContext(), CallLoc, CallerMap,
2590
5.01k
                  CalleeMap, Callee->getReturnType(),
2591
5.01k
                  /*IsArgument*/ false);
2592
5.01k
  }
2593
237k
}
2594
2595
75
static std::string qualifyWindowsLibrary(llvm::StringRef Lib) {
2596
  // If the argument does not end in .lib, automatically add the suffix.
2597
  // If the argument contains a space, enclose it in quotes.
2598
  // This matches the behavior of MSVC.
2599
75
  bool Quote = (Lib.find(' ') != StringRef::npos);
2600
71
  std::string ArgStr = Quote ? 
"\""4
: "";
2601
75
  ArgStr += Lib;
2602
75
  if (!Lib.endswith_lower(".lib") && 
!Lib.endswith_lower(".a")63
)
2603
63
    ArgStr += ".lib";
2604
71
  ArgStr += Quote ? 
"\""4
: "";
2605
75
  return ArgStr;
2606
75
}
2607
2608
class WinX86_32TargetCodeGenInfo : public X86_32TargetCodeGenInfo {
2609
public:
2610
  WinX86_32TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT,
2611
        bool DarwinVectorABI, bool RetSmallStructInRegABI, bool Win32StructABI,
2612
        unsigned NumRegisterParameters)
2613
    : X86_32TargetCodeGenInfo(CGT, DarwinVectorABI, RetSmallStructInRegABI,
2614
428
        Win32StructABI, NumRegisterParameters, false) {}
2615
2616
  void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
2617
                           CodeGen::CodeGenModule &CGM) const override;
2618
2619
  void getDependentLibraryOption(llvm::StringRef Lib,
2620
31
                                 llvm::SmallString<24> &Opt) const override {
2621
31
    Opt = "/DEFAULTLIB:";
2622
31
    Opt += qualifyWindowsLibrary(Lib);
2623
31
  }
2624
2625
  void getDetectMismatchOption(llvm::StringRef Name,
2626
                               llvm::StringRef Value,
2627
2
                               llvm::SmallString<32> &Opt) const override {
2628
2
    Opt = "/FAILIFMISMATCH:\"" + Name.str() + "=" + Value.str() + "\"";
2629
2
  }
2630
};
2631
2632
static void addStackProbeTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
2633
17.1k
                                          CodeGen::CodeGenModule &CGM) {
2634
17.1k
  if (llvm::Function *Fn = dyn_cast_or_null<llvm::Function>(GV)) {
2635
2636
14.7k
    if (CGM.getCodeGenOpts().StackProbeSize != 4096)
2637
1
      Fn->addFnAttr("stack-probe-size",
2638
1
                    llvm::utostr(CGM.getCodeGenOpts().StackProbeSize));
2639
14.7k
    if (CGM.getCodeGenOpts().NoStackArgProbe)
2640
4
      Fn->addFnAttr("no-stack-arg-probe");
2641
14.7k
  }
2642
17.1k
}
2643
2644
void WinX86_32TargetCodeGenInfo::setTargetAttributes(
2645
17.5k
    const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {
2646
17.5k
  X86_32TargetCodeGenInfo::setTargetAttributes(D, GV, CGM);
2647
17.5k
  if (GV->isDeclaration())
2648
10.0k
    return;
2649
7.55k
  addStackProbeTargetAttributes(D, GV, CGM);
2650
7.55k
}
2651
2652
class WinX86_64TargetCodeGenInfo : public TargetCodeGenInfo {
2653
public:
2654
  WinX86_64TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT,
2655
                             X86AVXABILevel AVXLevel)
2656
358
      : TargetCodeGenInfo(std::make_unique<WinX86_64ABIInfo>(CGT, AVXLevel)) {}
2657
2658
  void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
2659
                           CodeGen::CodeGenModule &CGM) const override;
2660
2661
0
  int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const override {
2662
0
    return 7;
2663
0
  }
2664
2665
  bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
2666
0
                               llvm::Value *Address) const override {
2667
0
    llvm::Value *Eight8 = llvm::ConstantInt::get(CGF.Int8Ty, 8);
2668
2669
    // 0-15 are the 16 integer registers.
2670
    // 16 is %rip.
2671
0
    AssignToArrayRange(CGF.Builder, Address, Eight8, 0, 16);
2672
0
    return false;
2673
0
  }
2674
2675
  void getDependentLibraryOption(llvm::StringRef Lib,
2676
33
                                 llvm::SmallString<24> &Opt) const override {
2677
33
    Opt = "/DEFAULTLIB:";
2678
33
    Opt += qualifyWindowsLibrary(Lib);
2679
33
  }
2680
2681
  void getDetectMismatchOption(llvm::StringRef Name,
2682
                               llvm::StringRef Value,
2683
4
                               llvm::SmallString<32> &Opt) const override {
2684
4
    Opt = "/FAILIFMISMATCH:\"" + Name.str() + "=" + Value.str() + "\"";
2685
4
  }
2686
};
2687
2688
void WinX86_64TargetCodeGenInfo::setTargetAttributes(
2689
19.2k
    const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {
2690
19.2k
  TargetCodeGenInfo::setTargetAttributes(D, GV, CGM);
2691
19.2k
  if (GV->isDeclaration())
2692
10.4k
    return;
2693
8.76k
  if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) {
2694
7.69k
    if (FD->hasAttr<X86ForceAlignArgPointerAttr>()) {
2695
0
      llvm::Function *Fn = cast<llvm::Function>(GV);
2696
0
      Fn->addFnAttr("stackrealign");
2697
0
    }
2698
2699
7.69k
    addX86InterruptAttrs(FD, GV, CGM);
2700
7.69k
  }
2701
2702
8.76k
  addStackProbeTargetAttributes(D, GV, CGM);
2703
8.76k
}
2704
}
2705
2706
void X86_64ABIInfo::postMerge(unsigned AggregateSize, Class &Lo,
2707
12.4k
                              Class &Hi) const {
2708
  // AMD64-ABI 3.2.3p2: Rule 5. Then a post merger cleanup is done:
2709
  //
2710
  // (a) If one of the classes is Memory, the whole argument is passed in
2711
  //     memory.
2712
  //
2713
  // (b) If X87UP is not preceded by X87, the whole argument is passed in
2714
  //     memory.
2715
  //
2716
  // (c) If the size of the aggregate exceeds two eightbytes and the first
2717
  //     eightbyte isn't SSE or any other eightbyte isn't SSEUP, the whole
2718
  //     argument is passed in memory. NOTE: This is necessary to keep the
2719
  //     ABI working for processors that don't support the __m256 type.
2720
  //
2721
  // (d) If SSEUP is not preceded by SSE or SSEUP, it is converted to SSE.
2722
  //
2723
  // Some of these are enforced by the merging logic.  Others can arise
2724
  // only with unions; for example:
2725
  //   union { _Complex double; unsigned; }
2726
  //
2727
  // Note that clauses (b) and (c) were added in 0.98.
2728
  //
2729
12.4k
  if (Hi == Memory)
2730
0
    Lo = Memory;
2731
12.4k
  if (Hi == X87Up && 
Lo != X87348
&&
honorsRevision0_98()17
)
2732
15
    Lo = Memory;
2733
12.4k
  if (AggregateSize > 128 && 
(323
Lo != SSE323
||
Hi != SSEUp24
))
2734
299
    Lo = Memory;
2735
12.4k
  if (Hi == SSEUp && 
Lo != SSE33
)
2736
1
    Hi = SSE;
2737
12.4k
}
2738
2739
18.9k
X86_64ABIInfo::Class X86_64ABIInfo::merge(Class Accum, Class Field) {
2740
  // AMD64-ABI 3.2.3p2: Rule 4. Each field of an object is
2741
  // classified recursively so that always two fields are
2742
  // considered. The resulting class is calculated according to
2743
  // the classes of the fields in the eightbyte:
2744
  //
2745
  // (a) If both classes are equal, this is the resulting class.
2746
  //
2747
  // (b) If one of the classes is NO_CLASS, the resulting class is
2748
  // the other class.
2749
  //
2750
  // (c) If one of the classes is MEMORY, the result is the MEMORY
2751
  // class.
2752
  //
2753
  // (d) If one of the classes is INTEGER, the result is the
2754
  // INTEGER.
2755
  //
2756
  // (e) If one of the classes is X87, X87UP, COMPLEX_X87 class,
2757
  // MEMORY is used as class.
2758
  //
2759
  // (f) Otherwise class SSE is used.
2760
2761
  // Accum should never be memory (we should have returned) or
2762
  // ComplexX87 (because this cannot be passed in a structure).
2763
18.9k
  assert((Accum != Memory && Accum != ComplexX87) &&
2764
18.9k
         "Invalid accumulated classification during merge.");
2765
18.9k
  if (Accum == Field || 
Field == NoClass10.0k
)
2766
10.4k
    return Accum;
2767
8.53k
  if (Field == Memory)
2768
15
    return Memory;
2769
8.52k
  if (Accum == NoClass)
2770
8.41k
    return Field;
2771
101
  if (Accum == Integer || 
Field == Integer73
)
2772
98
    return Integer;
2773
3
  if (Field == X87 || Field == X87Up || Field == ComplexX87 ||
2774
3
      Accum == X87 || 
Accum == X87Up0
)
2775
3
    return Memory;
2776
0
  return SSE;
2777
0
}
2778
2779
void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase,
2780
393k
                             Class &Lo, Class &Hi, bool isNamedArg) const {
2781
  // FIXME: This code can be simplified by introducing a simple value class for
2782
  // Class pairs with appropriate constructor methods for the various
2783
  // situations.
2784
2785
  // FIXME: Some of the split computations are wrong; unaligned vectors
2786
  // shouldn't be passed in registers for example, so there is no chance they
2787
  // can straddle an eightbyte. Verify & simplify.
2788
2789
393k
  Lo = Hi = NoClass;
2790
2791
392k
  Class &Current = OffsetBase < 64 ? Lo : 
Hi1.62k
;
2792
393k
  Current = Memory;
2793
2794
393k
  if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
2795
150k
    BuiltinType::Kind k = BT->getKind();
2796
2797
150k
    if (k == BuiltinType::Void) {
2798
57.0k
      Current = NoClass;
2799
93.1k
    } else if (k == BuiltinType::Int128 || 
k == BuiltinType::UInt12893.1k
) {
2800
26
      Lo = Integer;
2801
26
      Hi = Integer;
2802
93.1k
    } else if (k >= BuiltinType::Bool && 
k <= BuiltinType::LongLong93.1k
) {
2803
88.1k
      Current = Integer;
2804
4.99k
    } else if (k == BuiltinType::Float || 
k == BuiltinType::Double2.93k
) {
2805
3.71k
      Current = SSE;
2806
1.27k
    } else if (k == BuiltinType::LongDouble) {
2807
980
      const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();
2808
980
      if (LDF == &llvm::APFloat::IEEEquad()) {
2809
40
        Lo = SSE;
2810
40
        Hi = SSEUp;
2811
940
      } else if (LDF == &llvm::APFloat::x87DoubleExtended()) {
2812
918
        Lo = X87;
2813
918
        Hi = X87Up;
2814
22
      } else if (LDF == &llvm::APFloat::IEEEdouble()) {
2815
22
        Current = SSE;
2816
22
      } else
2817
0
        llvm_unreachable("unexpected long double representation!");
2818
980
    }
2819
    // FIXME: _Decimal32 and _Decimal64 are SSE.
2820
    // FIXME: _float128 and _Decimal128 are (SSE, SSEUp).
2821
150k
    return;
2822
243k
  }
2823
2824
243k
  if (const EnumType *ET = Ty->getAs<EnumType>()) {
2825
    // Classify the underlying integer type.
2826
926
    classify(ET->getDecl()->getIntegerType(), OffsetBase, Lo, Hi, isNamedArg);
2827
926
    return;
2828
926
  }
2829
2830
242k
  if (Ty->hasPointerRepresentation()) {
2831
223k
    Current = Integer;
2832
223k
    return;
2833
223k
  }
2834
2835
19.5k
  if (Ty->isMemberPointerType()) {
2836
122
    if (Ty->isMemberFunctionPointerType()) {
2837
74
      if (Has64BitPointers) {
2838
        // If Has64BitPointers, this is an {i64, i64}, so classify both
2839
        // Lo and Hi now.
2840
66
        Lo = Hi = Integer;
2841
8
      } else {
2842
        // Otherwise, with 32-bit pointers, this is an {i32, i32}. If that
2843
        // straddles an eightbyte boundary, Hi should be classified as well.
2844
8
        uint64_t EB_FuncPtr = (OffsetBase) / 64;
2845
8
        uint64_t EB_ThisAdj = (OffsetBase + 64 - 1) / 64;
2846
8
        if (EB_FuncPtr != EB_ThisAdj) {
2847
2
          Lo = Hi = Integer;
2848
6
        } else {
2849
6
          Current = Integer;
2850
6
        }
2851
8
      }
2852
48
    } else {
2853
48
      Current = Integer;
2854
48
    }
2855
122
    return;
2856
122
  }
2857
2858
19.4k
  if (const VectorType *VT = Ty->getAs<VectorType>()) {
2859
5.87k
    uint64_t Size = getContext().getTypeSize(VT);
2860
5.87k
    if (Size == 1 || Size == 8 || 
Size == 165.87k
||
Size == 325.87k
) {
2861
      // gcc passes the following as integer:
2862
      // 4 bytes - <4 x char>, <2 x short>, <1 x int>, <1 x float>
2863
      // 2 bytes - <2 x char>, <1 x short>
2864
      // 1 byte  - <1 x char>
2865
5
      Current = Integer;
2866
2867
      // If this type crosses an eightbyte boundary, it should be
2868
      // split.
2869
5
      uint64_t EB_Lo = (OffsetBase) / 64;
2870
5
      uint64_t EB_Hi = (OffsetBase + Size - 1) / 64;
2871
5
      if (EB_Lo != EB_Hi)
2872
0
        Hi = Lo;
2873
5.87k
    } else if (Size == 64) {
2874
184
      QualType ElementType = VT->getElementType();
2875
2876
      // gcc passes <1 x double> in memory. :(
2877
184
      if (ElementType->isSpecificBuiltinType(BuiltinType::Double))
2878
0
        return;
2879
2880
      // gcc passes <1 x long long> as SSE but clang used to unconditionally
2881
      // pass them as integer.  For platforms where clang is the de facto
2882
      // platform compiler, we must continue to use integer.
2883
184
      if (!classifyIntegerMMXAsSSE() &&
2884
125
          (ElementType->isSpecificBuiltinType(BuiltinType::LongLong) ||
2885
45
           ElementType->isSpecificBuiltinType(BuiltinType::ULongLong) ||
2886
45
           ElementType->isSpecificBuiltinType(BuiltinType::Long) ||
2887
45
           ElementType->isSpecificBuiltinType(BuiltinType::ULong)))
2888
80
        Current = Integer;
2889
104
      else
2890
104
        Current = SSE;
2891
2892
      // If this type crosses an eightbyte boundary, it should be
2893
      // split.
2894
184
      if (OffsetBase && 
OffsetBase != 640
)
2895
0
        Hi = Lo;
2896
5.68k
    } else if (Size == 128 ||
2897
5.49k
               
(3.42k
isNamedArg3.42k
&&
Size <= getNativeVectorSizeForAVXABI(AVXLevel)3.39k
)) {
2898
5.49k
      QualType ElementType = VT->getElementType();
2899
2900
      // gcc passes 256 and 512 bit <X x __int128> vectors in memory. :(
2901
5.49k
      if (passInt128VectorsInMem() && 
Size != 128516
&&
2902
255
          (ElementType->isSpecificBuiltinType(BuiltinType::Int128) ||
2903
255
           ElementType->isSpecificBuiltinType(BuiltinType::UInt128)))
2904
6
        return;
2905
2906
      // Arguments of 256-bits are split into four eightbyte chunks. The
2907
      // least significant one belongs to class SSE and all the others to class
2908
      // SSEUP. The original Lo and Hi design considers that types can't be
2909
      // greater than 128-bits, so a 64-bit split in Hi and Lo makes sense.
2910
      // This design isn't correct for 256-bits, but since there're no cases
2911
      // where the upper parts would need to be inspected, avoid adding
2912
      // complexity and just consider Hi to match the 64-256 part.
2913
      //
2914
      // Note that per 3.5.7 of AMD64-ABI, 256-bit args are only passed in
2915
      // registers if they are "named", i.e. not part of the "..." of a
2916
      // variadic function.
2917
      //
2918
      // Similarly, per 3.2.3. of the AVX512 draft, 512-bits ("named") args are
2919
      // split into eight eightbyte chunks, one SSE and seven SSEUP.
2920
5.49k
      Lo = SSE;
2921
5.49k
      Hi = SSEUp;
2922
5.49k
    }
2923
5.86k
    return;
2924
13.5k
  }
2925
2926
13.5k
  if (const ComplexType *CT = Ty->getAs<ComplexType>()) {
2927
390
    QualType ET = getContext().getCanonicalType(CT->getElementType());
2928
2929
390
    uint64_t Size = getContext().getTypeSize(Ty);
2930
390
    if (ET->isIntegralOrEnumerationType()) {
2931
36
      if (Size <= 64)
2932
32
        Current = Integer;
2933
4
      else if (Size <= 128)
2934
4
        Lo = Hi = Integer;
2935
354
    } else if (ET == getContext().FloatTy) {
2936
134
      Current = SSE;
2937
220
    } else if (ET == getContext().DoubleTy) {
2938
111
      Lo = Hi = SSE;
2939
109
    } else if (ET == getContext().LongDoubleTy) {
2940
108
      const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();
2941
108
      if (LDF == &llvm::APFloat::IEEEquad())
2942
8
        Current = Memory;
2943
100
      else if (LDF == &llvm::APFloat::x87DoubleExtended())
2944
92
        Current = ComplexX87;
2945
8
      else if (LDF == &llvm::APFloat::IEEEdouble())
2946
8
        Lo = Hi = SSE;
2947
8
      else
2948
0
        llvm_unreachable("unexpected long double representation!");
2949
108
    }
2950
2951
    // If this complex type crosses an eightbyte boundary then it
2952
    // should be split.
2953
390
    uint64_t EB_Real = (OffsetBase) / 64;
2954
390
    uint64_t EB_Imag = (OffsetBase + getContext().getTypeSize(ET)) / 64;
2955
390
    if (Hi == NoClass && 
EB_Real != EB_Imag267
)
2956
104
      Hi = Lo;
2957
2958
390
    return;
2959
13.1k
  }
2960
2961
13.1k
  if (const auto *EITy = Ty->getAs<ExtIntType>()) {
2962
78
    if (EITy->getNumBits() <= 64)
2963
52
      Current = Integer;
2964
26
    else if (EITy->getNumBits() <= 128)
2965
16
      Lo = Hi = Integer;
2966
    // Larger values need to get passed in memory.
2967
78
    return;
2968
78
  }
2969
2970
13.0k
  if (const ConstantArrayType *AT = getContext().getAsConstantArrayType(Ty)) {
2971
    // Arrays are treated like structures.
2972
2973
54
    uint64_t Size = getContext().getTypeSize(Ty);
2974
2975
    // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger
2976
    // than eight eightbytes, ..., it has class MEMORY.
2977
54
    if (Size > 512)
2978
0
      return;
2979
2980
    // AMD64-ABI 3.2.3p2: Rule 1. If ..., or it contains unaligned
2981
    // fields, it has class MEMORY.
2982
    //
2983
    // Only need to check alignment of array base.
2984
54
    if (OffsetBase % getContext().getTypeAlign(AT->getElementType()))
2985
0
      return;
2986
2987
    // Otherwise implement simplified merge. We could be smarter about
2988
    // this, but it isn't worth it and would be harder to verify.
2989
54
    Current = NoClass;
2990
54
    uint64_t EltSize = getContext().getTypeSize(AT->getElementType());
2991
54
    uint64_t ArraySize = AT->getSize().getZExtValue();
2992
2993
    // The only case a 256-bit wide vector could be used is when the array
2994
    // contains a single 256-bit element. Since Lo and Hi logic isn't extended
2995
    // to work for sizes wider than 128, early check and fallback to memory.
2996
    //
2997
54
    if (Size > 128 &&
2998
15
        (Size != EltSize || 
Size > getNativeVectorSizeForAVXABI(AVXLevel)4
))
2999
11
      return;
3000
3001
196
    
for (uint64_t i=0, Offset=OffsetBase; 43
i<ArraySize;
++i, Offset += EltSize153
) {
3002
153
      Class FieldLo, FieldHi;
3003
153
      classify(AT->getElementType(), Offset, FieldLo, FieldHi, isNamedArg);
3004
153
      Lo = merge(Lo, FieldLo);
3005
153
      Hi = merge(Hi, FieldHi);
3006
153
      if (Lo == Memory || Hi == Memory)
3007
0
        break;
3008
153
    }
3009
3010
43
    postMerge(Size, Lo, Hi);
3011
43
    assert((Hi != SSEUp || Lo == SSE) && "Invalid SSEUp array classification.");
3012
43
    return;
3013
43
  }
3014
3015
13.0k
  if (const RecordType *RT = Ty->getAs<RecordType>()) {
3016
12.8k
    uint64_t Size = getContext().getTypeSize(Ty);
3017
3018
    // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger
3019
    // than eight eightbytes, ..., it has class MEMORY.
3020
12.8k
    if (Size > 512)
3021
131
      return;
3022
3023
    // AMD64-ABI 3.2.3p2: Rule 2. If a C++ object has either a non-trivial
3024
    // copy constructor or a non-trivial destructor, it is passed by invisible
3025
    // reference.
3026
12.7k
    if (getRecordArgABI(RT, getCXXABI()))
3027
387
      return;
3028
3029
12.3k
    const RecordDecl *RD = RT->getDecl();
3030
3031
    // Assume variable sized types are passed in memory.
3032
12.3k
    if (RD->hasFlexibleArrayMember())
3033
0
      return;
3034
3035
12.3k
    const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
3036
3037
    // Reset Lo class, this will be recomputed.
3038
12.3k
    Current = NoClass;
3039
3040
    // If this is a C++ record, classify the bases first.
3041
12.3k
    if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
3042
1.12k
      for (const auto &I : CXXRD->bases()) {
3043
1.12k
        assert(!I.isVirtual() && !I.getType()->isDependentType() &&
3044
1.12k
               "Unexpected base class!");
3045
1.12k
        const auto *Base =
3046
1.12k
            cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
3047
3048
        // Classify this field.
3049
        //
3050
        // AMD64-ABI 3.2.3p2: Rule 3. If the size of the aggregate exceeds a
3051
        // single eightbyte, each is classified separately. Each eightbyte gets
3052
        // initialized to class NO_CLASS.
3053
1.12k
        Class FieldLo, FieldHi;
3054
1.12k
        uint64_t Offset =
3055
1.12k
          OffsetBase + getContext().toBits(Layout.getBaseClassOffset(Base));
3056
1.12k
        classify(I.getType(), Offset, FieldLo, FieldHi, isNamedArg);
3057
1.12k
        Lo = merge(Lo, FieldLo);
3058
1.12k
        Hi = merge(Hi, FieldHi);
3059
1.12k
        if (Lo == Memory || 
Hi == Memory1.12k
) {
3060
6
          postMerge(Size, Lo, Hi);
3061
6
          return;
3062
6
        }
3063
1.12k
      }
3064
11.8k
    }
3065
3066
    // Classify the fields one at a time, merging the results.
3067
12.3k
    unsigned idx = 0;
3068
12.3k
    bool UseClang11Compat = getContext().getLangOpts().getClangABICompat() <=
3069
12.3k
                                LangOptions::ClangABI::Ver11 ||
3070
12.3k
                            getContext().getTargetInfo().getTriple().isPS4();
3071
12.3k
    bool IsUnion = RT->isUnionType() && 
!UseClang11Compat61
;
3072
3073
12.3k
    for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
3074
20.5k
           i != e; 
++i, ++idx8.20k
) {
3075
8.52k
      uint64_t Offset = OffsetBase + Layout.getFieldOffset(idx);
3076
8.52k
      bool BitField = i->isBitField();
3077
3078
      // Ignore padding bit-fields.
3079
8.52k
      if (BitField && 
i->isUnnamedBitfield()42
)
3080
12
        continue;
3081
3082
      // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger than
3083
      // eight eightbytes, or it contains unaligned fields, it has class MEMORY.
3084
      //
3085
      // The only case a 256-bit or a 512-bit wide vector could be used is when
3086
      // the struct contains a single 256-bit or 512-bit element. Early check
3087
      // and fallback to memory.
3088
      //
3089
      // FIXME: Extended the Lo and Hi logic properly to work for size wider
3090
      // than 128.
3091
8.51k
      if (Size > 128 &&
3092
320
          ((!IsUnion && 
Size != getContext().getTypeSize(i->getType())309
) ||
3093
287
           
Size > getNativeVectorSizeForAVXABI(AVXLevel)84
)) {
3094
287
        Lo = Memory;
3095
287
        postMerge(Size, Lo, Hi);
3096
287
        return;
3097
287
      }
3098
      // Note, skip this test for bit-fields, see below.
3099
8.22k
      if (!BitField && 
Offset % getContext().getTypeAlign(i->getType())8.19k
) {
3100
20
        Lo = Memory;
3101
20
        postMerge(Size, Lo, Hi);
3102
20
        return;
3103
20
      }
3104
3105
      // Classify this field.
3106
      //
3107
      // AMD64-ABI 3.2.3p2: Rule 3. If the size of the aggregate
3108
      // exceeds a single eightbyte, each is classified
3109
      // separately. Each eightbyte gets initialized to class
3110
      // NO_CLASS.
3111
8.20k
      Class FieldLo, FieldHi;
3112
3113
      // Bit-fields require special handling, they do not force the
3114
      // structure to be passed in memory even if unaligned, and
3115
      // therefore they can straddle an eightbyte.
3116
8.20k
      if (BitField) {
3117
30
        assert(!i->isUnnamedBitfield());
3118
30
        uint64_t Offset = OffsetBase + Layout.getFieldOffset(idx);
3119
30
        uint64_t Size = i->getBitWidthValue(getContext());
3120
3121
30
        uint64_t EB_Lo = Offset / 64;
3122
30
        uint64_t EB_Hi = (Offset + Size - 1) / 64;
3123
3124
30
        if (EB_Lo) {
3125
0
          assert(EB_Hi == EB_Lo && "Invalid classification, type > 16 bytes.");
3126
0
          FieldLo = NoClass;
3127
0
          FieldHi = Integer;
3128
30
        } else {
3129
30
          FieldLo = Integer;
3130
30
          FieldHi = EB_Hi ? 
Integer0
: NoClass;
3131
30
        }
3132
30
      } else
3133
8.17k
        classify(i->getType(), Offset, FieldLo, FieldHi, isNamedArg);
3134
8.20k
      Lo = merge(Lo, FieldLo);
3135
8.20k
      Hi = merge(Hi, FieldHi);
3136
8.20k
      if (Lo == Memory || 
Hi == Memory8.19k
)
3137
12
        break;
3138
8.20k
    }
3139
3140
12.0k
    postMerge(Size, Lo, Hi);
3141
12.0k
  }
3142
13.0k
}
3143
3144
433
ABIArgInfo X86_64ABIInfo::getIndirectReturnResult(QualType Ty) const {
3145
  // If this is a scalar LLVM value then assume LLVM will pass it in the right
3146
  // place naturally.
3147
433
  if (!isAggregateTypeForABI(Ty)) {
3148
    // Treat an enum type as its underlying type.
3149
219
    if (const EnumType *EnumTy = Ty->getAs<EnumType>())
3150
0
      Ty = EnumTy->getDecl()->getIntegerType();
3151
3152
219
    if (Ty->isExtIntType())
3153
3
      return getNaturalAlignIndirect(Ty);
3154
3155
216
    return (isPromotableIntegerTypeForABI(Ty) ? 
ABIArgInfo::getExtend(Ty)0
3156
216
                                              : ABIArgInfo::getDirect());
3157
216
  }
3158
3159
214
  return getNaturalAlignIndirect(Ty);
3160
214
}
3161
3162
2.78k
bool X86_64ABIInfo::IsIllegalVectorType(QualType Ty) const {
3163
2.78k
  if (const VectorType *VecTy = Ty->getAs<VectorType>()) {
3164
138
    uint64_t Size = getContext().getTypeSize(VecTy);
3165
138
    unsigned LargestVector = getNativeVectorSizeForAVXABI(AVXLevel);
3166
138
    if (Size <= 64 || 
Size > LargestVector132
)
3167
117
      return true;
3168
21
    QualType EltTy = VecTy->getElementType();
3169
21
    if (passInt128VectorsInMem() &&
3170
9
        (EltTy->isSpecificBuiltinType(BuiltinType::Int128) ||
3171
9
         EltTy->isSpecificBuiltinType(BuiltinType::UInt128)))
3172
6
      return true;
3173
2.65k
  }
3174
3175
2.65k
  return false;
3176
2.65k
}
3177
3178
ABIArgInfo X86_64ABIInfo::getIndirectResult(QualType Ty,
3179
3.51k
                                            unsigned freeIntRegs) const {
3180
  // If this is a scalar LLVM value then assume LLVM will pass it in the right
3181
  // place naturally.
3182
  //
3183
  // This assumption is optimistic, as there could be free registers available
3184
  // when we need to pass this argument in memory, and LLVM could try to pass
3185
  // the argument in the free register. This does not seem to happen currently,
3186
  // but this code would be much safer if we could mark the argument with
3187
  // 'onstack'. See PR12193.
3188
3.51k
  if (!isAggregateTypeForABI(Ty) && 
!IsIllegalVectorType(Ty)2.78k
&&
3189
2.65k
      !Ty->isExtIntType()) {
3190
    // Treat an enum type as its underlying type.
3191
2.65k
    if (const EnumType *EnumTy = Ty->getAs<EnumType>())
3192
0
      Ty = EnumTy->getDecl()->getIntegerType();
3193
3194
286
    return (isPromotableIntegerTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty)
3195
2.36k
                                              : ABIArgInfo::getDirect());
3196
2.65k
  }
3197
3198
859
  if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))
3199
397
    return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
3200
3201
  // Compute the byval alignment. We specify the alignment of the byval in all
3202
  // cases so that the mid-level optimizer knows the alignment of the byval.
3203
462
  unsigned Align = std::max(getContext().getTypeAlign(Ty) / 8, 8U);
3204
3205
  // Attempt to avoid passing indirect results using byval when possible. This
3206
  // is important for good codegen.
3207
  //
3208
  // We do this by coercing the value into a scalar type which the backend can
3209
  // handle naturally (i.e., without using byval).
3210
  //
3211
  // For simplicity, we currently only do this when we have exhausted all of the
3212
  // free integer registers. Doing this when there are free integer registers
3213
  // would require more care, as we would have to ensure that the coerced value
3214
  // did not claim the unused register. That would require either reording the
3215
  // arguments to the function (so that any subsequent inreg values came first),
3216
  // or only doing this optimization when there were no following arguments that
3217
  // might be inreg.
3218
  //
3219
  // We currently expect it to be rare (particularly in well written code) for
3220
  // arguments to be passed on the stack when there are still free integer
3221
  // registers available (this would typically imply large structs being passed
3222
  // by value), so this seems like a fair tradeoff for now.
3223
  //
3224
  // We can revisit this if the backend grows support for 'onstack' parameter
3225
  // attributes. See PR12193.
3226
462
  if (freeIntRegs == 0) {
3227
32
    uint64_t Size = getContext().getTypeSize(Ty);
3228
3229
    // If this type fits in an eightbyte, coerce it into the matching integral
3230
    // type, which will end up on the stack (with alignment 8).
3231
32
    if (Align == 8 && 
Size <= 6422
)
3232
4
      return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(),
3233
4
                                                          Size));
3234
458
  }
3235
3236
458
  return ABIArgInfo::getIndirect(CharUnits::fromQuantity(Align));
3237
458
}
3238
3239
/// The ABI specifies that a value should be passed in a full vector XMM/YMM
3240
/// register. Pick an LLVM IR type that will be passed as a vector register.
3241
5.53k
llvm::Type *X86_64ABIInfo::GetByteVectorType(QualType Ty) const {
3242
  // Wrapper structs/arrays that only contain vectors are passed just like
3243
  // vectors; strip them off if present.
3244
5.53k
  if (const Type *InnerTy = isSingleElementStruct(Ty, getContext()))
3245
25
    Ty = QualType(InnerTy, 0);
3246
3247
5.53k
  llvm::Type *IRType = CGT.ConvertType(Ty);
3248
5.53k
  if (isa<llvm::VectorType>(IRType)) {
3249
    // Don't pass vXi128 vectors in their native type, the backend can't
3250
    // legalize them.
3251
5.48k
    if (passInt128VectorsInMem() &&
3252
507
        cast<llvm::VectorType>(IRType)->getElementType()->isIntegerTy(128)) {
3253
      // Use a vXi64 vector.
3254
6
      uint64_t Size = getContext().getTypeSize(Ty);
3255
6
      return llvm::FixedVectorType::get(llvm::Type::getInt64Ty(getVMContext()),
3256
6
                                        Size / 64);
3257
6
    }
3258
3259
5.48k
    return IRType;
3260
5.48k
  }
3261
3262
43
  if (IRType->getTypeID() == llvm::Type::FP128TyID)
3263
40
    return IRType;
3264
3265
  // We couldn't find the preferred IR vector type for 'Ty'.
3266
3
  uint64_t Size = getContext().getTypeSize(Ty);
3267
3
  assert((Size == 128 || Size == 256 || Size == 512) && "Invalid type found!");
3268
3269
3270
  // Return a LLVM IR vector type based on the size of 'Ty'.
3271
3
  return llvm::FixedVectorType::get(llvm::Type::getDoubleTy(getVMContext()),
3272
3
                                    Size / 64);
3273
3
}
3274
3275
/// BitsContainNoUserData - Return true if the specified [start,end) bit range
3276
/// is known to either be off the end of the specified type or being in
3277
/// alignment padding.  The user type specified is known to be at most 128 bits
3278
/// in size, and have passed through X86_64ABIInfo::classify with a successful
3279
/// classification that put one of the two halves in the INTEGER class.
3280
///
3281
/// It is conservatively correct to return false.
3282
static bool BitsContainNoUserData(QualType Ty, unsigned StartBit,
3283
77.0k
                                  unsigned EndBit, ASTContext &Context) {
3284
  // If the bytes being queried are off the end of the type, there is no user
3285
  // data hiding here.  This handles analysis of builtins, vectors and other
3286
  // types that don't contain interesting padding.
3287
77.0k
  unsigned TySize = (unsigned)Context.getTypeSize(Ty);
3288
77.0k
  if (TySize <= StartBit)
3289
67.5k
    return true;
3290
3291
9.49k
  if (const ConstantArrayType *AT = Context.getAsConstantArrayType(Ty)) {
3292
32
    unsigned EltSize = (unsigned)Context.getTypeSize(AT->getElementType());
3293
32
    unsigned NumElts = (unsigned)AT->getSize().getZExtValue();
3294
3295
    // Check each element to see if the element overlaps with the queried range.
3296
89
    for (unsigned i = 0; i != NumElts; 
++i57
) {
3297
      // If the element is after the span we care about, then we're done..
3298
86
      unsigned EltOffset = i*EltSize;
3299
86
      if (EltOffset >= EndBit) 
break0
;
3300
3301
86
      unsigned EltStart = EltOffset < StartBit ? 
StartBit-EltOffset64
:
022
;
3302
86
      if (!BitsContainNoUserData(AT->getElementType(), EltStart,
3303
86
                                 EndBit-EltOffset, Context))
3304
29
        return false;
3305
86
    }
3306
    // If it overlaps no elements, then it is safe to process as padding.
3307
3
    return true;
3308
9.45k
  }
3309
3310
9.45k
  if (const RecordType *RT = Ty->getAs<RecordType>()) {
3311
1.20k
    const RecordDecl *RD = RT->getDecl();
3312
1.20k
    const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
3313
3314
    // If this is a C++ record, check the bases first.
3315
1.20k
    if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
3316
29
      for (const auto &I : CXXRD->bases()) {
3317
29
        assert(!I.isVirtual() && !I.getType()->isDependentType() &&
3318
29
               "Unexpected base class!");
3319
29
        const auto *Base =
3320
29
            cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
3321
3322
        // If the base is after the span we care about, ignore it.
3323
29
        unsigned BaseOffset = Context.toBits(Layout.getBaseClassOffset(Base));
3324
29
        if (BaseOffset >= EndBit) 
continue0
;
3325
3326
29
        unsigned BaseStart = BaseOffset < StartBit ? 
StartBit-BaseOffset26
:
03
;
3327
29
        if (!BitsContainNoUserData(I.getType(), BaseStart,
3328
29
                                   EndBit-BaseOffset, Context))
3329
24
          return false;
3330
29
      }
3331
927
    }
3332
3333
    // Verify that no field has data that overlaps the region of interest.  Yes
3334
    // this could be sped up a lot by being smarter about queried fields,
3335
    // however we're only looking at structs up to 16 bytes, so we don't care
3336
    // much.
3337
1.18k
    unsigned idx = 0;
3338
1.18k
    for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
3339
2.59k
         i != e; 
++i, ++idx1.41k
) {
3340
2.39k
      unsigned FieldOffset = (unsigned)Layout.getFieldOffset(idx);
3341
3342
      // If we found a field after the region we care about, then we're done.
3343
2.39k
      if (FieldOffset >= EndBit) 
break140
;
3344
3345
2.25k
      unsigned FieldStart = FieldOffset < StartBit ? 
StartBit-FieldOffset1.67k
:
0587
;
3346
2.25k
      if (!BitsContainNoUserData(i->getType(), FieldStart, EndBit-FieldOffset,
3347
2.25k
                                 Context))
3348
840
        return false;
3349
2.25k
    }
3350
3351
    // If nothing in this record overlapped the area of interest, then we're
3352
    // clean.
3353
342
    return true;
3354
8.25k
  }
3355
3356
8.25k
  return false;
3357
8.25k
}
3358
3359
/// ContainsFloatAtOffset - Return true if the specified LLVM IR type has a
3360
/// float member at the specified offset.  For example, {int,{float}} has a
3361
/// float at offset 4.  It is conservatively correct for this routine to return
3362
/// false.
3363
static bool ContainsFloatAtOffset(llvm::Type *IRType, unsigned IROffset,
3364
9.18k
                                  const llvm::DataLayout &TD) {
3365
  // Base case if we find a float.
3366
9.18k
  if (IROffset == 0 && 
IRType->isFloatTy()8.63k
)
3367
560
    return true;
3368
3369
  // If this is a struct, recurse into the field at the specified offset.
3370
8.62k
  if (llvm::StructType *STy = dyn_cast<llvm::StructType>(IRType)) {
3371
1.06k
    const llvm::StructLayout *SL = TD.getStructLayout(STy);
3372
1.06k
    unsigned Elt = SL->getElementContainingOffset(IROffset);
3373
1.06k
    IROffset -= SL->getElementOffset(Elt);
3374
1.06k
    return ContainsFloatAtOffset(STy->getElementType(Elt), IROffset, TD);
3375
1.06k
  }
3376
3377
  // If this is an array, recurse into the field at the specified offset.
3378
7.56k
  if (llvm::ArrayType *ATy = dyn_cast<llvm::ArrayType>(IRType)) {
3379
17
    llvm::Type *EltTy = ATy->getElementType();
3380
17
    unsigned EltSize = TD.getTypeAllocSize(EltTy);
3381
17
    IROffset -= IROffset/EltSize*EltSize;
3382
17
    return ContainsFloatAtOffset(EltTy, IROffset, TD);
3383
17
  }
3384
3385
7.54k
  return false;
3386
7.54k
}
3387
3388
3389
/// GetSSETypeAtOffset - Return a type that will be passed by the backend in the
3390
/// low 8 bytes of an XMM register, corresponding to the SSE class.
3391
llvm::Type *X86_64ABIInfo::
3392
GetSSETypeAtOffset(llvm::Type *IRType, unsigned IROffset,
3393
9.50k
                   QualType SourceTy, unsigned SourceOffset) const {
3394
  // The only three choices we have are either double, <2 x float>, or float. We
3395
  // pass as float if the last 4 bytes is just padding.  This happens for
3396
  // structs that contain 3 floats.
3397
9.50k
  if (BitsContainNoUserData(SourceTy, SourceOffset*8+32,
3398
9.50k
                            SourceOffset*8+64, getContext()))
3399
1.67k
    return llvm::Type::getFloatTy(getVMContext());
3400
3401
  // We want to pass as <2 x float> if the LLVM IR type contains a float at
3402
  // offset+0 and offset+4.  Walk the LLVM IR type to find out if this is the
3403
  // case.
3404
7.82k
  if (ContainsFloatAtOffset(IRType, IROffset, getDataLayout()) &&
3405
280
      ContainsFloatAtOffset(IRType, IROffset+4, getDataLayout()))
3406
280
    return llvm::FixedVectorType::get(llvm::Type::getFloatTy(getVMContext()),
3407
280
                                      2);
3408
3409
7.54k
  return llvm::Type::getDoubleTy(getVMContext());
3410
7.54k
}
3411
3412
3413
/// GetINTEGERTypeAtOffset - The ABI specifies that a value should be passed in
3414
/// an 8-byte GPR.  This means that we either have a scalar or we are talking
3415
/// about the high or low part of an up-to-16-byte struct.  This routine picks
3416
/// the best LLVM IR type to represent this, which may be i64 or may be anything
3417
/// else that the backend will pass in a GPR that works better (e.g. i8, %foo*,
3418
/// etc).
3419
///
3420
/// PrefType is an LLVM IR type that corresponds to (part of) the IR type for
3421
/// the source type.  IROffset is an offset in bytes into the LLVM IR type that
3422
/// the 8-byte value references.  PrefType may be null.
3423
///
3424
/// SourceTy is the source-level type for the entire argument.  SourceOffset is
3425
/// an offset into this that we're processing (which is always either 0 or 8).
3426
///
3427
llvm::Type *X86_64ABIInfo::
3428
GetINTEGERTypeAtOffset(llvm::Type *IRType, unsigned IROffset,
3429
318k
                       QualType SourceTy, unsigned SourceOffset) const {
3430
  // If we're dealing with an un-offset LLVM IR type, then it means that we're
3431
  // returning an 8-byte unit starting with it.  See if we can safely use it.
3432
318k
  if (IROffset == 0) {
3433
    // Pointers and int64's always fill the 8-byte unit.
3434
316k
    if ((isa<llvm::PointerType>(IRType) && 
Has64BitPointers223k
) ||
3435
93.8k
        IRType->isIntegerTy(64))
3436
242k
      return IRType;
3437
3438
    // If we have a 1/2/4-byte integer, we can use it only if the rest of the
3439
    // goodness in the source type is just tail padding.  This is allowed to
3440
    // kick in for struct {double,int} on the int, but not on
3441
    // struct{double,int,int} because we wouldn't return the second int.  We
3442
    // have to do this analysis on the source type because we can't depend on
3443
    // unions being lowered a specific way etc.
3444
74.4k
    if (IRType->isIntegerTy(8) || 
IRType->isIntegerTy(16)70.9k
||
3445
70.2k
        IRType->isIntegerTy(32) ||
3446
65.1k
        
(9.36k
isa<llvm::PointerType>(IRType)9.36k
&&
!Has64BitPointers92
)) {
3447
92
      unsigned BitWidth = isa<llvm::PointerType>(IRType) ? 32 :
3448
65.0k
          cast<llvm::IntegerType>(IRType)->getBitWidth();
3449
3450
65.1k
      if (BitsContainNoUserData(SourceTy, SourceOffset*8+BitWidth,
3451
65.1k
                                SourceOffset*8+64, getContext()))
3452
64.7k
        return IRType;
3453
11.0k
    }
3454
74.4k
  }
3455
3456
11.0k
  if (llvm::StructType *STy = dyn_cast<llvm::StructType>(IRType)) {
3457
    // If this is a struct, recurse into the field at the specified offset.
3458
7.21k
    const llvm::StructLayout *SL = getDataLayout().getStructLayout(STy);
3459
7.21k
    if (IROffset < SL->getSizeInBytes()) {
3460
7.21k
      unsigned FieldIdx = SL->getElementContainingOffset(IROffset);
3461
7.21k
      IROffset -= SL->getElementOffset(FieldIdx);
3462
3463
7.21k
      return GetINTEGERTypeAtOffset(STy->getElementType(FieldIdx), IROffset,
3464
7.21k
                                    SourceTy, SourceOffset);
3465
7.21k
    }
3466
3.88k
  }
3467
3468
3.88k
  if (llvm::ArrayType *ATy = dyn_cast<llvm::ArrayType>(IRType)) {
3469
52
    llvm::Type *EltTy = ATy->getElementType();
3470
52
    unsigned EltSize = getDataLayout().getTypeAllocSize(EltTy);
3471
52
    unsigned EltOffset = IROffset/EltSize*EltSize;
3472
52
    return GetINTEGERTypeAtOffset(EltTy, IROffset-EltOffset, SourceTy,
3473
52
                                  SourceOffset);
3474
52
  }
3475
3476
  // Okay, we don't have any better idea of what to pass, so we pass this in an
3477
  // integer register that isn't too big to fit the rest of the struct.
3478
3.82k
  unsigned TySizeInBytes =
3479
3.82k
    (unsigned)getContext().getTypeSizeInChars(SourceTy).getQuantity();
3480
3481
3.82k
  assert(TySizeInBytes != SourceOffset && "Empty field?");
3482
3483
  // It is always safe to classify this as an integer type up to i64 that
3484
  // isn't larger than the structure.
3485
3.82k
  return llvm::IntegerType::get(getVMContext(),
3486
3.82k
                                std::min(TySizeInBytes-SourceOffset, 8U)*8);
3487
3.82k
}
3488
3489
3490
/// GetX86_64ByValArgumentPair - Given a high and low type that can ideally
3491
/// be used as elements of a two register pair to pass or return, return a
3492
/// first class aggregate to represent them.  For example, if the low part of
3493
/// a by-value argument should be passed as i32* and the high part as float,
3494
/// return {i32*, float}.
3495
static llvm::Type *
3496
GetX86_64ByValArgumentPair(llvm::Type *Lo, llvm::Type *Hi,
3497
1.58k
                           const llvm::DataLayout &TD) {
3498
  // In order to correctly satisfy the ABI, we need to the high part to start
3499
  // at offset 8.  If the high and low parts we inferred are both 4-byte types
3500
  // (e.g. i32 and i32) then the resultant struct type ({i32,i32}) won't have
3501
  // the second element at offset 8.  Check for this:
3502
1.58k
  unsigned LoSize = (unsigned)TD.getTypeAllocSize(Lo);
3503
1.58k
  unsigned HiAlign = TD.getABITypeAlignment(Hi);
3504
1.58k
  unsigned HiStart = llvm::alignTo(LoSize, HiAlign);
3505
1.58k
  assert(HiStart != 0 && HiStart <= 8 && "Invalid x86-64 argument pair!");
3506
3507
  // To handle this, we have to increase the size of the low part so that the
3508
  // second element will start at an 8 byte offset.  We can't increase the size
3509
  // of the second element because it might make us access off the end of the
3510
  // struct.
3511
1.58k
  if (HiStart != 8) {
3512
    // There are usually two sorts of types the ABI generation code can produce
3513
    // for the low part of a pair that aren't 8 bytes in size: float or
3514
    // i8/i16/i32.  This can also include pointers when they are 32-bit (X32 and
3515
    // NaCl).
3516
    // Promote these to a larger type.
3517
5
    if (Lo->isFloatTy())
3518
0
      Lo = llvm::Type::getDoubleTy(Lo->getContext());
3519
5
    else {
3520
5
      assert((Lo->isIntegerTy() || Lo->isPointerTy())
3521
5
             && "Invalid/unknown lo type");
3522
5
      Lo = llvm::Type::getInt64Ty(Lo->getContext());
3523
5
    }
3524
5
  }
3525
3526
1.58k
  llvm::StructType *Result = llvm::StructType::get(Lo, Hi);
3527
3528
  // Verify that the second element is at an 8-byte offset.
3529
1.58k
  assert(TD.getStructLayout(Result)->getElementOffset(1) == 8 &&
3530
1.58k
         "Invalid x86-64 argument pair!");
3531
1.58k
  return Result;
3532
1.58k
}
3533
3534
ABIArgInfo X86_64ABIInfo::
3535
139k
classifyReturnType(QualType RetTy) const {
3536
  // AMD64-ABI 3.2.3p4: Rule 1. Classify the return type with the
3537
  // classification algorithm.
3538
139k
  X86_64ABIInfo::Class Lo, Hi;
3539
139k
  classify(RetTy, 0, Lo, Hi, /*isNamedArg*/ true);
3540
3541
  // Check some invariants.
3542
139k
  assert((Hi != Memory || Lo == Memory) && "Invalid memory classification.");
3543
139k
  assert((Hi != SSEUp || Lo == SSE) && "Invalid SSEUp classification.");
3544
3545
139k
  llvm::Type *ResType = nullptr;
3546
139k
  switch (Lo) {
3547
57.3k
  case NoClass:
3548
57.3k
    if (Hi == NoClass)
3549
57.3k
      return ABIArgInfo::getIgnore();
3550
    // If the low part is just padding, it takes no register, leave ResType
3551
    // null.
3552
1
    assert((Hi == SSE || Hi == Integer || Hi == X87Up) &&
3553
1
           "Unknown missing lo part");
3554
1
    break;
3555
3556
0
  case SSEUp:
3557
0
  case X87Up:
3558
0
    llvm_unreachable("Invalid classification for lo word.");
3559
3560
    // AMD64-ABI 3.2.3p4: Rule 2. Types of class memory are returned via
3561
    // hidden argument.
3562
432
  case Memory:
3563
432
    return getIndirectReturnResult(RetTy);
3564
3565
    // AMD64-ABI 3.2.3p4: Rule 3. If the class is INTEGER, the next
3566
    // available register of the sequence %rax, %rdx is used.
3567
77.8k
  case Integer:
3568
77.8k
    ResType = GetINTEGERTypeAtOffset(CGT.ConvertType(RetTy), 0, RetTy, 0);
3569
3570
    // If we have a sign or zero extended integer, make sure to return Extend
3571
    // so that the parameter gets the right LLVM IR attributes.
3572
77.8k
    if (Hi == NoClass && 
isa<llvm::IntegerType>(ResType)77.1k
) {
3573
      // Treat an enum type as its underlying type.
3574
41.4k
      if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
3575
100
        RetTy = EnumTy->getDecl()->getIntegerType();
3576
3577
41.4k
      if (RetTy->isIntegralOrEnumerationType() &&
3578
40.6k
          isPromotableIntegerTypeForABI(RetTy))
3579
4.01k
        return ABIArgInfo::getExtend(RetTy);
3580
73.8k
    }
3581
73.8k
    break;
3582
3583
    // AMD64-ABI 3.2.3p4: Rule 4. If the class is SSE, the next
3584
    // available SSE register of the sequence %xmm0, %xmm1 is used.
3585
3.10k
  case SSE:
3586
3.10k
    ResType = GetSSETypeAtOffset(CGT.ConvertType(RetTy), 0, RetTy, 0);
3587
3.10k
    break;
3588
3589
    // AMD64-ABI 3.2.3p4: Rule 6. If the class is X87, the value is
3590
    // returned on the X87 stack in %st0 as 80-bit x87 number.
3591
646
  case X87:
3592
646
    ResType = llvm::Type::getX86_FP80Ty(getVMContext());
3593
646
    break;
3594
3595
    // AMD64-ABI 3.2.3p4: Rule 8. If the class is COMPLEX_X87, the real
3596
    // part of the value is returned in %st0 and the imaginary part in
3597
    // %st1.
3598
38
  case ComplexX87:
3599
38
    assert(Hi == ComplexX87 && "Unexpected ComplexX87 classification.");
3600
38
    ResType = llvm::StructType::get(llvm::Type::getX86_FP80Ty(getVMContext()),
3601
38
                                    llvm::Type::getX86_FP80Ty(getVMContext()));
3602
38
    break;
3603
77.6k
  }
3604
3605
77.6k
  llvm::Type *HighPart = nullptr;
3606
77.6k
  switch (Hi) {
3607
    // Memory was handled previously and X87 should
3608
    // never occur as a hi class.
3609
0
  case Memory:
3610
0
  case X87:
3611
0
    llvm_unreachable("Invalid classification for hi word.");
3612
3613
38
  case ComplexX87: // Previously handled.
3614
74.1k
  case NoClass:
3615
74.1k
    break;
3616
3617
649
  case Integer:
3618
649
    HighPart = GetINTEGERTypeAtOffset(CGT.ConvertType(RetTy), 8, RetTy, 8);
3619
649
    if (Lo == NoClass)  // Return HighPart at offset 8 in memory.
3620
0
      return ABIArgInfo::getDirect(HighPart, 8);
3621
649
    break;
3622
137
  case SSE:
3623
137
    HighPart = GetSSETypeAtOffset(CGT.ConvertType(RetTy), 8, RetTy, 8);
3624
137
    if (Lo == NoClass)  // Return HighPart at offset 8 in memory.
3625
1
      return ABIArgInfo::getDirect(HighPart, 8);
3626
136
    break;
3627
3628
    // AMD64-ABI 3.2.3p4: Rule 5. If the class is SSEUP, the eightbyte
3629
    // is passed in the next available eightbyte chunk if the last used
3630
    // vector register.
3631
    //
3632
    // SSEUP should always be preceded by SSE, just widen.
3633
2.04k
  case SSEUp:
3634
2.04k
    assert(Lo == SSE && "Unexpected SSEUp classification.");
3635
2.04k
    ResType = GetByteVectorType(RetTy);
3636
2.04k
    break;
3637
3638
    // AMD64-ABI 3.2.3p4: Rule 7. If the class is X87UP, the value is
3639
    // returned together with the previous X87 value in %st0.
3640
646
  case X87Up:
3641
    // If X87Up is preceded by X87, we don't need to do
3642
    // anything. However, in some cases with unions it may not be
3643
    // preceded by X87. In such situations we follow gcc and pass the
3644
    // extra bits in an SSE reg.
3645
646
    if (Lo != X87) {
3646
0
      HighPart = GetSSETypeAtOffset(CGT.ConvertType(RetTy), 8, RetTy, 8);
3647
0
      if (Lo == NoClass)  // Return HighPart at offset 8 in memory.
3648
0
        return ABIArgInfo::getDirect(HighPart, 8);
3649
646
    }
3650
646
    break;
3651
77.6k
  }
3652
3653
  // If a high part was specified, merge it together with the low part.  It is
3654
  // known to pass in the high eightbyte of the result.  We do this by forming a
3655
  // first class struct aggregate with the high and low part: {low, high}
3656
77.6k
  if (HighPart)
3657
785
    ResType = GetX86_64ByValArgumentPair(ResType, HighPart, getDataLayout());
3658
3659
77.6k
  return ABIArgInfo::getDirect(ResType);
3660
77.6k
}
3661
3662
ABIArgInfo X86_64ABIInfo::classifyArgumentType(
3663
  QualType Ty, unsigned freeIntRegs, unsigned &neededInt, unsigned &neededSSE,
3664
  bool isNamedArg)
3665
  const
3666
243k
{
3667
243k
  Ty = useFirstFieldIfTransparentUnion(Ty);
3668
3669
243k
  X86_64ABIInfo::Class Lo, Hi;
3670
243k
  classify(Ty, 0, Lo, Hi, isNamedArg);
3671
3672
  // Check some invariants.
3673
  // FIXME: Enforce these by construction.
3674
243k
  assert((Hi != Memory || Lo == Memory) && "Invalid memory classification.");
3675
243k
  assert((Hi != SSEUp || Lo == SSE) && "Invalid SSEUp classification.");
3676
3677
243k
  neededInt = 0;
3678
243k
  neededSSE = 0;
3679
243k
  llvm::Type *ResType = nullptr;
3680
243k
  switch (Lo) {
3681
4.67k
  case NoClass:
3682
4.67k
    if (Hi == NoClass)
3683
4.67k
      return ABIArgInfo::getIgnore();
3684
    // If the low part is just padding, it takes no register, leave ResType
3685
    // null.
3686
2
    assert((Hi == SSE || Hi == Integer || Hi == X87Up) &&
3687
2
           "Unknown missing lo part");
3688
2
    break;
3689
3690
    // AMD64-ABI 3.2.3p3: Rule 1. If the class is MEMORY, pass the argument
3691
    // on the stack.
3692
1.07k
  case Memory:
3693
3694
    // AMD64-ABI 3.2.3p3: Rule 5. If the class is X87, X87UP or
3695
    // COMPLEX_X87, it is passed in memory.
3696
1.32k
  case X87:
3697
1.38k
  case ComplexX87:
3698
1.38k
    if (getRecordArgABI(Ty, getCXXABI()) == CGCXXABI::RAA_Indirect)
3699
397
      ++neededInt;
3700
1.38k
    return getIndirectResult(Ty, freeIntRegs);
3701
3702
0
  case SSEUp:
3703
0
  case X87Up:
3704
0
    llvm_unreachable("Invalid classification for lo word.");
3705
3706
    // AMD64-ABI 3.2.3p3: Rule 2. If the class is INTEGER, the next
3707
    // available register of the sequence %rdi, %rsi, %rdx, %rcx, %r8
3708
    // and %r9 is used.
3709
231k
  case Integer:
3710
231k
    ++neededInt;
3711
3712
    // Pick an 8-byte type based on the preferred type.
3713
231k
    ResType = GetINTEGERTypeAtOffset(CGT.ConvertType(Ty), 0, Ty, 0);
3714
3715
    // If we have a sign or zero extended integer, make sure to return Extend
3716
    // so that the parameter gets the right LLVM IR attributes.
3717
231k
    if (Hi == NoClass && 
isa<llvm::IntegerType>(ResType)231k
) {
3718
      // Treat an enum type as its underlying type.
3719
45.0k
      if (const EnumType *EnumTy = Ty->getAs<EnumType>())
3720
826
        Ty = EnumTy->getDecl()->getIntegerType();
3721
3722
45.0k
      if (Ty->isIntegralOrEnumerationType() &&
3723
44.6k
          isPromotableIntegerTypeForABI(Ty))
3724
2.96k
        return ABIArgInfo::getExtend(Ty);
3725
228k
    }
3726
3727
228k
    break;
3728
3729
    // AMD64-ABI 3.2.3p3: Rule 3. If the class is SSE, the next
3730
    // available SSE register is used, the registers are taken in the
3731
    // order from %xmm0 to %xmm7.
3732
6.10k
  case SSE: {
3733
6.10k
    llvm::Type *IRType = CGT.ConvertType(Ty);
3734
6.10k
    ResType = GetSSETypeAtOffset(IRType, 0, Ty, 0);
3735
6.10k
    ++neededSSE;
3736
6.10k
    break;
3737
234k
  }
3738
234k
  }
3739
3740
234k
  llvm::Type *HighPart = nullptr;
3741
234k
  switch (Hi) {
3742
    // Memory was handled previously, ComplexX87 and X87 should
3743
    // never occur as hi classes, and X87Up must be preceded by X87,
3744
    // which is passed in memory.
3745
0
  case Memory:
3746
0
  case X87:
3747
0
  case ComplexX87:
3748
0
    llvm_unreachable("Invalid classification for hi word.");
3749
3750
230k
  case NoClass: break;
3751
3752
653
  case Integer:
3753
653
    ++neededInt;
3754
    // Pick an 8-byte type based on the preferred type.
3755
653
    HighPart = GetINTEGERTypeAtOffset(CGT.ConvertType(Ty), 8, Ty, 8);
3756
3757
653
    if (Lo == NoClass)  // Pass HighPart at offset 8 in memory.
3758
2
      return ABIArgInfo::getDirect(HighPart, 8);
3759
651
    break;
3760
3761
    // X87Up generally doesn't occur here (long double is passed in
3762
    // memory), except in situations involving unions.
3763
1
  case X87Up:
3764
152
  case SSE:
3765
152
    HighPart = GetSSETypeAtOffset(CGT.ConvertType(Ty), 8, Ty, 8);
3766
3767
152
    if (Lo == NoClass)  // Pass HighPart at offset 8 in memory.
3768
0
      return ABIArgInfo::getDirect(HighPart, 8);
3769
3770
152
    ++neededSSE;
3771
152
    break;
3772
3773
    // AMD64-ABI 3.2.3p3: Rule 4. If the class is SSEUP, the
3774
    // eightbyte is passed in the upper half of the last used SSE
3775
    // register.  This only happens when 128-bit vectors are passed.
3776
3.48k
  case SSEUp:
3777
3.48k
    assert(Lo == SSE && "Unexpected SSEUp classification");
3778
3.48k
    ResType = GetByteVectorType(Ty);
3779
3.48k
    break;
3780
234k
  }
3781
3782
  // If a high part was specified, merge it together with the low part.  It is
3783
  // known to pass in the high eightbyte of the result.  We do this by forming a
3784
  // first class struct aggregate with the high and low part: {low, high}
3785
234k
  if (HighPart)
3786
803
    ResType = GetX86_64ByValArgumentPair(ResType, HighPart, getDataLayout());
3787
3788
234k
  return ABIArgInfo::getDirect(ResType);
3789
234k
}
3790
3791
ABIArgInfo
3792
X86_64ABIInfo::classifyRegCallStructTypeImpl(QualType Ty, unsigned &NeededInt,
3793
18
                                             unsigned &NeededSSE) const {
3794
18
  auto RT = Ty->getAs<RecordType>();
3795
18
  assert(RT && "classifyRegCallStructType only valid with struct types");
3796
3797
18
  if (RT->getDecl()->hasFlexibleArrayMember())
3798
0
    return getIndirectReturnResult(Ty);
3799
3800
  // Sum up bases
3801
18
  if (auto CXXRD = dyn_cast<CXXRecordDecl>(RT->getDecl())) {
3802
0
    if (CXXRD->isDynamicClass()) {
3803
0
      NeededInt = NeededSSE = 0;
3804
0
      return getIndirectReturnResult(Ty);
3805
0
    }
3806
3807
0
    for (const auto &I : CXXRD->bases())
3808
0
      if (classifyRegCallStructTypeImpl(I.getType(), NeededInt, NeededSSE)
3809
0
              .isIndirect()) {
3810
0
        NeededInt = NeededSSE = 0;
3811
0
        return getIndirectReturnResult(Ty);
3812
0
      }
3813
0
  }
3814
3815
  // Sum up members
3816
44
  
for (const auto *FD : RT->getDecl()->fields())18
{
3817
44
    if (FD->getType()->isRecordType() && 
!FD->getType()->isUnionType()0
) {
3818
0
      if (classifyRegCallStructTypeImpl(FD->getType(), NeededInt, NeededSSE)
3819
0
              .isIndirect()) {
3820
0
        NeededInt = NeededSSE = 0;
3821
0
        return getIndirectReturnResult(Ty);
3822
0
      }
3823
44
    } else {
3824
44
      unsigned LocalNeededInt, LocalNeededSSE;
3825
44
      if (classifyArgumentType(FD->getType(), UINT_MAX, LocalNeededInt,
3826
44
                               LocalNeededSSE, true)
3827
0
              .isIndirect()) {
3828
0
        NeededInt = NeededSSE = 0;
3829
0
        return getIndirectReturnResult(Ty);
3830
0
      }
3831
44
      NeededInt += LocalNeededInt;
3832
44
      NeededSSE += LocalNeededSSE;
3833
44
    }
3834
44
  }
3835
3836
18
  return ABIArgInfo::getDirect();
3837
18
}
3838
3839
ABIArgInfo X86_64ABIInfo::classifyRegCallStructType(QualType Ty,
3840
                                                    unsigned &NeededInt,
3841
18
                                                    unsigned &NeededSSE) const {
3842
3843
18
  NeededInt = 0;
3844
18
  NeededSSE = 0;
3845
3846
18
  return classifyRegCallStructTypeImpl(Ty, NeededInt, NeededSSE);
3847
18
}
3848
3849
140k
void X86_64ABIInfo::computeInfo(CGFunctionInfo &FI) const {
3850
3851
140k
  const unsigned CallingConv = FI.getCallingConvention();
3852
  // It is possible to force Win64 calling convention on any x86_64 target by
3853
  // using __attribute__((ms_abi)). In such case to correctly emit Win64
3854
  // compatible code delegate this call to WinX86_64ABIInfo::computeInfo.
3855
140k
  if (CallingConv == llvm::CallingConv::Win64) {
3856
11
    WinX86_64ABIInfo Win64ABIInfo(CGT, AVXLevel);
3857
11
    Win64ABIInfo.computeInfo(FI);
3858
11
    return;
3859
11
  }
3860
3861
140k
  bool IsRegCall = CallingConv == llvm::CallingConv::X86_RegCall;
3862
3863
  // Keep track of the number of assigned registers.
3864
140k
  unsigned FreeIntRegs = IsRegCall ? 
1124
: 6;
3865
140k
  unsigned FreeSSERegs = IsRegCall ? 
1624
: 8;
3866
140k
  unsigned NeededInt, NeededSSE;
3867
3868
140k
  if (!::classifyReturnType(getCXXABI(), FI, *this)) {
3869
139k
    if (IsRegCall && 
FI.getReturnType()->getTypePtr()->isRecordType()23
&&
3870
2
        !FI.getReturnType()->getTypePtr()->isUnionType()) {
3871
2
      FI.getReturnInfo() =
3872
2
          classifyRegCallStructType(FI.getReturnType(), NeededInt, NeededSSE);
3873
2
      if (FreeIntRegs >= NeededInt && FreeSSERegs >= NeededSSE) {
3874
2
        FreeIntRegs -= NeededInt;
3875
2
        FreeSSERegs -= NeededSSE;
3876
0
      } else {
3877
0
        FI.getReturnInfo() = getIndirectReturnResult(FI.getReturnType());
3878
0
      }
3879
139k
    } else if (IsRegCall && 
FI.getReturnType()->getAs<ComplexType>()21
&&
3880
2
               getContext().getCanonicalType(FI.getReturnType()
3881
2
                                                 ->getAs<ComplexType>()
3882
2
                                                 ->getElementType()) ==
3883
2
                   getContext().LongDoubleTy)
3884
      // Complex Long Double Type is passed in Memory when Regcall
3885
      // calling convention is used.
3886
1
      FI.getReturnInfo() = getIndirectReturnResult(FI.getReturnType());
3887
139k
    else
3888
139k
      FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
3889
139k
  }
3890
3891
  // If the return value is indirect, then the hidden argument is consuming one
3892
  // integer register.
3893
140k
  if (FI.getReturnInfo().isIndirect())
3894
888
    --FreeIntRegs;
3895
3896
  // The chain argument effectively gives us another free register.
3897
140k
  if (FI.isChainCall())
3898
4
    ++FreeIntRegs;
3899
3900
140k
  unsigned NumRequiredArgs = FI.getNumRequiredArgs();
3901
  // AMD64-ABI 3.2.3p3: Once arguments are classified, the registers
3902
  // get assigned (in left-to-right order) for passing as follows...
3903
140k
  unsigned ArgNo = 0;
3904
140k
  for (CGFunctionInfo::arg_iterator it = FI.arg_begin(), ie = FI.arg_end();
3905
383k
       it != ie; 
++it, ++ArgNo243k
) {
3906
243k
    bool IsNamedArg = ArgNo < NumRequiredArgs;
3907
3908
243k
    if (IsRegCall && 
it->type->isStructureOrClassType()54
)
3909
16
      it->info = classifyRegCallStructType(it->type, NeededInt, NeededSSE);
3910
243k
    else
3911
243k
      it->info = classifyArgumentType(it->type, FreeIntRegs, NeededInt,
3912
243k
                                      NeededSSE, IsNamedArg);
3913
3914
    // AMD64-ABI 3.2.3p3: If there are no registers available for any
3915
    // eightbyte of an argument, the whole argument is passed on the
3916
    // stack. If registers have already been assigned for some
3917
    // eightbytes of such an argument, the assignments get reverted.
3918
243k
    if (FreeIntRegs >= NeededInt && 
FreeSSERegs >= NeededSSE241k
) {
3919
241k
      FreeIntRegs -= NeededInt;
3920
241k
      FreeSSERegs -= NeededSSE;
3921
2.12k
    } else {
3922
2.12k
      it->info = getIndirectResult(it->type, FreeIntRegs);
3923
2.12k
    }
3924
243k
  }
3925
140k
}
3926
3927
static Address EmitX86_64VAArgFromMemory(CodeGenFunction &CGF,
3928
97
                                         Address VAListAddr, QualType Ty) {
3929
97
  Address overflow_arg_area_p =
3930
97
      CGF.Builder.CreateStructGEP(VAListAddr, 2, "overflow_arg_area_p");
3931
97
  llvm::Value *overflow_arg_area =
3932
97
    CGF.Builder.CreateLoad(overflow_arg_area_p, "overflow_arg_area");
3933
3934
  // AMD64-ABI 3.5.7p5: Step 7. Align l->overflow_arg_area upwards to a 16
3935
  // byte boundary if alignment needed by type exceeds 8 byte boundary.
3936
  // It isn't stated explicitly in the standard, but in practice we use
3937
  // alignment greater than 16 where necessary.
3938
97
  CharUnits Align = CGF.getContext().getTypeAlignInChars(Ty);
3939
97
  if (Align > CharUnits::fromQuantity(8)) {
3940
18
    overflow_arg_area = emitRoundPointerUpToAlignment(CGF, overflow_arg_area,
3941
18
                                                      Align);
3942
18
  }
3943
3944
  // AMD64-ABI 3.5.7p5: Step 8. Fetch type from l->overflow_arg_area.
3945
97
  llvm::Type *LTy = CGF.ConvertTypeForMem(Ty);
3946
97
  llvm::Value *Res =
3947
97
    CGF.Builder.CreateBitCast(overflow_arg_area,
3948
97
                              llvm::PointerType::getUnqual(LTy));
3949
3950
  // AMD64-ABI 3.5.7p5: Step 9. Set l->overflow_arg_area to:
3951
  // l->overflow_arg_area + sizeof(type).
3952
  // AMD64-ABI 3.5.7p5: Step 10. Align l->overflow_arg_area upwards to
3953
  // an 8 byte boundary.
3954
3955
97
  uint64_t SizeInBytes = (CGF.getContext().getTypeSize(Ty) + 7) / 8;
3956
97
  llvm::Value *Offset =
3957
97
      llvm::ConstantInt::get(CGF.Int32Ty, (SizeInBytes + 7)  & ~7);
3958
97
  overflow_arg_area = CGF.Builder.CreateGEP(overflow_arg_area, Offset,
3959
97
                                            "overflow_arg_area.next");
3960
97
  CGF.Builder.CreateStore(overflow_arg_area, overflow_arg_area_p);
3961
3962
  // AMD64-ABI 3.5.7p5: Step 11. Return the fetched type.
3963
97
  return Address(Res, Align);
3964
97
}
3965
3966
Address X86_64ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
3967
97
                                 QualType Ty) const {
3968
  // Assume that va_list type is correct; should be pointer to LLVM type:
3969
  // struct {
3970
  //   i32 gp_offset;
3971
  //   i32 fp_offset;
3972
  //   i8* overflow_arg_area;
3973
  //   i8* reg_save_area;
3974
  // };
3975
97
  unsigned neededInt, neededSSE;
3976
3977
97
  Ty = getContext().getCanonicalType(Ty);
3978
97
  ABIArgInfo AI = classifyArgumentType(Ty, 0, neededInt, neededSSE,
3979
97
                                       /*isNamedArg*/false);
3980
3981
  // AMD64-ABI 3.5.7p5: Step 1. Determine whether type may be passed
3982
  // in the registers. If not go to step 7.
3983
97
  if (!neededInt && 
!neededSSE34
)
3984
18
    return EmitX86_64VAArgFromMemory(CGF, VAListAddr, Ty);
3985
3986
  // AMD64-ABI 3.5.7p5: Step 2. Compute num_gp to hold the number of
3987
  // general purpose registers needed to pass type and num_fp to hold
3988
  // the number of floating point registers needed.
3989
3990
  // AMD64-ABI 3.5.7p5: Step 3. Verify whether arguments fit into
3991
  // registers. In the case: l->gp_offset > 48 - num_gp * 8 or
3992
  // l->fp_offset > 304 - num_fp * 16 go to step 7.
3993
  //
3994
  // NOTE: 304 is a typo, there are (6 * 8 + 8 * 16) = 176 bytes of
3995
  // register save space).
3996
3997
79
  llvm::Value *InRegs = nullptr;
3998
79
  Address gp_offset_p = Address::invalid(), fp_offset_p = Address::invalid();
3999
79
  llvm::Value *gp_offset = nullptr, *fp_offset = nullptr;
4000
79
  if (neededInt) {
4001
63
    gp_offset_p = CGF.Builder.CreateStructGEP(VAListAddr, 0, "gp_offset_p");
4002
63
    gp_offset = CGF.Builder.CreateLoad(gp_offset_p, "gp_offset");
4003
63
    InRegs = llvm::ConstantInt::get(CGF.Int32Ty, 48 - neededInt * 8);
4004
63
    InRegs = CGF.Builder.CreateICmpULE(gp_offset, InRegs, "fits_in_gp");
4005
63
  }
4006
4007
79
  if (neededSSE) {
4008
20
    fp_offset_p = CGF.Builder.CreateStructGEP(VAListAddr, 1, "fp_offset_p");
4009
20
    fp_offset = CGF.Builder.CreateLoad(fp_offset_p, "fp_offset");
4010
20
    llvm::Value *FitsInFP =
4011
20
      llvm::ConstantInt::get(CGF.Int32Ty, 176 - neededSSE * 16);
4012
20
    FitsInFP = CGF.Builder.CreateICmpULE(fp_offset, FitsInFP, "fits_in_fp");
4013
16
    InRegs = InRegs ? 
CGF.Builder.CreateAnd(InRegs, FitsInFP)4
: FitsInFP;
4014
20
  }
4015
4016
79
  llvm::BasicBlock *InRegBlock = CGF.createBasicBlock("vaarg.in_reg");
4017
79
  llvm::BasicBlock *InMemBlock = CGF.createBasicBlock("vaarg.in_mem");
4018
79
  llvm::BasicBlock *ContBlock = CGF.createBasicBlock("vaarg.end");
4019
79
  CGF.Builder.CreateCondBr(InRegs, InRegBlock, InMemBlock);
4020
4021
  // Emit code to load the value if it was passed in registers.
4022
4023
79
  CGF.EmitBlock(InRegBlock);
4024
4025
  // AMD64-ABI 3.5.7p5: Step 4. Fetch type from l->reg_save_area with
4026
  // an offset of l->gp_offset and/or l->fp_offset. This may require
4027
  // copying to a temporary location in case the parameter is passed
4028
  // in different register classes or requires an alignment greater
4029
  // than 8 for general purpose registers and 16 for XMM registers.
4030
  //
4031
  // FIXME: This really results in shameful code when we end up needing to
4032
  // collect arguments from different places; often what should result in a
4033
  // simple assembling of a structure from scattered addresses has many more
4034
  // loads than necessary. Can we clean this up?
4035
79
  llvm::Type *LTy = CGF.ConvertTypeForMem(Ty);
4036
79
  llvm::Value *RegSaveArea = CGF.Builder.CreateLoad(
4037
79
      CGF.Builder.CreateStructGEP(VAListAddr, 3), "reg_save_area");
4038
4039
79
  Address RegAddr = Address::invalid();
4040
79
  if (neededInt && 
neededSSE63
) {
4041
    // FIXME: Cleanup.
4042
4
    assert(AI.isDirect() && "Unexpected ABI info for mixed regs");
4043
4
    llvm::StructType *ST = cast<llvm::StructType>(AI.getCoerceToType());
4044
4
    Address Tmp = CGF.CreateMemTemp(Ty);
4045
4
    Tmp = CGF.Builder.CreateElementBitCast(Tmp, ST);
4046
4
    assert(ST->getNumElements() == 2 && "Unexpected ABI info for mixed regs");
4047
4
    llvm::Type *TyLo = ST->getElementType(0);
4048
4
    llvm::Type *TyHi = ST->getElementType(1);
4049
4
    assert((TyLo->isFPOrFPVectorTy() ^ TyHi->isFPOrFPVectorTy()) &&
4050
4
           "Unexpected ABI info for mixed regs");
4051
4
    llvm::Type *PTyLo = llvm::PointerType::getUnqual(TyLo);
4052
4
    llvm::Type *PTyHi = llvm::PointerType::getUnqual(TyHi);
4053
4
    llvm::Value *GPAddr = CGF.Builder.CreateGEP(RegSaveArea, gp_offset);
4054
4
    llvm::Value *FPAddr = CGF.Builder.CreateGEP(RegSaveArea, fp_offset);
4055
3
    llvm::Value *RegLoAddr = TyLo->isFPOrFPVectorTy() ? 
FPAddr1
: GPAddr;
4056
3
    llvm::Value *RegHiAddr = TyLo->isFPOrFPVectorTy() ? 
GPAddr1
: FPAddr;
4057
4058
    // Copy the first element.
4059
    // FIXME: Our choice of alignment here and below is probably pessimistic.
4060
4
    llvm::Value *V = CGF.Builder.CreateAlignedLoad(
4061
4
        TyLo, CGF.Builder.CreateBitCast(RegLoAddr, PTyLo),
4062
4
        CharUnits::fromQuantity(getDataLayout().getABITypeAlignment(TyLo)));
4063
4
    CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 0));
4064
4065
    // Copy the second element.
4066
4
    V = CGF.Builder.CreateAlignedLoad(
4067
4
        TyHi, CGF.Builder.CreateBitCast(RegHiAddr, PTyHi),
4068
4
        CharUnits::fromQuantity(getDataLayout().getABITypeAlignment(TyHi)));
4069
4
    CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 1));
4070
4071
4
    RegAddr = CGF.Builder.CreateElementBitCast(Tmp, LTy);
4072
75
  } else if (neededInt) {
4073
59
    RegAddr = Address(CGF.Builder.CreateGEP(RegSaveArea, gp_offset),
4074
59
                      CharUnits::fromQuantity(8));
4075
59
    RegAddr = CGF.Builder.CreateElementBitCast(RegAddr, LTy);
4076
4077
    // Copy to a temporary if necessary to ensure the appropriate alignment.
4078
59
    auto TInfo = getContext().getTypeInfoInChars(Ty);
4079
59
    uint64_t TySize = TInfo.Width.getQuantity();
4080
59
    CharUnits TyAlign = TInfo.Align;
4081
4082
    // Copy into a temporary if the type is more aligned than the
4083
    // register save area.
4084
59
    if (TyAlign.getQuantity() > 8) {
4085
3
      Address Tmp = CGF.CreateMemTemp(Ty);
4086
3
      CGF.Builder.CreateMemCpy(Tmp, RegAddr, TySize, false);
4087
3
      RegAddr = Tmp;
4088
3
    }
4089
4090
16
  } else if (neededSSE == 1) {
4091
7
    RegAddr = Address(CGF.Builder.CreateGEP(RegSaveArea, fp_offset),
4092
7
                      CharUnits::fromQuantity(16));
4093
7
    RegAddr = CGF.Builder.CreateElementBitCast(RegAddr, LTy);
4094
9
  } else {
4095
9
    assert(neededSSE == 2 && "Invalid number of needed registers!");
4096
    // SSE registers are spaced 16 bytes apart in the register save
4097
    // area, we need to collect the two eightbytes together.
4098
    // The ABI isn't explicit about this, but it seems reasonable
4099
    // to assume that the slots are 16-byte aligned, since the stack is
4100
    // naturally 16-byte aligned and the prologue is expected to store
4101
    // all the SSE registers to the RSA.
4102
9
    Address RegAddrLo = Address(CGF.Builder.CreateGEP(RegSaveArea, fp_offset),
4103
9
                                CharUnits::fromQuantity(16));
4104
9
    Address RegAddrHi =
4105
9
      CGF.Builder.CreateConstInBoundsByteGEP(RegAddrLo,
4106
9
                                             CharUnits::fromQuantity(16));
4107
9
    llvm::Type *ST = AI.canHaveCoerceToType()
4108
9
                         ? AI.getCoerceToType()
4109
0
                         : llvm::StructType::get(CGF.DoubleTy, CGF.DoubleTy);
4110
9
    llvm::Value *V;
4111
9
    Address Tmp = CGF.CreateMemTemp(Ty);
4112
9
    Tmp = CGF.Builder.CreateElementBitCast(Tmp, ST);
4113
9
    V = CGF.Builder.CreateLoad(CGF.Builder.CreateElementBitCast(
4114
9
        RegAddrLo, ST->getStructElementType(0)));
4115
9
    CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 0));
4116
9
    V = CGF.Builder.CreateLoad(CGF.Builder.CreateElementBitCast(
4117
9
        RegAddrHi, ST->getStructElementType(1)));
4118
9
    CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 1));
4119
4120
9
    RegAddr = CGF.Builder.CreateElementBitCast(Tmp, LTy);
4121
9
  }
4122
4123
  // AMD64-ABI 3.5.7p5: Step 5. Set:
4124
  // l->gp_offset = l->gp_offset + num_gp * 8
4125
  // l->fp_offset = l->fp_offset + num_fp * 16.
4126
79
  if (neededInt) {
4127
63
    llvm::Value *Offset = llvm::ConstantInt::get(CGF.Int32Ty, neededInt * 8);
4128
63
    CGF.Builder.CreateStore(CGF.Builder.CreateAdd(gp_offset, Offset),
4129
63
                            gp_offset_p);
4130
63
  }
4131
79
  if (neededSSE) {
4132
20
    llvm::Value *Offset = llvm::ConstantInt::get(CGF.Int32Ty, neededSSE * 16);
4133
20
    CGF.Builder.CreateStore(CGF.Builder.CreateAdd(fp_offset, Offset),
4134
20
                            fp_offset_p);
4135
20
  }
4136
79
  CGF.EmitBranch(ContBlock);
4137
4138
  // Emit code to load the value if it was passed in memory.
4139
4140
79
  CGF.EmitBlock(InMemBlock);
4141
79
  Address MemAddr = EmitX86_64VAArgFromMemory(CGF, VAListAddr, Ty);
4142
4143
  // Return the appropriate result.
4144
4145
79
  CGF.EmitBlock(ContBlock);
4146
79
  Address ResAddr = emitMergePHI(CGF, RegAddr, InRegBlock, MemAddr, InMemBlock,
4147
79
                                 "vaarg.addr");
4148
79
  return ResAddr;
4149
79
}
4150
4151
Address X86_64ABIInfo::EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr,
4152
8
                                   QualType Ty) const {
4153
8
  return emitVoidPtrVAArg(CGF, VAListAddr, Ty, /*indirect*/ false,
4154
8
                          CGF.getContext().getTypeInfoInChars(Ty),
4155
8
                          CharUnits::fromQuantity(8),
4156
8
                          /*allowHigherAlign*/ false);
4157
8
}
4158
4159
ABIArgInfo WinX86_64ABIInfo::reclassifyHvaArgForVectorCall(
4160
97
    QualType Ty, unsigned &FreeSSERegs, const ABIArgInfo &current) const {
4161
97
  const Type *Base = nullptr;
4162
97
  uint64_t NumElts = 0;
4163
4164
97
  if (!Ty->isBuiltinType() && 
!Ty->isVectorType()49
&&
4165
36
      isHomogeneousAggregate(Ty, Base, NumElts) && 
FreeSSERegs >= NumElts27
) {
4166
18
    FreeSSERegs -= NumElts;
4167
18
    return getDirectX86Hva();
4168
18
  }
4169
79
  return current;
4170
79
}
4171
4172
ABIArgInfo WinX86_64ABIInfo::classify(QualType Ty, unsigned &FreeSSERegs,
4173
                                      bool IsReturnType, bool IsVectorCall,
4174
9.74k
                                      bool IsRegCall) const {
4175
4176
9.74k
  if (Ty->isVoidType())
4177
1.59k
    return ABIArgInfo::getIgnore();
4178
4179
8.14k
  if (const EnumType *EnumTy = Ty->getAs<EnumType>())
4180
18
    Ty = EnumTy->getDecl()->getIntegerType();
4181
4182
8.14k
  TypeInfo Info = getContext().getTypeInfo(Ty);
4183
8.14k
  uint64_t Width = Info.Width;
4184
8.14k
  CharUnits Align = getContext().toCharUnitsFromBits(Info.Align);
4185
4186
8.14k
  const RecordType *RT = Ty->getAs<RecordType>();
4187
8.14k
  if (RT) {
4188
290
    if (!IsReturnType) {
4189
255
      if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(RT, getCXXABI()))
4190
71
        return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
4191
219
    }
4192
4193
219
    if (RT->getDecl()->hasFlexibleArrayMember())
4194
0
      return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
4195
4196
8.07k
  }
4197
4198
8.07k
  const Type *Base = nullptr;
4199
8.07k
  uint64_t NumElts = 0;
4200
  // vectorcall adds the concept of a homogenous vector aggregate, similar to
4201
  // other targets.
4202
8.07k
  if ((IsVectorCall || 
IsRegCall7.96k
) &&
4203
171
      isHomogeneousAggregate(Ty, Base, NumElts)) {
4204
94
    if (IsRegCall) {
4205
31
      if (FreeSSERegs >= NumElts) {
4206
31
        FreeSSERegs -= NumElts;
4207
31
        if (IsReturnType || 
Ty->isBuiltinType()27
||
Ty->isVectorType()21
)
4208
16
          return ABIArgInfo::getDirect();
4209
15
        return ABIArgInfo::getExpand();
4210
15
      }
4211
0
      return ABIArgInfo::getIndirect(Align, /*ByVal=*/false);
4212
63
    } else if (IsVectorCall) {
4213
63
      if (FreeSSERegs >= NumElts &&
4214
57
          (IsReturnType || 
Ty->isBuiltinType()45
||
Ty->isVectorType()36
)) {
4215
34
        FreeSSERegs -= NumElts;
4216
34
        return ABIArgInfo::getDirect();
4217
29
      } else if (IsReturnType) {
4218
0
        return ABIArgInfo::getExpand();
4219
29
      } else if (!Ty->isBuiltinType() && 
!Ty->isVectorType()27
) {
4220
        // HVAs are delayed and reclassified in the 2nd step.
4221
27
        return ABIArgInfo::getIndirect(Align, /*ByVal=*/false);
4222
27
      }
4223
7.98k
    }
4224
94
  }
4225
4226
7.98k
  if (Ty->isMemberPointerType()) {
4227
    // If the member pointer is represented by an LLVM int or ptr, pass it
4228
    // directly.
4229
159
    llvm::Type *LLTy = CGT.ConvertType(Ty);
4230
159
    if (LLTy->isPointerTy() || 
LLTy->isIntegerTy()76
)
4231
91
      return ABIArgInfo::getDirect();
4232
7.89k
  }
4233
4234
7.89k
  if (RT || 
Ty->isAnyComplexType()7.72k
||
Ty->isMemberPointerType()7.70k
) {
4235
    // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
4236
    // not 1, 2, 4, or 8 bytes, must be passed by reference."
4237
256
    if (Width > 64 || 
!llvm::isPowerOf2_64(Width)148
)
4238
108
      return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
4239
4240
    // Otherwise, coerce it to a small integer.
4241
148
    return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(), Width));
4242
148
  }
4243
4244
7.63k
  if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
4245
2.07k
    switch (BT->getKind()) {
4246
29
    case BuiltinType::Bool:
4247
      // Bool type is always extended to the ABI, other builtin types are not
4248
      // extended.
4249
29
      return ABIArgInfo::getExtend(Ty);
4250
4251
56
    case BuiltinType::LongDouble:
4252
      // Mingw64 GCC uses the old 80 bit extended precision floating point
4253
      // unit. It passes them indirectly through memory.
4254
56
      if (IsMingw64) {
4255
6
        const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();
4256
6
        if (LDF == &llvm::APFloat::x87DoubleExtended())
4257
6
          return ABIArgInfo::getIndirect(Align, /*ByVal=*/false);
4258
50
      }
4259
50
      break;
4260
4261
8
    case BuiltinType::Int128:
4262
8
    case BuiltinType::UInt128:
4263
      // If it's a parameter type, the normal ABI rule is that arguments larger
4264
      // than 8 bytes are passed indirectly. GCC follows it. We follow it too,
4265
      // even though it isn't particularly efficient.
4266
8
      if (!IsReturnType)
4267
4
        return ABIArgInfo::getIndirect(Align, /*ByVal=*/false);
4268
4269
      // Mingw64 GCC returns i128 in XMM0. Coerce to v2i64 to handle that.
4270
      // Clang matches them for compatibility.
4271
4
      return ABIArgInfo::getDirect(llvm::FixedVectorType::get(
4272
4
          llvm::Type::getInt64Ty(getVMContext()), 2));
4273
4274
1.98k
    default:
4275
1.98k
      break;
4276
7.59k
    }
4277
7.59k
  }
4278
4279
7.59k
  if (Ty->isExtIntType()) {
4280
    // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
4281
    // not 1, 2, 4, or 8 bytes, must be passed by reference."
4282
    // However, non-power-of-two _ExtInts will be passed as 1,2,4 or 8 bytes
4283
    // anyway as long is it fits in them, so we don't have to check the power of
4284
    // 2.
4285
54
    if (Width <= 64)
4286
35
      return ABIArgInfo::getDirect();
4287
19
    return ABIArgInfo::getIndirect(Align, /*ByVal=*/false);
4288
19
  }
4289
4290
7.54k
  return ABIArgInfo::getDirect();
4291
7.54k
}
4292
4293
4.00k
void WinX86_64ABIInfo::computeInfo(CGFunctionInfo &FI) const {
4294
4.00k
  const unsigned CC = FI.getCallingConvention();
4295
4.00k
  bool IsVectorCall = CC == llvm::CallingConv::X86_VectorCall;
4296
4.00k
  bool IsRegCall = CC == llvm::CallingConv::X86_RegCall;
4297
4298
  // If __attribute__((sysv_abi)) is in use, use the SysV argument
4299
  // classification rules.
4300
4.00k
  if (CC == llvm::CallingConv::X86_64_SysV) {
4301
14
    X86_64ABIInfo SysVABIInfo(CGT, AVXLevel);
4302
14
    SysVABIInfo.computeInfo(FI);
4303
14
    return;
4304
14
  }
4305
4306
3.99k
  unsigned FreeSSERegs = 0;
4307
3.99k
  if (IsVectorCall) {
4308
    // We can use up to 4 SSE return registers with vectorcall.
4309
48
    FreeSSERegs = 4;
4310
3.94k
  } else if (IsRegCall) {
4311
    // RegCall gives us 16 SSE registers.
4312
23
    FreeSSERegs = 16;
4313
23
  }
4314
4315
3.99k
  if (!getCXXABI().classifyReturnType(FI))
4316
3.95k
    FI.getReturnInfo() = classify(FI.getReturnType(), FreeSSERegs, true,
4317
3.95k
                                  IsVectorCall, IsRegCall);
4318
4319
3.99k
  if (IsVectorCall) {
4320
    // We can use up to 6 SSE register parameters with vectorcall.
4321
48
    FreeSSERegs = 6;
4322
3.94k
  } else if (IsRegCall) {
4323
    // RegCall gives us 16 SSE registers, we can reuse the return registers.
4324
23
    FreeSSERegs = 16;
4325
23
  }
4326
4327
3.99k
  unsigned ArgNum = 0;
4328
3.99k
  unsigned ZeroSSERegs = 0;
4329
5.78k
  for (auto &I : FI.arguments()) {
4330
    // Vectorcall in x64 only permits the first 6 arguments to be passed as
4331
    // XMM/YMM registers. After the sixth argument, pretend no vector
4332
    // registers are left.
4333
5.78k
    unsigned *MaybeFreeSSERegs =
4334
5.78k
        (IsVectorCall && 
ArgNum >= 697
) ?
&ZeroSSERegs5
:
&FreeSSERegs5.78k
;
4335
5.78k
    I.info =
4336
5.78k
        classify(I.type, *MaybeFreeSSERegs, false, IsVectorCall, IsRegCall);
4337
5.78k
    ++ArgNum;
4338
5.78k
  }
4339
4340
3.99k
  if (IsVectorCall) {
4341
    // For vectorcall, assign aggregate HVAs to any free vector registers in a
4342
    // second pass.
4343
48
    for (auto &I : FI.arguments())
4344
97
      I.info = reclassifyHvaArgForVectorCall(I.type, FreeSSERegs, I.info);
4345
48
  }
4346
3.99k
}
4347
4348
Address WinX86_64ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
4349
19
                                    QualType Ty) const {
4350
4351
19
  bool IsIndirect = false;
4352
4353
  // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
4354
  // not 1, 2, 4, or 8 bytes, must be passed by reference."
4355
19
  if (isAggregateTypeForABI(Ty) || 
Ty->isMemberPointerType()13
) {
4356
6
    uint64_t Width = getContext().getTypeSize(Ty);
4357
6
    IsIndirect = Width > 64 || 
!llvm::isPowerOf2_64(Width)0
;
4358
6
  }
4359
4360
19
  return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect,
4361
19
                          CGF.getContext().getTypeInfoInChars(Ty),
4362
19
                          CharUnits::fromQuantity(8),
4363
19
                          /*allowHigherAlign*/ false);
4364
19
}
4365
4366
static bool PPC_initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
4367
                                        llvm::Value *Address, bool Is64Bit,
4368
4
                                        bool IsAIX) {
4369
  // This is calculated from the LLVM and GCC tables and verified
4370
  // against gcc output.  AFAIK all PPC ABIs use the same encoding.
4371
4372
4
  CodeGen::CGBuilderTy &Builder = CGF.Builder;
4373
4374
4
  llvm::IntegerType *i8 = CGF.Int8Ty;
4375
4
  llvm::Value *Four8 = llvm::ConstantInt::get(i8, 4);
4376
4
  llvm::Value *Eight8 = llvm::ConstantInt::get(i8, 8);
4377
4
  llvm::Value *Sixteen8 = llvm::ConstantInt::get(i8, 16);
4378
4379
  // 0-31: r0-31, the 4-byte or 8-byte general-purpose registers
4380
2
  AssignToArrayRange(Builder, Address, Is64Bit ? Eight8 : Four8, 0, 31);
4381
4382
  // 32-63: fp0-31, the 8-byte floating-point registers
4383
4
  AssignToArrayRange(Builder, Address, Eight8, 32, 63);
4384
4385
  // 64-67 are various 4-byte or 8-byte special-purpose registers:
4386
  // 64: mq
4387
  // 65: lr
4388
  // 66: ctr
4389
  // 67: ap
4390
2
  AssignToArrayRange(Builder, Address, Is64Bit ? Eight8 : Four8, 64, 67);
4391
4392
  // 68-76 are various 4-byte special-purpose registers:
4393
  // 68-75 cr0-7
4394
  // 76: xer
4395
4
  AssignToArrayRange(Builder, Address, Four8, 68, 76);
4396
4397
  // 77-108: v0-31, the 16-byte vector registers
4398
4
  AssignToArrayRange(Builder, Address, Sixteen8, 77, 108);
4399
4400
  // 109: vrsave
4401
  // 110: vscr
4402
2
  AssignToArrayRange(Builder, Address, Is64Bit ? Eight8 : Four8, 109, 110);
4403
4404
  // AIX does not utilize the rest of the registers.
4405
4
  if (IsAIX)
4406
2
    return false;
4407
4408
  // 111: spe_acc
4409
  // 112: spefscr
4410
  // 113: sfp
4411
2
  AssignToArrayRange(Builder, Address, Is64Bit ? 
Eight81
:
Four81
, 111, 113);
4412
4413
2
  if (!Is64Bit)
4414
1
    return false;
4415
4416
  // TODO: Need to verify if these registers are used on 64 bit AIX with Power8
4417
  // or above CPU.
4418
  // 64-bit only registers:
4419
  // 114: tfhar
4420
  // 115: tfiar
4421
  // 116: texasr
4422
1
  AssignToArrayRange(Builder, Address, Eight8, 114, 116);
4423
4424
1
  return false;
4425
1
}
4426
4427
// AIX
4428
namespace {
4429
/// AIXABIInfo - The AIX XCOFF ABI information.
4430
class AIXABIInfo : public ABIInfo {
4431
  const bool Is64Bit;
4432
  const unsigned PtrByteSize;
4433
  CharUnits getParamTypeAlignment(QualType Ty) const;
4434
4435
public:
4436
  AIXABIInfo(CodeGen::CodeGenTypes &CGT, bool Is64Bit)
4437
60
      : ABIInfo(CGT), Is64Bit(Is64Bit), PtrByteSize(Is64Bit ? 8 : 4) {}
4438
4439
  bool isPromotableTypeForABI(QualType Ty) const;
4440
4441
  ABIArgInfo classifyReturnType(QualType RetTy) const;
4442
  ABIArgInfo classifyArgumentType(QualType Ty) const;
4443
4444
206
  void computeInfo(CGFunctionInfo &FI) const override {
4445
206
    if (!getCXXABI().classifyReturnType(FI))
4446
204
      FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
4447
4448
206
    for (auto &I : FI.arguments())
4449
128
      I.info = classifyArgumentType(I.type);
4450
206
  }
4451
4452
  Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
4453
                    QualType Ty) const override;
4454
};
4455
4456
class AIXTargetCodeGenInfo : public TargetCodeGenInfo {
4457
  const bool Is64Bit;
4458
4459
public:
4460
  AIXTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, bool Is64Bit)
4461
      : TargetCodeGenInfo(std::make_unique<AIXABIInfo>(CGT, Is64Bit)),
4462
60
        Is64Bit(Is64Bit) {}
4463
2
  int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
4464
2
    return 1; // r1 is the dedicated stack pointer
4465
2
  }
4466
4467
  bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
4468
                               llvm::Value *Address) const override;
4469
};
4470
} // namespace
4471
4472
// Return true if the ABI requires Ty to be passed sign- or zero-
4473
// extended to 32/64 bits.
4474
156
bool AIXABIInfo::isPromotableTypeForABI(QualType Ty) const {
4475
  // Treat an enum type as its underlying type.
4476
156
  if (const EnumType *EnumTy = Ty->getAs<EnumType>())
4477
8
    Ty = EnumTy->getDecl()->getIntegerType();
4478
4479
  // Promotable integer types are required to be promoted by the ABI.
4480
156
  if (Ty->isPromotableIntegerType())
4481
12
    return true;
4482
4483
144
  if (!Is64Bit)
4484
96
    return false;
4485
4486
  // For 64 bit mode, in addition to the usual promotable integer types, we also
4487
  // need to extend all 32-bit types, since the ABI requires promotion to 64
4488
  // bits.
4489
48
  if (const BuiltinType *BT = Ty->getAs<BuiltinType>())
4490
28
    switch (BT->getKind()) {
4491
22
    case BuiltinType::Int:
4492
22
    case BuiltinType::UInt:
4493
22
      return true;
4494
6
    default:
4495
6
      break;
4496
26
    }
4497
4498
26
  return false;
4499
26
}
4500
4501
204
ABIArgInfo AIXABIInfo::classifyReturnType(QualType RetTy) const {
4502
204
  if (RetTy->isAnyComplexType())
4503
6
    return ABIArgInfo::getDirect();
4504
4505
198
  if (RetTy->isVectorType())
4506
10
    return ABIArgInfo::getDirect();
4507
4508
188
  if (RetTy->isVoidType())
4509
96
    return ABIArgInfo::getIgnore();
4510
4511
92
  if (isAggregateTypeForABI(RetTy))
4512
22
    return getNaturalAlignIndirect(RetTy);
4513
4514
70
  return (isPromotableTypeForABI(RetTy) ? 
ABIArgInfo::getExtend(RetTy)20
4515
50
                                        : ABIArgInfo::getDirect());
4516
70
}
4517
4518
128
ABIArgInfo AIXABIInfo::classifyArgumentType(QualType Ty) const {
4519
128
  Ty = useFirstFieldIfTransparentUnion(Ty);
4520
4521
128
  if (Ty->isAnyComplexType())
4522
6
    return ABIArgInfo::getDirect();
4523
4524
122
  if (Ty->isVectorType())
4525
10
    return ABIArgInfo::getDirect();
4526
4527
112
  if (isAggregateTypeForABI(Ty)) {
4528
    // Records with non-trivial destructors/copy-constructors should not be
4529
    // passed by value.
4530
26
    if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))
4531
2
      return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
4532
4533
24
    CharUnits CCAlign = getParamTypeAlignment(Ty);
4534
24
    CharUnits TyAlign = getContext().getTypeAlignInChars(Ty);
4535
4536
24
    return ABIArgInfo::getIndirect(CCAlign, /*ByVal*/ true,
4537
24
                                   /*Realign*/ TyAlign > CCAlign);
4538
24
  }
4539
4540
86
  return (isPromotableTypeForABI(Ty) ? 
ABIArgInfo::getExtend(Ty)14
4541
72
                                     : ABIArgInfo::getDirect());
4542
86
}
4543
4544
28
CharUnits AIXABIInfo::getParamTypeAlignment(QualType Ty) const {
4545
  // Complex types are passed just like their elements.
4546
28
  if (const ComplexType *CTy = Ty->getAs<ComplexType>())
4547
0
    Ty = CTy->getElementType();
4548
4549
28
  if (Ty->isVectorType())
4550
0
    return CharUnits::fromQuantity(16);
4551
4552
  // If the structure contains a vector type, the alignment is 16.
4553
28
  if (isRecordWithSIMDVectorType(getContext(), Ty))
4554
2
    return CharUnits::fromQuantity(16);
4555
4556
26
  return CharUnits::fromQuantity(PtrByteSize);
4557
26
}
4558
4559
Address AIXABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
4560
4
                              QualType Ty) const {
4561
4
  if (Ty->isAnyComplexType())
4562
0
    llvm::report_fatal_error("complex type is not supported on AIX yet");
4563
4564
4
  if (Ty->isVectorType())
4565
0
    llvm::report_fatal_error(
4566
0
        "vector types are not yet supported for variadic functions on AIX");
4567
4568
4
  auto TypeInfo = getContext().getTypeInfoInChars(Ty);
4569
4
  TypeInfo.Align = getParamTypeAlignment(Ty);
4570
4571
4
  CharUnits SlotSize = CharUnits::fromQuantity(PtrByteSize);
4572
4573
4
  return emitVoidPtrVAArg(CGF, VAListAddr, Ty, /*Indirect*/ false, TypeInfo,
4574
4
                          SlotSize, /*AllowHigher*/ true);
4575
4
}
4576
4577
bool AIXTargetCodeGenInfo::initDwarfEHRegSizeTable(
4578
2
    CodeGen::CodeGenFunction &CGF, llvm::Value *Address) const {
4579
2
  return PPC_initDwarfEHRegSizeTable(CGF, Address, Is64Bit, /*IsAIX*/ true);
4580
2
}
4581
4582
// PowerPC-32
4583
namespace {
4584
/// PPC32_SVR4_ABIInfo - The 32-bit PowerPC ELF (SVR4) ABI information.
4585
class PPC32_SVR4_ABIInfo : public DefaultABIInfo {
4586
  bool IsSoftFloatABI;
4587
  bool IsRetSmallStructInRegABI;
4588
4589
  CharUnits getParamTypeAlignment(QualType Ty) const;
4590
4591
public:
4592
  PPC32_SVR4_ABIInfo(CodeGen::CodeGenTypes &CGT, bool SoftFloatABI,
4593
                     bool RetSmallStructInRegABI)
4594
      : DefaultABIInfo(CGT), IsSoftFloatABI(SoftFloatABI),
4595
48
        IsRetSmallStructInRegABI(RetSmallStructInRegABI) {}
4596
4597
  ABIArgInfo classifyReturnType(QualType RetTy) const;
4598
4599
830
  void computeInfo(CGFunctionInfo &FI) const override {
4600
830
    if (!getCXXABI().classifyReturnType(FI))
4601
830
      FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
4602
830
    for (auto &I : FI.arguments())
4603
1.47k
      I.info = classifyArgumentType(I.type);
4604
830
  }
4605
4606
  Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
4607
                    QualType Ty) const override;
4608
};
4609
4610
class PPC32TargetCodeGenInfo : public TargetCodeGenInfo {
4611
public:
4612
  PPC32TargetCodeGenInfo(CodeGenTypes &CGT, bool SoftFloatABI,
4613
                         bool RetSmallStructInRegABI)
4614
      : TargetCodeGenInfo(std::make_unique<PPC32_SVR4_ABIInfo>(
4615
48
            CGT, SoftFloatABI, RetSmallStructInRegABI)) {}
4616
4617
  static bool isStructReturnInRegABI(const llvm::Triple &Triple,
4618
                                     const CodeGenOptions &Opts);
4619
4620
1
  int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
4621
    // This is recovered from gcc output.
4622
1
    return 1; // r1 is the dedicated stack pointer
4623
1
  }
4624
4625
  bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
4626
                               llvm::Value *Address) const override;
4627
};
4628
}
4629
4630
0
CharUnits PPC32_SVR4_ABIInfo::getParamTypeAlignment(QualType Ty) const {
4631
  // Complex types are passed just like their elements.
4632
0
  if (const ComplexType *CTy = Ty->getAs<ComplexType>())
4633
0
    Ty = CTy->getElementType();
4634
4635
0
  if (Ty->isVectorType())
4636
0
    return CharUnits::fromQuantity(getContext().getTypeSize(Ty) == 128 ? 16
4637
0
                                                                       : 4);
4638
4639
  // For single-element float/vector structs, we consider the whole type
4640
  // to have the same alignment requirements as its single element.
4641
0
  const Type *AlignTy = nullptr;
4642
0
  if (const Type *EltType = isSingleElementStruct(Ty, getContext())) {
4643
0
    const BuiltinType *BT = EltType->getAs<BuiltinType>();
4644
0
    if ((EltType->isVectorType() && getContext().getTypeSize(EltType) == 128) ||
4645
0
        (BT && BT->isFloatingPoint()))
4646
0
      AlignTy = EltType;
4647
0
  }
4648
4649
0
  if (AlignTy)
4650
0
    return CharUnits::fromQuantity(AlignTy->isVectorType() ? 16 : 4);
4651
0
  return CharUnits::fromQuantity(4);
4652
0
}
4653
4654
830
ABIArgInfo PPC32_SVR4_ABIInfo::classifyReturnType(QualType RetTy) const {
4655
830
  uint64_t Size;
4656
4657
  // -msvr4-struct-return puts small aggregates in GPR3 and GPR4.
4658
830
  if (isAggregateTypeForABI(RetTy) && 
IsRetSmallStructInRegABI141
&&
4659
88
      (Size = getContext().getTypeSize(RetTy)) <= 64) {
4660
    // System V ABI (1995), page 3-22, specified:
4661
    // > A structure or union whose size is less than or equal to 8 bytes
4662
    // > shall be returned in r3 and r4, as if it were first stored in the
4663
    // > 8-byte aligned memory area and then the low addressed word were
4664
    // > loaded into r3 and the high-addressed word into r4.  Bits beyond
4665
    // > the last member of the structure or union are not defined.
4666
    //
4667
    // GCC for big-endian PPC32 inserts the pad before the first member,
4668
    // not "beyond the last member" of the struct.  To stay compatible
4669
    // with GCC, we coerce the struct to an integer of the same size.
4670
    // LLVM will extend it and return i32 in r3, or i64 in r3:r4.
4671
70
    if (Size == 0)
4672
8
      return ABIArgInfo::getIgnore();
4673
62
    else {
4674
62
      llvm::Type *CoerceTy = llvm::Type::getIntNTy(getVMContext(), Size);
4675
62
      return ABIArgInfo::getDirect(CoerceTy);
4676
62
    }
4677
760
  }
4678
4679
760
  return DefaultABIInfo::classifyReturnType(RetTy);
4680
760
}
4681
4682
// TODO: this implementation is now likely redundant with
4683
// DefaultABIInfo::EmitVAArg.
4684
Address PPC32_SVR4_ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAList,
4685
3
                                      QualType Ty) const {
4686
3
  if (getTarget().getTriple().isOSDarwin()) {
4687
0
    auto TI = getContext().getTypeInfoInChars(Ty);
4688
0
    TI.Align = getParamTypeAlignment(Ty);
4689
4690
0
    CharUnits SlotSize = CharUnits::fromQuantity(4);
4691
0
    return emitVoidPtrVAArg(CGF, VAList, Ty,
4692
0
                            classifyArgumentType(Ty).isIndirect(), TI, SlotSize,
4693
0
                            /*AllowHigherAlign=*/true);
4694
0
  }
4695
4696
3
  const unsigned OverflowLimit = 8;
4697
3
  if (const ComplexType *CTy = Ty->getAs<ComplexType>()) {
4698
    // TODO: Implement this. For now ignore.
4699
0
    (void)CTy;
4700
0
    return Address::invalid(); // FIXME?
4701
0
  }
4702
4703
  // struct __va_list_tag {
4704
  //   unsigned char gpr;
4705
  //   unsigned char fpr;
4706
  //   unsigned short reserved;
4707
  //   void *overflow_arg_area;
4708
  //   void *reg_save_area;
4709
  // };
4710
4711
3
  bool isI64 = Ty->isIntegerType() && 
getContext().getTypeSize(Ty) == 641
;
4712
3
  bool isInt =
4713
3
      Ty->isIntegerType() || 
Ty->isPointerType()2
||
Ty->isAggregateType()2
;
4714
3
  bool isF64 = Ty->isFloatingType() && 
getContext().getTypeSize(Ty) == 641
;
4715
4716
  // All aggregates are passed indirectly?  That doesn't seem consistent
4717
  // with the argument-lowering code.
4718
3
  bool isIndirect = Ty->isAggregateType();
4719
4720
3
  CGBuilderTy &Builder = CGF.Builder;
4721
4722
  // The calling convention either uses 1-2 GPRs or 1 FPR.
4723
3
  Address NumRegsAddr = Address::invalid();
4724
3
  if (isInt || 
IsSoftFloatABI1
) {
4725
3
    NumRegsAddr = Builder.CreateStructGEP(VAList, 0, "gpr");
4726
0
  } else {
4727
0
    NumRegsAddr = Builder.CreateStructGEP(VAList, 1, "fpr");
4728
0
  }
4729
4730
3
  llvm::Value *NumRegs = Builder.CreateLoad(NumRegsAddr, "numUsedRegs");
4731
4732
  // "Align" the register count when TY is i64.
4733
3
  if (isI64 || (isF64 && 
IsSoftFloatABI1
)) {
4734
1
    NumRegs = Builder.CreateAdd(NumRegs, Builder.getInt8(1));
4735
1
    NumRegs = Builder.CreateAnd(NumRegs, Builder.getInt8((uint8_t) ~1U));
4736
1
  }
4737
4738
3
  llvm::Value *CC =
4739
3
      Builder.CreateICmpULT(NumRegs, Builder.getInt8(OverflowLimit), "cond");
4740
4741
3
  llvm::BasicBlock *UsingRegs = CGF.createBasicBlock("using_regs");
4742
3
  llvm::BasicBlock *UsingOverflow = CGF.createBasicBlock("using_overflow");
4743
3
  llvm::BasicBlock *Cont = CGF.createBasicBlock("cont");
4744
4745
3
  Builder.CreateCondBr(CC, UsingRegs, UsingOverflow);
4746
4747
3
  llvm::Type *DirectTy = CGF.ConvertType(Ty);
4748
3
  if (isIndirect) 
DirectTy = DirectTy->getPointerTo(0)1
;
4749
4750
  // Case 1: consume registers.
4751
3
  Address RegAddr = Address::invalid();
4752
3
  {
4753
3
    CGF.EmitBlock(UsingRegs);
4754
4755
3
    Address RegSaveAreaPtr = Builder.CreateStructGEP(VAList, 4);
4756
3
    RegAddr = Address(Builder.CreateLoad(RegSaveAreaPtr),
4757
3
                      CharUnits::fromQuantity(8));
4758
3
    assert(RegAddr.getElementType() == CGF.Int8Ty);
4759
4760
    // Floating-point registers start after the general-purpose registers.
4761
3
    if (!(isInt || 
IsSoftFloatABI1
)) {
4762
0
      RegAddr = Builder.CreateConstInBoundsByteGEP(RegAddr,
4763
0
                                                   CharUnits::fromQuantity(32));
4764
0
    }
4765
4766
    // Get the address of the saved value by scaling the number of
4767
    // registers we've used by the number of
4768
3
    CharUnits RegSize = CharUnits::fromQuantity((isInt || 
IsSoftFloatABI1
) ? 4 :
80
);
4769
3
    llvm::Value *RegOffset =
4770
3
      Builder.CreateMul(NumRegs, Builder.getInt8(RegSize.getQuantity()));
4771
3
    RegAddr = Address(Builder.CreateInBoundsGEP(CGF.Int8Ty,
4772
3
                                            RegAddr.getPointer(), RegOffset),
4773
3
                      RegAddr.getAlignment().alignmentOfArrayElement(RegSize));
4774
3
    RegAddr = Builder.CreateElementBitCast(RegAddr, DirectTy);
4775
4776
    // Increase the used-register count.
4777
3
    NumRegs =
4778
3
      Builder.CreateAdd(NumRegs,
4779
3
                        Builder.getInt8((isI64 || (isF64 && 
IsSoftFloatABI1
)) ?
21
:
12
));
4780
3
    Builder.CreateStore(NumRegs, NumRegsAddr);
4781
4782
3
    CGF.EmitBranch(Cont);
4783
3
  }
4784
4785
  // Case 2: consume space in the overflow area.
4786
3
  Address MemAddr = Address::invalid();
4787
3
  {
4788
3
    CGF.EmitBlock(UsingOverflow);
4789
4790
3
    Builder.CreateStore(Builder.getInt8(OverflowLimit), NumRegsAddr);
4791
4792
    // Everything in the overflow area is rounded up to a size of at least 4.
4793
3
    CharUnits OverflowAreaAlign = CharUnits::fromQuantity(4);
4794
4795
3
    CharUnits Size;
4796
3
    if (!isIndirect) {
4797
2
      auto TypeInfo = CGF.getContext().getTypeInfoInChars(Ty);
4798
2
      Size = TypeInfo.Width.alignTo(OverflowAreaAlign);
4799
1
    } else {
4800
1
      Size = CGF.getPointerSize();
4801
1
    }
4802
4803
3
    Address OverflowAreaAddr = Builder.CreateStructGEP(VAList, 3);
4804
3
    Address OverflowArea(Builder.CreateLoad(OverflowAreaAddr, "argp.cur"),
4805
3
                         OverflowAreaAlign);
4806
    // Round up address of argument to alignment
4807
3
    CharUnits Align = CGF.getContext().getTypeAlignInChars(Ty);
4808
3
    if (Align > OverflowAreaAlign) {
4809
2
      llvm::Value *Ptr = OverflowArea.getPointer();
4810
2
      OverflowArea = Address(emitRoundPointerUpToAlignment(CGF, Ptr, Align),
4811
2
                                                           Align);
4812
2
    }
4813
4814
3
    MemAddr = Builder.CreateElementBitCast(OverflowArea, DirectTy);
4815
4816
    // Increase the overflow area.
4817
3
    OverflowArea = Builder.CreateConstInBoundsByteGEP(OverflowArea, Size);
4818
3
    Builder.CreateStore(OverflowArea.getPointer(), OverflowAreaAddr);
4819
3
    CGF.EmitBranch(Cont);
4820
3
  }
4821
4822
3
  CGF.EmitBlock(Cont);
4823
4824
  // Merge the cases with a phi.
4825
3
  Address Result = emitMergePHI(CGF, RegAddr, UsingRegs, MemAddr, UsingOverflow,
4826
3
                                "vaarg.addr");
4827
4828
  // Load the pointer if the argument was passed indirectly.
4829
3
  if (isIndirect) {
4830
1
    Result = Address(Builder.CreateLoad(Result, "aggr"),
4831
1
                     getContext().getTypeAlignInChars(Ty));
4832
1
  }
4833
4834
3
  return Result;
4835
3
}
4836
4837
bool PPC32TargetCodeGenInfo::isStructReturnInRegABI(
4838
48
    const llvm::Triple &Triple, const CodeGenOptions &Opts) {
4839
48
  assert(Triple.isPPC32());
4840
4841
48
  switch (Opts.getStructReturnConvention()) {
4842
42
  case CodeGenOptions::SRCK_Default:
4843
42
    break;
4844
3
  case CodeGenOptions::SRCK_OnStack: // -maix-struct-return
4845
3
    return false;
4846
3
  case CodeGenOptions::SRCK_InRegs: // -msvr4-struct-return
4847
3
    return true;
4848
42
  }
4849
4850
42
  if (Triple.isOSBinFormatELF() && !Triple.isOSLinux())
4851
26
    return true;
4852
4853
16
  return false;
4854
16
}
4855
4856
bool
4857
PPC32TargetCodeGenInfo::initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
4858
1
                                                llvm::Value *Address) const {
4859
1
  return PPC_initDwarfEHRegSizeTable(CGF, Address, /*Is64Bit*/ false,
4860
1
                                     /*IsAIX*/ false);
4861
1
}
4862
4863
// PowerPC-64
4864
4865
namespace {
4866
/// PPC64_SVR4_ABIInfo - The 64-bit PowerPC ELF (SVR4) ABI information.
4867
class PPC64_SVR4_ABIInfo : public SwiftABIInfo {
4868
public:
4869
  enum ABIKind {
4870
    ELFv1 = 0,
4871
    ELFv2
4872
  };
4873
4874
private:
4875
  static const unsigned GPRBits = 64;
4876
  ABIKind Kind;
4877
  bool IsSoftFloatABI;
4878
4879
public:
4880
  PPC64_SVR4_ABIInfo(CodeGen::CodeGenTypes &CGT, ABIKind Kind,
4881
                     bool SoftFloatABI)
4882
2.51k
      : SwiftABIInfo(CGT), Kind(Kind), IsSoftFloatABI(SoftFloatABI) {}
4883
4884
  bool isPromotableTypeForABI(QualType Ty) const;
4885
  CharUnits getParamTypeAlignment(QualType Ty) const;
4886
4887
  ABIArgInfo classifyReturnType(QualType RetTy) const;
4888
  ABIArgInfo classifyArgumentType(QualType Ty) const;
4889
4890
  bool isHomogeneousAggregateBaseType(QualType Ty) const override;
4891
  bool isHomogeneousAggregateSmallEnough(const Type *Ty,
4892
                                         uint64_t Members) const override;
4893
4894
  // TODO: We can add more logic to computeInfo to improve performance.
4895
  // Example: For aggregate arguments that fit in a register, we could
4896
  // use getDirectInReg (as is done below for structs containing a single
4897
  // floating-point value) to avoid pushing them to memory on function
4898
  // entry.  This would require changing the logic in PPCISelLowering
4899
  // when lowering the parameters in the caller and args in the callee.
4900
15.8k
  void computeInfo(CGFunctionInfo &FI) const override {
4901
15.8k
    if (!getCXXABI().classifyReturnType(FI))
4902
15.8k
      FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
4903
31.2k
    for (auto &I : FI.arguments()) {
4904
      // We rely on the default argument classification for the most part.
4905
      // One exception:  An aggregate containing a single floating-point
4906
      // or vector item must be passed in a register if one is available.
4907
31.2k
      const Type *T = isSingleElementStruct(I.type, getContext());
4908
31.2k
      if (T) {
4909
25
        const BuiltinType *BT = T->getAs<BuiltinType>();
4910
25
        if ((T->isVectorType() && 
getContext().getTypeSize(T) == 1284
) ||
4911
22
            (BT && 
BT->isFloatingPoint()19
)) {
4912
21
          QualType QT(T, 0);
4913
21
          I.info = ABIArgInfo::getDirectInReg(CGT.ConvertType(QT));
4914
21
          continue;
4915
21
        }
4916
31.2k
      }
4917
31.2k
      I.info = classifyArgumentType(I.type);
4918
31.2k
    }
4919
15.8k
  }
4920
4921
  Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
4922
                    QualType Ty) const override;
4923
4924
  bool shouldPassIndirectlyForSwift(ArrayRef<llvm::Type*> scalars,
4925
0
                                    bool asReturnValue) const override {
4926
0
    return occupiesMoreThan(CGT, scalars, /*total*/ 4);
4927
0
  }
4928
4929
0
  bool isSwiftErrorInRegister() const override {
4930
0
    return false;
4931
0
  }
4932
};
4933
4934
class PPC64_SVR4_TargetCodeGenInfo : public TargetCodeGenInfo {
4935
4936
public:
4937
  PPC64_SVR4_TargetCodeGenInfo(CodeGenTypes &CGT,
4938
                               PPC64_SVR4_ABIInfo::ABIKind Kind,
4939
                               bool SoftFloatABI)
4940
      : TargetCodeGenInfo(
4941
2.51k
            std::make_unique<PPC64_SVR4_ABIInfo>(CGT, Kind, SoftFloatABI)) {}
4942
4943
1
  int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
4944
    // This is recovered from gcc output.
4945
1
    return 1; // r1 is the dedicated stack pointer
4946
1
  }
4947
4948
  bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
4949
                               llvm::Value *Address) const override;
4950
};
4951
4952
class PPC64TargetCodeGenInfo : public DefaultTargetCodeGenInfo {
4953
public:
4954
0
  PPC64TargetCodeGenInfo(CodeGenTypes &CGT) : DefaultTargetCodeGenInfo(CGT) {}
4955
4956
0
  int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
4957
    // This is recovered from gcc output.
4958
0
    return 1; // r1 is the dedicated stack pointer
4959
0
  }
4960
4961
  bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
4962
                               llvm::Value *Address) const override;
4963
};
4964
4965
}
4966
4967
// Return true if the ABI requires Ty to be passed sign- or zero-
4968
// extended to 64 bits.
4969
bool
4970
35.9k
PPC64_SVR4_ABIInfo::isPromotableTypeForABI(QualType Ty) const {
4971
  // Treat an enum type as its underlying type.
4972
35.9k
  if (const EnumType *EnumTy = Ty->getAs<EnumType>())
4973
2
    Ty = EnumTy->getDecl()->getIntegerType();
4974
4975
  // Promotable integer types are required to be promoted by the ABI.
4976
35.9k
  if (isPromotableIntegerTypeForABI(Ty))
4977
289
    return true;
4978
4979
  // In addition to the usual promotable integer types, we also need to
4980
  // extend all 32-bit types, since the ABI requires promotion to 64 bits.
4981
35.6k
  if (const BuiltinType *BT = Ty->getAs<BuiltinType>())
4982
13.6k
    switch (BT->getKind()) {
4983
5.64k
    case BuiltinType::Int:
4984
5.83k
    case BuiltinType::UInt:
4985
5.83k
      return true;
4986
7.80k
    default:
4987
7.80k
      break;
4988
29.8k
    }
4989
4990
29.8k
  if (const auto *EIT = Ty->getAs<ExtIntType>())
4991
8
    if (EIT->getNumBits() < 64)
4992
2
      return true;
4993
4994
29.8k
  return false;
4995
29.8k
}
4996
4997
/// isAlignedParamType - Determine whether a type requires 16-byte or
4998
/// higher alignment in the parameter area.  Always returns at least 8.
4999
138
CharUnits PPC64_SVR4_ABIInfo::getParamTypeAlignment(QualType Ty) const {
5000
  // Complex types are passed just like their elements.
5001
138
  if (const ComplexType *CTy = Ty->getAs<ComplexType>())
5002
8
    Ty = CTy->getElementType();
5003
5004
  // Only vector types of size 16 bytes need alignment (larger types are
5005
  // passed via reference, smaller types are not aligned).
5006
138
  if (Ty->isVectorType()) {
5007
0
    return CharUnits::fromQuantity(getContext().getTypeSize(Ty) == 128 ? 16 : 8);
5008
138
  } else if (Ty->isRealFloatingType() &&
5009
15
             &getContext().getFloatTypeSemantics(Ty) ==
5010
9
                 &llvm::APFloat::IEEEquad()) {
5011
    // According to ABI document section 'Optional Save Areas': If extended
5012
    // precision floating-point values in IEEE BINARY 128 QUADRUPLE PRECISION
5013
    // format are supported, map them to a single quadword, quadword aligned.
5014
9
    return CharUnits::fromQuantity(16);
5015
9
  }
5016
5017
  // For single-element float/vector structs, we consider the whole type
5018
  // to have the same alignment requirements as its single element.
5019
129
  const Type *AlignAsType = nullptr;
5020
129
  const Type *EltType = isSingleElementStruct(Ty, getContext());
5021
129
  if (EltType) {
5022
3
    const BuiltinType *BT = EltType->getAs<BuiltinType>();
5023
3
    if ((EltType->isVectorType() && 
getContext().getTypeSize(EltType) == 1282
) ||
5024
2
        (BT && 
BT->isFloatingPoint()1
))
5025
2
      AlignAsType = EltType;
5026
3
  }
5027
5028
  // Likewise for ELFv2 homogeneous aggregates.
5029
129
  const Type *Base = nullptr;
5030
129
  uint64_t Members = 0;
5031
129
  if (!AlignAsType && 
Kind == ELFv2127
&&
5032
90
      isAggregateTypeForABI(Ty) && 
isHomogeneousAggregate(Ty, Base, Members)82
)
5033
50
    AlignAsType = Base;
5034
5035
  // With special case aggregates, only vector base types need alignment.
5036
129
  if (AlignAsType) {
5037
33
    return CharUnits::fromQuantity(AlignAsType->isVectorType() ? 
1619
: 8);
5038
52
  }
5039
5040
  // Otherwise, we only need alignment for any aggregate type that
5041
  // has an alignment requirement of >= 16 bytes.
5042
77
  if (isAggregateTypeForABI(Ty) && 
getContext().getTypeAlign(Ty) >= 12863
) {
5043
14
    return CharUnits::fromQuantity(16);
5044
14
  }
5045
5046
63
  return CharUnits::fromQuantity(8);
5047
63
}
5048
5049
/// isHomogeneousAggregate - Return true if a type is an ELFv2 homogeneous
5050
/// aggregate.  Base is set to the base element type, and Members is set
5051
/// to the number of base elements.
5052
bool ABIInfo::isHomogeneousAggregate(QualType Ty, const Type *&Base,
5053
5.97k
                                     uint64_t &Members) const {
5054
5.97k
  if (const ConstantArrayType *AT = getContext().getAsConstantArrayType(Ty)) {
5055
1.10k
    uint64_t NElements = AT->getSize().getZExtValue();
5056
1.10k
    if (NElements == 0)
5057
0
      return false;
5058
1.10k
    if (!isHomogeneousAggregate(AT->getElementType(), Base, Members))
5059
147
      return false;
5060
961
    Members *= NElements;
5061
4.86k
  } else if (const RecordType *RT = Ty->getAs<RecordType>()) {
5062
1.99k
    const RecordDecl *RD = RT->getDecl();
5063
1.99k
    if (RD->hasFlexibleArrayMember())
5064
2
      return false;
5065
5066
1.99k
    Members = 0;
5067
5068
    // If this is a C++ record, check the properties of the record such as
5069
    // bases and ABI specific restrictions
5070
1.99k
    if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
5071
497
      if (!getCXXABI().isPermittedToBeHomogeneousAggregate(CXXRD))
5072
24
        return false;
5073
5074
473
      for (const auto &I : CXXRD->bases()) {
5075
        // Ignore empty records.
5076
174
        if (isEmptyRecord(getContext(), I.getType(), true))
5077
10
          continue;
5078
5079
164
        uint64_t FldMembers;
5080
164
        if (!isHomogeneousAggregate(I.getType(), Base, FldMembers))
5081
27
          return false;
5082
5083
137
        Members += FldMembers;
5084
137
      }
5085
473
    }
5086
5087
2.44k
    
for (const auto *FD : RD->fields())1.94k
{
5088
      // Ignore (non-zero arrays of) empty records.
5089
2.44k
      QualType FT = FD->getType();
5090
3.56k
      while (const ConstantArrayType *AT =
5091
1.11k
             getContext().getAsConstantArrayType(FT)) {
5092
1.11k
        if (AT->getSize().getZExtValue() == 0)
5093
0
          return false;
5094
1.11k
        FT = AT->getElementType();
5095
1.11k
      }
5096
2.44k
      if (isEmptyRecord(getContext(), FT, true))
5097
22
        continue;
5098
5099
      // For compatibility with GCC, ignore empty bitfields in C++ mode.
5100
2.42k
      if (getContext().getLangOpts().CPlusPlus &&
5101
431
          FD->isZeroLengthBitField(getContext()))
5102
7
        continue;
5103
5104
2.42k
      uint64_t FldMembers;
5105
2.42k
      if (!isHomogeneousAggregate(FD->getType(), Base, FldMembers))
5106
542
        return false;
5107
5108
1.87k
      Members = (RD->isUnion() ?
5109
1.85k
                 
std::max(Members, FldMembers)27
: Members + FldMembers);
5110
1.87k
    }
5111
5112
1.39k
    if (!Base)
5113
29
      return false;
5114
5115
    // Ensure there is no padding.
5116
1.37k
    if (getContext().getTypeSize(Base) * Members !=
5117
1.37k
        getContext().getTypeSize(Ty))
5118
6
      return false;
5119
2.87k
  } else {
5120
2.87k
    Members = 1;
5121
2.87k
    if (const ComplexType *CT = Ty->getAs<ComplexType>()) {
5122
138
      Members = 2;
5123
138
      Ty = CT->getElementType();
5124
138
    }
5125
5126
    // Most ABIs only support float, double, and some vector type widths.
5127
2.87k
    if (!isHomogeneousAggregateBaseType(Ty))
5128
747
      return false;
5129
5130
    // The base type must be the same for all members.  Types that
5131
    // agree in both total size and mode (float vs. vector) are
5132
    // treated as being equivalent here.
5133
2.12k
    const Type *TyPtr = Ty.getTypePtr();
5134
2.12k
    if (!Base) {
5135
1.51k
      Base = TyPtr;
5136
      // If it's a non-power-of-2 vector, its size is already a power-of-2,
5137
      // so make sure to widen it explicitly.
5138
1.51k
      if (const VectorType *VT = Base->getAs<VectorType>()) {
5139
995
        QualType EltTy = VT->getElementType();
5140
995
        unsigned NumElements =
5141
995
            getContext().getTypeSize(VT) / getContext().getTypeSize(EltTy);
5142
995
        Base = getContext()
5143
995
                   .getVectorType(EltTy, NumElements, VT->getVectorKind())
5144
995
                   .getTypePtr();
5145
995
      }
5146
1.51k
    }
5147
5148
2.12k
    if (Base->isVectorType() != TyPtr->isVectorType() ||
5149
2.12k
        getContext().getTypeSize(Base) != getContext().getTypeSize(TyPtr))
5150
7
      return false;
5151
4.44k
  }
5152
4.44k
  return Members > 0 && isHomogeneousAggregateSmallEnough(Base, Members);
5153
4.44k
}
5154
5155
352
bool PPC64_SVR4_ABIInfo::isHomogeneousAggregateBaseType(QualType Ty) const {
5156
  // Homogeneous aggregates for ELFv2 must have base types of float,
5157
  // double, long double, or 128-bit vectors.
5158
352
  if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
5159
268
    if (BT->getKind() == BuiltinType::Float ||
5160
132
        BT->getKind() == BuiltinType::Double ||
5161
80
        BT->getKind() == BuiltinType::LongDouble ||
5162
80
        (getContext().getTargetInfo().hasFloat128Type() &&
5163
234
          
(BT->getKind() == BuiltinType::Float128)46
)) {
5164
234
      if (IsSoftFloatABI)
5165
34
        return false;
5166
200
      return true;
5167
200
    }
5168
268
  }
5169
118
  if (const VectorType *VT = Ty->getAs<VectorType>()) {
5170
80
    if (getContext().getTypeSize(VT) == 128)
5171
80
      return true;
5172
38
  }
5173
38
  return false;
5174
38
}
5175
5176
bool PPC64_SVR4_ABIInfo::isHomogeneousAggregateSmallEnough(
5177
602
    const Type *Base, uint64_t Members) const {
5178
  // Vector and fp128 types require one register, other floating point types
5179
  // require one or two registers depending on their size.
5180
602
  uint32_t NumRegs =
5181
602
      ((getContext().getTargetInfo().hasFloat128Type() &&
5182
108
          Base->isFloat128Type()) ||
5183
494
        Base->isVectorType()) ? 
1294
5184
308
                              : (getContext().getTypeSize(Base) + 63) / 64;
5185
5186
  // Homogeneous Aggregates may occupy at most 8 registers.
5187
602
  return Members * NumRegs <= 8;
5188
602
}
5189
5190
ABIArgInfo
5191
31.2k
PPC64_SVR4_ABIInfo::classifyArgumentType(QualType Ty) const {
5192
31.2k
  Ty = useFirstFieldIfTransparentUnion(Ty);
5193
5194
31.2k
  if (Ty->isAnyComplexType())
5195
46
    return ABIArgInfo::getDirect();
5196
5197
  // Non-Altivec vector types are passed in GPRs (smaller than 16 bytes)
5198
  // or via reference (larger than 16 bytes).
5199
31.2k
  if (Ty->isVectorType()) {
5200
3.11k
    uint64_t Size = getContext().getTypeSize(Ty);
5201
3.11k
    if (Size > 128)
5202
1
      return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
5203
3.11k
    else if (Size < 128) {
5204
3
      llvm::Type *CoerceTy = llvm::IntegerType::get(getVMContext(), Size);
5205
3
      return ABIArgInfo::getDirect(CoerceTy);
5206
3
    }
5207
31.1k
  }
5208
5209
31.1k
  if (const auto *EIT = Ty->getAs<ExtIntType>())
5210
8
    if (EIT->getNumBits() > 128)
5211
2
      return getNaturalAlignIndirect(Ty, /*ByVal=*/true);
5212
5213
31.1k
  if (isAggregateTypeForABI(Ty)) {
5214
157
    if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))
5215
49
      return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
5216
5217
108
    uint64_t ABIAlign = getParamTypeAlignment(Ty).getQuantity();
5218
108
    uint64_t TyAlign = getContext().getTypeAlignInChars(Ty).getQuantity();
5219
5220
    // ELFv2 homogeneous aggregates are passed as array types.
5221
108
    const Type *Base = nullptr;
5222
108
    uint64_t Members = 0;
5223
108
    if (Kind == ELFv2 &&
5224
82
        isHomogeneousAggregate(Ty, Base, Members)) {
5225
50
      llvm::Type *BaseTy = CGT.ConvertType(QualType(Base, 0));
5226