Coverage Report

Created: 2022-01-18 06:27

/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/CodeGen/TargetInfo.cpp
Line
Count
Source (jump to first uncovered line)
1
//===---- TargetInfo.cpp - Encapsulate target details -----------*- C++ -*-===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// These classes wrap the information about a call or function
10
// definition used to handle ABI compliancy.
11
//
12
//===----------------------------------------------------------------------===//
13
14
#include "TargetInfo.h"
15
#include "ABIInfo.h"
16
#include "CGBlocks.h"
17
#include "CGCXXABI.h"
18
#include "CGValue.h"
19
#include "CodeGenFunction.h"
20
#include "clang/AST/Attr.h"
21
#include "clang/AST/RecordLayout.h"
22
#include "clang/Basic/CodeGenOptions.h"
23
#include "clang/Basic/DiagnosticFrontend.h"
24
#include "clang/Basic/Builtins.h"
25
#include "clang/CodeGen/CGFunctionInfo.h"
26
#include "clang/CodeGen/SwiftCallingConv.h"
27
#include "llvm/ADT/SmallBitVector.h"
28
#include "llvm/ADT/StringExtras.h"
29
#include "llvm/ADT/StringSwitch.h"
30
#include "llvm/ADT/Triple.h"
31
#include "llvm/ADT/Twine.h"
32
#include "llvm/IR/DataLayout.h"
33
#include "llvm/IR/IntrinsicsNVPTX.h"
34
#include "llvm/IR/IntrinsicsS390.h"
35
#include "llvm/IR/Type.h"
36
#include "llvm/Support/raw_ostream.h"
37
#include <algorithm> // std::sort
38
39
using namespace clang;
40
using namespace CodeGen;
41
42
// Helper for coercing an aggregate argument or return value into an integer
43
// array of the same size (including padding) and alignment.  This alternate
44
// coercion happens only for the RenderScript ABI and can be removed after
45
// runtimes that rely on it are no longer supported.
46
//
47
// RenderScript assumes that the size of the argument / return value in the IR
48
// is the same as the size of the corresponding qualified type. This helper
49
// coerces the aggregate type into an array of the same size (including
50
// padding).  This coercion is used in lieu of expansion of struct members or
51
// other canonical coercions that return a coerced-type of larger size.
52
//
53
// Ty          - The argument / return value type
54
// Context     - The associated ASTContext
55
// LLVMContext - The associated LLVMContext
56
static ABIArgInfo coerceToIntArray(QualType Ty,
57
                                   ASTContext &Context,
58
19
                                   llvm::LLVMContext &LLVMContext) {
59
  // Alignment and Size are measured in bits.
60
19
  const uint64_t Size = Context.getTypeSize(Ty);
61
19
  const uint64_t Alignment = Context.getTypeAlign(Ty);
62
19
  llvm::Type *IntType = llvm::Type::getIntNTy(LLVMContext, Alignment);
63
19
  const uint64_t NumElements = (Size + Alignment - 1) / Alignment;
64
19
  return ABIArgInfo::getDirect(llvm::ArrayType::get(IntType, NumElements));
65
19
}
66
67
static void AssignToArrayRange(CodeGen::CGBuilderTy &Builder,
68
                               llvm::Value *Array,
69
                               llvm::Value *Value,
70
                               unsigned FirstIndex,
71
31
                               unsigned LastIndex) {
72
  // Alternatively, we could emit this as a loop in the source.
73
572
  for (unsigned I = FirstIndex; I <= LastIndex; 
++I541
) {
74
541
    llvm::Value *Cell =
75
541
        Builder.CreateConstInBoundsGEP1_32(Builder.getInt8Ty(), Array, I);
76
541
    Builder.CreateAlignedStore(Value, Cell, CharUnits::One());
77
541
  }
78
31
}
79
80
173k
static bool isAggregateTypeForABI(QualType T) {
81
173k
  return !CodeGenFunction::hasScalarEvaluationKind(T) ||
82
173k
         
T->isMemberFunctionPointerType()167k
;
83
173k
}
84
85
ABIArgInfo ABIInfo::getNaturalAlignIndirect(QualType Ty, bool ByVal,
86
                                            bool Realign,
87
3.36k
                                            llvm::Type *Padding) const {
88
3.36k
  return ABIArgInfo::getIndirect(getContext().getTypeAlignInChars(Ty), ByVal,
89
3.36k
                                 Realign, Padding);
90
3.36k
}
91
92
ABIArgInfo
93
28
ABIInfo::getNaturalAlignIndirectInReg(QualType Ty, bool Realign) const {
94
28
  return ABIArgInfo::getIndirectInReg(getContext().getTypeAlignInChars(Ty),
95
28
                                      /*ByVal*/ false, Realign);
96
28
}
97
98
Address ABIInfo::EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr,
99
0
                             QualType Ty) const {
100
0
  return Address::invalid();
101
0
}
102
103
271k
bool ABIInfo::isPromotableIntegerTypeForABI(QualType Ty) const {
104
271k
  if (Ty->isPromotableIntegerType())
105
12.7k
    return true;
106
107
258k
  if (const auto *EIT = Ty->getAs<BitIntType>())
108
341
    if (EIT->getNumBits() < getContext().getTypeSize(getContext().IntTy))
109
130
      return true;
110
111
258k
  return false;
112
258k
}
113
114
35.2k
ABIInfo::~ABIInfo() {}
115
116
/// Does the given lowering require more than the given number of
117
/// registers when expanded?
118
///
119
/// This is intended to be the basis of a reasonable basic implementation
120
/// of should{Pass,Return}IndirectlyForSwift.
121
///
122
/// For most targets, a limit of four total registers is reasonable; this
123
/// limits the amount of code required in order to move around the value
124
/// in case it wasn't produced immediately prior to the call by the caller
125
/// (or wasn't produced in exactly the right registers) or isn't used
126
/// immediately within the callee.  But some targets may need to further
127
/// limit the register count due to an inability to support that many
128
/// return registers.
129
static bool occupiesMoreThan(CodeGenTypes &cgt,
130
                             ArrayRef<llvm::Type*> scalarTypes,
131
1.05k
                             unsigned maxAllRegisters) {
132
1.05k
  unsigned intCount = 0, fpCount = 0;
133
4.18k
  for (llvm::Type *type : scalarTypes) {
134
4.18k
    if (type->isPointerTy()) {
135
0
      intCount++;
136
4.18k
    } else if (auto intTy = dyn_cast<llvm::IntegerType>(type)) {
137
2.30k
      auto ptrWidth = cgt.getTarget().getPointerWidth(0);
138
2.30k
      intCount += (intTy->getBitWidth() + ptrWidth - 1) / ptrWidth;
139
2.30k
    } else {
140
1.88k
      assert(type->isVectorTy() || type->isFloatingPointTy());
141
0
      fpCount++;
142
1.88k
    }
143
4.18k
  }
144
145
1.05k
  return (intCount + fpCount > maxAllRegisters);
146
1.05k
}
147
148
bool SwiftABIInfo::isLegalVectorTypeForSwift(CharUnits vectorSize,
149
                                             llvm::Type *eltTy,
150
440
                                             unsigned numElts) const {
151
  // The default implementation of this assumes that the target guarantees
152
  // 128-bit SIMD support but nothing more.
153
440
  return (vectorSize.getQuantity() > 8 && 
vectorSize.getQuantity() <= 16432
);
154
440
}
155
156
static CGCXXABI::RecordArgABI getRecordArgABI(const RecordType *RT,
157
19.9k
                                              CGCXXABI &CXXABI) {
158
19.9k
  const CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(RT->getDecl());
159
19.9k
  if (!RD) {
160
3.85k
    if (!RT->getDecl()->canPassInRegisters())
161
16
      return CGCXXABI::RAA_Indirect;
162
3.83k
    return CGCXXABI::RAA_Default;
163
3.85k
  }
164
16.1k
  return CXXABI.getRecordArgABI(RD);
165
19.9k
}
166
167
static CGCXXABI::RecordArgABI getRecordArgABI(QualType T,
168
8.37k
                                              CGCXXABI &CXXABI) {
169
8.37k
  const RecordType *RT = T->getAs<RecordType>();
170
8.37k
  if (!RT)
171
3.48k
    return CGCXXABI::RAA_Default;
172
4.88k
  return getRecordArgABI(RT, CXXABI);
173
8.37k
}
174
175
static bool classifyReturnType(const CGCXXABI &CXXABI, CGFunctionInfo &FI,
176
214k
                               const ABIInfo &Info) {
177
214k
  QualType Ty = FI.getReturnType();
178
179
214k
  if (const auto *RT = Ty->getAs<RecordType>())
180
5.92k
    if (!isa<CXXRecordDecl>(RT->getDecl()) &&
181
5.92k
        
!RT->getDecl()->canPassInRegisters()1.29k
) {
182
12
      FI.getReturnInfo() = Info.getNaturalAlignIndirect(Ty);
183
12
      return true;
184
12
    }
185
186
214k
  return CXXABI.classifyReturnType(FI);
187
214k
}
188
189
/// Pass transparent unions as if they were the type of the first element. Sema
190
/// should ensure that all elements of the union have the same "machine type".
191
425k
static QualType useFirstFieldIfTransparentUnion(QualType Ty) {
192
425k
  if (const RecordType *UT = Ty->getAsUnionType()) {
193
139
    const RecordDecl *UD = UT->getDecl();
194
139
    if (UD->hasAttr<TransparentUnionAttr>()) {
195
18
      assert(!UD->field_empty() && "sema created an empty transparent union");
196
0
      return UD->field_begin()->getType();
197
18
    }
198
139
  }
199
425k
  return Ty;
200
425k
}
201
202
260k
CGCXXABI &ABIInfo::getCXXABI() const {
203
260k
  return CGT.getCXXABI();
204
260k
}
205
206
421k
ASTContext &ABIInfo::getContext() const {
207
421k
  return CGT.getContext();
208
421k
}
209
210
16.9k
llvm::LLVMContext &ABIInfo::getVMContext() const {
211
16.9k
  return CGT.getLLVMContext();
212
16.9k
}
213
214
21.3k
const llvm::DataLayout &ABIInfo::getDataLayout() const {
215
21.3k
  return CGT.getDataLayout();
216
21.3k
}
217
218
34.3k
const TargetInfo &ABIInfo::getTarget() const {
219
34.3k
  return CGT.getTarget();
220
34.3k
}
221
222
1.04k
const CodeGenOptions &ABIInfo::getCodeGenOpts() const {
223
1.04k
  return CGT.getCodeGenOpts();
224
1.04k
}
225
226
5.04k
bool ABIInfo::isAndroid() const { return getTarget().getTriple().isAndroid(); }
227
228
0
bool ABIInfo::isHomogeneousAggregateBaseType(QualType Ty) const {
229
0
  return false;
230
0
}
231
232
bool ABIInfo::isHomogeneousAggregateSmallEnough(const Type *Base,
233
0
                                                uint64_t Members) const {
234
0
  return false;
235
0
}
236
237
0
LLVM_DUMP_METHOD void ABIArgInfo::dump() const {
238
0
  raw_ostream &OS = llvm::errs();
239
0
  OS << "(ABIArgInfo Kind=";
240
0
  switch (TheKind) {
241
0
  case Direct:
242
0
    OS << "Direct Type=";
243
0
    if (llvm::Type *Ty = getCoerceToType())
244
0
      Ty->print(OS);
245
0
    else
246
0
      OS << "null";
247
0
    break;
248
0
  case Extend:
249
0
    OS << "Extend";
250
0
    break;
251
0
  case Ignore:
252
0
    OS << "Ignore";
253
0
    break;
254
0
  case InAlloca:
255
0
    OS << "InAlloca Offset=" << getInAllocaFieldIndex();
256
0
    break;
257
0
  case Indirect:
258
0
    OS << "Indirect Align=" << getIndirectAlign().getQuantity()
259
0
       << " ByVal=" << getIndirectByVal()
260
0
       << " Realign=" << getIndirectRealign();
261
0
    break;
262
0
  case IndirectAliased:
263
0
    OS << "Indirect Align=" << getIndirectAlign().getQuantity()
264
0
       << " AadrSpace=" << getIndirectAddrSpace()
265
0
       << " Realign=" << getIndirectRealign();
266
0
    break;
267
0
  case Expand:
268
0
    OS << "Expand";
269
0
    break;
270
0
  case CoerceAndExpand:
271
0
    OS << "CoerceAndExpand Type=";
272
0
    getCoerceAndExpandType()->print(OS);
273
0
    break;
274
0
  }
275
0
  OS << ")\n";
276
0
}
277
278
// Dynamically round a pointer up to a multiple of the given alignment.
279
static llvm::Value *emitRoundPointerUpToAlignment(CodeGenFunction &CGF,
280
                                                  llvm::Value *Ptr,
281
123
                                                  CharUnits Align) {
282
123
  llvm::Value *PtrAsInt = Ptr;
283
  // OverflowArgArea = (OverflowArgArea + Align - 1) & -Align;
284
123
  PtrAsInt = CGF.Builder.CreatePtrToInt(PtrAsInt, CGF.IntPtrTy);
285
123
  PtrAsInt = CGF.Builder.CreateAdd(PtrAsInt,
286
123
        llvm::ConstantInt::get(CGF.IntPtrTy, Align.getQuantity() - 1));
287
123
  PtrAsInt = CGF.Builder.CreateAnd(PtrAsInt,
288
123
           llvm::ConstantInt::get(CGF.IntPtrTy, -Align.getQuantity()));
289
123
  PtrAsInt = CGF.Builder.CreateIntToPtr(PtrAsInt,
290
123
                                        Ptr->getType(),
291
123
                                        Ptr->getName() + ".aligned");
292
123
  return PtrAsInt;
293
123
}
294
295
/// Emit va_arg for a platform using the common void* representation,
296
/// where arguments are simply emitted in an array of slots on the stack.
297
///
298
/// This version implements the core direct-value passing rules.
299
///
300
/// \param SlotSize - The size and alignment of a stack slot.
301
///   Each argument will be allocated to a multiple of this number of
302
///   slots, and all the slots will be aligned to this value.
303
/// \param AllowHigherAlign - The slot alignment is not a cap;
304
///   an argument type with an alignment greater than the slot size
305
///   will be emitted on a higher-alignment address, potentially
306
///   leaving one or more empty slots behind as padding.  If this
307
///   is false, the returned address might be less-aligned than
308
///   DirectAlign.
309
static Address emitVoidPtrDirectVAArg(CodeGenFunction &CGF,
310
                                      Address VAListAddr,
311
                                      llvm::Type *DirectTy,
312
                                      CharUnits DirectSize,
313
                                      CharUnits DirectAlign,
314
                                      CharUnits SlotSize,
315
319
                                      bool AllowHigherAlign) {
316
  // Cast the element type to i8* if necessary.  Some platforms define
317
  // va_list as a struct containing an i8* instead of just an i8*.
318
319
  if (VAListAddr.getElementType() != CGF.Int8PtrTy)
319
59
    VAListAddr = CGF.Builder.CreateElementBitCast(VAListAddr, CGF.Int8PtrTy);
320
321
319
  llvm::Value *Ptr = CGF.Builder.CreateLoad(VAListAddr, "argp.cur");
322
323
  // If the CC aligns values higher than the slot size, do so if needed.
324
319
  Address Addr = Address::invalid();
325
319
  if (AllowHigherAlign && 
DirectAlign > SlotSize278
) {
326
104
    Addr = Address(emitRoundPointerUpToAlignment(CGF, Ptr, DirectAlign),
327
104
                                                 DirectAlign);
328
215
  } else {
329
215
    Addr = Address(Ptr, SlotSize);
330
215
  }
331
332
  // Advance the pointer past the argument, then store that back.
333
319
  CharUnits FullDirectSize = DirectSize.alignTo(SlotSize);
334
319
  Address NextPtr =
335
319
      CGF.Builder.CreateConstInBoundsByteGEP(Addr, FullDirectSize, "argp.next");
336
319
  CGF.Builder.CreateStore(NextPtr.getPointer(), VAListAddr);
337
338
  // If the argument is smaller than a slot, and this is a big-endian
339
  // target, the argument will be right-adjusted in its slot.
340
319
  if (DirectSize < SlotSize && 
CGF.CGM.getDataLayout().isBigEndian()41
&&
341
319
      
!DirectTy->isStructTy()6
) {
342
0
    Addr = CGF.Builder.CreateConstInBoundsByteGEP(Addr, SlotSize - DirectSize);
343
0
  }
344
345
319
  Addr = CGF.Builder.CreateElementBitCast(Addr, DirectTy);
346
319
  return Addr;
347
319
}
348
349
/// Emit va_arg for a platform using the common void* representation,
350
/// where arguments are simply emitted in an array of slots on the stack.
351
///
352
/// \param IsIndirect - Values of this type are passed indirectly.
353
/// \param ValueInfo - The size and alignment of this type, generally
354
///   computed with getContext().getTypeInfoInChars(ValueTy).
355
/// \param SlotSizeAndAlign - The size and alignment of a stack slot.
356
///   Each argument will be allocated to a multiple of this number of
357
///   slots, and all the slots will be aligned to this value.
358
/// \param AllowHigherAlign - The slot alignment is not a cap;
359
///   an argument type with an alignment greater than the slot size
360
///   will be emitted on a higher-alignment address, potentially
361
///   leaving one or more empty slots behind as padding.
362
static Address emitVoidPtrVAArg(CodeGenFunction &CGF, Address VAListAddr,
363
                                QualType ValueTy, bool IsIndirect,
364
                                TypeInfoChars ValueInfo,
365
                                CharUnits SlotSizeAndAlign,
366
319
                                bool AllowHigherAlign) {
367
  // The size and alignment of the value that was passed directly.
368
319
  CharUnits DirectSize, DirectAlign;
369
319
  if (IsIndirect) {
370
48
    DirectSize = CGF.getPointerSize();
371
48
    DirectAlign = CGF.getPointerAlign();
372
271
  } else {
373
271
    DirectSize = ValueInfo.Width;
374
271
    DirectAlign = ValueInfo.Align;
375
271
  }
376
377
  // Cast the address we've calculated to the right type.
378
319
  llvm::Type *DirectTy = CGF.ConvertTypeForMem(ValueTy);
379
319
  if (IsIndirect)
380
48
    DirectTy = DirectTy->getPointerTo(0);
381
382
319
  Address Addr = emitVoidPtrDirectVAArg(CGF, VAListAddr, DirectTy,
383
319
                                        DirectSize, DirectAlign,
384
319
                                        SlotSizeAndAlign,
385
319
                                        AllowHigherAlign);
386
387
319
  if (IsIndirect) {
388
48
    Addr = Address(CGF.Builder.CreateLoad(Addr), ValueInfo.Align);
389
48
  }
390
391
319
  return Addr;
392
393
319
}
394
395
static Address complexTempStructure(CodeGenFunction &CGF, Address VAListAddr,
396
                                    QualType Ty, CharUnits SlotSize,
397
0
                                    CharUnits EltSize, const ComplexType *CTy) {
398
0
  Address Addr =
399
0
      emitVoidPtrDirectVAArg(CGF, VAListAddr, CGF.Int8Ty, SlotSize * 2,
400
0
                             SlotSize, SlotSize, /*AllowHigher*/ true);
401
402
0
  Address RealAddr = Addr;
403
0
  Address ImagAddr = RealAddr;
404
0
  if (CGF.CGM.getDataLayout().isBigEndian()) {
405
0
    RealAddr =
406
0
        CGF.Builder.CreateConstInBoundsByteGEP(RealAddr, SlotSize - EltSize);
407
0
    ImagAddr = CGF.Builder.CreateConstInBoundsByteGEP(ImagAddr,
408
0
                                                      2 * SlotSize - EltSize);
409
0
  } else {
410
0
    ImagAddr = CGF.Builder.CreateConstInBoundsByteGEP(RealAddr, SlotSize);
411
0
  }
412
413
0
  llvm::Type *EltTy = CGF.ConvertTypeForMem(CTy->getElementType());
414
0
  RealAddr = CGF.Builder.CreateElementBitCast(RealAddr, EltTy);
415
0
  ImagAddr = CGF.Builder.CreateElementBitCast(ImagAddr, EltTy);
416
0
  llvm::Value *Real = CGF.Builder.CreateLoad(RealAddr, ".vareal");
417
0
  llvm::Value *Imag = CGF.Builder.CreateLoad(ImagAddr, ".vaimag");
418
419
0
  Address Temp = CGF.CreateMemTemp(Ty, "vacplx");
420
0
  CGF.EmitStoreOfComplex({Real, Imag}, CGF.MakeAddrLValue(Temp, Ty),
421
0
                         /*init*/ true);
422
0
  return Temp;
423
0
}
424
425
static Address emitMergePHI(CodeGenFunction &CGF,
426
                            Address Addr1, llvm::BasicBlock *Block1,
427
                            Address Addr2, llvm::BasicBlock *Block2,
428
388
                            const llvm::Twine &Name = "") {
429
388
  assert(Addr1.getType() == Addr2.getType());
430
0
  llvm::PHINode *PHI = CGF.Builder.CreatePHI(Addr1.getType(), 2, Name);
431
388
  PHI->addIncoming(Addr1.getPointer(), Block1);
432
388
  PHI->addIncoming(Addr2.getPointer(), Block2);
433
388
  CharUnits Align = std::min(Addr1.getAlignment(), Addr2.getAlignment());
434
388
  return Address(PHI, Addr1.getElementType(), Align);
435
388
}
436
437
35.0k
TargetCodeGenInfo::~TargetCodeGenInfo() = default;
438
439
// If someone can figure out a general rule for this, that would be great.
440
// It's probably just doomed to be platform-dependent, though.
441
5
unsigned TargetCodeGenInfo::getSizeOfUnwindException() const {
442
  // Verified for:
443
  //   x86-64     FreeBSD, Linux, Darwin
444
  //   x86-32     FreeBSD, Linux, Darwin
445
  //   PowerPC    Linux, Darwin
446
  //   ARM        Darwin (*not* EABI)
447
  //   AArch64    Linux
448
5
  return 32;
449
5
}
450
451
bool TargetCodeGenInfo::isNoProtoCallVariadic(const CallArgList &args,
452
189
                                     const FunctionNoProtoType *fnType) const {
453
  // The following conventions are known to require this to be false:
454
  //   x86_stdcall
455
  //   MIPS
456
  // For everything else, we just prefer false unless we opt out.
457
189
  return false;
458
189
}
459
460
void
461
TargetCodeGenInfo::getDependentLibraryOption(llvm::StringRef Lib,
462
2
                                             llvm::SmallString<24> &Opt) const {
463
  // This assumes the user is passing a library name like "rt" instead of a
464
  // filename like "librt.a/so", and that they don't care whether it's static or
465
  // dynamic.
466
2
  Opt = "-l";
467
2
  Opt += Lib;
468
2
}
469
470
58
unsigned TargetCodeGenInfo::getOpenCLKernelCallingConv() const {
471
  // OpenCL kernels are called via an explicit runtime API with arguments
472
  // set with clSetKernelArg(), not as normal sub-functions.
473
  // Return SPIR_KERNEL by default as the kernel calling convention to
474
  // ensure the fingerprint is fixed such way that each OpenCL argument
475
  // gets one matching argument in the produced kernel function argument
476
  // list to enable feasible implementation of clSetKernelArg() with
477
  // aggregates etc. In case we would use the default C calling conv here,
478
  // clSetKernelArg() might break depending on the target-specific
479
  // conventions; different targets might split structs passed as values
480
  // to multiple function arguments etc.
481
58
  return llvm::CallingConv::SPIR_KERNEL;
482
58
}
483
484
llvm::Constant *TargetCodeGenInfo::getNullPointer(const CodeGen::CodeGenModule &CGM,
485
34.1k
    llvm::PointerType *T, QualType QT) const {
486
34.1k
  return llvm::ConstantPointerNull::get(T);
487
34.1k
}
488
489
LangAS TargetCodeGenInfo::getGlobalVarAddressSpace(CodeGenModule &CGM,
490
69.6k
                                                   const VarDecl *D) const {
491
69.6k
  assert(!CGM.getLangOpts().OpenCL &&
492
69.6k
         !(CGM.getLangOpts().CUDA && CGM.getLangOpts().CUDAIsDevice) &&
493
69.6k
         "Address space agnostic languages only");
494
69.6k
  return D ? 
D->getType().getAddressSpace()67.6k
:
LangAS::Default1.97k
;
495
69.6k
}
496
497
llvm::Value *TargetCodeGenInfo::performAddrSpaceCast(
498
    CodeGen::CodeGenFunction &CGF, llvm::Value *Src, LangAS SrcAddr,
499
5.88k
    LangAS DestAddr, llvm::Type *DestTy, bool isNonNull) const {
500
  // Since target may map different address spaces in AST to the same address
501
  // space, an address space conversion may end up as a bitcast.
502
5.88k
  if (auto *C = dyn_cast<llvm::Constant>(Src))
503
50
    return performAddrSpaceCast(CGF.CGM, C, SrcAddr, DestAddr, DestTy);
504
  // Try to preserve the source's name to make IR more readable.
505
5.83k
  return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
506
5.83k
      Src, DestTy, Src->hasName() ? 
Src->getName() + ".ascast"5.63k
:
""202
);
507
5.88k
}
508
509
llvm::Constant *
510
TargetCodeGenInfo::performAddrSpaceCast(CodeGenModule &CGM, llvm::Constant *Src,
511
                                        LangAS SrcAddr, LangAS DestAddr,
512
238
                                        llvm::Type *DestTy) const {
513
  // Since target may map different address spaces in AST to the same address
514
  // space, an address space conversion may end up as a bitcast.
515
238
  return llvm::ConstantExpr::getPointerCast(Src, DestTy);
516
238
}
517
518
llvm::SyncScope::ID
519
TargetCodeGenInfo::getLLVMSyncScopeID(const LangOptions &LangOpts,
520
                                      SyncScope Scope,
521
                                      llvm::AtomicOrdering Ordering,
522
0
                                      llvm::LLVMContext &Ctx) const {
523
0
  return Ctx.getOrInsertSyncScopeID(""); /* default sync scope */
524
0
}
525
526
static bool isEmptyRecord(ASTContext &Context, QualType T, bool AllowArrays);
527
528
/// isEmptyField - Return true iff a the field is "empty", that is it
529
/// is an unnamed bit-field or an (array of) empty record(s).
530
static bool isEmptyField(ASTContext &Context, const FieldDecl *FD,
531
5.26k
                         bool AllowArrays) {
532
5.26k
  if (FD->isUnnamedBitfield())
533
127
    return true;
534
535
5.13k
  QualType FT = FD->getType();
536
537
  // Constant arrays of empty records count as empty, strip them off.
538
  // Constant arrays of zero length always count as empty.
539
5.13k
  bool WasArray = false;
540
5.13k
  if (AllowArrays)
541
6.83k
    
while (const ConstantArrayType *4.96k
AT = Context.getAsConstantArrayType(FT)) {
542
1.88k
      if (AT->getSize() == 0)
543
11
        return true;
544
1.86k
      FT = AT->getElementType();
545
      // The [[no_unique_address]] special case below does not apply to
546
      // arrays of C++ empty records, so we need to remember this fact.
547
1.86k
      WasArray = true;
548
1.86k
    }
549
550
5.12k
  const RecordType *RT = FT->getAs<RecordType>();
551
5.12k
  if (!RT)
552
4.71k
    return false;
553
554
  // C++ record fields are never empty, at least in the Itanium ABI.
555
  //
556
  // FIXME: We should use a predicate for whether this behavior is true in the
557
  // current ABI.
558
  //
559
  // The exception to the above rule are fields marked with the
560
  // [[no_unique_address]] attribute (since C++20).  Those do count as empty
561
  // according to the Itanium ABI.  The exception applies only to records,
562
  // not arrays of records, so we must also check whether we stripped off an
563
  // array type above.
564
411
  if (isa<CXXRecordDecl>(RT->getDecl()) &&
565
411
      
(85
WasArray85
||
!FD->hasAttr<NoUniqueAddressAttr>()68
))
566
78
    return false;
567
568
333
  return isEmptyRecord(Context, FT, AllowArrays);
569
411
}
570
571
/// isEmptyRecord - Return true iff a structure contains only empty
572
/// fields. Note that a structure with a flexible array member is not
573
/// considered empty.
574
9.18k
static bool isEmptyRecord(ASTContext &Context, QualType T, bool AllowArrays) {
575
9.18k
  const RecordType *RT = T->getAs<RecordType>();
576
9.18k
  if (!RT)
577
4.51k
    return false;
578
4.66k
  const RecordDecl *RD = RT->getDecl();
579
4.66k
  if (RD->hasFlexibleArrayMember())
580
10
    return false;
581
582
  // If this is a C++ record, check the bases first.
583
4.65k
  if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD))
584
1.01k
    for (const auto &I : CXXRD->bases())
585
115
      if (!isEmptyRecord(Context, I.getType(), true))
586
107
        return false;
587
588
4.54k
  for (const auto *I : RD->fields())
589
4.21k
    if (!isEmptyField(Context, I, AllowArrays))
590
3.93k
      return false;
591
619
  return true;
592
4.54k
}
593
594
/// isSingleElementStruct - Determine if a structure is a "single
595
/// element struct", i.e. it has exactly one non-empty field or
596
/// exactly one field which is itself a single element
597
/// struct. Structures with flexible array members are never
598
/// considered single element structs.
599
///
600
/// \return The field declaration for the single non-empty field, if
601
/// it exists.
602
65.6k
static const Type *isSingleElementStruct(QualType T, ASTContext &Context) {
603
65.6k
  const RecordType *RT = T->getAs<RecordType>();
604
65.6k
  if (!RT)
605
64.9k
    return nullptr;
606
607
715
  const RecordDecl *RD = RT->getDecl();
608
715
  if (RD->hasFlexibleArrayMember())
609
0
    return nullptr;
610
611
715
  const Type *Found = nullptr;
612
613
  // If this is a C++ record, check the bases first.
614
715
  if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
615
269
    for (const auto &I : CXXRD->bases()) {
616
      // Ignore empty records.
617
49
      if (isEmptyRecord(Context, I.getType(), true))
618
6
        continue;
619
620
      // If we already found an element then this isn't a single-element struct.
621
43
      if (Found)
622
11
        return nullptr;
623
624
      // If this is non-empty and not a single element struct, the composite
625
      // cannot be a single element struct.
626
32
      Found = isSingleElementStruct(I.getType(), Context);
627
32
      if (!Found)
628
9
        return nullptr;
629
32
    }
630
269
  }
631
632
  // Check for single element.
633
937
  
for (const auto *FD : RD->fields())695
{
634
937
    QualType FT = FD->getType();
635
636
    // Ignore empty fields.
637
937
    if (isEmptyField(Context, FD, true))
638
19
      continue;
639
640
    // If we already found an element then this isn't a single-element
641
    // struct.
642
918
    if (Found)
643
258
      return nullptr;
644
645
    // Treat single element arrays as the element.
646
674
    
while (const ConstantArrayType *660
AT = Context.getAsConstantArrayType(FT)) {
647
176
      if (AT->getSize().getZExtValue() != 1)
648
162
        break;
649
14
      FT = AT->getElementType();
650
14
    }
651
652
660
    if (!isAggregateTypeForABI(FT)) {
653
480
      Found = FT.getTypePtr();
654
480
    } else {
655
180
      Found = isSingleElementStruct(FT, Context);
656
180
      if (!Found)
657
178
        return nullptr;
658
180
    }
659
660
  }
660
661
  // We don't consider a struct a single-element struct if it has
662
  // padding beyond the element type.
663
259
  if (Found && 
Context.getTypeSize(Found) != Context.getTypeSize(T)236
)
664
3
    return nullptr;
665
666
256
  return Found;
667
259
}
668
669
namespace {
670
Address EmitVAArgInstr(CodeGenFunction &CGF, Address VAListAddr, QualType Ty,
671
8
                       const ABIArgInfo &AI) {
672
  // This default implementation defers to the llvm backend's va_arg
673
  // instruction. It can handle only passing arguments directly
674
  // (typically only handled in the backend for primitive types), or
675
  // aggregates passed indirectly by pointer (NOTE: if the "byval"
676
  // flag has ABI impact in the callee, this implementation cannot
677
  // work.)
678
679
  // Only a few cases are covered here at the moment -- those needed
680
  // by the default abi.
681
8
  llvm::Value *Val;
682
683
8
  if (AI.isIndirect()) {
684
1
    assert(!AI.getPaddingType() &&
685
1
           "Unexpected PaddingType seen in arginfo in generic VAArg emitter!");
686
0
    assert(
687
1
        !AI.getIndirectRealign() &&
688
1
        "Unexpected IndirectRealign seen in arginfo in generic VAArg emitter!");
689
690
0
    auto TyInfo = CGF.getContext().getTypeInfoInChars(Ty);
691
1
    CharUnits TyAlignForABI = TyInfo.Align;
692
693
1
    llvm::Type *BaseTy =
694
1
        llvm::PointerType::getUnqual(CGF.ConvertTypeForMem(Ty));
695
1
    llvm::Value *Addr =
696
1
        CGF.Builder.CreateVAArg(VAListAddr.getPointer(), BaseTy);
697
1
    return Address(Addr, TyAlignForABI);
698
7
  } else {
699
7
    assert((AI.isDirect() || AI.isExtend()) &&
700
7
           "Unexpected ArgInfo Kind in generic VAArg emitter!");
701
702
0
    assert(!AI.getInReg() &&
703
7
           "Unexpected InReg seen in arginfo in generic VAArg emitter!");
704
0
    assert(!AI.getPaddingType() &&
705
7
           "Unexpected PaddingType seen in arginfo in generic VAArg emitter!");
706
0
    assert(!AI.getDirectOffset() &&
707
7
           "Unexpected DirectOffset seen in arginfo in generic VAArg emitter!");
708
0
    assert(!AI.getCoerceToType() &&
709
7
           "Unexpected CoerceToType seen in arginfo in generic VAArg emitter!");
710
711
0
    Address Temp = CGF.CreateMemTemp(Ty, "varet");
712
7
    Val = CGF.Builder.CreateVAArg(VAListAddr.getPointer(), CGF.ConvertType(Ty));
713
7
    CGF.Builder.CreateStore(Val, Temp);
714
7
    return Temp;
715
7
  }
716
8
}
717
718
/// DefaultABIInfo - The default implementation for ABI specific
719
/// details. This implementation provides information which results in
720
/// self-consistent and sensible LLVM IR generation, but does not
721
/// conform to any particular ABI.
722
class DefaultABIInfo : public ABIInfo {
723
public:
724
696
  DefaultABIInfo(CodeGen::CodeGenTypes &CGT) : ABIInfo(CGT) {}
725
726
  ABIArgInfo classifyReturnType(QualType RetTy) const;
727
  ABIArgInfo classifyArgumentType(QualType RetTy) const;
728
729
606
  void computeInfo(CGFunctionInfo &FI) const override {
730
606
    if (!getCXXABI().classifyReturnType(FI))
731
605
      FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
732
606
    for (auto &I : FI.arguments())
733
806
      I.info = classifyArgumentType(I.type);
734
606
  }
735
736
  Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
737
3
                    QualType Ty) const override {
738
3
    return EmitVAArgInstr(CGF, VAListAddr, Ty, classifyArgumentType(Ty));
739
3
  }
740
};
741
742
class DefaultTargetCodeGenInfo : public TargetCodeGenInfo {
743
public:
744
  DefaultTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT)
745
13
      : TargetCodeGenInfo(std::make_unique<DefaultABIInfo>(CGT)) {}
746
};
747
748
2.07k
ABIArgInfo DefaultABIInfo::classifyArgumentType(QualType Ty) const {
749
2.07k
  Ty = useFirstFieldIfTransparentUnion(Ty);
750
751
2.07k
  if (isAggregateTypeForABI(Ty)) {
752
    // Records with non-trivial destructors/copy-constructors should not be
753
    // passed by value.
754
106
    if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))
755
2
      return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
756
757
104
    return getNaturalAlignIndirect(Ty);
758
106
  }
759
760
  // Treat an enum type as its underlying type.
761
1.96k
  if (const EnumType *EnumTy = Ty->getAs<EnumType>())
762
8
    Ty = EnumTy->getDecl()->getIntegerType();
763
764
1.96k
  ASTContext &Context = getContext();
765
1.96k
  if (const auto *EIT = Ty->getAs<BitIntType>())
766
64
    if (EIT->getNumBits() >
767
64
        Context.getTypeSize(Context.getTargetInfo().hasInt128Type()
768
64
                                ? 
Context.Int128Ty16
769
64
                                : 
Context.LongLongTy48
))
770
28
      return getNaturalAlignIndirect(Ty);
771
772
1.94k
  return (isPromotableIntegerTypeForABI(Ty) ? 
ABIArgInfo::getExtend(Ty)73
773
1.94k
                                            : 
ABIArgInfo::getDirect()1.86k
);
774
1.96k
}
775
776
1.78k
ABIArgInfo DefaultABIInfo::classifyReturnType(QualType RetTy) const {
777
1.78k
  if (RetTy->isVoidType())
778
962
    return ABIArgInfo::getIgnore();
779
780
819
  if (isAggregateTypeForABI(RetTy))
781
56
    return getNaturalAlignIndirect(RetTy);
782
783
  // Treat an enum type as its underlying type.
784
763
  if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
785
0
    RetTy = EnumTy->getDecl()->getIntegerType();
786
787
763
  if (const auto *EIT = RetTy->getAs<BitIntType>())
788
67
    if (EIT->getNumBits() >
789
67
        getContext().getTypeSize(getContext().getTargetInfo().hasInt128Type()
790
67
                                     ? 
getContext().Int128Ty14
791
67
                                     : 
getContext().LongLongTy53
))
792
43
      return getNaturalAlignIndirect(RetTy);
793
794
720
  return (isPromotableIntegerTypeForABI(RetTy) ? 
ABIArgInfo::getExtend(RetTy)68
795
720
                                               : 
ABIArgInfo::getDirect()652
);
796
763
}
797
798
//===----------------------------------------------------------------------===//
799
// WebAssembly ABI Implementation
800
//
801
// This is a very simple ABI that relies a lot on DefaultABIInfo.
802
//===----------------------------------------------------------------------===//
803
804
class WebAssemblyABIInfo final : public SwiftABIInfo {
805
public:
806
  enum ABIKind {
807
    MVP = 0,
808
    ExperimentalMV = 1,
809
  };
810
811
private:
812
  DefaultABIInfo defaultInfo;
813
  ABIKind Kind;
814
815
public:
816
  explicit WebAssemblyABIInfo(CodeGen::CodeGenTypes &CGT, ABIKind Kind)
817
34
      : SwiftABIInfo(CGT), defaultInfo(CGT), Kind(Kind) {}
818
819
private:
820
  ABIArgInfo classifyReturnType(QualType RetTy) const;
821
  ABIArgInfo classifyArgumentType(QualType Ty) const;
822
823
  // DefaultABIInfo's classifyReturnType and classifyArgumentType are
824
  // non-virtual, but computeInfo and EmitVAArg are virtual, so we
825
  // overload them.
826
344
  void computeInfo(CGFunctionInfo &FI) const override {
827
344
    if (!getCXXABI().classifyReturnType(FI))
828
336
      FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
829
344
    for (auto &Arg : FI.arguments())
830
336
      Arg.info = classifyArgumentType(Arg.type);
831
344
  }
832
833
  Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
834
                    QualType Ty) const override;
835
836
  bool shouldPassIndirectlyForSwift(ArrayRef<llvm::Type*> scalars,
837
0
                                    bool asReturnValue) const override {
838
0
    return occupiesMoreThan(CGT, scalars, /*total*/ 4);
839
0
  }
840
841
0
  bool isSwiftErrorInRegister() const override {
842
0
    return false;
843
0
  }
844
};
845
846
class WebAssemblyTargetCodeGenInfo final : public TargetCodeGenInfo {
847
public:
848
  explicit WebAssemblyTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT,
849
                                        WebAssemblyABIInfo::ABIKind K)
850
34
      : TargetCodeGenInfo(std::make_unique<WebAssemblyABIInfo>(CGT, K)) {}
851
852
  void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
853
1.17k
                           CodeGen::CodeGenModule &CGM) const override {
854
1.17k
    TargetCodeGenInfo::setTargetAttributes(D, GV, CGM);
855
1.17k
    if (const auto *FD = dyn_cast_or_null<FunctionDecl>(D)) {
856
1.03k
      if (const auto *Attr = FD->getAttr<WebAssemblyImportModuleAttr>()) {
857
1
        llvm::Function *Fn = cast<llvm::Function>(GV);
858
1
        llvm::AttrBuilder B(GV->getContext());
859
1
        B.addAttribute("wasm-import-module", Attr->getImportModule());
860
1
        Fn->addFnAttrs(B);
861
1
      }
862
1.03k
      if (const auto *Attr = FD->getAttr<WebAssemblyImportNameAttr>()) {
863
1
        llvm::Function *Fn = cast<llvm::Function>(GV);
864
1
        llvm::AttrBuilder B(GV->getContext());
865
1
        B.addAttribute("wasm-import-name", Attr->getImportName());
866
1
        Fn->addFnAttrs(B);
867
1
      }
868
1.03k
      if (const auto *Attr = FD->getAttr<WebAssemblyExportNameAttr>()) {
869
2
        llvm::Function *Fn = cast<llvm::Function>(GV);
870
2
        llvm::AttrBuilder B(GV->getContext());
871
2
        B.addAttribute("wasm-export-name", Attr->getExportName());
872
2
        Fn->addFnAttrs(B);
873
2
      }
874
1.03k
    }
875
876
1.17k
    if (auto *FD = dyn_cast_or_null<FunctionDecl>(D)) {
877
1.03k
      llvm::Function *Fn = cast<llvm::Function>(GV);
878
1.03k
      if (!FD->doesThisDeclarationHaveABody() && 
!FD->hasPrototype()60
)
879
0
        Fn->addFnAttr("no-prototype");
880
1.03k
    }
881
1.17k
  }
882
};
883
884
/// Classify argument of given type \p Ty.
885
336
ABIArgInfo WebAssemblyABIInfo::classifyArgumentType(QualType Ty) const {
886
336
  Ty = useFirstFieldIfTransparentUnion(Ty);
887
888
336
  if (isAggregateTypeForABI(Ty)) {
889
    // Records with non-trivial destructors/copy-constructors should not be
890
    // passed by value.
891
42
    if (auto RAA = getRecordArgABI(Ty, getCXXABI()))
892
8
      return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
893
    // Ignore empty structs/unions.
894
34
    if (isEmptyRecord(getContext(), Ty, true))
895
4
      return ABIArgInfo::getIgnore();
896
    // Lower single-element structs to just pass a regular value. TODO: We
897
    // could do reasonable-size multiple-element structs too, using getExpand(),
898
    // though watch out for things like bitfields.
899
30
    if (const Type *SeltTy = isSingleElementStruct(Ty, getContext()))
900
11
      return ABIArgInfo::getDirect(CGT.ConvertType(QualType(SeltTy, 0)));
901
    // For the experimental multivalue ABI, fully expand all other aggregates
902
19
    if (Kind == ABIKind::ExperimentalMV) {
903
3
      const RecordType *RT = Ty->getAs<RecordType>();
904
3
      assert(RT);
905
0
      bool HasBitField = false;
906
5
      for (auto *Field : RT->getDecl()->fields()) {
907
5
        if (Field->isBitField()) {
908
1
          HasBitField = true;
909
1
          break;
910
1
        }
911
5
      }
912
3
      if (!HasBitField)
913
2
        return ABIArgInfo::getExpand();
914
3
    }
915
19
  }
916
917
  // Otherwise just do the default thing.
918
311
  return defaultInfo.classifyArgumentType(Ty);
919
336
}
920
921
336
ABIArgInfo WebAssemblyABIInfo::classifyReturnType(QualType RetTy) const {
922
336
  if (isAggregateTypeForABI(RetTy)) {
923
    // Records with non-trivial destructors/copy-constructors should not be
924
    // returned by value.
925
30
    if (!getRecordArgABI(RetTy, getCXXABI())) {
926
      // Ignore empty structs/unions.
927
30
      if (isEmptyRecord(getContext(), RetTy, true))
928
4
        return ABIArgInfo::getIgnore();
929
      // Lower single-element structs to just return a regular value. TODO: We
930
      // could do reasonable-size multiple-element structs too, using
931
      // ABIArgInfo::getDirect().
932
26
      if (const Type *SeltTy = isSingleElementStruct(RetTy, getContext()))
933
11
        return ABIArgInfo::getDirect(CGT.ConvertType(QualType(SeltTy, 0)));
934
      // For the experimental multivalue ABI, return all other aggregates
935
15
      if (Kind == ABIKind::ExperimentalMV)
936
3
        return ABIArgInfo::getDirect();
937
15
    }
938
30
  }
939
940
  // Otherwise just do the default thing.
941
318
  return defaultInfo.classifyReturnType(RetTy);
942
336
}
943
944
Address WebAssemblyABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
945
5
                                      QualType Ty) const {
946
5
  bool IsIndirect = isAggregateTypeForABI(Ty) &&
947
5
                    
!isEmptyRecord(getContext(), Ty, true)3
&&
948
5
                    
!isSingleElementStruct(Ty, getContext())2
;
949
5
  return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect,
950
5
                          getContext().getTypeInfoInChars(Ty),
951
5
                          CharUnits::fromQuantity(4),
952
5
                          /*AllowHigherAlign=*/true);
953
5
}
954
955
//===----------------------------------------------------------------------===//
956
// le32/PNaCl bitcode ABI Implementation
957
//
958
// This is a simplified version of the x86_32 ABI.  Arguments and return values
959
// are always passed on the stack.
960
//===----------------------------------------------------------------------===//
961
962
class PNaClABIInfo : public ABIInfo {
963
 public:
964
3
  PNaClABIInfo(CodeGen::CodeGenTypes &CGT) : ABIInfo(CGT) {}
965
966
  ABIArgInfo classifyReturnType(QualType RetTy) const;
967
  ABIArgInfo classifyArgumentType(QualType RetTy) const;
968
969
  void computeInfo(CGFunctionInfo &FI) const override;
970
  Address EmitVAArg(CodeGenFunction &CGF,
971
                    Address VAListAddr, QualType Ty) const override;
972
};
973
974
class PNaClTargetCodeGenInfo : public TargetCodeGenInfo {
975
 public:
976
   PNaClTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT)
977
3
       : TargetCodeGenInfo(std::make_unique<PNaClABIInfo>(CGT)) {}
978
};
979
980
2
void PNaClABIInfo::computeInfo(CGFunctionInfo &FI) const {
981
2
  if (!getCXXABI().classifyReturnType(FI))
982
2
    FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
983
984
2
  for (auto &I : FI.arguments())
985
0
    I.info = classifyArgumentType(I.type);
986
2
}
987
988
Address PNaClABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
989
0
                                QualType Ty) const {
990
  // The PNaCL ABI is a bit odd, in that varargs don't use normal
991
  // function classification. Structs get passed directly for varargs
992
  // functions, through a rewriting transform in
993
  // pnacl-llvm/lib/Transforms/NaCl/ExpandVarArgs.cpp, which allows
994
  // this target to actually support a va_arg instructions with an
995
  // aggregate type, unlike other targets.
996
0
  return EmitVAArgInstr(CGF, VAListAddr, Ty, ABIArgInfo::getDirect());
997
0
}
998
999
/// Classify argument of given type \p Ty.
1000
0
ABIArgInfo PNaClABIInfo::classifyArgumentType(QualType Ty) const {
1001
0
  if (isAggregateTypeForABI(Ty)) {
1002
0
    if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))
1003
0
      return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
1004
0
    return getNaturalAlignIndirect(Ty);
1005
0
  } else if (const EnumType *EnumTy = Ty->getAs<EnumType>()) {
1006
    // Treat an enum type as its underlying type.
1007
0
    Ty = EnumTy->getDecl()->getIntegerType();
1008
0
  } else if (Ty->isFloatingType()) {
1009
    // Floating-point types don't go inreg.
1010
0
    return ABIArgInfo::getDirect();
1011
0
  } else if (const auto *EIT = Ty->getAs<BitIntType>()) {
1012
    // Treat bit-precise integers as integers if <= 64, otherwise pass
1013
    // indirectly.
1014
0
    if (EIT->getNumBits() > 64)
1015
0
      return getNaturalAlignIndirect(Ty);
1016
0
    return ABIArgInfo::getDirect();
1017
0
  }
1018
1019
0
  return (isPromotableIntegerTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty)
1020
0
                                            : ABIArgInfo::getDirect());
1021
0
}
1022
1023
2
ABIArgInfo PNaClABIInfo::classifyReturnType(QualType RetTy) const {
1024
2
  if (RetTy->isVoidType())
1025
2
    return ABIArgInfo::getIgnore();
1026
1027
  // In the PNaCl ABI we always return records/structures on the stack.
1028
0
  if (isAggregateTypeForABI(RetTy))
1029
0
    return getNaturalAlignIndirect(RetTy);
1030
1031
  // Treat bit-precise integers as integers if <= 64, otherwise pass indirectly.
1032
0
  if (const auto *EIT = RetTy->getAs<BitIntType>()) {
1033
0
    if (EIT->getNumBits() > 64)
1034
0
      return getNaturalAlignIndirect(RetTy);
1035
0
    return ABIArgInfo::getDirect();
1036
0
  }
1037
1038
  // Treat an enum type as its underlying type.
1039
0
  if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
1040
0
    RetTy = EnumTy->getDecl()->getIntegerType();
1041
1042
0
  return (isPromotableIntegerTypeForABI(RetTy) ? ABIArgInfo::getExtend(RetTy)
1043
0
                                               : ABIArgInfo::getDirect());
1044
0
}
1045
1046
/// IsX86_MMXType - Return true if this is an MMX type.
1047
78
bool IsX86_MMXType(llvm::Type *IRType) {
1048
  // Return true if the type is an MMX type <2 x i32>, <4 x i16>, or <8 x i8>.
1049
78
  return IRType->isVectorTy() && IRType->getPrimitiveSizeInBits() == 64 &&
1050
78
    
cast<llvm::VectorType>(IRType)->getElementType()->isIntegerTy()18
&&
1051
78
    
IRType->getScalarSizeInBits() != 6416
;
1052
78
}
1053
1054
static llvm::Type* X86AdjustInlineAsmType(CodeGen::CodeGenFunction &CGF,
1055
                                          StringRef Constraint,
1056
1.60k
                                          llvm::Type* Ty) {
1057
1.60k
  bool IsMMXCons = llvm::StringSwitch<bool>(Constraint)
1058
1.60k
                     .Cases("y", "&y", "^Ym", true)
1059
1.60k
                     .Default(false);
1060
1.60k
  if (IsMMXCons && 
Ty->isVectorTy()18
) {
1061
14
    if (cast<llvm::VectorType>(Ty)->getPrimitiveSizeInBits().getFixedSize() !=
1062
14
        64) {
1063
      // Invalid MMX constraint
1064
0
      return nullptr;
1065
0
    }
1066
1067
14
    return llvm::Type::getX86_MMXTy(CGF.getLLVMContext());
1068
14
  }
1069
1070
  // No operation needed
1071
1.59k
  return Ty;
1072
1.60k
}
1073
1074
/// Returns true if this type can be passed in SSE registers with the
1075
/// X86_VectorCall calling convention. Shared between x86_32 and x86_64.
1076
771
static bool isX86VectorTypeForVectorCall(ASTContext &Context, QualType Ty) {
1077
771
  if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
1078
475
    if (BT->isFloatingPoint() && 
BT->getKind() != BuiltinType::Half281
) {
1079
281
      if (BT->getKind() == BuiltinType::LongDouble) {
1080
6
        if (&Context.getTargetInfo().getLongDoubleFormat() ==
1081
6
            &llvm::APFloat::x87DoubleExtended())
1082
2
          return false;
1083
6
      }
1084
279
      return true;
1085
281
    }
1086
475
  } else 
if (const VectorType *296
VT296
= Ty->getAs<VectorType>()) {
1087
    // vectorcall can pass XMM, YMM, and ZMM vectors. We don't pass SSE1 MMX
1088
    // registers specially.
1089
263
    unsigned VecSize = Context.getTypeSize(VT);
1090
263
    if (VecSize == 128 || 
VecSize == 2560
||
VecSize == 5120
)
1091
263
      return true;
1092
263
  }
1093
227
  return false;
1094
771
}
1095
1096
/// Returns true if this aggregate is small enough to be passed in SSE registers
1097
/// in the X86_VectorCall calling convention. Shared between x86_32 and x86_64.
1098
730
static bool isX86VectorCallAggregateSmallEnough(uint64_t NumMembers) {
1099
730
  return NumMembers <= 4;
1100
730
}
1101
1102
/// Returns a Homogeneous Vector Aggregate ABIArgInfo, used in X86.
1103
30
static ABIArgInfo getDirectX86Hva(llvm::Type* T = nullptr) {
1104
30
  auto AI = ABIArgInfo::getDirect(T);
1105
30
  AI.setInReg(true);
1106
30
  AI.setCanBeFlattened(false);
1107
30
  return AI;
1108
30
}
1109
1110
//===----------------------------------------------------------------------===//
1111
// X86-32 ABI Implementation
1112
//===----------------------------------------------------------------------===//
1113
1114
/// Similar to llvm::CCState, but for Clang.
1115
struct CCState {
1116
  CCState(CGFunctionInfo &FI)
1117
18.6k
      : IsPreassigned(FI.arg_size()), CC(FI.getCallingConvention()) {}
1118
1119
  llvm::SmallBitVector IsPreassigned;
1120
  unsigned CC = CallingConv::CC_C;
1121
  unsigned FreeRegs = 0;
1122
  unsigned FreeSSERegs = 0;
1123
};
1124
1125
/// X86_32ABIInfo - The X86-32 ABI information.
1126
class X86_32ABIInfo : public SwiftABIInfo {
1127
  enum Class {
1128
    Integer,
1129
    Float
1130
  };
1131
1132
  static const unsigned MinABIStackAlignInBytes = 4;
1133
1134
  bool IsDarwinVectorABI;
1135
  bool IsRetSmallStructInRegABI;
1136
  bool IsWin32StructABI;
1137
  bool IsSoftFloatABI;
1138
  bool IsMCUABI;
1139
  bool IsLinuxABI;
1140
  unsigned DefaultNumRegisterParameters;
1141
1142
519
  static bool isRegisterSize(unsigned Size) {
1143
519
    return (Size == 8 || 
Size == 16488
||
Size == 32469
||
Size == 64231
);
1144
519
  }
1145
1146
400
  bool isHomogeneousAggregateBaseType(QualType Ty) const override {
1147
    // FIXME: Assumes vectorcall is in use.
1148
400
    return isX86VectorTypeForVectorCall(getContext(), Ty);
1149
400
  }
1150
1151
  bool isHomogeneousAggregateSmallEnough(const Type *Ty,
1152
322
                                         uint64_t NumMembers) const override {
1153
    // FIXME: Assumes vectorcall is in use.
1154
322
    return isX86VectorCallAggregateSmallEnough(NumMembers);
1155
322
  }
1156
1157
  bool shouldReturnTypeInRegister(QualType Ty, ASTContext &Context) const;
1158
1159
  /// getIndirectResult - Give a source type \arg Ty, return a suitable result
1160
  /// such that the argument will be passed in memory.
1161
  ABIArgInfo getIndirectResult(QualType Ty, bool ByVal, CCState &State) const;
1162
1163
  ABIArgInfo getIndirectReturnResult(QualType Ty, CCState &State) const;
1164
1165
  /// Return the alignment to use for the given type on the stack.
1166
  unsigned getTypeStackAlignInBytes(QualType Ty, unsigned Align) const;
1167
1168
  Class classify(QualType Ty) const;
1169
  ABIArgInfo classifyReturnType(QualType RetTy, CCState &State) const;
1170
  ABIArgInfo classifyArgumentType(QualType RetTy, CCState &State) const;
1171
1172
  /// Updates the number of available free registers, returns
1173
  /// true if any registers were allocated.
1174
  bool updateFreeRegs(QualType Ty, CCState &State) const;
1175
1176
  bool shouldAggregateUseDirect(QualType Ty, CCState &State, bool &InReg,
1177
                                bool &NeedsPadding) const;
1178
  bool shouldPrimitiveUseInReg(QualType Ty, CCState &State) const;
1179
1180
  bool canExpandIndirectArgument(QualType Ty) const;
1181
1182
  /// Rewrite the function info so that all memory arguments use
1183
  /// inalloca.
1184
  void rewriteWithInAlloca(CGFunctionInfo &FI) const;
1185
1186
  void addFieldToArgStruct(SmallVector<llvm::Type *, 6> &FrameFields,
1187
                           CharUnits &StackOffset, ABIArgInfo &Info,
1188
                           QualType Type) const;
1189
  void runVectorCallFirstPass(CGFunctionInfo &FI, CCState &State) const;
1190
1191
public:
1192
1193
  void computeInfo(CGFunctionInfo &FI) const override;
1194
  Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
1195
                    QualType Ty) const override;
1196
1197
  X86_32ABIInfo(CodeGen::CodeGenTypes &CGT, bool DarwinVectorABI,
1198
                bool RetSmallStructInRegABI, bool Win32StructABI,
1199
                unsigned NumRegisterParameters, bool SoftFloatABI)
1200
    : SwiftABIInfo(CGT), IsDarwinVectorABI(DarwinVectorABI),
1201
      IsRetSmallStructInRegABI(RetSmallStructInRegABI),
1202
      IsWin32StructABI(Win32StructABI), IsSoftFloatABI(SoftFloatABI),
1203
      IsMCUABI(CGT.getTarget().getTriple().isOSIAMCU()),
1204
      IsLinuxABI(CGT.getTarget().getTriple().isOSLinux() ||
1205
                 CGT.getTarget().getTriple().isOSCygMing()),
1206
3.09k
      DefaultNumRegisterParameters(NumRegisterParameters) {}
1207
1208
  bool shouldPassIndirectlyForSwift(ArrayRef<llvm::Type*> scalars,
1209
0
                                    bool asReturnValue) const override {
1210
    // LLVM's x86-32 lowering currently only assigns up to three
1211
    // integer registers and three fp registers.  Oddly, it'll use up to
1212
    // four vector registers for vectors, but those can overlap with the
1213
    // scalar registers.
1214
0
    return occupiesMoreThan(CGT, scalars, /*total*/ 3);
1215
0
  }
1216
1217
0
  bool isSwiftErrorInRegister() const override {
1218
    // x86-32 lowering does not support passing swifterror in a register.
1219
0
    return false;
1220
0
  }
1221
};
1222
1223
class X86_32TargetCodeGenInfo : public TargetCodeGenInfo {
1224
public:
1225
  X86_32TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, bool DarwinVectorABI,
1226
                          bool RetSmallStructInRegABI, bool Win32StructABI,
1227
                          unsigned NumRegisterParameters, bool SoftFloatABI)
1228
      : TargetCodeGenInfo(std::make_unique<X86_32ABIInfo>(
1229
            CGT, DarwinVectorABI, RetSmallStructInRegABI, Win32StructABI,
1230
3.09k
            NumRegisterParameters, SoftFloatABI)) {}
1231
1232
  static bool isStructReturnInRegABI(
1233
      const llvm::Triple &Triple, const CodeGenOptions &Opts);
1234
1235
  void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
1236
                           CodeGen::CodeGenModule &CGM) const override;
1237
1238
0
  int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const override {
1239
    // Darwin uses different dwarf register numbers for EH.
1240
0
    if (CGM.getTarget().getTriple().isOSDarwin()) return 5;
1241
0
    return 4;
1242
0
  }
1243
1244
  bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
1245
                               llvm::Value *Address) const override;
1246
1247
  llvm::Type* adjustInlineAsmType(CodeGen::CodeGenFunction &CGF,
1248
                                  StringRef Constraint,
1249
700
                                  llvm::Type* Ty) const override {
1250
700
    return X86AdjustInlineAsmType(CGF, Constraint, Ty);
1251
700
  }
1252
1253
  void addReturnRegisterOutputs(CodeGenFunction &CGF, LValue ReturnValue,
1254
                                std::string &Constraints,
1255
                                std::vector<llvm::Type *> &ResultRegTypes,
1256
                                std::vector<llvm::Type *> &ResultTruncRegTypes,
1257
                                std::vector<LValue> &ResultRegDests,
1258
                                std::string &AsmString,
1259
                                unsigned NumOutputs) const override;
1260
1261
  llvm::Constant *
1262
37
  getUBSanFunctionSignature(CodeGen::CodeGenModule &CGM) const override {
1263
37
    unsigned Sig = (0xeb << 0) |  // jmp rel8
1264
37
                   (0x06 << 8) |  //           .+0x08
1265
37
                   ('v' << 16) |
1266
37
                   ('2' << 24);
1267
37
    return llvm::ConstantInt::get(CGM.Int32Ty, Sig);
1268
37
  }
1269
1270
7
  StringRef getARCRetainAutoreleasedReturnValueMarker() const override {
1271
7
    return "movl\t%ebp, %ebp"
1272
7
           "\t\t// marker for objc_retainAutoreleaseReturnValue";
1273
7
  }
1274
};
1275
1276
}
1277
1278
/// Rewrite input constraint references after adding some output constraints.
1279
/// In the case where there is one output and one input and we add one output,
1280
/// we need to replace all operand references greater than or equal to 1:
1281
///     mov $0, $1
1282
///     mov eax, $1
1283
/// The result will be:
1284
///     mov $0, $2
1285
///     mov eax, $2
1286
static void rewriteInputConstraintReferences(unsigned FirstIn,
1287
                                             unsigned NumNewOuts,
1288
27
                                             std::string &AsmString) {
1289
27
  std::string Buf;
1290
27
  llvm::raw_string_ostream OS(Buf);
1291
27
  size_t Pos = 0;
1292
88
  while (Pos < AsmString.size()) {
1293
61
    size_t DollarStart = AsmString.find('$', Pos);
1294
61
    if (DollarStart == std::string::npos)
1295
19
      DollarStart = AsmString.size();
1296
61
    size_t DollarEnd = AsmString.find_first_not_of('$', DollarStart);
1297
61
    if (DollarEnd == std::string::npos)
1298
19
      DollarEnd = AsmString.size();
1299
61
    OS << StringRef(&AsmString[Pos], DollarEnd - Pos);
1300
61
    Pos = DollarEnd;
1301
61
    size_t NumDollars = DollarEnd - DollarStart;
1302
61
    if (NumDollars % 2 != 0 && 
Pos < AsmString.size()19
) {
1303
      // We have an operand reference.
1304
19
      size_t DigitStart = Pos;
1305
19
      if (AsmString[DigitStart] == '{') {
1306
3
        OS << '{';
1307
3
        ++DigitStart;
1308
3
      }
1309
19
      size_t DigitEnd = AsmString.find_first_not_of("0123456789", DigitStart);
1310
19
      if (DigitEnd == std::string::npos)
1311
6
        DigitEnd = AsmString.size();
1312
19
      StringRef OperandStr(&AsmString[DigitStart], DigitEnd - DigitStart);
1313
19
      unsigned OperandIndex;
1314
19
      if (!OperandStr.getAsInteger(10, OperandIndex)) {
1315
19
        if (OperandIndex >= FirstIn)
1316
14
          OperandIndex += NumNewOuts;
1317
19
        OS << OperandIndex;
1318
19
      } else {
1319
0
        OS << OperandStr;
1320
0
      }
1321
19
      Pos = DigitEnd;
1322
19
    }
1323
61
  }
1324
27
  AsmString = std::move(OS.str());
1325
27
}
1326
1327
/// Add output constraints for EAX:EDX because they are return registers.
1328
void X86_32TargetCodeGenInfo::addReturnRegisterOutputs(
1329
    CodeGenFunction &CGF, LValue ReturnSlot, std::string &Constraints,
1330
    std::vector<llvm::Type *> &ResultRegTypes,
1331
    std::vector<llvm::Type *> &ResultTruncRegTypes,
1332
    std::vector<LValue> &ResultRegDests, std::string &AsmString,
1333
27
    unsigned NumOutputs) const {
1334
27
  uint64_t RetWidth = CGF.getContext().getTypeSize(ReturnSlot.getType());
1335
1336
  // Use the EAX constraint if the width is 32 or smaller and EAX:EDX if it is
1337
  // larger.
1338
27
  if (!Constraints.empty())
1339
4
    Constraints += ',';
1340
27
  if (RetWidth <= 32) {
1341
22
    Constraints += "={eax}";
1342
22
    ResultRegTypes.push_back(CGF.Int32Ty);
1343
22
  } else {
1344
    // Use the 'A' constraint for EAX:EDX.
1345
5
    Constraints += "=A";
1346
5
    ResultRegTypes.push_back(CGF.Int64Ty);
1347
5
  }
1348
1349
  // Truncate EAX or EAX:EDX to an integer of the appropriate size.
1350
27
  llvm::Type *CoerceTy = llvm::IntegerType::get(CGF.getLLVMContext(), RetWidth);
1351
27
  ResultTruncRegTypes.push_back(CoerceTy);
1352
1353
  // Coerce the integer by bitcasting the return slot pointer.
1354
27
  ReturnSlot.setAddress(CGF.Builder.CreateBitCast(ReturnSlot.getAddress(CGF),
1355
27
                                                  CoerceTy->getPointerTo()));
1356
27
  ResultRegDests.push_back(ReturnSlot);
1357
1358
27
  rewriteInputConstraintReferences(NumOutputs, 1, AsmString);
1359
27
}
1360
1361
/// shouldReturnTypeInRegister - Determine if the given type should be
1362
/// returned in a register (for the Darwin and MCU ABI).
1363
bool X86_32ABIInfo::shouldReturnTypeInRegister(QualType Ty,
1364
538
                                               ASTContext &Context) const {
1365
538
  uint64_t Size = Context.getTypeSize(Ty);
1366
1367
  // For i386, type must be register sized.
1368
  // For the MCU ABI, it only needs to be <= 8-byte
1369
538
  if ((IsMCUABI && 
Size > 6419
) ||
(535
!IsMCUABI535
&&
!isRegisterSize(Size)519
))
1370
70
   return false;
1371
1372
468
  if (Ty->isVectorType()) {
1373
    // 64- and 128- bit vectors inside structures are not returned in
1374
    // registers.
1375
4
    if (Size == 64 || 
Size == 1281
)
1376
3
      return false;
1377
1378
1
    return true;
1379
4
  }
1380
1381
  // If this is a builtin, pointer, enum, complex type, member pointer, or
1382
  // member function pointer it is ok.
1383
464
  if (Ty->getAs<BuiltinType>() || 
Ty->hasPointerRepresentation()376
||
1384
464
      
Ty->isAnyComplexType()372
||
Ty->isEnumeralType()349
||
1385
464
      
Ty->isBlockPointerType()347
||
Ty->isMemberPointerType()347
)
1386
379
    return true;
1387
1388
  // Arrays are treated like records.
1389
85
  if (const ConstantArrayType *AT = Context.getAsConstantArrayType(Ty))
1390
6
    return shouldReturnTypeInRegister(AT->getElementType(), Context);
1391
1392
  // Otherwise, it must be a record type.
1393
79
  const RecordType *RT = Ty->getAs<RecordType>();
1394
79
  if (!RT) 
return false0
;
1395
1396
  // FIXME: Traverse bases here too.
1397
1398
  // Structure types are passed in register if all fields would be
1399
  // passed in a register.
1400
113
  
for (const auto *FD : RT->getDecl()->fields())79
{
1401
    // Empty fields are ignored.
1402
113
    if (isEmptyField(Context, FD, true))
1403
11
      continue;
1404
1405
    // Check fields recursively.
1406
102
    if (!shouldReturnTypeInRegister(FD->getType(), Context))
1407
4
      return false;
1408
102
  }
1409
75
  return true;
1410
79
}
1411
1412
207
static bool is32Or64BitBasicType(QualType Ty, ASTContext &Context) {
1413
  // Treat complex types as the element type.
1414
207
  if (const ComplexType *CTy = Ty->getAs<ComplexType>())
1415
2
    Ty = CTy->getElementType();
1416
1417
  // Check for a type which we know has a simple scalar argument-passing
1418
  // convention without any padding.  (We're specifically looking for 32
1419
  // and 64-bit integer and integer-equivalents, float, and double.)
1420
207
  if (!Ty->getAs<BuiltinType>() && 
!Ty->hasPointerRepresentation()36
&&
1421
207
      
!Ty->isEnumeralType()31
&&
!Ty->isBlockPointerType()28
)
1422
28
    return false;
1423
1424
179
  uint64_t Size = Context.getTypeSize(Ty);
1425
179
  return Size == 32 || 
Size == 6429
;
1426
207
}
1427
1428
static bool addFieldSizes(ASTContext &Context, const RecordDecl *RD,
1429
246
                          uint64_t &Size) {
1430
246
  for (const auto *FD : RD->fields()) {
1431
    // Scalar arguments on the stack get 4 byte alignment on x86. If the
1432
    // argument is smaller than 32-bits, expanding the struct will create
1433
    // alignment padding.
1434
207
    if (!is32Or64BitBasicType(FD->getType(), Context))
1435
50
      return false;
1436
1437
    // FIXME: Reject bit-fields wholesale; there are two problems, we don't know
1438
    // how to expand them yet, and the predicate for telling if a bitfield still
1439
    // counts as "basic" is more complicated than what we were doing previously.
1440
157
    if (FD->isBitField())
1441
2
      return false;
1442
1443
155
    Size += Context.getTypeSize(FD->getType());
1444
155
  }
1445
194
  return true;
1446
246
}
1447
1448
static bool addBaseAndFieldSizes(ASTContext &Context, const CXXRecordDecl *RD,
1449
144
                                 uint64_t &Size) {
1450
  // Don't do this if there are any non-empty bases.
1451
144
  for (const CXXBaseSpecifier &Base : RD->bases()) {
1452
1
    if (!addBaseAndFieldSizes(Context, Base.getType()->getAsCXXRecordDecl(),
1453
1
                              Size))
1454
0
      return false;
1455
1
  }
1456
144
  if (!addFieldSizes(Context, RD, Size))
1457
10
    return false;
1458
134
  return true;
1459
144
}
1460
1461
/// Test whether an argument type which is to be passed indirectly (on the
1462
/// stack) would have the equivalent layout if it was expanded into separate
1463
/// arguments. If so, we prefer to do the latter to avoid inhibiting
1464
/// optimizations.
1465
334
bool X86_32ABIInfo::canExpandIndirectArgument(QualType Ty) const {
1466
  // We can only expand structure types.
1467
334
  const RecordType *RT = Ty->getAs<RecordType>();
1468
334
  if (!RT)
1469
76
    return false;
1470
258
  const RecordDecl *RD = RT->getDecl();
1471
258
  uint64_t Size = 0;
1472
258
  if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
1473
156
    if (!IsWin32StructABI) {
1474
      // On non-Windows, we have to conservatively match our old bitcode
1475
      // prototypes in order to be ABI-compatible at the bitcode level.
1476
35
      if (!CXXRD->isCLike())
1477
13
        return false;
1478
121
    } else {
1479
      // Don't do this for dynamic classes.
1480
121
      if (CXXRD->isDynamicClass())
1481
0
        return false;
1482
121
    }
1483
143
    if (!addBaseAndFieldSizes(getContext(), CXXRD, Size))
1484
10
      return false;
1485
143
  } else {
1486
102
    if (!addFieldSizes(getContext(), RD, Size))
1487
42
      return false;
1488
102
  }
1489
1490
  // We can do this if there was no alignment padding.
1491
193
  return Size == getContext().getTypeSize(Ty);
1492
258
}
1493
1494
149
ABIArgInfo X86_32ABIInfo::getIndirectReturnResult(QualType RetTy, CCState &State) const {
1495
  // If the return value is indirect, then the hidden argument is consuming one
1496
  // integer register.
1497
149
  if (State.FreeRegs) {
1498
6
    --State.FreeRegs;
1499
6
    if (!IsMCUABI)
1500
3
      return getNaturalAlignIndirectInReg(RetTy);
1501
6
  }
1502
146
  return getNaturalAlignIndirect(RetTy, /*ByVal=*/false);
1503
149
}
1504
1505
ABIArgInfo X86_32ABIInfo::classifyReturnType(QualType RetTy,
1506
18.4k
                                             CCState &State) const {
1507
18.4k
  if (RetTy->isVoidType())
1508
12.5k
    return ABIArgInfo::getIgnore();
1509
1510
5.97k
  const Type *Base = nullptr;
1511
5.97k
  uint64_t NumElts = 0;
1512
5.97k
  if ((State.CC == llvm::CallingConv::X86_VectorCall ||
1513
5.97k
       
State.CC == llvm::CallingConv::X86_RegCall5.96k
) &&
1514
5.97k
      
isHomogeneousAggregate(RetTy, Base, NumElts)26
) {
1515
    // The LLVM struct type for such an aggregate should lower properly.
1516
17
    return ABIArgInfo::getDirect();
1517
17
  }
1518
1519
5.95k
  if (const VectorType *VT = RetTy->getAs<VectorType>()) {
1520
    // On Darwin, some vectors are returned in registers.
1521
31
    if (IsDarwinVectorABI) {
1522
15
      uint64_t Size = getContext().getTypeSize(RetTy);
1523
1524
      // 128-bit vectors are a special case; they are returned in
1525
      // registers and we need to make sure to pick a type the LLVM
1526
      // backend will like.
1527
15
      if (Size == 128)
1528
5
        return ABIArgInfo::getDirect(llvm::FixedVectorType::get(
1529
5
            llvm::Type::getInt64Ty(getVMContext()), 2));
1530
1531
      // Always return in register if it fits in a general purpose
1532
      // register, or if it is 64 bits and has a single element.
1533
10
      if ((Size == 8 || Size == 16 || Size == 32) ||
1534
10
          
(7
Size == 647
&&
VT->getNumElements() == 17
))
1535
7
        return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(),
1536
7
                                                            Size));
1537
1538
3
      return getIndirectReturnResult(RetTy, State);
1539
10
    }
1540
1541
16
    return ABIArgInfo::getDirect();
1542
31
  }
1543
1544
5.92k
  if (isAggregateTypeForABI(RetTy)) {
1545
528
    if (const RecordType *RT = RetTy->getAs<RecordType>()) {
1546
      // Structures with flexible arrays are always indirect.
1547
197
      if (RT->getDecl()->hasFlexibleArrayMember())
1548
1
        return getIndirectReturnResult(RetTy, State);
1549
197
    }
1550
1551
    // If specified, structs and unions are always indirect.
1552
527
    if (!IsRetSmallStructInRegABI && 
!RetTy->isAnyComplexType()115
)
1553
62
      return getIndirectReturnResult(RetTy, State);
1554
1555
    // Ignore empty structs/unions.
1556
465
    if (isEmptyRecord(getContext(), RetTy, true))
1557
34
      return ABIArgInfo::getIgnore();
1558
1559
    // Return complex of _Float16 as <2 x half> so the backend will use xmm0.
1560
431
    if (const ComplexType *CT = RetTy->getAs<ComplexType>()) {
1561
64
      QualType ET = getContext().getCanonicalType(CT->getElementType());
1562
64
      if (ET->isFloat16Type())
1563
1
        return ABIArgInfo::getDirect(llvm::FixedVectorType::get(
1564
1
            llvm::Type::getHalfTy(getVMContext()), 2));
1565
64
    }
1566
1567
    // Small structures which are register sized are generally returned
1568
    // in a register.
1569
430
    if (shouldReturnTypeInRegister(RetTy, getContext())) {
1570
357
      uint64_t Size = getContext().getTypeSize(RetTy);
1571
1572
      // As a special-case, if the struct is a "single-element" struct, and
1573
      // the field is of type "float" or "double", return it in a
1574
      // floating-point register. (MSVC does not apply this special case.)
1575
      // We apply a similar transformation for pointer types to improve the
1576
      // quality of the generated IR.
1577
357
      if (const Type *SeltTy = isSingleElementStruct(RetTy, getContext()))
1578
48
        if ((!IsWin32StructABI && 
SeltTy->isRealFloatingType()36
)
1579
48
            || 
SeltTy->hasPointerRepresentation()32
)
1580
18
          return ABIArgInfo::getDirect(CGT.ConvertType(QualType(SeltTy, 0)));
1581
1582
      // FIXME: We should be able to narrow this integer in cases with dead
1583
      // padding.
1584
339
      return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(),Size));
1585
357
    }
1586
1587
73
    return getIndirectReturnResult(RetTy, State);
1588
430
  }
1589
1590
  // Treat an enum type as its underlying type.
1591
5.39k
  if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
1592
35
    RetTy = EnumTy->getDecl()->getIntegerType();
1593
1594
5.39k
  if (const auto *EIT = RetTy->getAs<BitIntType>())
1595
26
    if (EIT->getNumBits() > 64)
1596
10
      return getIndirectReturnResult(RetTy, State);
1597
1598
5.38k
  return (isPromotableIntegerTypeForABI(RetTy) ? 
ABIArgInfo::getExtend(RetTy)122
1599
5.38k
                                               : 
ABIArgInfo::getDirect()5.26k
);
1600
5.39k
}
1601
1602
36
static bool isSIMDVectorType(ASTContext &Context, QualType Ty) {
1603
36
  return Ty->getAs<VectorType>() && 
Context.getTypeSize(Ty) == 12812
;
1604
36
}
1605
1606
28
static bool isRecordWithSIMDVectorType(ASTContext &Context, QualType Ty) {
1607
28
  const RecordType *RT = Ty->getAs<RecordType>();
1608
28
  if (!RT)
1609
12
    return false;
1610
16
  const RecordDecl *RD = RT->getDecl();
1611
1612
  // If this is a C++ record, check the bases first.
1613
16
  if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD))
1614
0
    for (const auto &I : CXXRD->bases())
1615
0
      if (!isRecordWithSIMDVectorType(Context, I.getType()))
1616
0
        return false;
1617
1618
19
  
for (const auto *i : RD->fields())16
{
1619
19
    QualType FT = i->getType();
1620
1621
19
    if (isSIMDVectorType(Context, FT))
1622
7
      return true;
1623
1624
12
    if (isRecordWithSIMDVectorType(Context, FT))
1625
0
      return true;
1626
12
  }
1627
1628
9
  return false;
1629
16
}
1630
1631
unsigned X86_32ABIInfo::getTypeStackAlignInBytes(QualType Ty,
1632
349
                                                 unsigned Align) const {
1633
  // Otherwise, if the alignment is less than or equal to the minimum ABI
1634
  // alignment, just use the default; the backend will handle this.
1635
349
  if (Align <= MinABIStackAlignInBytes)
1636
273
    return 0; // Use default alignment.
1637
1638
76
  if (IsLinuxABI) {
1639
    // Exclude other System V OS (e.g Darwin, PS4 and FreeBSD) since we don't
1640
    // want to spend any effort dealing with the ramifications of ABI breaks.
1641
    //
1642
    // If the vector type is __m128/__m256/__m512, return the default alignment.
1643
30
    if (Ty->isVectorType() && 
(18
Align == 1618
||
Align == 3212
||
Align == 646
))
1644
18
      return Align;
1645
30
  }
1646
  // On non-Darwin, the stack type alignment is always 4.
1647
58
  if (!IsDarwinVectorABI) {
1648
    // Set explicit alignment, since we may need to realign the top.
1649
31
    return MinABIStackAlignInBytes;
1650
31
  }
1651
1652
  // Otherwise, if the type contains an SSE vector type, the alignment is 16.
1653
27
  if (Align >= 16 && 
(17
isSIMDVectorType(getContext(), Ty)17
||
1654
17
                      
isRecordWithSIMDVectorType(getContext(), Ty)16
))
1655
8
    return 16;
1656
1657
19
  return MinABIStackAlignInBytes;
1658
27
}
1659
1660
ABIArgInfo X86_32ABIInfo::getIndirectResult(QualType Ty, bool ByVal,
1661
465
                                            CCState &State) const {
1662
465
  if (!ByVal) {
1663
166
    if (State.FreeRegs) {
1664
22
      --State.FreeRegs; // Non-byval indirects just use one pointer.
1665
22
      if (!IsMCUABI)
1666
22
        return getNaturalAlignIndirectInReg(Ty);
1667
22
    }
1668
144
    return getNaturalAlignIndirect(Ty, false);
1669
166
  }
1670
1671
  // Compute the byval alignment.
1672
299
  unsigned TypeAlign = getContext().getTypeAlign(Ty) / 8;
1673
299
  unsigned StackAlign = getTypeStackAlignInBytes(Ty, TypeAlign);
1674
299
  if (StackAlign == 0)
1675
249
    return ABIArgInfo::getIndirect(CharUnits::fromQuantity(4), /*ByVal=*/true);
1676
1677
  // If the stack alignment is less than the type alignment, realign the
1678
  // argument.
1679
50
  bool Realign = TypeAlign > StackAlign;
1680
50
  return ABIArgInfo::getIndirect(CharUnits::fromQuantity(StackAlign),
1681
50
                                 /*ByVal=*/true, Realign);
1682
299
}
1683
1684
31.8k
X86_32ABIInfo::Class X86_32ABIInfo::classify(QualType Ty) const {
1685
31.8k
  const Type *T = isSingleElementStruct(Ty, getContext());
1686
31.8k
  if (!T)
1687
31.7k
    T = Ty.getTypePtr();
1688
1689
31.8k
  if (const BuiltinType *BT = T->getAs<BuiltinType>()) {
1690
10.8k
    BuiltinType::Kind K = BT->getKind();
1691
10.8k
    if (K == BuiltinType::Float || 
K == BuiltinType::Double10.5k
)
1692
480
      return Float;
1693
10.8k
  }
1694
31.3k
  return Integer;
1695
31.8k
}
1696
1697
31.8k
bool X86_32ABIInfo::updateFreeRegs(QualType Ty, CCState &State) const {
1698
31.8k
  if (!IsSoftFloatABI) {
1699
31.8k
    Class C = classify(Ty);
1700
31.8k
    if (C == Float)
1701
480
      return false;
1702
31.8k
  }
1703
1704
31.3k
  unsigned Size = getContext().getTypeSize(Ty);
1705
31.3k
  unsigned SizeInRegs = (Size + 31) / 32;
1706
1707
31.3k
  if (SizeInRegs == 0)
1708
2
    return false;
1709
1710
31.3k
  if (!IsMCUABI) {
1711
31.3k
    if (SizeInRegs > State.FreeRegs) {
1712
31.1k
      State.FreeRegs = 0;
1713
31.1k
      return false;
1714
31.1k
    }
1715
31.3k
  } else {
1716
    // The MCU psABI allows passing parameters in-reg even if there are
1717
    // earlier parameters that are passed on the stack. Also,
1718
    // it does not allow passing >8-byte structs in-register,
1719
    // even if there are 3 free registers available.
1720
42
    if (SizeInRegs > State.FreeRegs || 
SizeInRegs > 234
)
1721
9
      return false;
1722
42
  }
1723
1724
243
  State.FreeRegs -= SizeInRegs;
1725
243
  return true;
1726
31.3k
}
1727
1728
bool X86_32ABIInfo::shouldAggregateUseDirect(QualType Ty, CCState &State,
1729
                                             bool &InReg,
1730
401
                                             bool &NeedsPadding) const {
1731
  // On Windows, aggregates other than HFAs are never passed in registers, and
1732
  // they do not consume register slots. Homogenous floating-point aggregates
1733
  // (HFAs) have already been dealt with at this point.
1734
401
  if (IsWin32StructABI && 
isAggregateTypeForABI(Ty)172
)
1735
172
    return false;
1736
1737
229
  NeedsPadding = false;
1738
229
  InReg = !IsMCUABI;
1739
1740
229
  if (!updateFreeRegs(Ty, State))
1741
209
    return false;
1742
1743
20
  if (IsMCUABI)
1744
7
    return true;
1745
1746
13
  if (State.CC == llvm::CallingConv::X86_FastCall ||
1747
13
      
State.CC == llvm::CallingConv::X86_VectorCall10
||
1748
13
      
State.CC == llvm::CallingConv::X86_RegCall10
) {
1749
4
    if (getContext().getTypeSize(Ty) <= 32 && State.FreeRegs)
1750
3
      NeedsPadding = true;
1751
1752
4
    return false;
1753
4
  }
1754
1755
9
  return true;
1756
13
}
1757
1758
31.6k
bool X86_32ABIInfo::shouldPrimitiveUseInReg(QualType Ty, CCState &State) const {
1759
31.6k
  if (!updateFreeRegs(Ty, State))
1760
31.4k
    return false;
1761
1762
223
  if (IsMCUABI)
1763
26
    return false;
1764
1765
197
  if (State.CC == llvm::CallingConv::X86_FastCall ||
1766
197
      
State.CC == llvm::CallingConv::X86_VectorCall137
||
1767
197
      
State.CC == llvm::CallingConv::X86_RegCall94
) {
1768
146
    if (getContext().getTypeSize(Ty) > 32)
1769
13
      return false;
1770
1771
133
    return (Ty->isIntegralOrEnumerationType() || 
Ty->isPointerType()27
||
1772
133
        
Ty->isReferenceType()7
);
1773
146
  }
1774
1775
51
  return true;
1776
197
}
1777
1778
57
void X86_32ABIInfo::runVectorCallFirstPass(CGFunctionInfo &FI, CCState &State) const {
1779
  // Vectorcall x86 works subtly different than in x64, so the format is
1780
  // a bit different than the x64 version.  First, all vector types (not HVAs)
1781
  // are assigned, with the first 6 ending up in the [XYZ]MM0-5 registers.
1782
  // This differs from the x64 implementation, where the first 6 by INDEX get
1783
  // registers.
1784
  // In the second pass over the arguments, HVAs are passed in the remaining
1785
  // vector registers if possible, or indirectly by address. The address will be
1786
  // passed in ECX/EDX if available. Any other arguments are passed according to
1787
  // the usual fastcall rules.
1788
57
  MutableArrayRef<CGFunctionInfoArgInfo> Args = FI.arguments();
1789
171
  for (int I = 0, E = Args.size(); I < E; 
++I114
) {
1790
114
    const Type *Base = nullptr;
1791
114
    uint64_t NumElts = 0;
1792
114
    const QualType &Ty = Args[I].type;
1793
114
    if ((Ty->isVectorType() || 
Ty->isBuiltinType()93
) &&
1794
114
        
isHomogeneousAggregate(Ty, Base, NumElts)84
) {
1795
42
      if (State.FreeSSERegs >= NumElts) {
1796
38
        State.FreeSSERegs -= NumElts;
1797
38
        Args[I].info = ABIArgInfo::getDirectInReg();
1798
38
        State.IsPreassigned.set(I);
1799
38
      }
1800
42
    }
1801
114
  }
1802
57
}
1803
1804
ABIArgInfo X86_32ABIInfo::classifyArgumentType(QualType Ty,
1805
32.4k
                                               CCState &State) const {
1806
  // FIXME: Set alignment on indirect arguments.
1807
32.4k
  bool IsFastCall = State.CC == llvm::CallingConv::X86_FastCall;
1808
32.4k
  bool IsRegCall = State.CC == llvm::CallingConv::X86_RegCall;
1809
32.4k
  bool IsVectorCall = State.CC == llvm::CallingConv::X86_VectorCall;
1810
1811
32.4k
  Ty = useFirstFieldIfTransparentUnion(Ty);
1812
32.4k
  TypeInfo TI = getContext().getTypeInfo(Ty);
1813
1814
  // Check with the C++ ABI first.
1815
32.4k
  const RecordType *RT = Ty->getAs<RecordType>();
1816
32.4k
  if (RT) {
1817
575
    CGCXXABI::RecordArgABI RAA = getRecordArgABI(RT, getCXXABI());
1818
575
    if (RAA == CGCXXABI::RAA_Indirect) {
1819
102
      return getIndirectResult(Ty, false, State);
1820
473
    } else if (RAA == CGCXXABI::RAA_DirectInMemory) {
1821
      // The field index doesn't matter, we'll fix it up later.
1822
86
      return ABIArgInfo::getInAlloca(/*FieldIndex=*/0);
1823
86
    }
1824
575
  }
1825
1826
  // Regcall uses the concept of a homogenous vector aggregate, similar
1827
  // to other targets.
1828
32.2k
  const Type *Base = nullptr;
1829
32.2k
  uint64_t NumElts = 0;
1830
32.2k
  if ((IsRegCall || 
IsVectorCall32.1k
) &&
1831
32.2k
      
isHomogeneousAggregate(Ty, Base, NumElts)179
) {
1832
79
    if (State.FreeSSERegs >= NumElts) {
1833
57
      State.FreeSSERegs -= NumElts;
1834
1835
      // Vectorcall passes HVAs directly and does not flatten them, but regcall
1836
      // does.
1837
57
      if (IsVectorCall)
1838
12
        return getDirectX86Hva();
1839
1840
45
      if (Ty->isBuiltinType() || 
Ty->isVectorType()35
)
1841
20
        return ABIArgInfo::getDirect();
1842
25
      return ABIArgInfo::getExpand();
1843
45
    }
1844
22
    return getIndirectResult(Ty, /*ByVal=*/false, State);
1845
79
  }
1846
1847
32.1k
  if (isAggregateTypeForABI(Ty)) {
1848
    // Structures with flexible arrays are always indirect.
1849
    // FIXME: This should not be byval!
1850
426
    if (RT && 
RT->getDecl()->hasFlexibleArrayMember()339
)
1851
1
      return getIndirectResult(Ty, true, State);
1852
1853
    // Ignore empty structs/unions on non-Windows.
1854
425
    if (!IsWin32StructABI && 
isEmptyRecord(getContext(), Ty, true)253
)
1855
24
      return ABIArgInfo::getIgnore();
1856
1857
401
    llvm::LLVMContext &LLVMContext = getVMContext();
1858
401
    llvm::IntegerType *Int32 = llvm::Type::getInt32Ty(LLVMContext);
1859
401
    bool NeedsPadding = false;
1860
401
    bool InReg;
1861
401
    if (shouldAggregateUseDirect(Ty, State, InReg, NeedsPadding)) {
1862
16
      unsigned SizeInRegs = (TI.Width + 31) / 32;
1863
16
      SmallVector<llvm::Type*, 3> Elements(SizeInRegs, Int32);
1864
16
      llvm::Type *Result = llvm::StructType::get(LLVMContext, Elements);
1865
16
      if (InReg)
1866
9
        return ABIArgInfo::getDirectInReg(Result);
1867
7
      else
1868
7
        return ABIArgInfo::getDirect(Result);
1869
16
    }
1870
385
    llvm::IntegerType *PaddingType = NeedsPadding ? 
Int323
:
nullptr382
;
1871
1872
    // Pass over-aligned aggregates on Windows indirectly. This behavior was
1873
    // added in MSVC 2015.
1874
385
    if (IsWin32StructABI && 
TI.isAlignRequired()172
&&
TI.Align > 321
)
1875
1
      return getIndirectResult(Ty, /*ByVal=*/false, State);
1876
1877
    // Expand small (<= 128-bit) record types when we know that the stack layout
1878
    // of those arguments will match the struct. This is important because the
1879
    // LLVM backend isn't smart enough to remove byval, which inhibits many
1880
    // optimizations.
1881
    // Don't do this for the MCU if there are still free integer registers
1882
    // (see X86_64 ABI for full explanation).
1883
384
    if (TI.Width <= 4 * 32 && 
(337
!IsMCUABI337
||
State.FreeRegs == 05
) &&
1884
384
        
canExpandIndirectArgument(Ty)334
)
1885
86
      return ABIArgInfo::getExpandWithPadding(
1886
86
          IsFastCall || 
IsVectorCall82
||
IsRegCall81
, PaddingType);
1887
1888
298
    return getIndirectResult(Ty, true, State);
1889
384
  }
1890
1891
31.7k
  if (const VectorType *VT = Ty->getAs<VectorType>()) {
1892
    // On Windows, vectors are passed directly if registers are available, or
1893
    // indirectly if not. This avoids the need to align argument memory. Pass
1894
    // user-defined vector types larger than 512 bits indirectly for simplicity.
1895
130
    if (IsWin32StructABI) {
1896
45
      if (TI.Width <= 512 && 
State.FreeSSERegs > 040
) {
1897
28
        --State.FreeSSERegs;
1898
28
        return ABIArgInfo::getDirectInReg();
1899
28
      }
1900
17
      return getIndirectResult(Ty, /*ByVal=*/false, State);
1901
45
    }
1902
1903
    // On Darwin, some vectors are passed in memory, we handle this by passing
1904
    // it as an i8/i16/i32/i64.
1905
85
    if (IsDarwinVectorABI) {
1906
24
      if ((TI.Width == 8 || TI.Width == 16 || TI.Width == 32) ||
1907
24
          
(22
TI.Width == 6422
&&
VT->getNumElements() == 110
))
1908
7
        return ABIArgInfo::getDirect(
1909
7
            llvm::IntegerType::get(getVMContext(), TI.Width));
1910
24
    }
1911
1912
78
    if (IsX86_MMXType(CGT.ConvertType(Ty)))
1913
8
      return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(), 64));
1914
1915
70
    return ABIArgInfo::getDirect();
1916
78
  }
1917
1918
1919
31.6k
  if (const EnumType *EnumTy = Ty->getAs<EnumType>())
1920
25
    Ty = EnumTy->getDecl()->getIntegerType();
1921
1922
31.6k
  bool InReg = shouldPrimitiveUseInReg(Ty, State);
1923
1924
31.6k
  if (isPromotableIntegerTypeForABI(Ty)) {
1925
284
    if (InReg)
1926
25
      return ABIArgInfo::getExtendInReg(Ty);
1927
259
    return ABIArgInfo::getExtend(Ty);
1928
284
  }
1929
1930
31.3k
  if (const auto *EIT = Ty->getAs<BitIntType>()) {
1931
44
    if (EIT->getNumBits() <= 64) {
1932
20
      if (InReg)
1933
0
        return ABIArgInfo::getDirectInReg();
1934
20
      return ABIArgInfo::getDirect();
1935
20
    }
1936
24
    return getIndirectResult(Ty, /*ByVal=*/false, State);
1937
44
  }
1938
1939
31.3k
  if (InReg)
1940
159
    return ABIArgInfo::getDirectInReg();
1941
31.1k
  return ABIArgInfo::getDirect();
1942
31.3k
}
1943
1944
18.5k
void X86_32ABIInfo::computeInfo(CGFunctionInfo &FI) const {
1945
18.5k
  CCState State(FI);
1946
18.5k
  if (IsMCUABI)
1947
33
    State.FreeRegs = 3;
1948
18.5k
  else if (State.CC == llvm::CallingConv::X86_FastCall) {
1949
77
    State.FreeRegs = 2;
1950
77
    State.FreeSSERegs = 3;
1951
18.4k
  } else if (State.CC == llvm::CallingConv::X86_VectorCall) {
1952
57
    State.FreeRegs = 2;
1953
57
    State.FreeSSERegs = 6;
1954
18.4k
  } else if (FI.getHasRegParm())
1955
31
    State.FreeRegs = FI.getRegParm();
1956
18.3k
  else if (State.CC == llvm::CallingConv::X86_RegCall) {
1957
54
    State.FreeRegs = 5;
1958
54
    State.FreeSSERegs = 8;
1959
18.3k
  } else if (IsWin32StructABI) {
1960
    // Since MSVC 2015, the first three SSE vectors have been passed in
1961
    // registers. The rest are passed indirectly.
1962
4.04k
    State.FreeRegs = DefaultNumRegisterParameters;
1963
4.04k
    State.FreeSSERegs = 3;
1964
4.04k
  } else
1965
14.2k
    State.FreeRegs = DefaultNumRegisterParameters;
1966
1967
18.5k
  if (!::classifyReturnType(getCXXABI(), FI, *this)) {
1968
18.4k
    FI.getReturnInfo() = classifyReturnType(FI.getReturnType(), State);
1969
18.4k
  } else 
if (83
FI.getReturnInfo().isIndirect()83
) {
1970
    // The C++ ABI is not aware of register usage, so we have to check if the
1971
    // return value was sret and put it in a register ourselves if appropriate.
1972
83
    if (State.FreeRegs) {
1973
4
      --State.FreeRegs;  // The sret parameter consumes a register.
1974
4
      if (!IsMCUABI)
1975
4
        FI.getReturnInfo().setInReg(true);
1976
4
    }
1977
83
  }
1978
1979
  // The chain argument effectively gives us another free register.
1980
18.5k
  if (FI.isChainCall())
1981
4
    ++State.FreeRegs;
1982
1983
  // For vectorcall, do a first pass over the arguments, assigning FP and vector
1984
  // arguments to XMM registers as available.
1985
18.5k
  if (State.CC == llvm::CallingConv::X86_VectorCall)
1986
57
    runVectorCallFirstPass(FI, State);
1987
1988
18.5k
  bool UsedInAlloca = false;
1989
18.5k
  MutableArrayRef<CGFunctionInfoArgInfo> Args = FI.arguments();
1990
51.0k
  for (int I = 0, E = Args.size(); I < E; 
++I32.5k
) {
1991
    // Skip arguments that have already been assigned.
1992
32.5k
    if (State.IsPreassigned.test(I))
1993
38
      continue;
1994
1995
32.4k
    Args[I].info = classifyArgumentType(Args[I].type, State);
1996
32.4k
    UsedInAlloca |= (Args[I].info.getKind() == ABIArgInfo::InAlloca);
1997
32.4k
  }
1998
1999
  // If we needed to use inalloca for any argument, do a second pass and rewrite
2000
  // all the memory arguments to use inalloca.
2001
18.5k
  if (UsedInAlloca)
2002
73
    rewriteWithInAlloca(FI);
2003
18.5k
}
2004
2005
void
2006
X86_32ABIInfo::addFieldToArgStruct(SmallVector<llvm::Type *, 6> &FrameFields,
2007
                                   CharUnits &StackOffset, ABIArgInfo &Info,
2008
167
                                   QualType Type) const {
2009
  // Arguments are always 4-byte-aligned.
2010
167
  CharUnits WordSize = CharUnits::fromQuantity(4);
2011
167
  assert(StackOffset.isMultipleOf(WordSize) && "unaligned inalloca struct");
2012
2013
  // sret pointers and indirect things will require an extra pointer
2014
  // indirection, unless they are byval. Most things are byval, and will not
2015
  // require this indirection.
2016
0
  bool IsIndirect = false;
2017
167
  if (Info.isIndirect() && 
!Info.getIndirectByVal()17
)
2018
14
    IsIndirect = true;
2019
167
  Info = ABIArgInfo::getInAlloca(FrameFields.size(), IsIndirect);
2020
167
  llvm::Type *LLTy = CGT.ConvertTypeForMem(Type);
2021
167
  if (IsIndirect)
2022
14
    LLTy = LLTy->getPointerTo(0);
2023
167
  FrameFields.push_back(LLTy);
2024
167
  StackOffset += IsIndirect ? 
WordSize14
:
getContext().getTypeSizeInChars(Type)153
;
2025
2026
  // Insert padding bytes to respect alignment.
2027
167
  CharUnits FieldEnd = StackOffset;
2028
167
  StackOffset = FieldEnd.alignTo(WordSize);
2029
167
  if (StackOffset != FieldEnd) {
2030
40
    CharUnits NumBytes = StackOffset - FieldEnd;
2031
40
    llvm::Type *Ty = llvm::Type::getInt8Ty(getVMContext());
2032
40
    Ty = llvm::ArrayType::get(Ty, NumBytes.getQuantity());
2033
40
    FrameFields.push_back(Ty);
2034
40
  }
2035
167
}
2036
2037
177
static bool isArgInAlloca(const ABIArgInfo &Info) {
2038
  // Leave ignored and inreg arguments alone.
2039
177
  switch (Info.getKind()) {
2040
86
  case ABIArgInfo::InAlloca:
2041
86
    return true;
2042
0
  case ABIArgInfo::Ignore:
2043
0
  case ABIArgInfo::IndirectAliased:
2044
0
    return false;
2045
11
  case ABIArgInfo::Indirect:
2046
84
  case ABIArgInfo::Direct:
2047
90
  case ABIArgInfo::Extend:
2048
90
    return !Info.getInReg();
2049
1
  case ABIArgInfo::Expand:
2050
1
  case ABIArgInfo::CoerceAndExpand:
2051
    // These are aggregate types which are never passed in registers when
2052
    // inalloca is involved.
2053
1
    return true;
2054
177
  }
2055
0
  llvm_unreachable("invalid enum");
2056
0
}
2057
2058
73
void X86_32ABIInfo::rewriteWithInAlloca(CGFunctionInfo &FI) const {
2059
73
  assert(IsWin32StructABI && "inalloca only supported on win32");
2060
2061
  // Build a packed struct type for all of the arguments in memory.
2062
0
  SmallVector<llvm::Type *, 6> FrameFields;
2063
2064
  // The stack alignment is always 4.
2065
73
  CharUnits StackAlign = CharUnits::fromQuantity(4);
2066
2067
73
  CharUnits StackOffset;
2068
73
  CGFunctionInfo::arg_iterator I = FI.arg_begin(), E = FI.arg_end();
2069
2070
  // Put 'this' into the struct before 'sret', if necessary.
2071
73
  bool IsThisCall =
2072
73
      FI.getCallingConvention() == llvm::CallingConv::X86_ThisCall;
2073
73
  ABIArgInfo &Ret = FI.getReturnInfo();
2074
73
  if (Ret.isIndirect() && 
Ret.isSRetAfterThis()9
&&
!IsThisCall7
&&
2075
73
      
isArgInAlloca(I->info)6
) {
2076
5
    addFieldToArgStruct(FrameFields, StackOffset, I->info, I->type);
2077
5
    ++I;
2078
5
  }
2079
2080
  // Put the sret parameter into the inalloca struct if it's in memory.
2081
73
  if (Ret.isIndirect() && 
!Ret.getInReg()9
) {
2082
8
    addFieldToArgStruct(FrameFields, StackOffset, Ret, FI.getReturnType());
2083
    // On Windows, the hidden sret parameter is always returned in eax.
2084
8
    Ret.setInAllocaSRet(IsWin32StructABI);
2085
8
  }
2086
2087
  // Skip the 'this' parameter in ecx.
2088
73
  if (IsThisCall)
2089
17
    ++I;
2090
2091
  // Put arguments passed in memory into the struct.
2092
244
  for (; I != E; 
++I171
) {
2093
171
    if (isArgInAlloca(I->info))
2094
154
      addFieldToArgStruct(FrameFields, StackOffset, I->info, I->type);
2095
171
  }
2096
2097
73
  FI.setArgStruct(llvm::StructType::get(getVMContext(), FrameFields,
2098
73
                                        /*isPacked=*/true),
2099
73
                  StackAlign);
2100
73
}
2101
2102
Address X86_32ABIInfo::EmitVAArg(CodeGenFunction &CGF,
2103
50
                                 Address VAListAddr, QualType Ty) const {
2104
2105
50
  auto TypeInfo = getContext().getTypeInfoInChars(Ty);
2106
2107
  // x86-32 changes the alignment of certain arguments on the stack.
2108
  //
2109
  // Just messing with TypeInfo like this works because we never pass
2110
  // anything indirectly.
2111
50
  TypeInfo.Align = CharUnits::fromQuantity(
2112
50
                getTypeStackAlignInBytes(Ty, TypeInfo.Align.getQuantity()));
2113
2114
50
  return emitVoidPtrVAArg(CGF, VAListAddr, Ty, /*Indirect*/ false,
2115
50
                          TypeInfo, CharUnits::fromQuantity(4),
2116
50
                          /*AllowHigherAlign*/ true);
2117
50
}
2118
2119
bool X86_32TargetCodeGenInfo::isStructReturnInRegABI(
2120
3.09k
    const llvm::Triple &Triple, const CodeGenOptions &Opts) {
2121
3.09k
  assert(Triple.getArch() == llvm::Triple::x86);
2122
2123
0
  switch (Opts.getStructReturnConvention()) {
2124
3.08k
  case CodeGenOptions::SRCK_Default:
2125
3.08k
    break;
2126
3
  case CodeGenOptions::SRCK_OnStack:  // -fpcc-struct-return
2127
3
    return false;
2128
3
  case CodeGenOptions::SRCK_InRegs:  // -freg-struct-return
2129
3
    return true;
2130
3.09k
  }
2131
2132
3.08k
  if (Triple.isOSDarwin() || 
Triple.isOSIAMCU()2.91k
)
2133
180
    return true;
2134
2135
2.90k
  switch (Triple.getOS()) {
2136
1
  case llvm::Triple::DragonFly:
2137
1
  case llvm::Triple::FreeBSD:
2138
2
  case llvm::Triple::OpenBSD:
2139
437
  case llvm::Triple::Win32:
2140
437
    return true;
2141
2.47k
  default:
2142
2.47k
    return false;
2143
2.90k
  }
2144
2.90k
}
2145
2146
static void addX86InterruptAttrs(const FunctionDecl *FD, llvm::GlobalValue *GV,
2147
192k
                                 CodeGen::CodeGenModule &CGM) {
2148
192k
  if (!FD->hasAttr<AnyX86InterruptAttr>())
2149
192k
    return;
2150
2151
25
  llvm::Function *Fn = cast<llvm::Function>(GV);
2152
25
  Fn->setCallingConv(llvm::CallingConv::X86_INTR);
2153
25
  if (FD->getNumParams() == 0)
2154
0
    return;
2155
2156
25
  auto PtrTy = cast<PointerType>(FD->getParamDecl(0)->getType());
2157
25
  llvm::Type *ByValTy = CGM.getTypes().ConvertType(PtrTy->getPointeeType());
2158
25
  llvm::Attribute NewAttr = llvm::Attribute::getWithByValType(
2159
25
    Fn->getContext(), ByValTy);
2160
25
  Fn->addParamAttr(0, NewAttr);
2161
25
}
2162
2163
void X86_32TargetCodeGenInfo::setTargetAttributes(
2164
73.1k
    const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {
2165
73.1k
  if (GV->isDeclaration())
2166
42.6k
    return;
2167
30.4k
  if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) {
2168
21.2k
    if (FD->hasAttr<X86ForceAlignArgPointerAttr>()) {
2169
2
      llvm::Function *Fn = cast<llvm::Function>(GV);
2170
2
      Fn->addFnAttr("stackrealign");
2171
2
    }
2172
2173
21.2k
    addX86InterruptAttrs(FD, GV, CGM);
2174
21.2k
  }
2175
30.4k
}
2176
2177
bool X86_32TargetCodeGenInfo::initDwarfEHRegSizeTable(
2178
                                               CodeGen::CodeGenFunction &CGF,
2179
0
                                               llvm::Value *Address) const {
2180
0
  CodeGen::CGBuilderTy &Builder = CGF.Builder;
2181
2182
0
  llvm::Value *Four8 = llvm::ConstantInt::get(CGF.Int8Ty, 4);
2183
2184
  // 0-7 are the eight integer registers;  the order is different
2185
  //   on Darwin (for EH), but the range is the same.
2186
  // 8 is %eip.
2187
0
  AssignToArrayRange(Builder, Address, Four8, 0, 8);
2188
2189
0
  if (CGF.CGM.getTarget().getTriple().isOSDarwin()) {
2190
    // 12-16 are st(0..4).  Not sure why we stop at 4.
2191
    // These have size 16, which is sizeof(long double) on
2192
    // platforms with 8-byte alignment for that type.
2193
0
    llvm::Value *Sixteen8 = llvm::ConstantInt::get(CGF.Int8Ty, 16);
2194
0
    AssignToArrayRange(Builder, Address, Sixteen8, 12, 16);
2195
2196
0
  } else {
2197
    // 9 is %eflags, which doesn't get a size on Darwin for some
2198
    // reason.
2199
0
    Builder.CreateAlignedStore(
2200
0
        Four8, Builder.CreateConstInBoundsGEP1_32(CGF.Int8Ty, Address, 9),
2201
0
                               CharUnits::One());
2202
2203
    // 11-16 are st(0..5).  Not sure why we stop at 5.
2204
    // These have size 12, which is sizeof(long double) on
2205
    // platforms with 4-byte alignment for that type.
2206
0
    llvm::Value *Twelve8 = llvm::ConstantInt::get(CGF.Int8Ty, 12);
2207
0
    AssignToArrayRange(Builder, Address, Twelve8, 11, 16);
2208
0
  }
2209
2210
0
  return false;
2211
0
}
2212
2213
//===----------------------------------------------------------------------===//
2214
// X86-64 ABI Implementation
2215
//===----------------------------------------------------------------------===//
2216
2217
2218
namespace {
2219
/// The AVX ABI level for X86 targets.
2220
enum class X86AVXABILevel {
2221
  None,
2222
  AVX,
2223
  AVX512
2224
};
2225
2226
/// \p returns the size in bits of the largest (native) vector for \p AVXLevel.
2227
3.98k
static unsigned getNativeVectorSizeForAVXABI(X86AVXABILevel AVXLevel) {
2228
3.98k
  switch (AVXLevel) {
2229
3.06k
  case X86AVXABILevel::AVX512:
2230
3.06k
    return 512;
2231
697
  case X86AVXABILevel::AVX:
2232
697
    return 256;
2233
229
  case X86AVXABILevel::None:
2234
229
    return 128;
2235
3.98k
  }
2236
0
  llvm_unreachable("Unknown AVXLevel");
2237
0
}
2238
2239
/// X86_64ABIInfo - The X86_64 ABI information.
2240
class X86_64ABIInfo : public SwiftABIInfo {
2241
  enum Class {
2242
    Integer = 0,
2243
    SSE,
2244
    SSEUp,
2245
    X87,
2246
    X87Up,
2247
    ComplexX87,
2248
    NoClass,
2249
    Memory
2250
  };
2251
2252
  /// merge - Implement the X86_64 ABI merging algorithm.
2253
  ///
2254
  /// Merge an accumulating classification \arg Accum with a field
2255
  /// classification \arg Field.
2256
  ///
2257
  /// \param Accum - The accumulating classification. This should
2258
  /// always be either NoClass or the result of a previous merge
2259
  /// call. In addition, this should never be Memory (the caller
2260
  /// should just return Memory for the aggregate).
2261
  static Class merge(Class Accum, Class Field);
2262
2263
  /// postMerge - Implement the X86_64 ABI post merging algorithm.
2264
  ///
2265
  /// Post merger cleanup, reduces a malformed Hi and Lo pair to
2266
  /// final MEMORY or SSE classes when necessary.
2267
  ///
2268
  /// \param AggregateSize - The size of the current aggregate in
2269
  /// the classification process.
2270
  ///
2271
  /// \param Lo - The classification for the parts of the type
2272
  /// residing in the low word of the containing object.
2273
  ///
2274
  /// \param Hi - The classification for the parts of the type
2275
  /// residing in the higher words of the containing object.
2276
  ///
2277
  void postMerge(unsigned AggregateSize, Class &Lo, Class &Hi) const;
2278
2279
  /// classify - Determine the x86_64 register classes in which the
2280
  /// given type T should be passed.
2281
  ///
2282
  /// \param Lo - The classification for the parts of the type
2283
  /// residing in the low word of the containing object.
2284
  ///
2285
  /// \param Hi - The classification for the parts of the type
2286
  /// residing in the high word of the containing object.
2287
  ///
2288
  /// \param OffsetBase - The bit offset of this type in the
2289
  /// containing object.  Some parameters are classified different
2290
  /// depending on whether they straddle an eightbyte boundary.
2291
  ///
2292
  /// \param isNamedArg - Whether the argument in question is a "named"
2293
  /// argument, as used in AMD64-ABI 3.5.7.
2294
  ///
2295
  /// If a word is unused its result will be NoClass; if a type should
2296
  /// be passed in Memory then at least the classification of \arg Lo
2297
  /// will be Memory.
2298
  ///
2299
  /// The \arg Lo class will be NoClass iff the argument is ignored.
2300
  ///
2301
  /// If the \arg Lo class is ComplexX87, then the \arg Hi class will
2302
  /// also be ComplexX87.
2303
  void classify(QualType T, uint64_t OffsetBase, Class &Lo, Class &Hi,
2304
                bool isNamedArg) const;
2305
2306
  llvm::Type *GetByteVectorType(QualType Ty) const;
2307
  llvm::Type *GetSSETypeAtOffset(llvm::Type *IRType,
2308
                                 unsigned IROffset, QualType SourceTy,
2309
                                 unsigned SourceOffset) const;
2310
  llvm::Type *GetINTEGERTypeAtOffset(llvm::Type *IRType,
2311
                                     unsigned IROffset, QualType SourceTy,
2312
                                     unsigned SourceOffset) const;
2313
2314
  /// getIndirectResult - Give a source type \arg Ty, return a suitable result
2315
  /// such that the argument will be returned in memory.
2316
  ABIArgInfo getIndirectReturnResult(QualType Ty) const;
2317
2318
  /// getIndirectResult - Give a source type \arg Ty, return a suitable result
2319
  /// such that the argument will be passed in memory.
2320
  ///
2321
  /// \param freeIntRegs - The number of free integer registers remaining
2322
  /// available.
2323
  ABIArgInfo getIndirectResult(QualType Ty, unsigned freeIntRegs) const;
2324
2325
  ABIArgInfo classifyReturnType(QualType RetTy) const;
2326
2327
  ABIArgInfo classifyArgumentType(QualType Ty, unsigned freeIntRegs,
2328
                                  unsigned &neededInt, unsigned &neededSSE,
2329
                                  bool isNamedArg) const;
2330
2331
  ABIArgInfo classifyRegCallStructType(QualType Ty, unsigned &NeededInt,
2332
                                       unsigned &NeededSSE) const;
2333
2334
  ABIArgInfo classifyRegCallStructTypeImpl(QualType Ty, unsigned &NeededInt,
2335
                                           unsigned &NeededSSE) const;
2336
2337
  bool IsIllegalVectorType(QualType Ty) const;
2338
2339
  /// The 0.98 ABI revision clarified a lot of ambiguities,
2340
  /// unfortunately in ways that were not always consistent with
2341
  /// certain previous compilers.  In particular, platforms which
2342
  /// required strict binary compatibility with older versions of GCC
2343
  /// may need to exempt themselves.
2344
17
  bool honorsRevision0_98() const {
2345
17
    return !getTarget().getTriple().isOSDarwin();
2346
17
  }
2347
2348
  /// GCC classifies <1 x long long> as SSE but some platform ABIs choose to
2349
  /// classify it as INTEGER (for compatibility with older clang compilers).
2350
186
  bool classifyIntegerMMXAsSSE() const {
2351
    // Clang <= 3.8 did not do this.
2352
186
    if (getContext().getLangOpts().getClangABICompat() <=
2353
186
        LangOptions::ClangABI::Ver3_8)
2354
3
      return false;
2355
2356
183
    const llvm::Triple &Triple = getTarget().getTriple();
2357
183
    if (Triple.isOSDarwin() || 
Triple.getOS() == llvm::Triple::PS466
)
2358
122
      return false;
2359
61
    if (Triple.isOSFreeBSD() && 
Triple.getOSMajorVersion() >= 100
)
2360
0
      return false;
2361
61
    return true;
2362
61
  }
2363
2364
  // GCC classifies vectors of __int128 as memory.
2365
12.3k
  bool passInt128VectorsInMem() const {
2366
    // Clang <= 9.0 did not do this.
2367
12.3k
    if (getContext().getLangOpts().getClangABICompat() <=
2368
12.3k
        LangOptions::ClangABI::Ver9)
2369
36
      return false;
2370
2371
12.2k
    const llvm::Triple &T = getTarget().getTriple();
2372
12.2k
    return T.isOSLinux() || 
T.isOSNetBSD()11.2k
;
2373
12.3k
  }
2374
2375
  X86AVXABILevel AVXLevel;
2376
  // Some ABIs (e.g. X32 ABI and Native Client OS) use 32 bit pointers on
2377
  // 64-bit hardware.
2378
  bool Has64BitPointers;
2379
2380
public:
2381
  X86_64ABIInfo(CodeGen::CodeGenTypes &CGT, X86AVXABILevel AVXLevel) :
2382
      SwiftABIInfo(CGT), AVXLevel(AVXLevel),
2383
24.9k
      Has64BitPointers(CGT.getDataLayout().getPointerSize(0) == 8) {
2384
24.9k
  }
2385
2386
122
  bool isPassedUsingAVXType(QualType type) const {
2387
122
    unsigned neededInt, neededSSE;
2388
    // The freeIntRegs argument doesn't matter here.
2389
122
    ABIArgInfo info = classifyArgumentType(type, 0, neededInt, neededSSE,
2390
122
                                           /*isNamedArg*/true);
2391
122
    if (info.isDirect()) {
2392
119
      llvm::Type *ty = info.getCoerceToType();
2393
119
      if (llvm::VectorType *vectorTy = dyn_cast_or_null<llvm::VectorType>(ty))
2394
2
        return vectorTy->getPrimitiveSizeInBits().getFixedSize() > 128;
2395
119
    }
2396
120
    return false;
2397
122
  }
2398
2399
  void computeInfo(CGFunctionInfo &FI) const override;
2400
2401
  Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
2402
                    QualType Ty) const override;
2403
  Address EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr,
2404
                      QualType Ty) const override;
2405
2406
0
  bool has64BitPointers() const {
2407
0
    return Has64BitPointers;
2408
0
  }
2409
2410
  bool shouldPassIndirectlyForSwift(ArrayRef<llvm::Type*> scalars,
2411
302
                                    bool asReturnValue) const override {
2412
302
    return occupiesMoreThan(CGT, scalars, /*total*/ 4);
2413
302
  }
2414
0
  bool isSwiftErrorInRegister() const override {
2415
0
    return true;
2416
0
  }
2417
};
2418
2419
/// WinX86_64ABIInfo - The Windows X86_64 ABI information.
2420
class WinX86_64ABIInfo : public SwiftABIInfo {
2421
public:
2422
  WinX86_64ABIInfo(CodeGen::CodeGenTypes &CGT, X86AVXABILevel AVXLevel)
2423
      : SwiftABIInfo(CGT), AVXLevel(AVXLevel),
2424
411
        IsMingw64(getTarget().getTriple().isWindowsGNUEnvironment()) {}
2425
2426
  void computeInfo(CGFunctionInfo &FI) const override;
2427
2428
  Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
2429
                    QualType Ty) const override;
2430
2431
371
  bool isHomogeneousAggregateBaseType(QualType Ty) const override {
2432
    // FIXME: Assumes vectorcall is in use.
2433
371
    return isX86VectorTypeForVectorCall(getContext(), Ty);
2434
371
  }
2435
2436
  bool isHomogeneousAggregateSmallEnough(const Type *Ty,
2437
408
                                         uint64_t NumMembers) const override {
2438
    // FIXME: Assumes vectorcall is in use.
2439
408
    return isX86VectorCallAggregateSmallEnough(NumMembers);
2440
408
  }
2441
2442
  bool shouldPassIndirectlyForSwift(ArrayRef<llvm::Type *> scalars,
2443
25
                                    bool asReturnValue) const override {
2444
25
    return occupiesMoreThan(CGT, scalars, /*total*/ 4);
2445
25
  }
2446
2447
0
  bool isSwiftErrorInRegister() const override {
2448
0
    return true;
2449
0
  }
2450
2451
private:
2452
  ABIArgInfo classify(QualType Ty, unsigned &FreeSSERegs, bool IsReturnType,
2453
                      bool IsVectorCall, bool IsRegCall) const;
2454
  ABIArgInfo reclassifyHvaArgForVectorCall(QualType Ty, unsigned &FreeSSERegs,
2455
                                           const ABIArgInfo &current) const;
2456
2457
  X86AVXABILevel AVXLevel;
2458
2459
  bool IsMingw64;
2460
};
2461
2462
class X86_64TargetCodeGenInfo : public TargetCodeGenInfo {
2463
public:
2464
  X86_64TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, X86AVXABILevel AVXLevel)
2465
24.8k
      : TargetCodeGenInfo(std::make_unique<X86_64ABIInfo>(CGT, AVXLevel)) {}
2466
2467
122
  const X86_64ABIInfo &getABIInfo() const {
2468
122
    return static_cast<const X86_64ABIInfo&>(TargetCodeGenInfo::getABIInfo());
2469
122
  }
2470
2471
  /// Disable tail call on x86-64. The epilogue code before the tail jump blocks
2472
  /// autoreleaseRV/retainRV and autoreleaseRV/unsafeClaimRV optimizations.
2473
161
  bool markARCOptimizedReturnCallsAsNoTail() const override { return true; }
2474
2475
0
  int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const override {
2476
0
    return 7;
2477
0
  }
2478
2479
  bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
2480
0
                               llvm::Value *Address) const override {
2481
0
    llvm::Value *Eight8 = llvm::ConstantInt::get(CGF.Int8Ty, 8);
2482
2483
    // 0-15 are the 16 integer registers.
2484
    // 16 is %rip.
2485
0
    AssignToArrayRange(CGF.Builder, Address, Eight8, 0, 16);
2486
0
    return false;
2487
0
  }
2488
2489
  llvm::Type* adjustInlineAsmType(CodeGen::CodeGenFunction &CGF,
2490
                                  StringRef Constraint,
2491
907
                                  llvm::Type* Ty) const override {
2492
907
    return X86AdjustInlineAsmType(CGF, Constraint, Ty);
2493
907
  }
2494
2495
  bool isNoProtoCallVariadic(const CallArgList &args,
2496
742
                             const FunctionNoProtoType *fnType) const override {
2497
    // The default CC on x86-64 sets %al to the number of SSA
2498
    // registers used, and GCC sets this when calling an unprototyped
2499
    // function, so we override the default behavior.  However, don't do
2500
    // that when AVX types are involved: the ABI explicitly states it is
2501
    // undefined, and it doesn't work in practice because of how the ABI
2502
    // defines varargs anyway.
2503
742
    if (fnType->getCallConv() == CC_C) {
2504
742
      bool HasAVXType = false;
2505
742
      for (CallArgList::const_iterator
2506
862
             it = args.begin(), ie = args.end(); it != ie; 
++it120
) {
2507
122
        if (getABIInfo().isPassedUsingAVXType(it->Ty)) {
2508
2
          HasAVXType = true;
2509
2
          break;
2510
2
        }
2511
122
      }
2512
2513
742
      if (!HasAVXType)
2514
740
        return true;
2515
742
    }
2516
2517
2
    return TargetCodeGenInfo::isNoProtoCallVariadic(args, fnType);
2518
742
  }
2519
2520
  llvm::Constant *
2521
82
  getUBSanFunctionSignature(CodeGen::CodeGenModule &CGM) const override {
2522
82
    unsigned Sig = (0xeb << 0) | // jmp rel8
2523
82
                   (0x06 << 8) | //           .+0x08
2524
82
                   ('v' << 16) |
2525
82
                   ('2' << 24);
2526
82
    return llvm::ConstantInt::get(CGM.Int32Ty, Sig);
2527
82
  }
2528
2529
  void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
2530
400k
                           CodeGen::CodeGenModule &CGM) const override {
2531
400k
    if (GV->isDeclaration())
2532
224k
      return;
2533
175k
    if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) {
2534
163k
      if (FD->hasAttr<X86ForceAlignArgPointerAttr>()) {
2535
1
        llvm::Function *Fn = cast<llvm::Function>(GV);
2536
1
        Fn->addFnAttr("stackrealign");
2537
1
      }
2538
2539
163k
      addX86InterruptAttrs(FD, GV, CGM);
2540
163k
    }
2541
175k
  }
2542
2543
  void checkFunctionCallABI(CodeGenModule &CGM, SourceLocation CallLoc,
2544
                            const FunctionDecl *Caller,
2545
                            const FunctionDecl *Callee,
2546
                            const CallArgList &Args) const override;
2547
};
2548
2549
static void initFeatureMaps(const ASTContext &Ctx,
2550
                            llvm::StringMap<bool> &CallerMap,
2551
                            const FunctionDecl *Caller,
2552
                            llvm::StringMap<bool> &CalleeMap,
2553
13.4k
                            const FunctionDecl *Callee) {
2554
13.4k
  if (CalleeMap.empty() && 
CallerMap.empty()6.25k
) {
2555
    // The caller is potentially nullptr in the case where the call isn't in a
2556
    // function.  In this case, the getFunctionFeatureMap ensures we just get
2557
    // the TU level setting (since it cannot be modified by 'target'..
2558
6.25k
    Ctx.getFunctionFeatureMap(CallerMap, Caller);
2559
6.25k
    Ctx.getFunctionFeatureMap(CalleeMap, Callee);
2560
6.25k
  }
2561
13.4k
}
2562
2563
static bool checkAVXParamFeature(DiagnosticsEngine &Diag,
2564
                                 SourceLocation CallLoc,
2565
                                 const llvm::StringMap<bool> &CallerMap,
2566
                                 const llvm::StringMap<bool> &CalleeMap,
2567
                                 QualType Ty, StringRef Feature,
2568
13.4k
                                 bool IsArgument) {
2569
13.4k
  bool CallerHasFeat = CallerMap.lookup(Feature);
2570
13.4k
  bool CalleeHasFeat = CalleeMap.lookup(Feature);
2571
13.4k
  if (!CallerHasFeat && 
!CalleeHasFeat62
)
2572
56
    return Diag.Report(CallLoc, diag::warn_avx_calling_convention)
2573
56
           << IsArgument << Ty << Feature;
2574
2575
  // Mixing calling conventions here is very clearly an error.
2576
13.3k
  if (!CallerHasFeat || 
!CalleeHasFeat13.3k
)
2577
6
    return Diag.Report(CallLoc, diag::err_avx_calling_convention)
2578
6
           << IsArgument << Ty << Feature;
2579
2580
  // Else, both caller and callee have the required feature, so there is no need
2581
  // to diagnose.
2582
13.3k
  return false;
2583
13.3k
}
2584
2585
static bool checkAVXParam(DiagnosticsEngine &Diag, ASTContext &Ctx,
2586
                          SourceLocation CallLoc,
2587
                          const llvm::StringMap<bool> &CallerMap,
2588
                          const llvm::StringMap<bool> &CalleeMap, QualType Ty,
2589
13.4k
                          bool IsArgument) {
2590
13.4k
  uint64_t Size = Ctx.getTypeSize(Ty);
2591
13.4k
  if (Size > 256)
2592
6.35k
    return checkAVXParamFeature(Diag, CallLoc, CallerMap, CalleeMap, Ty,
2593
6.35k
                                "avx512f", IsArgument);
2594
2595
7.06k
  if (Size > 128)
2596
7.06k
    return checkAVXParamFeature(Diag, CallLoc, CallerMap, CalleeMap, Ty, "avx",
2597
7.06k
                                IsArgument);
2598
2599
0
  return false;
2600
7.06k
}
2601
2602
void X86_64TargetCodeGenInfo::checkFunctionCallABI(
2603
    CodeGenModule &CGM, SourceLocation CallLoc, const FunctionDecl *Caller,
2604
295k
    const FunctionDecl *Callee, const CallArgList &Args) const {
2605
295k
  llvm::StringMap<bool> CallerMap;
2606
295k
  llvm::StringMap<bool> CalleeMap;
2607
295k
  unsigned ArgIndex = 0;
2608
2609
  // We need to loop through the actual call arguments rather than the the
2610
  // function's parameters, in case this variadic.
2611
494k
  for (const CallArg &Arg : Args) {
2612
    // The "avx" feature changes how vectors >128 in size are passed. "avx512f"
2613
    // additionally changes how vectors >256 in size are passed. Like GCC, we
2614
    // warn when a function is called with an argument where this will change.
2615
    // Unlike GCC, we also error when it is an obvious ABI mismatch, that is,
2616
    // the caller and callee features are mismatched.
2617
    // Unfortunately, we cannot do this diagnostic in SEMA, since the callee can
2618
    // change its ABI with attribute-target after this call.
2619
494k
    if (Arg.getType()->isVectorType() &&
2620
494k
        
CGM.getContext().getTypeSize(Arg.getType()) > 12814.6k
) {
2621
7.75k
      initFeatureMaps(CGM.getContext(), CallerMap, Caller, CalleeMap, Callee);
2622
7.75k
      QualType Ty = Arg.getType();
2623
      // The CallArg seems to have desugared the type already, so for clearer
2624
      // diagnostics, replace it with the type in the FunctionDecl if possible.
2625
7.75k
      if (ArgIndex < Callee->getNumParams())
2626
7.72k
        Ty = Callee->getParamDecl(ArgIndex)->getType();
2627
2628
7.75k
      if (checkAVXParam(CGM.getDiags(), CGM.getContext(), CallLoc, CallerMap,
2629
7.75k
                        CalleeMap, Ty, /*IsArgument*/ true))
2630
54
        return;
2631
7.75k
    }
2632
494k
    ++ArgIndex;
2633
494k
  }
2634
2635
  // Check return always, as we don't have a good way of knowing in codegen
2636
  // whether this value is used, tail-called, etc.
2637
295k
  if (Callee->getReturnType()->isVectorType() &&
2638
295k
      
CGM.getContext().getTypeSize(Callee->getReturnType()) > 12810.0k
) {
2639
5.66k
    initFeatureMaps(CGM.getContext(), CallerMap, Caller, CalleeMap, Callee);
2640
5.66k
    checkAVXParam(CGM.getDiags(), CGM.getContext(), CallLoc, CallerMap,
2641
5.66k
                  CalleeMap, Callee->getReturnType(),
2642
5.66k
                  /*IsArgument*/ false);
2643
5.66k
  }
2644
295k
}
2645
2646
87
static std::string qualifyWindowsLibrary(llvm::StringRef Lib) {
2647
  // If the argument does not end in .lib, automatically add the suffix.
2648
  // If the argument contains a space, enclose it in quotes.
2649
  // This matches the behavior of MSVC.
2650
87
  bool Quote = Lib.contains(' ');
2651
87
  std::string ArgStr = Quote ? 
"\""4
:
""83
;
2652
87
  ArgStr += Lib;
2653
87
  if (!Lib.endswith_insensitive(".lib") && 
!Lib.endswith_insensitive(".a")75
)
2654
75
    ArgStr += ".lib";
2655
87
  ArgStr += Quote ? 
"\""4
:
""83
;
2656
87
  return ArgStr;
2657
87
}
2658
2659
class WinX86_32TargetCodeGenInfo : public X86_32TargetCodeGenInfo {
2660
public:
2661
  WinX86_32TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT,
2662
        bool DarwinVectorABI, bool RetSmallStructInRegABI, bool Win32StructABI,
2663
        unsigned NumRegisterParameters)
2664
    : X86_32TargetCodeGenInfo(CGT, DarwinVectorABI, RetSmallStructInRegABI,
2665
437
        Win32StructABI, NumRegisterParameters, false) {}
2666
2667
  void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
2668
                           CodeGen::CodeGenModule &CGM) const override;
2669
2670
  void getDependentLibraryOption(llvm::StringRef Lib,
2671
31
                                 llvm::SmallString<24> &Opt) const override {
2672
31
    Opt = "/DEFAULTLIB:";
2673
31
    Opt += qualifyWindowsLibrary(Lib);
2674
31
  }
2675
2676
  void getDetectMismatchOption(llvm::StringRef Name,
2677
                               llvm::StringRef Value,
2678
2
                               llvm::SmallString<32> &Opt) const override {
2679
2
    Opt = "/FAILIFMISMATCH:\"" + Name.str() + "=" + Value.str() + "\"";
2680
2
  }
2681
};
2682
2683
static void addStackProbeTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
2684
17.6k
                                          CodeGen::CodeGenModule &CGM) {
2685
17.6k
  if (llvm::Function *Fn = dyn_cast_or_null<llvm::Function>(GV)) {
2686
2687
15.1k
    if (CGM.getCodeGenOpts().StackProbeSize != 4096)
2688
1
      Fn->addFnAttr("stack-probe-size",
2689
1
                    llvm::utostr(CGM.getCodeGenOpts().StackProbeSize));
2690
15.1k
    if (CGM.getCodeGenOpts().NoStackArgProbe)
2691
4
      Fn->addFnAttr("no-stack-arg-probe");
2692
15.1k
  }
2693
17.6k
}
2694
2695
void WinX86_32TargetCodeGenInfo::setTargetAttributes(
2696
17.8k
    const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {
2697
17.8k
  X86_32TargetCodeGenInfo::setTargetAttributes(D, GV, CGM);
2698
17.8k
  if (GV->isDeclaration())
2699
10.1k
    return;
2700
7.69k
  addStackProbeTargetAttributes(D, GV, CGM);
2701
7.69k
}
2702
2703
class WinX86_64TargetCodeGenInfo : public TargetCodeGenInfo {
2704
public:
2705
  WinX86_64TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT,
2706
                             X86AVXABILevel AVXLevel)
2707
398
      : TargetCodeGenInfo(std::make_unique<WinX86_64ABIInfo>(CGT, AVXLevel)) {}
2708
2709
  void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
2710
                           CodeGen::CodeGenModule &CGM) const override;
2711
2712
0
  int getDwarfEHStackPointer(CodeGen::CodeGenModule &CGM) const override {
2713
0
    return 7;
2714
0
  }
2715
2716
  bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
2717
0
                               llvm::Value *Address) const override {
2718
0
    llvm::Value *Eight8 = llvm::ConstantInt::get(CGF.Int8Ty, 8);
2719
2720
    // 0-15 are the 16 integer registers.
2721
    // 16 is %rip.
2722
0
    AssignToArrayRange(CGF.Builder, Address, Eight8, 0, 16);
2723
0
    return false;
2724
0
  }
2725
2726
  void getDependentLibraryOption(llvm::StringRef Lib,
2727
45
                                 llvm::SmallString<24> &Opt) const override {
2728
45
    Opt = "/DEFAULTLIB:";
2729
45
    Opt += qualifyWindowsLibrary(Lib);
2730
45
  }
2731
2732
  void getDetectMismatchOption(llvm::StringRef Name,
2733
                               llvm::StringRef Value,
2734
4
                               llvm::SmallString<32> &Opt) const override {
2735
4
    Opt = "/FAILIFMISMATCH:\"" + Name.str() + "=" + Value.str() + "\"";
2736
4
  }
2737
};
2738
2739
void WinX86_64TargetCodeGenInfo::setTargetAttributes(
2740
19.8k
    const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const {
2741
19.8k
  TargetCodeGenInfo::setTargetAttributes(D, GV, CGM);
2742
19.8k
  if (GV->isDeclaration())
2743
10.8k
    return;
2744
9.05k
  if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) {
2745
7.95k
    if (FD->hasAttr<X86ForceAlignArgPointerAttr>()) {
2746
0
      llvm::Function *Fn = cast<llvm::Function>(GV);
2747
0
      Fn->addFnAttr("stackrealign");
2748
0
    }
2749
2750
7.95k
    addX86InterruptAttrs(FD, GV, CGM);
2751
7.95k
  }
2752
2753
9.05k
  addStackProbeTargetAttributes(D, GV, CGM);
2754
9.05k
}
2755
}
2756
2757
void X86_64ABIInfo::postMerge(unsigned AggregateSize, Class &Lo,
2758
13.7k
                              Class &Hi) const {
2759
  // AMD64-ABI 3.2.3p2: Rule 5. Then a post merger cleanup is done:
2760
  //
2761
  // (a) If one of the classes is Memory, the whole argument is passed in
2762
  //     memory.
2763
  //
2764
  // (b) If X87UP is not preceded by X87, the whole argument is passed in
2765
  //     memory.
2766
  //
2767
  // (c) If the size of the aggregate exceeds two eightbytes and the first
2768
  //     eightbyte isn't SSE or any other eightbyte isn't SSEUP, the whole
2769
  //     argument is passed in memory. NOTE: This is necessary to keep the
2770
  //     ABI working for processors that don't support the __m256 type.
2771
  //
2772
  // (d) If SSEUP is not preceded by SSE or SSEUP, it is converted to SSE.
2773
  //
2774
  // Some of these are enforced by the merging logic.  Others can arise
2775
  // only with unions; for example:
2776
  //   union { _Complex double; unsigned; }
2777
  //
2778
  // Note that clauses (b) and (c) were added in 0.98.
2779
  //
2780
13.7k
  if (Hi == Memory)
2781
0
    Lo = Memory;
2782
13.7k
  if (Hi == X87Up && 
Lo != X87374
&&
honorsRevision0_98()17
)
2783
15
    Lo = Memory;
2784
13.7k
  if (AggregateSize > 128 && 
(335
Lo != SSE335
||
Hi != SSEUp24
))
2785
311
    Lo = Memory;
2786
13.7k
  if (Hi == SSEUp && 
Lo != SSE33
)
2787
1
    Hi = SSE;
2788
13.7k
}
2789
2790
20.4k
X86_64ABIInfo::Class X86_64ABIInfo::merge(Class Accum, Class Field) {
2791
  // AMD64-ABI 3.2.3p2: Rule 4. Each field of an object is
2792
  // classified recursively so that always two fields are
2793
  // considered. The resulting class is calculated according to
2794
  // the classes of the fields in the eightbyte:
2795
  //
2796
  // (a) If both classes are equal, this is the resulting class.
2797
  //
2798
  // (b) If one of the classes is NO_CLASS, the resulting class is
2799
  // the other class.
2800
  //
2801
  // (c) If one of the classes is MEMORY, the result is the MEMORY
2802
  // class.
2803
  //
2804
  // (d) If one of the classes is INTEGER, the result is the
2805
  // INTEGER.
2806
  //
2807
  // (e) If one of the classes is X87, X87UP, COMPLEX_X87 class,
2808
  // MEMORY is used as class.
2809
  //
2810
  // (f) Otherwise class SSE is used.
2811
2812
  // Accum should never be memory (we should have returned) or
2813
  // ComplexX87 (because this cannot be passed in a structure).
2814
20.4k
  assert((Accum != Memory && Accum != ComplexX87) &&
2815
20.4k
         "Invalid accumulated classification during merge.");
2816
20.4k
  if (Accum == Field || 
Field == NoClass10.6k
)
2817
11.3k
    return Accum;
2818
9.09k
  if (Field == Memory)
2819
17
    return Memory;
2820
9.08k
  if (Accum == NoClass)
2821
8.97k
    return Field;
2822
101
  if (Accum == Integer || 
Field == Integer73
)
2823
98
    return Integer;
2824
3
  if (Field == X87 || Field == X87Up || Field == ComplexX87 ||
2825
3
      Accum == X87 || 
Accum == X87Up0
)
2826
3
    return Memory;
2827
0
  return SSE;
2828
3
}
2829
2830
void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase,
2831
472k
                             Class &Lo, Class &Hi, bool isNamedArg) const {
2832
  // FIXME: This code can be simplified by introducing a simple value class for
2833
  // Class pairs with appropriate constructor methods for the various
2834
  // situations.
2835
2836
  // FIXME: Some of the split computations are wrong; unaligned vectors
2837
  // shouldn't be passed in registers for example, so there is no chance they
2838
  // can straddle an eightbyte. Verify & simplify.
2839
2840
472k
  Lo = Hi = NoClass;
2841
2842
472k
  Class &Current = OffsetBase < 64 ? 
Lo470k
:
Hi1.70k
;
2843
472k
  Current = Memory;
2844
2845
472k
  if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
2846
181k
    BuiltinType::Kind k = BT->getKind();
2847
2848
181k
    if (k == BuiltinType::Void) {
2849
65.4k
      Current = NoClass;
2850
116k
    } else if (k == BuiltinType::Int128 || 
k == BuiltinType::UInt128116k
) {
2851
32
      Lo = Integer;
2852
32
      Hi = Integer;
2853
116k
    } else if (k >= BuiltinType::Bool && 
k <= BuiltinType::LongLong116k
) {
2854
110k
      Current = Integer;
2855
110k
    } else 
if (5.35k
k == BuiltinType::Float5.35k
||
k == BuiltinType::Double3.08k
||
2856
5.35k
               
k == BuiltinType::Float161.40k
) {
2857
4.12k
      Current = SSE;
2858
4.12k
    } else 
if (1.22k
k == BuiltinType::LongDouble1.22k
) {
2859
1.00k
      const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();
2860
1.00k
      if (LDF == &llvm::APFloat::IEEEquad()) {
2861
40
        Lo = SSE;
2862
40
        Hi = SSEUp;
2863
960
      } else if (LDF == &llvm::APFloat::x87DoubleExtended()) {
2864
938
        Lo = X87;
2865
938
        Hi = X87Up;
2866
938
      } else 
if (22
LDF == &llvm::APFloat::IEEEdouble()22
) {
2867
22
        Current = SSE;
2868
22
      } else
2869
0
        llvm_unreachable("unexpected long double representation!");
2870
1.00k
    }
2871
    // FIXME: _Decimal32 and _Decimal64 are SSE.
2872
    // FIXME: _float128 and _Decimal128 are (SSE, SSEUp).
2873
181k
    return;
2874
181k
  }
2875
2876
290k
  if (const EnumType *ET = Ty->getAs<EnumType>()) {
2877
    // Classify the underlying integer type.
2878
954
    classify(ET->getDecl()->getIntegerType(), OffsetBase, Lo, Hi, isNamedArg);
2879
954
    return;
2880
954
  }
2881
2882
289k
  if (Ty->hasPointerRepresentation()) {
2883
267k
    Current = Integer;
2884
267k
    return;
2885
267k
  }
2886
2887
21.8k
  if (Ty->isMemberPointerType()) {
2888
126
    if (Ty->isMemberFunctionPointerType()) {
2889
77
      if (Has64BitPointers) {
2890
        // If Has64BitPointers, this is an {i64, i64}, so classify both
2891
        // Lo and Hi now.
2892
69
        Lo = Hi = Integer;
2893
69
      } else {
2894
        // Otherwise, with 32-bit pointers, this is an {i32, i32}. If that
2895
        // straddles an eightbyte boundary, Hi should be classified as well.
2896
8
        uint64_t EB_FuncPtr = (OffsetBase) / 64;
2897
8
        uint64_t EB_ThisAdj = (OffsetBase + 64 - 1) / 64;
2898
8
        if (EB_FuncPtr != EB_ThisAdj) {
2899
2
          Lo = Hi = Integer;
2900
6
        } else {
2901
6
          Current = Integer;
2902
6
        }
2903
8
      }
2904
77
    } else {
2905
49
      Current = Integer;
2906
49
    }
2907
126
    return;
2908
126
  }
2909
2910
21.7k
  if (const VectorType *VT = Ty->getAs<VectorType>()) {
2911
6.53k
    uint64_t Size = getContext().getTypeSize(VT);
2912
6.53k
    if (Size == 1 || Size == 8 || 
Size == 166.52k
||
Size == 326.52k
) {
2913
      // gcc passes the following as integer:
2914
      // 4 bytes - <4 x char>, <2 x short>, <1 x int>, <1 x float>
2915
      // 2 bytes - <2 x char>, <1 x short>
2916
      // 1 byte  - <1 x char>
2917
5
      Current = Integer;
2918
2919
      // If this type crosses an eightbyte boundary, it should be
2920
      // split.
2921
5
      uint64_t EB_Lo = (OffsetBase) / 64;
2922
5
      uint64_t EB_Hi = (OffsetBase + Size - 1) / 64;
2923
5
      if (EB_Lo != EB_Hi)
2924
0
        Hi = Lo;
2925
6.52k
    } else if (Size == 64) {
2926
186
      QualType ElementType = VT->getElementType();
2927
2928
      // gcc passes <1 x double> in memory. :(
2929
186
      if (ElementType->isSpecificBuiltinType(BuiltinType::Double))
2930
0
        return;
2931
2932
      // gcc passes <1 x long long> as SSE but clang used to unconditionally
2933
      // pass them as integer.  For platforms where clang is the de facto
2934
      // platform compiler, we must continue to use integer.
2935
186
      if (!classifyIntegerMMXAsSSE() &&
2936
186
          
(125
ElementType->isSpecificBuiltinType(BuiltinType::LongLong)125
||
2937
125
           
ElementType->isSpecificBuiltinType(BuiltinType::ULongLong)45
||
2938
125
           
ElementType->isSpecificBuiltinType(BuiltinType::Long)45
||
2939
125
           
ElementType->isSpecificBuiltinType(BuiltinType::ULong)45
))
2940
80
        Current = Integer;
2941
106
      else
2942
106
        Current = SSE;
2943
2944
      // If this type crosses an eightbyte boundary, it should be
2945
      // split.
2946
186
      if (OffsetBase && 
OffsetBase != 640
)
2947
0
        Hi = Lo;
2948
6.33k
    } else if (Size == 128 ||
2949
6.33k
               
(3.78k
isNamedArg3.78k
&&
Size <= getNativeVectorSizeForAVXABI(AVXLevel)3.75k
)) {
2950
6.15k
      QualType ElementType = VT->getElementType();
2951
2952
      // gcc passes 256 and 512 bit <X x __int128> vectors in memory. :(
2953
6.15k
      if (passInt128VectorsInMem() && 
Size != 128523
&&
2954
6.15k
          
(256
ElementType->isSpecificBuiltinType(BuiltinType::Int128)256
||
2955
256
           ElementType->isSpecificBuiltinType(BuiltinType::UInt128)))
2956
6
        return;
2957
2958
      // Arguments of 256-bits are split into four eightbyte chunks. The
2959
      // least significant one belongs to class SSE and all the others to class
2960
      // SSEUP. The original Lo and Hi design considers that types can't be
2961
      // greater than 128-bits, so a 64-bit split in Hi and Lo makes sense.
2962
      // This design isn't correct for 256-bits, but since there're no cases
2963
      // where the upper parts would need to be inspected, avoid adding
2964
      // complexity and just consider Hi to match the 64-256 part.
2965
      //
2966
      // Note that per 3.5.7 of AMD64-ABI, 256-bit args are only passed in
2967
      // registers if they are "named", i.e. not part of the "..." of a
2968
      // variadic function.
2969
      //
2970
      // Similarly, per 3.2.3. of the AVX512 draft, 512-bits ("named") args are
2971
      // split into eight eightbyte chunks, one SSE and seven SSEUP.
2972
6.14k
      Lo = SSE;
2973
6.14k
      Hi = SSEUp;
2974
6.14k
    }
2975
6.52k
    return;
2976
6.53k
  }
2977
2978
15.2k
  if (const ComplexType *CT = Ty->getAs<ComplexType>()) {
2979
410
    QualType ET = getContext().getCanonicalType(CT->getElementType());
2980
2981
410
    uint64_t Size = getContext().getTypeSize(Ty);
2982
410
    if (ET->isIntegralOrEnumerationType()) {
2983
34
      if (Size <= 64)
2984
30
        Current = Integer;
2985
4
      else if (Size <= 128)
2986
4
        Lo = Hi = Integer;
2987
376
    } else if (ET->isFloat16Type() || 
ET == getContext().FloatTy364
) {
2988
154
      Current = SSE;
2989
222
    } else if (ET == getContext().DoubleTy) {
2990
113
      Lo = Hi = SSE;
2991
113
    } else 
if (109
ET == getContext().LongDoubleTy109
) {
2992
108
      const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();
2993
108
      if (LDF == &llvm::APFloat::IEEEquad())
2994
8
        Current = Memory;
2995
100
      else if (LDF == &llvm::APFloat::x87DoubleExtended())
2996
92
        Current = ComplexX87;
2997
8
      else if (LDF == &llvm::APFloat::IEEEdouble())
2998
8
        Lo = Hi = SSE;
2999
0
      else
3000
0
        llvm_unreachable("unexpected long double representation!");
3001
108
    }
3002
3003
    // If this complex type crosses an eightbyte boundary then it
3004
    // should be split.
3005
410
    uint64_t EB_Real = (OffsetBase) / 64;
3006
410
    uint64_t EB_Imag = (OffsetBase + getContext().getTypeSize(ET)) / 64;
3007
410
    if (Hi == NoClass && 
EB_Real != EB_Imag285
)
3008
104
      Hi = Lo;
3009
3010
410
    return;
3011
410
  }
3012
3013
14.8k
  if (const auto *EITy = Ty->getAs<BitIntType>()) {
3014
97
    if (EITy->getNumBits() <= 64)
3015
66
      Current = Integer;
3016
31
    else if (EITy->getNumBits() <= 128)
3017
21
      Lo = Hi = Integer;
3018
    // Larger values need to get passed in memory.
3019
97
    return;
3020
97
  }
3021
3022
14.7k
  if (const ConstantArrayType *AT = getContext().getAsConstantArrayType(Ty)) {
3023
    // Arrays are treated like structures.
3024
3025
54
    uint64_t Size = getContext().getTypeSize(Ty);
3026
3027
    // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger
3028
    // than eight eightbytes, ..., it has class MEMORY.
3029
54
    if (Size > 512)
3030
0
      return;
3031
3032
    // AMD64-ABI 3.2.3p2: Rule 1. If ..., or it contains unaligned
3033
    // fields, it has class MEMORY.
3034
    //
3035
    // Only need to check alignment of array base.
3036
54
    if (OffsetBase % getContext().getTypeAlign(AT->getElementType()))
3037
0
      return;
3038
3039
    // Otherwise implement simplified merge. We could be smarter about
3040
    // this, but it isn't worth it and would be harder to verify.
3041
54
    Current = NoClass;
3042
54
    uint64_t EltSize = getContext().getTypeSize(AT->getElementType());
3043
54
    uint64_t ArraySize = AT->getSize().getZExtValue();
3044
3045
    // The only case a 256-bit wide vector could be used is when the array
3046
    // contains a single 256-bit element. Since Lo and Hi logic isn't extended
3047
    // to work for sizes wider than 128, early check and fallback to memory.
3048
    //
3049
54
    if (Size > 128 &&
3050
54
        
(15
Size != EltSize15
||
Size > getNativeVectorSizeForAVXABI(AVXLevel)4
))
3051
11
      return;
3052
3053
197
    
for (uint64_t i=0, Offset=OffsetBase; 43
i<ArraySize;
++i, Offset += EltSize154
) {
3054
154
      Class FieldLo, FieldHi;
3055
154
      classify(AT->getElementType(), Offset, FieldLo, FieldHi, isNamedArg);
3056
154
      Lo = merge(Lo, FieldLo);
3057
154
      Hi = merge(Hi, FieldHi);
3058
154
      if (Lo == Memory || Hi == Memory)
3059
0
        break;
3060
154
    }
3061
3062
43
    postMerge(Size, Lo, Hi);
3063
43
    assert((Hi != SSEUp || Lo == SSE) && "Invalid SSEUp array classification.");
3064
0
    return;
3065
54
  }
3066
3067
14.6k
  if (const RecordType *RT = Ty->getAs<RecordType>()) {
3068
14.3k
    uint64_t Size = getContext().getTypeSize(Ty);
3069
3070
    // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger
3071
    // than eight eightbytes, ..., it has class MEMORY.
3072
14.3k
    if (Size > 512)
3073
141
      return;
3074
3075
    // AMD64-ABI 3.2.3p2: Rule 2. If a C++ object has either a non-trivial
3076
    // copy constructor or a non-trivial destructor, it is passed by invisible
3077
    // reference.
3078
14.2k
    if (getRecordArgABI(RT, getCXXABI()))
3079
464
      return;
3080
3081
13.7k
    const RecordDecl *RD = RT->getDecl();
3082
3083
    // Assume variable sized types are passed in memory.
3084
13.7k
    if (RD->hasFlexibleArrayMember())
3085
0
      return;
3086
3087
13.7k
    const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
3088
3089
    // Reset Lo class, this will be recomputed.
3090
13.7k
    Current = NoClass;
3091
3092
    // If this is a C++ record, classify the bases first.
3093
13.7k
    if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
3094
13.1k
      for (const auto &I : CXXRD->bases()) {
3095
1.27k
        assert(!I.isVirtual() && !I.getType()->isDependentType() &&
3096
1.27k
               "Unexpected base class!");
3097
0
        const auto *Base =
3098
1.27k
            cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
3099
3100
        // Classify this field.
3101
        //
3102
        // AMD64-ABI 3.2.3p2: Rule 3. If the size of the aggregate exceeds a
3103
        // single eightbyte, each is classified separately. Each eightbyte gets
3104
        // initialized to class NO_CLASS.
3105
1.27k
        Class FieldLo, FieldHi;
3106
1.27k
        uint64_t Offset =
3107
1.27k
          OffsetBase + getContext().toBits(Layout.getBaseClassOffset(Base));
3108
1.27k
        classify(I.getType(), Offset, FieldLo, FieldHi, isNamedArg);
3109
1.27k
        Lo = merge(Lo, FieldLo);
3110
1.27k
        Hi = merge(Hi, FieldHi);
3111
1.27k
        if (Lo == Memory || 
Hi == Memory1.27k
) {
3112
6
          postMerge(Size, Lo, Hi);
3113
6
          return;
3114
6
        }
3115
1.27k
      }
3116
13.1k
    }
3117
3118
    // Classify the fields one at a time, merging the results.
3119
13.7k
    unsigned idx = 0;
3120
13.7k
    bool UseClang11Compat = getContext().getLangOpts().getClangABICompat() <=
3121
13.7k
                                LangOptions::ClangABI::Ver11 ||
3122
13.7k
                            
getContext().getTargetInfo().getTriple().isPS4()13.6k
;
3123
13.7k
    bool IsUnion = RT->isUnionType() && 
!UseClang11Compat79
;
3124
3125
13.7k
    for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
3126
22.5k
           i != e; 
++i, ++idx8.79k
) {
3127
9.12k
      uint64_t Offset = OffsetBase + Layout.getFieldOffset(idx);
3128
9.12k
      bool BitField = i->isBitField();
3129
3130
      // Ignore padding bit-fields.
3131
9.12k
      if (BitField && 
i->isUnnamedBitfield()42
)
3132
12
        continue;
3133
3134
      // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger than
3135
      // eight eightbytes, or it contains unaligned fields, it has class MEMORY.
3136
      //
3137
      // The only case a 256-bit or a 512-bit wide vector could be used is when
3138
      // the struct contains a single 256-bit or 512-bit element. Early check
3139
      // and fallback to memory.
3140
      //
3141
      // FIXME: Extended the Lo and Hi logic properly to work for size wider
3142
      // than 128.
3143
9.11k
      if (Size > 128 &&
3144
9.11k
          
(332
(332
!IsUnion332
&&
Size != getContext().getTypeSize(i->getType())321
) ||
3145
332
           
Size > getNativeVectorSizeForAVXABI(AVXLevel)87
)) {
3146
299
        Lo = Memory;
3147
299
        postMerge(Size, Lo, Hi);
3148
299
        return;
3149
299
      }
3150
      // Note, skip this test for bit-fields, see below.
3151
8.81k
      if (!BitField && 
Offset % getContext().getTypeAlign(i->getType())8.78k
) {
3152
20
        Lo = Memory;
3153
20
        postMerge(Size, Lo, Hi);
3154
20
        return;
3155
20
      }
3156
3157
      // Classify this field.
3158
      //
3159
      // AMD64-ABI 3.2.3p2: Rule 3. If the size of the aggregate
3160
      // exceeds a single eightbyte, each is classified
3161
      // separately. Each eightbyte gets initialized to class
3162
      // NO_CLASS.
3163
8.79k
      Class FieldLo, FieldHi;
3164
3165
      // Bit-fields require special handling, they do not force the
3166
      // structure to be passed in memory even if unaligned, and
3167
      // therefore they can straddle an eightbyte.
3168
8.79k
      if (BitField) {
3169
30
        assert(!i->isUnnamedBitfield());
3170
0
        uint64_t Offset = OffsetBase + Layout.getFieldOffset(idx);
3171
30
        uint64_t Size = i->getBitWidthValue(getContext());
3172
3173
30
        uint64_t EB_Lo = Offset / 64;
3174
30
        uint64_t EB_Hi = (Offset + Size - 1) / 64;
3175
3176
30
        if (EB_Lo) {
3177
0
          assert(EB_Hi == EB_Lo && "Invalid classification, type > 16 bytes.");
3178
0
          FieldLo = NoClass;
3179
0
          FieldHi = Integer;
3180
30
        } else {
3181
30
          FieldLo = Integer;
3182
30
          FieldHi = EB_Hi ? 
Integer0
: NoClass;
3183
30
        }
3184
30
      } else
3185
8.76k
        classify(i->getType(), Offset, FieldLo, FieldHi, isNamedArg);
3186
0
      Lo = merge(Lo, FieldLo);
3187
8.79k
      Hi = merge(Hi, FieldHi);
3188
8.79k
      if (Lo == Memory || 
Hi == Memory8.78k
)
3189
14
        break;
3190
8.79k
    }
3191
3192
13.4k
    postMerge(Size, Lo, Hi);
3193
13.4k
  }
3194
14.6k
}
3195
3196
383
ABIArgInfo X86_64ABIInfo::getIndirectReturnResult(QualType Ty) const {
3197
  // If this is a scalar LLVM value then assume LLVM will pass it in the right
3198
  // place naturally.
3199
383
  if (!isAggregateTypeForABI(Ty)) {
3200
    // Treat an enum type as its underlying type.
3201
156
    if (const EnumType *EnumTy = Ty->getAs<EnumType>())
3202
0
      Ty = EnumTy->getDecl()->getIntegerType();
3203
3204
156
    if (Ty->isBitIntType())
3205
3
      return getNaturalAlignIndirect(Ty);
3206
3207
153
    return (isPromotableIntegerTypeForABI(Ty) ? 
ABIArgInfo::getExtend(Ty)0
3208
153
                                              : ABIArgInfo::getDirect());
3209
156
  }
3210
3211
227
  return getNaturalAlignIndirect(Ty);
3212
383
}
3213
3214
2.99k
bool X86_64ABIInfo::IsIllegalVectorType(QualType Ty) const {
3215
2.99k
  if (const VectorType *VecTy = Ty->getAs<VectorType>()) {
3216
140
    uint64_t Size = getContext().getTypeSize(VecTy);
3217
140
    unsigned LargestVector = getNativeVectorSizeForAVXABI(AVXLevel);
3218
140
    if (Size <= 64 || 
Size > LargestVector134
)
3219
117
      return true;
3220
23
    QualType EltTy = VecTy->getElementType();
3221
23
    if (passInt128VectorsInMem() &&
3222
23
        
(9
EltTy->isSpecificBuiltinType(BuiltinType::Int128)9
||
3223
9
         EltTy->isSpecificBuiltinType(BuiltinType::UInt128)))
3224
6
      return true;
3225
23
  }
3226
3227
2.87k
  return false;
3228
2.99k
}
3229
3230
ABIArgInfo X86_64ABIInfo::getIndirectResult(QualType Ty,
3231
3.81k
                                            unsigned freeIntRegs) const {
3232
  // If this is a scalar LLVM value then assume LLVM will pass it in the right
3233
  // place naturally.
3234
  //
3235
  // This assumption is optimistic, as there could be free registers available
3236
  // when we need to pass this argument in memory, and LLVM could try to pass
3237
  // the argument in the free register. This does not seem to happen currently,
3238
  // but this code would be much safer if we could mark the argument with
3239
  // 'onstack'. See PR12193.
3240
3.81k
  if (!isAggregateTypeForABI(Ty) && 
!IsIllegalVectorType(Ty)2.99k
&&
3241
3.81k
      
!Ty->isBitIntType()2.87k
) {
3242
    // Treat an enum type as its underlying type.
3243
2.86k
    if (const EnumType *EnumTy = Ty->getAs<EnumType>())
3244
0
      Ty = EnumTy->getDecl()->getIntegerType();
3245
3246
2.86k
    return (isPromotableIntegerTypeForABI(Ty) ? 
ABIArgInfo::getExtend(Ty)286
3247
2.86k
                                              : 
ABIArgInfo::getDirect()2.58k
);
3248
2.86k
  }
3249
3250
947
  if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))
3251
477
    return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
3252
3253
  // Compute the byval alignment. We specify the alignment of the byval in all
3254
  // cases so that the mid-level optimizer knows the alignment of the byval.
3255
470
  unsigned Align = std::max(getContext().getTypeAlign(Ty) / 8, 8U);
3256
3257
  // Attempt to avoid passing indirect results using byval when possible. This
3258
  // is important for good codegen.
3259
  //
3260
  // We do this by coercing the value into a scalar type which the backend can
3261
  // handle naturally (i.e., without using byval).
3262
  //
3263
  // For simplicity, we currently only do this when we have exhausted all of the
3264
  // free integer registers. Doing this when there are free integer registers
3265
  // would require more care, as we would have to ensure that the coerced value
3266
  // did not claim the unused register. That would require either reording the
3267
  // arguments to the function (so that any subsequent inreg values came first),
3268
  // or only doing this optimization when there were no following arguments that
3269
  // might be inreg.
3270
  //
3271
  // We currently expect it to be rare (particularly in well written code) for
3272
  // arguments to be passed on the stack when there are still free integer
3273
  // registers available (this would typically imply large structs being passed
3274
  // by value), so this seems like a fair tradeoff for now.
3275
  //
3276
  // We can revisit this if the backend grows support for 'onstack' parameter
3277
  // attributes. See PR12193.
3278
470
  if (freeIntRegs == 0) {
3279
32
    uint64_t Size = getContext().getTypeSize(Ty);
3280
3281
    // If this type fits in an eightbyte, coerce it into the matching integral
3282
    // type, which will end up on the stack (with alignment 8).
3283
32
    if (Align == 8 && 
Size <= 6422
)
3284
4
      return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(),
3285
4
                                                          Size));
3286
32
  }
3287
3288
466
  return ABIArgInfo::getIndirect(CharUnits::fromQuantity(Align));
3289
470
}
3290
3291
/// The ABI specifies that a value should be passed in a full vector XMM/YMM
3292
/// register. Pick an LLVM IR type that will be passed as a vector register.
3293
6.18k
llvm::Type *X86_64ABIInfo::GetByteVectorType(QualType Ty) const {
3294
  // Wrapper structs/arrays that only contain vectors are passed just like
3295
  // vectors; strip them off if present.
3296
6.18k
  if (const Type *InnerTy = isSingleElementStruct(Ty, getContext()))
3297
25
    Ty = QualType(InnerTy, 0);
3298
3299
6.18k
  llvm::Type *IRType = CGT.ConvertType(Ty);
3300
6.18k
  if (isa<llvm::VectorType>(IRType)) {
3301
    // Don't pass vXi128 vectors in their native type, the backend can't
3302
    // legalize them.
3303
6.14k
    if (passInt128VectorsInMem() &&
3304
6.14k
        
cast<llvm::VectorType>(IRType)->getElementType()->isIntegerTy(128)514
) {
3305
      // Use a vXi64 vector.
3306
6
      uint64_t Size = getContext().getTypeSize(Ty);
3307
6
      return llvm::FixedVectorType::get(llvm::Type::getInt64Ty(getVMContext()),
3308
6
                                        Size / 64);
3309
6
    }
3310
3311
6.13k
    return IRType;
3312
6.14k
  }
3313
3314
43
  if (IRType->getTypeID() == llvm::Type::FP128TyID)
3315
40
    return IRType;
3316
3317
  // We couldn't find the preferred IR vector type for 'Ty'.
3318
3
  uint64_t Size = getContext().getTypeSize(Ty);
3319
3
  assert((Size == 128 || Size == 256 || Size == 512) && "Invalid type found!");
3320
3321
3322
  // Return a LLVM IR vector type based on the size of 'Ty'.
3323
0
  return llvm::FixedVectorType::get(llvm::Type::getDoubleTy(getVMContext()),
3324
3
                                    Size / 64);
3325
43
}
3326
3327
/// BitsContainNoUserData - Return true if the specified [start,end) bit range
3328
/// is known to either be off the end of the specified type or being in
3329
/// alignment padding.  The user type specified is known to be at most 128 bits
3330
/// in size, and have passed through X86_64ABIInfo::classify with a successful
3331
/// classification that put one of the two halves in the INTEGER class.
3332
///
3333
/// It is conservatively correct to return false.
3334
static bool BitsContainNoUserData(QualType Ty, unsigned StartBit,
3335
84.6k
                                  unsigned EndBit, ASTContext &Context) {
3336
  // If the bytes being queried are off the end of the type, there is no user
3337
  // data hiding here.  This handles analysis of builtins, vectors and other
3338
  // types that don't contain interesting padding.
3339
84.6k
  unsigned TySize = (unsigned)Context.getTypeSize(Ty);
3340
84.6k
  if (TySize <= StartBit)
3341
83.3k
    return true;
3342
3343
1.29k
  if (const ConstantArrayType *AT = Context.getAsConstantArrayType(Ty)) {
3344
20
    unsigned EltSize = (unsigned)Context.getTypeSize(AT->getElementType());
3345
20
    unsigned NumElts = (unsigned)AT->getSize().getZExtValue();
3346
3347
    // Check each element to see if the element overlaps with the queried range.
3348
72
    for (unsigned i = 0; i != NumElts; 
++i52
) {
3349
      // If the element is after the span we care about, then we're done..
3350
69
      unsigned EltOffset = i*EltSize;
3351
69
      if (EltOffset >= EndBit) 
break0
;
3352
3353
69
      unsigned EltStart = EltOffset < StartBit ? 
StartBit-EltOffset52
:
017
;
3354
69
      if (!BitsContainNoUserData(AT->getElementType(), EltStart,
3355
69
                                 EndBit-EltOffset, Context))
3356
17
        return false;
3357
69
    }
3358
    // If it overlaps no elements, then it is safe to process as padding.
3359
3
    return true;
3360
20
  }
3361
3362
1.27k
  if (const RecordType *RT = Ty->getAs<RecordType>()) {
3363
822
    const RecordDecl *RD = RT->getDecl();
3364
822
    const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
3365
3366
    // If this is a C++ record, check the bases first.
3367
822
    if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
3368
669
      for (const auto &I : CXXRD->bases()) {
3369
27
        assert(!I.isVirtual() && !I.getType()->isDependentType() &&
3370
27
               "Unexpected base class!");
3371
0
        const auto *Base =
3372
27
            cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
3373
3374
        // If the base is after the span we care about, ignore it.
3375
27
        unsigned BaseOffset = Context.toBits(Layout.getBaseClassOffset(Base));
3376
27
        if (BaseOffset >= EndBit) 
continue0
;
3377
3378
27
        unsigned BaseStart = BaseOffset < StartBit ? 
StartBit-BaseOffset24
:
03
;
3379
27
        if (!BitsContainNoUserData(I.getType(), BaseStart,
3380
27
                                   EndBit-BaseOffset, Context))
3381
23
          return false;
3382
27
      }
3383
669
    }
3384
3385
    // Verify that no field has data that overlaps the region of interest.  Yes
3386
    // this could be sped up a lot by being smarter about queried fields,
3387
    // however we're only looking at structs up to 16 bytes, so we don't care
3388
    // much.
3389
799
    unsigned idx = 0;
3390
799
    for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
3391
1.80k
         i != e; 
++i, ++idx1.00k
) {
3392
1.60k
      unsigned FieldOffset = (unsigned)Layout.getFieldOffset(idx);
3393
3394
      // If we found a field after the region we care about, then we're done.
3395
1.60k
      if (FieldOffset >= EndBit) 
break121
;
3396
3397
1.48k
      unsigned FieldStart = FieldOffset < StartBit ? 
StartBit-FieldOffset1.04k
:
0433
;
3398
1.48k
      if (!BitsContainNoUserData(i->getType(), FieldStart, EndBit-FieldOffset,
3399
1.48k
                                 Context))
3400
473
        return false;
3401
1.48k
    }
3402
3403
    // If nothing in this record overlapped the area of interest, then we're
3404
    // clean.
3405
326
    return true;
3406
799
  }
3407
3408
449
  return false;
3409
1.27k
}
3410
3411
/// getFPTypeAtOffset - Return a floating point type at the specified offset.
3412
static llvm::Type *getFPTypeAtOffset(llvm::Type *IRType, unsigned IROffset,
3413
12.2k
                                     const llvm::DataLayout &TD) {
3414
12.2k
  if (IROffset == 0 && 
IRType->isFloatingPointTy()11.5k
)
3415
4.63k
    return IRType;
3416
3417
  // If this is a struct, recurse into the field at the specified offset.
3418
7.64k
  if (llvm::StructType *STy = dyn_cast<llvm::StructType>(IRType)) {
3419
1.33k
    if (!STy->getNumContainedTypes())
3420
4
      return nullptr;
3421
3422
1.33k
    const llvm::StructLayout *SL = TD.getStructLayout(STy);
3423
1.33k
    unsigned Elt = SL->getElementContainingOffset(IROffset);
3424
1.33k
    IROffset -= SL->getElementOffset(Elt);
3425
1.33k
    return getFPTypeAtOffset(STy->getElementType(Elt), IROffset, TD);
3426
1.33k
  }
3427
3428
  // If this is an array, recurse into the field at the specified offset.
3429
6.30k
  if (llvm::ArrayType *ATy = dyn_cast<llvm::ArrayType>(IRType)) {
3430
20
    llvm::Type *EltTy = ATy->getElementType();
3431
20
    unsigned EltSize = TD.getTypeAllocSize(EltTy);
3432
20
    IROffset -= IROffset / EltSize * EltSize;
3433
20
    return getFPTypeAtOffset(EltTy, IROffset, TD);
3434
20
  }
3435
3436
6.28k
  return nullptr;
3437
6.30k
}
3438
3439
/// GetSSETypeAtOffset - Return a type that will be passed by the backend in the
3440
/// low 8 bytes of an XMM register, corresponding to the SSE class.
3441
llvm::Type *X86_64ABIInfo::
3442
GetSSETypeAtOffset(llvm::Type *IRType, unsigned IROffset,
3443
10.5k
                   QualType SourceTy, unsigned SourceOffset) const {
3444
10.5k
  const llvm::DataLayout &TD = getDataLayout();
3445
10.5k
  unsigned SourceSize =
3446
10.5k
      (unsigned)getContext().getTypeSize(SourceTy) / 8 - SourceOffset;
3447
10.5k
  llvm::Type *T0 = getFPTypeAtOffset(IRType, IROffset, TD);
3448
10.5k
  if (!T0 || 
T0->isDoubleTy()4.28k
)
3449
8.19k
    return llvm::Type::getDoubleTy(getVMContext());
3450
3451
  // Get the adjacent FP type.
3452
2.34k
  llvm::Type *T1 = nullptr;
3453
2.34k
  unsigned T0Size = TD.getTypeAllocSize(T0);
3454
2.34k
  if (SourceSize > T0Size)
3455
369
      T1 = getFPTypeAtOffset(IRType, IROffset + T0Size, TD);
3456
2.34k
  if (T1 == nullptr) {
3457
    // Check if IRType is a half + float. float type will be in IROffset+4 due
3458
    // to its alignment.
3459
2.00k
    if (T0->isHalfTy() && 
SourceSize > 4132
)
3460
6
      T1 = getFPTypeAtOffset(IRType, IROffset + 4, TD);
3461
    // If we can't get a second FP type, return a simple half or float.
3462
    // avx512fp16-abi.c:pr51813_2 shows it works to return float for
3463
    // {float, i8} too.
3464
2.00k
    if (T1 == nullptr)
3465
2.00k
      return T0;
3466
2.00k
  }
3467
3468
341
  if (T0->isFloatTy() && 
T1->isFloatTy()314
)
3469
308
    return llvm::FixedVectorType::get(T0, 2);
3470
3471
33
  if (T0->isHalfTy() && 
T1->isHalfTy()27
) {
3472
23
    llvm::Type *T2 = nullptr;
3473
23
    if (SourceSize > 4)
3474
8
      T2 = getFPTypeAtOffset(IRType, IROffset + 4, TD);
3475
23
    if (T2 == nullptr)
3476
15
      return llvm::FixedVectorType::get(T0, 2);
3477
8
    return llvm::FixedVectorType::get(T0, 4);
3478
23
  }
3479
3480
10
  if (T0->isHalfTy() || 
T1->isHalfTy()6
)
3481
10
    return llvm::FixedVectorType::get(llvm::Type::getHalfTy(getVMContext()), 4);
3482
3483
0
  return llvm::Type::getDoubleTy(getVMContext());
3484
10
}
3485
3486
3487
/// GetINTEGERTypeAtOffset - The ABI specifies that a value should be passed in
3488
/// an 8-byte GPR.  This means that we either have a scalar or we are talking
3489
/// about the high or low part of an up-to-16-byte struct.  This routine picks
3490
/// the best LLVM IR type to represent this, which may be i64 or may be anything
3491
/// else that the backend will pass in a GPR that works better (e.g. i8, %foo*,
3492
/// etc).
3493
///
3494
/// PrefType is an LLVM IR type that corresponds to (part of) the IR type for
3495
/// the source type.  IROffset is an offset in bytes into the LLVM IR type that
3496
/// the 8-byte value references.  PrefType may be null.
3497
///
3498
/// SourceTy is the source-level type for the entire argument.  SourceOffset is
3499
/// an offset into this that we're processing (which is always either 0 or 8).
3500
///
3501
llvm::Type *X86_64ABIInfo::
3502
GetINTEGERTypeAtOffset(llvm::Type *IRType, unsigned IROffset,
3503
385k
                       QualType SourceTy, unsigned SourceOffset) const {
3504
  // If we're dealing with an un-offset LLVM IR type, then it means that we're
3505
  // returning an 8-byte unit starting with it.  See if we can safely use it.
3506
385k
  if (IROffset == 0) {
3507
    // Pointers and int64's always fill the 8-byte unit.
3508
384k
    if ((isa<llvm::PointerType>(IRType) && 
Has64BitPointers267k
) ||
3509
384k
        
IRType->isIntegerTy(64)117k
)
3510
291k
      return IRType;
3511
3512
    // If we have a 1/2/4-byte integer, we can use it only if the rest of the
3513
    // goodness in the source type is just tail padding.  This is allowed to
3514
    // kick in for struct {double,int} on the int, but not on
3515
    // struct{double,int,int} because we wouldn't return the second int.  We
3516
    // have to do this analysis on the source type because we can't depend on
3517
    // unions being lowered a specific way etc.
3518
93.2k
    if (IRType->isIntegerTy(8) || 
IRType->isIntegerTy(16)89.5k
||
3519
93.2k
        
IRType->isIntegerTy(32)88.7k
||
3520
93.2k
        
(10.2k
isa<llvm::PointerType>(IRType)10.2k
&&
!Has64BitPointers92
)) {
3521
83.1k
      unsigned BitWidth = isa<llvm::PointerType>(IRType) ? 
3292
:
3522
83.1k
          
cast<llvm::IntegerType>(IRType)->getBitWidth()83.0k
;
3523
3524
83.1k
      if (BitsContainNoUserData(SourceTy, SourceOffset*8+BitWidth,
3525
83.1k
                                SourceOffset*8+64, getContext()))
3526
82.6k
        return IRType;
3527
83.1k
    }
3528
93.2k
  }
3529
3530
12.0k
  if (llvm::StructType *STy = dyn_cast<llvm::StructType>(IRType)) {
3531
    // If this is a struct, recurse into the field at the specified offset.
3532
7.65k
    const llvm::StructLayout *SL = getDataLayout().getStructLayout(STy);
3533
7.65k
    if (IROffset < SL->getSizeInBytes()) {
3534
7.65k
      unsigned FieldIdx = SL->getElementContainingOffset(IROffset);
3535
7.65k
      IROffset -= SL->getElementOffset(FieldIdx);
3536
3537
7.65k
      return GetINTEGERTypeAtOffset(STy->getElementType(FieldIdx), IROffset,
3538
7.65k
                                    SourceTy, SourceOffset);
3539
7.65k
    }
3540
7.65k
  }
3541
3542
4.41k
  if (llvm::ArrayType *ATy = dyn_cast<llvm::ArrayType>(IRType)) {
3543
50
    llvm::Type *EltTy = ATy->getElementType();
3544
50
    unsigned EltSize = getDataLayout().getTypeAllocSize(EltTy);
3545
50
    unsigned EltOffset = IROffset/EltSize*EltSize;
3546
50
    return GetINTEGERTypeAtOffset(EltTy, IROffset-EltOffset, SourceTy,
3547
50
                                  SourceOffset);
3548
50
  }
3549
3550
  // Okay, we don't have any better idea of what to pass, so we pass this in an
3551
  // integer register that isn't too big to fit the rest of the struct.
3552
4.36k
  unsigned TySizeInBytes =
3553
4.36k
    (unsigned)getContext().getTypeSizeInChars(SourceTy).getQuantity();
3554
3555
4.36k
  assert(TySizeInBytes != SourceOffset && "Empty field?");
3556
3557
  // It is always safe to classify this as an integer type up to i64 that
3558
  // isn't larger than the structure.
3559
0
  return llvm::IntegerType::get(getVMContext(),
3560
4.36k
                                std::min(TySizeInBytes-SourceOffset, 8U)*8);
3561
4.41k
}
3562
3563
3564
/// GetX86_64ByValArgumentPair - Given a high and low type that can ideally
3565
/// be used as elements of a two register pair to pass or return, return a
3566
/// first class aggregate to represent them.  For example, if the low part of
3567
/// a by-value argument should be passed as i32* and the high part as float,
3568
/// return {i32*, float}.
3569
static llvm::Type *
3570
GetX86_64ByValArgumentPair(llvm::Type *Lo, llvm::Type *Hi,
3571
1.69k
                           const llvm::DataLayout &TD) {
3572
  // In order to correctly satisfy the ABI, we need to the high part to start
3573
  // at offset 8.  If the high and low parts we inferred are both 4-byte types
3574
  // (e.g. i32 and i32) then the resultant struct type ({i32,i32}) won't have
3575
  // the second element at offset 8.  Check for this:
3576
1.69k
  unsigned LoSize = (unsigned)TD.getTypeAllocSize(Lo);
3577
1.69k
  unsigned HiAlign = TD.getABITypeAlignment(Hi);
3578
1.69k
  unsigned HiStart = llvm::alignTo(LoSize, HiAlign);
3579
1.69k
  assert(HiStart != 0 && HiStart <= 8 && "Invalid x86-64 argument pair!");
3580
3581
  // To handle this, we have to increase the size of the low part so that the
3582
  // second element will start at an 8 byte offset.  We can't increase the size
3583
  // of the second element because it might make us access off the end of the
3584
  // struct.
3585
1.69k
  if (HiStart != 8) {
3586
    // There are usually two sorts of types the ABI generation code can produce
3587
    // for the low part of a pair that aren't 8 bytes in size: half, float or
3588
    // i8/i16/i32.  This can also include pointers when they are 32-bit (X32 and
3589
    // NaCl).
3590
    // Promote these to a larger type.
3591
6
    if (Lo->isHalfTy() || Lo->isFloatTy())
3592
1
      Lo = llvm::Type::getDoubleTy(Lo->getContext());
3593
5
    else {
3594
5
      assert((Lo->isIntegerTy() || Lo->isPointerTy())
3595
5
             && "Invalid/unknown lo type");
3596
0
      Lo = llvm::Type::getInt64Ty(Lo->getContext());
3597
5
    }
3598
6
  }
3599
3600
0
  llvm::StructType *Result = llvm::StructType::get(Lo, Hi);
3601
3602
  // Verify that the second element is at an 8-byte offset.
3603
1.69k
  assert(TD.getStructLayout(Result)->getElementOffset(1) == 8 &&
3604
1.69k
         "Invalid x86-64 argument pair!");
3605
0
  return Result;
3606
1.69k
}
3607
3608
ABIArgInfo X86_64ABIInfo::
3609
164k
classifyReturnType(QualType RetTy) const {
3610
  // AMD64-ABI 3.2.3p4: Rule 1. Classify the return type with the
3611
  // classification algorithm.
3612
164k
  X86_64ABIInfo::Class Lo, Hi;
3613
164k
  classify(RetTy, 0, Lo, Hi, /*isNamedArg*/ true);
3614
3615
  // Check some invariants.
3616
164k
  assert((Hi != Memory || Lo == Memory) && "Invalid memory classification.");
3617
0
  assert((Hi != SSEUp || Lo == SSE) && "Invalid SSEUp classification.");
3618
3619
0
  llvm::Type *ResType = nullptr;
3620
164k
  switch (Lo) {
3621
65.8k
  case NoClass:
3622
65.8k
    if (Hi == NoClass)
3623
65.8k
      return ABIArgInfo::getIgnore();
3624
    // If the low part is just padding, it takes no register, leave ResType
3625
    // null.
3626
1
    assert((Hi == SSE || Hi == Integer || Hi == X87Up) &&
3627
1
           "Unknown missing lo part");
3628
0
    break;
3629
3630
0
  case SSEUp:
3631
0
  case X87Up:
3632
0
    llvm_unreachable("Invalid classification for lo word.");
3633
3634
    // AMD64-ABI 3.2.3p4: Rule 2. Types of class memory are returned via
3635
    // hidden argument.
3636
382
  case Memory:
3637
382
    return getIndirectReturnResult(RetTy);
3638
3639
    // AMD64-ABI 3.2.3p4: Rule 3. If the class is INTEGER, the next
3640
    // available register of the sequence %rax, %rdx is used.
3641
94.4k
  case Integer:
3642
94.4k
    ResType = GetINTEGERTypeAtOffset(CGT.ConvertType(RetTy), 0, RetTy, 0);
3643
3644
    // If we have a sign or zero extended integer, make sure to return Extend
3645
    // so that the parameter gets the right LLVM IR attributes.
3646
94.4k
    if (Hi == NoClass && 
isa<llvm::IntegerType>(ResType)93.7k
) {
3647
      // Treat an enum type as its underlying type.
3648
52.4k
      if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
3649
101
        RetTy = EnumTy->getDecl()->getIntegerType();
3650
3651
52.4k
      if (RetTy->isIntegralOrEnumerationType() &&
3652
52.4k
          
isPromotableIntegerTypeForABI(RetTy)51.5k
)
3653
4.51k
        return ABIArgInfo::getExtend(RetTy);
3654
52.4k
    }
3655
89.9k
    break;
3656
3657
    // AMD64-ABI 3.2.3p4: Rule 4. If the class is SSE, the next
3658
    // available SSE register of the sequence %xmm0, %xmm1 is used.
3659
89.9k
  case SSE:
3660
3.44k
    ResType = GetSSETypeAtOffset(CGT.ConvertType(RetTy), 0, RetTy, 0);
3661
3.44k
    break;
3662
3663
    // AMD64-ABI 3.2.3p4: Rule 6. If the class is X87, the value is
3664
    // returned on the X87 stack in %st0 as 80-bit x87 number.
3665
678
  case X87:
3666
678
    ResType = llvm::Type::getX86_FP80Ty(getVMContext());
3667
678
    break;
3668
3669
    // AMD64-ABI 3.2.3p4: Rule 8. If the class is COMPLEX_X87, the real
3670
    // part of the value is returned in %st0 and the imaginary part in
3671
    // %st1.
3672
38
  case ComplexX87:
3673
38
    assert(Hi == ComplexX87 && "Unexpected ComplexX87 classification.");
3674
0
    ResType = llvm::StructType::get(llvm::Type::getX86_FP80Ty(getVMContext()),
3675
38
                                    llvm::Type::getX86_FP80Ty(getVMContext()));
3676
38
    break;
3677
164k
  }
3678
3679
94.0k
  llvm::Type *HighPart = nullptr;
3680
94.0k
  switch (Hi) {
3681
    // Memory was handled previously and X87 should
3682
    // never occur as a hi class.
3683
0
  case Memory:
3684
0
  case X87:
3685
0
    llvm_unreachable("Invalid classification for hi word.");
3686
3687
38
  case ComplexX87: // Previously handled.
3688
90.2k
  case NoClass:
3689
90.2k
    break;
3690
3691
693
  case Integer:
3692
693
    HighPart = GetINTEGERTypeAtOffset(CGT.ConvertType(RetTy), 8, RetTy, 8);
3693
693
    if (Lo == NoClass)  // Return HighPart at offset 8 in memory.
3694
0
      return ABIArgInfo::getDirect(HighPart, 8);
3695
693
    break;
3696
693
  case SSE:
3697
147
    HighPart = GetSSETypeAtOffset(CGT.ConvertType(RetTy), 8, RetTy, 8);
3698
147
    if (Lo == NoClass)  // Return HighPart at offset 8 in memory.
3699
1
      return ABIArgInfo::getDirect(HighPart, 8);
3700
146
    break;
3701
3702
    // AMD64-ABI 3.2.3p4: Rule 5. If the class is SSEUP, the eightbyte
3703
    // is passed in the next available eightbyte chunk if the last used
3704
    // vector register.
3705
    //
3706
    // SSEUP should always be preceded by SSE, just widen.
3707
2.30k
  case SSEUp:
3708
2.30k
    assert(Lo == SSE && "Unexpected SSEUp classification.");
3709
0
    ResType = GetByteVectorType(RetTy);
3710
2.30k
    break;
3711
3712
    // AMD64-ABI 3.2.3p4: Rule 7. If the class is X87UP, the value is
3713
    // returned together with the previous X87 value in %st0.
3714
678
  case X87Up:
3715
    // If X87Up is preceded by X87, we don't need to do
3716
    // anything. However, in some cases with unions it may not be
3717
    // preceded by X87. In such situations we follow gcc and pass the
3718
    // extra bits in an SSE reg.
3719
678
    if (Lo != X87) {
3720
0
      HighPart = GetSSETypeAtOffset(CGT.ConvertType(RetTy), 8, RetTy, 8);
3721
0
      if (Lo == NoClass)  // Return HighPart at offset 8 in memory.
3722
0
        return ABIArgInfo::getDirect(HighPart, 8);
3723
0
    }
3724
678
    break;
3725
94.0k
  }
3726
3727
  // If a high part was specified, merge it together with the low part.  It is
3728
  // known to pass in the high eightbyte of the result.  We do this by forming a
3729
  // first class struct aggregate with the high and low part: {low, high}
3730
94.0k
  if (HighPart)
3731
839
    ResType = GetX86_64ByValArgumentPair(ResType, HighPart, getDataLayout());
3732
3733
94.0k
  return ABIArgInfo::getDirect(ResType);
3734
94.0k
}
3735
3736
ABIArgInfo X86_64ABIInfo::classifyArgumentType(
3737
  QualType Ty, unsigned freeIntRegs, unsigned &neededInt, unsigned &neededSSE,
3738
  bool isNamedArg)
3739
  const
3740
296k
{
3741
296k
  Ty = useFirstFieldIfTransparentUnion(Ty);
3742
3743
296k
  X86_64ABIInfo::Class Lo, Hi;
3744
296k
  classify(Ty, 0, Lo, Hi, isNamedArg);
3745
3746
  // Check some invariants.
3747
  // FIXME: Enforce these by construction.
3748
296k
  assert((Hi != Memory || Lo == Memory) && "Invalid memory classification.");
3749
0
  assert((Hi != SSEUp || Lo == SSE) && "Invalid SSEUp classification.");
3750
3751
0
  neededInt = 0;
3752
296k
  neededSSE = 0;
3753
296k
  llvm::Type *ResType = nullptr;
3754
296k
  switch (Lo) {
3755
5.30k
  case NoClass:
3756
5.30k
    if (Hi == NoClass)
3757
5.30k
      return ABIArgInfo::getIgnore();
3758
    // If the low part is just padding, it takes no register, leave ResType
3759
    // null.
3760
2
    assert((Hi == SSE || Hi == Integer || Hi == X87Up) &&
3761
2
           "Unknown missing lo part");
3762
0
    break;
3763
3764
    // AMD64-ABI 3.2.3p3: Rule 1. If the class is MEMORY, pass the argument
3765
    // on the stack.
3766
1.32k
  case Memory:
3767
3768
    // AMD64-ABI 3.2.3p3: Rule 5. If the class is X87, X87UP or
3769
    // COMPLEX_X87, it is passed in memory.
3770
1.56k
  case X87:
3771
1.62k
  case ComplexX87:
3772
1.62k
    if (getRecordArgABI(Ty, getCXXABI()) == CGCXXABI::RAA_Indirect)
3773
477
      ++neededInt;
3774
1.62k
    return getIndirectResult(Ty, freeIntRegs);
3775
3776
0
  case SSEUp:
3777
0
  case X87Up:
3778
0
    llvm_unreachable("Invalid classification for lo word.");
3779
3780
    // AMD64-ABI 3.2.3p3: Rule 2. If the class is INTEGER, the next
3781
    // available register of the sequence %rdi, %rsi, %rdx, %rcx, %r8
3782
    // and %r9 is used.
3783
282k
  case Integer:
3784
282k
    ++neededInt;
3785
3786
    // Pick an 8-byte type based on the preferred type.
3787
282k
    ResType = GetINTEGERTypeAtOffset(CGT.ConvertType(Ty), 0, Ty, 0);
3788
3789
    // If we have a sign or zero extended integer, make sure to return Extend
3790
    // so that the parameter gets the right LLVM IR attributes.
3791
282k
    if (Hi == NoClass && 
isa<llvm::IntegerType>(ResType)281k
) {
3792
      // Treat an enum type as its underlying type.
3793
56.7k
      if (const EnumType *EnumTy = Ty->getAs<EnumType>())
3794
851
        Ty = EnumTy->getDecl()->getIntegerType();
3795
3796
56.7k
      if (Ty->isIntegralOrEnumerationType() &&
3797
56.7k
          
isPromotableIntegerTypeForABI(Ty)56.3k
)
3798
3.21k
        return ABIArgInfo::getExtend(Ty);
3799
56.7k
    }
3800
3801
279k
    break;
3802
3803
    // AMD64-ABI 3.2.3p3: Rule 3. If the class is SSE, the next
3804
    // available SSE register is used, the registers are taken in the
3805
    // order from %xmm0 to %xmm7.
3806
279k
  case SSE: {
3807
6.78k
    llvm::Type *IRType = CGT.ConvertType(Ty);
3808
6.78k
    ResType = GetSSETypeAtOffset(IRType, 0, Ty, 0);
3809
6.78k
    ++neededSSE;
3810
6.78k
    break;
3811
282k
  }
3812
296k
  }
3813
3814
286k
  llvm::Type *HighPart = nullptr;
3815
286k
  switch (Hi) {
3816
    // Memory was handled previously, ComplexX87 and X87 should
3817
    // never occur as hi classes, and X87Up must be preceded by X87,
3818
    // which is passed in memory.
3819
0
  case Memory:
3820
0
  case X87:
3821
0
  case ComplexX87:
3822
0
    llvm_unreachable("Invalid classification for hi word.");
3823
3824
281k
  case NoClass: break;
3825
3826
696
  case Integer:
3827
696
    ++neededInt;
3828
    // Pick an 8-byte type based on the preferred type.
3829
696
    HighPart = GetINTEGERTypeAtOffset(CGT.ConvertType(Ty), 8, Ty, 8);
3830
3831
696
    if (Lo == NoClass)  // Pass HighPart at offset 8 in memory.
3832
2
      return ABIArgInfo::getDirect(HighPart, 8);
3833
694
    break;
3834
3835
    // X87Up generally doesn't occur here (long double is passed in
3836
    // memory), except in situations involving unions.
3837
694
  case X87Up:
3838
158
  case SSE:
3839
158
    HighPart = GetSSETypeAtOffset(CGT.ConvertType(Ty), 8, Ty, 8);
3840
3841
158
    if (Lo == NoClass)  // Pass HighPart at offset 8 in memory.
3842
0
      return ABIArgInfo::getDirect(HighPart, 8);
3843
3844
158
    ++neededSSE;
3845
158
    break;
3846
3847
    // AMD64-ABI 3.2.3p3: Rule 4. If the class is SSEUP, the
3848
    // eightbyte is passed in the upper half of the last used SSE
3849
    // register.  This only happens when 128-bit vectors are passed.
3850
3.88k
  case SSEUp:
3851
3.88k
    assert(Lo == SSE && "Unexpected SSEUp classification");
3852
0
    ResType = GetByteVectorType(Ty);
3853
3.88k
    break;
3854
286k
  }
3855
3856
  // If a high part was specified, merge it together with the low part.  It is
3857
  // known to pass in the high eightbyte of the result.  We do this by forming a
3858
  // first class struct aggregate with the high and low part: {low, high}
3859
286k
  if (HighPart)
3860
852
    ResType = GetX86_64ByValArgumentPair(ResType, HighPart, getDataLayout());
3861
3862
286k
  return ABIArgInfo::getDirect(ResType);
3863
286k
}
3864
3865
ABIArgInfo
3866
X86_64ABIInfo::classifyRegCallStructTypeImpl(QualType Ty, unsigned &NeededInt,
3867
18
                                             unsigned &NeededSSE) const {
3868
18
  auto RT = Ty->getAs<RecordType>();
3869
18
  assert(RT && "classifyRegCallStructType only valid with struct types");
3870
3871
18
  if (RT->getDecl()->hasFlexibleArrayMember())
3872
0
    return getIndirectReturnResult(Ty);
3873
3874
  // Sum up bases
3875
18
  if (auto CXXRD = dyn_cast<CXXRecordDecl>(RT->getDecl())) {
3876
0
    if (CXXRD->isDynamicClass()) {
3877
0
      NeededInt = NeededSSE = 0;
3878
0
      return getIndirectReturnResult(Ty);
3879
0
    }
3880
3881
0
    for (const auto &I : CXXRD->bases())
3882
0
      if (classifyRegCallStructTypeImpl(I.getType(), NeededInt, NeededSSE)
3883
0
              .isIndirect()) {
3884
0
        NeededInt = NeededSSE = 0;
3885
0
        return getIndirectReturnResult(Ty);
3886
0
      }
3887
0
  }
3888
3889
  // Sum up members
3890
44
  
for (const auto *FD : RT->getDecl()->fields())18
{
3891
44
    if (FD->getType()->isRecordType() && 
!FD->getType()->isUnionType()0
) {
3892
0
      if (classifyRegCallStructTypeImpl(FD->getType(), NeededInt, NeededSSE)
3893
0
              .isIndirect()) {
3894
0
        NeededInt = NeededSSE = 0;
3895
0
        return getIndirectReturnResult(Ty);
3896
0
      }
3897
44
    } else {
3898
44
      unsigned LocalNeededInt, LocalNeededSSE;
3899
44
      if (classifyArgumentType(FD->getType(), UINT_MAX, LocalNeededInt,
3900
44
                               LocalNeededSSE, true)
3901
44
              .isIndirect()) {
3902
0
        NeededInt = NeededSSE = 0;
3903
0
        return getIndirectReturnResult(Ty);
3904
0
      }
3905
44
      NeededInt += LocalNeededInt;
3906
44
      NeededSSE += LocalNeededSSE;
3907
44
    }
3908
44
  }
3909
3910
18
  return ABIArgInfo::getDirect();
3911
18
}
3912
3913
ABIArgInfo X86_64ABIInfo::classifyRegCallStructType(QualType Ty,
3914
                                                    unsigned &NeededInt,
3915
18
                                                    unsigned &NeededSSE) const {
3916
3917
18
  NeededInt = 0;
3918
18
  NeededSSE = 0;
3919
3920
18
  return classifyRegCallStructTypeImpl(Ty, NeededInt, NeededSSE);
3921
18
}
3922
3923
165k
void X86_64ABIInfo::computeInfo(CGFunctionInfo &FI) const {
3924
3925
165k
  const unsigned CallingConv = FI.getCallingConvention();
3926
  // It is possible to force Win64 calling convention on any x86_64 target by
3927
  // using __attribute__((ms_abi)). In such case to correctly emit Win64
3928
  // compatible code delegate this call to WinX86_64ABIInfo::computeInfo.
3929
165k
  if (CallingConv == llvm::CallingConv::Win64) {
3930
13
    WinX86_64ABIInfo Win64ABIInfo(CGT, AVXLevel);
3931
13
    Win64ABIInfo.computeInfo(FI);
3932
13
    return;
3933
13
  }
3934
3935
165k
  bool IsRegCall = CallingConv == llvm::CallingConv::X86_RegCall;
3936
3937
  // Keep track of the number of assigned registers.
3938
165k
  unsigned FreeIntRegs = IsRegCall ? 
1126
:
6165k
;
3939
165k
  unsigned FreeSSERegs = IsRegCall ? 
1626
:
8165k
;
3940
165k
  unsigned NeededInt, NeededSSE;
3941
3942
165k
  if (!::classifyReturnType(getCXXABI(), FI, *this)) {
3943
164k
    if (IsRegCall && 
FI.getReturnType()->getTypePtr()->isRecordType()25
&&
3944
164k
        
!FI.getReturnType()->getTypePtr()->isUnionType()2
) {
3945
2
      FI.getReturnInfo() =
3946
2
          classifyRegCallStructType(FI.getReturnType(), NeededInt, NeededSSE);
3947
2
      if (FreeIntRegs >= NeededInt && FreeSSERegs >= NeededSSE) {
3948
2
        FreeIntRegs -= NeededInt;
3949
2
        FreeSSERegs -= NeededSSE;
3950
2
      } else {
3951
0
        FI.getReturnInfo() = getIndirectReturnResult(FI.getReturnType());
3952
0
      }
3953
164k
    } else if (IsRegCall && 
FI.getReturnType()->getAs<ComplexType>()23
&&
3954
164k
               getContext().getCanonicalType(FI.getReturnType()
3955
2
                                                 ->getAs<ComplexType>()
3956
2
                                                 ->getElementType()) ==
3957
2
                   getContext().LongDoubleTy)
3958
      // Complex Long Double Type is passed in Memory when Regcall
3959
      // calling convention is used.
3960
1
      FI.getReturnInfo() = getIndirectReturnResult(FI.getReturnType());
3961
164k
    else
3962
164k
      FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
3963
164k
  }
3964
3965
  // If the return value is indirect, then the hidden argument is consuming one
3966
  // integer register.
3967
165k
  if (FI.getReturnInfo().isIndirect())
3968
1.08k
    --FreeIntRegs;
3969
3970
  // The chain argument effectively gives us another free register.
3971
165k
  if (FI.isChainCall())
3972
4
    ++FreeIntRegs;
3973
3974
165k
  unsigned NumRequiredArgs = FI.getNumRequiredArgs();
3975
  // AMD64-ABI 3.2.3p3: Once arguments are classified, the registers
3976
  // get assigned (in left-to-right order) for passing as follows...
3977
165k
  unsigned ArgNo = 0;
3978
165k
  for (CGFunctionInfo::arg_iterator it = FI.arg_begin(), ie = FI.arg_end();
3979
461k
       it != ie; 
++it, ++ArgNo295k
) {
3980
295k
    bool IsNamedArg = ArgNo < NumRequiredArgs;
3981
3982
295k
    if (IsRegCall && 
it->type->isStructureOrClassType()62
)
3983
16
      it->info = classifyRegCallStructType(it->type, NeededInt, NeededSSE);
3984
295k
    else
3985
295k
      it->info = classifyArgumentType(it->type, FreeIntRegs, NeededInt,
3986
295k
                                      NeededSSE, IsNamedArg);
3987
3988
    // AMD64-ABI 3.2.3p3: If there are no registers available for any
3989
    // eightbyte of an argument, the whole argument is passed on the
3990
    // stack. If registers have already been assigned for some
3991
    // eightbytes of such an argument, the assignments get reverted.
3992
295k
    if (FreeIntRegs >= NeededInt && 
FreeSSERegs >= NeededSSE293k
) {
3993
293k
      FreeIntRegs -= NeededInt;
3994
293k
      FreeSSERegs -= NeededSSE;
3995
293k
    } else {
3996
2.19k
      it->info = getIndirectResult(it->type, FreeIntRegs);
3997
2.19k
    }
3998
295k
  }
3999
165k
}
4000
4001
static Address EmitX86_64VAArgFromMemory(CodeGenFunction &CGF,
4002
98
                                         Address VAListAddr, QualType Ty) {
4003
98
  Address overflow_arg_area_p =
4004
98
      CGF.Builder.CreateStructGEP(VAListAddr, 2, "overflow_arg_area_p");
4005
98
  llvm::Value *overflow_arg_area =
4006
98
    CGF.Builder.CreateLoad(overflow_arg_area_p, "overflow_arg_area");
4007
4008
  // AMD64-ABI 3.5.7p5: Step 7. Align l->overflow_arg_area upwards to a 16
4009
  // byte boundary if alignment needed by type exceeds 8 byte boundary.
4010
  // It isn't stated explicitly in the standard, but in practice we use
4011
  // alignment greater than 16 where necessary.
4012
98
  CharUnits Align = CGF.getContext().getTypeAlignInChars(Ty);
4013
98
  if (Align > CharUnits::fromQuantity(8)) {
4014
18
    overflow_arg_area = emitRoundPointerUpToAlignment(CGF, overflow_arg_area,
4015
18
                                                      Align);
4016
18
  }
4017
4018
  // AMD64-ABI 3.5.7p5: Step 8. Fetch type from l->overflow_arg_area.
4019
98
  llvm::Type *LTy = CGF.ConvertTypeForMem(Ty);
4020
98
  llvm::Value *Res =
4021
98
    CGF.Builder.CreateBitCast(overflow_arg_area,
4022
98
                              llvm::PointerType::getUnqual(LTy));
4023
4024
  // AMD64-ABI 3.5.7p5: Step 9. Set l->overflow_arg_area to:
4025
  // l->overflow_arg_area + sizeof(type).
4026
  // AMD64-ABI 3.5.7p5: Step 10. Align l->overflow_arg_area upwards to
4027
  // an 8 byte boundary.
4028
4029
98
  uint64_t SizeInBytes = (CGF.getContext().getTypeSize(Ty) + 7) / 8;
4030
98
  llvm::Value *Offset =
4031
98
      llvm::ConstantInt::get(CGF.Int32Ty, (SizeInBytes + 7)  & ~7);
4032
98
  overflow_arg_area = CGF.Builder.CreateGEP(CGF.Int8Ty, overflow_arg_area,
4033
98
                                            Offset, "overflow_arg_area.next");
4034
98
  CGF.Builder.CreateStore(overflow_arg_area, overflow_arg_area_p);
4035
4036
  // AMD64-ABI 3.5.7p5: Step 11. Return the fetched type.
4037
98
  return Address(Res, LTy, Align);
4038
98
}
4039
4040
Address X86_64ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
4041
98
                                 QualType Ty) const {
4042
  // Assume that va_list type is correct; should be pointer to LLVM type:
4043
  // struct {
4044
  //   i32 gp_offset;
4045
  //   i32 fp_offset;
4046
  //   i8* overflow_arg_area;
4047
  //   i8* reg_save_area;
4048
  // };
4049
98
  unsigned neededInt, neededSSE;
4050
4051
98
  Ty = getContext().getCanonicalType(Ty);
4052
98
  ABIArgInfo AI = classifyArgumentType(Ty, 0, neededInt, neededSSE,
4053
98
                                       /*isNamedArg*/false);
4054
4055
  // AMD64-ABI 3.5.7p5: Step 1. Determine whether type may be passed
4056
  // in the registers. If not go to step 7.
4057
98
  if (!neededInt && 
!neededSSE35
)
4058
18
    return EmitX86_64VAArgFromMemory(CGF, VAListAddr, Ty);
4059
4060
  // AMD64-ABI 3.5.7p5: Step 2. Compute num_gp to hold the number of
4061
  // general purpose registers needed to pass type and num_fp to hold
4062
  // the number of floating point registers needed.
4063
4064
  // AMD64-ABI 3.5.7p5: Step 3. Verify whether arguments fit into
4065
  // registers. In the case: l->gp_offset > 48 - num_gp * 8 or
4066
  // l->fp_offset > 304 - num_fp * 16 go to step 7.
4067
  //
4068
  // NOTE: 304 is a typo, there are (6 * 8 + 8 * 16) = 176 bytes of
4069
  // register save space).
4070
4071
80
  llvm::Value *InRegs = nullptr;
4072
80
  Address gp_offset_p = Address::invalid(), fp_offset_p = Address::invalid();
4073
80
  llvm::Value *gp_offset = nullptr, *fp_offset = nullptr;
4074
80
  if (neededInt) {
4075
63
    gp_offset_p = CGF.Builder.CreateStructGEP(VAListAddr, 0, "gp_offset_p");
4076
63
    gp_offset = CGF.Builder.CreateLoad(gp_offset_p, "gp_offset");
4077
63
    InRegs = llvm::ConstantInt::get(CGF.Int32Ty, 48 - neededInt * 8);
4078
63
    InRegs = CGF.Builder.CreateICmpULE(gp_offset, InRegs, "fits_in_gp");
4079
63
  }
4080
4081
80
  if (neededSSE) {
4082
21
    fp_offset_p = CGF.Builder.CreateStructGEP(VAListAddr, 1, "fp_offset_p");
4083
21
    fp_offset = CGF.Builder.CreateLoad(fp_offset_p, "fp_offset");
4084
21
    llvm::Value *FitsInFP =
4085
21
      llvm::ConstantInt::get(CGF.Int32Ty, 176 - neededSSE * 16);
4086
21
    FitsInFP = CGF.Builder.CreateICmpULE(fp_offset, FitsInFP, "fits_in_fp");
4087
21
    InRegs = InRegs ? 
CGF.Builder.CreateAnd(InRegs, FitsInFP)4
:
FitsInFP17
;
4088
21
  }
4089
4090
80
  llvm::BasicBlock *InRegBlock = CGF.createBasicBlock("vaarg.in_reg");
4091
80
  llvm::BasicBlock *InMemBlock = CGF.createBasicBlock("vaarg.in_mem");
4092
80
  llvm::BasicBlock *ContBlock = CGF.createBasicBlock("vaarg.end");
4093
80
  CGF.Builder.CreateCondBr(InRegs, InRegBlock, InMemBlock);
4094
4095
  // Emit code to load the value if it was passed in registers.
4096
4097
80
  CGF.EmitBlock(InRegBlock);
4098
4099
  // AMD64-ABI 3.5.7p5: Step 4. Fetch type from l->reg_save_area with
4100
  // an offset of l->gp_offset and/or l->fp_offset. This may require
4101
  // copying to a temporary location in case the parameter is passed
4102
  // in different register classes or requires an alignment greater
4103
  // than 8 for general purpose registers and 16 for XMM registers.
4104
  //
4105
  // FIXME: This really results in shameful code when we end up needing to
4106
  // collect arguments from different places; often what should result in a
4107
  // simple assembling of a structure from scattered addresses has many more
4108
  // loads than necessary. Can we clean this up?
4109
80
  llvm::Type *LTy = CGF.ConvertTypeForMem(Ty);
4110
80
  llvm::Value *RegSaveArea = CGF.Builder.CreateLoad(
4111
80
      CGF.Builder.CreateStructGEP(VAListAddr, 3), "reg_save_area");
4112
4113
80
  Address RegAddr = Address::invalid();
4114
80
  if (neededInt && 
neededSSE63
) {
4115
    // FIXME: Cleanup.
4116
4
    assert(AI.isDirect() && "Unexpected ABI info for mixed regs");
4117
0
    llvm::StructType *ST = cast<llvm::StructType>(AI.getCoerceToType());
4118
4
    Address Tmp = CGF.CreateMemTemp(Ty);
4119
4
    Tmp = CGF.Builder.CreateElementBitCast(Tmp, ST);
4120
4
    assert(ST->getNumElements() == 2 && "Unexpected ABI info for mixed regs");
4121
0
    llvm::Type *TyLo = ST->getElementType(0);
4122
4
    llvm::Type *TyHi = ST->getElementType(1);
4123
4
    assert((TyLo->isFPOrFPVectorTy() ^ TyHi->isFPOrFPVectorTy()) &&
4124
4
           "Unexpected ABI info for mixed regs");
4125
0
    llvm::Type *PTyLo = llvm::PointerType::getUnqual(TyLo);
4126
4
    llvm::Type *PTyHi = llvm::PointerType::getUnqual(TyHi);
4127
4
    llvm::Value *GPAddr =
4128
4
        CGF.Builder.CreateGEP(CGF.Int8Ty, RegSaveArea, gp_offset);
4129
4
    llvm::Value *FPAddr =
4130
4
        CGF.Builder.CreateGEP(CGF.Int8Ty, RegSaveArea, fp_offset);
4131
4
    llvm::Value *RegLoAddr = TyLo->isFPOrFPVectorTy() ? 
FPAddr1
:
GPAddr3
;
4132
4
    llvm::Value *RegHiAddr = TyLo->isFPOrFPVectorTy() ? 
GPAddr1
:
FPAddr3
;
4133
4134
    // Copy the first element.
4135
    // FIXME: Our choice of alignment here and below is probably pessimistic.
4136
4
    llvm::Value *V = CGF.Builder.CreateAlignedLoad(
4137
4
        TyLo, CGF.Builder.CreateBitCast(RegLoAddr, PTyLo),
4138
4
        CharUnits::fromQuantity(getDataLayout().getABITypeAlignment(TyLo)));
4139
4
    CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 0));
4140
4141
    // Copy the second element.
4142
4
    V = CGF.Builder.CreateAlignedLoad(
4143
4
        TyHi, CGF.Builder.CreateBitCast(RegHiAddr, PTyHi),
4144
4
        CharUnits::fromQuantity(getDataLayout().getABITypeAlignment(TyHi)));
4145
4
    CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 1));
4146
4147
4
    RegAddr = CGF.Builder.CreateElementBitCast(Tmp, LTy);
4148
76
  } else if (neededInt) {
4149
59
    RegAddr = Address(CGF.Builder.CreateGEP(CGF.Int8Ty, RegSaveArea, gp_offset),
4150
59
                      CGF.Int8Ty, CharUnits::fromQuantity(8));
4151
59
    RegAddr = CGF.Builder.CreateElementBitCast(RegAddr, LTy);
4152
4153
    // Copy to a temporary if necessary to ensure the appropriate alignment.
4154
59
    auto TInfo = getContext().getTypeInfoInChars(Ty);
4155
59
    uint64_t TySize = TInfo.Width.getQuantity();
4156
59
    CharUnits TyAlign = TInfo.Align;
4157
4158
    // Copy into a temporary if the type is more aligned than the
4159
    // register save area.
4160
59
    if (TyAlign.getQuantity() > 8) {
4161
3
      Address Tmp = CGF.CreateMemTemp(Ty);
4162
3
      CGF.Builder.CreateMemCpy(Tmp, RegAddr, TySize, false);
4163
3
      RegAddr = Tmp;
4164
3
    }
4165
4166
59
  } else 
if (17
neededSSE == 117
) {
4167
7
    RegAddr = Address(CGF.Builder.CreateGEP(CGF.Int8Ty, RegSaveArea, fp_offset),
4168
7
                      CGF.Int8Ty, CharUnits::fromQuantity(16));
4169
7
    RegAddr = CGF.Builder.CreateElementBitCast(RegAddr, LTy);
4170
10
  } else {
4171
10
    assert(neededSSE == 2 && "Invalid number of needed registers!");
4172
    // SSE registers are spaced 16 bytes apart in the register save
4173
    // area, we need to collect the two eightbytes together.
4174
    // The ABI isn't explicit about this, but it seems reasonable
4175
    // to assume that the slots are 16-byte aligned, since the stack is
4176
    // naturally 16-byte aligned and the prologue is expected to store
4177
    // all the SSE registers to the RSA.
4178
0
    Address RegAddrLo = Address(CGF.Builder.CreateGEP(CGF.Int8Ty, RegSaveArea,
4179
10
                                                      fp_offset),
4180
10
                                CGF.Int8Ty, CharUnits::fromQuantity(16));
4181
10
    Address RegAddrHi =
4182
10
      CGF.Builder.CreateConstInBoundsByteGEP(RegAddrLo,
4183
10
                                             CharUnits::fromQuantity(16));
4184
10
    llvm::Type *ST = AI.canHaveCoerceToType()
4185
10
                         ? AI.getCoerceToType()
4186
10
                         : 
llvm::StructType::get(CGF.DoubleTy, CGF.DoubleTy)0
;
4187
10
    llvm::Value *V;
4188
10
    Address Tmp = CGF.CreateMemTemp(Ty);
4189
10
    Tmp = CGF.Builder.CreateElementBitCast(Tmp, ST);
4190
10
    V = CGF.Builder.CreateLoad(CGF.Builder.CreateElementBitCast(
4191
10
        RegAddrLo, ST->getStructElementType(0)));
4192
10
    CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 0));
4193
10
    V = CGF.Builder.CreateLoad(CGF.Builder.CreateElementBitCast(
4194
10
        RegAddrHi, ST->getStructElementType(1)));
4195
10
    CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 1));
4196
4197
10
    RegAddr = CGF.Builder.CreateElementBitCast(Tmp, LTy);
4198
10
  }
4199
4200
  // AMD64-ABI 3.5.7p5: Step 5. Set:
4201
  // l->gp_offset = l->gp_offset + num_gp * 8
4202
  // l->fp_offset = l->fp_offset + num_fp * 16.
4203
80
  if (neededInt) {
4204
63
    llvm::Value *Offset = llvm::ConstantInt::get(CGF.Int32Ty, neededInt * 8);
4205
63
    CGF.Builder.CreateStore(CGF.Builder.CreateAdd(gp_offset, Offset),
4206
63
                            gp_offset_p);
4207
63
  }
4208
80
  if (neededSSE) {
4209
21
    llvm::Value *Offset = llvm::ConstantInt::get(CGF.Int32Ty, neededSSE * 16);
4210
21
    CGF.Builder.CreateStore(CGF.Builder.CreateAdd(fp_offset, Offset),
4211
21
                            fp_offset_p);
4212
21
  }
4213
80
  CGF.EmitBranch(ContBlock);
4214
4215
  // Emit code to load the value if it was passed in memory.
4216
4217
80
  CGF.EmitBlock(InMemBlock);
4218
80
  Address MemAddr = EmitX86_64VAArgFromMemory(CGF, VAListAddr, Ty);
4219
4220
  // Return the appropriate result.
4221
4222
80
  CGF.EmitBlock(ContBlock);
4223
80
  Address ResAddr = emitMergePHI(CGF, RegAddr, InRegBlock, MemAddr, InMemBlock,
4224
80
                                 "vaarg.addr");
4225
80
  return ResAddr;
4226
98
}
4227
4228
Address X86_64ABIInfo::EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr,
4229
8
                                   QualType Ty) const {
4230
  // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
4231
  // not 1, 2, 4, or 8 bytes, must be passed by reference."
4232
8
  uint64_t Width = getContext().getTypeSize(Ty);
4233
8
  bool IsIndirect = Width > 64 || 
!llvm::isPowerOf2_64(Width)4
;
4234
4235
8
  return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect,
4236
8
                          CGF.getContext().getTypeInfoInChars(Ty),
4237
8
                          CharUnits::fromQuantity(8),
4238
8
                          /*allowHigherAlign*/ false);
4239
8
}
4240
4241
ABIArgInfo WinX86_64ABIInfo::reclassifyHvaArgForVectorCall(
4242
97
    QualType Ty, unsigned &FreeSSERegs, const ABIArgInfo &current) const {
4243
97
  const Type *Base = nullptr;
4244
97
  uint64_t NumElts = 0;
4245
4246
97
  if (!Ty->isBuiltinType() && 
!Ty->isVectorType()49
&&
4247
97
      
isHomogeneousAggregate(Ty, Base, NumElts)36
&&
FreeSSERegs >= NumElts27
) {
4248
18
    FreeSSERegs -= NumElts;
4249
18
    return getDirectX86Hva();
4250
18
  }
4251
79
  return current;
4252
97
}
4253
4254
ABIArgInfo WinX86_64ABIInfo::classify(QualType Ty, unsigned &FreeSSERegs,
4255
                                      bool IsReturnType, bool IsVectorCall,
4256
10.1k
                                      bool IsRegCall) const {
4257
4258
10.1k
  if (Ty->isVoidType())
4259
1.69k
    return ABIArgInfo::getIgnore();
4260
4261
8.42k
  if (const EnumType *EnumTy = Ty->getAs<EnumType>())
4262
20
    Ty = EnumTy->getDecl()->getIntegerType();
4263
4264
8.42k
  TypeInfo Info = getContext().getTypeInfo(Ty);
4265
8.42k
  uint64_t Width = Info.Width;
4266
8.42k
  CharUnits Align = getContext().toCharUnitsFromBits(Info.Align);
4267
4268
8.42k
  const RecordType *RT = Ty->getAs<RecordType>();
4269
8.42k
  if (RT) {
4270
300
    if (!IsReturnType) {
4271
262
      if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(RT, getCXXABI()))
4272
71
        return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
4273
262
    }
4274
4275
229
    if (RT->getDecl()->hasFlexibleArrayMember())
4276
0
      return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
4277
4278
229
  }
4279
4280
8.35k
  const Type *Base = nullptr;
4281
8.35k
  uint64_t NumElts = 0;
4282
  // vectorcall adds the concept of a homogenous vector aggregate, similar to
4283
  // other targets.
4284
8.35k
  if ((IsVectorCall || 
IsRegCall8.24k
) &&
4285
8.35k
      
isHomogeneousAggregate(Ty, Base, NumElts)171
) {
4286
94
    if (IsRegCall) {
4287
31
      if (FreeSSERegs >= NumElts) {
4288
31
        FreeSSERegs -= NumElts;
4289
31
        if (IsReturnType || 
Ty->isBuiltinType()27
||
Ty->isVectorType()21
)
4290
16
          return ABIArgInfo::getDirect();
4291
15
        return ABIArgInfo::getExpand();
4292
31
      }
4293
0
      return ABIArgInfo::getIndirect(Align, /*ByVal=*/false);
4294
63
    } else if (IsVectorCall) {
4295
63
      if (FreeSSERegs >= NumElts &&
4296
63
          
(57
IsReturnType57
||
Ty->isBuiltinType()45
||
Ty->isVectorType()36
)) {
4297
34
        FreeSSERegs -= NumElts;
4298
34
        return ABIArgInfo::getDirect();
4299
34
      } else 
if (29
IsReturnType29
) {
4300
0
        return ABIArgInfo::getExpand();
4301
29
      } else if (!Ty->isBuiltinType() && 
!Ty->isVectorType()27
) {
4302
        // HVAs are delayed and reclassified in the 2nd step.
4303
27
        return ABIArgInfo::getIndirect(Align, /*ByVal=*/false);
4304
27
      }
4305
63
    }
4306
94
  }
4307
4308
8.26k
  if (Ty->isMemberPointerType()) {
4309
    // If the member pointer is represented by an LLVM int or ptr, pass it
4310
    // directly.
4311
160
    llvm::Type *LLTy = CGT.ConvertType(Ty);
4312
160
    if (LLTy->isPointerTy() || 
LLTy->isIntegerTy()77
)
4313
91
      return ABIArgInfo::getDirect();
4314
160
  }
4315
4316
8.17k
  if (RT || 
Ty->isAnyComplexType()7.99k
||
Ty->isMemberPointerType()7.97k
) {
4317
    // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
4318
    // not 1, 2, 4, or 8 bytes, must be passed by reference."
4319
267
    if (Width > 64 || 
!llvm::isPowerOf2_64(Width)150
)
4320
117
      return getNaturalAlignIndirect(Ty, /*ByVal=*/false);
4321
4322
    // Otherwise, coerce it to a small integer.
4323
150
    return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(), Width));
4324
267
  }
4325
4326
7.90k
  if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
4327
2.21k
    switch (BT->getKind()) {
4328
32
    case BuiltinType::Bool:
4329
      // Bool type is always extended to the ABI, other builtin types are not
4330
      // extended.
4331
32
      return ABIArgInfo::getExtend(Ty);
4332
4333
56
    case BuiltinType::LongDouble:
4334
      // Mingw64 GCC uses the old 80 bit extended precision floating point
4335
      // unit. It passes them indirectly through memory.
4336
56
      if (IsMingw64) {
4337
6
        const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();
4338
6
        if (LDF == &llvm::APFloat::x87DoubleExtended())
4339
6
          return ABIArgInfo::getIndirect(Align, /*ByVal=*/false);
4340
6
      }
4341
50
      break;
4342
4343
50
    case BuiltinType::Int128:
4344
10
    case BuiltinType::UInt128:
4345
      // If it's a parameter type, the normal ABI rule is that arguments larger
4346
      // than 8 bytes are passed indirectly. GCC follows it. We follow it too,
4347
      // even though it isn't particularly efficient.
4348
10
      if (!IsReturnType)
4349
4
        return ABIArgInfo::getIndirect(Align, /*ByVal=*/false);
4350
4351
      // Mingw64 GCC returns i128 in XMM0. Coerce to v2i64 to handle that.
4352
      // Clang matches them for compatibility.
4353
6
      return ABIArgInfo::getDirect(llvm::FixedVectorType::get(
4354
6
          llvm::Type::getInt64Ty(getVMContext()), 2));
4355
4356
2.12k
    default:
4357
2.12k
      break;
4358
2.21k
    }
4359
2.21k
  }
4360
4361
7.85k
  if (Ty->isBitIntType()) {
4362
    // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
4363
    // not 1, 2, 4, or 8 bytes, must be passed by reference."
4364
    // However, non-power-of-two bit-precise integers will be passed as 1, 2, 4,
4365
    // or 8 bytes anyway as long is it fits in them, so we don't have to check
4366
    // the power of 2.
4367
56
    if (Width <= 64)
4368
37
      return ABIArgInfo::getDirect();
4369
19
    return ABIArgInfo::getIndirect(Align, /*ByVal=*/false);
4370
56
  }
4371
4372
7.80k
  return ABIArgInfo::getDirect();
4373
7.85k
}
4374
4375
4.18k
void WinX86_64ABIInfo::computeInfo(CGFunctionInfo &FI) const {
4376
4.18k
  const unsigned CC = FI.getCallingConvention();
4377
4.18k
  bool IsVectorCall = CC == llvm::CallingConv::X86_VectorCall;
4378
4.18k
  bool IsRegCall = CC == llvm::CallingConv::X86_RegCall;
4379
4380
  // If __attribute__((sysv_abi)) is in use, use the SysV argument
4381
  // classification rules.
4382
4.18k
  if (CC == llvm::CallingConv::X86_64_SysV) {
4383
14
    X86_64ABIInfo SysVABIInfo(CGT, AVXLevel);
4384
14
    SysVABIInfo.computeInfo(FI);
4385
14
    return;
4386
14
  }
4387
4388
4.17k
  unsigned FreeSSERegs = 0;
4389
4.17k
  if (IsVectorCall) {
4390
    // We can use up to 4 SSE return registers with vectorcall.
4391
49
    FreeSSERegs = 4;
4392
4.12k
  } else if (IsRegCall) {
4393
    // RegCall gives us 16 SSE registers.
4394
24
    FreeSSERegs = 16;
4395
24
  }
4396
4397
4.17k
  if (!getCXXABI().classifyReturnType(FI))
4398
4.12k
    FI.getReturnInfo() = classify(FI.getReturnType(), FreeSSERegs, true,
4399
4.12k
                                  IsVectorCall, IsRegCall);
4400
4401
4.17k
  if (IsVectorCall) {
4402
    // We can use up to 6 SSE register parameters with vectorcall.
4403
49
    FreeSSERegs = 6;
4404
4.12k
  } else if (IsRegCall) {
4405
    // RegCall gives us 16 SSE registers, we can reuse the return registers.
4406
24
    FreeSSERegs = 16;
4407
24
  }
4408
4409
4.17k
  unsigned ArgNum = 0;
4410
4.17k
  unsigned ZeroSSERegs = 0;
4411
6.00k
  for (auto &I : FI.arguments()) {
4412
    // Vectorcall in x64 only permits the first 6 arguments to be passed as
4413
    // XMM/YMM registers. After the sixth argument, pretend no vector
4414
    // registers are left.
4415
6.00k
    unsigned *MaybeFreeSSERegs =
4416
6.00k
        (IsVectorCall && 
ArgNum >= 697
) ?
&ZeroSSERegs5
:
&FreeSSERegs5.99k
;
4417
6.00k
    I.info =
4418
6.00k
        classify(I.type, *MaybeFreeSSERegs, false, IsVectorCall, IsRegCall);
4419
6.00k
    ++ArgNum;
4420
6.00k
  }
4421
4422
4.17k
  if (IsVectorCall) {
4423
    // For vectorcall, assign aggregate HVAs to any free vector registers in a
4424
    // second pass.
4425
49
    for (auto &I : FI.arguments())
4426
97
      I.info = reclassifyHvaArgForVectorCall(I.type, FreeSSERegs, I.info);
4427
49
  }
4428
4.17k
}
4429
4430
Address WinX86_64ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
4431
23
                                    QualType Ty) const {
4432
  // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is
4433
  // not 1, 2, 4, or 8 bytes, must be passed by reference."
4434
23
  uint64_t Width = getContext().getTypeSize(Ty);
4435
23
  bool IsIndirect = Width > 64 || 
!llvm::isPowerOf2_64(Width)8
;
4436
4437
23
  return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect,
4438
23
                          CGF.getContext().getTypeInfoInChars(Ty),
4439
23
                          CharUnits::fromQuantity(8),
4440
23
                          /*allowHigherAlign*/ false);
4441
23
}
4442
4443
static bool PPC_initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
4444
                                        llvm::Value *Address, bool Is64Bit,
4445
4
                                        bool IsAIX) {
4446
  // This is calculated from the LLVM and GCC tables and verified
4447
  // against gcc output.  AFAIK all PPC ABIs use the same encoding.
4448
4449
4
  CodeGen::CGBuilderTy &Builder = CGF.Builder;
4450
4451
4
  llvm::IntegerType *i8 = CGF.Int8Ty;
4452
4
  llvm::Value *Four8 = llvm::ConstantInt::get(i8, 4);
4453
4
  llvm::Value *Eight8 = llvm::ConstantInt::get(i8, 8);
4454
4
  llvm::Value *Sixteen8 = llvm::ConstantInt::get(i8, 16);
4455
4456
  // 0-31: r0-31, the 4-byte or 8-byte general-purpose registers
4457
4
  AssignToArrayRange(Builder, Address, Is64Bit ? 
Eight82
:
Four82
, 0, 31);
4458
4459
  // 32-63: fp0-31, the 8-byte floating-point registers
4460
4
  AssignToArrayRange(Builder, Address, Eight8, 32, 63);
4461
4462
  // 64-67 are various 4-byte or 8-byte special-purpose registers:
4463
  // 64: mq
4464
  // 65: lr
4465
  // 66: ctr
4466
  // 67: ap
4467
4
  AssignToArrayRange(Builder, Address, Is64Bit ? 
Eight82
:
Four82
, 64, 67);
4468
4469
  // 68-76 are various 4-byte special-purpose registers:
4470
  // 68-75 cr0-7
4471
  // 76: xer
4472
4
  AssignToArrayRange(Builder, Address, Four8, 68, 76);
4473
4474
  // 77-108: v0-31, the 16-byte vector registers
4475
4
  AssignToArrayRange(Builder, Address, Sixteen8, 77, 108);
4476
4477
  // 109: vrsave
4478
  // 110: vscr
4479
4
  AssignToArrayRange(Builder, Address, Is64Bit ? 
Eight82
:
Four82
, 109, 110);
4480
4481
  // AIX does not utilize the rest of the registers.
4482
4
  if (IsAIX)
4483
2
    return false;
4484
4485
  // 111: spe_acc
4486
  // 112: spefscr
4487
  // 113: sfp
4488
2
  AssignToArrayRange(Builder, Address, Is64Bit ? 
Eight81
:
Four81
, 111, 113);
4489
4490
2
  if (!Is64Bit)
4491
1
    return false;
4492
4493
  // TODO: Need to verify if these registers are used on 64 bit AIX with Power8
4494
  // or above CPU.
4495
  // 64-bit only registers:
4496
  // 114: tfhar
4497
  // 115: tfiar
4498
  // 116: texasr
4499
1
  AssignToArrayRange(Builder, Address, Eight8, 114, 116);
4500
4501
1
  return false;
4502
2
}
4503
4504
// AIX
4505
namespace {
4506
/// AIXABIInfo - The AIX XCOFF ABI information.
4507
class AIXABIInfo : public ABIInfo {
4508
  const bool Is64Bit;
4509
  const unsigned PtrByteSize;
4510
  CharUnits getParamTypeAlignment(QualType Ty) const;
4511
4512
public:
4513
  AIXABIInfo(CodeGen::CodeGenTypes &CGT, bool Is64Bit)
4514
82
      : ABIInfo(CGT), Is64Bit(Is64Bit), PtrByteSize(Is64Bit ? 8 : 4) {}
4515
4516
  bool isPromotableTypeForABI(QualType Ty) const;
4517
4518
  ABIArgInfo classifyReturnType(QualType RetTy) const;
4519
  ABIArgInfo classifyArgumentType(QualType Ty) const;
4520
4521
191
  void computeInfo(CGFunctionInfo &FI) const override {
4522
191
    if (!getCXXABI().classifyReturnType(FI))
4523
191
      FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
4524
4525
191
    for (auto &I : FI.arguments())
4526
101
      I.info = classifyArgumentType(I.type);
4527
191
  }
4528
4529
  Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
4530
                    QualType Ty) const override;
4531
};
4532
4533
class AIXTargetCodeGenInfo : public TargetCodeGenInfo {
4534
  const bool Is64Bit;
4535
4536
public:
4537
  AIXTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, bool Is64Bit)
4538
      : TargetCodeGenInfo(std::make_unique<AIXABIInfo>(CGT, Is64Bit)),
4539
82
        Is64Bit(Is64Bit) {}
4540
2
  int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
4541
2
    return 1; // r1 is the dedicated stack pointer
4542
2
  }
4543
4544
  bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
4545
                               llvm::Value *Address) const override;
4546
};
4547
} // namespace
4548
4549
// Return true if the ABI requires Ty to be passed sign- or zero-
4550
// extended to 32/64 bits.
4551
183
bool AIXABIInfo::isPromotableTypeForABI(QualType Ty) const {
4552
  // Treat an enum type as its underlying type.
4553
183
  if (const EnumType *EnumTy = Ty->getAs<EnumType>())
4554
2
    Ty = EnumTy->getDecl()->getIntegerType();
4555
4556
  // Promotable integer types are required to be promoted by the ABI.
4557
183
  if (Ty->isPromotableIntegerType())
4558
0
    return true;
4559
4560
183
  if (!Is64Bit)
4561
90
    return false;
4562
4563
  // For 64 bit mode, in addition to the usual promotable integer types, we also
4564
  // need to extend all 32-bit types, since the ABI requires promotion to 64
4565
  // bits.
4566
93
  if (const BuiltinType *BT = Ty->getAs<BuiltinType>())
4567
64
    switch (BT->getKind()) {
4568
43
    case BuiltinType::Int:
4569
47
    case BuiltinType::UInt:
4570
47
      return true;
4571
17
    default:
4572
17
      break;
4573
64
    }
4574
4575
46
  return false;
4576
93
}
4577
4578
191
ABIArgInfo AIXABIInfo::classifyReturnType(QualType RetTy) const {
4579
191
  if (RetTy->isAnyComplexType())
4580
6
    return ABIArgInfo::getDirect();
4581
4582
185
  if (RetTy->isVectorType())
4583
0
    return ABIArgInfo::getDirect();
4584
4585
185
  if (RetTy->isVoidType())
4586
97
    return ABIArgInfo::getIgnore();
4587
4588
88
  if (isAggregateTypeForABI(RetTy))
4589
0
    return getNaturalAlignIndirect(RetTy);
4590
4591
88
  return (isPromotableTypeForABI(RetTy) ? 
ABIArgInfo::getExtend(RetTy)25
4592
88
                                        : 
ABIArgInfo::getDirect()63
);
4593
88
}
4594
4595
101
ABIArgInfo AIXABIInfo::classifyArgumentType(QualType Ty) const {
4596
101
  Ty = useFirstFieldIfTransparentUnion(Ty);
4597
4598
101
  if (Ty->isAnyComplexType())
4599
6
    return ABIArgInfo::getDirect();
4600
4601
95
  if (Ty->isVectorType())
4602
0
    return ABIArgInfo::getDirect();
4603
4604
95
  if (isAggregateTypeForABI(Ty)) {
4605
    // Records with non-trivial destructors/copy-constructors should not be
4606
    // passed by value.
4607
0
    if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI()))
4608
0
      return getNaturalAlignIndirect(Ty, RAA == CGCXXABI::RAA_DirectInMemory);
4609
4610
0
    CharUnits CCAlign = getParamTypeAlignment(Ty);
4611
0
    CharUnits TyAlign = getContext().getTypeAlignInChars(Ty);
4612
4613
0
    return ABIArgInfo::getIndirect(CCAlign, /*ByVal*/ true,
4614
0
                                   /*Realign*/ TyAlign > CCAlign);
4615
0
  }
4616
4617
95
  return (isPromotableTypeForABI(Ty) ? 
ABIArgInfo::getExtend(Ty)22
4618
95
                                     : 
ABIArgInfo::getDirect()73
);
4619
95
}
4620
4621
0
CharUnits AIXABIInfo::getParamTypeAlignment(QualType Ty) const {
4622
  // Complex types are passed just like their elements.
4623
0
  if (const ComplexType *CTy = Ty->getAs<ComplexType>())
4624
0
    Ty = CTy->getElementType();
4625
4626
0
  if (Ty->isVectorType())
4627
0
    return CharUnits::fromQuantity(16);
4628
4629
  // If the structure contains a vector type, the alignment is 16.
4630
0
  if (isRecordWithSIMDVectorType(getContext(), Ty))
4631
0
    return CharUnits::fromQuantity(16);
4632
4633
0
  return CharUnits::fromQuantity(PtrByteSize);
4634
0
}
4635
4636
Address AIXABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
4637
0
                              QualType Ty) const {
4638
4639
0
  auto TypeInfo = getContext().getTypeInfoInChars(Ty);
4640
0
  TypeInfo.Align = getParamTypeAlignment(Ty);
4641
4642
0
  CharUnits SlotSize = CharUnits::fromQuantity(PtrByteSize);
4643
4644
  // If we have a complex type and the base type is smaller than the register
4645
  // size, the ABI calls for the real and imaginary parts to be right-adjusted
4646
  // in separate words in 32bit mode or doublewords in 64bit mode. However,
4647
  // Clang expects us to produce a pointer to a structure with the two parts
4648
  // packed tightly. So generate loads of the real and imaginary parts relative
4649
  // to the va_list pointer, and store them to a temporary structure. We do the
4650
  // same as the PPC64ABI here.
4651
0
  if (const ComplexType *CTy = Ty->getAs<ComplexType>()) {
4652
0
    CharUnits EltSize = TypeInfo.Width / 2;
4653
0
    if (EltSize < SlotSize)
4654
0
      return complexTempStructure(CGF, VAListAddr, Ty, SlotSize, EltSize, CTy);
4655
0
  }
4656
4657
0
  return emitVoidPtrVAArg(CGF, VAListAddr, Ty, /*Indirect*/ false, TypeInfo,
4658
0
                          SlotSize, /*AllowHigher*/ true);
4659
0
}
4660
4661
bool AIXTargetCodeGenInfo::initDwarfEHRegSizeTable(
4662
2
    CodeGen::CodeGenFunction &CGF, llvm::Value *Address) const {
4663
2
  return PPC_initDwarfEHRegSizeTable(CGF, Address, Is64Bit, /*IsAIX*/ true);
4664
2
}
4665
4666
// PowerPC-32
4667
namespace {
4668
/// PPC32_SVR4_ABIInfo - The 32-bit PowerPC ELF (SVR4) ABI information.
4669
class PPC32_SVR4_ABIInfo : public DefaultABIInfo {
4670
  bool IsSoftFloatABI;
4671
  bool IsRetSmallStructInRegABI;
4672
4673
  CharUnits getParamTypeAlignment(QualType Ty) const;
4674
4675
public:
4676
  PPC32_SVR4_ABIInfo(CodeGen::CodeGenTypes &CGT, bool SoftFloatABI,
4677
                     bool RetSmallStructInRegABI)
4678
      : DefaultABIInfo(CGT), IsSoftFloatABI(SoftFloatABI),
4679
64
        IsRetSmallStructInRegABI(RetSmallStructInRegABI) {}
4680
4681
  ABIArgInfo classifyReturnType(QualType RetTy) const;
4682
4683
169
  void computeInfo(CGFunctionInfo &FI) const override {
4684
169
    if (!getCXXABI().classifyReturnType(FI))
4685
169
      FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
4686
169
    for (auto &I : FI.arguments())
4687
279
      I.info = classifyArgumentType(I.type);
4688
169
  }
4689
4690
  Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
4691
                    QualType Ty) const override;
4692
};
4693
4694
class PPC32TargetCodeGenInfo : public TargetCodeGenInfo {
4695
public:
4696
  PPC32TargetCodeGenInfo(CodeGenTypes &CGT, bool SoftFloatABI,
4697
                         bool RetSmallStructInRegABI)
4698
      : TargetCodeGenInfo(std::make_unique<PPC32_SVR4_ABIInfo>(
4699
64
            CGT, SoftFloatABI, RetSmallStructInRegABI)) {}
4700
4701
  static bool isStructReturnInRegABI(const llvm::Triple &Triple,
4702
                                     const CodeGenOptions &Opts);
4703
4704
1
  int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
4705
    // This is recovered from gcc output.
4706
1
    return 1; // r1 is the dedicated stack pointer
4707
1
  }
4708
4709
  bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
4710
                               llvm::Value *Address) const override;
4711
};
4712
}
4713
4714
0
CharUnits PPC32_SVR4_ABIInfo::getParamTypeAlignment(QualType Ty) const {
4715
  // Complex types are passed just like their elements.
4716
0
  if (const ComplexType *CTy = Ty->getAs<ComplexType>())
4717
0
    Ty = CTy->getElementType();
4718
4719
0
  if (Ty->isVectorType())
4720
0
    return CharUnits::fromQuantity(getContext().getTypeSize(Ty) == 128 ? 16
4721
0
                                                                       : 4);
4722
4723
  // For single-element float/vector structs, we consider the whole type
4724
  // to have the same alignment requirements as its single element.
4725
0
  const Type *AlignTy = nullptr;
4726
0
  if (const Type *EltType = isSingleElementStruct(Ty, getContext())) {
4727
0
    const BuiltinType *BT = EltType->getAs<BuiltinType>();
4728
0
    if ((EltType->isVectorType() && getContext().getTypeSize(EltType) == 128) ||
4729
0
        (BT && BT->isFloatingPoint()))
4730
0
      AlignTy = EltType;
4731
0
  }
4732
4733
0
  if (AlignTy)
4734
0
    return CharUnits::fromQuantity(AlignTy->isVectorType() ? 16 : 4);
4735
0
  return CharUnits::fromQuantity(4);
4736
0
}
4737
4738
169
ABIArgInfo PPC32_SVR4_ABIInfo::classifyReturnType(QualType RetTy) const {
4739
169
  uint64_t Size;
4740
4741
  // -msvr4-struct-return puts small aggregates in GPR3 and GPR4.
4742
169
  if (isAggregateTypeForABI(RetTy) && 
IsRetSmallStructInRegABI21
&&
4743
169
      
(Size = getContext().getTypeSize(RetTy)) <= 6418
) {
4744
    // System V ABI (1995), page 3-22, specified:
4745
    // > A structure or union whose size is less than or equal to 8 bytes
4746
    // > shall be returned in r3 and r4, as if it were first stored in the
4747
    // > 8-byte aligned memory area and then the low addressed word were
4748
    // > loaded into r3 and the high-addressed word into r4.  Bits beyond
4749
    // > the last member of the structure or union are not defined.
4750
    //
4751
    // GCC for big-endian PPC32 inserts the pad before the first member,
4752
    // not "beyond the last member" of the struct.  To stay compatible
4753
    // with GCC, we coerce the struct to an integer of the same size.
4754
    // LLVM will extend it and return i32 in r3, or i64 in r3:r4.
4755
7
    if (Size == 0)
4756
1
      return ABIArgInfo::getIgnore();
4757
6
    else {
4758
6
      llvm::Type *CoerceTy = llvm::Type::getIntNTy(getVMContext(), Size);
4759
6
      return ABIArgInfo::getDirect(CoerceTy);
4760
6
    }
4761
7
  }
4762
4763
162
  return DefaultABIInfo::classifyReturnType(RetTy);
4764
169
}
4765
4766
// TODO: this implementation is now likely redundant with
4767
// DefaultABIInfo::EmitVAArg.
4768
Address PPC32_SVR4_ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAList,
4769
1
                                      QualType Ty) const {
4770
1
  if (getTarget().getTriple().isOSDarwin()) {
4771
0
    auto TI = getContext().getTypeInfoInChars(Ty);
4772
0
    TI.Align = getParamTypeAlignment(Ty);
4773
4774
0
    CharUnits SlotSize = CharUnits::fromQuantity(4);
4775
0
    return emitVoidPtrVAArg(CGF, VAList, Ty,
4776
0
                            classifyArgumentType(Ty).isIndirect(), TI, SlotSize,
4777
0
                            /*AllowHigherAlign=*/true);
4778
0
  }
4779
4780
1
  const unsigned OverflowLimit = 8;
4781
1
  if (const ComplexType *CTy = Ty->getAs<ComplexType>()) {
4782
    // TODO: Implement this. For now ignore.
4783
0
    (void)CTy;
4784
0
    return Address::invalid(); // FIXME?
4785
0
  }
4786
4787
  // struct __va_list_tag {
4788
  //   unsigned char gpr;
4789
  //   unsigned char fpr;
4790
  //   unsigned short reserved;
4791
  //   void *overflow_arg_area;
4792
  //   void *reg_save_area;
4793
  // };
4794
4795
1
  bool isI64 = Ty->isIntegerType() && 
getContext().getTypeSize(Ty) == 640
;
4796
1
  bool isInt = !Ty->isFloatingType();
4797
1
  bool isF64 = Ty->isFloatingType() && getContext().getTypeSize(Ty) == 64;
4798
4799
  // All aggregates are passed indirectly?  That doesn't seem consistent
4800
  // with the argument-lowering code.
4801
1
  bool isIndirect = isAggregateTypeForABI(Ty);
4802
4803
1
  CGBuilderTy &Builder = CGF.Builder;
4804
4805
  // The calling convention either uses 1-2 GPRs or 1 FPR.
4806
1
  Address NumRegsAddr = Address::invalid();
4807
1
  if (isInt || IsSoftFloatABI) {
4808
1
    NumRegsAddr = Builder.CreateStructGEP(VAList, 0, "gpr");
4809
1
  } else {
4810
0
    NumRegsAddr = Builder.CreateStructGEP(VAList, 1, "fpr");
4811
0
  }
4812
4813
1
  llvm::Value *NumRegs = Builder.CreateLoad(NumRegsAddr, "numUsedRegs");
4814
4815
  // "Align" the register count when TY is i64.
4816
1
  if (isI64 || (isF64 && IsSoftFloatABI)) {
4817
1
    NumRegs = Builder.CreateAdd(NumRegs, Builder.getInt8(1));
4818
1
    NumRegs = Builder.CreateAnd(NumRegs, Builder.getInt8((uint8_t) ~1U));
4819
1
  }
4820
4821
1
  llvm::Value *CC =
4822
1
      Builder.CreateICmpULT(NumRegs, Builder.getInt8(OverflowLimit), "cond");
4823
4824
1
  llvm::BasicBlock *UsingRegs = CGF.createBasicBlock("using_regs");
4825
1
  llvm::BasicBlock *UsingOverflow = CGF.createBasicBlock("using_overflow");
4826
1
  llvm::BasicBlock *Cont = CGF.createBasicBlock("cont");
4827
4828
1
  Builder.CreateCondBr(CC, UsingRegs, UsingOverflow);
4829
4830
1
  llvm::Type *DirectTy = CGF.ConvertType(Ty);
4831
1
  if (isIndirect) 
DirectTy = DirectTy->getPointerTo(0)0
;
4832
4833
  // Case 1: consume registers.
4834
1
  Address RegAddr = Address::invalid();
4835
1
  {
4836
1
    CGF.EmitBlock(UsingRegs);
4837
4838
1
    Address RegSaveAreaPtr = Builder.CreateStructGEP(VAList, 4);
4839
1
    RegAddr = Address(Builder.CreateLoad(RegSaveAreaPtr),
4840
1
                      CharUnits::fromQuantity(8));
4841
1
    assert(RegAddr.getElementType() == CGF.Int8Ty);
4842
4843
    // Floating-point registers start after the general-purpose registers.
4844
1
    if (!(isInt || IsSoftFloatABI)) {
4845
0
      RegAddr = Builder.CreateConstInBoundsByteGEP(RegAddr,
4846
0
                                                   CharUnits::fromQuantity(32));
4847
0
    }
4848
4849
    // Get the address of the saved value by scaling the number of
4850
    // registers we've used by the number of
4851
1
    CharUnits RegSize = CharUnits::fromQuantity((isInt || IsSoftFloatABI) ? 4 : 
80
);
4852
1
    llvm::Value *RegOffset =
4853
1
      Builder.CreateMul(NumRegs, Builder.getInt8(RegSize.getQuantity()));
4854
1
    RegAddr = Address(Builder.CreateInBoundsGEP(CGF.Int8Ty,
4855
1
                                            RegAddr.getPointer(), RegOffset),
4856
1
                      RegAddr.getAlignment().alignmentOfArrayElement(RegSize));
4857
1
    RegAddr = Builder.CreateElementBitCast(RegAddr, DirectTy);
4858
4859
    // Increase the used-register count.
4860
1
    NumRegs =
4861
1
      Builder.CreateAdd(NumRegs,
4862
1
                        Builder.getInt8((isI64 || (isF64 && IsSoftFloatABI)) ? 2 : 
10
));
4863
1
    Builder.CreateStore(NumRegs, NumRegsAddr);
4864
4865
1
    CGF.EmitBranch(Cont);
4866
1
  }
4867
4868
  // Case 2: consume space in the overflow area.
4869
0
  Address MemAddr = Address::invalid();
4870
1
  {
4871
1
    CGF.EmitBlock(UsingOverflow);
4872
4873
1
    Builder.CreateStore(Builder.getInt8(OverflowLimit), NumRegsAddr);
4874
4875
    // Everything in the overflow area is rounded up to a size of at least 4.
4876
1
    CharUnits OverflowAreaAlign = CharUnits::fromQuantity(4);
4877
4878
1
    CharUnits Size;
4879
1
    if (!isIndirect) {
4880
1
      auto TypeInfo = CGF.getContext().getTypeInfoInChars(Ty);
4881
1
      Size = TypeInfo.Width.alignTo(OverflowAreaAlign);
4882
1
    } else {
4883
0
      Size = CGF.getPointerSize();
4884
0
    }
4885
4886
1
    Address OverflowAreaAddr = Builder.CreateStructGEP(VAList, 3);
4887
1
    Address OverflowArea(Builder.CreateLoad(OverflowAreaAddr, "argp.cur"),
4888
1
                         OverflowAreaAlign);
4889
    // Round up address of argument to alignment
4890
1
    CharUnits Align = CGF.getContext().getTypeAlignInChars(Ty);
4891
1
    if (Align > OverflowAreaAlign) {
4892
1
      llvm::Value *Ptr = OverflowArea.getPointer();
4893
1
      OverflowArea = Address(emitRoundPointerUpToAlignment(CGF, Ptr, Align),
4894
1
                                                           Align);
4895
1
    }
4896
4897
1
    MemAddr = Builder.CreateElementBitCast(OverflowArea, DirectTy);
4898
4899
    // Increase the overflow area.
4900
1
    OverflowArea = Builder.CreateConstInBoundsByteGEP(OverflowArea, Size);
4901
1
    Builder.CreateStore(OverflowArea.getPointer(), OverflowAreaAddr);
4902
1
    CGF.EmitBranch(Cont);
4903
1
  }
4904
4905
1
  CGF.EmitBlock(Cont);
4906
4907
  // Merge the cases with a phi.
4908
1
  Address Result = emitMergePHI(CGF, RegAddr, UsingRegs, MemAddr, UsingOverflow,
4909
1
                                "vaarg.addr");
4910
4911
  // Load the pointer if the argument was passed indirectly.
4912
1
  if (isIndirect) {
4913
0
    Result = Address(Builder.CreateLoad(Result, "aggr"),
4914
0
                     getContext().getTypeAlignInChars(Ty));
4915
0
  }
4916
4917
1
  return Result;
4918
1
}
4919
4920
bool PPC32TargetCodeGenInfo::isStructReturnInRegABI(
4921
64
    const llvm::Triple &Triple, const CodeGenOptions &Opts) {
4922
64
  assert(Triple.isPPC32());
4923
4924
0
  switch (Opts.getStructReturnConvention()) {
4925
64
  case CodeGenOptions::SRCK_Default:
4926
64
    break;
4927
0
  case CodeGenOptions::SRCK_OnStack: // -maix-struct-return
4928
0
    return false;
4929
0
  case CodeGenOptions::SRCK_InRegs: // -msvr4-struct-return
4930
0
    return true;
4931
64
  }
4932
4933
64
  if (Triple.isOSBinFormatELF() && !Triple.isOSLinux())
4934
55
    return true;
4935
4936
9
  return false;
4937
64
}
4938
4939
bool
4940
PPC32TargetCodeGenInfo::initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
4941
1
                                                llvm::Value *Address) const {
4942
1
  return PPC_initDwarfEHRegSizeTable(CGF, Address, /*Is64Bit*/ false,
4943
1
                                     /*IsAIX*/ false);
4944
1
}
4945
4946
// PowerPC-64
4947
4948
namespace {
4949
/// PPC64_SVR4_ABIInfo - The 64-bit PowerPC ELF (SVR4) ABI information.
4950
class PPC64_SVR4_ABIInfo : public SwiftABIInfo {
4951
public:
4952
  enum ABIKind {
4953
    ELFv1 = 0,
4954
    ELFv2
4955
  };
4956
4957
private:
4958
  static const unsigned GPRBits = 64;
4959
  ABIKind Kind;
4960
  bool IsSoftFloatABI;
4961
4962
public:
4963
  PPC64_SVR4_ABIInfo(CodeGen::CodeGenTypes &CGT, ABIKind Kind,
4964
                     bool SoftFloatABI)
4965
2.52k
      : SwiftABIInfo(CGT), Kind(Kind), IsSoftFloatABI(SoftFloatABI) {}
4966
4967
  bool isPromotableTypeForABI(QualType Ty) const;
4968
  CharUnits getParamTypeAlignment(QualType Ty) const;
4969
4970
  ABIArgInfo classifyReturnType(QualType RetTy) const;
4971
  ABIArgInfo classifyArgumentType(QualType Ty) const;
4972
4973
  bool isHomogeneousAggregateBaseType(QualType Ty) const override;
4974
  bool isHomogeneousAggregateSmallEnough(const Type *Ty,
4975
                                         uint64_t Members) const override;
4976
4977
  // TODO: We can add more logic to computeInfo to improve performance.
4978
  // Example: For aggregate arguments that fit in a register, we could
4979
  // use getDirectInReg (as is done below for structs containing a single
4980
  // floating-point value) to avoid pushing them to memory on function
4981
  // entry.  This would require changing the logic in PPCISelLowering
4982
  // when lowering the parameters in the caller and args in the callee.
4983
13.5k
  void computeInfo(CGFunctionInfo &FI) const override {
4984
13.5k
    if (!getCXXABI().classifyReturnType(FI))
4985
13.5k
      FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
4986
26.7k
    for (auto &I : FI.arguments()) {
4987
      // We rely on the default argument classification for the most part.
4988
      // One exception:  An aggregate containing a single floating-point
4989
      // or vector item must be passed in a register if one is available.
4990
26.7k
      const Type *T = isSingleElementStruct(I.type, getContext());
4991
26.7k
      if (T) {
4992
23
        const BuiltinType *BT = T->getAs<BuiltinType>();
4993
23
        if ((T->isVectorType() && 
getContext().getTypeSize(T) == 1283
) ||
4994
23
            
(21
BT21
&&
BT->isFloatingPoint()18
)) {
4995
19
          QualType QT(T, 0);
4996
19
          I.info = ABIArgInfo::getDirectInReg(CGT.ConvertType(QT));
4997
19
          continue;
4998
19
        }
4999
23
      }
5000
26.7k
      I.info = classifyArgumentType(I.type);
5001
26.7k
    }
5002
13.5k
  }
5003
5004
  Address EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
5005
                    QualType Ty) const override;
5006
5007
  bool shouldPassIndirectlyForSwift(ArrayRef<llvm::Type*> scalars,
5008
0
                                    bool asReturnValue) const override {
5009
0
    return occupiesMoreThan(CGT, scalars, /*total*/ 4);
5010
0
  }
5011
5012
0
  bool isSwiftErrorInRegister() const override {
5013
0
    return false;
5014
0
  }
5015
};
5016
5017
class PPC64_SVR4_TargetCodeGenInfo : public TargetCodeGenInfo {
5018
5019
public:
5020
  PPC64_SVR4_TargetCodeGenInfo(CodeGenTypes &CGT,
5021
                               PPC64_SVR4_ABIInfo::ABIKind Kind,
5022
                               bool SoftFloatABI)
5023
      : TargetCodeGenInfo(
5024
2.52k
            std::make_unique<PPC64_SVR4_ABIInfo>(CGT, Kind, SoftFloatABI)) {}
5025
5026
1
  int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
5027
    // This is recovered from gcc output.
5028
1
    return 1; // r1 is the dedicated stack pointer
5029
1
  }
5030
5031
  bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
5032
                               llvm::Value *Address) const override;
5033
};
5034
5035
class PPC64TargetCodeGenInfo : public DefaultTargetCodeGenInfo {
5036
public:
5037
0
  PPC64TargetCodeGenInfo(CodeGenTypes &CGT) : DefaultTargetCodeGenInfo(CGT) {}
5038
5039
0
  int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override {
5040
    // This is recovered from gcc output.
5041
0
    return 1; // r1 is the dedicated stack pointer
5042
0
  }
5043
5044
  bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
5045
                               llvm::Value *Address) const override;
5046
};
5047
5048
}
5049
5050
// Return true if the ABI requires Ty to be passed sign- or zero-
5051
// extended to 64 bits.
5052
bool
5053
29.4k
PPC64_SVR4_ABIInfo::isPromotableTypeForABI(QualType Ty) const {
5054
  // Treat an enum type as its underlying type.
5055
29.4k
  if (const EnumType *EnumTy = Ty->getAs<EnumType>())
5056
0
    Ty = EnumTy->getDecl()->getIntegerType();
5057
5058
  // Promotable integer types are required to be promoted by the ABI.
5059
29.4k
  if (isPromotableIntegerTypeForABI(Ty))
5060
60
    return true;
5061
5062
  // In addition to the usual promotable integer types, we also need to
5063
  // extend all 32-bit types, since the ABI requires promotion to 64 bits.
5064
29.3k
  if (const BuiltinType *BT = Ty->getAs<BuiltinType>())
5065
12.5k
    switch (BT->getKind()) {
5066
5.22k
    case BuiltinType::Int:
5067
5.24k
    case BuiltinType::UInt:
5068
5.24k
      return true;
5069
7.27k
    default:
5070
7.27k
      break;
5071
12.5k
    }
5072
5073
24.1k
  if (const auto *EIT = Ty->getAs<BitIntType>())
5074
9
    if (EIT->getNumBits() < 64)
5075
2
      return true;
5076
5077
24.1k
  return false;
5078
24.1k
}
5079
5080
/// isAlignedParamType - Determine whether a type requires 16-byte or
5081
/// higher alignment in the parameter area.  Always returns at least 8.
5082
124
CharUnits PPC64_SVR4_ABIInfo::getParamTypeAlignment(QualType Ty) const {
5083
  // Complex types are passed just like their elements.
5084
124
  if (const ComplexType *CTy = Ty->getAs<ComplexType>())
5085
0
    Ty = CTy->getElementType();
5086
5087
153
  auto FloatUsesVector = [this](QualType Ty){
5088
153
    return Ty->isRealFloatingType() && &getContext().getFloatTypeSemantics(
5089
44
                                           Ty) == &llvm::APFloat::IEEEquad();
5090
153
  };
5091
5092
  // Only vector types of size 16 bytes need alignment (larger types are
5093
  // passed via reference, smaller types are not aligned).
5094
124
  if (Ty->isVectorType()) {
5095
0
    return CharUnits::fromQuantity(getContext().getTypeSize(Ty) == 128 ? 16 : 8);
5096
124
  } else if (FloatUsesVector(Ty)) {
5097
    // According to ABI document section 'Optional Save Areas': If extended
5098
    // precision floating-point values in IEEE BINARY 128 QUADRUPLE PRECISION
5099
    // format are supported, map them to a single quadword, quadword aligned.
5100
9
    return CharUnits::fromQuantity(16);
5101
9
  }
5102
5103
  // For single-element float/vector structs, we consider the whole type
5104
  // to have the same alignment requirements as its single element.
5105
115
  const Type *AlignAsType = nullptr;
5106
115
  const Type *EltType = isSingleElementStruct(Ty, getContext());
5107
115
  if (EltType) {
5108
6
    const BuiltinType *BT = EltType->getAs<BuiltinType>();
5109
6
    if ((EltType->isVectorType() && 
getContext().getTypeSize(EltType) == 1282
) ||
5110
6
        
(5
BT5
&&
BT->isFloatingPoint()4
))
5111
5
      AlignAsType = EltType;
5112
6
  }
5113
5114
  // Likewise for ELFv2 homogeneous aggregates.
5115
115
  const Type *Base = nullptr;
5116
115
  uint64_t Members = 0;
5117
115
  if (!AlignAsType && 
Kind == ELFv2110
&&
5118
115
      
isAggregateTypeForABI(Ty)82
&&
isHomogeneousAggregate(Ty, Base, Members)77
)
5119
43
    AlignAsType = Base;
5120
5121
  // With special case aggregates, only vector base types need alignment.
5122
115
  if (AlignAsType) {
5123
48
    bool UsesVector = AlignAsType->isVectorType() ||
5124
48
                      
FloatUsesVector(QualType(AlignAsType, 0))29
;
5125
48
    return CharUnits::fromQuantity(UsesVector ? 
1631
:
817
);
5126
48
  }
5127
5128
  // Otherwise, we only need alignment for any aggregate type that
5129
  // has an alignment requirement of >= 16 bytes.
5130
67
  if (isAggregateTypeForABI(Ty) && 
getContext().getTypeAlign(Ty) >= 12861
) {
5131
14
    return CharUnits::fromQuantity(16);
5132
14
  }
5133
5134
53
  return CharUnits::fromQuantity(8);
5135
67
}
5136
5137
/// isHomogeneousAggregate - Return true if a type is an ELFv2 homogeneous
5138
/// aggregate.  Base is set to the base element type, and Members is set
5139
/// to the number of base elements.
5140
bool ABIInfo::isHomogeneousAggregate(QualType Ty, const Type *&Base,
5141
6.18k
                                     uint64_t &Members) const {
5142
6.18k
  if (const ConstantArrayType *AT = getContext().getAsConstantArrayType(Ty)) {
5143
1.14k
    uint64_t NElements = AT->getSize().getZExtValue();
5144
1.14k
    if (NElements == 0)
5145
0
      return false;
5146
1.14k
    if (!isHomogeneousAggregate(AT->getElementType(), Base, Members))
5147
157
      return false;
5148
984
    Members *= NElements;
5149
5.03k
  } else if (const RecordType *RT = Ty->getAs<RecordType>()) {
5150
2.08k
    const RecordDecl *RD = RT->getDecl();
5151
2.08k
    if (RD->hasFlexibleArrayMember())
5152
2
      return false;
5153
5154
2.07k
    Members = 0;
5155
5156
    // If this is a C++ record, check the properties of the record such as
5157
    // bases and ABI specific restricti