Coverage Report

Created: 2020-02-25 14:32

/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/CodeGen/CGCall.cpp
Line
Count
Source (jump to first uncovered line)
1
//===--- CGCall.cpp - Encapsulate calling convention details --------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// These classes wrap the information about a call or function
10
// definition used to handle ABI compliancy.
11
//
12
//===----------------------------------------------------------------------===//
13
14
#include "CGCall.h"
15
#include "ABIInfo.h"
16
#include "CGBlocks.h"
17
#include "CGCXXABI.h"
18
#include "CGCleanup.h"
19
#include "CodeGenFunction.h"
20
#include "CodeGenModule.h"
21
#include "TargetInfo.h"
22
#include "clang/AST/Attr.h"
23
#include "clang/AST/Decl.h"
24
#include "clang/AST/DeclCXX.h"
25
#include "clang/AST/DeclObjC.h"
26
#include "clang/Basic/CodeGenOptions.h"
27
#include "clang/Basic/TargetBuiltins.h"
28
#include "clang/Basic/TargetInfo.h"
29
#include "clang/CodeGen/CGFunctionInfo.h"
30
#include "clang/CodeGen/SwiftCallingConv.h"
31
#include "llvm/ADT/StringExtras.h"
32
#include "llvm/Analysis/ValueTracking.h"
33
#include "llvm/IR/Attributes.h"
34
#include "llvm/IR/CallingConv.h"
35
#include "llvm/IR/DataLayout.h"
36
#include "llvm/IR/InlineAsm.h"
37
#include "llvm/IR/IntrinsicInst.h"
38
#include "llvm/IR/Intrinsics.h"
39
#include "llvm/Transforms/Utils/Local.h"
40
using namespace clang;
41
using namespace CodeGen;
42
43
/***/
44
45
180k
unsigned CodeGenTypes::ClangCallConvToLLVMCallConv(CallingConv CC) {
46
180k
  switch (CC) {
47
175k
  default: return llvm::CallingConv::C;
48
51
  case CC_X86StdCall: return llvm::CallingConv::X86_StdCall;
49
70
  case CC_X86FastCall: return llvm::CallingConv::X86_FastCall;
50
95
  case CC_X86RegCall: return llvm::CallingConv::X86_RegCall;
51
2.58k
  case CC_X86ThisCall: return llvm::CallingConv::X86_ThisCall;
52
14
  case CC_Win64: return llvm::CallingConv::Win64;
53
14
  case CC_X86_64SysV: return llvm::CallingConv::X86_64_SysV;
54
5
  case CC_AAPCS: return llvm::CallingConv::ARM_AAPCS;
55
11
  case CC_AAPCS_VFP: return llvm::CallingConv::ARM_AAPCS_VFP;
56
3
  case CC_IntelOclBicc: return llvm::CallingConv::Intel_OCL_BI;
57
0
  // TODO: Add support for __pascal to LLVM.
58
3
  case CC_X86Pascal: return llvm::CallingConv::C;
59
0
  // TODO: Add support for __vectorcall to LLVM.
60
102
  case CC_X86VectorCall: return llvm::CallingConv::X86_VectorCall;
61
2
  case CC_AArch64VectorCall: return llvm::CallingConv::AArch64_VectorCall;
62
247
  case CC_SpirFunction: return llvm::CallingConv::SPIR_FUNC;
63
307
  case CC_OpenCLKernel: return CGM.getTargetCodeGenInfo().getOpenCLKernelCallingConv();
64
10
  case CC_PreserveMost: return llvm::CallingConv::PreserveMost;
65
6
  case CC_PreserveAll: return llvm::CallingConv::PreserveAll;
66
1.12k
  case CC_Swift: return llvm::CallingConv::Swift;
67
180k
  }
68
180k
}
69
70
/// Derives the 'this' type for codegen purposes, i.e. ignoring method CVR
71
/// qualification. Either or both of RD and MD may be null. A null RD indicates
72
/// that there is no meaningful 'this' type, and a null MD can occur when
73
/// calling a method pointer.
74
CanQualType CodeGenTypes::DeriveThisType(const CXXRecordDecl *RD,
75
532k
                                         const CXXMethodDecl *MD) {
76
532k
  QualType RecTy;
77
532k
  if (RD)
78
531k
    RecTy = Context.getTagDeclType(RD)->getCanonicalTypeInternal();
79
680
  else
80
680
    RecTy = Context.VoidTy;
81
532k
82
532k
  if (MD)
83
532k
    RecTy = Context.getAddrSpaceQualType(RecTy, MD->getMethodQualifiers().getAddressSpace());
84
532k
  return Context.getPointerType(CanQualType::CreateUnsafe(RecTy));
85
532k
}
86
87
/// Returns the canonical formal type of the given C++ method.
88
497k
static CanQual<FunctionProtoType> GetFormalType(const CXXMethodDecl *MD) {
89
497k
  return MD->getType()->getCanonicalTypeUnqualified()
90
497k
           .getAs<FunctionProtoType>();
91
497k
}
92
93
/// Returns the "extra-canonicalized" return type, which discards
94
/// qualifiers on the return type.  Codegen doesn't care about them,
95
/// and it makes ABI code a little easier to be able to assume that
96
/// all parameter and return types are top-level unqualified.
97
258k
static CanQualType GetReturnType(QualType RetTy) {
98
258k
  return RetTy->getCanonicalTypeUnqualified().getUnqualifiedType();
99
258k
}
100
101
/// Arrange the argument and result information for a value of the given
102
/// unprototyped freestanding function type.
103
const CGFunctionInfo &
104
577
CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionNoProtoType> FTNP) {
105
577
  // When translating an unprototyped function type, always use a
106
577
  // variadic type.
107
577
  return arrangeLLVMFunctionInfo(FTNP->getReturnType().getUnqualifiedType(),
108
577
                                 /*instanceMethod=*/false,
109
577
                                 /*chainCall=*/false, None,
110
577
                                 FTNP->getExtInfo(), {}, RequiredArgs(0));
111
577
}
112
113
static void addExtParameterInfosForCall(
114
         llvm::SmallVectorImpl<FunctionProtoType::ExtParameterInfo> &paramInfos,
115
                                        const FunctionProtoType *proto,
116
                                        unsigned prefixArgs,
117
701
                                        unsigned totalArgs) {
118
701
  assert(proto->hasExtParameterInfos());
119
701
  assert(paramInfos.size() <= prefixArgs);
120
701
  assert(proto->getNumParams() + prefixArgs <= totalArgs);
121
701
122
701
  paramInfos.reserve(totalArgs);
123
701
124
701
  // Add default infos for any prefix args that don't already have infos.
125
701
  paramInfos.resize(prefixArgs);
126
701
127
701
  // Add infos for the prototype.
128
1.33k
  for (const auto &ParamInfo : proto->getExtParameterInfos()) {
129
1.33k
    paramInfos.push_back(ParamInfo);
130
1.33k
    // pass_object_size params have no parameter info.
131
1.33k
    if (ParamInfo.hasPassObjectSize())
132
447
      paramInfos.emplace_back();
133
1.33k
  }
134
701
135
701
  assert(paramInfos.size() <= totalArgs &&
136
701
         "Did we forget to insert pass_object_size args?");
137
701
  // Add default infos for the variadic and/or suffix arguments.
138
701
  paramInfos.resize(totalArgs);
139
701
}
140
141
/// Adds the formal parameters in FPT to the given prefix. If any parameter in
142
/// FPT has pass_object_size attrs, then we'll add parameters for those, too.
143
static void appendParameterTypes(const CodeGenTypes &CGT,
144
                                 SmallVectorImpl<CanQualType> &prefix,
145
              SmallVectorImpl<FunctionProtoType::ExtParameterInfo> &paramInfos,
146
807k
                                 CanQual<FunctionProtoType> FPT) {
147
807k
  // Fast path: don't touch param info if we don't need to.
148
807k
  if (!FPT->hasExtParameterInfos()) {
149
807k
    assert(paramInfos.empty() &&
150
807k
           "We have paramInfos, but the prototype doesn't?");
151
807k
    prefix.append(FPT->param_type_begin(), FPT->param_type_end());
152
807k
    return;
153
807k
  }
154
497
155
497
  unsigned PrefixSize = prefix.size();
156
497
  // In the vast majority of cases, we'll have precisely FPT->getNumParams()
157
497
  // parameters; the only thing that can change this is the presence of
158
497
  // pass_object_size. So, we preallocate for the common case.
159
497
  prefix.reserve(prefix.size() + FPT->getNumParams());
160
497
161
497
  auto ExtInfos = FPT->getExtParameterInfos();
162
497
  assert(ExtInfos.size() == FPT->getNumParams());
163
1.47k
  for (unsigned I = 0, E = FPT->getNumParams(); I != E; 
++I974
) {
164
974
    prefix.push_back(FPT->getParamType(I));
165
974
    if (ExtInfos[I].hasPassObjectSize())
166
299
      prefix.push_back(CGT.getContext().getSizeType());
167
974
  }
168
497
169
497
  addExtParameterInfosForCall(paramInfos, FPT.getTypePtr(), PrefixSize,
170
497
                              prefix.size());
171
497
}
172
173
/// Arrange the LLVM function layout for a value of the given function
174
/// type, on top of any implicit parameters already stored.
175
static const CGFunctionInfo &
176
arrangeLLVMFunctionInfo(CodeGenTypes &CGT, bool instanceMethod,
177
                        SmallVectorImpl<CanQualType> &prefix,
178
553k
                        CanQual<FunctionProtoType> FTP) {
179
553k
  SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos;
180
553k
  RequiredArgs Required = RequiredArgs::forPrototypePlus(FTP, prefix.size());
181
553k
  // FIXME: Kill copy.
182
553k
  appendParameterTypes(CGT, prefix, paramInfos, FTP);
183
553k
  CanQualType resultType = FTP->getReturnType().getUnqualifiedType();
184
553k
185
553k
  return CGT.arrangeLLVMFunctionInfo(resultType, instanceMethod,
186
553k
                                     /*chainCall=*/false, prefix,
187
553k
                                     FTP->getExtInfo(), paramInfos,
188
553k
                                     Required);
189
553k
}
190
191
/// Arrange the argument and result information for a value of the
192
/// given freestanding function type.
193
const CGFunctionInfo &
194
364k
CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionProtoType> FTP) {
195
364k
  SmallVector<CanQualType, 16> argTypes;
196
364k
  return ::arrangeLLVMFunctionInfo(*this, /*instanceMethod=*/false, argTypes,
197
364k
                                   FTP);
198
364k
}
199
200
16.6k
static CallingConv getCallingConventionForDecl(const Decl *D, bool IsWindows) {
201
16.6k
  // Set the appropriate calling convention for the Function.
202
16.6k
  if (D->hasAttr<StdCallAttr>())
203
2
    return CC_X86StdCall;
204
16.6k
205
16.6k
  if (D->hasAttr<FastCallAttr>())
206
0
    return CC_X86FastCall;
207
16.6k
208
16.6k
  if (D->hasAttr<RegCallAttr>())
209
0
    return CC_X86RegCall;
210
16.6k
211
16.6k
  if (D->hasAttr<ThisCallAttr>())
212
0
    return CC_X86ThisCall;
213
16.6k
214
16.6k
  if (D->hasAttr<VectorCallAttr>())
215
0
    return CC_X86VectorCall;
216
16.6k
217
16.6k
  if (D->hasAttr<PascalAttr>())
218
0
    return CC_X86Pascal;
219
16.6k
220
16.6k
  if (PcsAttr *PCS = D->getAttr<PcsAttr>())
221
0
    return (PCS->getPCS() == PcsAttr::AAPCS ? CC_AAPCS : CC_AAPCS_VFP);
222
16.6k
223
16.6k
  if (D->hasAttr<AArch64VectorPcsAttr>())
224
0
    return CC_AArch64VectorCall;
225
16.6k
226
16.6k
  if (D->hasAttr<IntelOclBiccAttr>())
227
0
    return CC_IntelOclBicc;
228
16.6k
229
16.6k
  if (D->hasAttr<MSABIAttr>())
230
2
    return IsWindows ? 
CC_C0
: CC_Win64;
231
16.6k
232
16.6k
  if (D->hasAttr<SysVABIAttr>())
233
0
    return IsWindows ? CC_X86_64SysV : CC_C;
234
16.6k
235
16.6k
  if (D->hasAttr<PreserveMostAttr>())
236
0
    return CC_PreserveMost;
237
16.6k
238
16.6k
  if (D->hasAttr<PreserveAllAttr>())
239
0
    return CC_PreserveAll;
240
16.6k
241
16.6k
  return CC_C;
242
16.6k
}
243
244
/// Arrange the argument and result information for a call to an
245
/// unknown C++ non-static member function of the given abstract type.
246
/// (A null RD means we don't have any meaningful "this" argument type,
247
///  so fall back to a generic pointer type).
248
/// The member function must be an ordinary function, i.e. not a
249
/// constructor or destructor.
250
const CGFunctionInfo &
251
CodeGenTypes::arrangeCXXMethodType(const CXXRecordDecl *RD,
252
                                   const FunctionProtoType *FTP,
253
189k
                                   const CXXMethodDecl *MD) {
254
189k
  SmallVector<CanQualType, 16> argTypes;
255
189k
256
189k
  // Add the 'this' pointer.
257
189k
  argTypes.push_back(DeriveThisType(RD, MD));
258
189k
259
189k
  return ::arrangeLLVMFunctionInfo(
260
189k
      *this, true, argTypes,
261
189k
      FTP->getCanonicalTypeUnqualified().getAs<FunctionProtoType>());
262
189k
}
263
264
/// Set calling convention for CUDA/HIP kernel.
265
static void setCUDAKernelCallingConvention(CanQualType &FTy, CodeGenModule &CGM,
266
515k
                                           const FunctionDecl *FD) {
267
515k
  if (FD->hasAttr<CUDAGlobalAttr>()) {
268
206
    const FunctionType *FT = FTy->getAs<FunctionType>();
269
206
    CGM.getTargetCodeGenInfo().setCUDAKernelCallingConvention(FT);
270
206
    FTy = FT->getCanonicalTypeUnqualified();
271
206
  }
272
515k
}
273
274
/// Arrange the argument and result information for a declaration or
275
/// definition of the given C++ non-static member function.  The
276
/// member function must be an ordinary function, i.e. not a
277
/// constructor or destructor.
278
const CGFunctionInfo &
279
196k
CodeGenTypes::arrangeCXXMethodDeclaration(const CXXMethodDecl *MD) {
280
196k
  assert(!isa<CXXConstructorDecl>(MD) && "wrong method for constructors!");
281
196k
  assert(!isa<CXXDestructorDecl>(MD) && "wrong method for destructors!");
282
196k
283
196k
  CanQualType FT = GetFormalType(MD).getAs<Type>();
284
196k
  setCUDAKernelCallingConvention(FT, CGM, MD);
285
196k
  auto prototype = FT.getAs<FunctionProtoType>();
286
196k
287
196k
  if (MD->isInstance()) {
288
189k
    // The abstract case is perfectly fine.
289
189k
    const CXXRecordDecl *ThisType = TheCXXABI.getThisArgumentTypeForMethod(MD);
290
189k
    return arrangeCXXMethodType(ThisType, prototype.getTypePtr(), MD);
291
189k
  }
292
6.96k
293
6.96k
  return arrangeFreeFunctionType(prototype);
294
6.96k
}
295
296
bool CodeGenTypes::inheritingCtorHasParams(
297
1.18k
    const InheritedConstructor &Inherited, CXXCtorType Type) {
298
1.18k
  // Parameters are unnecessary if we're constructing a base class subobject
299
1.18k
  // and the inherited constructor lives in a virtual base.
300
1.18k
  return Type == Ctor_Complete ||
301
1.18k
         
!Inherited.getShadowDecl()->constructsVirtualBase()798
||
302
1.18k
         
!Target.getCXXABI().hasConstructorVariants()74
;
303
1.18k
}
304
305
const CGFunctionInfo &
306
254k
CodeGenTypes::arrangeCXXStructorDeclaration(GlobalDecl GD) {
307
254k
  auto *MD = cast<CXXMethodDecl>(GD.getDecl());
308
254k
309
254k
  SmallVector<CanQualType, 16> argTypes;
310
254k
  SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos;
311
254k
  argTypes.push_back(DeriveThisType(MD->getParent(), MD));
312
254k
313
254k
  bool PassParams = true;
314
254k
315
254k
  if (auto *CD = dyn_cast<CXXConstructorDecl>(MD)) {
316
153k
    // A base class inheriting constructor doesn't get forwarded arguments
317
153k
    // needed to construct a virtual base (or base class thereof).
318
153k
    if (auto Inherited = CD->getInheritedConstructor())
319
722
      PassParams = inheritingCtorHasParams(Inherited, GD.getCtorType());
320
153k
  }
321
254k
322
254k
  CanQual<FunctionProtoType> FTP = GetFormalType(MD);
323
254k
324
254k
  // Add the formal parameters.
325
254k
  if (PassParams)
326
254k
    appendParameterTypes(*this, argTypes, paramInfos, FTP);
327
254k
328
254k
  CGCXXABI::AddedStructorArgs AddedArgs =
329
254k
      TheCXXABI.buildStructorSignature(GD, argTypes);
330
254k
  if (!paramInfos.empty()) {
331
175
    // Note: prefix implies after the first param.
332
175
    if (AddedArgs.Prefix)
333
2
      paramInfos.insert(paramInfos.begin() + 1, AddedArgs.Prefix,
334
2
                        FunctionProtoType::ExtParameterInfo{});
335
175
    if (AddedArgs.Suffix)
336
20
      paramInfos.append(AddedArgs.Suffix,
337
20
                        FunctionProtoType::ExtParameterInfo{});
338
175
  }
339
254k
340
254k
  RequiredArgs required =
341
254k
      (PassParams && 
MD->isVariadic()254k
?
RequiredArgs(argTypes.size())91
342
254k
                                      : 
RequiredArgs::All254k
);
343
254k
344
254k
  FunctionType::ExtInfo extInfo = FTP->getExtInfo();
345
254k
  CanQualType resultType = TheCXXABI.HasThisReturn(GD)
346
254k
                               ? 
argTypes.front()7.38k
347
254k
                               : TheCXXABI.hasMostDerivedReturn(GD)
348
246k
                                     ? 
CGM.getContext().VoidPtrTy1.37k
349
246k
                                     : 
Context.VoidTy245k
;
350
254k
  return arrangeLLVMFunctionInfo(resultType, /*instanceMethod=*/true,
351
254k
                                 /*chainCall=*/false, argTypes, extInfo,
352
254k
                                 paramInfos, required);
353
254k
}
354
355
static SmallVector<CanQualType, 16>
356
65.5k
getArgTypesForCall(ASTContext &ctx, const CallArgList &args) {
357
65.5k
  SmallVector<CanQualType, 16> argTypes;
358
65.5k
  for (auto &arg : args)
359
93.6k
    argTypes.push_back(ctx.getCanonicalParamType(arg.Ty));
360
65.5k
  return argTypes;
361
65.5k
}
362
363
static SmallVector<CanQualType, 16>
364
20.4k
getArgTypesForDeclaration(ASTContext &ctx, const FunctionArgList &args) {
365
20.4k
  SmallVector<CanQualType, 16> argTypes;
366
20.4k
  for (auto &arg : args)
367
59.0k
    argTypes.push_back(ctx.getCanonicalParamType(arg->getType()));
368
20.4k
  return argTypes;
369
20.4k
}
370
371
static llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16>
372
getExtParameterInfosForCall(const FunctionProtoType *proto,
373
64.9k
                            unsigned prefixArgs, unsigned totalArgs) {
374
64.9k
  llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16> result;
375
64.9k
  if (proto->hasExtParameterInfos()) {
376
11
    addExtParameterInfosForCall(result, proto, prefixArgs, totalArgs);
377
11
  }
378
64.9k
  return result;
379
64.9k
}
380
381
/// Arrange a call to a C++ method, passing the given arguments.
382
///
383
/// ExtraPrefixArgs is the number of ABI-specific args passed after the `this`
384
/// parameter.
385
/// ExtraSuffixArgs is the number of ABI-specific args passed at the end of
386
/// args.
387
/// PassProtoArgs indicates whether `args` has args for the parameters in the
388
/// given CXXConstructorDecl.
389
const CGFunctionInfo &
390
CodeGenTypes::arrangeCXXConstructorCall(const CallArgList &args,
391
                                        const CXXConstructorDecl *D,
392
                                        CXXCtorType CtorKind,
393
                                        unsigned ExtraPrefixArgs,
394
                                        unsigned ExtraSuffixArgs,
395
47.0k
                                        bool PassProtoArgs) {
396
47.0k
  // FIXME: Kill copy.
397
47.0k
  SmallVector<CanQualType, 16> ArgTypes;
398
47.0k
  for (const auto &Arg : args)
399
95.0k
    ArgTypes.push_back(Context.getCanonicalParamType(Arg.Ty));
400
47.0k
401
47.0k
  // +1 for implicit this, which should always be args[0].
402
47.0k
  unsigned TotalPrefixArgs = 1 + ExtraPrefixArgs;
403
47.0k
404
47.0k
  CanQual<FunctionProtoType> FPT = GetFormalType(D);
405
47.0k
  RequiredArgs Required = PassProtoArgs
406
47.0k
                              ? RequiredArgs::forPrototypePlus(
407
47.0k
                                    FPT, TotalPrefixArgs + ExtraSuffixArgs)
408
47.0k
                              : 
RequiredArgs::All10
;
409
47.0k
410
47.0k
  GlobalDecl GD(D, CtorKind);
411
47.0k
  CanQualType ResultType = TheCXXABI.HasThisReturn(GD)
412
47.0k
                               ? 
ArgTypes.front()2.20k
413
47.0k
                               : TheCXXABI.hasMostDerivedReturn(GD)
414
44.8k
                                     ? 
CGM.getContext().VoidPtrTy0
415
44.8k
                                     : Context.VoidTy;
416
47.0k
417
47.0k
  FunctionType::ExtInfo Info = FPT->getExtInfo();
418
47.0k
  llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16> ParamInfos;
419
47.0k
  // If the prototype args are elided, we should only have ABI-specific args,
420
47.0k
  // which never have param info.
421
47.0k
  if (PassProtoArgs && 
FPT->hasExtParameterInfos()47.0k
) {
422
70
    // ABI-specific suffix arguments are treated the same as variadic arguments.
423
70
    addExtParameterInfosForCall(ParamInfos, FPT.getTypePtr(), TotalPrefixArgs,
424
70
                                ArgTypes.size());
425
70
  }
426
47.0k
  return arrangeLLVMFunctionInfo(ResultType, /*instanceMethod=*/true,
427
47.0k
                                 /*chainCall=*/false, ArgTypes, Info,
428
47.0k
                                 ParamInfos, Required);
429
47.0k
}
430
431
/// Arrange the argument and result information for the declaration or
432
/// definition of the given function.
433
const CGFunctionInfo &
434
405k
CodeGenTypes::arrangeFunctionDeclaration(const FunctionDecl *FD) {
435
405k
  if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(FD))
436
101k
    if (MD->isInstance())
437
86.2k
      return arrangeCXXMethodDeclaration(MD);
438
319k
439
319k
  CanQualType FTy = FD->getType()->getCanonicalTypeUnqualified();
440
319k
441
319k
  assert(isa<FunctionType>(FTy));
442
319k
  setCUDAKernelCallingConvention(FTy, CGM, FD);
443
319k
444
319k
  // When declaring a function without a prototype, always use a
445
319k
  // non-variadic type.
446
319k
  if (CanQual<FunctionNoProtoType> noProto = FTy.getAs<FunctionNoProtoType>()) {
447
11.5k
    return arrangeLLVMFunctionInfo(
448
11.5k
        noProto->getReturnType(), /*instanceMethod=*/false,
449
11.5k
        /*chainCall=*/false, None, noProto->getExtInfo(), {},RequiredArgs::All);
450
11.5k
  }
451
308k
452
308k
  return arrangeFreeFunctionType(FTy.castAs<FunctionProtoType>());
453
308k
}
454
455
/// Arrange the argument and result information for the declaration or
456
/// definition of an Objective-C method.
457
const CGFunctionInfo &
458
5.10k
CodeGenTypes::arrangeObjCMethodDeclaration(const ObjCMethodDecl *MD) {
459
5.10k
  // It happens that this is the same as a call with no optional
460
5.10k
  // arguments, except also using the formal 'self' type.
461
5.10k
  return arrangeObjCMessageSendSignature(MD, MD->getSelfDecl()->getType());
462
5.10k
}
463
464
/// Arrange the argument and result information for the function type
465
/// through which to perform a send to the given Objective-C method,
466
/// using the given receiver type.  The receiver type is not always
467
/// the 'self' type of the method or even an Objective-C pointer type.
468
/// This is *not* the right method for actually performing such a
469
/// message send, due to the possibility of optional arguments.
470
const CGFunctionInfo &
471
CodeGenTypes::arrangeObjCMessageSendSignature(const ObjCMethodDecl *MD,
472
16.6k
                                              QualType receiverType) {
473
16.6k
  SmallVector<CanQualType, 16> argTys;
474
16.6k
  SmallVector<FunctionProtoType::ExtParameterInfo, 4> extParamInfos(2);
475
16.6k
  argTys.push_back(Context.getCanonicalParamType(receiverType));
476
16.6k
  argTys.push_back(Context.getCanonicalParamType(Context.getObjCSelType()));
477
16.6k
  // FIXME: Kill copy?
478
16.6k
  for (const auto *I : MD->parameters()) {
479
13.8k
    argTys.push_back(Context.getCanonicalParamType(I->getType()));
480
13.8k
    auto extParamInfo = FunctionProtoType::ExtParameterInfo().withIsNoEscape(
481
13.8k
        I->hasAttr<NoEscapeAttr>());
482
13.8k
    extParamInfos.push_back(extParamInfo);
483
13.8k
  }
484
16.6k
485
16.6k
  FunctionType::ExtInfo einfo;
486
16.6k
  bool IsWindows = getContext().getTargetInfo().getTriple().isOSWindows();
487
16.6k
  einfo = einfo.withCallingConv(getCallingConventionForDecl(MD, IsWindows));
488
16.6k
489
16.6k
  if (getContext().getLangOpts().ObjCAutoRefCount &&
490
16.6k
      
MD->hasAttr<NSReturnsRetainedAttr>()802
)
491
114
    einfo = einfo.withProducesResult(true);
492
16.6k
493
16.6k
  RequiredArgs required =
494
16.6k
    (MD->isVariadic() ? 
RequiredArgs(argTys.size())924
:
RequiredArgs::All15.7k
);
495
16.6k
496
16.6k
  return arrangeLLVMFunctionInfo(
497
16.6k
      GetReturnType(MD->getReturnType()), /*instanceMethod=*/false,
498
16.6k
      /*chainCall=*/false, argTys, einfo, extParamInfos, required);
499
16.6k
}
500
501
const CGFunctionInfo &
502
CodeGenTypes::arrangeUnprototypedObjCMessageSend(QualType returnType,
503
757
                                                 const CallArgList &args) {
504
757
  auto argTypes = getArgTypesForCall(Context, args);
505
757
  FunctionType::ExtInfo einfo;
506
757
507
757
  return arrangeLLVMFunctionInfo(
508
757
      GetReturnType(returnType), /*instanceMethod=*/false,
509
757
      /*chainCall=*/false, argTypes, einfo, {}, RequiredArgs::All);
510
757
}
511
512
const CGFunctionInfo &
513
461k
CodeGenTypes::arrangeGlobalDeclaration(GlobalDecl GD) {
514
461k
  // FIXME: Do we need to handle ObjCMethodDecl?
515
461k
  const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
516
461k
517
461k
  if (isa<CXXConstructorDecl>(GD.getDecl()) ||
518
461k
      
isa<CXXDestructorDecl>(GD.getDecl())424k
)
519
55.6k
    return arrangeCXXStructorDeclaration(GD);
520
405k
521
405k
  return arrangeFunctionDeclaration(FD);
522
405k
}
523
524
/// Arrange a thunk that takes 'this' as the first parameter followed by
525
/// varargs.  Return a void pointer, regardless of the actual return type.
526
/// The body of the thunk will end in a musttail call to a function of the
527
/// correct type, and the caller will bitcast the function to the correct
528
/// prototype.
529
const CGFunctionInfo &
530
54
CodeGenTypes::arrangeUnprototypedMustTailThunk(const CXXMethodDecl *MD) {
531
54
  assert(MD->isVirtual() && "only methods have thunks");
532
54
  CanQual<FunctionProtoType> FTP = GetFormalType(MD);
533
54
  CanQualType ArgTys[] = {DeriveThisType(MD->getParent(), MD)};
534
54
  return arrangeLLVMFunctionInfo(Context.VoidTy, /*instanceMethod=*/false,
535
54
                                 /*chainCall=*/false, ArgTys,
536
54
                                 FTP->getExtInfo(), {}, RequiredArgs(1));
537
54
}
538
539
const CGFunctionInfo &
540
CodeGenTypes::arrangeMSCtorClosure(const CXXConstructorDecl *CD,
541
34
                                   CXXCtorType CT) {
542
34
  assert(CT == Ctor_CopyingClosure || CT == Ctor_DefaultClosure);
543
34
544
34
  CanQual<FunctionProtoType> FTP = GetFormalType(CD);
545
34
  SmallVector<CanQualType, 2> ArgTys;
546
34
  const CXXRecordDecl *RD = CD->getParent();
547
34
  ArgTys.push_back(DeriveThisType(RD, CD));
548
34
  if (CT == Ctor_CopyingClosure)
549
6
    ArgTys.push_back(*FTP->param_type_begin());
550
34
  if (RD->getNumVBases() > 0)
551
0
    ArgTys.push_back(Context.IntTy);
552
34
  CallingConv CC = Context.getDefaultCallingConvention(
553
34
      /*IsVariadic=*/false, /*IsCXXMethod=*/true);
554
34
  return arrangeLLVMFunctionInfo(Context.VoidTy, /*instanceMethod=*/true,
555
34
                                 /*chainCall=*/false, ArgTys,
556
34
                                 FunctionType::ExtInfo(CC), {},
557
34
                                 RequiredArgs::All);
558
34
}
559
560
/// Arrange a call as unto a free function, except possibly with an
561
/// additional number of formal parameters considered required.
562
static const CGFunctionInfo &
563
arrangeFreeFunctionLikeCall(CodeGenTypes &CGT,
564
                            CodeGenModule &CGM,
565
                            const CallArgList &args,
566
                            const FunctionType *fnType,
567
                            unsigned numExtraRequiredArgs,
568
155k
                            bool chainCall) {
569
155k
  assert(args.size() >= numExtraRequiredArgs);
570
155k
571
155k
  llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos;
572
155k
573
155k
  // In most cases, there are no optional arguments.
574
155k
  RequiredArgs required = RequiredArgs::All;
575
155k
576
155k
  // If we have a variadic prototype, the required arguments are the
577
155k
  // extra prefix plus the arguments in the prototype.
578
155k
  if (const FunctionProtoType *proto = dyn_cast<FunctionProtoType>(fnType)) {
579
154k
    if (proto->isVariadic())
580
41.9k
      required = RequiredArgs::forPrototypePlus(proto, numExtraRequiredArgs);
581
154k
582
154k
    if (proto->hasExtParameterInfos())
583
123
      addExtParameterInfosForCall(paramInfos, proto, numExtraRequiredArgs,
584
123
                                  args.size());
585
154k
586
154k
  // If we don't have a prototype at all, but we're supposed to
587
154k
  // explicitly use the variadic convention for unprototyped calls,
588
154k
  // treat all of the arguments as required but preserve the nominal
589
154k
  // possibility of variadics.
590
154k
  } else 
if (849
CGM.getTargetCodeGenInfo()
591
849
                .isNoProtoCallVariadic(args,
592
849
                                       cast<FunctionNoProtoType>(fnType))) {
593
662
    required = RequiredArgs(args.size());
594
662
  }
595
155k
596
155k
  // FIXME: Kill copy.
597
155k
  SmallVector<CanQualType, 16> argTypes;
598
155k
  for (const auto &arg : args)
599
270k
    argTypes.push_back(CGT.getContext().getCanonicalParamType(arg.Ty));
600
155k
  return CGT.arrangeLLVMFunctionInfo(GetReturnType(fnType->getReturnType()),
601
155k
                                     /*instanceMethod=*/false, chainCall,
602
155k
                                     argTypes, fnType->getExtInfo(), paramInfos,
603
155k
                                     required);
604
155k
}
605
606
/// Figure out the rules for calling a function with the given formal
607
/// type using the given arguments.  The arguments are necessary
608
/// because the function might be unprototyped, in which case it's
609
/// target-dependent in crazy ways.
610
const CGFunctionInfo &
611
CodeGenTypes::arrangeFreeFunctionCall(const CallArgList &args,
612
                                      const FunctionType *fnType,
613
154k
                                      bool chainCall) {
614
154k
  return arrangeFreeFunctionLikeCall(*this, CGM, args, fnType,
615
154k
                                     chainCall ? 
18
:
0154k
, chainCall);
616
154k
}
617
618
/// A block function is essentially a free function with an
619
/// extra implicit argument.
620
const CGFunctionInfo &
621
CodeGenTypes::arrangeBlockFunctionCall(const CallArgList &args,
622
567
                                       const FunctionType *fnType) {
623
567
  return arrangeFreeFunctionLikeCall(*this, CGM, args, fnType, 1,
624
567
                                     /*chainCall=*/false);
625
567
}
626
627
const CGFunctionInfo &
628
CodeGenTypes::arrangeBlockFunctionDeclaration(const FunctionProtoType *proto,
629
1.05k
                                              const FunctionArgList &params) {
630
1.05k
  auto paramInfos = getExtParameterInfosForCall(proto, 1, params.size());
631
1.05k
  auto argTypes = getArgTypesForDeclaration(Context, params);
632
1.05k
633
1.05k
  return arrangeLLVMFunctionInfo(GetReturnType(proto->getReturnType()),
634
1.05k
                                 /*instanceMethod*/ false, /*chainCall*/ false,
635
1.05k
                                 argTypes, proto->getExtInfo(), paramInfos,
636
1.05k
                                 RequiredArgs::forPrototypePlus(proto, 1));
637
1.05k
}
638
639
const CGFunctionInfo &
640
CodeGenTypes::arrangeBuiltinFunctionCall(QualType resultType,
641
934
                                         const CallArgList &args) {
642
934
  // FIXME: Kill copy.
643
934
  SmallVector<CanQualType, 16> argTypes;
644
934
  for (const auto &Arg : args)
645
3.39k
    argTypes.push_back(Context.getCanonicalParamType(Arg.Ty));
646
934
  return arrangeLLVMFunctionInfo(
647
934
      GetReturnType(resultType), /*instanceMethod=*/false,
648
934
      /*chainCall=*/false, argTypes, FunctionType::ExtInfo(),
649
934
      /*paramInfos=*/ {}, RequiredArgs::All);
650
934
}
651
652
const CGFunctionInfo &
653
CodeGenTypes::arrangeBuiltinFunctionDeclaration(QualType resultType,
654
19.4k
                                                const FunctionArgList &args) {
655
19.4k
  auto argTypes = getArgTypesForDeclaration(Context, args);
656
19.4k
657
19.4k
  return arrangeLLVMFunctionInfo(
658
19.4k
      GetReturnType(resultType), /*instanceMethod=*/false, /*chainCall=*/false,
659
19.4k
      argTypes, FunctionType::ExtInfo(), {}, RequiredArgs::All);
660
19.4k
}
661
662
const CGFunctionInfo &
663
CodeGenTypes::arrangeBuiltinFunctionDeclaration(CanQualType resultType,
664
384
                                              ArrayRef<CanQualType> argTypes) {
665
384
  return arrangeLLVMFunctionInfo(
666
384
      resultType, /*instanceMethod=*/false, /*chainCall=*/false,
667
384
      argTypes, FunctionType::ExtInfo(), {}, RequiredArgs::All);
668
384
}
669
670
/// Arrange a call to a C++ method, passing the given arguments.
671
///
672
/// numPrefixArgs is the number of ABI-specific prefix arguments we have. It
673
/// does not count `this`.
674
const CGFunctionInfo &
675
CodeGenTypes::arrangeCXXMethodCall(const CallArgList &args,
676
                                   const FunctionProtoType *proto,
677
                                   RequiredArgs required,
678
63.8k
                                   unsigned numPrefixArgs) {
679
63.8k
  assert(numPrefixArgs + 1 <= args.size() &&
680
63.8k
         "Emitting a call with less args than the required prefix?");
681
63.8k
  // Add one to account for `this`. It's a bit awkward here, but we don't count
682
63.8k
  // `this` in similar places elsewhere.
683
63.8k
  auto paramInfos =
684
63.8k
    getExtParameterInfosForCall(proto, numPrefixArgs + 1, args.size());
685
63.8k
686
63.8k
  // FIXME: Kill copy.
687
63.8k
  auto argTypes = getArgTypesForCall(Context, args);
688
63.8k
689
63.8k
  FunctionType::ExtInfo info = proto->getExtInfo();
690
63.8k
  return arrangeLLVMFunctionInfo(
691
63.8k
      GetReturnType(proto->getReturnType()), /*instanceMethod=*/true,
692
63.8k
      /*chainCall=*/false, argTypes, info, paramInfos, required);
693
63.8k
}
694
695
12.7k
const CGFunctionInfo &CodeGenTypes::arrangeNullaryFunction() {
696
12.7k
  return arrangeLLVMFunctionInfo(
697
12.7k
      getContext().VoidTy, /*instanceMethod=*/false, /*chainCall=*/false,
698
12.7k
      None, FunctionType::ExtInfo(), {}, RequiredArgs::All);
699
12.7k
}
700
701
const CGFunctionInfo &
702
CodeGenTypes::arrangeCall(const CGFunctionInfo &signature,
703
11.5k
                          const CallArgList &args) {
704
11.5k
  assert(signature.arg_size() <= args.size());
705
11.5k
  if (signature.arg_size() == args.size())
706
10.6k
    return signature;
707
906
708
906
  SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos;
709
906
  auto sigParamInfos = signature.getExtParameterInfos();
710
906
  if (!sigParamInfos.empty()) {
711
906
    paramInfos.append(sigParamInfos.begin(), sigParamInfos.end());
712
906
    paramInfos.resize(args.size());
713
906
  }
714
906
715
906
  auto argTypes = getArgTypesForCall(Context, args);
716
906
717
906
  assert(signature.getRequiredArgs().allowsOptionalArgs());
718
906
  return arrangeLLVMFunctionInfo(signature.getReturnType(),
719
906
                                 signature.isInstanceMethod(),
720
906
                                 signature.isChainCall(),
721
906
                                 argTypes,
722
906
                                 signature.getExtInfo(),
723
906
                                 paramInfos,
724
906
                                 signature.getRequiredArgs());
725
906
}
726
727
namespace clang {
728
namespace CodeGen {
729
void computeSPIRKernelABIInfo(CodeGenModule &CGM, CGFunctionInfo &FI);
730
}
731
}
732
733
/// Arrange the argument and result information for an abstract value
734
/// of a given function type.  This is the method which all of the
735
/// above functions ultimately defer to.
736
const CGFunctionInfo &
737
CodeGenTypes::arrangeLLVMFunctionInfo(CanQualType resultType,
738
                                      bool instanceMethod,
739
                                      bool chainCall,
740
                                      ArrayRef<CanQualType> argTypes,
741
                                      FunctionType::ExtInfo info,
742
                     ArrayRef<FunctionProtoType::ExtParameterInfo> paramInfos,
743
1.13M
                                      RequiredArgs required) {
744
1.13M
  assert(llvm::all_of(argTypes,
745
1.13M
                      [](CanQualType T) { return T.isCanonicalAsParam(); }));
746
1.13M
747
1.13M
  // Lookup or create unique function info.
748
1.13M
  llvm::FoldingSetNodeID ID;
749
1.13M
  CGFunctionInfo::Profile(ID, instanceMethod, chainCall, info, paramInfos,
750
1.13M
                          required, resultType, argTypes);
751
1.13M
752
1.13M
  void *insertPos = nullptr;
753
1.13M
  CGFunctionInfo *FI = FunctionInfos.FindNodeOrInsertPos(ID, insertPos);
754
1.13M
  if (FI)
755
958k
    return *FI;
756
180k
757
180k
  unsigned CC = ClangCallConvToLLVMCallConv(info.getCC());
758
180k
759
180k
  // Construct the function info.  We co-allocate the ArgInfos.
760
180k
  FI = CGFunctionInfo::create(CC, instanceMethod, chainCall, info,
761
180k
                              paramInfos, resultType, argTypes, required);
762
180k
  FunctionInfos.InsertNode(FI, insertPos);
763
180k
764
180k
  bool inserted = FunctionsBeingProcessed.insert(FI).second;
765
180k
  (void)inserted;
766
180k
  assert(inserted && "Recursively being processed?");
767
180k
768
180k
  // Compute ABI information.
769
180k
  if (CC == llvm::CallingConv::SPIR_KERNEL) {
770
124
    // Force target independent argument handling for the host visible
771
124
    // kernel functions.
772
124
    computeSPIRKernelABIInfo(CGM, *FI);
773
180k
  } else if (info.getCC() == CC_Swift) {
774
1.12k
    swiftcall::computeABIInfo(CGM, *FI);
775
179k
  } else {
776
179k
    getABIInfo().computeInfo(*FI);
777
179k
  }
778
180k
779
180k
  // Loop over all of the computed argument and return value info.  If any of
780
180k
  // them are direct or extend without a specified coerce type, specify the
781
180k
  // default now.
782
180k
  ABIArgInfo &retInfo = FI->getReturnInfo();
783
180k
  if (retInfo.canHaveCoerceToType() && 
retInfo.getCoerceToType() == nullptr99.3k
)
784
23.8k
    retInfo.setCoerceToType(ConvertType(FI->getReturnType()));
785
180k
786
180k
  for (auto &I : FI->arguments())
787
316k
    if (I.info.canHaveCoerceToType() && 
I.info.getCoerceToType() == nullptr308k
)
788
80.7k
      I.info.setCoerceToType(ConvertType(I.type));
789
180k
790
180k
  bool erased = FunctionsBeingProcessed.erase(FI); (void)erased;
791
180k
  assert(erased && "Not in set?");
792
180k
793
180k
  return *FI;
794
180k
}
795
796
CGFunctionInfo *CGFunctionInfo::create(unsigned llvmCC,
797
                                       bool instanceMethod,
798
                                       bool chainCall,
799
                                       const FunctionType::ExtInfo &info,
800
                                       ArrayRef<ExtParameterInfo> paramInfos,
801
                                       CanQualType resultType,
802
                                       ArrayRef<CanQualType> argTypes,
803
180k
                                       RequiredArgs required) {
804
180k
  assert(paramInfos.empty() || paramInfos.size() == argTypes.size());
805
180k
  assert(!required.allowsOptionalArgs() ||
806
180k
         required.getNumRequiredArgs() <= argTypes.size());
807
180k
808
180k
  void *buffer =
809
180k
    operator new(totalSizeToAlloc<ArgInfo,             ExtParameterInfo>(
810
180k
                                  argTypes.size() + 1, paramInfos.size()));
811
180k
812
180k
  CGFunctionInfo *FI = new(buffer) CGFunctionInfo();
813
180k
  FI->CallingConvention = llvmCC;
814
180k
  FI->EffectiveCallingConvention = llvmCC;
815
180k
  FI->ASTCallingConvention = info.getCC();
816
180k
  FI->InstanceMethod = instanceMethod;
817
180k
  FI->ChainCall = chainCall;
818
180k
  FI->NoReturn = info.getNoReturn();
819
180k
  FI->ReturnsRetained = info.getProducesResult();
820
180k
  FI->NoCallerSavedRegs = info.getNoCallerSavedRegs();
821
180k
  FI->NoCfCheck = info.getNoCfCheck();
822
180k
  FI->Required = required;
823
180k
  FI->HasRegParm = info.getHasRegParm();
824
180k
  FI->RegParm = info.getRegParm();
825
180k
  FI->ArgStruct = nullptr;
826
180k
  FI->ArgStructAlign = 0;
827
180k
  FI->NumArgs = argTypes.size();
828
180k
  FI->HasExtParameterInfos = !paramInfos.empty();
829
180k
  FI->getArgsBuffer()[0].type = resultType;
830
497k
  for (unsigned i = 0, e = argTypes.size(); i != e; 
++i316k
)
831
316k
    FI->getArgsBuffer()[i + 1].type = argTypes[i];
832
200k
  for (unsigned i = 0, e = paramInfos.size(); i != e; 
++i20.5k
)
833
20.5k
    FI->getExtParameterInfosBuffer()[i] = paramInfos[i];
834
180k
  return FI;
835
180k
}
836
837
/***/
838
839
namespace {
840
// ABIArgInfo::Expand implementation.
841
842
// Specifies the way QualType passed as ABIArgInfo::Expand is expanded.
843
struct TypeExpansion {
844
  enum TypeExpansionKind {
845
    // Elements of constant arrays are expanded recursively.
846
    TEK_ConstantArray,
847
    // Record fields are expanded recursively (but if record is a union, only
848
    // the field with the largest size is expanded).
849
    TEK_Record,
850
    // For complex types, real and imaginary parts are expanded recursively.
851
    TEK_Complex,
852
    // All other types are not expandable.
853
    TEK_None
854
  };
855
856
  const TypeExpansionKind Kind;
857
858
2.24k
  TypeExpansion(TypeExpansionKind K) : Kind(K) {}
859
2.24k
  virtual ~TypeExpansion() {}
860
};
861
862
struct ConstantArrayExpansion : TypeExpansion {
863
  QualType EltTy;
864
  uint64_t NumElts;
865
866
  ConstantArrayExpansion(QualType EltTy, uint64_t NumElts)
867
40
      : TypeExpansion(TEK_ConstantArray), EltTy(EltTy), NumElts(NumElts) {}
868
2.28k
  static bool classof(const TypeExpansion *TE) {
869
2.28k
    return TE->Kind == TEK_ConstantArray;
870
2.28k
  }
871
};
872
873
struct RecordExpansion : TypeExpansion {
874
  SmallVector<const CXXBaseSpecifier *, 1> Bases;
875
876
  SmallVector<const FieldDecl *, 1> Fields;
877
878
  RecordExpansion(SmallVector<const CXXBaseSpecifier *, 1> &&Bases,
879
                  SmallVector<const FieldDecl *, 1> &&Fields)
880
      : TypeExpansion(TEK_Record), Bases(std::move(Bases)),
881
721
        Fields(std::move(Fields)) {}
882
2.92k
  static bool classof(const TypeExpansion *TE) {
883
2.92k
    return TE->Kind == TEK_Record;
884
2.92k
  }
885
};
886
887
struct ComplexExpansion : TypeExpansion {
888
  QualType EltTy;
889
890
22
  ComplexExpansion(QualType EltTy) : TypeExpansion(TEK_Complex), EltTy(EltTy) {}
891
1.49k
  static bool classof(const TypeExpansion *TE) {
892
1.49k
    return TE->Kind == TEK_Complex;
893
1.49k
  }
894
};
895
896
struct NoExpansion : TypeExpansion {
897
1.46k
  NoExpansion() : TypeExpansion(TEK_None) {}
898
1.46k
  static bool classof(const TypeExpansion *TE) {
899
1.46k
    return TE->Kind == TEK_None;
900
1.46k
  }
901
};
902
}  // namespace
903
904
static std::unique_ptr<TypeExpansion>
905
2.24k
getTypeExpansion(QualType Ty, const ASTContext &Context) {
906
2.24k
  if (const ConstantArrayType *AT = Context.getAsConstantArrayType(Ty)) {
907
40
    return std::make_unique<ConstantArrayExpansion>(
908
40
        AT->getElementType(), AT->getSize().getZExtValue());
909
40
  }
910
2.20k
  if (const RecordType *RT = Ty->getAs<RecordType>()) {
911
721
    SmallVector<const CXXBaseSpecifier *, 1> Bases;
912
721
    SmallVector<const FieldDecl *, 1> Fields;
913
721
    const RecordDecl *RD = RT->getDecl();
914
721
    assert(!RD->hasFlexibleArrayMember() &&
915
721
           "Cannot expand structure with flexible array.");
916
721
    if (RD->isUnion()) {
917
5
      // Unions can be here only in degenerative cases - all the fields are same
918
5
      // after flattening. Thus we have to use the "largest" field.
919
5
      const FieldDecl *LargestFD = nullptr;
920
5
      CharUnits UnionSize = CharUnits::Zero();
921
5
922
10
      for (const auto *FD : RD->fields()) {
923
10
        if (FD->isZeroLengthBitField(Context))
924
0
          continue;
925
10
        assert(!FD->isBitField() &&
926
10
               "Cannot expand structure with bit-field members.");
927
10
        CharUnits FieldSize = Context.getTypeSizeInChars(FD->getType());
928
10
        if (UnionSize < FieldSize) {
929
5
          UnionSize = FieldSize;
930
5
          LargestFD = FD;
931
5
        }
932
10
      }
933
5
      if (LargestFD)
934
5
        Fields.push_back(LargestFD);
935
716
    } else {
936
716
      if (const auto *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
937
201
        assert(!CXXRD->isDynamicClass() &&
938
201
               "cannot expand vtable pointers in dynamic classes");
939
201
        for (const CXXBaseSpecifier &BS : CXXRD->bases())
940
5
          Bases.push_back(&BS);
941
201
      }
942
716
943
1.42k
      for (const auto *FD : RD->fields()) {
944
1.42k
        if (FD->isZeroLengthBitField(Context))
945
0
          continue;
946
1.42k
        assert(!FD->isBitField() &&
947
1.42k
               "Cannot expand structure with bit-field members.");
948
1.42k
        Fields.push_back(FD);
949
1.42k
      }
950
716
    }
951
721
    return std::make_unique<RecordExpansion>(std::move(Bases),
952
721
                                              std::move(Fields));
953
721
  }
954
1.48k
  if (const ComplexType *CT = Ty->getAs<ComplexType>()) {
955
22
    return std::make_unique<ComplexExpansion>(CT->getElementType());
956
22
  }
957
1.46k
  return std::make_unique<NoExpansion>();
958
1.46k
}
959
960
1.31k
static int getExpansionSize(QualType Ty, const ASTContext &Context) {
961
1.31k
  auto Exp = getTypeExpansion(Ty, Context);
962
1.31k
  if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
963
24
    return CAExp->NumElts * getExpansionSize(CAExp->EltTy, Context);
964
24
  }
965
1.29k
  if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
966
433
    int Res = 0;
967
433
    for (auto BS : RExp->Bases)
968
3
      Res += getExpansionSize(BS->getType(), Context);
969
433
    for (auto FD : RExp->Fields)
970
856
      Res += getExpansionSize(FD->getType(), Context);
971
433
    return Res;
972
433
  }
973
862
  if (isa<ComplexExpansion>(Exp.get()))
974
13
    return 2;
975
849
  assert(isa<NoExpansion>(Exp.get()));
976
849
  return 1;
977
849
}
978
979
void
980
CodeGenTypes::getExpandedTypes(QualType Ty,
981
512
                               SmallVectorImpl<llvm::Type *>::iterator &TI) {
982
512
  auto Exp = getTypeExpansion(Ty, Context);
983
512
  if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
984
40
    for (int i = 0, n = CAExp->NumElts; i < n; 
i++32
) {
985
32
      getExpandedTypes(CAExp->EltTy, TI);
986
32
    }
987
504
  } else if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
988
162
    for (auto BS : RExp->Bases)
989
1
      getExpandedTypes(BS->getType(), TI);
990
162
    for (auto FD : RExp->Fields)
991
316
      getExpandedTypes(FD->getType(), TI);
992
342
  } else if (auto CExp = dyn_cast<ComplexExpansion>(Exp.get())) {
993
5
    llvm::Type *EltTy = ConvertType(CExp->EltTy);
994
5
    *TI++ = EltTy;
995
5
    *TI++ = EltTy;
996
337
  } else {
997
337
    assert(isa<NoExpansion>(Exp.get()));
998
337
    *TI++ = ConvertType(Ty);
999
337
  }
1000
512
}
1001
1002
static void forConstantArrayExpansion(CodeGenFunction &CGF,
1003
                                      ConstantArrayExpansion *CAE,
1004
                                      Address BaseAddr,
1005
8
                                      llvm::function_ref<void(Address)> Fn) {
1006
8
  CharUnits EltSize = CGF.getContext().getTypeSizeInChars(CAE->EltTy);
1007
8
  CharUnits EltAlign =
1008
8
    BaseAddr.getAlignment().alignmentOfArrayElement(EltSize);
1009
8
1010
40
  for (int i = 0, n = CAE->NumElts; i < n; 
i++32
) {
1011
32
    llvm::Value *EltAddr =
1012
32
      CGF.Builder.CreateConstGEP2_32(nullptr, BaseAddr.getPointer(), 0, i);
1013
32
    Fn(Address(EltAddr, EltAlign));
1014
32
  }
1015
8
}
1016
1017
void CodeGenFunction::ExpandTypeFromArgs(
1018
344
    QualType Ty, LValue LV, SmallVectorImpl<llvm::Value *>::iterator &AI) {
1019
344
  assert(LV.isSimple() &&
1020
344
         "Unexpected non-simple lvalue during struct expansion.");
1021
344
1022
344
  auto Exp = getTypeExpansion(Ty, getContext());
1023
344
  if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
1024
8
    forConstantArrayExpansion(
1025
32
        *this, CAExp, LV.getAddress(*this), [&](Address EltAddr) {
1026
32
          LValue LV = MakeAddrLValue(EltAddr, CAExp->EltTy);
1027
32
          ExpandTypeFromArgs(CAExp->EltTy, LV, AI);
1028
32
        });
1029
336
  } else if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
1030
99
    Address This = LV.getAddress(*this);
1031
99
    for (const CXXBaseSpecifier *BS : RExp->Bases) {
1032
1
      // Perform a single step derived-to-base conversion.
1033
1
      Address Base =
1034
1
          GetAddressOfBaseClass(This, Ty->getAsCXXRecordDecl(), &BS, &BS + 1,
1035
1
                                /*NullCheckValue=*/false, SourceLocation());
1036
1
      LValue SubLV = MakeAddrLValue(Base, BS->getType());
1037
1
1038
1
      // Recurse onto bases.
1039
1
      ExpandTypeFromArgs(BS->getType(), SubLV, AI);
1040
1
    }
1041
211
    for (auto FD : RExp->Fields) {
1042
211
      // FIXME: What are the right qualifiers here?
1043
211
      LValue SubLV = EmitLValueForFieldInitialization(LV, FD);
1044
211
      ExpandTypeFromArgs(FD->getType(), SubLV, AI);
1045
211
    }
1046
237
  } else if (isa<ComplexExpansion>(Exp.get())) {
1047
3
    auto realValue = *AI++;
1048
3
    auto imagValue = *AI++;
1049
3
    EmitStoreOfComplex(ComplexPairTy(realValue, imagValue), LV, /*init*/ true);
1050
234
  } else {
1051
234
    // Call EmitStoreOfScalar except when the lvalue is a bitfield to emit a
1052
234
    // primitive store.
1053
234
    assert(isa<NoExpansion>(Exp.get()));
1054
234
    if (LV.isBitField())
1055
0
      EmitStoreThroughLValue(RValue::get(*AI++), LV);
1056
234
    else
1057
234
      EmitStoreOfScalar(*AI++, LV);
1058
234
  }
1059
344
}
1060
1061
void CodeGenFunction::ExpandTypeToArgs(
1062
    QualType Ty, CallArg Arg, llvm::FunctionType *IRFuncTy,
1063
71
    SmallVectorImpl<llvm::Value *> &IRCallArgs, unsigned &IRCallArgPos) {
1064
71
  auto Exp = getTypeExpansion(Ty, getContext());
1065
71
  if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
1066
0
    Address Addr = Arg.hasLValue() ? Arg.getKnownLValue().getAddress(*this)
1067
0
                                   : Arg.getKnownRValue().getAggregateAddress();
1068
0
    forConstantArrayExpansion(
1069
0
        *this, CAExp, Addr, [&](Address EltAddr) {
1070
0
          CallArg EltArg = CallArg(
1071
0
              convertTempToRValue(EltAddr, CAExp->EltTy, SourceLocation()),
1072
0
              CAExp->EltTy);
1073
0
          ExpandTypeToArgs(CAExp->EltTy, EltArg, IRFuncTy, IRCallArgs,
1074
0
                           IRCallArgPos);
1075
0
        });
1076
71
  } else if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
1077
27
    Address This = Arg.hasLValue() ? 
Arg.getKnownLValue().getAddress(*this)12
1078
27
                                   : 
Arg.getKnownRValue().getAggregateAddress()15
;
1079
27
    for (const CXXBaseSpecifier *BS : RExp->Bases) {
1080
0
      // Perform a single step derived-to-base conversion.
1081
0
      Address Base =
1082
0
          GetAddressOfBaseClass(This, Ty->getAsCXXRecordDecl(), &BS, &BS + 1,
1083
0
                                /*NullCheckValue=*/false, SourceLocation());
1084
0
      CallArg BaseArg = CallArg(RValue::getAggregate(Base), BS->getType());
1085
0
1086
0
      // Recurse onto bases.
1087
0
      ExpandTypeToArgs(BS->getType(), BaseArg, IRFuncTy, IRCallArgs,
1088
0
                       IRCallArgPos);
1089
0
    }
1090
27
1091
27
    LValue LV = MakeAddrLValue(This, Ty);
1092
44
    for (auto FD : RExp->Fields) {
1093
44
      CallArg FldArg =
1094
44
          CallArg(EmitRValueForField(LV, FD, SourceLocation()), FD->getType());
1095
44
      ExpandTypeToArgs(FD->getType(), FldArg, IRFuncTy, IRCallArgs,
1096
44
                       IRCallArgPos);
1097
44
    }
1098
44
  } else if (isa<ComplexExpansion>(Exp.get())) {
1099
1
    ComplexPairTy CV = Arg.getKnownRValue().getComplexVal();
1100
1
    IRCallArgs[IRCallArgPos++] = CV.first;
1101
1
    IRCallArgs[IRCallArgPos++] = CV.second;
1102
43
  } else {
1103
43
    assert(isa<NoExpansion>(Exp.get()));
1104
43
    auto RV = Arg.getKnownRValue();
1105
43
    assert(RV.isScalar() &&
1106
43
           "Unexpected non-scalar rvalue during struct expansion.");
1107
43
1108
43
    // Insert a bitcast as needed.
1109
43
    llvm::Value *V = RV.getScalarVal();
1110
43
    if (IRCallArgPos < IRFuncTy->getNumParams() &&
1111
43
        V->getType() != IRFuncTy->getParamType(IRCallArgPos))
1112
0
      V = Builder.CreateBitCast(V, IRFuncTy->getParamType(IRCallArgPos));
1113
43
1114
43
    IRCallArgs[IRCallArgPos++] = V;
1115
43
  }
1116
71
}
1117
1118
/// Create a temporary allocation for the purposes of coercion.
1119
static Address CreateTempAllocaForCoercion(CodeGenFunction &CGF, llvm::Type *Ty,
1120
284
                                           CharUnits MinAlign) {
1121
284
  // Don't use an alignment that's worse than what LLVM would prefer.
1122
284
  auto PrefAlign = CGF.CGM.getDataLayout().getPrefTypeAlignment(Ty);
1123
284
  CharUnits Align = std::max(MinAlign, CharUnits::fromQuantity(PrefAlign));
1124
284
1125
284
  return CGF.CreateTempAlloca(Ty, Align);
1126
284
}
1127
1128
/// EnterStructPointerForCoercedAccess - Given a struct pointer that we are
1129
/// accessing some number of bytes out of it, try to gep into the struct to get
1130
/// at its inner goodness.  Dive as deep as possible without entering an element
1131
/// with an in-memory size smaller than DstSize.
1132
static Address
1133
EnterStructPointerForCoercedAccess(Address SrcPtr,
1134
                                   llvm::StructType *SrcSTy,
1135
17.8k
                                   uint64_t DstSize, CodeGenFunction &CGF) {
1136
17.8k
  // We can't dive into a zero-element struct.
1137
17.8k
  if (SrcSTy->getNumElements() == 0) 
return SrcPtr0
;
1138
17.8k
1139
17.8k
  llvm::Type *FirstElt = SrcSTy->getElementType(0);
1140
17.8k
1141
17.8k
  // If the first elt is at least as large as what we're looking for, or if the
1142
17.8k
  // first element is the same size as the whole struct, we can enter it. The
1143
17.8k
  // comparison must be made on the store size and not the alloca size. Using
1144
17.8k
  // the alloca size may overstate the size of the load.
1145
17.8k
  uint64_t FirstEltSize =
1146
17.8k
    CGF.CGM.getDataLayout().getTypeStoreSize(FirstElt);
1147
17.8k
  if (FirstEltSize < DstSize &&
1148
17.8k
      
FirstEltSize < CGF.CGM.getDataLayout().getTypeStoreSize(SrcSTy)4.44k
)
1149
4.30k
    return SrcPtr;
1150
13.5k
1151
13.5k
  // GEP into the first element.
1152
13.5k
  SrcPtr = CGF.Builder.CreateStructGEP(SrcPtr, 0, "coerce.dive");
1153
13.5k
1154
13.5k
  // If the first element is a struct, recurse.
1155
13.5k
  llvm::Type *SrcTy = SrcPtr.getElementType();
1156
13.5k
  if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy))
1157
3.98k
    return EnterStructPointerForCoercedAccess(SrcPtr, SrcSTy, DstSize, CGF);
1158
9.56k
1159
9.56k
  return SrcPtr;
1160
9.56k
}
1161
1162
/// CoerceIntOrPtrToIntOrPtr - Convert a value Val to the specific Ty where both
1163
/// are either integers or pointers.  This does a truncation of the value if it
1164
/// is too large or a zero extension if it is too small.
1165
///
1166
/// This behaves as if the value were coerced through memory, so on big-endian
1167
/// targets the high bits are preserved in a truncation, while little-endian
1168
/// targets preserve the low bits.
1169
static llvm::Value *CoerceIntOrPtrToIntOrPtr(llvm::Value *Val,
1170
                                             llvm::Type *Ty,
1171
8.54k
                                             CodeGenFunction &CGF) {
1172
8.54k
  if (Val->getType() == Ty)
1173
8.13k
    return Val;
1174
407
1175
407
  if (isa<llvm::PointerType>(Val->getType())) {
1176
328
    // If this is Pointer->Pointer avoid conversion to and from int.
1177
328
    if (isa<llvm::PointerType>(Ty))
1178
142
      return CGF.Builder.CreateBitCast(Val, Ty, "coerce.val");
1179
186
1180
186
    // Convert the pointer to an integer so we can play with its width.
1181
186
    Val = CGF.Builder.CreatePtrToInt(Val, CGF.IntPtrTy, "coerce.val.pi");
1182
186
  }
1183
407
1184
407
  llvm::Type *DestIntTy = Ty;
1185
265
  if (isa<llvm::PointerType>(DestIntTy))
1186
19
    DestIntTy = CGF.IntPtrTy;
1187
265
1188
265
  if (Val->getType() != DestIntTy) {
1189
60
    const llvm::DataLayout &DL = CGF.CGM.getDataLayout();
1190
60
    if (DL.isBigEndian()) {
1191
19
      // Preserve the high bits on big-endian targets.
1192
19
      // That is what memory coercion does.
1193
19
      uint64_t SrcSize = DL.getTypeSizeInBits(Val->getType());
1194
19
      uint64_t DstSize = DL.getTypeSizeInBits(DestIntTy);
1195
19
1196
19
      if (SrcSize > DstSize) {
1197
10
        Val = CGF.Builder.CreateLShr(Val, SrcSize - DstSize, "coerce.highbits");
1198
10
        Val = CGF.Builder.CreateTrunc(Val, DestIntTy, "coerce.val.ii");
1199
10
      } else {
1200
9
        Val = CGF.Builder.CreateZExt(Val, DestIntTy, "coerce.val.ii");
1201
9
        Val = CGF.Builder.CreateShl(Val, DstSize - SrcSize, "coerce.highbits");
1202
9
      }
1203
41
    } else {
1204
41
      // Little-endian targets preserve the low bits. No shifts required.
1205
41
      Val = CGF.Builder.CreateIntCast(Val, DestIntTy, false, "coerce.val.ii");
1206
41
    }
1207
60
  }
1208
265
1209
265
  if (isa<llvm::PointerType>(Ty))
1210
19
    Val = CGF.Builder.CreateIntToPtr(Val, Ty, "coerce.val.ip");
1211
265
  return Val;
1212
407
}
1213
1214
1215
1216
/// CreateCoercedLoad - Create a load from \arg SrcPtr interpreted as
1217
/// a pointer to an object of type \arg Ty, known to be aligned to
1218
/// \arg SrcAlign bytes.
1219
///
1220
/// This safely handles the case when the src type is smaller than the
1221
/// destination type; in this situation the values of bits which not
1222
/// present in the src are undefined.
1223
static llvm::Value *CreateCoercedLoad(Address Src, llvm::Type *Ty,
1224
7.42k
                                      CodeGenFunction &CGF) {
1225
7.42k
  llvm::Type *SrcTy = Src.getElementType();
1226
7.42k
1227
7.42k
  // If SrcTy and Ty are the same, just do a load.
1228
7.42k
  if (SrcTy == Ty)
1229
24
    return CGF.Builder.CreateLoad(Src);
1230
7.40k
1231
7.40k
  uint64_t DstSize = CGF.CGM.getDataLayout().getTypeAllocSize(Ty);
1232
7.40k
1233
7.40k
  if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy)) {
1234
6.08k
    Src = EnterStructPointerForCoercedAccess(Src, SrcSTy, DstSize, CGF);
1235
6.08k
    SrcTy = Src.getType()->getElementType();
1236
6.08k
  }
1237
7.40k
1238
7.40k
  uint64_t SrcSize = CGF.CGM.getDataLayout().getTypeAllocSize(SrcTy);
1239
7.40k
1240
7.40k
  // If the source and destination are integer or pointer types, just do an
1241
7.40k
  // extension or truncation to the desired type.
1242
7.40k
  if ((isa<llvm::IntegerType>(Ty) || 
isa<llvm::PointerType>(Ty)4.72k
) &&
1243
7.40k
      
(5.56k
isa<llvm::IntegerType>(SrcTy)5.56k
||
isa<llvm::PointerType>(SrcTy)4.44k
)) {
1244
4.19k
    llvm::Value *Load = CGF.Builder.CreateLoad(Src);
1245
4.19k
    return CoerceIntOrPtrToIntOrPtr(Load, Ty, CGF);
1246
4.19k
  }
1247
3.20k
1248
3.20k
  // If load is legal, just bitcast the src pointer.
1249
3.20k
  if (SrcSize >= DstSize) {
1250
3.07k
    // Generally SrcSize is never greater than DstSize, since this means we are
1251
3.07k
    // losing bits. However, this can happen in cases where the structure has
1252
3.07k
    // additional padding, for example due to a user specified alignment.
1253
3.07k
    //
1254
3.07k
    // FIXME: Assert that we aren't truncating non-padding bits when have access
1255
3.07k
    // to that information.
1256
3.07k
    Src = CGF.Builder.CreateBitCast(Src,
1257
3.07k
                                    Ty->getPointerTo(Src.getAddressSpace()));
1258
3.07k
    return CGF.Builder.CreateLoad(Src);
1259
3.07k
  }
1260
131
1261
131
  // Otherwise do coercion through memory. This is stupid, but simple.
1262
131
  Address Tmp = CreateTempAllocaForCoercion(CGF, Ty, Src.getAlignment());
1263
131
  Address Casted = CGF.Builder.CreateElementBitCast(Tmp,CGF.Int8Ty);
1264
131
  Address SrcCasted = CGF.Builder.CreateElementBitCast(Src,CGF.Int8Ty);
1265
131
  CGF.Builder.CreateMemCpy(Casted, SrcCasted,
1266
131
      llvm::ConstantInt::get(CGF.IntPtrTy, SrcSize),
1267
131
      false);
1268
131
  return CGF.Builder.CreateLoad(Tmp);
1269
131
}
1270
1271
// Function to store a first-class aggregate into memory.  We prefer to
1272
// store the elements rather than the aggregate to be more friendly to
1273
// fast-isel.
1274
// FIXME: Do we need to recurse here?
1275
static void BuildAggStore(CodeGenFunction &CGF, llvm::Value *Val,
1276
4.81k
                          Address Dest, bool DestIsVolatile) {
1277
4.81k
  // Prefer scalar stores to first-class aggregate stores.
1278
4.81k
  if (llvm::StructType *STy =
1279
1.32k
        dyn_cast<llvm::StructType>(Val->getType())) {
1280
3.95k
    for (unsigned i = 0, e = STy->getNumElements(); i != e; 
++i2.62k
) {
1281
2.62k
      Address EltPtr = CGF.Builder.CreateStructGEP(Dest, i);
1282
2.62k
      llvm::Value *Elt = CGF.Builder.CreateExtractValue(Val, i);
1283
2.62k
      CGF.Builder.CreateStore(Elt, EltPtr, DestIsVolatile);
1284
2.62k
    }
1285
3.48k
  } else {
1286
3.48k
    CGF.Builder.CreateStore(Val, Dest, DestIsVolatile);
1287
3.48k
  }
1288
4.81k
}
1289
1290
/// CreateCoercedStore - Create a store to \arg DstPtr from \arg Src,
1291
/// where the source and destination may have different types.  The
1292
/// destination is known to be aligned to \arg DstAlign bytes.
1293
///
1294
/// This safely handles the case when the src type is larger than the
1295
/// destination type; the upper bits of the src will be lost.
1296
static void CreateCoercedStore(llvm::Value *Src,
1297
                               Address Dst,
1298
                               bool DstIsVolatile,
1299
9.39k
                               CodeGenFunction &CGF) {
1300
9.39k
  llvm::Type *SrcTy = Src->getType();
1301
9.39k
  llvm::Type *DstTy = Dst.getType()->getElementType();
1302
9.39k
  if (SrcTy == DstTy) {
1303
156
    CGF.Builder.CreateStore(Src, Dst, DstIsVolatile);
1304
156
    return;
1305
156
  }
1306
9.24k
1307
9.24k
  uint64_t SrcSize = CGF.CGM.getDataLayout().getTypeAllocSize(SrcTy);
1308
9.24k
1309
9.24k
  if (llvm::StructType *DstSTy = dyn_cast<llvm::StructType>(DstTy)) {
1310
7.78k
    Dst = EnterStructPointerForCoercedAccess(Dst, DstSTy, SrcSize, CGF);
1311
7.78k
    DstTy = Dst.getType()->getElementType();
1312
7.78k
  }
1313
9.24k
1314
9.24k
  llvm::PointerType *SrcPtrTy = llvm::dyn_cast<llvm::PointerType>(SrcTy);
1315
9.24k
  llvm::PointerType *DstPtrTy = llvm::dyn_cast<llvm::PointerType>(DstTy);
1316
9.24k
  if (SrcPtrTy && 
DstPtrTy3.08k
&&
1317
9.24k
      
SrcPtrTy->getAddressSpace() != DstPtrTy->getAddressSpace()3.08k
) {
1318
4
    Src = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Src, DstTy);
1319
4
    CGF.Builder.CreateStore(Src, Dst, DstIsVolatile);
1320
4
    return;
1321
4
  }
1322
9.23k
1323
9.23k
  // If the source and destination are integer or pointer types, just do an
1324
9.23k
  // extension or truncation to the desired type.
1325
9.23k
  if ((isa<llvm::IntegerType>(SrcTy) || 
isa<llvm::PointerType>(SrcTy)6.49k
) &&
1326
9.23k
      
(5.82k
isa<llvm::IntegerType>(DstTy)5.82k
||
isa<llvm::PointerType>(DstTy)4.58k
)) {
1327
4.35k
    Src = CoerceIntOrPtrToIntOrPtr(Src, DstTy, CGF);
1328
4.35k
    CGF.Builder.CreateStore(Src, Dst, DstIsVolatile);
1329
4.35k
    return;
1330
4.35k
  }
1331
4.88k
1332
4.88k
  uint64_t DstSize = CGF.CGM.getDataLayout().getTypeAllocSize(DstTy);
1333
4.88k
1334
4.88k
  // If store is legal, just bitcast the src pointer.
1335
4.88k
  if (SrcSize <= DstSize) {
1336
4.73k
    Dst = CGF.Builder.CreateElementBitCast(Dst, SrcTy);
1337
4.73k
    BuildAggStore(CGF, Src, Dst, DstIsVolatile);
1338
4.73k
  } else {
1339
153
    // Otherwise do coercion through memory. This is stupid, but
1340
153
    // simple.
1341
153
1342
153
    // Generally SrcSize is never greater than DstSize, since this means we are
1343
153
    // losing bits. However, this can happen in cases where the structure has
1344
153
    // additional padding, for example due to a user specified alignment.
1345
153
    //
1346
153
    // FIXME: Assert that we aren't truncating non-padding bits when have access
1347
153
    // to that information.
1348
153
    Address Tmp = CreateTempAllocaForCoercion(CGF, SrcTy, Dst.getAlignment());
1349
153
    CGF.Builder.CreateStore(Src, Tmp);
1350
153
    Address Casted = CGF.Builder.CreateElementBitCast(Tmp,CGF.Int8Ty);
1351
153
    Address DstCasted = CGF.Builder.CreateElementBitCast(Dst,CGF.Int8Ty);
1352
153
    CGF.Builder.CreateMemCpy(DstCasted, Casted,
1353
153
        llvm::ConstantInt::get(CGF.IntPtrTy, DstSize),
1354
153
        false);
1355
153
  }
1356
4.88k
}
1357
1358
static Address emitAddressAtOffset(CodeGenFunction &CGF, Address addr,
1359
18.7k
                                   const ABIArgInfo &info) {
1360
18.7k
  if (unsigned offset = info.getDirectOffset()) {
1361
3
    addr = CGF.Builder.CreateElementBitCast(addr, CGF.Int8Ty);
1362
3
    addr = CGF.Builder.CreateConstInBoundsByteGEP(addr,
1363
3
                                             CharUnits::fromQuantity(offset));
1364
3
    addr = CGF.Builder.CreateElementBitCast(addr, info.getCoerceToType());
1365
3
  }
1366
18.7k
  return addr;
1367
18.7k
}
1368
1369
namespace {
1370
1371
/// Encapsulates information about the way function arguments from
1372
/// CGFunctionInfo should be passed to actual LLVM IR function.
1373
class ClangToLLVMArgMapping {
1374
  static const unsigned InvalidIndex = ~0U;
1375
  unsigned InallocaArgNo;
1376
  unsigned SRetArgNo;
1377
  unsigned TotalIRArgs;
1378
1379
  /// Arguments of LLVM IR function corresponding to single Clang argument.
1380
  struct IRArgs {
1381
    unsigned PaddingArgIndex;
1382
    // Argument is expanded to IR arguments at positions
1383
    // [FirstArgIndex, FirstArgIndex + NumberOfArgs).
1384
    unsigned FirstArgIndex;
1385
    unsigned NumberOfArgs;
1386
1387
    IRArgs()
1388
        : PaddingArgIndex(InvalidIndex), FirstArgIndex(InvalidIndex),
1389
2.02M
          NumberOfArgs(0) {}
1390
  };
1391
1392
  SmallVector<IRArgs, 8> ArgInfo;
1393
1394
public:
1395
  ClangToLLVMArgMapping(const ASTContext &Context, const CGFunctionInfo &FI,
1396
                        bool OnlyRequiredArgs = false)
1397
      : InallocaArgNo(InvalidIndex), SRetArgNo(InvalidIndex), TotalIRArgs(0),
1398
2.02M
        ArgInfo(OnlyRequiredArgs ? FI.getNumRequiredArgs() : FI.arg_size()) {
1399
2.02M
    construct(Context, FI, OnlyRequiredArgs);
1400
2.02M
  }
1401
1402
2.23M
  bool hasInallocaArg() const { return InallocaArgNo != InvalidIndex; }
1403
542
  unsigned getInallocaArgNo() const {
1404
542
    assert(hasInallocaArg());
1405
542
    return InallocaArgNo;
1406
542
  }
1407
1408
1.73M
  bool hasSRetArg() const { return SRetArgNo != InvalidIndex; }
1409
12.7k
  unsigned getSRetArgNo() const {
1410
12.7k
    assert(hasSRetArg());
1411
12.7k
    return SRetArgNo;
1412
12.7k
  }
1413
1414
2.26M
  unsigned totalIRArgs() const { return TotalIRArgs; }
1415
1416
2.89M
  bool hasPaddingArg(unsigned ArgNo) const {
1417
2.89M
    assert(ArgNo < ArgInfo.size());
1418
2.89M
    return ArgInfo[ArgNo].PaddingArgIndex != InvalidIndex;
1419
2.89M
  }
1420
69
  unsigned getPaddingArgNo(unsigned ArgNo) const {
1421
69
    assert(hasPaddingArg(ArgNo));
1422
69
    return ArgInfo[ArgNo].PaddingArgIndex;
1423
69
  }
1424
1425
  /// Returns index of first IR argument corresponding to ArgNo, and their
1426
  /// quantity.
1427
2.50M
  std::pair<unsigned, unsigned> getIRArgs(unsigned ArgNo) const {
1428
2.50M
    assert(ArgNo < ArgInfo.size());
1429
2.50M
    return std::make_pair(ArgInfo[ArgNo].FirstArgIndex,
1430
2.50M
                          ArgInfo[ArgNo].NumberOfArgs);
1431
2.50M
  }
1432
1433
private:
1434
  void construct(const ASTContext &Context, const CGFunctionInfo &FI,
1435
                 bool OnlyRequiredArgs);
1436
};
1437
1438
void ClangToLLVMArgMapping::construct(const ASTContext &Context,
1439
                                      const CGFunctionInfo &FI,
1440
2.02M
                                      bool OnlyRequiredArgs) {
1441
2.02M
  unsigned IRArgNo = 0;
1442
2.02M
  bool SwapThisWithSRet = false;
1443
2.02M
  const ABIArgInfo &RetAI = FI.getReturnInfo();
1444
2.02M
1445
2.02M
  if (RetAI.getKind() == ABIArgInfo::Indirect) {
1446
12.7k
    SwapThisWithSRet = RetAI.isSRetAfterThis();
1447
12.7k
    SRetArgNo = SwapThisWithSRet ? 
1389
:
IRArgNo++12.3k
;
1448
12.7k
  }
1449
2.02M
1450
2.02M
  unsigned ArgNo = 0;
1451
2.02M
  unsigned NumArgs = OnlyRequiredArgs ? 
FI.getNumRequiredArgs()899k
:
FI.arg_size()1.12M
;
1452
5.31M
  for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(); ArgNo < NumArgs;
1453
3.29M
       ++I, ++ArgNo) {
1454
3.29M
    assert(I != FI.arg_end());
1455
3.29M
    QualType ArgType = I->type;
1456
3.29M
    const ABIArgInfo &AI = I->info;
1457
3.29M
    // Collect data about IR arguments corresponding to Clang argument ArgNo.
1458
3.29M
    auto &IRArgs = ArgInfo[ArgNo];
1459
3.29M
1460
3.29M
    if (AI.getPaddingType())
1461
124
      IRArgs.PaddingArgIndex = IRArgNo++;
1462
3.29M
1463
3.29M
    switch (AI.getKind()) {
1464
3.21M
    case ABIArgInfo::Extend:
1465
3.21M
    case ABIArgInfo::Direct: {
1466
3.21M
      // FIXME: handle sseregparm someday...
1467
3.21M
      llvm::StructType *STy = dyn_cast<llvm::StructType>(AI.getCoerceToType());
1468
3.21M
      if (AI.isDirect() && 
AI.getCanBeFlattened()3.11M
&&
STy3.11M
) {
1469
7.37k
        IRArgs.NumberOfArgs = STy->getNumElements();
1470
3.20M
      } else {
1471
3.20M
        IRArgs.NumberOfArgs = 1;
1472
3.20M
      }
1473
3.21M
      break;
1474
3.21M
    }
1475
3.21M
    case ABIArgInfo::Indirect:
1476
14.0k
      IRArgs.NumberOfArgs = 1;
1477
14.0k
      break;
1478
3.21M
    case ABIArgInfo::Ignore:
1479
61.3k
    case ABIArgInfo::InAlloca:
1480
61.3k
      // ignore and inalloca doesn't have matching LLVM parameters.
1481
61.3k
      IRArgs.NumberOfArgs = 0;
1482
61.3k
      break;
1483
61.3k
    case ABIArgInfo::CoerceAndExpand:
1484
3.39k
      IRArgs.NumberOfArgs = AI.getCoerceAndExpandTypeSequence().size();
1485
3.39k
      break;
1486
61.3k
    case ABIArgInfo::Expand:
1487
436
      IRArgs.NumberOfArgs = getExpansionSize(ArgType, Context);
1488
436
      break;
1489
3.29M
    }
1490
3.29M
1491
3.29M
    if (IRArgs.NumberOfArgs > 0) {
1492
3.23M
      IRArgs.FirstArgIndex = IRArgNo;
1493
3.23M
      IRArgNo += IRArgs.NumberOfArgs;
1494
3.23M
    }
1495
3.29M
1496
3.29M
    // Skip over the sret parameter when it comes second.  We already handled it
1497
3.29M
    // above.
1498
3.29M
    if (IRArgNo == 1 && 
SwapThisWithSRet1.88M
)
1499
389
      IRArgNo++;
1500
3.29M
  }
1501
2.02M
  assert(ArgNo == ArgInfo.size());
1502
2.02M
1503
2.02M
  if (FI.usesInAlloca())
1504
444
    InallocaArgNo = IRArgNo++;
1505
2.02M
1506
2.02M
  TotalIRArgs = IRArgNo;
1507
2.02M
}
1508
}  // namespace
1509
1510
/***/
1511
1512
21.0k
bool CodeGenModule::ReturnTypeUsesSRet(const CGFunctionInfo &FI) {
1513
21.0k
  const auto &RI = FI.getReturnInfo();
1514
21.0k
  return RI.isIndirect() || 
(21.0k
RI.isInAlloca()21.0k
&&
RI.getInAllocaSRet()1
);
1515
21.0k
}
1516
1517
13.2k
bool CodeGenModule::ReturnSlotInterferesWithArgs(const CGFunctionInfo &FI) {
1518
13.2k
  return ReturnTypeUsesSRet(FI) &&
1519
13.2k
         
getTargetCodeGenInfo().doesReturnSlotInterfereWithArgs()40
;
1520
13.2k
}
1521
1522
12.1k
bool CodeGenModule::ReturnTypeUsesFPRet(QualType ResultType) {
1523
12.1k
  if (const BuiltinType *BT = ResultType->getAs<BuiltinType>()) {
1524
5.12k
    switch (BT->getKind()) {
1525
5.05k
    default:
1526
5.05k
      return false;
1527
18
    case BuiltinType::Float:
1528
18
      return getTarget().useObjCFPRetForRealType(TargetInfo::Float);
1529
46
    case BuiltinType::Double:
1530
46
      return getTarget().useObjCFPRetForRealType(TargetInfo::Double);
1531
4
    case BuiltinType::LongDouble:
1532
4
      return getTarget().useObjCFPRetForRealType(TargetInfo::LongDouble);
1533
7.05k
    }
1534
7.05k
  }
1535
7.05k
1536
7.05k
  return false;
1537
7.05k
}
1538
1539
12.1k
bool CodeGenModule::ReturnTypeUsesFP2Ret(QualType ResultType) {
1540
12.1k
  if (const ComplexType *CT = ResultType->getAs<ComplexType>()) {
1541
14
    if (const BuiltinType *BT = CT->getElementType()->getAs<BuiltinType>()) {
1542
14
      if (BT->getKind() == BuiltinType::LongDouble)
1543
2
        return getTarget().useObjCFP2RetForComplexLongDouble();
1544
12.1k
    }
1545
14
  }
1546
12.1k
1547
12.1k
  return false;
1548
12.1k
}
1549
1550
5.38k
llvm::FunctionType *CodeGenTypes::GetFunctionType(GlobalDecl GD) {
1551
5.38k
  const CGFunctionInfo &FI = arrangeGlobalDeclaration(GD);
1552
5.38k
  return GetFunctionType(FI);
1553
5.38k
}
1554
1555
llvm::FunctionType *
1556
899k
CodeGenTypes::GetFunctionType(const CGFunctionInfo &FI) {
1557
899k
1558
899k
  bool Inserted = FunctionsBeingProcessed.insert(&FI).second;
1559
899k
  (void)Inserted;
1560
899k
  assert(Inserted && "Recursively being processed?");
1561
899k
1562
899k
  llvm::Type *resultType = nullptr;
1563
899k
  const ABIArgInfo &retAI = FI.getReturnInfo();
1564
899k
  switch (retAI.getKind()) {
1565
0
  case ABIArgInfo::Expand:
1566
0
    llvm_unreachable("Invalid ABI kind for return argument");
1567
0
1568
470k
  case ABIArgInfo::Extend:
1569
470k
  case ABIArgInfo::Direct:
1570
470k
    resultType = retAI.getCoerceToType();
1571
470k
    break;
1572
470k
1573
470k
  case ABIArgInfo::InAlloca:
1574
18
    if (retAI.getInAllocaSRet()) {
1575
18
      // sret things on win32 aren't void, they return the sret pointer.
1576
18
      QualType ret = FI.getReturnType();
1577
18
      llvm::Type *ty = ConvertType(ret);
1578
18
      unsigned addressSpace = Context.getTargetAddressSpace(ret);
1579
18
      resultType = llvm::PointerType::get(ty, addressSpace);
1580
18
    } else {
1581
0
      resultType = llvm::Type::getVoidTy(getLLVMContext());
1582
0
    }
1583
18
    break;
1584
470k
1585
470k
  case ABIArgInfo::Indirect:
1586
427k
  case ABIArgInfo::Ignore:
1587
427k
    resultType = llvm::Type::getVoidTy(getLLVMContext());
1588
427k
    break;
1589
427k
1590
427k
  case ABIArgInfo::CoerceAndExpand:
1591
1.42k
    resultType = retAI.getUnpaddedCoerceAndExpandType();
1592
1.42k
    break;
1593
899k
  }
1594
899k
1595
899k
  ClangToLLVMArgMapping IRFunctionArgs(getContext(), FI, true);
1596
899k
  SmallVector<llvm::Type*, 8> ArgTypes(IRFunctionArgs.totalIRArgs());
1597
899k
1598
899k
  // Add type for sret argument.
1599
899k
  if (IRFunctionArgs.hasSRetArg()) {
1600
5.24k
    QualType Ret = FI.getReturnType();
1601
5.24k
    llvm::Type *Ty = ConvertType(Ret);
1602
5.24k
    unsigned AddressSpace = Context.getTargetAddressSpace(Ret);
1603
5.24k
    ArgTypes[IRFunctionArgs.getSRetArgNo()] =
1604
5.24k
        llvm::PointerType::get(Ty, AddressSpace);
1605
5.24k
  }
1606
899k
1607
899k
  // Add type for inalloca argument.
1608
899k
  if (IRFunctionArgs.hasInallocaArg()) {
1609
184
    auto ArgStruct = FI.getArgStruct();
1610
184
    assert(ArgStruct);
1611
184
    ArgTypes[IRFunctionArgs.getInallocaArgNo()] = ArgStruct->getPointerTo();
1612
184
  }
1613
899k
1614
899k
  // Add in all of the required arguments.
1615
899k
  unsigned ArgNo = 0;
1616
899k
  CGFunctionInfo::const_arg_iterator it = FI.arg_begin(),
1617
899k
                                     ie = it + FI.getNumRequiredArgs();
1618
2.33M
  for (; it != ie; 
++it, ++ArgNo1.43M
) {
1619
1.43M
    const ABIArgInfo &ArgInfo = it->info;
1620
1.43M
1621
1.43M
    // Insert a padding type to ensure proper alignment.
1622
1.43M
    if (IRFunctionArgs.hasPaddingArg(ArgNo))
1623
47
      ArgTypes[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
1624
47
          ArgInfo.getPaddingType();
1625
1.43M
1626
1.43M
    unsigned FirstIRArg, NumIRArgs;
1627
1.43M
    std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
1628
1.43M
1629
1.43M
    switch (ArgInfo.getKind()) {
1630
30.2k
    case ABIArgInfo::Ignore:
1631
30.2k
    case ABIArgInfo::InAlloca:
1632
30.2k
      assert(NumIRArgs == 0);
1633
30.2k
      break;
1634
30.2k
1635
30.2k
    case ABIArgInfo::Indirect: {
1636
5.88k
      assert(NumIRArgs == 1);
1637
5.88k
      // indirect arguments are always on the stack, which is alloca addr space.
1638
5.88k
      llvm::Type *LTy = ConvertTypeForMem(it->type);
1639
5.88k
      ArgTypes[FirstIRArg] = LTy->getPointerTo(
1640
5.88k
          CGM.getDataLayout().getAllocaAddrSpace());
1641
5.88k
      break;
1642
30.2k
    }
1643
30.2k
1644
1.39M
    case ABIArgInfo::Extend:
1645
1.39M
    case ABIArgInfo::Direct: {
1646
1.39M
      // Fast-isel and the optimizer generally like scalar values better than
1647
1.39M
      // FCAs, so we flatten them if this is safe to do for this argument.
1648
1.39M
      llvm::Type *argType = ArgInfo.getCoerceToType();
1649
1.39M
      llvm::StructType *st = dyn_cast<llvm::StructType>(argType);
1650
1.39M
      if (st && 
ArgInfo.isDirect()3.37k
&&
ArgInfo.getCanBeFlattened()3.37k
) {
1651
3.16k
        assert(NumIRArgs == st->getNumElements());
1652
9.61k
        for (unsigned i = 0, e = st->getNumElements(); i != e; 
++i6.45k
)
1653
6.45k
          ArgTypes[FirstIRArg + i] = st->getElementType(i);
1654
1.39M
      } else {
1655
1.39M
        assert(NumIRArgs == 1);
1656
1.39M
        ArgTypes[FirstIRArg] = argType;
1657
1.39M
      }
1658
1.39M
      break;
1659
1.39M
    }
1660
1.39M
1661
1.39M
    case ABIArgInfo::CoerceAndExpand: {
1662
1.42k
      auto ArgTypesIter = ArgTypes.begin() + FirstIRArg;
1663
3.06k
      for (auto EltTy : ArgInfo.getCoerceAndExpandTypeSequence()) {
1664
3.06k
        *ArgTypesIter++ = EltTy;
1665
3.06k
      }
1666
1.42k
      assert(ArgTypesIter == ArgTypes.begin() + FirstIRArg + NumIRArgs);
1667
1.42k
      break;
1668
1.39M
    }
1669
1.39M
1670
1.39M
    case ABIArgInfo::Expand:
1671
163
      auto ArgTypesIter = ArgTypes.begin() + FirstIRArg;
1672
163
      getExpandedTypes(it->type, ArgTypesIter);
1673
163
      assert(ArgTypesIter == ArgTypes.begin() + FirstIRArg + NumIRArgs);
1674
163
      break;
1675
1.43M
    }
1676
1.43M
  }
1677
899k
1678
899k
  bool Erased = FunctionsBeingProcessed.erase(&FI); (void)Erased;
1679
899k
  assert(Erased && "Not in set?");
1680
899k
1681
899k
  return llvm::FunctionType::get(resultType, ArgTypes, FI.isVariadic());
1682
899k
}
1683
1684
5.36k
llvm::Type *CodeGenTypes::GetFunctionTypeForVTable(GlobalDecl GD) {
1685
5.36k
  const CXXMethodDecl *MD = cast<CXXMethodDecl>(GD.getDecl());
1686
5.36k
  const FunctionProtoType *FPT = MD->getType()->getAs<FunctionProtoType>();
1687
5.36k
1688
5.36k
  if (!isFuncTypeConvertible(FPT))
1689
39
    return llvm::StructType::get(getLLVMContext());
1690
5.32k
1691
5.32k
  return GetFunctionType(GD);
1692
5.32k
}
1693
1694
static void AddAttributesFromFunctionProtoType(ASTContext &Ctx,
1695
                                               llvm::AttrBuilder &FuncAttrs,
1696
1.10M
                                               const FunctionProtoType *FPT) {
1697
1.10M
  if (!FPT)
1698
577k
    return;
1699
524k
1700
524k
  if (!isUnresolvedExceptionSpec(FPT->getExceptionSpecType()) &&
1701
524k
      FPT->isNothrow())
1702
165k
    FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
1703
524k
}
1704
1705
void CodeGenModule::ConstructDefaultFnAttrList(StringRef Name, bool HasOptnone,
1706
                                               bool AttrOnCallSite,
1707
575k
                                               llvm::AttrBuilder &FuncAttrs) {
1708
575k
  // OptimizeNoneAttr takes precedence over -Os or -Oz. No warning needed.
1709
575k
  if (!HasOptnone) {
1710
575k
    if (CodeGenOpts.OptimizeSize)
1711
1.06k
      FuncAttrs.addAttribute(llvm::Attribute::OptimizeForSize);
1712
575k
    if (CodeGenOpts.OptimizeSize == 2)
1713
666
      FuncAttrs.addAttribute(llvm::Attribute::MinSize);
1714
575k
  }
1715
575k
1716
575k
  if (CodeGenOpts.DisableRedZone)
1717
23
    FuncAttrs.addAttribute(llvm::Attribute::NoRedZone);
1718
575k
  if (CodeGenOpts.IndirectTlsSegRefs)
1719
1
    FuncAttrs.addAttribute("indirect-tls-seg-refs");
1720
575k
  if (CodeGenOpts.NoImplicitFloat)
1721
16
    FuncAttrs.addAttribute(llvm::Attribute::NoImplicitFloat);
1722
575k
1723
575k
  if (AttrOnCallSite) {
1724
305k
    // Attributes that should go on the call site only.
1725
305k
    if (!CodeGenOpts.SimplifyLibCalls ||
1726
305k
        
CodeGenOpts.isNoBuiltinFunc(Name.data())132k
)
1727
173k
      FuncAttrs.addAttribute(llvm::Attribute::NoBuiltin);
1728
305k
    if (!CodeGenOpts.TrapFuncName.empty())
1729
0
      FuncAttrs.addAttribute("trap-func-name", CodeGenOpts.TrapFuncName);
1730
305k
  } else {
1731
269k
    StringRef FpKind;
1732
269k
    switch (CodeGenOpts.getFramePointer()) {
1733
152k
    case CodeGenOptions::FramePointerKind::None:
1734
152k
      FpKind = "none";
1735
152k
      break;
1736
186
    case CodeGenOptions::FramePointerKind::NonLeaf:
1737
186
      FpKind = "non-leaf";
1738
186
      break;
1739
116k
    case CodeGenOptions::FramePointerKind::All:
1740
116k
      FpKind = "all";
1741
116k
      break;
1742
269k
    }
1743
269k
    FuncAttrs.addAttribute("frame-pointer", FpKind);
1744
269k
1745
269k
    FuncAttrs.addAttribute("less-precise-fpmad",
1746
269k
                           llvm::toStringRef(CodeGenOpts.LessPreciseFPMAD));
1747
269k
1748
269k
    if (CodeGenOpts.NullPointerIsValid)
1749
228
      FuncAttrs.addAttribute("null-pointer-is-valid", "true");
1750
269k
1751
269k
    // TODO: Omit attribute when the default is IEEE.
1752
269k
    if (CodeGenOpts.FPDenormalMode.isValid())
1753
2.66k
      FuncAttrs.addAttribute("denormal-fp-math",
1754
2.66k
                             CodeGenOpts.FPDenormalMode.str());
1755
269k
    if (CodeGenOpts.FP32DenormalMode.isValid()) {
1756
2.67k
      FuncAttrs.addAttribute(
1757
2.67k
          "denormal-fp-math-f32",
1758
2.67k
          CodeGenOpts.FP32DenormalMode.str());
1759
2.67k
    }
1760
269k
1761
269k
    FuncAttrs.addAttribute("no-trapping-math",
1762
269k
                           llvm::toStringRef(CodeGenOpts.NoTrappingMath));
1763
269k
1764
269k
    // Strict (compliant) code is the default, so only add this attribute to
1765
269k
    // indicate that we are trying to workaround a problem case.
1766
269k
    if (!CodeGenOpts.StrictFloatCastOverflow)
1767
1
      FuncAttrs.addAttribute("strict-float-cast-overflow", "false");
1768
269k
1769
269k
    // TODO: Are these all needed?
1770
269k
    // unsafe/inf/nan/nsz are handled by instruction-level FastMathFlags.
1771
269k
    FuncAttrs.addAttribute("no-infs-fp-math",
1772
269k
                           llvm::toStringRef(CodeGenOpts.NoInfsFPMath));
1773
269k
    FuncAttrs.addAttribute("no-nans-fp-math",
1774
269k
                           llvm::toStringRef(CodeGenOpts.NoNaNsFPMath));
1775
269k
    FuncAttrs.addAttribute("unsafe-fp-math",
1776
269k
                           llvm::toStringRef(CodeGenOpts.UnsafeFPMath));
1777
269k
    FuncAttrs.addAttribute("use-soft-float",
1778
269k
                           llvm::toStringRef(CodeGenOpts.SoftFloat));
1779
269k
    FuncAttrs.addAttribute("stack-protector-buffer-size",
1780
269k
                           llvm::utostr(CodeGenOpts.SSPBufferSize));
1781
269k
    FuncAttrs.addAttribute("no-signed-zeros-fp-math",
1782
269k
                           llvm::toStringRef(CodeGenOpts.NoSignedZeros));
1783
269k
    FuncAttrs.addAttribute(
1784
269k
        "correctly-rounded-divide-sqrt-fp-math",
1785
269k
        llvm::toStringRef(CodeGenOpts.CorrectlyRoundedDivSqrt));
1786
269k
1787
269k
    // TODO: Reciprocal estimate codegen options should apply to instructions?
1788
269k
    const std::vector<std::string> &Recips = CodeGenOpts.Reciprocals;
1789
269k
    if (!Recips.empty())
1790
1
      FuncAttrs.addAttribute("reciprocal-estimates",
1791
1
                             llvm::join(Recips, ","));
1792
269k
1793
269k
    if (!CodeGenOpts.PreferVectorWidth.empty() &&
1794
269k
        
CodeGenOpts.PreferVectorWidth != "none"3
)
1795
2
      FuncAttrs.addAttribute("prefer-vector-width",
1796
2
                             CodeGenOpts.PreferVectorWidth);
1797
269k
1798
269k
    if (CodeGenOpts.StackRealignment)
1799
1
      FuncAttrs.addAttribute("stackrealign");
1800
269k
    if (CodeGenOpts.Backchain)
1801
2
      FuncAttrs.addAttribute("backchain");
1802
269k
1803
269k
    if (CodeGenOpts.SpeculativeLoadHardening)
1804
9
      FuncAttrs.addAttribute(llvm::Attribute::SpeculativeLoadHardening);
1805
269k
  }
1806
575k
1807
575k
  if (getLangOpts().assumeFunctionsAreConvergent()) {
1808
69.9k
    // Conservatively, mark all functions and calls in CUDA and OpenCL as
1809
69.9k
    // convergent (meaning, they may call an intrinsically convergent op, such
1810
69.9k
    // as __syncthreads() / barrier(), and so can't have certain optimizations
1811
69.9k
    // applied around them).  LLVM will remove this attribute where it safely
1812
69.9k
    // can.
1813
69.9k
    FuncAttrs.addAttribute(llvm::Attribute::Convergent);
1814
69.9k
  }
1815
575k
1816
575k
  if (getLangOpts().CUDA && 
getLangOpts().CUDAIsDevice1.06k
) {
1817
683
    // Exceptions aren't supported in CUDA device code.
1818
683
    FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
1819
683
  }
1820
575k
1821
575k
  for (StringRef Attr : CodeGenOpts.DefaultFunctionAttrs) {
1822
24
    StringRef Var, Value;
1823
24
    std::tie(Var, Value) = Attr.split('=');
1824
24
    FuncAttrs.addAttribute(Var, Value);
1825
24
  }
1826
575k
}
1827
1828
22
void CodeGenModule::AddDefaultFnAttrs(llvm::Function &F) {
1829
22
  llvm::AttrBuilder FuncAttrs;
1830
22
  ConstructDefaultFnAttrList(F.getName(), F.hasOptNone(),
1831
22
                             /* AttrOnCallSite = */ false, FuncAttrs);
1832
22
  F.addAttributes(llvm::AttributeList::FunctionIndex, FuncAttrs);
1833
22
}
1834
1835
static void addNoBuiltinAttributes(llvm::AttrBuilder &FuncAttrs,
1836
                                   const LangOptions &LangOpts,
1837
575k
                                   const NoBuiltinAttr *NBA = nullptr) {
1838
575k
  auto AddNoBuiltinAttr = [&FuncAttrs](StringRef BuiltinName) {
1839
2.35k
    SmallString<32> AttributeName;
1840
2.35k
    AttributeName += "no-builtin-";
1841
2.35k
    AttributeName += BuiltinName;
1842
2.35k
    FuncAttrs.addAttribute(AttributeName);
1843
2.35k
  };
1844
575k
1845
575k
  // First, handle the language options passed through -fno-builtin.
1846
575k
  if (LangOpts.NoBuiltin) {
1847
354k
    // -fno-builtin disables them all.
1848
354k
    FuncAttrs.addAttribute("no-builtins");
1849
354k
    return;
1850
354k
  }
1851
220k
1852
220k
  // Then, add attributes for builtins specified through -fno-builtin-<name>.
1853
220k
  llvm::for_each(LangOpts.NoBuiltinFuncs, AddNoBuiltinAttr);
1854
220k
1855
220k
  // Now, let's check the __attribute__((no_builtin("...")) attribute added to
1856
220k
  // the source.
1857
220k
  if (!NBA)
1858
220k
    return;
1859
9
1860
9
  // If there is a wildcard in the builtin names specified through the
1861
9
  // attribute, disable them all.
1862
9
  if (llvm::is_contained(NBA->builtinNames(), "*")) {
1863
1
    FuncAttrs.addAttribute("no-builtins");
1864
1
    return;
1865
1
  }
1866
8
1867
8
  // And last, add the rest of the builtin names.
1868
8
  llvm::for_each(NBA->builtinNames(), AddNoBuiltinAttr);
1869
8
}
1870
1871
void CodeGenModule::ConstructAttributeList(
1872
    StringRef Name, const CGFunctionInfo &FI, CGCalleeInfo CalleeInfo,
1873
575k
    llvm::AttributeList &AttrList, unsigned &CallingConv, bool AttrOnCallSite) {
1874
575k
  llvm::AttrBuilder FuncAttrs;
1875
575k
  llvm::AttrBuilder RetAttrs;
1876
575k
1877
575k
  CallingConv = FI.getEffectiveCallingConvention();
1878
575k
  if (FI.isNoReturn())
1879
1.04k
    FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
1880
575k
1881
575k
  // If we have information about the function prototype, we can learn
1882
575k
  // attributes from there.
1883
575k
  AddAttributesFromFunctionProtoType(getContext(), FuncAttrs,
1884
575k
                                     CalleeInfo.getCalleeFunctionProtoType());
1885
575k
1886
575k
  const Decl *TargetDecl = CalleeInfo.getCalleeDecl().getDecl();
1887
575k
1888
575k
  bool HasOptnone = false;
1889
575k
  // The NoBuiltinAttr attached to a TargetDecl (only allowed on FunctionDecls).
1890
575k
  const NoBuiltinAttr *NBA = nullptr;
1891
575k
  // FIXME: handle sseregparm someday...
1892
575k
  if (TargetDecl) {
1893
548k
    if (TargetDecl->hasAttr<ReturnsTwiceAttr>())
1894
25
      FuncAttrs.addAttribute(llvm::Attribute::ReturnsTwice);
1895
548k
    if (TargetDecl->hasAttr<NoThrowAttr>())
1896
4.64k
      FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
1897
548k
    if (TargetDecl->hasAttr<NoReturnAttr>())
1898
0
      FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
1899
548k
    if (TargetDecl->hasAttr<ColdAttr>())
1900
16
      FuncAttrs.addAttribute(llvm::Attribute::Cold);
1901
548k
    if (TargetDecl->hasAttr<NoDuplicateAttr>())
1902
6
      FuncAttrs.addAttribute(llvm::Attribute::NoDuplicate);
1903
548k
    if (TargetDecl->hasAttr<ConvergentAttr>())
1904
6
      FuncAttrs.addAttribute(llvm::Attribute::Convergent);
1905
548k
1906
548k
    if (const FunctionDecl *Fn = dyn_cast<FunctionDecl>(TargetDecl)) {
1907
526k
      AddAttributesFromFunctionProtoType(
1908
526k
          getContext(), FuncAttrs, Fn->getType()->getAs<FunctionProtoType>());
1909
526k
      const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Fn);
1910
526k
      const bool IsVirtualCall = MD && 
MD->isVirtual()250k
;
1911
526k
      // Don't use [[noreturn]], _Noreturn or [[no_builtin]] for a call to a
1912
526k
      // virtual function. These attributes are not inherited by overloads.
1913
526k
      if (!(AttrOnCallSite && 
IsVirtualCall287k
)) {
1914
521k
        if (Fn->isNoReturn())
1915
2.67k
          FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
1916
521k
        NBA = Fn->getAttr<NoBuiltinAttr>();
1917
521k
      }
1918
526k
    }
1919
548k
1920
548k
    // 'const', 'pure' and 'noalias' attributed functions are also nounwind.
1921
548k
    if (TargetDecl->hasAttr<ConstAttr>()) {
1922
1.37k
      FuncAttrs.addAttribute(llvm::Attribute::ReadNone);
1923
1.37k
      FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
1924
546k
    } else if (TargetDecl->hasAttr<PureAttr>()) {
1925
193
      FuncAttrs.addAttribute(llvm::Attribute::ReadOnly);
1926
193
      FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
1927
546k
    } else if (TargetDecl->hasAttr<NoAliasAttr>()) {
1928
2
      FuncAttrs.addAttribute(llvm::Attribute::ArgMemOnly);
1929
2
      FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
1930
2
    }
1931
548k
    if (TargetDecl->hasAttr<RestrictAttr>())
1932
54
      RetAttrs.addAttribute(llvm::Attribute::NoAlias);
1933
548k
    if (TargetDecl->hasAttr<ReturnsNonNullAttr>() &&
1934
548k
        
!CodeGenOpts.NullPointerIsValid8
)
1935
7
      RetAttrs.addAttribute(llvm::Attribute::NonNull);
1936
548k
    if (TargetDecl->hasAttr<AnyX86NoCallerSavedRegistersAttr>())
1937
14
      FuncAttrs.addAttribute("no_caller_saved_registers");
1938
548k
    if (TargetDecl->hasAttr<AnyX86NoCfCheckAttr>())
1939
2
      FuncAttrs.addAttribute(llvm::Attribute::NoCfCheck);
1940
548k
1941
548k
    HasOptnone = TargetDecl->hasAttr<OptimizeNoneAttr>();
1942
548k
    if (auto *AllocSize = TargetDecl->getAttr<AllocSizeAttr>()) {
1943
269
      Optional<unsigned> NumElemsParam;
1944
269
      if (AllocSize->getNumElemsParam().isValid())
1945
133
        NumElemsParam = AllocSize->getNumElemsParam().getLLVMIndex();
1946
269
      FuncAttrs.addAllocSizeAttr(AllocSize->getElemSizeParam().getLLVMIndex(),
1947
269
                                 NumElemsParam);
1948
269
    }
1949
548k
  }
1950
575k
1951
575k
  // Attach "no-builtins" attributes to:
1952
575k
  // * call sites: both `nobuiltin` and "no-builtins" or "no-builtin-<name>".
1953
575k
  // * definitions: "no-builtins" or "no-builtin-<name>" only.
1954
575k
  // The attributes can come from:
1955
575k
  // * LangOpts: -ffreestanding, -fno-builtin, -fno-builtin-<name>
1956
575k
  // * FunctionDecl attributes: __attribute__((no_builtin(...)))
1957
575k
  addNoBuiltinAttributes(FuncAttrs, getLangOpts(), NBA);
1958
575k
1959
575k
  ConstructDefaultFnAttrList(Name, HasOptnone, AttrOnCallSite, FuncAttrs);
1960
575k
1961
575k
  // This must run after constructing the default function attribute list
1962
575k
  // to ensure that the speculative load hardening attribute is removed
1963
575k
  // in the case where the -mspeculative-load-hardening flag was passed.
1964
575k
  if (TargetDecl) {
1965
548k
    if (TargetDecl->hasAttr<NoSpeculativeLoadHardeningAttr>())
1966
13
      FuncAttrs.removeAttribute(llvm::Attribute::SpeculativeLoadHardening);
1967
548k
    if (TargetDecl->hasAttr<SpeculativeLoadHardeningAttr>())
1968
13
      FuncAttrs.addAttribute(llvm::Attribute::SpeculativeLoadHardening);
1969
548k
  }
1970
575k
1971
575k
  if (CodeGenOpts.EnableSegmentedStacks &&
1972
575k
      
!(8
TargetDecl8
&&
TargetDecl->hasAttr<NoSplitStackAttr>()8
))
1973
4
    FuncAttrs.addAttribute("split-stack");
1974
575k
1975
575k
  // Add NonLazyBind attribute to function declarations when -fno-plt
1976
575k
  // is used.
1977
575k
  if (TargetDecl && 
CodeGenOpts.NoPLT548k
) {
1978
45
    if (auto *Fn = dyn_cast<FunctionDecl>(TargetDecl)) {
1979
45
      if (!Fn->isDefined() && 
!AttrOnCallSite22
) {
1980
12
        FuncAttrs.addAttribute(llvm::Attribute::NonLazyBind);
1981
12
      }
1982
45
    }
1983
45
  }
1984
575k
1985
575k
  if (TargetDecl && 
TargetDecl->hasAttr<OpenCLKernelAttr>()548k
) {
1986
431
    if (getLangOpts().OpenCLVersion <= 120) {
1987
299
      // OpenCL v1.2 Work groups are always uniform
1988
299
      FuncAttrs.addAttribute("uniform-work-group-size", "true");
1989
299
    } else {
1990
132
      // OpenCL v2.0 Work groups may be whether uniform or not.
1991
132
      // '-cl-uniform-work-group-size' compile option gets a hint
1992
132
      // to the compiler that the global work-size be a multiple of
1993
132
      // the work-group size specified to clEnqueueNDRangeKernel
1994
132
      // (i.e. work groups are uniform).
1995
132
      FuncAttrs.addAttribute("uniform-work-group-size",
1996
132
                             llvm::toStringRef(CodeGenOpts.UniformWGSize));
1997
132
    }
1998
431
  }
1999
575k
2000
575k
  if (!AttrOnCallSite) {
2001
269k
    bool DisableTailCalls = false;
2002
269k
2003
269k
    if (CodeGenOpts.DisableTailCalls)
2004
3
      DisableTailCalls = true;
2005
269k
    else if (TargetDecl) {
2006
258k
      if (TargetDecl->hasAttr<DisableTailCallsAttr>() ||
2007
258k
          
TargetDecl->hasAttr<AnyX86InterruptAttr>()258k
)
2008
107
        DisableTailCalls = true;
2009
258k
      else if (CodeGenOpts.NoEscapingBlockTailCalls) {
2010
17
        if (const auto *BD = dyn_cast<BlockDecl>(TargetDecl))
2011
8
          if (!BD->doesNotEscape())
2012
4
            DisableTailCalls = true;
2013
17
      }
2014
258k
    }
2015
269k
2016
269k
    FuncAttrs.addAttribute("disable-tail-calls",
2017
269k
                           llvm::toStringRef(DisableTailCalls));
2018
269k
    GetCPUAndFeaturesAttributes(CalleeInfo.getCalleeDecl(), FuncAttrs);
2019
269k
  }
2020
575k
2021
575k
  ClangToLLVMArgMapping IRFunctionArgs(getContext(), FI);
2022
575k
2023
575k
  QualType RetTy = FI.getReturnType();
2024
575k
  const ABIArgInfo &RetAI = FI.getReturnInfo();
2025
575k
  switch (RetAI.getKind()) {
2026
20.3k
  case ABIArgInfo::Extend:
2027
20.3k
    if (RetAI.isSignExt())
2028
10.2k
      RetAttrs.addAttribute(llvm::Attribute::SExt);
2029
10.0k
    else
2030
10.0k
      RetAttrs.addAttribute(llvm::Attribute::ZExt);
2031
20.3k
    LLVM_FALLTHROUGH;
2032
315k
  case ABIArgInfo::Direct:
2033
315k
    if (RetAI.getInReg())
2034
30
      RetAttrs.addAttribute(llvm::Attribute::InReg);
2035
315k
    break;
2036
254k
  case ABIArgInfo::Ignore:
2037
254k
    break;
2038
20.3k
2039
20.3k
  case ABIArgInfo::InAlloca:
2040
3.91k
  case ABIArgInfo::Indirect: {
2041
3.91k
    // inalloca and sret disable readnone and readonly
2042
3.91k
    FuncAttrs.removeAttribute(llvm::Attribute::ReadOnly)
2043
3.91k
      .removeAttribute(llvm::Attribute::ReadNone);
2044
3.91k
    break;
2045
3.91k
  }
2046
3.91k
2047
3.91k
  case ABIArgInfo::CoerceAndExpand:
2048
983
    break;
2049
3.91k
2050
3.91k
  case ABIArgInfo::Expand:
2051
0
    llvm_unreachable("Invalid ABI kind for return argument");
2052
575k
  }
2053
575k
2054
575k
  if (const auto *RefTy = RetTy->getAs<ReferenceType>()) {
2055
62.8k
    QualType PTy = RefTy->getPointeeType();
2056
62.8k
    if (!PTy->isIncompleteType() && 
PTy->isConstantSizeType()62.8k
)
2057
62.8k
      RetAttrs.addDereferenceableAttr(
2058
62.8k
          getMinimumObjectSize(PTy).getQuantity());
2059
34
    else if (getContext().getTargetAddressSpace(PTy) == 0 &&
2060
34
             
!CodeGenOpts.NullPointerIsValid32
)
2061
29
      RetAttrs.addAttribute(llvm::Attribute::NonNull);
2062
62.8k
  }
2063
575k
2064
575k
  bool hasUsedSRet = false;
2065
575k
  SmallVector<llvm::AttributeSet, 4> ArgAttrs(IRFunctionArgs.totalIRArgs());
2066
575k
2067
575k
  // Attach attributes to sret.
2068
575k
  if (IRFunctionArgs.hasSRetArg()) {
2069
3.89k
    llvm::AttrBuilder SRETAttrs;
2070
3.89k
    SRETAttrs.addAttribute(llvm::Attribute::StructRet);
2071
3.89k
    hasUsedSRet = true;
2072
3.89k
    if (RetAI.getInReg())
2073
49
      SRETAttrs.addAttribute(llvm::Attribute::InReg);
2074
3.89k
    ArgAttrs[IRFunctionArgs.getSRetArgNo()] =
2075
3.89k
        llvm::AttributeSet::get(getLLVMContext(), SRETAttrs);
2076
3.89k
  }
2077
575k
2078
575k
  // Attach attributes to inalloca argument.
2079
575k
  if (IRFunctionArgs.hasInallocaArg()) {
2080
152
    llvm::AttrBuilder Attrs;
2081
152
    Attrs.addAttribute(llvm::Attribute::InAlloca);
2082
152
    ArgAttrs[IRFunctionArgs.getInallocaArgNo()] =
2083
152
        llvm::AttributeSet::get(getLLVMContext(), Attrs);
2084
152
  }
2085
575k
2086
575k
  unsigned ArgNo = 0;
2087
575k
  for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(),
2088
575k
                                          E = FI.arg_end();
2089
1.51M
       I != E; 
++I, ++ArgNo944k
) {
2090
944k
    QualType ParamType = I->type;
2091
944k
    const ABIArgInfo &AI = I->info;
2092
944k
    llvm::AttrBuilder Attrs;
2093
944k
2094
944k
    // Add attribute for padding argument, if necessary.
2095
944k
    if (IRFunctionArgs.hasPaddingArg(ArgNo)) {
2096
47
      if (AI.getPaddingInReg()) {
2097
5
        ArgAttrs[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
2098
5
            llvm::AttributeSet::get(
2099
5
                getLLVMContext(),
2100
5
                llvm::AttrBuilder().addAttribute(llvm::Attribute::InReg));
2101
5
      }
2102
47
    }
2103
944k
2104
944k
    // 'restrict' -> 'noalias' is done in EmitFunctionProlog when we
2105
944k
    // have the corresponding parameter variable.  It doesn't make
2106
944k
    // sense to do it here because parameters are so messed up.
2107
944k
    switch (AI.getKind()) {
2108
26.8k
    case ABIArgInfo::Extend:
2109
26.8k
      if (AI.isSignExt())
2110
12.8k
        Attrs.addAttribute(llvm::Attribute::SExt);
2111
14.0k
      else
2112
14.0k
        Attrs.addAttribute(llvm::Attribute::ZExt);
2113
26.8k
      LLVM_FALLTHROUGH;
2114
923k
    case ABIArgInfo::Direct:
2115
923k
      if (ArgNo == 0 && 
FI.isChainCall()516k
)
2116
8
        Attrs.addAttribute(llvm::Attribute::Nest);
2117
923k
      else if (AI.getInReg())
2118
558
        Attrs.addAttribute(llvm::Attribute::InReg);
2119
923k
      break;
2120
26.8k
2121
26.8k
    case ABIArgInfo::Indirect: {
2122
4.32k
      if (AI.getInReg())
2123
24
        Attrs.addAttribute(llvm::Attribute::InReg);
2124
4.32k
2125
4.32k
      if (AI.getIndirectByVal())
2126
1.67k
        Attrs.addByValAttr(getTypes().ConvertTypeForMem(ParamType));
2127
4.32k
2128
4.32k
      CharUnits Align = AI.getIndirectAlign();
2129
4.32k
2130
4.32k
      // In a byval argument, it is important that the required
2131
4.32k
      // alignment of the type is honored, as LLVM might be creating a
2132
4.32k
      // *new* stack object, and needs to know what alignment to give
2133
4.32k
      // it. (Sometimes it can deduce a sensible alignment on its own,
2134
4.32k
      // but not if clang decides it must emit a packed struct, or the
2135
4.32k
      // user specifies increased alignment requirements.)
2136
4.32k
      //
2137
4.32k
      // This is different from indirect *not* byval, where the object
2138
4.32k
      // exists already, and the align attribute is purely
2139
4.32k
      // informative.
2140
4.32k
      assert(!Align.isZero());
2141
4.32k
2142
4.32k
      // For now, only add this when we have a byval argument.
2143
4.32k
      // TODO: be less lazy about updating test cases.
2144
4.32k
      if (AI.getIndirectByVal())
2145
1.67k
        Attrs.addAlignmentAttr(Align.getQuantity());
2146
4.32k
2147
4.32k
      // byval disables readnone and readonly.
2148
4.32k
      FuncAttrs.removeAttribute(llvm::Attribute::ReadOnly)
2149
4.32k
        .removeAttribute(llvm::Attribute::ReadNone);
2150
4.32k
      break;
2151
26.8k
    }
2152
26.8k
    case ABIArgInfo::Ignore:
2153
16.4k
    case ABIArgInfo::Expand:
2154
16.4k
    case ABIArgInfo::CoerceAndExpand:
2155
16.4k
      break;
2156
16.4k
2157
16.4k
    case ABIArgInfo::InAlloca:
2158
304
      // inalloca disables readnone and readonly.
2159
304
      FuncAttrs.removeAttribute(llvm::Attribute::ReadOnly)
2160
304
          .removeAttribute(llvm::Attribute::ReadNone);
2161
304
      continue;
2162
943k
    }
2163
943k
2164
943k
    if (const auto *RefTy = ParamType->getAs<ReferenceType>()) {
2165
125k
      QualType PTy = RefTy->getPointeeType();
2166
125k
      if (!PTy->isIncompleteType() && 
PTy->isConstantSizeType()125k
)
2167
125k
        Attrs.addDereferenceableAttr(
2168
125k
            getMinimumObjectSize(PTy).getQuantity());
2169
187
      else if (getContext().getTargetAddressSpace(PTy) == 0 &&
2170
187
               
!CodeGenOpts.NullPointerIsValid183
)
2171
174
        Attrs.addAttribute(llvm::Attribute::NonNull);
2172
125k
    }
2173
943k
2174
943k
    switch (FI.getExtParameterInfo(ArgNo).getABI()) {
2175
943k
    case ParameterABI::Ordinary:
2176
943k
      break;
2177
0
2178
48
    case ParameterABI::SwiftIndirectResult: {
2179
48
      // Add 'sret' if we haven't already used it for something, but
2180
48
      // only if the result is void.
2181
48
      if (!hasUsedSRet && 
RetTy->isVoidType()24
) {
2182
8
        Attrs.addAttribute(llvm::Attribute::StructRet);
2183
8
        hasUsedSRet = true;
2184
8
      }
2185
48
2186
48
      // Add 'noalias' in either case.
2187
48
      Attrs.addAttribute(llvm::Attribute::NoAlias);
2188
48
2189
48
      // Add 'dereferenceable' and 'alignment'.
2190
48
      auto PTy = ParamType->getPointeeType();
2191
48
      if (!PTy->isIncompleteType() && PTy->isConstantSizeType()) {
2192
48
        auto info = getContext().getTypeInfoInChars(PTy);
2193
48
        Attrs.addDereferenceableAttr(info.first.getQuantity());
2194
48
        Attrs.addAttribute(llvm::Attribute::getWithAlignment(
2195
48
            getLLVMContext(), info.second.getAsAlign()));
2196
48
      }
2197
48
      break;
2198
0
    }
2199
0
2200
24
    case ParameterABI::SwiftErrorResult:
2201
24
      Attrs.addAttribute(llvm::Attribute::SwiftError);
2202
24
      break;
2203
0
2204
40
    case ParameterABI::SwiftContext:
2205
40
      Attrs.addAttribute(llvm::Attribute::SwiftSelf);
2206
40
      break;
2207
943k
    }
2208
943k
2209
943k
    if (FI.getExtParameterInfo(ArgNo).isNoEscape())
2210
58
      Attrs.addAttribute(llvm::Attribute::NoCapture);
2211
943k
2212
943k
    if (Attrs.hasAttributes()) {
2213
152k
      unsigned FirstIRArg, NumIRArgs;
2214
152k
      std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
2215
306k
      for (unsigned i = 0; i < NumIRArgs; 
i++153k
)
2216
153k
        ArgAttrs[FirstIRArg + i] =
2217
153k
            llvm::AttributeSet::get(getLLVMContext(), Attrs);
2218
152k
    }
2219
943k
  }
2220
575k
  assert(ArgNo == FI.arg_size());
2221
575k
2222
575k
  AttrList = llvm::AttributeList::get(
2223
575k
      getLLVMContext(), llvm::AttributeSet::get(getLLVMContext(), FuncAttrs),
2224
575k
      llvm::AttributeSet::get(getLLVMContext(), RetAttrs), ArgAttrs);
2225
575k
}
2226
2227
/// An argument came in as a promoted argument; demote it back to its
2228
/// declared type.
2229
static llvm::Value *emitArgumentDemotion(CodeGenFunction &CGF,
2230
                                         const VarDecl *var,
2231
14
                                         llvm::Value *value) {
2232
14
  llvm::Type *varType = CGF.ConvertType(var->getType());
2233
14
2234
14
  // This can happen with promotions that actually don't change the
2235
14
  // underlying type, like the enum promotions.
2236
14
  if (value->getType() == varType) 
return value0
;
2237
14
2238
14
  assert((varType->isIntegerTy() || varType->isFloatingPointTy())
2239
14
         && "unexpected promotion type");
2240
14
2241
14
  if (isa<llvm::IntegerType>(varType))
2242
8
    return CGF.Builder.CreateTrunc(value, varType, "arg.unpromote");
2243
6
2244
6
  return CGF.Builder.CreateFPCast(value, varType, "arg.unpromote");
2245
6
}
2246
2247
/// Returns the attribute (either parameter attribute, or function
2248
/// attribute), which declares argument ArgNo to be non-null.
2249
static const NonNullAttr *getNonNullAttr(const Decl *FD, const ParmVarDecl *PVD,
2250
224k
                                         QualType ArgType, unsigned ArgNo) {
2251
224k
  // FIXME: __attribute__((nonnull)) can also be applied to:
2252
224k
  //   - references to pointers, where the pointee is known to be
2253
224k
  //     nonnull (apparently a Clang extension)
2254
224k
  //   - transparent unions containing pointers
2255
224k
  // In the former case, LLVM IR cannot represent the constraint. In
2256
224k
  // the latter case, we have no guarantee that the transparent union
2257
224k
  // is in fact passed as a pointer.
2258
224k
  if (!ArgType->isAnyPointerType() && 
!ArgType->isBlockPointerType()173k
)
2259
173k
    return nullptr;
2260
50.5k
  // First, check attribute on parameter itself.
2261
50.5k
  if (PVD) {
2262
50.5k
    if (auto ParmNNAttr = PVD->getAttr<NonNullAttr>())
2263
10
      return ParmNNAttr;
2264
50.5k
  }
2265
50.5k
  // Check function attributes.
2266
50.5k
  if (!FD)
2267
5
    return nullptr;
2268
50.5k
  for (const auto *NNAttr : FD->specific_attrs<NonNullAttr>()) {
2269
32
    if (NNAttr->isNonNull(ArgNo))
2270
28
      return NNAttr;
2271
32
  }
2272
50.5k
  
return nullptr50.4k
;
2273
50.5k
}
2274
2275
namespace {
2276
  struct CopyBackSwiftError final : EHScopeStack::Cleanup {
2277
    Address Temp;
2278
    Address Arg;
2279
16
    CopyBackSwiftError(Address temp, Address arg) : Temp(temp), Arg(arg) {}
2280
16
    void Emit(CodeGenFunction &CGF, Flags flags) override {
2281
16
      llvm::Value *errorValue = CGF.Builder.CreateLoad(Temp);
2282
16
      CGF.Builder.CreateStore(errorValue, Arg);
2283
16
    }
2284
  };
2285
}
2286
2287
void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
2288
                                         llvm::Function *Fn,
2289
242k
                                         const FunctionArgList &Args) {
2290
242k
  if (CurCodeDecl && 
CurCodeDecl->hasAttr<NakedAttr>()236k
)
2291
7
    // Naked functions don't have prologues.
2292
7
    return;
2293
242k
2294
242k
  // If this is an implicit-return-zero function, go ahead and
2295
242k
  // initialize the return value.  TODO: it might be nice to have
2296
242k
  // a more general mechanism for this that didn't require synthesized
2297
242k
  // return statements.
2298
242k
  if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(CurCodeDecl)) {
2299
213k
    if (FD->hasImplicitReturnZero()) {
2300
4.81k
      QualType RetTy = FD->getReturnType().getUnqualifiedType();
2301
4.81k
      llvm::Type* LLVMTy = CGM.getTypes().ConvertType(RetTy);
2302
4.81k
      llvm::Constant* Zero = llvm::Constant::getNullValue(LLVMTy);
2303
4.81k
      Builder.CreateStore(Zero, ReturnValue);
2304
4.81k
    }
2305
213k
  }
2306
242k
2307
242k
  // FIXME: We no longer need the types from FunctionArgList; lift up and
2308
242k
  // simplify.
2309
242k
2310
242k
  ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), FI);
2311
242k
  // Flattened function arguments.
2312
242k
  SmallVector<llvm::Value *, 16> FnArgs;
2313
242k
  FnArgs.reserve(IRFunctionArgs.totalIRArgs());
2314
389k
  for (auto &Arg : Fn->args()) {
2315
389k
    FnArgs.push_back(&Arg);
2316
389k
  }
2317
242k
  assert(FnArgs.size() == IRFunctionArgs.totalIRArgs());
2318
242k
2319
242k
  // If we're using inalloca, all the memory arguments are GEPs off of the last
2320
242k
  // parameter, which is a pointer to the complete memory area.
2321
242k
  Address ArgStruct = Address::invalid();
2322
242k
  if (IRFunctionArgs.hasInallocaArg()) {
2323
39
    ArgStruct = Address(FnArgs[IRFunctionArgs.getInallocaArgNo()],
2324
39
                        FI.getArgStructAlignment());
2325
39
2326
39
    assert(ArgStruct.getType() == FI.getArgStruct()->getPointerTo());
2327
39
  }
2328
242k
2329
242k
  // Name the struct return parameter.
2330
242k
  if (IRFunctionArgs.hasSRetArg()) {
2331
2.24k
    auto AI = cast<llvm::Argument>(FnArgs[IRFunctionArgs.getSRetArgNo()]);
2332
2.24k
    AI->setName("agg.result");
2333
2.24k
    AI->addAttr(llvm::Attribute::NoAlias);
2334
2.24k
  }
2335
242k
2336
242k
  // Track if we received the parameter as a pointer (indirect, byval, or
2337
242k
  // inalloca).  If already have a pointer, EmitParmDecl doesn't need to copy it
2338
242k
  // into a local alloca for us.
2339
242k
  SmallVector<ParamValue, 16> ArgVals;
2340
242k
  ArgVals.reserve(Args.size());
2341
242k
2342
242k
  // Create a pointer value for every parameter declaration.  This usually
2343
242k
  // entails copying one or more LLVM IR arguments into an alloca.  Don't push
2344
242k
  // any cleanups or do anything that might unwind.  We do that separately, so
2345
242k
  // we can push the cleanups in the correct order for the ABI.
2346
242k
  assert(FI.arg_size() == Args.size() &&
2347
242k
         "Mismatch between function signature & arguments.");
2348
242k
  unsigned ArgNo = 0;
2349
242k
  CGFunctionInfo::const_arg_iterator info_it = FI.arg_begin();
2350
242k
  for (FunctionArgList::const_iterator i = Args.begin(), e = Args.end();
2351
636k
       i != e; 
++i, ++info_it, ++ArgNo393k
) {
2352
393k
    const VarDecl *Arg = *i;
2353
393k
    const ABIArgInfo &ArgI = info_it->info;
2354
393k
2355
393k
    bool isPromoted =
2356
393k
      isa<ParmVarDecl>(Arg) && 
cast<ParmVarDecl>(Arg)->isKNRPromoted()239k
;
2357
393k
    // We are converting from ABIArgInfo type to VarDecl type directly, unless
2358
393k
    // the parameter is promoted. In this case we convert to
2359
393k
    // CGFunctionInfo::ArgInfo type with subsequent argument demotion.
2360
393k
    QualType Ty = isPromoted ? 
info_it->type14
:
Arg->getType()393k
;
2361
393k
    assert(hasScalarEvaluationKind(Ty) ==
2362
393k
           hasScalarEvaluationKind(Arg->getType()));
2363
393k
2364
393k
    unsigned FirstIRArg, NumIRArgs;
2365
393k
    std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
2366
393k
2367
393k
    switch (ArgI.getKind()) {
2368
70
    case ABIArgInfo::InAlloca: {
2369
70
      assert(NumIRArgs == 0);
2370
70
      auto FieldIndex = ArgI.getInAllocaFieldIndex();
2371
70
      Address V =
2372
70
          Builder.CreateStructGEP(ArgStruct, FieldIndex, Arg->getName());
2373
70
      if (ArgI.getInAllocaIndirect())
2374
5
        V = Address(Builder.CreateLoad(V),
2375
5
                    getContext().getTypeAlignInChars(Ty));
2376
70
      ArgVals.push_back(ParamValue::forIndirect(V));
2377
70
      break;
2378
0
    }
2379
0
2380
2.04k
    case ABIArgInfo::Indirect: {
2381
2.04k
      assert(NumIRArgs == 1);
2382
2.04k
      Address ParamAddr = Address(FnArgs[FirstIRArg], ArgI.getIndirectAlign());
2383
2.04k
2384
2.04k
      if (!hasScalarEvaluationKind(Ty)) {
2385
1.78k
        // Aggregates and complex variables are accessed by reference.  All we
2386
1.78k
        // need to do is realign the value, if requested.
2387
1.78k
        Address V = ParamAddr;
2388
1.78k
        if (ArgI.getIndirectRealign()) {
2389
26
          Address AlignedTemp = CreateMemTemp(Ty, "coerce");
2390
26
2391
26
          // Copy from the incoming argument pointer to the temporary with the
2392
26
          // appropriate alignment.
2393
26
          //
2394
26
          // FIXME: We should have a common utility for generating an aggregate
2395
26
          // copy.
2396
26
          CharUnits Size = getContext().getTypeSizeInChars(Ty);
2397
26
          auto SizeVal = llvm::ConstantInt::get(IntPtrTy, Size.getQuantity());
2398
26
          Address Dst = Builder.CreateBitCast(AlignedTemp, Int8PtrTy);
2399
26
          Address Src = Builder.CreateBitCast(ParamAddr, Int8PtrTy);
2400
26
          Builder.CreateMemCpy(Dst, Src, SizeVal, false);
2401
26
          V = AlignedTemp;
2402
26
        }
2403
1.78k
        ArgVals.push_back(ParamValue::forIndirect(V));
2404
1.78k
      } else {
2405
253
        // Load scalar value from indirect argument.
2406
253
        llvm::Value *V =
2407
253
            EmitLoadOfScalar(ParamAddr, false, Ty, Arg->getBeginLoc());
2408
253
2409
253
        if (isPromoted)
2410
0
          V = emitArgumentDemotion(*this, Arg, V);
2411
253
        ArgVals.push_back(ParamValue::forDirect(V));
2412
253
      }
2413
2.04k
      break;
2414
0
    }
2415
0
2416
383k
    case ABIArgInfo::Extend:
2417
383k
    case ABIArgInfo::Direct: {
2418
383k
2419
383k
      // If we have the trivial case, handle it with no muss and fuss.
2420
383k
      if (!isa<llvm::StructType>(ArgI.getCoerceToType()) &&
2421
383k
          
ArgI.getCoerceToType() == ConvertType(Ty)382k
&&
2422
383k
          
ArgI.getDirectOffset() == 0377k
) {
2423
377k
        assert(NumIRArgs == 1);
2424
377k
        llvm::Value *V = FnArgs[FirstIRArg];
2425
377k
        auto AI = cast<llvm::Argument>(V);
2426
377k
2427
377k
        if (const ParmVarDecl *PVD = dyn_cast<ParmVarDecl>(Arg)) {
2428
224k
          if (getNonNullAttr(CurCodeDecl, PVD, PVD->getType(),
2429
224k
                             PVD->getFunctionScopeIndex()) &&
2430
224k
              
!CGM.getCodeGenOpts().NullPointerIsValid22
)
2431
13
            AI->addAttr(llvm::Attribute::NonNull);
2432
224k
2433
224k
          QualType OTy = PVD->getOriginalType();
2434
224k
          if (const auto *ArrTy =
2435
452
              getContext().getAsConstantArrayType(OTy)) {
2436
452
            // A C99 array parameter declaration with the static keyword also
2437
452
            // indicates dereferenceability, and if the size is constant we can
2438
452
            // use the dereferenceable attribute (which requires the size in
2439
452
            // bytes).
2440
452
            if (ArrTy->getSizeModifier() == ArrayType::Static) {
2441
4
              QualType ETy = ArrTy->getElementType();
2442
4
              uint64_t ArrSize = ArrTy->getSize().getZExtValue();
2443
4
              if (!ETy->isIncompleteType() && ETy->isConstantSizeType() &&
2444
4
                  ArrSize) {
2445
4
                llvm::AttrBuilder Attrs;
2446
4
                Attrs.addDereferenceableAttr(
2447
4
                  getContext().getTypeSizeInChars(ETy).getQuantity()*ArrSize);
2448
4
                AI->addAttrs(Attrs);
2449
4
              } else 
if (0
getContext().getTargetAddressSpace(ETy) == 00
&&
2450
0
                         !CGM.getCodeGenOpts().NullPointerIsValid) {
2451
0
                AI->addAttr(llvm::Attribute::NonNull);
2452
0
              }
2453
4
            }
2454
223k
          } else if (const auto *ArrTy =
2455
86
                     getContext().getAsVariableArrayType(OTy)) {
2456
86
            // For C99 VLAs with the static keyword, we don't know the size so
2457
86
            // we can't use the dereferenceable attribute, but in addrspace(0)
2458
86
            // we know that it must be nonnull.
2459
86
            if (ArrTy->getSizeModifier() == VariableArrayType::Static &&
2460
86
                
!getContext().getTargetAddressSpace(ArrTy->getElementType())2
&&
2461
86
                
!CGM.getCodeGenOpts().NullPointerIsValid2
)
2462
1
              AI->addAttr(llvm::Attribute::NonNull);
2463
86
          }
2464
224k
2465
224k
          const auto *AVAttr = PVD->getAttr<AlignValueAttr>();
2466
224k
          if (!AVAttr)
2467
224k
            if (const auto *TOTy = dyn_cast<TypedefType>(OTy))
2468
85.1k
              AVAttr = TOTy->getDecl()->getAttr<AlignValueAttr>();
2469
224k
          if (AVAttr && 
!SanOpts.has(SanitizerKind::Alignment)13
) {
2470
10
            // If alignment-assumption sanitizer is enabled, we do *not* add
2471
10
            // alignment attribute here, but emit normal alignment assumption,
2472
10
            // so the UBSAN check could function.
2473
10
            llvm::Value *AlignmentValue =
2474
10
              EmitScalarExpr(AVAttr->getAlignment());
2475
10
            llvm::ConstantInt *AlignmentCI =
2476
10
              cast<llvm::ConstantInt>(AlignmentValue);
2477
10
            AI->addAttrs(llvm::AttrBuilder().addAlignmentAttr(llvm::MaybeAlign(
2478
10
                AlignmentCI->getLimitedValue(llvm::Value::MaximumAlignment))));
2479
10
          }
2480
224k
        }
2481
377k
2482
377k
        if (Arg->getType().isRestrictQualified())
2483
21.5k
          AI->addAttr(llvm::Attribute::NoAlias);
2484
377k
2485
377k
        // LLVM expects swifterror parameters to be used in very restricted
2486
377k
        // ways.  Copy the value into a less-restricted temporary.
2487
377k
        if (FI.getExtParameterInfo(ArgNo).getABI()
2488
377k
              == ParameterABI::SwiftErrorResult) {
2489
16
          QualType pointeeTy = Ty->getPointeeType();
2490
16
          assert(pointeeTy->isPointerType());
2491
16
          Address temp =
2492
16
            CreateMemTemp(pointeeTy, getPointerAlign(), "swifterror.temp");
2493
16
          Address arg = Address(V, getContext().getTypeAlignInChars(pointeeTy));
2494
16
          llvm::Value *incomingErrorValue = Builder.CreateLoad(arg);
2495
16
          Builder.CreateStore(incomingErrorValue, temp);
2496
16
          V = temp.getPointer();
2497
16
2498
16
          // Push a cleanup to copy the value back at the end of the function.
2499
16
          // The convention does not guarantee that the value will be written
2500
16
          // back if the function exits with an unwind exception.
2501
16
          EHStack.pushCleanup<CopyBackSwiftError>(NormalCleanup, temp, arg);
2502
16
        }
2503
377k
2504
377k
        // Ensure the argument is the correct type.
2505
377k
        if (V->getType() != ArgI.getCoerceToType())
2506
0
          V = Builder.CreateBitCast(V, ArgI.getCoerceToType());
2507
377k
2508
377k
        if (isPromoted)
2509
14
          V = emitArgumentDemotion(*this, Arg, V);
2510
377k
2511
377k
        // Because of merging of function types from multiple decls it is
2512
377k
        // possible for the type of an argument to not match the corresponding
2513
377k
        // type in the function type. Since we are codegening the callee
2514
377k
        // in here, add a cast to the argument type.
2515
377k
        llvm::Type *LTy = ConvertType(Arg->getType());
2516
377k
        if (V->getType() != LTy)
2517
0
          V = Builder.CreateBitCast(V, LTy);
2518
377k
2519
377k
        ArgVals.push_back(ParamValue::forDirect(V));
2520
377k
        break;
2521
377k
      }
2522
5.43k
2523
5.43k
      Address Alloca = CreateMemTemp(Ty, getContext().getDeclAlign(Arg),
2524
5.43k
                                     Arg->getName());
2525
5.43k
2526
5.43k
      // Pointer to store into.
2527
5.43k
      Address Ptr = emitAddressAtOffset(*this, Alloca, ArgI);
2528
5.43k
2529
5.43k
      // Fast-isel and the optimizer generally like scalar values better than
2530
5.43k
      // FCAs, so we flatten them if this is safe to do for this argument.
2531
5.43k
      llvm::StructType *STy = dyn_cast<llvm::StructType>(ArgI.getCoerceToType());
2532
5.43k
      if (ArgI.isDirect() && ArgI.getCanBeFlattened() && 
STy5.19k
&&
2533
5.43k
          
STy->getNumElements() > 1854
) {
2534
829
        uint64_t SrcSize = CGM.getDataLayout().getTypeAllocSize(STy);
2535
829
        llvm::Type *DstTy = Ptr.getElementType();
2536
829
        uint64_t DstSize = CGM.getDataLayout().getTypeAllocSize(DstTy);
2537
829
2538
829
        Address AddrToStoreInto = Address::invalid();
2539
829
        if (SrcSize <= DstSize) {
2540
742
          AddrToStoreInto = Builder.CreateElementBitCast(Ptr, STy);
2541
742
        } else {
2542
87
          AddrToStoreInto =
2543
87
            CreateTempAlloca(STy, Alloca.getAlignment(), "coerce");
2544
87
        }
2545
829
2546
829
        assert(STy->getNumElements() == NumIRArgs);
2547
2.59k
        for (unsigned i = 0, e = STy->getNumElements(); i != e; 
++i1.76k
) {
2548
1.76k
          auto AI = FnArgs[FirstIRArg + i];
2549
1.76k
          AI->setName(Arg->getName() + ".coerce" + Twine(i));
2550
1.76k
          Address EltPtr = Builder.CreateStructGEP(AddrToStoreInto, i);
2551
1.76k
          Builder.CreateStore(AI, EltPtr);
2552
1.76k
        }
2553
829
2554
829
        if (SrcSize > DstSize) {
2555
87
          Builder.CreateMemCpy(Ptr, AddrToStoreInto, DstSize);
2556
87
        }
2557
829
2558
4.60k
      } else {
2559
4.60k
        // Simple case, just do a coerced store of the argument into the alloca.
2560
4.60k
        assert(NumIRArgs == 1);
2561
4.60k
        auto AI = FnArgs[FirstIRArg];
2562
4.60k
        AI->setName(Arg->getName() + ".coerce");
2563
4.60k
        CreateCoercedStore(AI, Ptr, /*DstIsVolatile=*/false, *this);
2564
4.60k
      }
2565
5.43k
2566
5.43k
      // Match to what EmitParmDecl is expecting for this type.
2567
5.43k
      if (CodeGenFunction::hasScalarEvaluationKind(Ty)) {
2568
1.25k
        llvm::Value *V =
2569
1.25k
            EmitLoadOfScalar(Alloca, false, Ty, Arg->getBeginLoc());
2570
1.25k
        if (isPromoted)
2571
0
          V = emitArgumentDemotion(*this, Arg, V);
2572
1.25k
        ArgVals.push_back(ParamValue::forDirect(V));
2573
4.17k
      } else {
2574
4.17k
        ArgVals.push_back(ParamValue::forIndirect(Alloca));
2575
4.17k
      }
2576
5.43k
      break;
2577
5.43k
    }
2578
5.43k
2579
5.43k
    case ABIArgInfo::CoerceAndExpand: {
2580
543
      // Reconstruct into a temporary.
2581
543
      Address alloca = CreateMemTemp(Ty, getContext().getDeclAlign(Arg));
2582
543
      ArgVals.push_back(ParamValue::forIndirect(alloca));
2583
543
2584
543
      auto coercionType = ArgI.getCoerceAndExpandType();
2585
543
      alloca = Builder.CreateElementBitCast(alloca, coercionType);
2586
543
2587
543
      unsigned argIndex = FirstIRArg;
2588
1.69k
      for (unsigned i = 0, e = coercionType->getNumElements(); i != e; 
++i1.15k
) {
2589
1.15k
        llvm::Type *eltType = coercionType->getElementType(i);
2590
1.15k
        if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType))
2591
7
          continue;
2592
1.14k
2593
1.14k
        auto eltAddr = Builder.CreateStructGEP(alloca, i);
2594
1.14k
        auto elt = FnArgs[argIndex++];
2595
1.14k
        Builder.CreateStore(elt, eltAddr);
2596
1.14k
      }
2597
543
      assert(argIndex == FirstIRArg + NumIRArgs);
2598
543
      break;
2599
5.43k
    }
2600
5.43k
2601
5.43k
    case ABIArgInfo::Expand: {
2602
100
      // If this structure was expanded into multiple arguments then
2603
100
      // we need to create a temporary and reconstruct it from the
2604
100
      // arguments.
2605
100
      Address Alloca = CreateMemTemp(Ty, getContext().getDeclAlign(Arg));
2606
100
      LValue LV = MakeAddrLValue(Alloca, Ty);
2607
100
      ArgVals.push_back(ParamValue::forIndirect(Alloca));
2608
100
2609
100
      auto FnArgIter = FnArgs.begin() + FirstIRArg;
2610
100
      ExpandTypeFromArgs(Ty, LV, FnArgIter);
2611
100
      assert(FnArgIter == FnArgs.begin() + FirstIRArg + NumIRArgs);
2612
340
      for (unsigned i = 0, e = NumIRArgs; i != e; 
++i240
) {
2613
240
        auto AI = FnArgs[FirstIRArg + i];
2614
240
        AI->setName(Arg->getName() + "." + Twine(i));
2615
240
      }
2616
100
      break;
2617
5.43k
    }
2618
5.43k
2619
7.48k
    case ABIArgInfo::Ignore:
2620
7.48k
      assert(NumIRArgs == 0);
2621
7.48k
      // Initialize the local variable appropriately.
2622
7.48k
      if (!hasScalarEvaluationKind(Ty)) {
2623
7.48k
        ArgVals.push_back(ParamValue::forIndirect(CreateMemTemp(Ty)));
2624
7.48k
      } else {
2625
0
        llvm::Value *U = llvm::UndefValue::get(ConvertType(Arg->getType()));
2626
0
        ArgVals.push_back(ParamValue::forDirect(U));
2627
0
      }
2628
7.48k
      break;
2629
393k
    }
2630
393k
  }
2631
242k
2632
242k
  if (getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) {
2633
29.8k
    for (int I = Args.size() - 1; I >= 0; 
--I17.2k
)
2634
17.2k
      EmitParmDecl(*Args[I], ArgVals[I], I + 1);
2635
230k
  } else {
2636
606k
    for (unsigned I = 0, E = Args.size(); I != E; 
++I375k
)
2637
375k
      EmitParmDecl(*Args[I], ArgVals[I], I + 1);
2638
230k
  }
2639
242k
}
2640
2641
8
static void eraseUnusedBitCasts(llvm::Instruction *insn) {
2642
12
  while (insn->use_empty()) {
2643
12
    llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(insn);
2644
12
    if (!bitcast) 
return8
;
2645
4
2646
4
    // This is "safe" because we would have used a ConstantExpr otherwise.
2647
4
    insn = cast<llvm::Instruction>(bitcast->getOperand(0));
2648
4
    bitcast->eraseFromParent();
2649
4
  }
2650
8
}
2651
2652
/// Try to emit a fused autorelease of a return result.
2653
static llvm::Value *tryEmitFusedAutoreleaseOfResult(CodeGenFunction &CGF,
2654
74
                                                    llvm::Value *result) {
2655
74
  // We must be immediately followed the cast.
2656
74
  llvm::BasicBlock *BB = CGF.Builder.GetInsertBlock();
2657
74
  if (BB->empty()) 
return nullptr0
;
2658
74
  if (&BB->back() != result) 
return nullptr17
;
2659
57
2660
57
  llvm::Type *resultType = result->getType();
2661
57
2662
57
  // result is in a BasicBlock and is therefore an Instruction.
2663
57
  llvm::Instruction *generator = cast<llvm::Instruction>(result);
2664
57
2665
57
  SmallVector<llvm::Instruction *, 4> InstsToKill;
2666
57
2667
57
  // Look for:
2668
57
  //  %generator = bitcast %type1* %generator2 to %type2*
2669
89
  while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(generator)) {
2670
32
    // We would have emitted this as a constant if the operand weren't
2671
32
    // an Instruction.
2672
32
    generator = cast<llvm::Instruction>(bitcast->getOperand(0));
2673
32
2674
32
    // Require the generator to be immediately followed by the cast.
2675
32
    if (generator->getNextNode() != bitcast)
2676
0
      return nullptr;
2677
32
2678
32
    InstsToKill.push_back(bitcast);
2679
32
  }
2680
57
2681
57
  // Look for:
2682
57
  //   %generator = call i8* @objc_retain(i8* %originalResult)
2683
57
  // or
2684
57
  //   %generator = call i8* @objc_retainAutoreleasedReturnValue(i8* %originalResult)
2685
57
  llvm::CallInst *call = dyn_cast<llvm::CallInst>(generator);
2686
57
  if (!call) 
return nullptr8
;
2687
49
2688
49
  bool doRetainAutorelease;
2689
49
2690
49
  if (call->getCalledValue() == CGF.CGM.getObjCEntrypoints().objc_retain) {
2691
7
    doRetainAutorelease = true;
2692
42
  } else if (call->getCalledValue() == CGF.CGM.getObjCEntrypoints()
2693
42
                                          .objc_retainAutoreleasedReturnValue) {
2694
19
    doRetainAutorelease = false;
2695
19
2696
19
    // If we emitted an assembly marker for this call (and the
2697
19
    // ARCEntrypoints field should have been set if so), go looking
2698
19
    // for that call.  If we can't find it, we can't do this
2699
19
    // optimization.  But it should always be the immediately previous
2700
19
    // instruction, unless we needed bitcasts around the call.
2701
19
    if (CGF.CGM.getObjCEntrypoints().retainAutoreleasedReturnValueMarker) {
2702
12
      llvm::Instruction *prev = call->getPrevNode();
2703
12
      assert(prev);
2704
12
      if (isa<llvm::BitCastInst>(prev)) {
2705
8
        prev = prev->getPrevNode();
2706
8
        assert(prev);
2707
8
      }
2708
12
      assert(isa<llvm::CallInst>(prev));
2709
12
      assert(cast<llvm::CallInst>(prev)->getCalledValue() ==
2710
12
               CGF.CGM.getObjCEntrypoints().retainAutoreleasedReturnValueMarker);
2711
12
      InstsToKill.push_back(prev);
2712
12
    }
2713
23
  } else {
2714
23
    return nullptr;
2715
23
  }
2716
26
2717
26
  result = call->getArgOperand(0);
2718
26
  InstsToKill.push_back(call);
2719
26
2720
26
  // Keep killing bitcasts, for sanity.  Note that we no longer care
2721
26
  // about precise ordering as long as there's exactly one use.
2722
42
  while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(result)) {
2723
16
    if (!bitcast->hasOneUse()) 
break0
;
2724
16
    InstsToKill.push_back(bitcast);
2725
16
    result = bitcast->getOperand(0);
2726
16
  }
2727
26
2728
26
  // Delete all the unnecessary instructions, from latest to earliest.
2729
26
  for (auto *I : InstsToKill)
2730
76
    I->eraseFromParent();
2731
26
2732
26
  // Do the fused retain/autorelease if we were asked to.
2733
26
  if (doRetainAutorelease)
2734
7
    result = CGF.EmitARCRetainAutoreleaseReturnValue(result);
2735
26
2736
26
  // Cast back to the result type.
2737
26
  return CGF.Builder.CreateBitCast(result, resultType);
2738
26
}
2739
2740
/// If this is a +1 of the value of an immutable 'self', remove it.
2741
static llvm::Value *tryRemoveRetainOfSelf(CodeGenFunction &CGF,
2742
98
                                          llvm::Value *result) {
2743
98
  // This is only applicable to a method with an immutable 'self'.
2744
98
  const ObjCMethodDecl *method =
2745
98
    dyn_cast_or_null<ObjCMethodDecl>(CGF.CurCodeDecl);
2746
98
  if (!method) 
return nullptr79
;
2747
19
  const VarDecl *self = method->getSelfDecl();
2748
19
  if (!self->getType().isConstQualified()) 
return nullptr0
;
2749
19
2750
19
  // Look for a retain call.
2751
19
  llvm::CallInst *retainCall =
2752
19
    dyn_cast<llvm::CallInst>(result->stripPointerCasts());
2753
19
  if (!retainCall ||
2754
19
      
retainCall->getCalledValue() != CGF.CGM.getObjCEntrypoints().objc_retain15
)
2755
12
    return nullptr;
2756
7
2757
7
  // Look for an ordinary load of 'self'.
2758
7
  llvm::Value *retainedValue = retainCall->getArgOperand(0);
2759
7
  llvm::LoadInst *load =
2760
7
    dyn_cast<llvm::LoadInst>(retainedValue->stripPointerCasts());
2761
7
  if (!load || load->isAtomic() || load->isVolatile() ||
2762
7
      load->getPointerOperand() != CGF.GetAddrOfLocalVar(self).getPointer())
2763
3
    return nullptr;
2764
4
2765
4
  // Okay!  Burn it all down.  This relies for correctness on the
2766
4
  // assumption that the retain is emitted as part of the return and
2767
4
  // that thereafter everything is used "linearly".
2768
4
  llvm::Type *resultType = result->getType();
2769
4
  eraseUnusedBitCasts(cast<llvm::Instruction>(result));
2770
4
  assert(retainCall->use_empty());
2771
4
  retainCall->eraseFromParent();
2772
4
  eraseUnusedBitCasts(cast<llvm::Instruction>(retainedValue));
2773
4
2774
4
  return CGF.Builder.CreateBitCast(load, resultType);
2775
4
}
2776
2777
/// Emit an ARC autorelease of the result of a function.
2778
///
2779
/// \return the value to actually return from the function
2780
static llvm::Value *emitAutoreleaseOfResult(CodeGenFunction &CGF,
2781
98
                                            llvm::Value *result) {
2782
98
  // If we're returning 'self', kill the initial retain.  This is a
2783
98
  // heuristic attempt to "encourage correctness" in the really unfortunate
2784
98
  // case where we have a return of self during a dealloc and we desperately
2785
98
  // need to avoid the possible autorelease.
2786
98
  if (llvm::Value *self = tryRemoveRetainOfSelf(CGF, result))
2787
4
    return self;
2788
94
2789
94
  // At -O0, try to emit a fused retain/autorelease.
2790
94
  if (CGF.shouldUseFusedARCCalls())
2791
74
    if (llvm::Value *fused = tryEmitFusedAutoreleaseOfResult(CGF, result))
2792
26
      return fused;
2793
68
2794
68
  return CGF.EmitARCAutoreleaseReturnValue(result);
2795
68
}
2796
2797
/// Heuristically search for a dominating store to the return-value slot.
2798
109k
static llvm::StoreInst *findDominatingStoreToReturnValue(CodeGenFunction &CGF) {
2799
109k
  // Check if a User is a store which pointerOperand is the ReturnValue.
2800
109k
  // We are looking for stores to the ReturnValue, not for stores of the
2801
109k
  // ReturnValue to some other location.
2802
109k
  auto GetStoreIfValid = [&CGF](llvm::User *U) -> llvm::StoreInst * {
2803
107k
    auto *SI = dyn_cast<llvm::StoreInst>(U);
2804
107k
    if (!SI || 
SI->getPointerOperand() != CGF.ReturnValue.getPointer()106k
)
2805
1.94k
      return nullptr;
2806
105k
    // These aren't actually possible for non-coerced returns, and we
2807
105k
    // only care about non-coerced returns on this code path.
2808
105k
    assert(!SI->isAtomic() && !SI->isVolatile());
2809
105k
    return SI;
2810
105k
  };
2811
109k
  // If there are multiple uses of the return-value slot, just check
2812
109k
  // for something immediately preceding the IP.  Sometimes this can
2813
109k
  // happen with how we generate implicit-returns; it can also happen
2814
109k
  // with noreturn cleanups.
2815
109k
  if (!CGF.ReturnValue.getPointer()->hasOneUse()) {
2816
6.29k
    llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock();
2817
6.29k
    if (IP->empty()) 
return nullptr1.64k
;
2818
4.65k
    llvm::Instruction *I = &IP->back();
2819
4.65k
2820
4.65k
    // Skip lifetime markers
2821
4.65k
    for (llvm::BasicBlock::reverse_iterator II = IP->rbegin(),
2822
4.65k
                                            IE = IP->rend();
2823
4.73k
         II != IE; 
++II72
) {
2824
4.69k
      if (llvm::IntrinsicInst *Intrinsic =
2825
305
              dyn_cast<llvm::IntrinsicInst>(&*II)) {
2826
305
        if (Intrinsic->getIntrinsicID() == llvm::Intrinsic::lifetime_end) {
2827
72
          const llvm::Value *CastAddr = Intrinsic->getArgOperand(1);
2828
72
          ++II;
2829
72
          if (II == IE)
2830
0
            break;
2831
72
          if (isa<llvm::BitCastInst>(&*II) && (CastAddr == &*II))
2832
72
            continue;
2833
4.62k
        }
2834
305
      }
2835
4.62k
      I = &*II;
2836
4.62k
      break;
2837
4.62k
    }
2838
4.65k
2839
4.65k
    return GetStoreIfValid(I);
2840
4.65k
  }
2841
103k
2842
103k
  llvm::StoreInst *store =
2843
103k
      GetStoreIfValid(CGF.ReturnValue.getPointer()->user_back());
2844
103k
  if (!store) 
return nullptr473
;
2845
102k
2846
102k
  // Now do a first-and-dirty dominance check: just walk up the
2847
102k
  // single-predecessors chain from the current insertion point.
2848
102k
  llvm::BasicBlock *StoreBB = store->getParent();
2849
102k
  llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock();
2850
103k
  while (IP != StoreBB) {
2851
1.97k
    if (!(IP = IP->getSinglePredecessor()))
2852
1.24k
      return nullptr;
2853
1.97k
  }
2854
102k
2855
102k
  // Okay, the store's basic block dominates the insertion point; we
2856
102k
  // can do our thing.
2857
102k
  
return store101k
;
2858
102k
}
2859
2860
void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI,
2861
                                         bool EmitRetDbgLoc,
2862
242k
                                         SourceLocation EndLoc) {
2863
242k
  if (FI.isNoReturn()) {
2864
12
    // Noreturn functions don't return.
2865
12
    EmitUnreachable(EndLoc);
2866
12
    return;
2867
12
  }
2868
242k
2869
242k
  if (CurCodeDecl && 
CurCodeDecl->hasAttr<NakedAttr>()236k
) {
2870
7
    // Naked functions don't have epilogues.
2871
7
    Builder.CreateUnreachable();
2872
7
    return;
2873
7
  }
2874
242k
2875
242k
  // Functions with no result always return void.
2876
242k
  if (!ReturnValue.isValid()) {
2877
126k
    Builder.CreateRetVoid();
2878
126k
    return;
2879
126k
  }
2880
116k
2881
116k
  llvm::DebugLoc RetDbgLoc;
2882
116k
  llvm::Value *RV = nullptr;
2883
116k
  QualType RetTy = FI.getReturnType();
2884
116k
  const ABIArgInfo &RetAI = FI.getReturnInfo();
2885
116k
2886
116k
  switch (RetAI.getKind()) {
2887
5
  case ABIArgInfo::InAlloca:
2888
5
    // Aggregrates get evaluated directly into the destination.  Sometimes we
2889
5
    // need to return the sret value in a register, though.
2890
5
    assert(hasAggregateEvaluationKind(RetTy));
2891
5
    if (RetAI.getInAllocaSRet()) {
2892
5
      llvm::Function::arg_iterator EI = CurFn->arg_end();
2893
5
      --EI;
2894
5
      llvm::Value *ArgStruct = &*EI;
2895
5
      llvm::Value *SRet = Builder.CreateStructGEP(
2896
5
          nullptr, ArgStruct, RetAI.getInAllocaFieldIndex());
2897
5
      RV = Builder.CreateAlignedLoad(SRet, getPointerAlign(), "sret");
2898
5
    }
2899
5
    break;
2900
0
2901
2.24k
  case ABIArgInfo::Indirect: {
2902
2.24k
    auto AI = CurFn->arg_begin();
2903
2.24k
    if (RetAI.isSRetAfterThis())
2904
49
      ++AI;
2905
2.24k
    switch (getEvaluationKind(RetTy)) {
2906
229
    case TEK_Complex: {
2907
229
      ComplexPairTy RT =
2908
229
        EmitLoadOfComplex(MakeAddrLValue(ReturnValue, RetTy), EndLoc);
2909
229
      EmitStoreOfComplex(RT, MakeNaturalAlignAddrLValue(&*AI, RetTy),
2910
229
                         /*isInit*/ true);
2911
229
      break;
2912
0
    }
2913
1.85k
    case TEK_Aggregate:
2914
1.85k
      // Do nothing; aggregrates get evaluated directly into the destination.
2915
1.85k
      break;
2916
163
    case TEK_Scalar:
2917
163
      EmitStoreOfScalar(Builder.CreateLoad(ReturnValue),
2918
163
                        MakeNaturalAlignAddrLValue(&*AI, RetTy),
2919
163
                        /*isInit*/ true);
2920
163
      break;
2921
2.24k
    }
2922
2.24k
    break;
2923
2.24k
  }
2924
2.24k
2925
113k
  case ABIArgInfo::Extend:
2926
113k
  case ABIArgInfo::Direct:
2927
113k
    if (RetAI.getCoerceToType() == ConvertType(RetTy) &&
2928
113k
        
RetAI.getDirectOffset() == 0109k
) {
2929
109k
      // The internal return value temp always will have pointer-to-return-type
2930
109k
      // type, just do a load.
2931
109k
2932
109k
      // If there is a dominating store to ReturnValue, we can elide
2933
109k
      // the load, zap the store, and usually zap the alloca.
2934
109k
      if (llvm::StoreInst *SI =
2935
104k
              findDominatingStoreToReturnValue(*this)) {
2936
104k
        // Reuse the debug location from the store unless there is
2937
104k
        // cleanup code to be emitted between the store and return
2938
104k
        // instruction.
2939
104k
        if (EmitRetDbgLoc && 
!AutoreleaseResult103k
)
2940
103k
          RetDbgLoc = SI->getDebugLoc();
2941
104k
        // Get the stored value and nuke the now-dead store.
2942
104k
        RV = SI->getValueOperand();
2943
104k
        SI->eraseFromParent();
2944
104k
2945
104k
      // Otherwise, we have to do a simple load.
2946
104k
      } else {
2947
4.83k
        RV = Builder.CreateLoad(ReturnValue);
2948
4.83k
      }
2949
109k
    } else {
2950
4.22k
      // If the value is offset in memory, apply the offset now.
2951
4.22k
      Address V = emitAddressAtOffset(*this, ReturnValue, RetAI);
2952
4.22k
2953
4.22k
      RV = CreateCoercedLoad(V, RetAI.getCoerceToType(), *this);
2954
4.22k
    }
2955
113k
2956
113k
    // In ARC, end functions that return a retainable type with a call
2957
113k
    // to objc_autoreleaseReturnValue.
2958
113k
    if (AutoreleaseResult) {
2959
98
#ifndef NDEBUG
2960
98
      // Type::isObjCRetainabletype has to be called on a QualType that hasn't
2961
98
      // been stripped of the typedefs, so we cannot use RetTy here. Get the
2962
98
      // original return type of FunctionDecl, CurCodeDecl, and BlockDecl from
2963
98
      // CurCodeDecl or BlockInfo.
2964
98
      QualType RT;
2965
98
2966
98
      if (auto *FD = dyn_cast<FunctionDecl>(CurCodeDecl))
2967
57
        RT = FD->getReturnType();
2968
41
      else if (auto *MD = dyn_cast<ObjCMethodDecl>(CurCodeDecl))
2969
19
        RT = MD->getReturnType();
2970
22
      else if (isa<BlockDecl>(CurCodeDecl))
2971
22
        RT = BlockInfo->BlockExpression->getFunctionType()->getReturnType();
2972
22
      else
2973
22
        
llvm_unreachable0
("Unexpected function/method type");
2974
98
2975
98
      assert(getLangOpts().ObjCAutoRefCount &&
2976
98
             !FI.isReturnsRetained() &&
2977
98
             RT->isObjCRetainableType());
2978
98
#endif
2979
98
      RV = emitAutoreleaseOfResult(*this, RV);
2980
98
    }
2981
113k
2982
113k
    break;
2983
113k
2984
113k
  case ABIArgInfo::Ignore:
2985
283
    break;
2986
113k
2987
113k
  case ABIArgInfo::CoerceAndExpand: {
2988
541
    auto coercionType = RetAI.getCoerceAndExpandType();
2989
541
2990
541
    // Load all of the coerced elements out into results.
2991
541
    llvm::SmallVector<llvm::Value*, 4> results;
2992
541
    Address addr = Builder.CreateElementBitCast(ReturnValue, coercionType);
2993
1.68k
    for (unsigned i = 0, e = coercionType->getNumElements(); i != e; 
++i1.14k
) {
2994
1.14k
      auto coercedEltType = coercionType->getElementType(i);
2995
1.14k
      if (ABIArgInfo::isPaddingForCoerceAndExpand(coercedEltType))
2996
7
        continue;
2997
1.14k
2998
1.14k
      auto eltAddr = Builder.CreateStructGEP(addr, i);
2999
1.14k
      auto elt = Builder.CreateLoad(eltAddr);
3000
1.14k
      results.push_back(elt);
3001
1.14k
    }
3002
541
3003
541
    // If we have one result, it's the single direct result type.
3004
541
    if (results.size() == 1) {
3005
180
      RV = results[0];
3006
180
3007
180
    // Otherwise, we need to make a first-class aggregate.
3008
361
    } else {
3009
361
      // Construct a return type that lacks padding elements.
3010
361
      llvm::Type *returnType = RetAI.getUnpaddedCoerceAndExpandType();
3011
361
3012
361
      RV = llvm::UndefValue::get(returnType);
3013
1.32k
      for (unsigned i = 0, e = results.size(); i != e; 
++i960
) {
3014
960
        RV = Builder.CreateInsertValue(RV, results[i], i);
3015
960
      }
3016
361
    }
3017
541
    break;
3018
113k
  }
3019
113k
3020
113k
  case ABIArgInfo::Expand:
3021
0
    llvm_unreachable("Invalid ABI kind for return argument");
3022
116k
  }
3023
116k
3024
116k
  llvm::Instruction *Ret;
3025
116k
  if (RV) {
3026
114k
    EmitReturnValueCheck(RV);
3027
114k
    Ret = Builder.CreateRet(RV);
3028
114k
  } else {
3029
2.53k
    Ret = Builder.CreateRetVoid();
3030
2.53k
  }
3031
116k
3032
116k
  if (RetDbgLoc)
3033
38.4k
    Ret->setDebugLoc(std::move(RetDbgLoc));
3034
116k
}
3035
3036
114k
void CodeGenFunction::EmitReturnValueCheck(llvm::Value *RV) {
3037
114k
  // A current decl may not be available when emitting vtable thunks.
3038
114k
  if (!CurCodeDecl)
3039
964
    return;
3040
113k
3041
113k
  // If the return block isn't reachable, neither is this check, so don't emit
3042
113k
  // it.
3043
113k
  if (ReturnBlock.isValid() && 
ReturnBlock.getBlock()->use_empty()1.66k
)
3044
204
    return;
3045
112k
3046
112k
  ReturnsNonNullAttr *RetNNAttr = nullptr;
3047
112k
  if (SanOpts.has(SanitizerKind::ReturnsNonnullAttribute))
3048
47
    RetNNAttr = CurCodeDecl->getAttr<ReturnsNonNullAttr>();
3049
112k
3050
112k
  if (!RetNNAttr && 
!requiresReturnValueNullabilityCheck()112k
)
3051
112k
    return;
3052
18
3053
18
  // Prefer the returns_nonnull attribute if it's present.
3054
18
  SourceLocation AttrLoc;
3055
18
  SanitizerMask CheckKind;
3056
18
  SanitizerHandler Handler;
3057
18
  if (RetNNAttr) {
3058
5
    assert(!requiresReturnValueNullabilityCheck() &&
3059
5
           "Cannot check nullability and the nonnull attribute");
3060
5
    AttrLoc = RetNNAttr->getLocation();
3061
5
    CheckKind = SanitizerKind::ReturnsNonnullAttribute;
3062
5
    Handler = SanitizerHandler::NonnullReturn;
3063
13
  } else {
3064
13
    if (auto *DD = dyn_cast<DeclaratorDecl>(CurCodeDecl))
3065
7
      if (auto *TSI = DD->getTypeSourceInfo())
3066
7
        if (auto FTL = TSI->getTypeLoc().getAsAdjusted<FunctionTypeLoc>())
3067
7
          AttrLoc = FTL.getReturnLoc().findNullabilityLoc();
3068
13
    CheckKind = SanitizerKind::NullabilityReturn;
3069
13
    Handler = SanitizerHandler::NullabilityReturn;
3070
13
  }
3071
18
3072
18
  SanitizerScope SanScope(this);
3073
18
3074
18
  // Make sure the "return" source location is valid. If we're checking a
3075
18
  // nullability annotation, make sure the preconditions for the check are met.
3076
18
  llvm::BasicBlock *Check = createBasicBlock("nullcheck");
3077
18
  llvm::BasicBlock *NoCheck = createBasicBlock("no.nullcheck");
3078
18
  llvm::Value *SLocPtr = Builder.CreateLoad(ReturnLocation, "return.sloc.load");
3079
18
  llvm::Value *CanNullCheck = Builder.CreateIsNotNull(SLocPtr);
3080
18
  if (requiresReturnValueNullabilityCheck())
3081
13
    CanNullCheck =
3082
13
        Builder.CreateAnd(CanNullCheck, RetValNullabilityPrecondition);
3083
18
  Builder.CreateCondBr(CanNullCheck, Check, NoCheck);
3084
18
  EmitBlock(Check);
3085
18
3086
18
  // Now do the null check.
3087
18
  llvm::Value *Cond = Builder.CreateIsNotNull(RV);
3088
18
  llvm::Constant *StaticData[] = {EmitCheckSourceLocation(AttrLoc)};
3089
18
  llvm::Value *DynamicData[] = {SLocPtr};
3090
18
  EmitCheck(std::make_pair(Cond, CheckKind), Handler, StaticData, DynamicData);
3091
18
3092
18
  EmitBlock(NoCheck);
3093
18
3094
18
#ifndef NDEBUG
3095
18
  // The return location should not be used after the check has been emitted.
3096
18
  ReturnLocation = Address::invalid();
3097
18
#endif
3098
18
}
3099
3100
23.2k
static bool isInAllocaArgument(CGCXXABI &ABI, QualType type) {
3101
23.2k
  const CXXRecordDecl *RD = type->getAsCXXRecordDecl();
3102
23.2k
  return RD && 
ABI.getRecordArgABI(RD) == CGCXXABI::RAA_DirectInMemory2.75k
;
3103
23.2k
}
3104
3105
static AggValueSlot createPlaceholderSlot(CodeGenFunction &CGF,
3106
96
                                          QualType Ty) {
3107
96
  // FIXME: Generate IR in one pass, rather than going back and fixing up these
3108
96
  // placeholders.
3109
96
  llvm::Type *IRTy = CGF.ConvertTypeForMem(Ty);
3110
96
  llvm::Type *IRPtrTy = IRTy->getPointerTo();
3111
96
  llvm::Value *Placeholder = llvm::UndefValue::get(IRPtrTy->getPointerTo());
3112
96
3113
96
  // FIXME: When we generate this IR in one pass, we shouldn't need
3114
96
  // this win32-specific alignment hack.
3115
96
  CharUnits Align = CharUnits::fromQuantity(4);
3116
96
  Placeholder = CGF.Builder.CreateAlignedLoad(IRPtrTy, Placeholder, Align);
3117
96
3118
96
  return AggValueSlot::forAddr(Address(Placeholder, Align),
3119
96
                               Ty.getQualifiers(),
3120
96
                               AggValueSlot::IsNotDestructed,
3121
96
                               AggValueSlot::DoesNotNeedGCBarriers,
3122
96
                               AggValueSlot::IsNotAliased,
3123
96
                               AggValueSlot::DoesNotOverlap);
3124
96
}
3125
3126
void CodeGenFunction::EmitDelegateCallArg(CallArgList &args,
3127
                                          const VarDecl *param,
3128
17.0k
                                          SourceLocation loc) {
3129
17.0k
  // StartFunction converted the ABI-lowered parameter(s) into a
3130
17.0k
  // local alloca.  We need to turn that into an r-value suitable
3131
17.0k
  // for EmitCall.
3132
17.0k
  Address local = GetAddrOfLocalVar(param);
3133
17.0k
3134
17.0k
  QualType type = param->getType();
3135
17.0k
3136
17.0k
  if (isInAllocaArgument(CGM.getCXXABI(), type)) {
3137
1
    CGM.ErrorUnsupported(param, "forwarded non-trivially copyable parameter");
3138
1
  }
3139
17.0k
3140
17.0k
  // GetAddrOfLocalVar returns a pointer-to-pointer for references,
3141
17.0k
  // but the argument needs to be the original pointer.
3142
17.0k
  if (type->isReferenceType()) {
3143
8.80k
    args.add(RValue::get(Builder.CreateLoad(local)), type);
3144
8.80k
3145
8.80k
  // In ARC, move out of consumed arguments so that the release cleanup
3146
8.80k
  // entered by StartFunction doesn't cause an over-release.  This isn't
3147
8.80k
  // optimal -O0 code generation, but it should get cleaned up when
3148
8.80k
  // optimization is enabled.  This also assumes that delegate calls are
3149
8.80k
  // performed exactly once for a set of arguments, but that should be safe.
3150
8.80k
  } else 
if (8.24k
getLangOpts().ObjCAutoRefCount8.24k
&&
3151
8.24k
             
param->hasAttr<NSConsumedAttr>()9
&&
3152
8.24k
             
type->isObjCRetainableType()2
) {
3153
2
    llvm::Value *ptr = Builder.CreateLoad(local);
3154
2
    auto null =
3155
2
      llvm::ConstantPointerNull::get(cast<llvm::PointerType>(ptr->getType()));
3156
2
    Builder.CreateStore(null, local);
3157
2
    args.add(RValue::get(ptr), type);
3158
2
3159
2
  // For the most part, we just need to load the alloca, except that
3160
2
  // aggregate r-values are actually pointers to temporaries.
3161
8.24k
  } else {
3162
8.24k
    args.add(convertTempToRValue(local, type, loc), type);
3163
8.24k
  }
3164
17.0k
3165
17.0k
  // Deactivate the cleanup for the callee-destructed param that was pushed.
3166
17.0k
  if (hasAggregateEvaluationKind(type) && 
!CurFuncIsThunk2.48k
&&
3167
17.0k
      
type->castAs<RecordType>()->getDecl()->isParamDestroyedInCallee()2.45k
&&
3168
17.0k
      
param->needsDestruction(getContext())8
) {
3169
6
    EHScopeStack::stable_iterator cleanup =
3170
6
        CalleeDestructedParamCleanups.lookup(cast<ParmVarDecl>(param));
3171
6
    assert(cleanup.isValid() &&
3172
6
           "cleanup for callee-destructed param not recorded");
3173
6
    // This unreachable is a temporary marker which will be removed later.
3174
6
    llvm::Instruction *isActive = Builder.CreateUnreachable();
3175
6
    args.addArgCleanupDeactivation(cleanup, isActive);
3176
6
  }
3177
17.0k
}
3178
3179
52
static bool isProvablyNull(llvm::Value *addr) {
3180
52
  return isa<llvm::ConstantPointerNull>(addr);
3181
52
}
3182
3183
/// Emit the actual writing-back of a writeback.
3184
static void emitWriteback(CodeGenFunction &CGF,
3185
26
                          const CallArgList::Writeback &writeback) {
3186
26
  const LValue &srcLV = writeback.Source;
3187
26
  Address srcAddr = srcLV.getAddress(CGF);
3188
26
  assert(!isProvablyNull(srcAddr.getPointer()) &&
3189
26
         "shouldn't have writeback for provably null argument");
3190
26
3191
26
  llvm::BasicBlock *contBB = nullptr;
3192
26
3193
26
  // If the argument wasn't provably non-null, we need to null check
3194
26
  // before doing the store.
3195
26
  bool provablyNonNull = llvm::isKnownNonZero(srcAddr.getPointer(),
3196
26
                                              CGF.CGM.getDataLayout());
3197
26
  if (!provablyNonNull) {
3198
5
    llvm::BasicBlock *writebackBB = CGF.createBasicBlock("icr.writeback");
3199
5
    contBB = CGF.createBasicBlock("icr.done");
3200
5
3201
5
    llvm::Value *isNull =
3202
5
      CGF.Builder.CreateIsNull(srcAddr.getPointer(), "icr.isnull");
3203
5
    CGF.Builder.CreateCondBr(isNull, contBB, writebackBB);
3204
5
    CGF.EmitBlock(writebackBB);
3205
5
  }
3206
26
3207
26
  // Load the value to writeback.
3208
26
  llvm::Value *value = CGF.Builder.CreateLoad(writeback.Temporary);
3209
26
3210
26
  // Cast it back, in case we're writing an id to a Foo* or something.
3211
26
  value = CGF.Builder.CreateBitCast(value, srcAddr.getElementType(),
3212
26
                                    "icr.writeback-cast");
3213
26
3214
26
  // Perform the writeback.
3215
26
3216
26
  // If we have a "to use" value, it's something we need to emit a use
3217
26
  // of.  This has to be carefully threaded in: if it's done after the
3218
26
  // release it's potentially undefined behavior (and the optimizer
3219
26
  // will ignore it), and if it happens before the retain then the
3220
26
  // optimizer could move the release there.
3221
26
  if (writeback.ToUse) {
3222
7
    assert(srcLV.getObjCLifetime() == Qualifiers::OCL_Strong);
3223
7
3224
7
    // Retain the new value.  No need to block-copy here:  the block's
3225
7
    // being passed up the stack.
3226
7
    value = CGF.EmitARCRetainNonBlock(value);
3227
7
3228
7
    // Emit the intrinsic use here.
3229
7
    CGF.EmitARCIntrinsicUse(writeback.ToUse);
3230
7
3231
7
    // Load the old value (primitively).
3232
7
    llvm::Value *oldValue = CGF.EmitLoadOfScalar(srcLV, SourceLocation());
3233
7
3234
7
    // Put the new value in place (primitively).
3235
7
    CGF.EmitStoreOfScalar(value, srcLV, /*init*/ false);
3236
7
3237
7
    // Release the old value.
3238
7
    CGF.EmitARCRelease(oldValue, srcLV.isARCPreciseLifetime());
3239
7
3240
7
  // Otherwise, we can just do a normal lvalue store.
3241
19
  } else {
3242
19
    CGF.EmitStoreThroughLValue(RValue::get(value), srcLV);
3243
19
  }
3244
26
3245
26
  // Jump to the continuation block.
3246
26
  if (!provablyNonNull)
3247
5
    CGF.EmitBlock(contBB);
3248
26
}
3249
3250
static void emitWritebacks(CodeGenFunction &CGF,
3251
26
                           const CallArgList &args) {
3252
26
  for (const auto &I : args.writebacks())
3253
26
    emitWriteback(CGF, I);
3254
26
}
3255
3256
static void deactivateArgCleanupsBeforeCall(CodeGenFunction &CGF,
3257
135
                                            const CallArgList &CallArgs) {
3258
135
  ArrayRef<CallArgList::CallArgCleanup> Cleanups =
3259
135
    CallArgs.getCleanupsToDeactivate();
3260
135
  // Iterate in reverse to increase the likelihood of popping the cleanup.
3261
173
  for (const auto &I : llvm::reverse(Cleanups)) {
3262
173
    CGF.DeactivateCleanupBlock(I.Cleanup, I.IsActiveIP);
3263
173
    I.IsActiveIP->eraseFromParent();
3264
173
  }
3265
135
}
3266
3267
26
static const Expr *maybeGetUnaryAddrOfOperand(const Expr *E) {
3268
26
  if (const UnaryOperator *uop = dyn_cast<UnaryOperator>(E->IgnoreParens()))
3269
21
    if (uop->getOpcode() == UO_AddrOf)
3270
21
      return uop->getSubExpr();
3271
5
  return nullptr;
3272
5
}
3273
3274
/// Emit an argument that's being passed call-by-writeback.  That is,
3275
/// we are passing the address of an __autoreleased temporary; it
3276
/// might be copy-initialized with the current value of the given
3277
/// address, but it will definitely be copied out of after the call.
3278
static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args,
3279
26
                             const ObjCIndirectCopyRestoreExpr *CRE) {
3280
26
  LValue srcLV;
3281
26
3282
26
  // Make an optimistic effort to emit the address as an l-value.
3283
26
  // This can fail if the argument expression is more complicated.
3284
26
  if (const Expr *lvExpr = maybeGetUnaryAddrOfOperand(CRE->getSubExpr())) {
3285
21
    srcLV = CGF.EmitLValue(lvExpr);
3286
21
3287
21
  // Otherwise, just emit it as a scalar.
3288
21
  } else {
3289
5
    Address srcAddr = CGF.EmitPointerWithAlignment(CRE->getSubExpr());
3290
5
3291
5
    QualType srcAddrType =
3292
5
      CRE->getSubExpr()->getType()->castAs<PointerType>()->getPointeeType();
3293
5
    srcLV = CGF.MakeAddrLValue(srcAddr, srcAddrType);
3294
5
  }
3295
26
  Address srcAddr = srcLV.getAddress(CGF);
3296
26
3297
26
  // The dest and src types don't necessarily match in LLVM terms
3298
26
  // because of the crazy ObjC compatibility rules.
3299
26
3300
26
  llvm::PointerType *destType =
3301
26
    cast<llvm::PointerType>(CGF.ConvertType(CRE->getType()));
3302
26
3303
26
  // If the address is a constant null, just pass the appropriate null.
3304
26
  if (isProvablyNull(srcAddr.getPointer())) {
3305
0
    args.add(RValue::get(llvm::ConstantPointerNull::get(destType)),
3306
0
             CRE->getType());
3307
0
    return;
3308
0
  }
3309
26
3310
26
  // Create the temporary.
3311
26
  Address temp = CGF.CreateTempAlloca(destType->getElementType(),
3312
26
                                      CGF.getPointerAlign(),
3313
26
                                      "icr.temp");
3314
26
  // Loading an l-value can introduce a cleanup if the l-value is __weak,
3315
26
  // and that cleanup will be conditional if we can't prove that the l-value
3316
26
  // isn't null, so we need to register a dominating point so that the cleanups
3317
26
  // system will make valid IR.
3318
26
  CodeGenFunction::ConditionalEvaluation condEval(CGF);
3319
26
3320
26
  // Zero-initialize it if we're not doing a copy-initialization.
3321
26
  bool shouldCopy = CRE->shouldCopy();
3322
26
  if (!shouldCopy) {
3323
4
    llvm::Value *null =
3324
4
      llvm::ConstantPointerNull::get(
3325
4
        cast<llvm::PointerType>(destType->getElementType()));
3326
4
    CGF.Builder.CreateStore(null, temp);
3327
4
  }
3328
26
3329
26
  llvm::BasicBlock *contBB = nullptr;
3330
26
  llvm::BasicBlock *originBB = nullptr;
3331
26
3332
26
  // If the address is *not* known to be non-null, we need to switch.
3333
26
  llvm::Value *finalArgument;
3334
26
3335
26
  bool provablyNonNull = llvm::isKnownNonZero(srcAddr.getPointer(),
3336
26
                                              CGF.CGM.getDataLayout());
3337
26
  if (provablyNonNull) {
3338
21
    finalArgument = temp.getPointer();
3339
21
  } else {
3340
5
    llvm::Value *isNull =
3341
5
      CGF.Builder.CreateIsNull(srcAddr.getPointer(), "icr.isnull");
3342
5
3343
5
    finalArgument = CGF.Builder.CreateSelect(isNull,
3344
5
                                   llvm::ConstantPointerNull::get(destType),
3345
5
                                             temp.getPointer(), "icr.argument");
3346
5
3347
5
    // If we need to copy, then the load has to be conditional, which
3348
5
    // means we need control flow.
3349
5
    if (shouldCopy) {
3350
5
      originBB = CGF.Builder.GetInsertBlock();
3351
5
      contBB = CGF.createBasicBlock("icr.cont");
3352
5
      llvm::BasicBlock *copyBB = CGF.createBasicBlock("icr.copy");
3353
5
      CGF.Builder.CreateCondBr(isNull, contBB, copyBB);
3354
5
      CGF.EmitBlock(copyBB);
3355
5
      condEval.begin(CGF);
3356
5
    }
3357
5
  }
3358
26
3359
26
  llvm::Value *valueToUse = nullptr;
3360
26
3361
26
  // Perform a copy if necessary.
3362
26
  if (shouldCopy) {
3363
22
    RValue srcRV = CGF.EmitLoadOfLValue(srcLV, SourceLocation());
3364
22
    assert(srcRV.isScalar());
3365
22
3366
22
    llvm::Value *src = srcRV.getScalarVal();
3367
22
    src = CGF.Builder.CreateBitCast(src, destType->getElementType(),
3368
22
                                    "icr.cast");
3369
22
3370
22
    // Use an ordinary store, not a store-to-lvalue.
3371
22
    CGF.Builder.CreateStore(src, temp);
3372
22
3373
22
    // If optimization is enabled, and the value was held in a
3374
22
    // __strong variable, we need to tell the optimizer that this
3375
22
    // value has to stay alive until we're doing the store back.
3376
22
    // This is because the temporary is effectively unretained,
3377
22
    // and so otherwise we can violate the high-level semantics.
3378
22
    if (CGF.CGM.getCodeGenOpts().OptimizationLevel != 0 &&
3379
22
        
srcLV.getObjCLifetime() == Qualifiers::OCL_Strong9
) {
3380
7
      valueToUse = src;
3381
7
    }
3382
22
  }
3383
26
3384
26
  // Finish the control flow if we needed it.
3385
26
  if (shouldCopy && 
!provablyNonNull22
) {
3386
5
    llvm::BasicBlock *copyBB = CGF.Builder.GetInsertBlock();
3387
5
    CGF.EmitBlock(contBB);
3388
5
3389
5
    // Make a phi for the value to intrinsically use.
3390
5
    if (valueToUse) {
3391
2
      llvm::PHINode *phiToUse = CGF.Builder.CreatePHI(valueToUse->getType(), 2,
3392
2
                                                      "icr.to-use");
3393
2
      phiToUse->addIncoming(valueToUse, copyBB);
3394
2
      phiToUse->addIncoming(llvm::UndefValue::get(valueToUse->getType()),
3395
2
                            originBB);
3396
2
      valueToUse = phiToUse;
3397
2
    }
3398
5
3399
5
    condEval.end(CGF);
3400
5
  }
3401
26
3402
26
  args.addWriteback(srcLV, temp, valueToUse);
3403
26
  args.add(RValue::get(finalArgument), CRE->getType());
3404
26
}
3405
3406
68
void CallArgList::allocateArgumentMemory(CodeGenFunction &CGF) {
3407
68
  assert(!StackBase);
3408
68
3409
68
  // Save the stack.
3410
68
  llvm::Function *F = CGF.CGM.getIntrinsic(llvm::Intrinsic::stacksave);
3411
68
  StackBase = CGF.Builder.CreateCall(F, {}, "inalloca.save");
3412
68
}
3413
3414
303k
void CallArgList::freeArgumentMemory(CodeGenFunction &CGF) const {
3415
303k
  if (StackBase) {
3416
68
    // Restore the stack after the call.
3417
68
    llvm::Function *F = CGF.CGM.getIntrinsic(llvm::Intrinsic::stackrestore);
3418
68
    CGF.Builder.CreateCall(F, StackBase);
3419
68
  }
3420
303k
}
3421
3422
void CodeGenFunction::EmitNonNullArgCheck(RValue RV, QualType ArgType,
3423
                                          SourceLocation ArgLoc,
3424
                                          AbstractCallee AC,
3425
330k
                                          unsigned ParmNum) {
3426
330k
  if (!AC.getDecl() || 
!(320k
SanOpts.has(SanitizerKind::NonnullAttribute)320k
||
3427
320k
                         
SanOpts.has(SanitizerKind::NullabilityArg)320k
))
3428
330k
    return;
3429
71
3430
71
  // The param decl may be missing in a variadic function.
3431
71
  auto PVD = ParmNum < AC.getNumParams() ? 
AC.getParamDecl(ParmNum)60
:
nullptr11
;
3432
71
  unsigned ArgNo = PVD ? 
PVD->getFunctionScopeIndex()60
:
ParmNum11
;
3433
71
3434
71
  // Prefer the nonnull attribute if it's present.
3435
71
  const NonNullAttr *NNAttr = nullptr;
3436
71
  if (SanOpts.has(SanitizerKind::NonnullAttribute))
3437
23
    NNAttr = getNonNullAttr(AC.getDecl(), PVD, ArgType, ArgNo);
3438
71
3439
71
  bool CanCheckNullability = false;
3440
71
  if (SanOpts.has(SanitizerKind::NullabilityArg) && 
!NNAttr49
&&
PVD48
) {
3441
40
    auto Nullability = PVD->getType()->getNullability(getContext());
3442
40
    CanCheckNullability = Nullability &&
3443
40
                          
*Nullability == NullabilityKind::NonNull16
&&
3444
40
                          
PVD->getTypeSourceInfo()14
;
3445
40
  }
3446
71
3447
71
  if (!NNAttr && 
!CanCheckNullability55
)
3448
41
    return;
3449
30
3450
30
  SourceLocation AttrLoc;
3451
30
  SanitizerMask CheckKind;
3452
30
  SanitizerHandler Handler;
3453
30
  if (NNAttr) {
3454
16
    AttrLoc = NNAttr->getLocation();
3455
16
    CheckKind = SanitizerKind::NonnullAttribute;
3456
16
    Handler = SanitizerHandler::NonnullArg;
3457
16
  } else {
3458
14
    AttrLoc = PVD->getTypeSourceInfo()->getTypeLoc().findNullabilityLoc();
3459
14
    CheckKind = SanitizerKind::NullabilityArg;
3460
14
    Handler = SanitizerHandler::NullabilityArg;
3461
14
  }
3462
30
3463
30
  SanitizerScope SanScope(this);
3464
30
  assert(RV.isScalar());
3465
30
  llvm::Value *V = RV.getScalarVal();
3466
30
  llvm::Value *Cond =
3467
30
      Builder.CreateICmpNE(V, llvm::Constant::getNullValue(V->getType()));
3468
30
  llvm::Constant *StaticData[] = {
3469
30
      EmitCheckSourceLocation(ArgLoc), EmitCheckSourceLocation(AttrLoc),
3470
30
      llvm::ConstantInt::get(Int32Ty, ArgNo + 1),
3471
30
  };
3472
30
  EmitCheck(std::make_pair(Cond, CheckKind), Handler, StaticData, None);
3473
30
}
3474
3475
void CodeGenFunction::EmitCallArgs(
3476
    CallArgList &Args, ArrayRef<QualType> ArgTypes,
3477
    llvm::iterator_range<CallExpr::const_arg_iterator> ArgRange,
3478
262k
    AbstractCallee AC, unsigned ParamsToSkip, EvaluationOrder Order) {
3479
262k
  assert((int)ArgTypes.size() == (ArgRange.end() - ArgRange.begin()));
3480
262k
3481
262k
  // We *have* to evaluate arguments from right to left in the MS C++ ABI,
3482
262k
  // because arguments are destroyed left to right in the callee. As a special
3483
262k
  // case, there are certain language constructs that require left-to-right
3484
262k
  // evaluation, and in those cases we consider the evaluation order requirement
3485
262k
  // to trump the "destruction order is reverse construction order" guarantee.
3486
262k
  bool LeftToRight =
3487
262k
      CGM.getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()
3488
262k
          ? 
Order == EvaluationOrder::ForceLeftToRight6.78k
3489
262k
          : 
Order != EvaluationOrder::ForceRightToLeft255k
;
3490
262k
3491
262k
  auto MaybeEmitImplicitObjectSize = [&](unsigned I, const Expr *Arg,
3492
330k
                                         RValue EmittedArg) {
3493
330k
    if (!AC.hasFunctionDecl() || 
I >= AC.getNumParams()308k
)
3494
69.9k
      return;
3495
260k
    auto *PS = AC.getParamDecl(I)->getAttr<PassObjectSizeAttr>();
3496
260k
    if (PS == nullptr)
3497
260k
      return;
3498
107
3499
107
    const auto &Context = getContext();
3500
107
    auto SizeTy = Context.getSizeType();
3501
107
    auto T = Builder.getIntNTy(Context.getTypeSize(SizeTy));
3502
107
    assert(EmittedArg.getScalarVal() && "We emitted nothing for the arg?");
3503
107
    llvm::Value *V = evaluateOrEmitBuiltinObjectSize(Arg, PS->getType(), T,
3504
107
                                                     EmittedArg.getScalarVal(),
3505
107
                                                     PS->isDynamic());
3506
107
    Args.add(RValue::get(V), SizeTy);
3507
107
    // If we're emitting args in reverse, be sure to do so with
3508
107
    // pass_object_size, as well.
3509
107
    if (!LeftToRight)
3510
9
      std::swap(Args.back(), *(&Args.back() - 1));
3511
107
  };
3512
262k
3513
262k
  // Insert a stack save if we're going to need any inalloca args.
3514
262k
  bool HasInAllocaArgs = false;
3515
262k
  if (CGM.getTarget().getCXXABI().isMicrosoft()) {
3516
6.78k
    for (ArrayRef<QualType>::iterator I = ArgTypes.begin(), E = ArgTypes.end();
3517
12.9k
         I != E && 
!HasInAllocaArgs6.21k
;
++I6.18k
)
3518
6.18k
      HasInAllocaArgs = isInAllocaArgument(CGM.getCXXABI(), *I);
3519
6.78k
    if (HasInAllocaArgs) {
3520
68
      assert(getTarget().getTriple().getArch() == llvm::Triple::x86);
3521
68
      Args.allocateArgumentMemory(*this);
3522
68
    }
3523
6.78k
  }
3524
262k
3525
262k
  // Evaluate each argument in the appropriate order.
3526
262k
  size_t CallArgsStart = Args.size();
3527
593k
  for (unsigned I = 0, E = ArgTypes.size(); I != E; 
++I330k
) {
3528
330k
    unsigned Idx = LeftToRight ? 
I323k
:
E - I - 16.86k
;
3529
330k
    CallExpr::const_arg_iterator Arg = ArgRange.begin() + Idx;
3530
330k
    unsigned InitialArgSize = Args.size();
3531
330k
    // If *Arg is an ObjCIndirectCopyRestoreExpr, check that either the types of
3532
330k
    // the argument and parameter match or the objc method is parameterized.
3533
330k
    assert((!isa<ObjCIndirectCopyRestoreExpr>(*Arg) ||
3534
330k
            getContext().hasSameUnqualifiedType((*Arg)->getType(),
3535
330k
                                                ArgTypes[Idx]) ||
3536
330k
            (isa<ObjCMethodDecl>(AC.getDecl()) &&
3537
330k
             isObjCMethodWithTypeParams(cast<ObjCMethodDecl>(AC.getDecl())))) &&
3538
330k
           "Argument and parameter types don't match");
3539
330k
    EmitCallArg(Args, *Arg, ArgTypes[Idx]);
3540
330k
    // In particular, we depend on it being the last arg in Args, and the
3541
330k
    // objectsize bits depend on there only being one arg if !LeftToRight.
3542
330k
    assert(InitialArgSize + 1 == Args.size() &&
3543
330k
           "The code below depends on only adding one arg per EmitCallArg");
3544
330k
    (void)InitialArgSize;
3545
330k
    // Since pointer argument are never emitted as LValue, it is safe to emit
3546
330k
    // non-null argument check for r-value only.
3547
330k
    if (!Args.back().hasLValue()) {
3548
330k
      RValue RVArg = Args.back().getKnownRValue();
3549
330k
      EmitNonNullArgCheck(RVArg, ArgTypes[Idx], (*Arg)->getExprLoc(), AC,
3550
330k
                          ParamsToSkip + Idx);
3551
330k
      // @llvm.objectsize should never have side-effects and shouldn't need
3552
330k
      // destruction/cleanups, so we can safely "emit" it after its arg,
3553
330k
      // regardless of right-to-leftness
3554
330k
      MaybeEmitImplicitObjectSize(Idx, *Arg, RVArg);
3555
330k
    }
3556
330k
  }
3557
262k
3558
262k
  if (!LeftToRight) {
3559
7.39k
    // Un-reverse the arguments we just evaluated so they match up with the LLVM
3560
7.39k
    // IR function.
3561
7.39k
    std::reverse(Args.begin() + CallArgsStart, Args.end());
3562
7.39k
  }
3563
262k
}
3564
3565
namespace {
3566
3567
struct DestroyUnpassedArg final : EHScopeStack::Cleanup {
3568
  DestroyUnpassedArg(Address Addr, QualType Ty)
3569
163
      : Addr(Addr), Ty(Ty) {}
3570
3571
  Address Addr;
3572
  QualType Ty;
3573
3574
18
  void Emit(CodeGenFunction &CGF, Flags flags) override {
3575
18
    QualType::DestructionKind DtorKind = Ty.isDestructedType();
3576
18
    if (DtorKind == QualType::DK_cxx_destructor) {
3577
16
      const CXXDestructorDecl *Dtor = Ty->getAsCXXRecordDecl()->getDestructor();
3578
16
      assert(!Dtor->isTrivial());
3579
16
      CGF.EmitCXXDestructorCall(Dtor, Dtor_Complete, /*for vbase*/ false,
3580
16
                                /*Delegating=*/false, Addr, Ty);
3581
16
    } else {
3582
2
      CGF.callCStructDestructor(CGF.MakeAddrLValue(Addr, Ty));
3583
2
    }
3584
18
  }
3585
};
3586
3587
struct DisableDebugLocationUpdates {
3588
  CodeGenFunction &CGF;
3589
  bool disabledDebugInfo;
3590
330k
  DisableDebugLocationUpdates(CodeGenFunction &CGF, const Expr *E) : CGF(CGF) {
3591
330k
    if ((disabledDebugInfo = isa<CXXDefaultArgExpr>(E) && 
CGF.getDebugInfo()4.05k
))
3592
2.99k
      CGF.disableDebugInfo();
3593
330k
  }
3594
330k
  ~DisableDebugLocationUpdates() {
3595
330k
    if (disabledDebugInfo)
3596
2.99k
      CGF.enableDebugInfo();
3597
330k
  }
3598
};
3599
3600
} // end anonymous namespace
3601
3602
229
RValue CallArg::getRValue(CodeGenFunction &CGF) const {
3603
229
  if (!HasLV)
3604
229
    return RV;
3605
0
  LValue Copy = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty), Ty);
3606
0
  CGF.EmitAggregateCopy(Copy, LV, Ty, AggValueSlot::DoesNotOverlap,
3607
0
                        LV.isVolatile());
3608
0
  IsUsed = true;
3609
0
  return RValue::getAggregate(Copy.getAddress(CGF));
3610
0
}
3611
3612
1.13k
void CallArg::copyInto(CodeGenFunction &CGF, Address Addr) const {
3613
1.13k
  LValue Dst = CGF.MakeAddrLValue(Addr, Ty);
3614
1.13k
  if (!HasLV && 
RV.isScalar()1.09k
)
3615
752
    CGF.EmitStoreOfScalar(RV.getScalarVal(), Dst, /*isInit=*/true);
3616
387
  else if (!HasLV && 
RV.isComplex()345
)
3617
345
    CGF.EmitStoreOfComplex(RV.getComplexVal(), Dst, /*init=*/true);
3618
42
  else {
3619
42
    auto Addr = HasLV ? LV.getAddress(CGF) : 
RV.getAggregateAddress()0
;
3620
42
    LValue SrcLV = CGF.MakeAddrLValue(Addr, Ty);
3621
42
    // We assume that call args are never copied into subobjects.
3622
42
    CGF.EmitAggregateCopy(Dst, SrcLV, Ty, AggValueSlot::DoesNotOverlap,
3623
42
                          HasLV ? LV.isVolatileQualified()
3624
42
                                : 
RV.isVolatileQualified()0
);
3625
42
  }
3626
1.13k
  IsUsed = true;
3627
1.13k
}
3628
3629
void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E,
3630
330k
                                  QualType type) {
3631
330k
  DisableDebugLocationUpdates Dis(*this, E);
3632
330k
  if (const ObjCIndirectCopyRestoreExpr *CRE
3633
26
        = dyn_cast<ObjCIndirectCopyRestoreExpr>(E)) {
3634
26
    assert(getLangOpts().ObjCAutoRefCount);
3635
26
    return emitWritebackArg(*this, args, CRE);
3636
26
  }
3637
330k
3638
330k
  assert(type->isReferenceType() == E->isGLValue() &&
3639
330k
         "reference binding to unmaterialized r-value!");
3640
330k
3641
330k
  if (E->isGLValue()) {
3642
59.0k
    assert(E->getObjectKind() == OK_Ordinary);
3643
59.0k
    return args.add(EmitReferenceBindingToExpr(E), type);
3644
59.0k
  }
3645
271k
3646
271k
  bool HasAggregateEvalKind = hasAggregateEvaluationKind(type);
3647
271k
3648
271k
  // In the Microsoft C++ ABI, aggregate arguments are destructed by the callee.
3649
271k
  // However, we still have to push an EH-only cleanup in case we unwind before
3650
271k
  // we make it to the call.
3651
271k
  if (HasAggregateEvalKind &&
3652
271k
      
type->castAs<RecordType>()->getDecl()->isParamDestroyedInCallee()10.7k
) {
3653
333
    // If we're using inalloca, use the argument memory.  Otherwise, use a
3654
333
    // temporary.
3655
333
    AggValueSlot Slot;
3656
333
    if (args.isUsingInAlloca())
3657
96
      Slot = createPlaceholderSlot(*this, type);
3658
237
    else
3659
237
      Slot = CreateAggTemp(type, "agg.tmp");
3660
333
3661
333
    bool DestroyedInCallee = true, NeedsEHCleanup = true;
3662
333
    if (const auto *RD = type->getAsCXXRecordDecl())
3663
323
      DestroyedInCallee = RD->hasNonTrivialDestructor();
3664
10
    else
3665
10
      NeedsEHCleanup = needsEHCleanup(type.isDestructedType());
3666
333
3667
333
    if (DestroyedInCallee)
3668
173
      Slot.setExternallyDestructed();
3669
333
3670
333
    EmitAggExpr(E, Slot);
3671
333
    RValue RV = Slot.asRValue();
3672
333
    args.add(RV, type);
3673
333
3674
333
    if (DestroyedInCallee && 
NeedsEHCleanup173
) {
3675
167
      // Create a no-op GEP between the placeholder and the cleanup so we can
3676
167
      // RAUW it successfully.  It also serves as a marker of the first
3677
167
      // instruction where the cleanup is active.
3678
167
      pushFullExprCleanup<DestroyUnpassedArg>(EHCleanup, Slot.getAddress(),
3679
167
                                              type);
3680
167
      // This unreachable is a temporary marker which will be removed later.
3681
167
      llvm::Instruction *IsActive = Builder.CreateUnreachable();
3682
167
      args.addArgCleanupDeactivation(EHStack.getInnermostEHScope(), IsActive);
3683
167
    }
3684
333
    return;
3685
333
  }
3686
271k
3687
271k
  if (HasAggregateEvalKind && 
isa<ImplicitCastExpr>(E)10.4k
&&
3688
271k
      
cast<CastExpr>(E)->getCastKind() == CK_LValueToRValue678
) {
3689
660
    LValue L = EmitLValue(cast<CastExpr>(E)->getSubExpr());
3690
660
    assert(L.isSimple());
3691
660
    args.addUncopiedAggregate(L, type);
3692
660
    return;
3693
660
  }
3694
270k
3695
270k
  args.add(EmitAnyExprToTemp(E), type);
3696
270k
}
3697
3698
50.2k
QualType CodeGenFunction::getVarArgType(const Expr *Arg) {
3699
50.2k
  // System headers on Windows define NULL to 0 instead of 0LL on Win64. MSVC
3700
50.2k
  // implicitly widens null pointer constants that are arguments to varargs
3701
50.2k
  // functions to pointer-sized ints.
3702
50.2k
  if (!getTarget().getTriple().isOSWindows())
3703
50.1k
    return Arg->getType();
3704
67
3705
67
  if (Arg->getType()->isIntegerType() &&
3706
67
      getContext().getTypeSize(Arg->getType()) <
3707
44
          getContext().getTargetInfo().getPointerWidth(0) &&
3708
67
      Arg->isNullPointerConstant(getContext(),
3709
20
                                 Expr::NPC_ValueDependentIsNotNull)) {
3710
1
    return getContext().getIntPtrType();
3711
1
  }
3712
66
3713
66
  return Arg->getType();
3714
66
}
3715
3716
// In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
3717
// optimizer it can aggressively ignore unwind edges.
3718
void
3719
1.60k
CodeGenFunction::AddObjCARCExceptionMetadata(llvm::Instruction *Inst) {
3720
1.60k
  if (CGM.getCodeGenOpts().OptimizationLevel != 0 &&
3721
1.60k
      
!CGM.getCodeGenOpts().ObjCAutoRefCountExceptions395
)
3722
307
    Inst->setMetadata("clang.arc.no_objc_arc_exceptions",
3723
307
                      CGM.getNoObjCARCExceptionsMetadata());
3724
1.60k
}
3725
3726
/// Emits a call to the given no-arguments nounwind runtime function.
3727
llvm::CallInst *
3728
CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
3729
427
                                         const llvm::Twine &name) {
3730
427
  return EmitNounwindRuntimeCall(callee, None, name);
3731
427
}
3732
3733
/// Emits a call to the given nounwind runtime function.
3734
llvm::CallInst *
3735
CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
3736
                                         ArrayRef<llvm::Value *> args,
3737
24.2k
                                         const llvm::Twine &name) {
3738
24.2k
  llvm::CallInst *call = EmitRuntimeCall(callee, args, name);
3739
24.2k
  call->setDoesNotThrow();
3740
24.2k
  return call;
3741
24.2k
}
3742
3743
/// Emits a simple call (never an invoke) to the given no-arguments
3744
/// runtime function.
3745
llvm::CallInst *CodeGenFunction::EmitRuntimeCall(llvm::FunctionCallee callee,
3746
6.10k
                                                 const llvm::Twine &name) {
3747
6.10k
  return EmitRuntimeCall(callee, None, name);
3748
6.10k
}
3749
3750
// Calls which may throw must have operand bundles indicating which funclet
3751
// they are nested within.
3752
SmallVector<llvm::OperandBundleDef, 1>
3753
377k
CodeGenFunction::getBundlesForFunclet(llvm::Value *Callee) {
3754
377k
  SmallVector<llvm::OperandBundleDef, 1> BundleList;
3755
377k
  // There is no need for a funclet operand bundle if we aren't inside a
3756
377k
  // funclet.
3757
377k
  if (!CurrentFuncletPad)
3758
377k
    return BundleList;
3759
399
3760
399
  // Skip intrinsics which cannot throw.
3761
399
  auto *CalleeFn = dyn_cast<llvm::Function>(Callee->stripPointerCasts());
3762
399
  if (CalleeFn && 
CalleeFn->isIntrinsic()393
&&
CalleeFn->doesNotThrow()20
)
3763
2
    return BundleList;
3764
397
3765
397
  BundleList.emplace_back("funclet", CurrentFuncletPad);
3766
397
  return BundleList;
3767
397
}
3768
3769
/// Emits a simple call (never an invoke) to the given runtime function.
3770
llvm::CallInst *CodeGenFunction::EmitRuntimeCall(llvm::FunctionCallee callee,
3771
                                                 ArrayRef<llvm::Value *> args,
3772
68.2k
                                                 const llvm::Twine &name) {
3773
68.2k
  llvm::CallInst *call = Builder.CreateCall(
3774
68.2k
      callee, args, getBundlesForFunclet(callee.getCallee()), name);
3775
68.2k
  call->setCallingConv(getRuntimeCC());
3776
68.2k
  return call;
3777
68.2k
}
3778
3779
/// Emits a call or invoke to the given noreturn runtime function.
3780
void CodeGenFunction::EmitNoreturnRuntimeCallOrInvoke(
3781
520
    llvm::FunctionCallee callee, ArrayRef<llvm::Value *> args) {
3782
520
  SmallVector<llvm::OperandBundleDef, 1> BundleList =
3783
520
      getBundlesForFunclet(callee.getCallee());
3784
520
3785
520
  if (getInvokeDest()) {
3786
121
    llvm::InvokeInst *invoke =
3787
121
      Builder.CreateInvoke(callee,
3788
121
                           getUnreachableBlock(),
3789
121
                           getInvokeDest(),
3790
121
                           args,
3791
121
                           BundleList);
3792
121
    invoke->setDoesNotReturn();
3793
121
    invoke->setCallingConv(getRuntimeCC());
3794
399
  } else {
3795
399
    llvm::CallInst *call = Builder.CreateCall(callee, args, BundleList);
3796
399
    call->setDoesNotReturn();
3797
399
    call->setCallingConv(getRuntimeCC());
3798
399
    Builder.CreateUnreachable();
3799
399
  }
3800
520
}
3801
3802
/// Emits a call or invoke instruction to the given nullary runtime function.
3803
llvm::CallBase *
3804
CodeGenFunction::EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
3805
369
                                         const Twine &name) {
3806
369
  return EmitRuntimeCallOrInvoke(callee, None, name);
3807
369
}
3808
3809
/// Emits a call or invoke instruction to the given runtime function.
3810
llvm::CallBase *
3811
CodeGenFunction::EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
3812
                                         ArrayRef<llvm::Value *> args,
3813
560
                                         const Twine &name) {
3814
560
  llvm::CallBase *call = EmitCallOrInvoke(callee, args, name);
3815
560
  call->setCallingConv(getRuntimeCC());
3816
560
  return call;
3817
560
}
3818
3819
/// Emits a call or invoke instruction to the given function, depending
3820
/// on the current state of the EH stack.
3821
llvm::CallBase *CodeGenFunction::EmitCallOrInvoke(llvm::FunctionCallee Callee,
3822
                                                  ArrayRef<llvm::Value *> Args,
3823
2.19k
                                                  const Twine &Name) {
3824
2.19k
  llvm::BasicBlock *InvokeDest = getInvokeDest();
3825
2.19k
  SmallVector<llvm::OperandBundleDef, 1> BundleList =
3826
2.19k
      getBundlesForFunclet(Callee.getCallee());
3827
2.19k
3828
2.19k
  llvm::CallBase *Inst;
3829
2.19k
  if (!InvokeDest)
3830
1.94k
    Inst = Builder.CreateCall(Callee, Args, BundleList, Name);
3831
254
  else {
3832
254
    llvm::BasicBlock *ContBB = createBasicBlock("invoke.cont");
3833
254
    Inst = Builder.CreateInvoke(Callee, ContBB, InvokeDest, Args, BundleList,
3834
254
                                Name);
3835
254
    EmitBlock(ContBB);
3836
254
  }
3837
2.19k
3838
2.19k
  // In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
3839
2.19k
  // optimizer it can aggressively ignore unwind edges.
3840
2.19k
  if (CGM.getLangOpts().ObjCAutoRefCount)
3841
3
    AddObjCARCExceptionMetadata(Inst);
3842
2.19k
3843
2.19k
  return Inst;
3844
2.19k
}
3845
3846
void CodeGenFunction::deferPlaceholderReplacement(llvm::Instruction *Old,
3847
97
                                                  llvm::Value *New) {
3848
97
  DeferredReplacements.push_back(std::make_pair(Old, New));
3849
97
}
3850
3851
namespace {
3852
3853
/// Specify given \p NewAlign as the alignment of return value attribute. If
3854
/// such attribute already exists, re-set it to the maximal one of two options.
3855
LLVM_NODISCARD llvm::AttributeList
3856
maybeRaiseRetAlignmentAttribute(llvm::LLVMContext &Ctx,
3857
                                const llvm::AttributeList &Attrs,
3858
11
                                llvm::Align NewAlign) {
3859
11
  llvm::Align CurAlign = Attrs.getRetAlignment().valueOrOne();
3860
11
  if (CurAlign >= NewAlign)
3861
2
    return Attrs;
3862
9
  llvm::Attribute AlignAttr = llvm::Attribute::getWithAlignment(Ctx, NewAlign);
3863
9
  return Attrs
3864
9
      .removeAttribute(Ctx, llvm::AttributeList::ReturnIndex,
3865
9
                       llvm::Attribute::AttrKind::Alignment)
3866
9
      .addAttribute(Ctx, llvm::AttributeList::ReturnIndex, AlignAttr);
3867
9
}
3868
3869
template <typename AlignedAttrTy> class AbstractAssumeAlignedAttrEmitter {
3870
protected:
3871
  CodeGenFunction &CGF;
3872
3873
  /// We do nothing if this is, or becomes, nullptr.
3874
  const AlignedAttrTy *AA = nullptr;
3875
3876
  llvm::Value *Alignment = nullptr;      // May or may not be a constant.
3877
  llvm::ConstantInt *OffsetCI = nullptr; // Constant, hopefully zero.
3878
3879
  AbstractAssumeAlignedAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl)
3880
610k
      : CGF(CGF_) {
3881
610k
    if (!FuncDecl)
3882
31.9k
      return;
3883
578k
    AA = FuncDecl->getAttr<AlignedAttrTy>();
3884
578k
  }
CGCall.cpp:(anonymous namespace)::AbstractAssumeAlignedAttrEmitter<clang::AssumeAlignedAttr>::AbstractAssumeAlignedAttrEmitter(clang::CodeGen::CodeGenFunction&, clang::Decl const*)
Line
Count
Source
3880
305k
      : CGF(CGF_) {
3881
305k
    if (!FuncDecl)
3882
15.9k
      return;
3883
289k
    AA = FuncDecl->getAttr<AlignedAttrTy>();
3884
289k
  }
CGCall.cpp:(anonymous namespace)::AbstractAssumeAlignedAttrEmitter<clang::AllocAlignAttr>::AbstractAssumeAlignedAttrEmitter(clang::CodeGen::CodeGenFunction&, clang::Decl const*)
Line
Count
Source
3880
305k
      : CGF(CGF_) {
3881
305k
    if (!FuncDecl)
3882
15.9k
      return;
3883
289k
    AA = FuncDecl->getAttr<AlignedAttrTy>();
3884
289k
  }
3885
3886
public:
3887
  /// If we can, materialize the alignment as an attribute on return value.
3888
  LLVM_NODISCARD llvm::AttributeList
3889
610k
  TryEmitAsCallSiteAttribute(const llvm::AttributeList &Attrs) {
3890
610k
    if (!AA || 
OffsetCI35
||
CGF.SanOpts.has(SanitizerKind::Alignment)30
)
3891
610k
      return Attrs;
3892
21
    const auto *AlignmentCI = dyn_cast<llvm::ConstantInt>(Alignment);
3893
21
    if (!AlignmentCI)
3894
9
      return Attrs;
3895
12
    // We may legitimately have non-power-of-2 alignment here.
3896
12
    // If so, this is UB land, emit it via `@llvm.assume` instead.
3897
12
    if (!AlignmentCI->getValue().isPowerOf2())
3898
1
      return Attrs;
3899
11
    llvm::AttributeList NewAttrs = maybeRaiseRetAlignmentAttribute(
3900
11
        CGF.getLLVMContext(), Attrs,
3901
11
        llvm::Align(
3902
11
            AlignmentCI->getLimitedValue(llvm::Value::MaximumAlignment)));
3903
11
    AA = nullptr; // We're done. Disallow doing anything else.
3904
11
    return NewAttrs;
3905
11
  }
CGCall.cpp:(anonymous namespace)::AbstractAssumeAlignedAttrEmitter<clang::AssumeAlignedAttr>::TryEmitAsCallSiteAttribute(llvm::AttributeList const&)
Line
Count
Source
3889
305k
  TryEmitAsCallSiteAttribute(const llvm::AttributeList &Attrs) {
3890
305k
    if (!AA || 
OffsetCI14
||
CGF.SanOpts.has(SanitizerKind::Alignment)9
)
3891
305k
      return Attrs;
3892
6
    const auto *AlignmentCI = dyn_cast<llvm::ConstantInt>(Alignment);
3893
6
    if (!AlignmentCI)
3894
0
      return Attrs;
3895
6
    // We may legitimately have non-power-of-2 alignment here.
3896
6
    // If so, this is UB land, emit it via `@llvm.assume` instead.
3897
6
    if (!AlignmentCI->getValue().isPowerOf2())
3898
0
      return Attrs;
3899
6
    llvm::AttributeList NewAttrs = maybeRaiseRetAlignmentAttribute(
3900
6
        CGF.getLLVMContext(), Attrs,
3901
6
        llvm::Align(
3902
6
            AlignmentCI->getLimitedValue(llvm::Value::MaximumAlignment)));
3903
6
    AA = nullptr; // We're done. Disallow doing anything else.
3904
6
    return NewAttrs;
3905
6
  }
CGCall.cpp:(anonymous namespace)::AbstractAssumeAlignedAttrEmitter<clang::AllocAlignAttr>::TryEmitAsCallSiteAttribute(llvm::AttributeList const&)
Line
Count
Source
3889
305k
  TryEmitAsCallSiteAttribute(const llvm::AttributeList &Attrs) {
3890
305k
    if (!AA || 
OffsetCI21
||
CGF.SanOpts.has(SanitizerKind::Alignment)21
)
3891
305k
      return Attrs;
3892
15
    const auto *AlignmentCI = dyn_cast<llvm::ConstantInt>(Alignment);
3893
15
    if (!AlignmentCI)
3894
9
      return Attrs;
3895
6
    // We may legitimately have non-power-of-2 alignment here.
3896
6
    // If so, this is UB land, emit it via `@llvm.assume` instead.
3897
6
    if (!AlignmentCI->getValue().isPowerOf2())
3898
1
      return Attrs;
3899
5
    llvm::AttributeList NewAttrs = maybeRaiseRetAlignmentAttribute(
3900
5
        CGF.getLLVMContext(), Attrs,
3901
5
        llvm::Align(
3902
5
            AlignmentCI->getLimitedValue(llvm::Value::MaximumAlignment)));
3903
5
    AA = nullptr; // We're done. Disallow doing anything else.
3904
5
    return NewAttrs;
3905
5
  }
3906
3907
  /// Emit alignment assumption.
3908
  /// This is a general fallback that we take if either there is an offset,
3909
  /// or the alignment is variable or we are sanitizing for alignment.
3910
562k
  void EmitAsAnAssumption(SourceLocation Loc, QualType RetTy, RValue &Ret) {
3911
562k
    if (!AA)
3912
562k
      return;
3913
24
    CGF.emitAlignmentAssumption(Ret.getScalarVal(), RetTy, Loc,
3914
24
                                AA->getLocation(), Alignment, OffsetCI);
3915
24
    AA = nullptr; // We're done. Disallow doing anything else.
3916
24
  }
CGCall.cpp:(anonymous namespace)::AbstractAssumeAlignedAttrEmitter<clang::AssumeAlignedAttr>::EmitAsAnAssumption(clang::SourceLocation, clang::QualType, clang::CodeGen::RValue&)
Line
Count
Source
3910
281k
  void EmitAsAnAssumption(SourceLocation Loc, QualType RetTy, RValue &Ret) {
3911
281k
    if (!AA)
3912
281k
      return;
3913
8
    CGF.emitAlignmentAssumption(Ret.getScalarVal(), RetTy, Loc,
3914
8
                                AA->getLocation(), Alignment, OffsetCI);
3915
8
    AA = nullptr; // We're done. Disallow doing anything else.
3916
8
  }
CGCall.cpp:(anonymous namespace)::AbstractAssumeAlignedAttrEmitter<clang::AllocAlignAttr>::EmitAsAnAssumption(clang::SourceLocation, clang::QualType, clang::CodeGen::RValue&)
Line
Count
Source
3910
281k
  void EmitAsAnAssumption(SourceLocation Loc, QualType RetTy, RValue &Ret) {
3911
281k
    if (!AA)
3912
281k
      return;
3913
16
    CGF.emitAlignmentAssumption(Ret.getScalarVal(), RetTy, Loc,
3914
16
                                AA->getLocation(), Alignment, OffsetCI);
3915
16
    AA = nullptr; // We're done. Disallow doing anything else.
3916
16
  }
3917
};
3918
3919
/// Helper data structure to emit `AssumeAlignedAttr`.
3920
class AssumeAlignedAttrEmitter final
3921
    : public AbstractAssumeAlignedAttrEmitter<AssumeAlignedAttr> {
3922
public:
3923
  AssumeAlignedAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl)
3924
305k
      : AbstractAssumeAlignedAttrEmitter(CGF_, FuncDecl) {
3925
305k
    if (!AA)
3926
305k
      return;
3927
14
    // It is guaranteed that the alignment/offset are constants.
3928
14
    Alignment = cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AA->getAlignment()));
3929
14
    if (Expr *Offset = AA->getOffset()) {
3930
5
      OffsetCI = cast<llvm::ConstantInt>(CGF.EmitScalarExpr(Offset));
3931
5
      if (OffsetCI->isNullValue()) // Canonicalize zero offset to no offset.
3932
0
        OffsetCI = nullptr;
3933
5
    }
3934
14
  }
3935
};
3936
3937
/// Helper data structure to emit `AllocAlignAttr`.
3938
class AllocAlignAttrEmitter final
3939
    : public AbstractAssumeAlignedAttrEmitter<AllocAlignAttr> {
3940
public:
3941
  AllocAlignAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl,
3942
                        const CallArgList &CallArgs)
3943
305k
      : AbstractAssumeAlignedAttrEmitter(CGF_, FuncDecl) {
3944
305k
    if (!AA)
3945
305k
      return;
3946
21
    // Alignment may or may not be a constant, and that is okay.
3947
21
    Alignment = CallArgs[AA->getParamIndex().getLLVMIndex()]
3948
21
                    .getRValue(CGF)
3949
21
                    .getScalarVal();
3950
21
  }
3951
};
3952
3953
} // namespace
3954
3955
RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
3956
                                 const CGCallee &Callee,
3957
                                 ReturnValueSlot ReturnValue,
3958
                                 const CallArgList &CallArgs,
3959
                                 llvm::CallBase **callOrInvoke,
3960
305k
                                 SourceLocation Loc) {
3961
305k
  // FIXME: We no longer need the types from CallArgs; lift up and simplify.
3962
305k
3963
305k
  assert(Callee.isOrdinary() || Callee.isVirtual());
3964
305k
3965
305k
  // Handle struct-return functions by passing a pointer to the
3966
305k
  // location that we would like to return into.
3967
305k
  QualType RetTy = CallInfo.getReturnType();
3968
305k
  const ABIArgInfo &RetAI = CallInfo.getReturnInfo();
3969
305k
3970
305k
  llvm::FunctionType *IRFuncTy = getTypes().GetFunctionType(CallInfo);
3971
305k
3972
305k
  const Decl *TargetDecl = Callee.getAbstractInfo().getCalleeDecl().getDecl();
3973
305k
  if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(TargetDecl))
3974
287k
    // We can only guarantee that a function is called from the correct
3975
287k
    // context/function based on the appropriate target attributes,
3976
287k
    // so only check in the case where we have both always_inline and target
3977
287k
    // since otherwise we could be making a conditional call after a check for
3978
287k
    // the proper cpu features (and it won't cause code generation issues due to
3979
287k
    // function based code generation).
3980
287k
    if (TargetDecl->hasAttr<AlwaysInlineAttr>() &&
3981
287k
        
TargetDecl->hasAttr<TargetAttr>()34.4k
)
3982
11.8k
      checkTargetFeatures(Loc, FD);
3983
305k
3984
305k
#ifndef NDEBUG
3985
305k
  if (!(CallInfo.isVariadic() && 
CallInfo.getArgStruct()43.6k
)) {
3986
305k
    // For an inalloca varargs function, we don't expect CallInfo to match the
3987
305k
    // function pointer's type, because the inalloca struct a will have extra
3988
305k
    // fields in it for the varargs parameters.  Code later in this function
3989
305k
    // bitcasts the function pointer to the type derived from CallInfo.
3990
305k
    //
3991
305k
    // In other cases, we assert that the types match up (until pointers stop
3992
305k
    // having pointee types).
3993
305k
    llvm::Type *TypeFromVal;
3994
305k
    if (Callee.isVirtual())
3995
946
      TypeFromVal = Callee.getVirtualFunctionType();
3996
304k
    else
3997
304k
      TypeFromVal =
3998
304k
          Callee.getFunctionPointer()->getType()->getPointerElementType();
3999
305k
    assert(IRFuncTy == TypeFromVal);
4000
305k
  }
4001
305k
#endif
4002
305k
4003
305k
  // 1. Set up the arguments.
4004
305k
4005
305k
  // If we're using inalloca, insert the allocation after the stack save.
4006
305k
  // FIXME: Do this earlier rather than hacking it in here!
4007
305k
  Address ArgMemory = Address::invalid();
4008
305k
  if (llvm::StructType *ArgStruct = CallInfo.getArgStruct()) {
4009
69
    const llvm::DataLayout &DL = CGM.getDataLayout();
4010
69
    llvm::Instruction *IP = CallArgs.getStackBase();
4011
69
    llvm::AllocaInst *AI;
4012
69
    if (IP) {
4013
68
      IP = IP->getNextNode();
4014
68
      AI = new llvm::AllocaInst(ArgStruct, DL.getAllocaAddrSpace(),
4015
68
                                "argmem", IP);
4016
68
    } else {
4017
1
      AI = CreateTempAlloca(ArgStruct, "argmem");
4018
1
    }
4019
69
    auto Align = CallInfo.getArgStructAlignment();
4020
69
    AI->setAlignment(Align.getAsAlign());
4021
69
    AI->setUsedWithInAlloca(true);
4022
69
    assert(AI->isUsedWithInAlloca() && !AI->isStaticAlloca());
4023
69
    ArgMemory = Address(AI, Align);
4024
69
  }
4025
305k
4026
305k
  ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), CallInfo);
4027
305k
  SmallVector<llvm::Value *, 16> IRCallArgs(IRFunctionArgs.totalIRArgs());
4028
305k
4029
305k
  // If the call returns a temporary with struct return, create a temporary
4030
305k
  // alloca to hold the result, unless one is given to us.
4031
305k
  Address SRetPtr = Address::invalid();
4032
305k
  Address SRetAlloca = Address::invalid();
4033
305k
  llvm::Value *UnusedReturnSizePtr = nullptr;
4034
305k
  if (RetAI.isIndirect() || 
RetAI.isInAlloca()304k
||
RetAI.isCoerceAndExpand()304k
) {
4035
1.78k
    if (!ReturnValue.isNull()) {
4036
1.58k
      SRetPtr = ReturnValue.getValue();
4037
1.58k
    } else {
4038
208
      SRetPtr = CreateMemTemp(RetTy, "tmp", &SRetAlloca);
4039
208
      if (HaveInsertPoint() && ReturnValue.isUnused()) {
4040
126
        uint64_t size =
4041
126
            CGM.getDataLayout().getTypeAllocSize(ConvertTypeForMem(RetTy));
4042
126
        UnusedReturnSizePtr = EmitLifetimeStart(size, SRetAlloca.getPointer());
4043
126
      }
4044
208
    }
4045
1.78k
    if (IRFunctionArgs.hasSRetArg()) {
4046
1.34k
      IRCallArgs[IRFunctionArgs.getSRetArgNo()] = SRetPtr.getPointer();
4047
1.34k
    } else 
if (446
RetAI.isInAlloca()446
) {
4048
4
      Address Addr =
4049
4
          Builder.CreateStructGEP(ArgMemory, RetAI.getInAllocaFieldIndex());
4050
4
      Builder.CreateStore(SRetPtr.getPointer(), Addr);
4051
4
    }
4052
1.78k
  }
4053
305k
4054
305k
  Address swiftErrorTemp = Address::invalid();
4055
305k
  Address swiftErrorArg = Address::invalid();
4056
305k
4057
305k
  // When passing arguments using temporary allocas, we need to add the
4058
305k
  // appropriate lifetime markers. This vector keeps track of all the lifetime
4059
305k
  // markers that need to be ended right after the call.
4060
305k
  SmallVector<CallLifetimeEnd, 2> CallLifetimeEndAfterCall;
4061
305k
4062
305k
  // Translate all of the arguments as necessary to match the IR lowering.
4063
305k
  assert(CallInfo.arg_size() == CallArgs.size() &&
4064
305k
         "Mismatch between function signature & arguments.");
4065
305k
  unsigned ArgNo = 0;
4066
305k
  CGFunctionInfo::const_arg_iterator info_it = CallInfo.arg_begin();
4067
305k
  for (CallArgList::const_iterator I = CallArgs.begin(), E = CallArgs.end();
4068
825k
       I != E; 
++I, ++info_it, ++ArgNo519k
) {
4069
519k
    const ABIArgInfo &ArgInfo = info_it->info;
4070
519k
4071
519k
    // Insert a padding argument to ensure proper alignment.
4072
519k
    if (IRFunctionArgs.hasPaddingArg(ArgNo))
4073
17
      IRCallArgs[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
4074
17
          llvm::UndefValue::get(ArgInfo.getPaddingType());
4075
519k
4076
519k
    unsigned FirstIRArg, NumIRArgs;
4077
519k
    std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
4078
519k
4079
519k
    switch (ArgInfo.getKind()) {
4080
150
    case ABIArgInfo::InAlloca: {
4081
150
      assert(NumIRArgs == 0);
4082
150
      assert(getTarget().getTriple().getArch() == llvm::Triple::x86);
4083
150
      if (I->isAggregate()) {
4084
97
        Address Addr = I->hasLValue()
4085
97
                           ? 
I->getKnownLValue().getAddress(*this)0
4086
97
                           : I->getKnownRValue().getAggregateAddress();
4087
97
        llvm::Instruction *Placeholder =
4088
97
            cast<llvm::Instruction>(Addr.getPointer());
4089
97
4090
97
        if (!ArgInfo.getInAllocaIndirect()) {
4091
96
          // Replace the placeholder with the appropriate argument slot GEP.
4092
96
          CGBuilderTy::InsertPoint IP = Builder.saveIP();
4093
96
          Builder.SetInsertPoint(Placeholder);
4094
96
          Addr = Builder.CreateStructGEP(ArgMemory,
4095
96
                                         ArgInfo.getInAllocaFieldIndex());
4096
96
          Builder.restoreIP(IP);
4097
96
        } else {
4098
1
          // For indirect things such as overaligned structs, replace the
4099
1
          // placeholder with a regular aggregate temporary alloca. Store the
4100
1
          // address of this alloca into the struct.
4101
1
          Addr = CreateMemTemp(info_it->type, "inalloca.indirect.tmp");
4102
1
          Address ArgSlot = Builder.CreateStructGEP(
4103
1
              ArgMemory, ArgInfo.getInAllocaFieldIndex());
4104
1
          Builder.CreateStore(Addr.getPointer(), ArgSlot);
4105
1
        }
4106
97
        deferPlaceholderReplacement(Placeholder, Addr.getPointer());
4107
97
      } else 
if (53
ArgInfo.getInAllocaIndirect()53
) {
4108
2
        // Make a temporary alloca and store the address of it into the argument
4109
2
        // struct.
4110
2
        Address Addr = CreateMemTempWithoutCast(
4111
2
            I->Ty, getContext().getTypeAlignInChars(I->Ty),
4112
2
            "indirect-arg-temp");
4113
2
        I->copyInto(*this, Addr);
4114
2
        Address ArgSlot =
4115
2
            Builder.CreateStructGEP(ArgMemory, ArgInfo.getInAllocaFieldIndex());
4116
2
        Builder.CreateStore(Addr.getPointer(), ArgSlot);
4117
51
      } else {
4118
51
        // Store the RValue into the argument struct.
4119
51
        Address Addr =
4120
51
            Builder.CreateStructGEP(ArgMemory, ArgInfo.getInAllocaFieldIndex());
4121
51
        unsigned AS = Addr.getType()->getPointerAddressSpace();
4122
51
        llvm::Type *MemType = ConvertTypeForMem(I->Ty)->getPointerTo(AS);
4123
51
        // There are some cases where a trivial bitcast is not avoidable.  The
4124
51
        // definition of a type later in a translation unit may change it's type
4125
51
        // from {}* to (%struct.foo*)*.
4126
51
        if (Addr.getType() != MemType)
4127
1
          Addr = Builder.CreateBitCast(Addr, MemType);
4128
51
        I->copyInto(*this, Addr);
4129
51
      }
4130
150
      break;
4131
0
    }
4132
0
4133
1.79k
    case ABIArgInfo::Indirect: {
4134
1.79k
      assert(NumIRArgs == 1);
4135
1.79k
      if (!I->isAggregate()) {
4136
196
        // Make a temporary alloca to pass the argument.
4137
196
        Address Addr = CreateMemTempWithoutCast(
4138
196
            I->Ty, ArgInfo.getIndirectAlign(), "indirect-arg-temp");
4139
196
        IRCallArgs[FirstIRArg] = Addr.getPointer();
4140
196
4141
196
        I->copyInto(*this, Addr);
4142
1.60k
      } else {
4143
1.60k
        // We want to avoid creating an unnecessary temporary+copy here;
4144
1.60k
        // however, we need one in three cases:
4145
1.60k
        // 1. If the argument is not byval, and we are required to copy the
4146
1.60k
        //    source.  (This case doesn't occur on any common architecture.)
4147
1.60k
        // 2. If the argument is byval, RV is not sufficiently aligned, and
4148
1.60k
        //    we cannot force it to be sufficiently aligned.
4149
1.60k
        // 3. If the argument is byval, but RV is not located in default
4150
1.60k
        //    or alloca address space.
4151
1.60k
        Address Addr = I->hasLValue()
4152
1.60k
                           ? 
I->getKnownLValue().getAddress(*this)282
4153
1.60k
                           : 
I->getKnownRValue().getAggregateAddress()1.31k
;
4154
1.60k
        llvm::Value *V = Addr.getPointer();
4155
1.60k
        CharUnits Align = ArgInfo.getIndirectAlign();
4156
1.60k
        const llvm::DataLayout *TD = &CGM.getDataLayout();
4157
1.60k
4158
1.60k
        assert((FirstIRArg >= IRFuncTy->getNumParams() ||
4159
1.60k
                IRFuncTy->getParamType(FirstIRArg)->getPointerAddressSpace() ==
4160
1.60k
                    TD->getAllocaAddrSpace()) &&
4161
1.60k
               "indirect argument must be in alloca address space");
4162
1.60k
4163
1.60k
        bool NeedCopy = false;
4164
1.60k
4165
1.60k
        if (Addr.getAlignment() < Align &&
4166
1.60k
            llvm::getOrEnforceKnownAlignment(V, Align.getQuantity(), *TD) <
4167
79
                Align.getQuantity()) {
4168
2
          NeedCopy = true;
4169
1.59k
        } else if (I->hasLValue()) {
4170
280
          auto LV = I->getKnownLValue();
4171
280
          auto AS = LV.getAddressSpace();
4172
280
4173
280
          if (!ArgInfo.getIndirectByVal() ||
4174
280
              
(LV.getAlignment() < getContext().getTypeAlignInChars(I->Ty))253
) {
4175
27
            NeedCopy = true;
4176
27
          }
4177
280
          if (!getLangOpts().OpenCL) {
4178
250
            if ((ArgInfo.getIndirectByVal() &&
4179
250
                
(223
AS != LangAS::Default223
&&
4180
223
                 
AS != CGM.getASTAllocaAddressSpace()0
))) {
4181
0
              NeedCopy = true;
4182
0
            }
4183
250
          }
4184
30
          // For OpenCL even if RV is located in default or alloca address space
4185
30
          // we don't want to perform address space cast for it.
4186
30
          else if ((ArgInfo.getIndirectByVal() &&
4187
30
                    Addr.getType()->getAddressSpace() != IRFuncTy->
4188
30
                      getParamType(FirstIRArg)->getPointerAddressSpace())) {
4189
13
            NeedCopy = true;
4190
13
          }
4191
280
        }
4192
1.60k
4193
1.60k
        if (NeedCopy) {
4194
42
          // Create an aligned temporary, and copy to it.
4195
42
          Address AI = CreateMemTempWithoutCast(
4196
42
              I->Ty, ArgInfo.getIndirectAlign(), "byval-temp");
4197
42
          IRCallArgs[FirstIRArg] = AI.getPointer();
4198
42
4199
42
          // Emit lifetime markers for the temporary alloca.
4200
42
          uint64_t ByvalTempElementSize =
4201
42
              CGM.getDataLayout().getTypeAllocSize(AI.getElementType());
4202
42
          llvm::Value *LifetimeSize =
4203
42
              EmitLifetimeStart(ByvalTempElementSize, AI.getPointer());
4204
42
4205
42
          // Add cleanup code to emit the end lifetime marker after the call.
4206
42
          if (LifetimeSize) // In case we disabled lifetime markers.
4207
2
            CallLifetimeEndAfterCall.emplace_back(AI, LifetimeSize);
4208
42
4209
42
          // Generate the copy.
4210
42
          I->copyInto(*this, AI);
4211
1.55k
        } else {
4212
1.55k
          // Skip the extra memcpy call.
4213
1.55k
          auto *T = V->getType()->getPointerElementType()->getPointerTo(
4214
1.55k
              CGM.getDataLayout().getAllocaAddrSpace());
4215
1.55k
          IRCallArgs[FirstIRArg] = getTargetHooks().performAddrSpaceCast(
4216
1.55k
              *this, V, LangAS::Default, CGM.getASTAllocaAddressSpace(), T,
4217
1.55k
              true);
4218
1.55k
        }
4219
1.60k
      }
4220
1.79k
      break;
4221
0
    }
4222
0
4223
7.70k
    case ABIArgInfo::Ignore:
4224
7.70k
      assert(NumIRArgs == 0);
4225
7.70k
      break;
4226
0
4227
509k
    case ABIArgInfo::Extend:
4228
509k
    case ABIArgInfo::Direct: {
4229
509k
      if (!isa<llvm::StructType>(ArgInfo.getCoerceToType()) &&
4230
509k
          
ArgInfo.getCoerceToType() == ConvertType(info_it->type)508k
&&
4231
509k
          
ArgInfo.getDirectOffset() == 0505k
) {
4232
505k
        assert(NumIRArgs == 1);
4233
505k
        llvm::Value *V;
4234
505k
        if (!I->isAggregate())
4235
505k
          V = I->getKnownRValue().getScalarVal();
4236
0
        else
4237
0
          V = Builder.CreateLoad(
4238
0
              I->hasLValue() ? I->getKnownLValue().getAddress(*this)
4239
0
                             : I->getKnownRValue().getAggregateAddress());
4240
505k
4241
505k
        // Implement swifterror by copying into a new swifterror argument.
4242
505k
        // We'll write back in the normal path out of the call.
4243
505k
        if (CallInfo.getExtParameterInfo(ArgNo).getABI()
4244
505k
              == ParameterABI::SwiftErrorResult) {
4245
8
          assert(!swiftErrorTemp.isValid() && "multiple swifterror args");
4246
8
4247
8
          QualType pointeeTy = I->Ty->getPointeeType();
4248
8
          swiftErrorArg =
4249
8
            Address(V, getContext().getTypeAlignInChars(pointeeTy));
4250
8
4251
8
          swiftErrorTemp =
4252
8
            CreateMemTemp(pointeeTy, getPointerAlign(), "swifterror.temp");
4253
8
          V = swiftErrorTemp.getPointer();
4254
8
          cast<llvm::AllocaInst>(V)->setSwiftError(true);
4255
8
4256
8
          llvm::Value *errorValue = Builder.CreateLoad(swiftErrorArg);
4257
8
          Builder.CreateStore(errorValue, swiftErrorTemp);
4258
8
        }
4259
505k
4260
505k
        // We might have to widen integers, but we should never truncate.
4261
505k
        if (ArgInfo.getCoerceToType() != V->getType() &&
4262
505k
            
V->getType()->isIntegerTy()668
)
4263
1
          V = Builder.CreateZExt(V, ArgInfo.getCoerceToType());
4264
505k
4265
505k
        // If the argument doesn't match, perform a bitcast to coerce it.  This
4266
505k
        // can happen due to trivial type mismatches.
4267
505k
        if (FirstIRArg < IRFuncTy->getNumParams() &&
4268
505k
            
V->getType() != IRFuncTy->getParamType(FirstIRArg)455k
)
4269
667
          V = Builder.CreateBitCast(V, IRFuncTy->getParamType(FirstIRArg));
4270
505k
4271
505k
        IRCallArgs[FirstIRArg] = V;
4272
505k
        break;
4273
505k
      }
4274
4.30k
4275
4.30k
      // FIXME: Avoid the conversion through memory if possible.
4276
4.30k
      Address Src = Address::invalid();
4277
4.30k
      if (!I->isAggregate()) {
4278
848
        Src = CreateMemTemp(I->Ty, "coerce");
4279
848
        I->copyInto(*this, Src);
4280
3.45k
      } else {
4281
3.45k
        Src = I->hasLValue() ? 
I->getKnownLValue().getAddress(*this)359
4282
3.45k
                             : 
I->getKnownRValue().getAggregateAddress()3.09k
;
4283
3.45k
      }
4284
4.30k
4285
4.30k
      // If the value is offset in memory, apply the offset now.
4286
4.30k
      Src = emitAddressAtOffset(*this, Src, ArgInfo);
4287
4.30k
4288
4.30k
      // Fast-isel and the optimizer generally like scalar values better than
4289
4.30k
      // FCAs, so we flatten them if this is safe to do for this argument.
4290
4.30k
      llvm::StructType *STy =
4291
4.30k
            dyn_cast<llvm::StructType>(ArgInfo.getCoerceToType());
4292
4.30k
      if (STy && 
ArgInfo.isDirect()1.12k
&&
ArgInfo.getCanBeFlattened()1.12k
) {
4293
1.10k
        llvm::Type *SrcTy = Src.getType()->getElementType();
4294
1.10k
        uint64_t SrcSize = CGM.getDataLayout().getTypeAllocSize(SrcTy);
4295
1.10k
        uint64_t DstSize = CGM.getDataLayout().getTypeAllocSize(STy);
4296
1.10k
4297
1.10k
        // If the source type is smaller than the destination type of the
4298
1.10k
        // coerce-to logic, copy the source value into a temp alloca the size
4299
1.10k
        // of the destination type to allow loading all of it. The bits past
4300
1.10k
        // the source value are left undef.
4301
1.10k
        if (SrcSize < DstSize) {
4302
116
          Address TempAlloca
4303
116
            = CreateTempAlloca(STy, Src.getAlignment(),
4304
116
                               Src.getName() + ".coerce");
4305
116
          Builder.CreateMemCpy(TempAlloca, Src, SrcSize);
4306
116
          Src = TempAlloca;
4307
985
        } else {
4308
985
          Src = Builder.CreateBitCast(Src,
4309
985
                                      STy->getPointerTo(Src.getAddressSpace()));
4310
985
        }
4311
1.10k
4312
1.10k
        assert(NumIRArgs == STy->getNumElements());
4313
3.31k
        for (unsigned i = 0, e = STy->getNumElements(); i != e; 
++i2.21k
) {
4314
2.21k
          Address EltPtr = Builder.CreateStructGEP(Src, i);
4315
2.21k
          llvm::Value *LI = Builder.CreateLoad(EltPtr);
4316
2.21k
          IRCallArgs[FirstIRArg + i] = LI;
4317
2.21k
        }
4318
3.20k
      } else {
4319
3.20k
        // In the simple case, just pass the coerced loaded value.
4320
3.20k
        assert(NumIRArgs == 1);
4321
3.20k
        IRCallArgs[FirstIRArg] =
4322
3.20k
          CreateCoercedLoad(Src, ArgInfo.getCoerceToType(), *this);
4323
3.20k
      }
4324
4.30k
4325
4.30k
      break;
4326
4.30k
    }
4327
4.30k
4328
4.30k
    case ABIArgInfo::CoerceAndExpand: {
4329
442
      auto coercionType = ArgInfo.getCoerceAndExpandType();
4330
442
      auto layout = CGM.getDataLayout().getStructLayout(coercionType);
4331
442
4332
442
      llvm::Value *tempSize = nullptr;
4333
442
      Address addr = Address::invalid();
4334
442
      Address AllocaAddr = Address::invalid();
4335
442
      if (I->isAggregate()) {
4336
414
        addr = I->hasLValue() ? 
I->getKnownLValue().getAddress(*this)0
4337
414
                              : I->getKnownRValue().getAggregateAddress();
4338
414
4339
414
      } else {
4340
28
        RValue RV = I->getKnownRValue();
4341
28
        assert(RV.isScalar()); // complex should always just be direct
4342
28
4343
28
        llvm::Type *scalarType = RV.getScalarVal()->getType();
4344
28
        auto scalarSize = CGM.getDataLayout().getTypeAllocSize(scalarType);
4345
28
        auto scalarAlign = CGM.getDataLayout().getPrefTypeAlignment(scalarType);
4346
28
4347
28
        // Materialize to a temporary.
4348
28
        addr = CreateTempAlloca(
4349
28
            RV.getScalarVal()->getType(),
4350
28
            CharUnits::fromQuantity(std::max(
4351
28
                (unsigned)layout->getAlignment().value(), scalarAlign)),
4352
28
            "tmp",
4353
28
            /*ArraySize=*/nullptr, &AllocaAddr);
4354
28
        tempSize = EmitLifetimeStart(scalarSize, AllocaAddr.getPointer());
4355
28
4356
28
        Builder.CreateStore(RV.getScalarVal(), addr);
4357
28
      }
4358
442
4359
442
      addr = Builder.CreateElementBitCast(addr, coercionType);
4360
442
4361
442
      unsigned IRArgPos = FirstIRArg;
4362
1.41k
      for (unsigned i = 0, e = coercionType->getNumElements(); i != e; 
++i973
) {
4363
973
        llvm::Type *eltType = coercionType->getElementType(i);
4364
973
        if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType)) 
continue7
;
4365
966
        Address eltAddr = Builder.CreateStructGEP(addr, i);
4366
966
        llvm::Value *elt = Builder.CreateLoad(eltAddr);
4367
966
        IRCallArgs[IRArgPos++] = elt;
4368
966
      }
4369
442
      assert(IRArgPos == FirstIRArg + NumIRArgs);
4370
442
4371
442
      if (tempSize) {
4372
0
        EmitLifetimeEnd(tempSize, AllocaAddr.getPointer());
4373
0
      }
4374
442
4375
442
      break;
4376
4.30k
    }
4377
4.30k
4378
4.30k
    case ABIArgInfo::Expand:
4379
27
      unsigned IRArgPos = FirstIRArg;
4380
27
      ExpandTypeToArgs(I->Ty, *I, IRFuncTy, IRCallArgs, IRArgPos);
4381
27
      assert(IRArgPos == FirstIRArg + NumIRArgs);
4382
27
      break;
4383
519k
    }
4384
519k
  }
4385
305k
4386
305k
  const CGCallee &ConcreteCallee = Callee.prepareConcreteCallee(*this);
4387
305k
  llvm::Value *CalleePtr = ConcreteCallee.getFunctionPointer();
4388
305k
4389
305k
  // If we're using inalloca, set up that argument.
4390
305k
  if (ArgMemory.isValid()) {
4391
69
    llvm::Value *Arg = ArgMemory.getPointer();
4392
69
    if (CallInfo.isVariadic()) {
4393
10
      // When passing non-POD arguments by value to variadic functions, we will
4394
10
      // end up with a variadic prototype and an inalloca call site.  In such
4395
10
      // cases, we can't do any parameter mismatch checks.  Give up and bitcast
4396
10
      // the callee.
4397
10
      unsigned CalleeAS = CalleePtr->getType()->getPointerAddressSpace();
4398
10
      CalleePtr =
4399
10
          Builder.CreateBitCast(CalleePtr, IRFuncTy->getPointerTo(CalleeAS));
4400
59
    } else {
4401
59
      llvm::Type *LastParamTy =
4402
59
          IRFuncTy->getParamType(IRFuncTy->getNumParams() - 1);
4403
59
      if (Arg->getType() != LastParamTy) {
4404
0
#ifndef NDEBUG
4405
0
        // Assert that these structs have equivalent element types.
4406
0
        llvm::StructType *FullTy = CallInfo.getArgStruct();
4407
0
        llvm::StructType *DeclaredTy = cast<llvm::StructType>(
4408
0
            cast<llvm::PointerType>(LastParamTy)->getElementType());
4409
0
        assert(DeclaredTy->getNumElements() == FullTy->getNumElements());
4410
0
        for (llvm::StructType::element_iterator DI = DeclaredTy->element_begin(),
4411
0
                                                DE = DeclaredTy->element_end(),
4412
0
                                                FI = FullTy->element_begin();
4413
0
             DI != DE; ++DI, ++FI)
4414
0
          assert(*DI == *FI);
4415
0
#endif
4416
0
        Arg = Builder.CreateBitCast(Arg, LastParamTy);
4417
0
      }
4418
59
    }
4419
69
    assert(IRFunctionArgs.hasInallocaArg());
4420
69
    IRCallArgs[IRFunctionArgs.getInallocaArgNo()] = Arg;
4421
69
  }
4422
305k
4423
305k
  // 2. Prepare the function pointer.
4424
305k
4425
305k
  // If the callee is a bitcast of a non-variadic function to have a
4426
305k
  // variadic function pointer type, check to see if we can remove the
4427
305k
  // bitcast.  This comes up with unprototyped functions.
4428
305k
  //
4429
305k
  // This makes the IR nicer, but more importantly it ensures that we
4430
305k
  // can inline the function at -O0 if it is marked always_inline.
4431
305k
  auto simplifyVariadicCallee = [](llvm::FunctionType *CalleeFT,
4432
305k
                                   llvm::Value *Ptr) -> llvm::Function * {
4433
305k
    if (!CalleeFT->isVarArg())
4434
261k
      return nullptr;
4435
43.6k
4436
43.6k
    // Get underlying value if it's a bitcast
4437
43.6k
    if (llvm::ConstantExpr *CE = dyn_cast<llvm::ConstantExpr>(Ptr)) {
4438
1.29k
      if (CE->getOpcode() == llvm::Instruction::BitCast)
4439
1.29k
        Ptr = CE->getOperand(0);
4440
1.29k
    }
4441
43.6k
4442
43.6k
    llvm::Function *OrigFn = dyn_cast<llvm::Function>(Ptr);
4443
43.6k
    if (!OrigFn)
4444
52
      return nullptr;
4445
43.5k
4446
43.5k
    llvm::FunctionType *OrigFT = OrigFn->getFunctionType();
4447
43.5k
4448
43.5k
    // If the original type is variadic, or if any of the component types
4449
43.5k
    // disagree, we cannot remove the cast.
4450
43.5k
    if (OrigFT->isVarArg() ||
4451
43.5k
        
OrigFT->getNumParams() != CalleeFT->getNumParams()310
||
4452
43.5k
        
OrigFT->getReturnType() != CalleeFT->getReturnType()309
)
4453
43.2k
      return nullptr;
4454
309
4455
312
    
for (unsigned i = 0, e = OrigFT->getNumParams(); 309
i != e;
++i3
)
4456
3
      if (OrigFT->getParamType(i) != CalleeFT->getParamType(i))
4457
0
        return nullptr;
4458
309
4459
309
    return OrigFn;
4460
309
  };
4461
305k
4462
305k
  if (llvm::Function *OrigFn = simplifyVariadicCallee(IRFuncTy, CalleePtr)) {
4463
309
    CalleePtr = OrigFn;
4464
309
    IRFuncTy = OrigFn->getFunctionType();
4465
309
  }
4466
305k
4467
305k
  // 3. Perform the actual call.
4468
305k
4469
305k
  // Deactivate any cleanups that we're supposed to do immediately before
4470
305k
  // the call.
4471
305k
  if (!CallArgs.getCleanupsToDeactivate().empty())
4472
135
    deactivateArgCleanupsBeforeCall(*this, CallArgs);
4473
305k
4474
305k
  // Assert that the arguments we computed match up.  The IR verifier
4475
305k
  // will catch this, but this is a common enough source of problems
4476
305k
  // during IRGen changes that it's way better for debugging to catch
4477
305k
  // it ourselves here.
4478
305k
#ifndef NDEBUG
4479
305k
  assert(IRCallArgs.size() == IRFuncTy->getNumParams() || IRFuncTy->isVarArg());
4480
820k
  for (unsigned i = 0; i < IRCallArgs.size(); 
++i514k
) {
4481
514k
    // Inalloca argument can have different type.
4482
514k
    if (IRFunctionArgs.hasInallocaArg() &&
4483
514k
        
i == IRFunctionArgs.getInallocaArgNo()98
)
4484
69
      continue;
4485
514k
    if (i < IRFuncTy->getNumParams())
4486
514k
      assert(IRCallArgs[i]->getType() == IRFuncTy->getParamType(i));
4487
514k
  }
4488
305k
#endif
4489
305k
4490
305k
  // Update the largest vector width if any arguments have vector types.
4491
820k
  for (unsigned i = 0; i < IRCallArgs.size(); 
++i514k
) {
4492
514k
    if (auto *VT = dyn_cast<llvm::VectorType>(IRCallArgs[i]->getType()))
4493
49.8k
      LargestVectorWidth = std::max((uint64_t)LargestVectorWidth,
4494
49.8k
                                   VT->getPrimitiveSizeInBits().getFixedSize());
4495
514k
  }
4496
305k
4497
305k
  // Compute the calling convention and attributes.
4498
305k
  unsigned CallingConv;
4499
305k
  llvm::AttributeList Attrs;
4500
305k
  CGM.ConstructAttributeList(CalleePtr->getName(), CallInfo,
4501
305k
                             Callee.getAbstractInfo(), Attrs, CallingConv,
4502
305k
                             /*AttrOnCallSite=*/true);
4503
305k
4504
305k
  if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(CurFuncDecl))
4505
297k
    if (FD->usesFPIntrin())
4506
204
      // All calls within a strictfp function are marked strictfp
4507
204
      Attrs =
4508
204
        Attrs.addAttribute(getLLVMContext(), llvm::AttributeList::FunctionIndex,
4509
204
                           llvm::Attribute::StrictFP);
4510
305k
4511
305k
  // Apply some call-site-specific attributes.
4512
305k
  // TODO: work this into building the attribute set.
4513
305k
4514
305k
  // Apply always_inline to all calls within flatten functions.
4515
305k
  // FIXME: should this really take priority over __try, below?
4516
305k
  if (CurCodeDecl && 
CurCodeDecl->hasAttr<FlattenAttr>()304k
&&
4517
305k
      
!(3
TargetDecl3
&&
TargetDecl->hasAttr<NoInlineAttr>()3
)) {
4518
2
    Attrs =
4519
2
        Attrs.addAttribute(getLLVMContext(), llvm::AttributeList::FunctionIndex,
4520
2
                           llvm::Attribute::AlwaysInline);
4521
2
  }
4522
305k
4523
305k
  // Disable inlining inside SEH __try blocks.
4524
305k
  if (isSEHTryScope()) {
4525
101
    Attrs =
4526
101
        Attrs.addAttribute(getLLVMContext(), llvm::AttributeList::FunctionIndex,
4527
101
                           llvm::Attribute::NoInline);
4528
101
  }
4529
305k
4530
305k
  // Decide whether to use a call or an invoke.
4531
305k
  bool CannotThrow;
4532
305k
  if (currentFunctionUsesSEHTry()) {
4533
244
    // SEH cares about asynchronous exceptions, so everything can "throw."
4534
244
    CannotThrow = false;
4535
305k
  } else if (isCleanupPadScope() &&
4536
305k
             
EHPersonality::get(*this).isMSVCXXPersonality()85
) {
4537
64
    // The MSVC++ personality will implicitly terminate the program if an
4538
64
    // exception is thrown during a cleanup outside of a try/catch.
4539
64
    // We don't need to model anything in IR to get this behavior.
4540
64
    CannotThrow = true;
4541
305k
  } else {
4542
305k
    // Otherwise, nounwind call sites will never throw.
4543
305k
    CannotThrow = Attrs.hasAttribute(llvm::AttributeList::FunctionIndex,
4544
305k
                                     llvm::Attribute::NoUnwind);
4545
305k
  }
4546
305k
4547
305k
  // If we made a temporary, be sure to clean up after ourselves. Note that we
4548
305k
  // can't depend on being inside of an ExprWithCleanups, so we need to manually
4549
305k
  // pop this cleanup later on. Being eager about this is OK, since this
4550
305k
  // temporary is 'invisible' outside of the callee.
4551
305k
  if (UnusedReturnSizePtr)
4552
25
    pushFullExprCleanup<CallLifetimeEnd>(NormalEHLifetimeMarker, SRetAlloca,
4553
25
                                         UnusedReturnSizePtr);
4554
305k
4555
305k
  llvm::BasicBlock *InvokeDest = CannotThrow ? 
nullptr105k
:
getInvokeDest()200k
;
4556
305k
4557
305k
  SmallVector<llvm::OperandBundleDef, 1> BundleList =
4558
305k
      getBundlesForFunclet(CalleePtr);
4559
305k
4560
305k
  if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(CurFuncDecl))
4561
297k
    if (FD->usesFPIntrin())
4562
204
      // All calls within a strictfp function are marked strictfp
4563
204
      Attrs =
4564
204
        Attrs.addAttribute(getLLVMContext(), llvm::AttributeList::FunctionIndex,
4565
204
                           llvm::Attribute::StrictFP);
4566
305k
4567
305k
  AssumeAlignedAttrEmitter AssumeAlignedAttrEmitter(*this, TargetDecl);
4568
305k
  Attrs = AssumeAlignedAttrEmitter.TryEmitAsCallSiteAttribute(Attrs);
4569
305k
4570
305k
  AllocAlignAttrEmitter AllocAlignAttrEmitter(*this, TargetDecl, CallArgs);
4571
305k
  Attrs = AllocAlignAttrEmitter.TryEmitAsCallSiteAttribute(Attrs);
4572
305k
4573
305k
  // Emit the actual call/invoke instruction.
4574
305k
  llvm::CallBase *CI;
4575
305k
  if (!InvokeDest) {
4576
277k
    CI = Builder.CreateCall(IRFuncTy, CalleePtr, IRCallArgs, BundleList);
4577
277k
  } else {
4578
28.0k
    llvm::BasicBlock *Cont = createBasicBlock("invoke.cont");
4579
28.0k
    CI = Builder.CreateInvoke(IRFuncTy, CalleePtr, Cont, InvokeDest, IRCallArgs,
4580
28.0k
                              BundleList);
4581
28.0k
    EmitBlock(Cont);
4582
28.0k
  }
4583
305k
  if (callOrInvoke)
4584
167k
    *callOrInvoke = CI;
4585
305k
4586
305k
  // If this is within a function that has the guard(nocf) attribute and is an
4587
305k
  // indirect call, add the "guard_nocf" attribute to this call to indicate that
4588
305k
  // Control Flow Guard checks should not be added, even if the call is inlined.
4589
305k
  if (const auto *FD = dyn_cast_or_null<FunctionDecl>(CurFuncDecl)) {
4590
297k
    if (const auto *A = FD->getAttr<CFGuardAttr>()) {
4591
9
      if (A->getGuard() == CFGuardAttr::GuardArg::nocf && !CI->getCalledFunction())
4592
7
        Attrs = Attrs.addAttribute(
4593
7
            getLLVMContext(), llvm::AttributeList::FunctionIndex, "guard_nocf");
4594
9
    }
4595
297k
  }
4596
305k
4597
305k
  // Apply the attributes and calling convention.
4598
305k
  CI->setAttributes(Attrs);
4599
305k
  CI->setCallingConv(static_cast<llvm::CallingConv::ID>(CallingConv));
4600
305k
4601
305k
  // Apply various metadata.
4602
305k
4603
305k
  if (!CI->getType()->isVoidTy())
4604
187k
    CI->setName("call");
4605
305k
4606
305k
  // Update largest vector width from the return type.
4607
305k
  if (auto *VT = dyn_cast<llvm::VectorType>(CI->getType()))
4608
27.5k
    LargestVectorWidth = std::max((uint64_t)LargestVectorWidth,
4609
27.5k
                                  VT->getPrimitiveSizeInBits().getFixedSize());
4610
305k
4611
305k
  // Insert instrumentation or attach profile metadata at indirect call sites.
4612
305k
  // For more details, see the comment before the definition of
4613
305k
  // IPVK_IndirectCallTarget in InstrProfData.inc.
4614
305k
  if (!CI->getCalledFunction())
4615
18.3k
    PGO.valueProfile(Builder, llvm::IPVK_IndirectCallTarget,
4616
18.3k
                     CI, CalleePtr);
4617
305k
4618
305k
  // In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
4619
305k
  // optimizer it can aggressively ignore unwind edges.
4620
305k
  if (CGM.getLangOpts().ObjCAutoRefCount)
4621
1.60k
    AddObjCARCExceptionMetadata(CI);
4622
305k
4623
305k
  // Suppress tail calls if requested.
4624
305k
  if (llvm::CallInst *Call = dyn_cast<llvm::CallInst>(CI)) {
4625
277k
    if (TargetDecl && 
TargetDecl->hasAttr<NotTailCalledAttr>()261k
)
4626
293
      Call->setTailCallKind(llvm::CallInst::TCK_NoTail);
4627
277k
  }
4628
305k
4629
305k
  // Add metadata for calls to MSAllocator functions
4630
305k
  if (getDebugInfo() && 
TargetDecl197k
&&
4631
305k
      
TargetDecl->hasAttr<MSAllocatorAttr>()184k
)
4632
3
    getDebugInfo()->addHeapAllocSiteMetadata(CI, RetTy, Loc);
4633
305k
4634
305k
  // 4. Finish the call.
4635
305k
4636
305k
  // If the call doesn't return, finish the basic block and clear the
4637
305k
  // insertion point; this allows the rest of IRGen to discard
4638
305k
  // unreachable code.
4639
305k
  if (CI->doesNotReturn()) {
4640
1.74k
    if (UnusedReturnSizePtr)
4641
0
      PopCleanupBlock();
4642
1.74k
4643
1.74k
    // Strip away the noreturn attribute to better diagnose unreachable UB.
4644
1.74k
    if (SanOpts.has(SanitizerKind::Unreachable)) {
4645
8
      // Also remove from function since CallBase::hasFnAttr additionally checks
4646
8
      // attributes of the called function.
4647
8
      if (auto *F = CI->getCalledFunction())
4648
7
        F->removeFnAttr(llvm::Attribute::NoReturn);
4649
8
      CI->removeAttribute(llvm::AttributeList::FunctionIndex,
4650
8
                          llvm::Attribute::NoReturn);
4651
8
4652
8
      // Avoid incompatibility with ASan which relies on the `noreturn`
4653
8
      // attribute to insert handler calls.
4654
8
      if (SanOpts.hasOneOf(SanitizerKind::Address |
4655
8
                           SanitizerKind::KernelAddress)) {
4656
2
        SanitizerScope SanScope(this);
4657
2
        llvm::IRBuilder<>::InsertPointGuard IPGuard(Builder);
4658
2
        Builder.SetInsertPoint(CI);
4659
2
        auto *FnType = llvm::FunctionType::get(CGM.VoidTy, /*isVarArg=*/false);
4660
2
        llvm::FunctionCallee Fn =
4661
2
            CGM.CreateRuntimeFunction(FnType, "__asan_handle_no_return");
4662
2
        EmitNounwindRuntimeCall(Fn);
4663
2
      }
4664
8
    }
4665
1.74k
4666
1.74k
    EmitUnreachable(Loc);
4667
1.74k
    Builder.ClearInsertionPoint();
4668
1.74k
4669
1.74k
    // FIXME: For now, emit a dummy basic block because expr emitters in
4670
1.74k
    // generally are not ready to handle emitting expressions at unreachable
4671
1.74k
    // points.
4672
1.74k
    EnsureInsertPoint();
4673
1.74k
4674
1.74k
    // Return a reasonable RValue.
4675
1.74k
    return GetUndefRValue(RetTy);
4676
1.74k
  }
4677
303k
4678
303k
  // Perform the swifterror writeback.
4679
303k
  if (swiftErrorTemp.isValid()) {
4680
8
    llvm::Value *errorResult = Builder.CreateLoad(swiftErrorTemp);
4681
8
    Builder.CreateStore(errorResult, swiftErrorArg);
4682
8
  }
4683
303k
4684
303k
  // Emit any call-associated writebacks immediately.  Arguably this
4685
303k
  // should happen after any return-value munging.
4686
303k
  if (CallArgs.hasWritebacks())
4687
26
    emitWritebacks(*this, CallArgs);
4688
303k
4689
303k
  // The stack cleanup for inalloca arguments has to run out of the normal
4690
303k
  // lexical order, so deactivate it and run it manually here.
4691
303k
  CallArgs.freeArgumentMemory(*this);
4692
303k
4693
303k
  // Extract the return value.
4694
303k
  RValue Ret = [&] {
4695
303k
    switch (RetAI.getKind()) {
4696
442
    case ABIArgInfo::CoerceAndExpand: {
4697
442
      auto coercionType = RetAI.getCoerceAndExpandType();
4698
442
4699
442
      Address addr = SRetPtr;
4700
442
      addr = Builder.CreateElementBitCast(addr, coercionType);
4701
442
4702
442
      assert(CI->getType() == RetAI.getUnpaddedCoerceAndExpandType());
4703
442
      bool requiresExtract = isa<llvm::StructType>(CI->getType());
4704
442
4705
442
      unsigned unpaddedIndex = 0;
4706
1.41k
      for (unsigned i = 0, e = coercionType->getNumElements(); i != e; 
++i973
) {
4707
973
        llvm::Type *eltType = coercionType->getElementType(i);
4708
973
        if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType)) 
continue7
;
4709
966
        Address eltAddr = Builder.CreateStructGEP(addr, i);
4710
966
        llvm::Value *elt = CI;
4711
966
        if (requiresExtract)
4712
810
          elt = Builder.CreateExtractValue(elt, unpaddedIndex++);
4713
966
        else
4714
966
          assert(unpaddedIndex == 0);
4715
966
        Builder.CreateStore(elt, eltAddr);
4716
966
      }
4717
442
      // FALLTHROUGH
4718
442
      LLVM_FALLTHROUGH;
4719
442
    }
4720
442
4721
1.78k
    case ABIArgInfo::InAlloca:
4722
1.78k
    case ABIArgInfo::Indirect: {
4723
1.78k
      RValue ret = convertTempToRValue(SRetPtr, RetTy, SourceLocation());
4724
1.78k
      if (UnusedReturnSizePtr)
4725
25
        PopCleanupBlock();
4726
1.78k
      return ret;
4727
1.78k
    }
4728
1.78k
4729
115k
    case ABIArgInfo::Ignore:
4730
115k
      // If we are ignoring an argument that had a result, make sure to
4731
115k
      // construct the appropriate return value for our caller.
4732
115k
      return GetUndefRValue(RetTy);
4733
1.78k
4734
186k
    case ABIArgInfo::Extend:
4735
186k
    case ABIArgInfo::Direct: {
4736
186k
      llvm::Type *RetIRTy = ConvertType(RetTy);
4737
186k
      if (RetAI.getCoerceToType() == RetIRTy && 
RetAI.getDirectOffset() == 0182k
) {
4738
182k
        switch (getEvaluationKind(RetTy)) {
4739
224
        case TEK_Complex: {
4740
224
          llvm::Value *Real = Builder.CreateExtractValue(CI, 0);
4741
224
          llvm::Value *Imag = Builder.CreateExtractValue(CI, 1);
4742
224
          return RValue::getComplex(std::make_pair(Real, Imag));
4743
0
        }
4744
85
        case TEK_Aggregate: {
4745
85
          Address DestPtr = ReturnValue.getValue();
4746
85
          bool DestIsVolatile = ReturnValue.isVolatile();
4747
85
4748
85
          if (!DestPtr.isValid()) {
4749
3
            DestPtr = CreateMemTemp(RetTy, "agg.tmp");
4750
3
            DestIsVolatile = false;
4751
3
          }
4752
85
          BuildAggStore(*this, CI, DestPtr, DestIsVolatile);
4753
85
          return RValue::getAggregate(DestPtr);
4754
0
        }
4755
181k
        case TEK_Scalar: {
4756
181k
          // If the argument doesn't match, perform a bitcast to coerce it.  This
4757
181k
          // can happen due to trivial type mismatches.
4758
181k
          llvm::Value *V = CI;
4759
181k
          if (V->getType() != RetIRTy)
4760
0
            V = Builder.CreateBitCast(V, RetIRTy);
4761
181k
          return RValue::get(V);
4762
0
        }
4763
0
        }
4764
0
        llvm_unreachable("bad evaluation kind");
4765
0
      }
4766
4.79k
4767
4.79k
      Address DestPtr = ReturnValue.getValue();
4768
4.79k
      bool DestIsVolatile = ReturnValue.isVolatile();
4769
4.79k
4770
4.79k
      if (!DestPtr.isValid()) {
4771
1.03k
        DestPtr = CreateMemTemp(RetTy, "coerce");
4772
1.03k
        DestIsVolatile = false;
4773
1.03k
      }
4774
4.79k
4775
4.79k
      // If the value is offset in memory, apply the offset now.
4776
4.79k
      Address StorePtr = emitAddressAtOffset(*this, DestPtr, RetAI);
4777
4.79k
      CreateCoercedStore(CI, StorePtr, DestIsVolatile, *this);
4778
4.79k
4779
4.79k
      return convertTempToRValue(DestPtr, RetTy, SourceLocation());
4780
4.79k
    }
4781
4.79k
4782
4.79k
    case ABIArgInfo::Expand:
4783
0
      llvm_unreachable("Invalid ABI kind for return argument");
4784
0
    }
4785
0
4786
0
    llvm_unreachable("Unhandled ABIArgInfo::Kind");
4787
0
  } ();
4788
303k
4789
303k
  // Emit the assume_aligned check on the return value.
4790
303k
  if (Ret.isScalar() && 
TargetDecl296k
) {
4791
281k
    AssumeAlignedAttrEmitter.EmitAsAnAssumption(Loc, RetTy, Ret);
4792
281k
    AllocAlignAttrEmitter.EmitAsAnAssumption(Loc, RetTy, Ret);
4793
281k
  }
4794
303k
4795
303k
  // Explicitly call CallLifetimeEnd::Emit just to re-use the code even though
4796
303k
  // we can't use the full cleanup mechanism.
4797
303k
  for (CallLifetimeEnd &LifetimeEnd : CallLifetimeEndAfterCall)
4798
2
    LifetimeEnd.Emit(*this, /*Flags=*/{});
4799
303k
4800
303k
  return Ret;
4801
303k
}
4802
4803
305k
CGCallee CGCallee::prepareConcreteCallee(CodeGenFunction &CGF) const {
4804
305k
  if (isVirtual()) {
4805
946
    const CallExpr *CE = getVirtualCallExpr();
4806
946
    return CGF.CGM.getCXXABI().getVirtualFunctionPointer(
4807
946
        CGF, getVirtualMethodDecl(), getThisAddress(), getVirtualFunctionType(),
4808
946
        CE ? 
CE->getBeginLoc()879
:
SourceLocation()67
);
4809
946
  }
4810
304k
4811
304k
  return *this;
4812
304k
}
4813
4814
/* VarArg handling */
4815
4816
699
Address CodeGenFunction::EmitVAArg(VAArgExpr *VE, Address &VAListAddr) {
4817
699
  VAListAddr = VE->isMicrosoftABI()
4818
699
                 ? 
EmitMSVAListRef(VE->getSubExpr())9
4819
699
                 : 
EmitVAListRef(VE->getSubExpr())690
;
4820
699
  QualType Ty = VE->getType();
4821
699
  if (VE->isMicrosoftABI())
4822
9
    return CGM.getTypes().getABIInfo().EmitMSVAArg(*this, VAListAddr, Ty);
4823
690
  return CGM.getTypes().getABIInfo().EmitVAArg(*this, VAListAddr, Ty);
4824
690
}