Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/tools/clang/lib/CodeGen/CGCall.cpp
Line
Count
Source (jump to first uncovered line)
1
//===--- CGCall.cpp - Encapsulate calling convention details --------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// These classes wrap the information about a call or function
10
// definition used to handle ABI compliancy.
11
//
12
//===----------------------------------------------------------------------===//
13
14
#include "CGCall.h"
15
#include "ABIInfo.h"
16
#include "CGBlocks.h"
17
#include "CGCXXABI.h"
18
#include "CGCleanup.h"
19
#include "CodeGenFunction.h"
20
#include "CodeGenModule.h"
21
#include "TargetInfo.h"
22
#include "clang/AST/Decl.h"
23
#include "clang/AST/DeclCXX.h"
24
#include "clang/AST/DeclObjC.h"
25
#include "clang/Basic/CodeGenOptions.h"
26
#include "clang/Basic/TargetBuiltins.h"
27
#include "clang/Basic/TargetInfo.h"
28
#include "clang/CodeGen/CGFunctionInfo.h"
29
#include "clang/CodeGen/SwiftCallingConv.h"
30
#include "llvm/ADT/StringExtras.h"
31
#include "llvm/Transforms/Utils/Local.h"
32
#include "llvm/Analysis/ValueTracking.h"
33
#include "llvm/IR/Attributes.h"
34
#include "llvm/IR/CallingConv.h"
35
#include "llvm/IR/DataLayout.h"
36
#include "llvm/IR/InlineAsm.h"
37
#include "llvm/IR/IntrinsicInst.h"
38
#include "llvm/IR/Intrinsics.h"
39
using namespace clang;
40
using namespace CodeGen;
41
42
/***/
43
44
300k
unsigned CodeGenTypes::ClangCallConvToLLVMCallConv(CallingConv CC) {
45
300k
  switch (CC) {
46
300k
  
default: return llvm::CallingConv::C296k
;
47
300k
  
case CC_X86StdCall: return llvm::CallingConv::X86_StdCall51
;
48
300k
  
case CC_X86FastCall: return llvm::CallingConv::X86_FastCall68
;
49
300k
  
case CC_X86RegCall: return llvm::CallingConv::X86_RegCall95
;
50
300k
  
case CC_X86ThisCall: return llvm::CallingConv::X86_ThisCall2.50k
;
51
300k
  
case CC_Win64: return llvm::CallingConv::Win6414
;
52
300k
  
case CC_X86_64SysV: return llvm::CallingConv::X86_64_SysV14
;
53
300k
  
case CC_AAPCS: return llvm::CallingConv::ARM_AAPCS37
;
54
300k
  
case CC_AAPCS_VFP: return llvm::CallingConv::ARM_AAPCS_VFP11
;
55
300k
  
case CC_IntelOclBicc: return llvm::CallingConv::Intel_OCL_BI3
;
56
300k
  // TODO: Add support for __pascal to LLVM.
57
300k
  
case CC_X86Pascal: return llvm::CallingConv::C3
;
58
300k
  // TODO: Add support for __vectorcall to LLVM.
59
300k
  
case CC_X86VectorCall: return llvm::CallingConv::X86_VectorCall100
;
60
300k
  
case CC_AArch64VectorCall: return llvm::CallingConv::AArch64_VectorCall2
;
61
300k
  
case CC_SpirFunction: return llvm::CallingConv::SPIR_FUNC204
;
62
300k
  
case CC_OpenCLKernel: return CGM.getTargetCodeGenInfo().getOpenCLKernelCallingConv()294
;
63
300k
  
case CC_PreserveMost: return llvm::CallingConv::PreserveMost10
;
64
300k
  
case CC_PreserveAll: return llvm::CallingConv::PreserveAll6
;
65
300k
  
case CC_Swift: return llvm::CallingConv::Swift1.12k
;
66
300k
  }
67
300k
}
68
69
/// Derives the 'this' type for codegen purposes, i.e. ignoring method CVR
70
/// qualification. Either or both of RD and MD may be null. A null RD indicates
71
/// that there is no meaningful 'this' type, and a null MD can occur when
72
/// calling a method pointer.
73
CanQualType CodeGenTypes::DeriveThisType(const CXXRecordDecl *RD,
74
1.26M
                                         const CXXMethodDecl *MD) {
75
1.26M
  QualType RecTy;
76
1.26M
  if (RD)
77
1.26M
    RecTy = Context.getTagDeclType(RD)->getCanonicalTypeInternal();
78
635
  else
79
635
    RecTy = Context.VoidTy;
80
1.26M
81
1.26M
  if (MD)
82
1.26M
    RecTy = Context.getAddrSpaceQualType(RecTy, MD->getMethodQualifiers().getAddressSpace());
83
1.26M
  return Context.getPointerType(CanQualType::CreateUnsafe(RecTy));
84
1.26M
}
85
86
/// Returns the canonical formal type of the given C++ method.
87
1.08M
static CanQual<FunctionProtoType> GetFormalType(const CXXMethodDecl *MD) {
88
1.08M
  return MD->getType()->getCanonicalTypeUnqualified()
89
1.08M
           .getAs<FunctionProtoType>();
90
1.08M
}
91
92
/// Returns the "extra-canonicalized" return type, which discards
93
/// qualifiers on the return type.  Codegen doesn't care about them,
94
/// and it makes ABI code a little easier to be able to assume that
95
/// all parameter and return types are top-level unqualified.
96
777k
static CanQualType GetReturnType(QualType RetTy) {
97
777k
  return RetTy->getCanonicalTypeUnqualified().getUnqualifiedType();
98
777k
}
99
100
/// Arrange the argument and result information for a value of the given
101
/// unprototyped freestanding function type.
102
const CGFunctionInfo &
103
1.37k
CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionNoProtoType> FTNP) {
104
1.37k
  // When translating an unprototyped function type, always use a
105
1.37k
  // variadic type.
106
1.37k
  return arrangeLLVMFunctionInfo(FTNP->getReturnType().getUnqualifiedType(),
107
1.37k
                                 /*instanceMethod=*/false,
108
1.37k
                                 /*chainCall=*/false, None,
109
1.37k
                                 FTNP->getExtInfo(), {}, RequiredArgs(0));
110
1.37k
}
111
112
static void addExtParameterInfosForCall(
113
         llvm::SmallVectorImpl<FunctionProtoType::ExtParameterInfo> &paramInfos,
114
                                        const FunctionProtoType *proto,
115
                                        unsigned prefixArgs,
116
861
                                        unsigned totalArgs) {
117
861
  assert(proto->hasExtParameterInfos());
118
861
  assert(paramInfos.size() <= prefixArgs);
119
861
  assert(proto->getNumParams() + prefixArgs <= totalArgs);
120
861
121
861
  paramInfos.reserve(totalArgs);
122
861
123
861
  // Add default infos for any prefix args that don't already have infos.
124
861
  paramInfos.resize(prefixArgs);
125
861
126
861
  // Add infos for the prototype.
127
1.70k
  for (const auto &ParamInfo : proto->getExtParameterInfos()) {
128
1.70k
    paramInfos.push_back(ParamInfo);
129
1.70k
    // pass_object_size params have no parameter info.
130
1.70k
    if (ParamInfo.hasPassObjectSize())
131
447
      paramInfos.emplace_back();
132
1.70k
  }
133
861
134
861
  assert(paramInfos.size() <= totalArgs &&
135
861
         "Did we forget to insert pass_object_size args?");
136
861
  // Add default infos for the variadic and/or suffix arguments.
137
861
  paramInfos.resize(totalArgs);
138
861
}
139
140
/// Adds the formal parameters in FPT to the given prefix. If any parameter in
141
/// FPT has pass_object_size attrs, then we'll add parameters for those, too.
142
static void appendParameterTypes(const CodeGenTypes &CGT,
143
                                 SmallVectorImpl<CanQualType> &prefix,
144
              SmallVectorImpl<FunctionProtoType::ExtParameterInfo> &paramInfos,
145
1.60M
                                 CanQual<FunctionProtoType> FPT) {
146
1.60M
  // Fast path: don't touch param info if we don't need to.
147
1.60M
  if (!FPT->hasExtParameterInfos()) {
148
1.60M
    assert(paramInfos.empty() &&
149
1.60M
           "We have paramInfos, but the prototype doesn't?");
150
1.60M
    prefix.append(FPT->param_type_begin(), FPT->param_type_end());
151
1.60M
    return;
152
1.60M
  }
153
617
154
617
  unsigned PrefixSize = prefix.size();
155
617
  // In the vast majority of cases, we'll have precisely FPT->getNumParams()
156
617
  // parameters; the only thing that can change this is the presence of
157
617
  // pass_object_size. So, we preallocate for the common case.
158
617
  prefix.reserve(prefix.size() + FPT->getNumParams());
159
617
160
617
  auto ExtInfos = FPT->getExtParameterInfos();
161
617
  assert(ExtInfos.size() == FPT->getNumParams());
162
1.86k
  for (unsigned I = 0, E = FPT->getNumParams(); I != E; 
++I1.24k
) {
163
1.24k
    prefix.push_back(FPT->getParamType(I));
164
1.24k
    if (ExtInfos[I].hasPassObjectSize())
165
299
      prefix.push_back(CGT.getContext().getSizeType());
166
1.24k
  }
167
617
168
617
  addExtParameterInfosForCall(paramInfos, FPT.getTypePtr(), PrefixSize,
169
617
                              prefix.size());
170
617
}
171
172
/// Arrange the LLVM function layout for a value of the given function
173
/// type, on top of any implicit parameters already stored.
174
static const CGFunctionInfo &
175
arrangeLLVMFunctionInfo(CodeGenTypes &CGT, bool instanceMethod,
176
                        SmallVectorImpl<CanQualType> &prefix,
177
1.15M
                        CanQual<FunctionProtoType> FTP) {
178
1.15M
  SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos;
179
1.15M
  RequiredArgs Required = RequiredArgs::forPrototypePlus(FTP, prefix.size());
180
1.15M
  // FIXME: Kill copy.
181
1.15M
  appendParameterTypes(CGT, prefix, paramInfos, FTP);
182
1.15M
  CanQualType resultType = FTP->getReturnType().getUnqualifiedType();
183
1.15M
184
1.15M
  return CGT.arrangeLLVMFunctionInfo(resultType, instanceMethod,
185
1.15M
                                     /*chainCall=*/false, prefix,
186
1.15M
                                     FTP->getExtInfo(), paramInfos,
187
1.15M
                                     Required);
188
1.15M
}
189
190
/// Arrange the argument and result information for a value of the
191
/// given freestanding function type.
192
const CGFunctionInfo &
193
623k
CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionProtoType> FTP) {
194
623k
  SmallVector<CanQualType, 16> argTypes;
195
623k
  return ::arrangeLLVMFunctionInfo(*this, /*instanceMethod=*/false, argTypes,
196
623k
                                   FTP);
197
623k
}
198
199
4.01k
static CallingConv getCallingConventionForDecl(const Decl *D, bool IsWindows) {
200
4.01k
  // Set the appropriate calling convention for the Function.
201
4.01k
  if (D->hasAttr<StdCallAttr>())
202
2
    return CC_X86StdCall;
203
4.00k
204
4.00k
  if (D->hasAttr<FastCallAttr>())
205
0
    return CC_X86FastCall;
206
4.00k
207
4.00k
  if (D->hasAttr<RegCallAttr>())
208
0
    return CC_X86RegCall;
209
4.00k
210
4.00k
  if (D->hasAttr<ThisCallAttr>())
211
0
    return CC_X86ThisCall;
212
4.00k
213
4.00k
  if (D->hasAttr<VectorCallAttr>())
214
0
    return CC_X86VectorCall;
215
4.00k
216
4.00k
  if (D->hasAttr<PascalAttr>())
217
0
    return CC_X86Pascal;
218
4.00k
219
4.00k
  if (PcsAttr *PCS = D->getAttr<PcsAttr>())
220
0
    return (PCS->getPCS() == PcsAttr::AAPCS ? CC_AAPCS : CC_AAPCS_VFP);
221
4.00k
222
4.00k
  if (D->hasAttr<AArch64VectorPcsAttr>())
223
0
    return CC_AArch64VectorCall;
224
4.00k
225
4.00k
  if (D->hasAttr<IntelOclBiccAttr>())
226
0
    return CC_IntelOclBicc;
227
4.00k
228
4.00k
  if (D->hasAttr<MSABIAttr>())
229
2
    return IsWindows ? 
CC_C0
: CC_Win64;
230
4.00k
231
4.00k
  if (D->hasAttr<SysVABIAttr>())
232
0
    return IsWindows ? CC_X86_64SysV : CC_C;
233
4.00k
234
4.00k
  if (D->hasAttr<PreserveMostAttr>())
235
0
    return CC_PreserveMost;
236
4.00k
237
4.00k
  if (D->hasAttr<PreserveAllAttr>())
238
0
    return CC_PreserveAll;
239
4.00k
240
4.00k
  return CC_C;
241
4.00k
}
242
243
/// Arrange the argument and result information for a call to an
244
/// unknown C++ non-static member function of the given abstract type.
245
/// (A null RD means we don't have any meaningful "this" argument type,
246
///  so fall back to a generic pointer type).
247
/// The member function must be an ordinary function, i.e. not a
248
/// constructor or destructor.
249
const CGFunctionInfo &
250
CodeGenTypes::arrangeCXXMethodType(const CXXRecordDecl *RD,
251
                                   const FunctionProtoType *FTP,
252
535k
                                   const CXXMethodDecl *MD) {
253
535k
  SmallVector<CanQualType, 16> argTypes;
254
535k
255
535k
  // Add the 'this' pointer.
256
535k
  argTypes.push_back(DeriveThisType(RD, MD));
257
535k
258
535k
  return ::arrangeLLVMFunctionInfo(
259
535k
      *this, true, argTypes,
260
535k
      FTP->getCanonicalTypeUnqualified().getAs<FunctionProtoType>());
261
535k
}
262
263
/// Set calling convention for CUDA/HIP kernel.
264
static void setCUDAKernelCallingConvention(CanQualType &FTy, CodeGenModule &CGM,
265
1.05M
                                           const FunctionDecl *FD) {
266
1.05M
  if (FD->hasAttr<CUDAGlobalAttr>()) {
267
154
    const FunctionType *FT = FTy->getAs<FunctionType>();
268
154
    CGM.getTargetCodeGenInfo().setCUDAKernelCallingConvention(FT);
269
154
    FTy = FT->getCanonicalTypeUnqualified();
270
154
  }
271
1.05M
}
272
273
/// Arrange the argument and result information for a declaration or
274
/// definition of the given C++ non-static member function.  The
275
/// member function must be an ordinary function, i.e. not a
276
/// constructor or destructor.
277
const CGFunctionInfo &
278
550k
CodeGenTypes::arrangeCXXMethodDeclaration(const CXXMethodDecl *MD) {
279
550k
  assert(!isa<CXXConstructorDecl>(MD) && "wrong method for constructors!");
280
550k
  assert(!isa<CXXDestructorDecl>(MD) && "wrong method for destructors!");
281
550k
282
550k
  CanQualType FT = GetFormalType(MD).getAs<Type>();
283
550k
  setCUDAKernelCallingConvention(FT, CGM, MD);
284
550k
  auto prototype = FT.getAs<FunctionProtoType>();
285
550k
286
550k
  if (MD->isInstance()) {
287
534k
    // The abstract case is perfectly fine.
288
534k
    const CXXRecordDecl *ThisType = TheCXXABI.getThisArgumentTypeForMethod(MD);
289
534k
    return arrangeCXXMethodType(ThisType, prototype.getTypePtr(), MD);
290
534k
  }
291
15.2k
292
15.2k
  return arrangeFreeFunctionType(prototype);
293
15.2k
}
294
295
bool CodeGenTypes::inheritingCtorHasParams(
296
588
    const InheritedConstructor &Inherited, CXXCtorType Type) {
297
588
  // Parameters are unnecessary if we're constructing a base class subobject
298
588
  // and the inherited constructor lives in a virtual base.
299
588
  return Type == Ctor_Complete ||
300
588
         
!Inherited.getShadowDecl()->constructsVirtualBase()240
||
301
588
         
!Target.getCXXABI().hasConstructorVariants()74
;
302
588
}
303
304
const CGFunctionInfo &
305
444k
CodeGenTypes::arrangeCXXStructorDeclaration(GlobalDecl GD) {
306
444k
  auto *MD = cast<CXXMethodDecl>(GD.getDecl());
307
444k
308
444k
  SmallVector<CanQualType, 16> argTypes;
309
444k
  SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos;
310
444k
  argTypes.push_back(DeriveThisType(MD->getParent(), MD));
311
444k
312
444k
  bool PassParams = true;
313
444k
314
444k
  if (auto *CD = dyn_cast<CXXConstructorDecl>(MD)) {
315
250k
    // A base class inheriting constructor doesn't get forwarded arguments
316
250k
    // needed to construct a virtual base (or base class thereof).
317
250k
    if (auto Inherited = CD->getInheritedConstructor())
318
326
      PassParams = inheritingCtorHasParams(Inherited, GD.getCtorType());
319
250k
  }
320
444k
321
444k
  CanQual<FunctionProtoType> FTP = GetFormalType(MD);
322
444k
323
444k
  // Add the formal parameters.
324
444k
  if (PassParams)
325
444k
    appendParameterTypes(*this, argTypes, paramInfos, FTP);
326
444k
327
444k
  CGCXXABI::AddedStructorArgs AddedArgs =
328
444k
      TheCXXABI.buildStructorSignature(GD, argTypes);
329
444k
  if (!paramInfos.empty()) {
330
175
    // Note: prefix implies after the first param.
331
175
    if (AddedArgs.Prefix)
332
2
      paramInfos.insert(paramInfos.begin() + 1, AddedArgs.Prefix,
333
2
                        FunctionProtoType::ExtParameterInfo{});
334
175
    if (AddedArgs.Suffix)
335
20
      paramInfos.append(AddedArgs.Suffix,
336
20
                        FunctionProtoType::ExtParameterInfo{});
337
175
  }
338
444k
339
444k
  RequiredArgs required =
340
444k
      (PassParams && 
MD->isVariadic()444k
?
RequiredArgs(argTypes.size())91
341
444k
                                      : 
RequiredArgs::All444k
);
342
444k
343
444k
  FunctionType::ExtInfo extInfo = FTP->getExtInfo();
344
444k
  CanQualType resultType = TheCXXABI.HasThisReturn(GD)
345
444k
                               ? 
argTypes.front()250k
346
444k
                               : TheCXXABI.hasMostDerivedReturn(GD)
347
194k
                                     ? 
CGM.getContext().VoidPtrTy1.36k
348
194k
                                     : 
Context.VoidTy192k
;
349
444k
  return arrangeLLVMFunctionInfo(resultType, /*instanceMethod=*/true,
350
444k
                                 /*chainCall=*/false, argTypes, extInfo,
351
444k
                                 paramInfos, required);
352
444k
}
353
354
static SmallVector<CanQualType, 16>
355
227k
getArgTypesForCall(ASTContext &ctx, const CallArgList &args) {
356
227k
  SmallVector<CanQualType, 16> argTypes;
357
227k
  for (auto &arg : args)
358
397k
    argTypes.push_back(ctx.getCanonicalParamType(arg.Ty));
359
227k
  return argTypes;
360
227k
}
361
362
static SmallVector<CanQualType, 16>
363
19.7k
getArgTypesForDeclaration(ASTContext &ctx, const FunctionArgList &args) {
364
19.7k
  SmallVector<CanQualType, 16> argTypes;
365
19.7k
  for (auto &arg : args)
366
52.7k
    argTypes.push_back(ctx.getCanonicalParamType(arg->getType()));
367
19.7k
  return argTypes;
368
19.7k
}
369
370
static llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16>
371
getExtParameterInfosForCall(const FunctionProtoType *proto,
372
228k
                            unsigned prefixArgs, unsigned totalArgs) {
373
228k
  llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16> result;
374
228k
  if (proto->hasExtParameterInfos()) {
375
11
    addExtParameterInfosForCall(result, proto, prefixArgs, totalArgs);
376
11
  }
377
228k
  return result;
378
228k
}
379
380
/// Arrange a call to a C++ method, passing the given arguments.
381
///
382
/// ExtraPrefixArgs is the number of ABI-specific args passed after the `this`
383
/// parameter.
384
/// ExtraSuffixArgs is the number of ABI-specific args passed at the end of
385
/// args.
386
/// PassProtoArgs indicates whether `args` has args for the parameters in the
387
/// given CXXConstructorDecl.
388
const CGFunctionInfo &
389
CodeGenTypes::arrangeCXXConstructorCall(const CallArgList &args,
390
                                        const CXXConstructorDecl *D,
391
                                        CXXCtorType CtorKind,
392
                                        unsigned ExtraPrefixArgs,
393
                                        unsigned ExtraSuffixArgs,
394
93.7k
                                        bool PassProtoArgs) {
395
93.7k
  // FIXME: Kill copy.
396
93.7k
  SmallVector<CanQualType, 16> ArgTypes;
397
93.7k
  for (const auto &Arg : args)
398
188k
    ArgTypes.push_back(Context.getCanonicalParamType(Arg.Ty));
399
93.7k
400
93.7k
  // +1 for implicit this, which should always be args[0].
401
93.7k
  unsigned TotalPrefixArgs = 1 + ExtraPrefixArgs;
402
93.7k
403
93.7k
  CanQual<FunctionProtoType> FPT = GetFormalType(D);
404
93.7k
  RequiredArgs Required = PassProtoArgs
405
93.7k
                              ? RequiredArgs::forPrototypePlus(
406
93.7k
                                    FPT, TotalPrefixArgs + ExtraSuffixArgs)
407
93.7k
                              : 
RequiredArgs::All10
;
408
93.7k
409
93.7k
  GlobalDecl GD(D, CtorKind);
410
93.7k
  CanQualType ResultType = TheCXXABI.HasThisReturn(GD)
411
93.7k
                               ? 
ArgTypes.front()54.4k
412
93.7k
                               : TheCXXABI.hasMostDerivedReturn(GD)
413
39.3k
                                     ? 
CGM.getContext().VoidPtrTy0
414
39.3k
                                     : Context.VoidTy;
415
93.7k
416
93.7k
  FunctionType::ExtInfo Info = FPT->getExtInfo();
417
93.7k
  llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16> ParamInfos;
418
93.7k
  // If the prototype args are elided, we should only have ABI-specific args,
419
93.7k
  // which never have param info.
420
93.7k
  if (PassProtoArgs && 
FPT->hasExtParameterInfos()93.7k
) {
421
70
    // ABI-specific suffix arguments are treated the same as variadic arguments.
422
70
    addExtParameterInfosForCall(ParamInfos, FPT.getTypePtr(), TotalPrefixArgs,
423
70
                                ArgTypes.size());
424
70
  }
425
93.7k
  return arrangeLLVMFunctionInfo(ResultType, /*instanceMethod=*/true,
426
93.7k
                                 /*chainCall=*/false, ArgTypes, Info,
427
93.7k
                                 ParamInfos, Required);
428
93.7k
}
429
430
/// Arrange the argument and result information for the declaration or
431
/// definition of the given function.
432
const CGFunctionInfo &
433
727k
CodeGenTypes::arrangeFunctionDeclaration(const FunctionDecl *FD) {
434
727k
  if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(FD))
435
252k
    if (MD->isInstance())
436
219k
      return arrangeCXXMethodDeclaration(MD);
437
507k
438
507k
  CanQualType FTy = FD->getType()->getCanonicalTypeUnqualified();
439
507k
440
507k
  assert(isa<FunctionType>(FTy));
441
507k
  setCUDAKernelCallingConvention(FTy, CGM, FD);
442
507k
443
507k
  // When declaring a function without a prototype, always use a
444
507k
  // non-variadic type.
445
507k
  if (CanQual<FunctionNoProtoType> noProto = FTy.getAs<FunctionNoProtoType>()) {
446
21.2k
    return arrangeLLVMFunctionInfo(
447
21.2k
        noProto->getReturnType(), /*instanceMethod=*/false,
448
21.2k
        /*chainCall=*/false, None, noProto->getExtInfo(), {},RequiredArgs::All);
449
21.2k
  }
450
486k
451
486k
  return arrangeFreeFunctionType(FTy.castAs<FunctionProtoType>());
452
486k
}
453
454
/// Arrange the argument and result information for the declaration or
455
/// definition of an Objective-C method.
456
const CGFunctionInfo &
457
2.65k
CodeGenTypes::arrangeObjCMethodDeclaration(const ObjCMethodDecl *MD) {
458
2.65k
  // It happens that this is the same as a call with no optional
459
2.65k
  // arguments, except also using the formal 'self' type.
460
2.65k
  return arrangeObjCMessageSendSignature(MD, MD->getSelfDecl()->getType());
461
2.65k
}
462
463
/// Arrange the argument and result information for the function type
464
/// through which to perform a send to the given Objective-C method,
465
/// using the given receiver type.  The receiver type is not always
466
/// the 'self' type of the method or even an Objective-C pointer type.
467
/// This is *not* the right method for actually performing such a
468
/// message send, due to the possibility of optional arguments.
469
const CGFunctionInfo &
470
CodeGenTypes::arrangeObjCMessageSendSignature(const ObjCMethodDecl *MD,
471
4.01k
                                              QualType receiverType) {
472
4.01k
  SmallVector<CanQualType, 16> argTys;
473
4.01k
  SmallVector<FunctionProtoType::ExtParameterInfo, 4> extParamInfos(2);
474
4.01k
  argTys.push_back(Context.getCanonicalParamType(receiverType));
475
4.01k
  argTys.push_back(Context.getCanonicalParamType(Context.getObjCSelType()));
476
4.01k
  // FIXME: Kill copy?
477
4.01k
  for (const auto *I : MD->parameters()) {
478
1.77k
    argTys.push_back(Context.getCanonicalParamType(I->getType()));
479
1.77k
    auto extParamInfo = FunctionProtoType::ExtParameterInfo().withIsNoEscape(
480
1.77k
        I->hasAttr<NoEscapeAttr>());
481
1.77k
    extParamInfos.push_back(extParamInfo);
482
1.77k
  }
483
4.01k
484
4.01k
  FunctionType::ExtInfo einfo;
485
4.01k
  bool IsWindows = getContext().getTargetInfo().getTriple().isOSWindows();
486
4.01k
  einfo = einfo.withCallingConv(getCallingConventionForDecl(MD, IsWindows));
487
4.01k
488
4.01k
  if (getContext().getLangOpts().ObjCAutoRefCount &&
489
4.01k
      
MD->hasAttr<NSReturnsRetainedAttr>()724
)
490
114
    einfo = einfo.withProducesResult(true);
491
4.01k
492
4.01k
  RequiredArgs required =
493
4.01k
    (MD->isVariadic() ? 
RequiredArgs(argTys.size())29
:
RequiredArgs::All3.98k
);
494
4.01k
495
4.01k
  return arrangeLLVMFunctionInfo(
496
4.01k
      GetReturnType(MD->getReturnType()), /*instanceMethod=*/false,
497
4.01k
      /*chainCall=*/false, argTys, einfo, extParamInfos, required);
498
4.01k
}
499
500
const CGFunctionInfo &
501
CodeGenTypes::arrangeUnprototypedObjCMessageSend(QualType returnType,
502
215
                                                 const CallArgList &args) {
503
215
  auto argTypes = getArgTypesForCall(Context, args);
504
215
  FunctionType::ExtInfo einfo;
505
215
506
215
  return arrangeLLVMFunctionInfo(
507
215
      GetReturnType(returnType), /*instanceMethod=*/false,
508
215
      /*chainCall=*/false, argTypes, einfo, {}, RequiredArgs::All);
509
215
}
510
511
const CGFunctionInfo &
512
819k
CodeGenTypes::arrangeGlobalDeclaration(GlobalDecl GD) {
513
819k
  // FIXME: Do we need to handle ObjCMethodDecl?
514
819k
  const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
515
819k
516
819k
  if (isa<CXXConstructorDecl>(GD.getDecl()) ||
517
819k
      
isa<CXXDestructorDecl>(GD.getDecl())764k
)
518
91.7k
    return arrangeCXXStructorDeclaration(GD);
519
727k
520
727k
  return arrangeFunctionDeclaration(FD);
521
727k
}
522
523
/// Arrange a thunk that takes 'this' as the first parameter followed by
524
/// varargs.  Return a void pointer, regardless of the actual return type.
525
/// The body of the thunk will end in a musttail call to a function of the
526
/// correct type, and the caller will bitcast the function to the correct
527
/// prototype.
528
const CGFunctionInfo &
529
54
CodeGenTypes::arrangeUnprototypedMustTailThunk(const CXXMethodDecl *MD) {
530
54
  assert(MD->isVirtual() && "only methods have thunks");
531
54
  CanQual<FunctionProtoType> FTP = GetFormalType(MD);
532
54
  CanQualType ArgTys[] = {DeriveThisType(MD->getParent(), MD)};
533
54
  return arrangeLLVMFunctionInfo(Context.VoidTy, /*instanceMethod=*/false,
534
54
                                 /*chainCall=*/false, ArgTys,
535
54
                                 FTP->getExtInfo(), {}, RequiredArgs(1));
536
54
}
537
538
const CGFunctionInfo &
539
CodeGenTypes::arrangeMSCtorClosure(const CXXConstructorDecl *CD,
540
34
                                   CXXCtorType CT) {
541
34
  assert(CT == Ctor_CopyingClosure || CT == Ctor_DefaultClosure);
542
34
543
34
  CanQual<FunctionProtoType> FTP = GetFormalType(CD);
544
34
  SmallVector<CanQualType, 2> ArgTys;
545
34
  const CXXRecordDecl *RD = CD->getParent();
546
34
  ArgTys.push_back(DeriveThisType(RD, CD));
547
34
  if (CT == Ctor_CopyingClosure)
548
6
    ArgTys.push_back(*FTP->param_type_begin());
549
34
  if (RD->getNumVBases() > 0)
550
0
    ArgTys.push_back(Context.IntTy);
551
34
  CallingConv CC = Context.getDefaultCallingConvention(
552
34
      /*IsVariadic=*/false, /*IsCXXMethod=*/true);
553
34
  return arrangeLLVMFunctionInfo(Context.VoidTy, /*instanceMethod=*/true,
554
34
                                 /*chainCall=*/false, ArgTys,
555
34
                                 FunctionType::ExtInfo(CC), {},
556
34
                                 RequiredArgs::All);
557
34
}
558
559
/// Arrange a call as unto a free function, except possibly with an
560
/// additional number of formal parameters considered required.
561
static const CGFunctionInfo &
562
arrangeFreeFunctionLikeCall(CodeGenTypes &CGT,
563
                            CodeGenModule &CGM,
564
                            const CallArgList &args,
565
                            const FunctionType *fnType,
566
                            unsigned numExtraRequiredArgs,
567
525k
                            bool chainCall) {
568
525k
  assert(args.size() >= numExtraRequiredArgs);
569
525k
570
525k
  llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos;
571
525k
572
525k
  // In most cases, there are no optional arguments.
573
525k
  RequiredArgs required = RequiredArgs::All;
574
525k
575
525k
  // If we have a variadic prototype, the required arguments are the
576
525k
  // extra prefix plus the arguments in the prototype.
577
525k
  if (const FunctionProtoType *proto = dyn_cast<FunctionProtoType>(fnType)) {
578
516k
    if (proto->isVariadic())
579
50.1k
      required = RequiredArgs::forPrototypePlus(proto, numExtraRequiredArgs);
580
516k
581
516k
    if (proto->hasExtParameterInfos())
582
163
      addExtParameterInfosForCall(paramInfos, proto, numExtraRequiredArgs,
583
163
                                  args.size());
584
516k
585
516k
  // If we don't have a prototype at all, but we're supposed to
586
516k
  // explicitly use the variadic convention for unprototyped calls,
587
516k
  // treat all of the arguments as required but preserve the nominal
588
516k
  // possibility of variadics.
589
516k
  } else 
if (9.48k
CGM.getTargetCodeGenInfo()
590
9.48k
                .isNoProtoCallVariadic(args,
591
9.48k
                                       cast<FunctionNoProtoType>(fnType))) {
592
482
    required = RequiredArgs(args.size());
593
482
  }
594
525k
595
525k
  // FIXME: Kill copy.
596
525k
  SmallVector<CanQualType, 16> argTypes;
597
525k
  for (const auto &arg : args)
598
994k
    argTypes.push_back(CGT.getContext().getCanonicalParamType(arg.Ty));
599
525k
  return CGT.arrangeLLVMFunctionInfo(GetReturnType(fnType->getReturnType()),
600
525k
                                     /*instanceMethod=*/false, chainCall,
601
525k
                                     argTypes, fnType->getExtInfo(), paramInfos,
602
525k
                                     required);
603
525k
}
604
605
/// Figure out the rules for calling a function with the given formal
606
/// type using the given arguments.  The arguments are necessary
607
/// because the function might be unprototyped, in which case it's
608
/// target-dependent in crazy ways.
609
const CGFunctionInfo &
610
CodeGenTypes::arrangeFreeFunctionCall(const CallArgList &args,
611
                                      const FunctionType *fnType,
612
524k
                                      bool chainCall) {
613
524k
  return arrangeFreeFunctionLikeCall(*this, CGM, args, fnType,
614
524k
                                     chainCall ? 
18
:
0524k
, chainCall);
615
524k
}
616
617
/// A block function is essentially a free function with an
618
/// extra implicit argument.
619
const CGFunctionInfo &
620
CodeGenTypes::arrangeBlockFunctionCall(const CallArgList &args,
621
616
                                       const FunctionType *fnType) {
622
616
  return arrangeFreeFunctionLikeCall(*this, CGM, args, fnType, 1,
623
616
                                     /*chainCall=*/false);
624
616
}
625
626
const CGFunctionInfo &
627
CodeGenTypes::arrangeBlockFunctionDeclaration(const FunctionProtoType *proto,
628
1.20k
                                              const FunctionArgList &params) {
629
1.20k
  auto paramInfos = getExtParameterInfosForCall(proto, 1, params.size());
630
1.20k
  auto argTypes = getArgTypesForDeclaration(Context, params);
631
1.20k
632
1.20k
  return arrangeLLVMFunctionInfo(GetReturnType(proto->getReturnType()),
633
1.20k
                                 /*instanceMethod*/ false, /*chainCall*/ false,
634
1.20k
                                 argTypes, proto->getExtInfo(), paramInfos,
635
1.20k
                                 RequiredArgs::forPrototypePlus(proto, 1));
636
1.20k
}
637
638
const CGFunctionInfo &
639
CodeGenTypes::arrangeBuiltinFunctionCall(QualType resultType,
640
872
                                         const CallArgList &args) {
641
872
  // FIXME: Kill copy.
642
872
  SmallVector<CanQualType, 16> argTypes;
643
872
  for (const auto &Arg : args)
644
2.89k
    argTypes.push_back(Context.getCanonicalParamType(Arg.Ty));
645
872
  return arrangeLLVMFunctionInfo(
646
872
      GetReturnType(resultType), /*instanceMethod=*/false,
647
872
      /*chainCall=*/false, argTypes, FunctionType::ExtInfo(),
648
872
      /*paramInfos=*/ {}, RequiredArgs::All);
649
872
}
650
651
const CGFunctionInfo &
652
CodeGenTypes::arrangeBuiltinFunctionDeclaration(QualType resultType,
653
18.5k
                                                const FunctionArgList &args) {
654
18.5k
  auto argTypes = getArgTypesForDeclaration(Context, args);
655
18.5k
656
18.5k
  return arrangeLLVMFunctionInfo(
657
18.5k
      GetReturnType(resultType), /*instanceMethod=*/false, /*chainCall=*/false,
658
18.5k
      argTypes, FunctionType::ExtInfo(), {}, RequiredArgs::All);
659
18.5k
}
660
661
const CGFunctionInfo &
662
CodeGenTypes::arrangeBuiltinFunctionDeclaration(CanQualType resultType,
663
252
                                              ArrayRef<CanQualType> argTypes) {
664
252
  return arrangeLLVMFunctionInfo(
665
252
      resultType, /*instanceMethod=*/false, /*chainCall=*/false,
666
252
      argTypes, FunctionType::ExtInfo(), {}, RequiredArgs::All);
667
252
}
668
669
/// Arrange a call to a C++ method, passing the given arguments.
670
///
671
/// numPrefixArgs is the number of ABI-specific prefix arguments we have. It
672
/// does not count `this`.
673
const CGFunctionInfo &
674
CodeGenTypes::arrangeCXXMethodCall(const CallArgList &args,
675
                                   const FunctionProtoType *proto,
676
                                   RequiredArgs required,
677
227k
                                   unsigned numPrefixArgs) {
678
227k
  assert(numPrefixArgs + 1 <= args.size() &&
679
227k
         "Emitting a call with less args than the required prefix?");
680
227k
  // Add one to account for `this`. It's a bit awkward here, but we don't count
681
227k
  // `this` in similar places elsewhere.
682
227k
  auto paramInfos =
683
227k
    getExtParameterInfosForCall(proto, numPrefixArgs + 1, args.size());
684
227k
685
227k
  // FIXME: Kill copy.
686
227k
  auto argTypes = getArgTypesForCall(Context, args);
687
227k
688
227k
  FunctionType::ExtInfo info = proto->getExtInfo();
689
227k
  return arrangeLLVMFunctionInfo(
690
227k
      GetReturnType(proto->getReturnType()), /*instanceMethod=*/true,
691
227k
      /*chainCall=*/false, argTypes, info, paramInfos, required);
692
227k
}
693
694
14.6k
const CGFunctionInfo &CodeGenTypes::arrangeNullaryFunction() {
695
14.6k
  return arrangeLLVMFunctionInfo(
696
14.6k
      getContext().VoidTy, /*instanceMethod=*/false, /*chainCall=*/false,
697
14.6k
      None, FunctionType::ExtInfo(), {}, RequiredArgs::All);
698
14.6k
}
699
700
const CGFunctionInfo &
701
CodeGenTypes::arrangeCall(const CGFunctionInfo &signature,
702
1.35k
                          const CallArgList &args) {
703
1.35k
  assert(signature.arg_size() <= args.size());
704
1.35k
  if (signature.arg_size() == args.size())
705
1.34k
    return signature;
706
17
707
17
  SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos;
708
17
  auto sigParamInfos = signature.getExtParameterInfos();
709
17
  if (!sigParamInfos.empty()) {
710
17
    paramInfos.append(sigParamInfos.begin(), sigParamInfos.end());
711
17
    paramInfos.resize(args.size());
712
17
  }
713
17
714
17
  auto argTypes = getArgTypesForCall(Context, args);
715
17
716
17
  assert(signature.getRequiredArgs().allowsOptionalArgs());
717
17
  return arrangeLLVMFunctionInfo(signature.getReturnType(),
718
17
                                 signature.isInstanceMethod(),
719
17
                                 signature.isChainCall(),
720
17
                                 argTypes,
721
17
                                 signature.getExtInfo(),
722
17
                                 paramInfos,
723
17
                                 signature.getRequiredArgs());
724
17
}
725
726
namespace clang {
727
namespace CodeGen {
728
void computeSPIRKernelABIInfo(CodeGenModule &CGM, CGFunctionInfo &FI);
729
}
730
}
731
732
/// Arrange the argument and result information for an abstract value
733
/// of a given function type.  This is the method which all of the
734
/// above functions ultimately defer to.
735
const CGFunctionInfo &
736
CodeGenTypes::arrangeLLVMFunctionInfo(CanQualType resultType,
737
                                      bool instanceMethod,
738
                                      bool chainCall,
739
                                      ArrayRef<CanQualType> argTypes,
740
                                      FunctionType::ExtInfo info,
741
                     ArrayRef<FunctionProtoType::ExtParameterInfo> paramInfos,
742
2.51M
                                      RequiredArgs required) {
743
2.51M
  assert(llvm::all_of(argTypes,
744
2.51M
                      [](CanQualType T) { return T.isCanonicalAsParam(); }));
745
2.51M
746
2.51M
  // Lookup or create unique function info.
747
2.51M
  llvm::FoldingSetNodeID ID;
748
2.51M
  CGFunctionInfo::Profile(ID, instanceMethod, chainCall, info, paramInfos,
749
2.51M
                          required, resultType, argTypes);
750
2.51M
751
2.51M
  void *insertPos = nullptr;
752
2.51M
  CGFunctionInfo *FI = FunctionInfos.FindNodeOrInsertPos(ID, insertPos);
753
2.51M
  if (FI)
754
2.21M
    return *FI;
755
300k
756
300k
  unsigned CC = ClangCallConvToLLVMCallConv(info.getCC());
757
300k
758
300k
  // Construct the function info.  We co-allocate the ArgInfos.
759
300k
  FI = CGFunctionInfo::create(CC, instanceMethod, chainCall, info,
760
300k
                              paramInfos, resultType, argTypes, required);
761
300k
  FunctionInfos.InsertNode(FI, insertPos);
762
300k
763
300k
  bool inserted = FunctionsBeingProcessed.insert(FI).second;
764
300k
  (void)inserted;
765
300k
  assert(inserted && "Recursively being processed?");
766
300k
767
300k
  // Compute ABI information.
768
300k
  if (CC == llvm::CallingConv::SPIR_KERNEL) {
769
118
    // Force target independent argument handling for the host visible
770
118
    // kernel functions.
771
118
    computeSPIRKernelABIInfo(CGM, *FI);
772
300k
  } else if (info.getCC() == CC_Swift) {
773
1.12k
    swiftcall::computeABIInfo(CGM, *FI);
774
299k
  } else {
775
299k
    getABIInfo().computeInfo(*FI);
776
299k
  }
777
300k
778
300k
  // Loop over all of the computed argument and return value info.  If any of
779
300k
  // them are direct or extend without a specified coerce type, specify the
780
300k
  // default now.
781
300k
  ABIArgInfo &retInfo = FI->getReturnInfo();
782
300k
  if (retInfo.canHaveCoerceToType() && 
retInfo.getCoerceToType() == nullptr185k
)
783
142k
    retInfo.setCoerceToType(ConvertType(FI->getReturnType()));
784
300k
785
300k
  for (auto &I : FI->arguments())
786
579k
    if (I.info.canHaveCoerceToType() && 
I.info.getCoerceToType() == nullptr571k
)
787
418k
      I.info.setCoerceToType(ConvertType(I.type));
788
300k
789
300k
  bool erased = FunctionsBeingProcessed.erase(FI); (void)erased;
790
300k
  assert(erased && "Not in set?");
791
300k
792
300k
  return *FI;
793
300k
}
794
795
CGFunctionInfo *CGFunctionInfo::create(unsigned llvmCC,
796
                                       bool instanceMethod,
797
                                       bool chainCall,
798
                                       const FunctionType::ExtInfo &info,
799
                                       ArrayRef<ExtParameterInfo> paramInfos,
800
                                       CanQualType resultType,
801
                                       ArrayRef<CanQualType> argTypes,
802
300k
                                       RequiredArgs required) {
803
300k
  assert(paramInfos.empty() || paramInfos.size() == argTypes.size());
804
300k
  assert(!required.allowsOptionalArgs() ||
805
300k
         required.getNumRequiredArgs() <= argTypes.size());
806
300k
807
300k
  void *buffer =
808
300k
    operator new(totalSizeToAlloc<ArgInfo,             ExtParameterInfo>(
809
300k
                                  argTypes.size() + 1, paramInfos.size()));
810
300k
811
300k
  CGFunctionInfo *FI = new(buffer) CGFunctionInfo();
812
300k
  FI->CallingConvention = llvmCC;
813
300k
  FI->EffectiveCallingConvention = llvmCC;
814
300k
  FI->ASTCallingConvention = info.getCC();
815
300k
  FI->InstanceMethod = instanceMethod;
816
300k
  FI->ChainCall = chainCall;
817
300k
  FI->NoReturn = info.getNoReturn();
818
300k
  FI->ReturnsRetained = info.getProducesResult();
819
300k
  FI->NoCallerSavedRegs = info.getNoCallerSavedRegs();
820
300k
  FI->NoCfCheck = info.getNoCfCheck();
821
300k
  FI->Required = required;
822
300k
  FI->HasRegParm = info.getHasRegParm();
823
300k
  FI->RegParm = info.getRegParm();
824
300k
  FI->ArgStruct = nullptr;
825
300k
  FI->ArgStructAlign = 0;
826
300k
  FI->NumArgs = argTypes.size();
827
300k
  FI->HasExtParameterInfos = !paramInfos.empty();
828
300k
  FI->getArgsBuffer()[0].type = resultType;
829
879k
  for (unsigned i = 0, e = argTypes.size(); i != e; 
++i579k
)
830
579k
    FI->getArgsBuffer()[i + 1].type = argTypes[i];
831
305k
  for (unsigned i = 0, e = paramInfos.size(); i != e; 
++i5.11k
)
832
5.11k
    FI->getExtParameterInfosBuffer()[i] = paramInfos[i];
833
300k
  return FI;
834
300k
}
835
836
/***/
837
838
namespace {
839
// ABIArgInfo::Expand implementation.
840
841
// Specifies the way QualType passed as ABIArgInfo::Expand is expanded.
842
struct TypeExpansion {
843
  enum TypeExpansionKind {
844
    // Elements of constant arrays are expanded recursively.
845
    TEK_ConstantArray,
846
    // Record fields are expanded recursively (but if record is a union, only
847
    // the field with the largest size is expanded).
848
    TEK_Record,
849
    // For complex types, real and imaginary parts are expanded recursively.
850
    TEK_Complex,
851
    // All other types are not expandable.
852
    TEK_None
853
  };
854
855
  const TypeExpansionKind Kind;
856
857
2.21k
  TypeExpansion(TypeExpansionKind K) : Kind(K) {}
858
2.21k
  virtual ~TypeExpansion() {}
859
};
860
861
struct ConstantArrayExpansion : TypeExpansion {
862
  QualType EltTy;
863
  uint64_t NumElts;
864
865
  ConstantArrayExpansion(QualType EltTy, uint64_t NumElts)
866
40
      : TypeExpansion(TEK_ConstantArray), EltTy(EltTy), NumElts(NumElts) {}
867
2.21k
  static bool classof(const TypeExpansion *TE) {
868
2.21k
    return TE->Kind == TEK_ConstantArray;
869
2.21k
  }
870
};
871
872
struct RecordExpansion : TypeExpansion {
873
  SmallVector<const CXXBaseSpecifier *, 1> Bases;
874
875
  SmallVector<const FieldDecl *, 1> Fields;
876
877
  RecordExpansion(SmallVector<const CXXBaseSpecifier *, 1> &&Bases,
878
                  SmallVector<const FieldDecl *, 1> &&Fields)
879
      : TypeExpansion(TEK_Record), Bases(std::move(Bases)),
880
706
        Fields(std::move(Fields)) {}
881
2.17k
  static bool classof(const TypeExpansion *TE) {
882
2.17k
    return TE->Kind == TEK_Record;
883
2.17k
  }
884
};
885
886
struct ComplexExpansion : TypeExpansion {
887
  QualType EltTy;
888
889
22
  ComplexExpansion(QualType EltTy) : TypeExpansion(TEK_Complex), EltTy(EltTy) {}
890
1.46k
  static bool classof(const TypeExpansion *TE) {
891
1.46k
    return TE->Kind == TEK_Complex;
892
1.46k
  }
893
};
894
895
struct NoExpansion : TypeExpansion {
896
1.44k
  NoExpansion() : TypeExpansion(TEK_None) {}
897
0
  static bool classof(const TypeExpansion *TE) {
898
0
    return TE->Kind == TEK_None;
899
0
  }
900
};
901
}  // namespace
902
903
static std::unique_ptr<TypeExpansion>
904
2.21k
getTypeExpansion(QualType Ty, const ASTContext &Context) {
905
2.21k
  if (const ConstantArrayType *AT = Context.getAsConstantArrayType(Ty)) {
906
40
    return llvm::make_unique<ConstantArrayExpansion>(
907
40
        AT->getElementType(), AT->getSize().getZExtValue());
908
40
  }
909
2.17k
  if (const RecordType *RT = Ty->getAs<RecordType>()) {
910
706
    SmallVector<const CXXBaseSpecifier *, 1> Bases;
911
706
    SmallVector<const FieldDecl *, 1> Fields;
912
706
    const RecordDecl *RD = RT->getDecl();
913
706
    assert(!RD->hasFlexibleArrayMember() &&
914
706
           "Cannot expand structure with flexible array.");
915
706
    if (RD->isUnion()) {
916
0
      // Unions can be here only in degenerative cases - all the fields are same
917
0
      // after flattening. Thus we have to use the "largest" field.
918
0
      const FieldDecl *LargestFD = nullptr;
919
0
      CharUnits UnionSize = CharUnits::Zero();
920
0
921
0
      for (const auto *FD : RD->fields()) {
922
0
        if (FD->isZeroLengthBitField(Context))
923
0
          continue;
924
0
        assert(!FD->isBitField() &&
925
0
               "Cannot expand structure with bit-field members.");
926
0
        CharUnits FieldSize = Context.getTypeSizeInChars(FD->getType());
927
0
        if (UnionSize < FieldSize) {
928
0
          UnionSize = FieldSize;
929
0
          LargestFD = FD;
930
0
        }
931
0
      }
932
0
      if (LargestFD)
933
0
        Fields.push_back(LargestFD);
934
706
    } else {
935
706
      if (const auto *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
936
201
        assert(!CXXRD->isDynamicClass() &&
937
201
               "cannot expand vtable pointers in dynamic classes");
938
201
        for (const CXXBaseSpecifier &BS : CXXRD->bases())
939
5
          Bases.push_back(&BS);
940
201
      }
941
706
942
1.40k
      for (const auto *FD : RD->fields()) {
943
1.40k
        if (FD->isZeroLengthBitField(Context))
944
0
          continue;
945
1.40k
        assert(!FD->isBitField() &&
946
1.40k
               "Cannot expand structure with bit-field members.");
947
1.40k
        Fields.push_back(FD);
948
1.40k
      }
949
706
    }
950
706
    return llvm::make_unique<RecordExpansion>(std::move(Bases),
951
706
                                              std::move(Fields));
952
706
  }
953
1.46k
  if (const ComplexType *CT = Ty->getAs<ComplexType>()) {
954
22
    return llvm::make_unique<ComplexExpansion>(CT->getElementType());
955
22
  }
956
1.44k
  return llvm::make_unique<NoExpansion>();
957
1.44k
}
958
959
1.29k
static int getExpansionSize(QualType Ty, const ASTContext &Context) {
960
1.29k
  auto Exp = getTypeExpansion(Ty, Context);
961
1.29k
  if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
962
24
    return CAExp->NumElts * getExpansionSize(CAExp->EltTy, Context);
963
24
  }
964
1.27k
  if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
965
424
    int Res = 0;
966
424
    for (auto BS : RExp->Bases)
967
3
      Res += getExpansionSize(BS->getType(), Context);
968
424
    for (auto FD : RExp->Fields)
969
844
      Res += getExpansionSize(FD->getType(), Context);
970
424
    return Res;
971
424
  }
972
850
  if (isa<ComplexExpansion>(Exp.get()))
973
13
    return 2;
974
837
  assert(isa<NoExpansion>(Exp.get()));
975
837
  return 1;
976
837
}
977
978
void
979
CodeGenTypes::getExpandedTypes(QualType Ty,
980
505
                               SmallVectorImpl<llvm::Type *>::iterator &TI) {
981
505
  auto Exp = getTypeExpansion(Ty, Context);
982
505
  if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
983
40
    for (int i = 0, n = CAExp->NumElts; i < n; 
i++32
) {
984
32
      getExpandedTypes(CAExp->EltTy, TI);
985
32
    }
986
497
  } else if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
987
159
    for (auto BS : RExp->Bases)
988
1
      getExpandedTypes(BS->getType(), TI);
989
159
    for (auto FD : RExp->Fields)
990
312
      getExpandedTypes(FD->getType(), TI);
991
338
  } else if (auto CExp = dyn_cast<ComplexExpansion>(Exp.get())) {
992
5
    llvm::Type *EltTy = ConvertType(CExp->EltTy);
993
5
    *TI++ = EltTy;
994
5
    *TI++ = EltTy;
995
333
  } else {
996
333
    assert(isa<NoExpansion>(Exp.get()));
997
333
    *TI++ = ConvertType(Ty);
998
333
  }
999
505
}
1000
1001
static void forConstantArrayExpansion(CodeGenFunction &CGF,
1002
                                      ConstantArrayExpansion *CAE,
1003
                                      Address BaseAddr,
1004
8
                                      llvm::function_ref<void(Address)> Fn) {
1005
8
  CharUnits EltSize = CGF.getContext().getTypeSizeInChars(CAE->EltTy);
1006
8
  CharUnits EltAlign =
1007
8
    BaseAddr.getAlignment().alignmentOfArrayElement(EltSize);
1008
8
1009
40
  for (int i = 0, n = CAE->NumElts; i < n; 
i++32
) {
1010
32
    llvm::Value *EltAddr =
1011
32
      CGF.Builder.CreateConstGEP2_32(nullptr, BaseAddr.getPointer(), 0, i);
1012
32
    Fn(Address(EltAddr, EltAlign));
1013
32
  }
1014
8
}
1015
1016
void CodeGenFunction::ExpandTypeFromArgs(
1017
337
    QualType Ty, LValue LV, SmallVectorImpl<llvm::Value *>::iterator &AI) {
1018
337
  assert(LV.isSimple() &&
1019
337
         "Unexpected non-simple lvalue during struct expansion.");
1020
337
1021
337
  auto Exp = getTypeExpansion(Ty, getContext());
1022
337
  if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
1023
8
    forConstantArrayExpansion(*this, CAExp, LV.getAddress(),
1024
32
                              [&](Address EltAddr) {
1025
32
      LValue LV = MakeAddrLValue(EltAddr, CAExp->EltTy);
1026
32
      ExpandTypeFromArgs(CAExp->EltTy, LV, AI);
1027
32
    });
1028
329
  } else if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
1029
96
    Address This = LV.getAddress();
1030
96
    for (const CXXBaseSpecifier *BS : RExp->Bases) {
1031
1
      // Perform a single step derived-to-base conversion.
1032
1
      Address Base =
1033
1
          GetAddressOfBaseClass(This, Ty->getAsCXXRecordDecl(), &BS, &BS + 1,
1034
1
                                /*NullCheckValue=*/false, SourceLocation());
1035
1
      LValue SubLV = MakeAddrLValue(Base, BS->getType());
1036
1
1037
1
      // Recurse onto bases.
1038
1
      ExpandTypeFromArgs(BS->getType(), SubLV, AI);
1039
1
    }
1040
207
    for (auto FD : RExp->Fields) {
1041
207
      // FIXME: What are the right qualifiers here?
1042
207
      LValue SubLV = EmitLValueForFieldInitialization(LV, FD);
1043
207
      ExpandTypeFromArgs(FD->getType(), SubLV, AI);
1044
207
    }
1045
233
  } else if (isa<ComplexExpansion>(Exp.get())) {
1046
3
    auto realValue = *AI++;
1047
3
    auto imagValue = *AI++;
1048
3
    EmitStoreOfComplex(ComplexPairTy(realValue, imagValue), LV, /*init*/ true);
1049
230
  } else {
1050
230
    assert(isa<NoExpansion>(Exp.get()));
1051
230
    EmitStoreThroughLValue(RValue::get(*AI++), LV);
1052
230
  }
1053
337
}
1054
1055
void CodeGenFunction::ExpandTypeToArgs(
1056
    QualType Ty, CallArg Arg, llvm::FunctionType *IRFuncTy,
1057
71
    SmallVectorImpl<llvm::Value *> &IRCallArgs, unsigned &IRCallArgPos) {
1058
71
  auto Exp = getTypeExpansion(Ty, getContext());
1059
71
  if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
1060
0
    Address Addr = Arg.hasLValue() ? Arg.getKnownLValue().getAddress()
1061
0
                                   : Arg.getKnownRValue().getAggregateAddress();
1062
0
    forConstantArrayExpansion(
1063
0
        *this, CAExp, Addr, [&](Address EltAddr) {
1064
0
          CallArg EltArg = CallArg(
1065
0
              convertTempToRValue(EltAddr, CAExp->EltTy, SourceLocation()),
1066
0
              CAExp->EltTy);
1067
0
          ExpandTypeToArgs(CAExp->EltTy, EltArg, IRFuncTy, IRCallArgs,
1068
0
                           IRCallArgPos);
1069
0
        });
1070
71
  } else if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
1071
27
    Address This = Arg.hasLValue() ? 
Arg.getKnownLValue().getAddress()12
1072
27
                                   : 
Arg.getKnownRValue().getAggregateAddress()15
;
1073
27
    for (const CXXBaseSpecifier *BS : RExp->Bases) {
1074
0
      // Perform a single step derived-to-base conversion.
1075
0
      Address Base =
1076
0
          GetAddressOfBaseClass(This, Ty->getAsCXXRecordDecl(), &BS, &BS + 1,
1077
0
                                /*NullCheckValue=*/false, SourceLocation());
1078
0
      CallArg BaseArg = CallArg(RValue::getAggregate(Base), BS->getType());
1079
0
1080
0
      // Recurse onto bases.
1081
0
      ExpandTypeToArgs(BS->getType(), BaseArg, IRFuncTy, IRCallArgs,
1082
0
                       IRCallArgPos);
1083
0
    }
1084
27
1085
27
    LValue LV = MakeAddrLValue(This, Ty);
1086
44
    for (auto FD : RExp->Fields) {
1087
44
      CallArg FldArg =
1088
44
          CallArg(EmitRValueForField(LV, FD, SourceLocation()), FD->getType());
1089
44
      ExpandTypeToArgs(FD->getType(), FldArg, IRFuncTy, IRCallArgs,
1090
44
                       IRCallArgPos);
1091
44
    }
1092
44
  } else if (isa<ComplexExpansion>(Exp.get())) {
1093
1
    ComplexPairTy CV = Arg.getKnownRValue().getComplexVal();
1094
1
    IRCallArgs[IRCallArgPos++] = CV.first;
1095
1
    IRCallArgs[IRCallArgPos++] = CV.second;
1096
43
  } else {
1097
43
    assert(isa<NoExpansion>(Exp.get()));
1098
43
    auto RV = Arg.getKnownRValue();
1099
43
    assert(RV.isScalar() &&
1100
43
           "Unexpected non-scalar rvalue during struct expansion.");
1101
43
1102
43
    // Insert a bitcast as needed.
1103
43
    llvm::Value *V = RV.getScalarVal();
1104
43
    if (IRCallArgPos < IRFuncTy->getNumParams() &&
1105
43
        V->getType() != IRFuncTy->getParamType(IRCallArgPos))
1106
0
      V = Builder.CreateBitCast(V, IRFuncTy->getParamType(IRCallArgPos));
1107
43
1108
43
    IRCallArgs[IRCallArgPos++] = V;
1109
43
  }
1110
71
}
1111
1112
/// Create a temporary allocation for the purposes of coercion.
1113
static Address CreateTempAllocaForCoercion(CodeGenFunction &CGF, llvm::Type *Ty,
1114
282
                                           CharUnits MinAlign) {
1115
282
  // Don't use an alignment that's worse than what LLVM would prefer.
1116
282
  auto PrefAlign = CGF.CGM.getDataLayout().getPrefTypeAlignment(Ty);
1117
282
  CharUnits Align = std::max(MinAlign, CharUnits::fromQuantity(PrefAlign));
1118
282
1119
282
  return CGF.CreateTempAlloca(Ty, Align);
1120
282
}
1121
1122
/// EnterStructPointerForCoercedAccess - Given a struct pointer that we are
1123
/// accessing some number of bytes out of it, try to gep into the struct to get
1124
/// at its inner goodness.  Dive as deep as possible without entering an element
1125
/// with an in-memory size smaller than DstSize.
1126
static Address
1127
EnterStructPointerForCoercedAccess(Address SrcPtr,
1128
                                   llvm::StructType *SrcSTy,
1129
30.1k
                                   uint64_t DstSize, CodeGenFunction &CGF) {
1130
30.1k
  // We can't dive into a zero-element struct.
1131
30.1k
  if (SrcSTy->getNumElements() == 0) 
return SrcPtr0
;
1132
30.1k
1133
30.1k
  llvm::Type *FirstElt = SrcSTy->getElementType(0);
1134
30.1k
1135
30.1k
  // If the first elt is at least as large as what we're looking for, or if the
1136
30.1k
  // first element is the same size as the whole struct, we can enter it. The
1137
30.1k
  // comparison must be made on the store size and not the alloca size. Using
1138
30.1k
  // the alloca size may overstate the size of the load.
1139
30.1k
  uint64_t FirstEltSize =
1140
30.1k
    CGF.CGM.getDataLayout().getTypeStoreSize(FirstElt);
1141
30.1k
  if (FirstEltSize < DstSize &&
1142
30.1k
      
FirstEltSize < CGF.CGM.getDataLayout().getTypeStoreSize(SrcSTy)8.64k
)
1143
7.34k
    return SrcPtr;
1144
22.8k
1145
22.8k
  // GEP into the first element.
1146
22.8k
  SrcPtr = CGF.Builder.CreateStructGEP(SrcPtr, 0, "coerce.dive");
1147
22.8k
1148
22.8k
  // If the first element is a struct, recurse.
1149
22.8k
  llvm::Type *SrcTy = SrcPtr.getElementType();
1150
22.8k
  if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy))
1151
3.52k
    return EnterStructPointerForCoercedAccess(SrcPtr, SrcSTy, DstSize, CGF);
1152
19.2k
1153
19.2k
  return SrcPtr;
1154
19.2k
}
1155
1156
/// CoerceIntOrPtrToIntOrPtr - Convert a value Val to the specific Ty where both
1157
/// are either integers or pointers.  This does a truncation of the value if it
1158
/// is too large or a zero extension if it is too small.
1159
///
1160
/// This behaves as if the value were coerced through memory, so on big-endian
1161
/// targets the high bits are preserved in a truncation, while little-endian
1162
/// targets preserve the low bits.
1163
static llvm::Value *CoerceIntOrPtrToIntOrPtr(llvm::Value *Val,
1164
                                             llvm::Type *Ty,
1165
17.9k
                                             CodeGenFunction &CGF) {
1166
17.9k
  if (Val->getType() == Ty)
1167
8.31k
    return Val;
1168
9.61k
1169
9.61k
  if (isa<llvm::PointerType>(Val->getType())) {
1170
4.41k
    // If this is Pointer->Pointer avoid conversion to and from int.
1171
4.41k
    if (isa<llvm::PointerType>(Ty))
1172
130
      return CGF.Builder.CreateBitCast(Val, Ty, "coerce.val");
1173
4.28k
1174
4.28k
    // Convert the pointer to an integer so we can play with its width.
1175
4.28k
    Val = CGF.Builder.CreatePtrToInt(Val, CGF.IntPtrTy, "coerce.val.pi");
1176
4.28k
  }
1177
9.61k
1178
9.61k
  llvm::Type *DestIntTy = Ty;
1179
9.48k
  if (isa<llvm::PointerType>(DestIntTy))
1180
4.51k
    DestIntTy = CGF.IntPtrTy;
1181
9.48k
1182
9.48k
  if (Val->getType() != DestIntTy) {
1183
688
    const llvm::DataLayout &DL = CGF.CGM.getDataLayout();
1184
688
    if (DL.isBigEndian()) {
1185
10
      // Preserve the high bits on big-endian targets.
1186
10
      // That is what memory coercion does.
1187
10
      uint64_t SrcSize = DL.getTypeSizeInBits(Val->getType());
1188
10
      uint64_t DstSize = DL.getTypeSizeInBits(DestIntTy);
1189
10
1190
10
      if (SrcSize > DstSize) {
1191
4
        Val = CGF.Builder.CreateLShr(Val, SrcSize - DstSize, "coerce.highbits");
1192
4
        Val = CGF.Builder.CreateTrunc(Val, DestIntTy, "coerce.val.ii");
1193
6
      } else {
1194
6
        Val = CGF.Builder.CreateZExt(Val, DestIntTy, "coerce.val.ii");
1195
6
        Val = CGF.Builder.CreateShl(Val, DstSize - SrcSize, "coerce.highbits");
1196
6
      }
1197
678
    } else {
1198
678
      // Little-endian targets preserve the low bits. No shifts required.
1199
678
      Val = CGF.Builder.CreateIntCast(Val, DestIntTy, false, "coerce.val.ii");
1200
678
    }
1201
688
  }
1202
9.48k
1203
9.48k
  if (isa<llvm::PointerType>(Ty))
1204
4.51k
    Val = CGF.Builder.CreateIntToPtr(Val, Ty, "coerce.val.ip");
1205
9.48k
  return Val;
1206
9.61k
}
1207
1208
1209
1210
/// CreateCoercedLoad - Create a load from \arg SrcPtr interpreted as
1211
/// a pointer to an object of type \arg Ty, known to be aligned to
1212
/// \arg SrcAlign bytes.
1213
///
1214
/// This safely handles the case when the src type is smaller than the
1215
/// destination type; in this situation the values of bits which not
1216
/// present in the src are undefined.
1217
static llvm::Value *CreateCoercedLoad(Address Src, llvm::Type *Ty,
1218
14.3k
                                      CodeGenFunction &CGF) {
1219
14.3k
  llvm::Type *SrcTy = Src.getElementType();
1220
14.3k
1221
14.3k
  // If SrcTy and Ty are the same, just do a load.
1222
14.3k
  if (SrcTy == Ty)
1223
24
    return CGF.Builder.CreateLoad(Src);
1224
14.2k
1225
14.2k
  uint64_t DstSize = CGF.CGM.getDataLayout().getTypeAllocSize(Ty);
1226
14.2k
1227
14.2k
  if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy)) {
1228
12.6k
    Src = EnterStructPointerForCoercedAccess(Src, SrcSTy, DstSize, CGF);
1229
12.6k
    SrcTy = Src.getType()->getElementType();
1230
12.6k
  }
1231
14.2k
1232
14.2k
  uint64_t SrcSize = CGF.CGM.getDataLayout().getTypeAllocSize(SrcTy);
1233
14.2k
1234
14.2k
  // If the source and destination are integer or pointer types, just do an
1235
14.2k
  // extension or truncation to the desired type.
1236
14.2k
  if ((isa<llvm::IntegerType>(Ty) || 
isa<llvm::PointerType>(Ty)6.31k
) &&
1237
14.2k
      
(10.2k
isa<llvm::IntegerType>(SrcTy)10.2k
||
isa<llvm::PointerType>(SrcTy)8.09k
)) {
1238
8.79k
    llvm::Value *Load = CGF.Builder.CreateLoad(Src);
1239
8.79k
    return CoerceIntOrPtrToIntOrPtr(Load, Ty, CGF);
1240
8.79k
  }
1241
5.50k
1242
5.50k
  // If load is legal, just bitcast the src pointer.
1243
5.50k
  if (SrcSize >= DstSize) {
1244
5.36k
    // Generally SrcSize is never greater than DstSize, since this means we are
1245
5.36k
    // losing bits. However, this can happen in cases where the structure has
1246
5.36k
    // additional padding, for example due to a user specified alignment.
1247
5.36k
    //
1248
5.36k
    // FIXME: Assert that we aren't truncating non-padding bits when have access
1249
5.36k
    // to that information.
1250
5.36k
    Src = CGF.Builder.CreateBitCast(Src,
1251
5.36k
                                    Ty->getPointerTo(Src.getAddressSpace()));
1252
5.36k
    return CGF.Builder.CreateLoad(Src);
1253
5.36k
  }
1254
140
1255
140
  // Otherwise do coercion through memory. This is stupid, but simple.
1256
140
  Address Tmp = CreateTempAllocaForCoercion(CGF, Ty, Src.getAlignment());
1257
140
  Address Casted = CGF.Builder.CreateElementBitCast(Tmp,CGF.Int8Ty);
1258
140
  Address SrcCasted = CGF.Builder.CreateElementBitCast(Src,CGF.Int8Ty);
1259
140
  CGF.Builder.CreateMemCpy(Casted, SrcCasted,
1260
140
      llvm::ConstantInt::get(CGF.IntPtrTy, SrcSize),
1261
140
      false);
1262
140
  return CGF.Builder.CreateLoad(Tmp);
1263
140
}
1264
1265
// Function to store a first-class aggregate into memory.  We prefer to
1266
// store the elements rather than the aggregate to be more friendly to
1267
// fast-isel.
1268
// FIXME: Do we need to recurse here?
1269
static void BuildAggStore(CodeGenFunction &CGF, llvm::Value *Val,
1270
9.51k
                          Address Dest, bool DestIsVolatile) {
1271
9.51k
  // Prefer scalar stores to first-class aggregate stores.
1272
9.51k
  if (llvm::StructType *STy =
1273
3.89k
        dyn_cast<llvm::StructType>(Val->getType())) {
1274
9.08k
    for (unsigned i = 0, e = STy->getNumElements(); i != e; 
++i5.18k
) {
1275
5.18k
      Address EltPtr = CGF.Builder.CreateStructGEP(Dest, i);
1276
5.18k
      llvm::Value *Elt = CGF.Builder.CreateExtractValue(Val, i);
1277
5.18k
      CGF.Builder.CreateStore(Elt, EltPtr, DestIsVolatile);
1278
5.18k
    }
1279
5.61k
  } else {
1280
5.61k
    CGF.Builder.CreateStore(Val, Dest, DestIsVolatile);
1281
5.61k
  }
1282
9.51k
}
1283
1284
/// CreateCoercedStore - Create a store to \arg DstPtr from \arg Src,
1285
/// where the source and destination may have different types.  The
1286
/// destination is known to be aligned to \arg DstAlign bytes.
1287
///
1288
/// This safely handles the case when the src type is larger than the
1289
/// destination type; the upper bits of the src will be lost.
1290
static void CreateCoercedStore(llvm::Value *Src,
1291
                               Address Dst,
1292
                               bool DstIsVolatile,
1293
15.8k
                               CodeGenFunction &CGF) {
1294
15.8k
  llvm::Type *SrcTy = Src->getType();
1295
15.8k
  llvm::Type *DstTy = Dst.getType()->getElementType();
1296
15.8k
  if (SrcTy == DstTy) {
1297
149
    CGF.Builder.CreateStore(Src, Dst, DstIsVolatile);
1298
149
    return;
1299
149
  }
1300
15.6k
1301
15.6k
  uint64_t SrcSize = CGF.CGM.getDataLayout().getTypeAllocSize(SrcTy);
1302
15.6k
1303
15.6k
  if (llvm::StructType *DstSTy = dyn_cast<llvm::StructType>(DstTy)) {
1304
14.0k
    Dst = EnterStructPointerForCoercedAccess(Dst, DstSTy, SrcSize, CGF);
1305
14.0k
    DstTy = Dst.getType()->getElementType();
1306
14.0k
  }
1307
15.6k
1308
15.6k
  // If the source and destination are integer or pointer types, just do an
1309
15.6k
  // extension or truncation to the desired type.
1310
15.6k
  if ((isa<llvm::IntegerType>(SrcTy) || 
isa<llvm::PointerType>(SrcTy)8.04k
) &&
1311
15.6k
      
(10.6k
isa<llvm::IntegerType>(DstTy)10.6k
||
isa<llvm::PointerType>(DstTy)9.02k
)) {
1312
9.13k
    Src = CoerceIntOrPtrToIntOrPtr(Src, DstTy, CGF);
1313
9.13k
    CGF.Builder.CreateStore(Src, Dst, DstIsVolatile);
1314
9.13k
    return;
1315
9.13k
  }
1316
6.55k
1317
6.55k
  uint64_t DstSize = CGF.CGM.getDataLayout().getTypeAllocSize(DstTy);
1318
6.55k
1319
6.55k
  // If store is legal, just bitcast the src pointer.
1320
6.55k
  if (SrcSize <= DstSize) {
1321
6.41k
    Dst = CGF.Builder.CreateElementBitCast(Dst, SrcTy);
1322
6.41k
    BuildAggStore(CGF, Src, Dst, DstIsVolatile);
1323
6.41k
  } else {
1324
142
    // Otherwise do coercion through memory. This is stupid, but
1325
142
    // simple.
1326
142
1327
142
    // Generally SrcSize is never greater than DstSize, since this means we are
1328
142
    // losing bits. However, this can happen in cases where the structure has
1329
142
    // additional padding, for example due to a user specified alignment.
1330
142
    //
1331
142
    // FIXME: Assert that we aren't truncating non-padding bits when have access
1332
142
    // to that information.
1333
142
    Address Tmp = CreateTempAllocaForCoercion(CGF, SrcTy, Dst.getAlignment());
1334
142
    CGF.Builder.CreateStore(Src, Tmp);
1335
142
    Address Casted = CGF.Builder.CreateElementBitCast(Tmp,CGF.Int8Ty);
1336
142
    Address DstCasted = CGF.Builder.CreateElementBitCast(Dst,CGF.Int8Ty);
1337
142
    CGF.Builder.CreateMemCpy(DstCasted, Casted,
1338
142
        llvm::ConstantInt::get(CGF.IntPtrTy, DstSize),
1339
142
        false);
1340
142
  }
1341
6.55k
}
1342
1343
static Address emitAddressAtOffset(CodeGenFunction &CGF, Address addr,
1344
32.4k
                                   const ABIArgInfo &info) {
1345
32.4k
  if (unsigned offset = info.getDirectOffset()) {
1346
3
    addr = CGF.Builder.CreateElementBitCast(addr, CGF.Int8Ty);
1347
3
    addr = CGF.Builder.CreateConstInBoundsByteGEP(addr,
1348
3
                                             CharUnits::fromQuantity(offset));
1349
3
    addr = CGF.Builder.CreateElementBitCast(addr, info.getCoerceToType());
1350
3
  }
1351
32.4k
  return addr;
1352
32.4k
}
1353
1354
namespace {
1355
1356
/// Encapsulates information about the way function arguments from
1357
/// CGFunctionInfo should be passed to actual LLVM IR function.
1358
class ClangToLLVMArgMapping {
1359
  static const unsigned InvalidIndex = ~0U;
1360
  unsigned InallocaArgNo;
1361
  unsigned SRetArgNo;
1362
  unsigned TotalIRArgs;
1363
1364
  /// Arguments of LLVM IR function corresponding to single Clang argument.
1365
  struct IRArgs {
1366
    unsigned PaddingArgIndex;
1367
    // Argument is expanded to IR arguments at positions
1368
    // [FirstArgIndex, FirstArgIndex + NumberOfArgs).
1369
    unsigned FirstArgIndex;
1370
    unsigned NumberOfArgs;
1371
1372
    IRArgs()
1373
        : PaddingArgIndex(InvalidIndex), FirstArgIndex(InvalidIndex),
1374
4.74M
          NumberOfArgs(0) {}
1375
  };
1376
1377
  SmallVector<IRArgs, 8> ArgInfo;
1378
1379
public:
1380
  ClangToLLVMArgMapping(const ASTContext &Context, const CGFunctionInfo &FI,
1381
                        bool OnlyRequiredArgs = false)
1382
      : InallocaArgNo(InvalidIndex), SRetArgNo(InvalidIndex), TotalIRArgs(0),
1383
4.74M
        ArgInfo(OnlyRequiredArgs ? FI.getNumRequiredArgs() : FI.arg_size()) {
1384
4.74M
    construct(Context, FI, OnlyRequiredArgs);
1385
4.74M
  }
1386
1387
3.83M
  bool hasInallocaArg() const { return InallocaArgNo != InvalidIndex; }
1388
424
  unsigned getInallocaArgNo() const {
1389
424
    assert(hasInallocaArg());
1390
424
    return InallocaArgNo;
1391
424
  }
1392
1393
3.84M
  bool hasSRetArg() const { return SRetArgNo != InvalidIndex; }
1394
42.1k
  unsigned getSRetArgNo() const {
1395
42.1k
    assert(hasSRetArg());
1396
42.1k
    return SRetArgNo;
1397
42.1k
  }
1398
1399
4.74M
  unsigned totalIRArgs() const { return TotalIRArgs; }
1400
1401
7.69M
  bool hasPaddingArg(unsigned ArgNo) const {
1402
7.69M
    assert(ArgNo < ArgInfo.size());
1403
7.69M
    return ArgInfo[ArgNo].PaddingArgIndex != InvalidIndex;
1404
7.69M
  }
1405
69
  unsigned getPaddingArgNo(unsigned ArgNo) const {
1406
69
    assert(hasPaddingArg(ArgNo));
1407
69
    return ArgInfo[ArgNo].PaddingArgIndex;
1408
69
  }
1409
1410
  /// Returns index of first IR argument corresponding to ArgNo, and their
1411
  /// quantity.
1412
6.23M
  std::pair<unsigned, unsigned> getIRArgs(unsigned ArgNo) const {
1413
6.23M
    assert(ArgNo < ArgInfo.size());
1414
6.23M
    return std::make_pair(ArgInfo[ArgNo].FirstArgIndex,
1415
6.23M
                          ArgInfo[ArgNo].NumberOfArgs);
1416
6.23M
  }
1417
1418
private:
1419
  void construct(const ASTContext &Context, const CGFunctionInfo &FI,
1420
                 bool OnlyRequiredArgs);
1421
};
1422
1423
void ClangToLLVMArgMapping::construct(const ASTContext &Context,
1424
                                      const CGFunctionInfo &FI,
1425
4.74M
                                      bool OnlyRequiredArgs) {
1426
4.74M
  unsigned IRArgNo = 0;
1427
4.74M
  bool SwapThisWithSRet = false;
1428
4.74M
  const ABIArgInfo &RetAI = FI.getReturnInfo();
1429
4.74M
1430
4.74M
  if (RetAI.getKind() == ABIArgInfo::Indirect) {
1431
42.1k
    SwapThisWithSRet = RetAI.isSRetAfterThis();
1432
42.1k
    SRetArgNo = SwapThisWithSRet ? 
1357
:
IRArgNo++41.7k
;
1433
42.1k
  }
1434
4.74M
1435
4.74M
  unsigned ArgNo = 0;
1436
4.74M
  unsigned NumArgs = OnlyRequiredArgs ? 
FI.getNumRequiredArgs()2.07M
:
FI.arg_size()2.66M
;
1437
13.0M
  for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(); ArgNo < NumArgs;
1438
8.35M
       ++I, ++ArgNo) {
1439
8.35M
    assert(I != FI.arg_end());
1440
8.35M
    QualType ArgType = I->type;
1441
8.35M
    const ABIArgInfo &AI = I->info;
1442
8.35M
    // Collect data about IR arguments corresponding to Clang argument ArgNo.
1443
8.35M
    auto &IRArgs = ArgInfo[ArgNo];
1444
8.35M
1445
8.35M
    if (AI.getPaddingType())
1446
124
      IRArgs.PaddingArgIndex = IRArgNo++;
1447
8.35M
1448
8.35M
    switch (AI.getKind()) {
1449
8.35M
    case ABIArgInfo::Extend:
1450
8.28M
    case ABIArgInfo::Direct: {
1451
8.28M
      // FIXME: handle sseregparm someday...
1452
8.28M
      llvm::StructType *STy = dyn_cast<llvm::StructType>(AI.getCoerceToType());
1453
8.28M
      if (AI.isDirect() && 
AI.getCanBeFlattened()8.04M
&&
STy8.04M
) {
1454
8.66k
        IRArgs.NumberOfArgs = STy->getNumElements();
1455
8.27M
      } else {
1456
8.27M
        IRArgs.NumberOfArgs = 1;
1457
8.27M
      }
1458
8.28M
      break;
1459
8.28M
    }
1460
8.28M
    case ABIArgInfo::Indirect:
1461
24.6k
      IRArgs.NumberOfArgs = 1;
1462
24.6k
      break;
1463
8.28M
    case ABIArgInfo::Ignore:
1464
43.5k
    case ABIArgInfo::InAlloca:
1465
43.5k
      // ignore and inalloca doesn't have matching LLVM parameters.
1466
43.5k
      IRArgs.NumberOfArgs = 0;
1467
43.5k
      break;
1468
43.5k
    case ABIArgInfo::CoerceAndExpand:
1469
3.38k
      IRArgs.NumberOfArgs = AI.getCoerceAndExpandTypeSequence().size();
1470
3.38k
      break;
1471
43.5k
    case ABIArgInfo::Expand:
1472
427
      IRArgs.NumberOfArgs = getExpansionSize(ArgType, Context);
1473
427
      break;
1474
8.35M
    }
1475
8.35M
1476
8.35M
    if (IRArgs.NumberOfArgs > 0) {
1477
8.31M
      IRArgs.FirstArgIndex = IRArgNo;
1478
8.31M
      IRArgNo += IRArgs.NumberOfArgs;
1479
8.31M
    }
1480
8.35M
1481
8.35M
    // Skip over the sret parameter when it comes second.  We already handled it
1482
8.35M
    // above.
1483
8.35M
    if (IRArgNo == 1 && 
SwapThisWithSRet4.26M
)
1484
357
      IRArgNo++;
1485
8.35M
  }
1486
4.74M
  assert(ArgNo == ArgInfo.size());
1487
4.74M
1488
4.74M
  if (FI.usesInAlloca())
1489
424
    InallocaArgNo = IRArgNo++;
1490
4.74M
1491
4.74M
  TotalIRArgs = IRArgNo;
1492
4.74M
}
1493
}  // namespace
1494
1495
/***/
1496
1497
3.71k
bool CodeGenModule::ReturnTypeUsesSRet(const CGFunctionInfo &FI) {
1498
3.71k
  const auto &RI = FI.getReturnInfo();
1499
3.71k
  return RI.isIndirect() || 
(3.66k
RI.isInAlloca()3.66k
&&
RI.getInAllocaSRet()1
);
1500
3.71k
}
1501
1502
2.73k
bool CodeGenModule::ReturnSlotInterferesWithArgs(const CGFunctionInfo &FI) {
1503
2.73k
  return ReturnTypeUsesSRet(FI) &&
1504
2.73k
         
getTargetCodeGenInfo().doesReturnSlotInterfereWithArgs()49
;
1505
2.73k
}
1506
1507
1.50k
bool CodeGenModule::ReturnTypeUsesFPRet(QualType ResultType) {
1508
1.50k
  if (const BuiltinType *BT = ResultType->getAs<BuiltinType>()) {
1509
680
    switch (BT->getKind()) {
1510
680
    default:
1511
631
      return false;
1512
680
    case BuiltinType::Float:
1513
25
      return getTarget().useObjCFPRetForRealType(TargetInfo::Float);
1514
680
    case BuiltinType::Double:
1515
20
      return getTarget().useObjCFPRetForRealType(TargetInfo::Double);
1516
680
    case BuiltinType::LongDouble:
1517
4
      return getTarget().useObjCFPRetForRealType(TargetInfo::LongDouble);
1518
829
    }
1519
829
  }
1520
829
1521
829
  return false;
1522
829
}
1523
1524
1.49k
bool CodeGenModule::ReturnTypeUsesFP2Ret(QualType ResultType) {
1525
1.49k
  if (const ComplexType *CT = ResultType->getAs<ComplexType>()) {
1526
25
    if (const BuiltinType *BT = CT->getElementType()->getAs<BuiltinType>()) {
1527
25
      if (BT->getKind() == BuiltinType::LongDouble)
1528
2
        return getTarget().useObjCFP2RetForComplexLongDouble();
1529
1.48k
    }
1530
25
  }
1531
1.48k
1532
1.48k
  return false;
1533
1.48k
}
1534
1535
20.7k
llvm::FunctionType *CodeGenTypes::GetFunctionType(GlobalDecl GD) {
1536
20.7k
  const CGFunctionInfo &FI = arrangeGlobalDeclaration(GD);
1537
20.7k
  return GetFunctionType(FI);
1538
20.7k
}
1539
1540
llvm::FunctionType *
1541
2.07M
CodeGenTypes::GetFunctionType(const CGFunctionInfo &FI) {
1542
2.07M
1543
2.07M
  bool Inserted = FunctionsBeingProcessed.insert(&FI).second;
1544
2.07M
  (void)Inserted;
1545
2.07M
  assert(Inserted && "Recursively being processed?");
1546
2.07M
1547
2.07M
  llvm::Type *resultType = nullptr;
1548
2.07M
  const ABIArgInfo &retAI = FI.getReturnInfo();
1549
2.07M
  switch (retAI.getKind()) {
1550
2.07M
  case ABIArgInfo::Expand:
1551
0
    llvm_unreachable("Invalid ABI kind for return argument");
1552
2.07M
1553
2.07M
  case ABIArgInfo::Extend:
1554
1.35M
  case ABIArgInfo::Direct:
1555
1.35M
    resultType = retAI.getCoerceToType();
1556
1.35M
    break;
1557
1.35M
1558
1.35M
  case ABIArgInfo::InAlloca:
1559
18
    if (retAI.getInAllocaSRet()) {
1560
18
      // sret things on win32 aren't void, they return the sret pointer.
1561
18
      QualType ret = FI.getReturnType();
1562
18
      llvm::Type *ty = ConvertType(ret);
1563
18
      unsigned addressSpace = Context.getTargetAddressSpace(ret);
1564
18
      resultType = llvm::PointerType::get(ty, addressSpace);
1565
18
    } else {
1566
0
      resultType = llvm::Type::getVoidTy(getLLVMContext());
1567
0
    }
1568
18
    break;
1569
1.35M
1570
1.35M
  case ABIArgInfo::Indirect:
1571
727k
  case ABIArgInfo::Ignore:
1572
727k
    resultType = llvm::Type::getVoidTy(getLLVMContext());
1573
727k
    break;
1574
727k
1575
727k
  case ABIArgInfo::CoerceAndExpand:
1576
1.41k
    resultType = retAI.getUnpaddedCoerceAndExpandType();
1577
1.41k
    break;
1578
2.07M
  }
1579
2.07M
1580
2.07M
  ClangToLLVMArgMapping IRFunctionArgs(getContext(), FI, true);
1581
2.07M
  SmallVector<llvm::Type*, 8> ArgTypes(IRFunctionArgs.totalIRArgs());
1582
2.07M
1583
2.07M
  // Add type for sret argument.
1584
2.07M
  if (IRFunctionArgs.hasSRetArg()) {
1585
18.9k
    QualType Ret = FI.getReturnType();
1586
18.9k
    llvm::Type *Ty = ConvertType(Ret);
1587
18.9k
    unsigned AddressSpace = Context.getTargetAddressSpace(Ret);
1588
18.9k
    ArgTypes[IRFunctionArgs.getSRetArgNo()] =
1589
18.9k
        llvm::PointerType::get(Ty, AddressSpace);
1590
18.9k
  }
1591
2.07M
1592
2.07M
  // Add type for inalloca argument.
1593
2.07M
  if (IRFunctionArgs.hasInallocaArg()) {
1594
176
    auto ArgStruct = FI.getArgStruct();
1595
176
    assert(ArgStruct);
1596
176
    ArgTypes[IRFunctionArgs.getInallocaArgNo()] = ArgStruct->getPointerTo();
1597
176
  }
1598
2.07M
1599
2.07M
  // Add in all of the required arguments.
1600
2.07M
  unsigned ArgNo = 0;
1601
2.07M
  CGFunctionInfo::const_arg_iterator it = FI.arg_begin(),
1602
2.07M
                                     ie = it + FI.getNumRequiredArgs();
1603
5.69M
  for (; it != ie; 
++it, ++ArgNo3.61M
) {
1604
3.61M
    const ABIArgInfo &ArgInfo = it->info;
1605
3.61M
1606
3.61M
    // Insert a padding type to ensure proper alignment.
1607
3.61M
    if (IRFunctionArgs.hasPaddingArg(ArgNo))
1608
47
      ArgTypes[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
1609
47
          ArgInfo.getPaddingType();
1610
3.61M
1611
3.61M
    unsigned FirstIRArg, NumIRArgs;
1612
3.61M
    std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
1613
3.61M
1614
3.61M
    switch (ArgInfo.getKind()) {
1615
3.61M
    case ABIArgInfo::Ignore:
1616
21.2k
    case ABIArgInfo::InAlloca:
1617
21.2k
      assert(NumIRArgs == 0);
1618
21.2k
      break;
1619
21.2k
1620
21.2k
    case ABIArgInfo::Indirect: {
1621
10.9k
      assert(NumIRArgs == 1);
1622
10.9k
      // indirect arguments are always on the stack, which is alloca addr space.
1623
10.9k
      llvm::Type *LTy = ConvertTypeForMem(it->type);
1624
10.9k
      ArgTypes[FirstIRArg] = LTy->getPointerTo(
1625
10.9k
          CGM.getDataLayout().getAllocaAddrSpace());
1626
10.9k
      break;
1627
21.2k
    }
1628
21.2k
1629
3.58M
    case ABIArgInfo::Extend:
1630
3.58M
    case ABIArgInfo::Direct: {
1631
3.58M
      // Fast-isel and the optimizer generally like scalar values better than
1632
3.58M
      // FCAs, so we flatten them if this is safe to do for this argument.
1633
3.58M
      llvm::Type *argType = ArgInfo.getCoerceToType();
1634
3.58M
      llvm::StructType *st = dyn_cast<llvm::StructType>(argType);
1635
3.58M
      if (st && 
ArgInfo.isDirect()3.77k
&&
ArgInfo.getCanBeFlattened()3.77k
) {
1636
3.57k
        assert(NumIRArgs == st->getNumElements());
1637
10.8k
        for (unsigned i = 0, e = st->getNumElements(); i != e; 
++i7.26k
)
1638
7.26k
          ArgTypes[FirstIRArg + i] = st->getElementType(i);
1639
3.58M
      } else {
1640
3.58M
        assert(NumIRArgs == 1);
1641
3.58M
        ArgTypes[FirstIRArg] = argType;
1642
3.58M
      }
1643
3.58M
      break;
1644
3.58M
    }
1645
3.58M
1646
3.58M
    case ABIArgInfo::CoerceAndExpand: {
1647
1.42k
      auto ArgTypesIter = ArgTypes.begin() + FirstIRArg;
1648
3.06k
      for (auto EltTy : ArgInfo.getCoerceAndExpandTypeSequence()) {
1649
3.06k
        *ArgTypesIter++ = EltTy;
1650
3.06k
      }
1651
1.42k
      assert(ArgTypesIter == ArgTypes.begin() + FirstIRArg + NumIRArgs);
1652
1.42k
      break;
1653
3.58M
    }
1654
3.58M
1655
3.58M
    case ABIArgInfo::Expand:
1656
160
      auto ArgTypesIter = ArgTypes.begin() + FirstIRArg;
1657
160
      getExpandedTypes(it->type, ArgTypesIter);
1658
160
      assert(ArgTypesIter == ArgTypes.begin() + FirstIRArg + NumIRArgs);
1659
160
      break;
1660
3.61M
    }
1661
3.61M
  }
1662
2.07M
1663
2.07M
  bool Erased = FunctionsBeingProcessed.erase(&FI); (void)Erased;
1664
2.07M
  assert(Erased && "Not in set?");
1665
2.07M
1666
2.07M
  return llvm::FunctionType::get(resultType, ArgTypes, FI.isVariadic());
1667
2.07M
}
1668
1669
20.7k
llvm::Type *CodeGenTypes::GetFunctionTypeForVTable(GlobalDecl GD) {
1670
20.7k
  const CXXMethodDecl *MD = cast<CXXMethodDecl>(GD.getDecl());
1671
20.7k
  const FunctionProtoType *FPT = MD->getType()->getAs<FunctionProtoType>();
1672
20.7k
1673
20.7k
  if (!isFuncTypeConvertible(FPT))
1674
39
    return llvm::StructType::get(getLLVMContext());
1675
20.7k
1676
20.7k
  return GetFunctionType(GD);
1677
20.7k
}
1678
1679
static void AddAttributesFromFunctionProtoType(ASTContext &Ctx,
1680
                                               llvm::AttrBuilder &FuncAttrs,
1681
2.70M
                                               const FunctionProtoType *FPT) {
1682
2.70M
  if (!FPT)
1683
1.38M
    return;
1684
1.31M
1685
1.31M
  if (!isUnresolvedExceptionSpec(FPT->getExceptionSpecType()) &&
1686
1.31M
      
FPT->isNothrow()1.31M
)
1687
254k
    FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
1688
1.31M
}
1689
1690
void CodeGenModule::ConstructDefaultFnAttrList(StringRef Name, bool HasOptnone,
1691
                                               bool AttrOnCallSite,
1692
1.36M
                                               llvm::AttrBuilder &FuncAttrs) {
1693
1.36M
  // OptimizeNoneAttr takes precedence over -Os or -Oz. No warning needed.
1694
1.36M
  if (!HasOptnone) {
1695
1.36M
    if (CodeGenOpts.OptimizeSize)
1696
8.08k
      FuncAttrs.addAttribute(llvm::Attribute::OptimizeForSize);
1697
1.36M
    if (CodeGenOpts.OptimizeSize == 2)
1698
7.71k
      FuncAttrs.addAttribute(llvm::Attribute::MinSize);
1699
1.36M
  }
1700
1.36M
1701
1.36M
  if (CodeGenOpts.DisableRedZone)
1702
2.62k
    FuncAttrs.addAttribute(llvm::Attribute::NoRedZone);
1703
1.36M
  if (CodeGenOpts.IndirectTlsSegRefs)
1704
1
    FuncAttrs.addAttribute("indirect-tls-seg-refs");
1705
1.36M
  if (CodeGenOpts.NoImplicitFloat)
1706
1.77k
    FuncAttrs.addAttribute(llvm::Attribute::NoImplicitFloat);
1707
1.36M
1708
1.36M
  if (AttrOnCallSite) {
1709
905k
    // Attributes that should go on the call site only.
1710
905k
    if (!CodeGenOpts.SimplifyLibCalls ||
1711
905k
        
CodeGenOpts.isNoBuiltinFunc(Name.data())531k
)
1712
373k
      FuncAttrs.addAttribute(llvm::Attribute::NoBuiltin);
1713
905k
    if (!CodeGenOpts.TrapFuncName.empty())
1714
0
      FuncAttrs.addAttribute("trap-func-name", CodeGenOpts.TrapFuncName);
1715
905k
  } else {
1716
464k
    StringRef FpKind;
1717
464k
    switch (CodeGenOpts.getFramePointer()) {
1718
464k
    case CodeGenOptions::FramePointerKind::None:
1719
145k
      FpKind = "none";
1720
145k
      break;
1721
464k
    case CodeGenOptions::FramePointerKind::NonLeaf:
1722
39
      FpKind = "non-leaf";
1723
39
      break;
1724
464k
    case CodeGenOptions::FramePointerKind::All:
1725
318k
      FpKind = "all";
1726
318k
      break;
1727
464k
    }
1728
464k
    FuncAttrs.addAttribute("frame-pointer", FpKind);
1729
464k
1730
464k
    FuncAttrs.addAttribute("less-precise-fpmad",
1731
464k
                           llvm::toStringRef(CodeGenOpts.LessPreciseFPMAD));
1732
464k
1733
464k
    if (CodeGenOpts.NullPointerIsValid)
1734
212
      FuncAttrs.addAttribute("null-pointer-is-valid", "true");
1735
464k
    if (!CodeGenOpts.FPDenormalMode.empty())
1736
3
      FuncAttrs.addAttribute("denormal-fp-math", CodeGenOpts.FPDenormalMode);
1737
464k
1738
464k
    FuncAttrs.addAttribute("no-trapping-math",
1739
464k
                           llvm::toStringRef(CodeGenOpts.NoTrappingMath));
1740
464k
1741
464k
    // Strict (compliant) code is the default, so only add this attribute to
1742
464k
    // indicate that we are trying to workaround a problem case.
1743
464k
    if (!CodeGenOpts.StrictFloatCastOverflow)
1744
1
      FuncAttrs.addAttribute("strict-float-cast-overflow", "false");
1745
464k
1746
464k
    // TODO: Are these all needed?
1747
464k
    // unsafe/inf/nan/nsz are handled by instruction-level FastMathFlags.
1748
464k
    FuncAttrs.addAttribute("no-infs-fp-math",
1749
464k
                           llvm::toStringRef(CodeGenOpts.NoInfsFPMath));
1750
464k
    FuncAttrs.addAttribute("no-nans-fp-math",
1751
464k
                           llvm::toStringRef(CodeGenOpts.NoNaNsFPMath));
1752
464k
    FuncAttrs.addAttribute("unsafe-fp-math",
1753
464k
                           llvm::toStringRef(CodeGenOpts.UnsafeFPMath));
1754
464k
    FuncAttrs.addAttribute("use-soft-float",
1755
464k
                           llvm::toStringRef(CodeGenOpts.SoftFloat));
1756
464k
    FuncAttrs.addAttribute("stack-protector-buffer-size",
1757
464k
                           llvm::utostr(CodeGenOpts.SSPBufferSize));
1758
464k
    FuncAttrs.addAttribute("no-signed-zeros-fp-math",
1759
464k
                           llvm::toStringRef(CodeGenOpts.NoSignedZeros));
1760
464k
    FuncAttrs.addAttribute(
1761
464k
        "correctly-rounded-divide-sqrt-fp-math",
1762
464k
        llvm::toStringRef(CodeGenOpts.CorrectlyRoundedDivSqrt));
1763
464k
1764
464k
    if (getLangOpts().OpenCL)
1765
1.66k
      FuncAttrs.addAttribute("denorms-are-zero",
1766
1.66k
                             llvm::toStringRef(CodeGenOpts.FlushDenorm));
1767
464k
1768
464k
    // TODO: Reciprocal estimate codegen options should apply to instructions?
1769
464k
    const std::vector<std::string> &Recips = CodeGenOpts.Reciprocals;
1770
464k
    if (!Recips.empty())
1771
1
      FuncAttrs.addAttribute("reciprocal-estimates",
1772
1
                             llvm::join(Recips, ","));
1773
464k
1774
464k
    if (!CodeGenOpts.PreferVectorWidth.empty() &&
1775
464k
        
CodeGenOpts.PreferVectorWidth != "none"3
)
1776
2
      FuncAttrs.addAttribute("prefer-vector-width",
1777
2
                             CodeGenOpts.PreferVectorWidth);
1778
464k
1779
464k
    if (CodeGenOpts.StackRealignment)
1780
1
      FuncAttrs.addAttribute("stackrealign");
1781
464k
    if (CodeGenOpts.Backchain)
1782
2
      FuncAttrs.addAttribute("backchain");
1783
464k
1784
464k
    if (CodeGenOpts.SpeculativeLoadHardening)
1785
9
      FuncAttrs.addAttribute(llvm::Attribute::SpeculativeLoadHardening);
1786
464k
  }
1787
1.36M
1788
1.36M
  if (getLangOpts().assumeFunctionsAreConvergent()) {
1789
2.84k
    // Conservatively, mark all functions and calls in CUDA and OpenCL as
1790
2.84k
    // convergent (meaning, they may call an intrinsically convergent op, such
1791
2.84k
    // as __syncthreads() / barrier(), and so can't have certain optimizations
1792
2.84k
    // applied around them).  LLVM will remove this attribute where it safely
1793
2.84k
    // can.
1794
2.84k
    FuncAttrs.addAttribute(llvm::Attribute::Convergent);
1795
2.84k
  }
1796
1.36M
1797
1.36M
  if (getLangOpts().CUDA && 
getLangOpts().CUDAIsDevice989
) {
1798
640
    // Exceptions aren't supported in CUDA device code.
1799
640
    FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
1800
640
1801
640
    // Respect -fcuda-flush-denormals-to-zero.
1802
640
    if (CodeGenOpts.FlushDenorm)
1803
10
      FuncAttrs.addAttribute("nvptx-f32ftz", "true");
1804
640
  }
1805
1.36M
1806
1.36M
  for (StringRef Attr : CodeGenOpts.DefaultFunctionAttrs) {
1807
5
    StringRef Var, Value;
1808
5
    std::tie(Var, Value) = Attr.split('=');
1809
5
    FuncAttrs.addAttribute(Var, Value);
1810
5
  }
1811
1.36M
}
1812
1813
22
void CodeGenModule::AddDefaultFnAttrs(llvm::Function &F) {
1814
22
  llvm::AttrBuilder FuncAttrs;
1815
22
  ConstructDefaultFnAttrList(F.getName(), F.hasOptNone(),
1816
22
                             /* AttrOnCallSite = */ false, FuncAttrs);
1817
22
  F.addAttributes(llvm::AttributeList::FunctionIndex, FuncAttrs);
1818
22
}
1819
1820
void CodeGenModule::ConstructAttributeList(
1821
    StringRef Name, const CGFunctionInfo &FI, CGCalleeInfo CalleeInfo,
1822
1.36M
    llvm::AttributeList &AttrList, unsigned &CallingConv, bool AttrOnCallSite) {
1823
1.36M
  llvm::AttrBuilder FuncAttrs;
1824
1.36M
  llvm::AttrBuilder RetAttrs;
1825
1.36M
1826
1.36M
  CallingConv = FI.getEffectiveCallingConvention();
1827
1.36M
  if (FI.isNoReturn())
1828
24.7k
    FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
1829
1.36M
1830
1.36M
  // If we have information about the function prototype, we can learn
1831
1.36M
  // attributes from there.
1832
1.36M
  AddAttributesFromFunctionProtoType(getContext(), FuncAttrs,
1833
1.36M
                                     CalleeInfo.getCalleeFunctionProtoType());
1834
1.36M
1835
1.36M
  const Decl *TargetDecl = CalleeInfo.getCalleeDecl().getDecl();
1836
1.36M
1837
1.36M
  bool HasOptnone = false;
1838
1.36M
  // FIXME: handle sseregparm someday...
1839
1.36M
  if (TargetDecl) {
1840
1.34M
    if (TargetDecl->hasAttr<ReturnsTwiceAttr>())
1841
55
      FuncAttrs.addAttribute(llvm::Attribute::ReturnsTwice);
1842
1.34M
    if (TargetDecl->hasAttr<NoThrowAttr>())
1843
9.03k
      FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
1844
1.34M
    if (TargetDecl->hasAttr<NoReturnAttr>())
1845
0
      FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
1846
1.34M
    if (TargetDecl->hasAttr<ColdAttr>())
1847
16
      FuncAttrs.addAttribute(llvm::Attribute::Cold);
1848
1.34M
    if (TargetDecl->hasAttr<NoDuplicateAttr>())
1849
6
      FuncAttrs.addAttribute(llvm::Attribute::NoDuplicate);
1850
1.34M
    if (TargetDecl->hasAttr<ConvergentAttr>())
1851
6
      FuncAttrs.addAttribute(llvm::Attribute::Convergent);
1852
1.34M
1853
1.34M
    if (const FunctionDecl *Fn = dyn_cast<FunctionDecl>(TargetDecl)) {
1854
1.33M
      AddAttributesFromFunctionProtoType(
1855
1.33M
          getContext(), FuncAttrs, Fn->getType()->getAs<FunctionProtoType>());
1856
1.33M
      // Don't use [[noreturn]] or _Noreturn for a call to a virtual function.
1857
1.33M
      // These attributes are not inherited by overloads.
1858
1.33M
      const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Fn);
1859
1.33M
      if (Fn->isNoReturn() && 
!(27.3k
AttrOnCallSite27.3k
&&
MD23.1k
&&
MD->isVirtual()985
))
1860
27.3k
        FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
1861
1.33M
    }
1862
1.34M
1863
1.34M
    // 'const', 'pure' and 'noalias' attributed functions are also nounwind.
1864
1.34M
    if (TargetDecl->hasAttr<ConstAttr>()) {
1865
2.00k
      FuncAttrs.addAttribute(llvm::Attribute::ReadNone);
1866
2.00k
      FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
1867
1.34M
    } else if (TargetDecl->hasAttr<PureAttr>()) {
1868
955
      FuncAttrs.addAttribute(llvm::Attribute::ReadOnly);
1869
955
      FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
1870
1.34M
    } else if (TargetDecl->hasAttr<NoAliasAttr>()) {
1871
2
      FuncAttrs.addAttribute(llvm::Attribute::ArgMemOnly);
1872
2
      FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
1873
2
    }
1874
1.34M
    if (TargetDecl->hasAttr<RestrictAttr>())
1875
14
      RetAttrs.addAttribute(llvm::Attribute::NoAlias);
1876
1.34M
    if (TargetDecl->hasAttr<ReturnsNonNullAttr>() &&
1877
1.34M
        
!CodeGenOpts.NullPointerIsValid6
)
1878
5
      RetAttrs.addAttribute(llvm::Attribute::NonNull);
1879
1.34M
    if (TargetDecl->hasAttr<AnyX86NoCallerSavedRegistersAttr>())
1880
14
      FuncAttrs.addAttribute("no_caller_saved_registers");
1881
1.34M
    if (TargetDecl->hasAttr<AnyX86NoCfCheckAttr>())
1882
2
      FuncAttrs.addAttribute(llvm::Attribute::NoCfCheck);
1883
1.34M
1884
1.34M
    HasOptnone = TargetDecl->hasAttr<OptimizeNoneAttr>();
1885
1.34M
    if (auto *AllocSize = TargetDecl->getAttr<AllocSizeAttr>()) {
1886
3.02k
      Optional<unsigned> NumElemsParam;
1887
3.02k
      if (AllocSize->getNumElemsParam().isValid())
1888
724
        NumElemsParam = AllocSize->getNumElemsParam().getLLVMIndex();
1889
3.02k
      FuncAttrs.addAllocSizeAttr(AllocSize->getElemSizeParam().getLLVMIndex(),
1890
3.02k
                                 NumElemsParam);
1891
3.02k
    }
1892
1.34M
  }
1893
1.36M
1894
1.36M
  ConstructDefaultFnAttrList(Name, HasOptnone, AttrOnCallSite, FuncAttrs);
1895
1.36M
1896
1.36M
  // This must run after constructing the default function attribute list
1897
1.36M
  // to ensure that the speculative load hardening attribute is removed
1898
1.36M
  // in the case where the -mspeculative-load-hardening flag was passed.
1899
1.36M
  if (TargetDecl) {
1900
1.34M
    if (TargetDecl->hasAttr<NoSpeculativeLoadHardeningAttr>())
1901
13
      FuncAttrs.removeAttribute(llvm::Attribute::SpeculativeLoadHardening);
1902
1.34M
    if (TargetDecl->hasAttr<SpeculativeLoadHardeningAttr>())
1903
13
      FuncAttrs.addAttribute(llvm::Attribute::SpeculativeLoadHardening);
1904
1.34M
  }
1905
1.36M
1906
1.36M
  if (CodeGenOpts.EnableSegmentedStacks &&
1907
1.36M
      
!(8
TargetDecl8
&&
TargetDecl->hasAttr<NoSplitStackAttr>()8
))
1908
4
    FuncAttrs.addAttribute("split-stack");
1909
1.36M
1910
1.36M
  // Add NonLazyBind attribute to function declarations when -fno-plt
1911
1.36M
  // is used.
1912
1.36M
  if (TargetDecl && 
CodeGenOpts.NoPLT1.34M
) {
1913
45
    if (auto *Fn = dyn_cast<FunctionDecl>(TargetDecl)) {
1914
45
      if (!Fn->isDefined() && 
!AttrOnCallSite22
) {
1915
12
        FuncAttrs.addAttribute(llvm::Attribute::NonLazyBind);
1916
12
      }
1917
45
    }
1918
45
  }
1919
1.36M
1920
1.36M
  if (TargetDecl && 
TargetDecl->hasAttr<OpenCLKernelAttr>()1.34M
) {
1921
419
    if (getLangOpts().OpenCLVersion <= 120) {
1922
294
      // OpenCL v1.2 Work groups are always uniform
1923
294
      FuncAttrs.addAttribute("uniform-work-group-size", "true");
1924
294
    } else {
1925
125
      // OpenCL v2.0 Work groups may be whether uniform or not.
1926
125
      // '-cl-uniform-work-group-size' compile option gets a hint
1927
125
      // to the compiler that the global work-size be a multiple of
1928
125
      // the work-group size specified to clEnqueueNDRangeKernel
1929
125
      // (i.e. work groups are uniform).
1930
125
      FuncAttrs.addAttribute("uniform-work-group-size",
1931
125
                             llvm::toStringRef(CodeGenOpts.UniformWGSize));
1932
125
    }
1933
419
  }
1934
1.36M
1935
1.36M
  if (!AttrOnCallSite) {
1936
464k
    bool DisableTailCalls = false;
1937
464k
1938
464k
    if (CodeGenOpts.DisableTailCalls)
1939
3
      DisableTailCalls = true;
1940
464k
    else if (TargetDecl) {
1941
451k
      if (TargetDecl->hasAttr<DisableTailCallsAttr>() ||
1942
451k
          
TargetDecl->hasAttr<AnyX86InterruptAttr>()451k
)
1943
45
        DisableTailCalls = true;
1944
451k
      else if (CodeGenOpts.NoEscapingBlockTailCalls) {
1945
17
        if (const auto *BD = dyn_cast<BlockDecl>(TargetDecl))
1946
8
          if (!BD->doesNotEscape())
1947
4
            DisableTailCalls = true;
1948
17
      }
1949
451k
    }
1950
464k
1951
464k
    FuncAttrs.addAttribute("disable-tail-calls",
1952
464k
                           llvm::toStringRef(DisableTailCalls));
1953
464k
    GetCPUAndFeaturesAttributes(CalleeInfo.getCalleeDecl(), FuncAttrs);
1954
464k
  }
1955
1.36M
1956
1.36M
  ClangToLLVMArgMapping IRFunctionArgs(getContext(), FI);
1957
1.36M
1958
1.36M
  QualType RetTy = FI.getReturnType();
1959
1.36M
  const ABIArgInfo &RetAI = FI.getReturnInfo();
1960
1.36M
  switch (RetAI.getKind()) {
1961
1.36M
  case ABIArgInfo::Extend:
1962
70.8k
    if (RetAI.isSignExt())
1963
10.2k
      RetAttrs.addAttribute(llvm::Attribute::SExt);
1964
60.6k
    else
1965
60.6k
      RetAttrs.addAttribute(llvm::Attribute::ZExt);
1966
70.8k
    LLVM_FALLTHROUGH;
1967
884k
  case ABIArgInfo::Direct:
1968
884k
    if (RetAI.getInReg())
1969
12
      RetAttrs.addAttribute(llvm::Attribute::InReg);
1970
884k
    break;
1971
472k
  case ABIArgInfo::Ignore:
1972
472k
    break;
1973
70.8k
1974
70.8k
  case ABIArgInfo::InAlloca:
1975
11.9k
  case ABIArgInfo::Indirect: {
1976
11.9k
    // inalloca and sret disable readnone and readonly
1977
11.9k
    FuncAttrs.removeAttribute(llvm::Attribute::ReadOnly)
1978
11.9k
      .removeAttribute(llvm::Attribute::ReadNone);
1979
11.9k
    break;
1980
11.9k
  }
1981
11.9k
1982
11.9k
  case ABIArgInfo::CoerceAndExpand:
1983
981
    break;
1984
11.9k
1985
11.9k
  case ABIArgInfo::Expand:
1986
0
    llvm_unreachable("Invalid ABI kind for return argument");
1987
1.36M
  }
1988
1.36M
1989
1.36M
  if (const auto *RefTy = RetTy->getAs<ReferenceType>()) {
1990
112k
    QualType PTy = RefTy->getPointeeType();
1991
112k
    if (!PTy->isIncompleteType() && 
PTy->isConstantSizeType()112k
)
1992
112k
      RetAttrs.addDereferenceableAttr(getContext().getTypeSizeInChars(PTy)
1993
112k
                                        .getQuantity());
1994
16
    else if (getContext().getTargetAddressSpace(PTy) == 0 &&
1995
16
             
!CodeGenOpts.NullPointerIsValid14
)
1996
11
      RetAttrs.addAttribute(llvm::Attribute::NonNull);
1997
112k
  }
1998
1.36M
1999
1.36M
  bool hasUsedSRet = false;
2000
1.36M
  SmallVector<llvm::AttributeSet, 4> ArgAttrs(IRFunctionArgs.totalIRArgs());
2001
1.36M
2002
1.36M
  // Attach attributes to sret.
2003
1.36M
  if (IRFunctionArgs.hasSRetArg()) {
2004
11.9k
    llvm::AttrBuilder SRETAttrs;
2005
11.9k
    SRETAttrs.addAttribute(llvm::Attribute::StructRet);
2006
11.9k
    hasUsedSRet = true;
2007
11.9k
    if (RetAI.getInReg())
2008
49
      SRETAttrs.addAttribute(llvm::Attribute::InReg);
2009
11.9k
    ArgAttrs[IRFunctionArgs.getSRetArgNo()] =
2010
11.9k
        llvm::AttributeSet::get(getLLVMContext(), SRETAttrs);
2011
11.9k
  }
2012
1.36M
2013
1.36M
  // Attach attributes to inalloca argument.
2014
1.36M
  if (IRFunctionArgs.hasInallocaArg()) {
2015
146
    llvm::AttrBuilder Attrs;
2016
146
    Attrs.addAttribute(llvm::Attribute::InAlloca);
2017
146
    ArgAttrs[IRFunctionArgs.getInallocaArgNo()] =
2018
146
        llvm::AttributeSet::get(getLLVMContext(), Attrs);
2019
146
  }
2020
1.36M
2021
1.36M
  unsigned ArgNo = 0;
2022
1.36M
  for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(),
2023
1.36M
                                          E = FI.arg_end();
2024
3.80M
       I != E; 
++I, ++ArgNo2.43M
) {
2025
2.43M
    QualType ParamType = I->type;
2026
2.43M
    const ABIArgInfo &AI = I->info;
2027
2.43M
    llvm::AttrBuilder Attrs;
2028
2.43M
2029
2.43M
    // Add attribute for padding argument, if necessary.
2030
2.43M
    if (IRFunctionArgs.hasPaddingArg(ArgNo)) {
2031
47
      if (AI.getPaddingInReg()) {
2032
5
        ArgAttrs[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
2033
5
            llvm::AttributeSet::get(
2034
5
                getLLVMContext(),
2035
5
                llvm::AttrBuilder().addAttribute(llvm::Attribute::InReg));
2036
5
      }
2037
47
    }
2038
2.43M
2039
2.43M
    // 'restrict' -> 'noalias' is done in EmitFunctionProlog when we
2040
2.43M
    // have the corresponding parameter variable.  It doesn't make
2041
2.43M
    // sense to do it here because parameters are so messed up.
2042
2.43M
    switch (AI.getKind()) {
2043
2.43M
    case ABIArgInfo::Extend:
2044
69.5k
      if (AI.isSignExt())
2045
19.0k
        Attrs.addAttribute(llvm::Attribute::SExt);
2046
50.5k
      else
2047
50.5k
        Attrs.addAttribute(llvm::Attribute::ZExt);
2048
69.5k
      LLVM_FALLTHROUGH;
2049
2.41M
    case ABIArgInfo::Direct:
2050
2.41M
      if (ArgNo == 0 && 
FI.isChainCall()1.21M
)
2051
8
        Attrs.addAttribute(llvm::Attribute::Nest);
2052
2.41M
      else if (AI.getInReg())
2053
475
        Attrs.addAttribute(llvm::Attribute::InReg);
2054
2.41M
      break;
2055
69.5k
2056
69.5k
    case ABIArgInfo::Indirect: {
2057
7.17k
      if (AI.getInReg())
2058
20
        Attrs.addAttribute(llvm::Attribute::InReg);
2059
7.17k
2060
7.17k
      if (AI.getIndirectByVal())
2061
2.48k
        Attrs.addByValAttr(getTypes().ConvertTypeForMem(ParamType));
2062
7.17k
2063
7.17k
      CharUnits Align = AI.getIndirectAlign();
2064
7.17k
2065
7.17k
      // In a byval argument, it is important that the required
2066
7.17k
      // alignment of the type is honored, as LLVM might be creating a
2067
7.17k
      // *new* stack object, and needs to know what alignment to give
2068
7.17k
      // it. (Sometimes it can deduce a sensible alignment on its own,
2069
7.17k
      // but not if clang decides it must emit a packed struct, or the
2070
7.17k
      // user specifies increased alignment requirements.)
2071
7.17k
      //
2072
7.17k
      // This is different from indirect *not* byval, where the object
2073
7.17k
      // exists already, and the align attribute is purely
2074
7.17k
      // informative.
2075
7.17k
      assert(!Align.isZero());
2076
7.17k
2077
7.17k
      // For now, only add this when we have a byval argument.
2078
7.17k
      // TODO: be less lazy about updating test cases.
2079
7.17k
      if (AI.getIndirectByVal())
2080
2.48k
        Attrs.addAlignmentAttr(Align.getQuantity());
2081
7.17k
2082
7.17k
      // byval disables readnone and readonly.
2083
7.17k
      FuncAttrs.removeAttribute(llvm::Attribute::ReadOnly)
2084
7.17k
        .removeAttribute(llvm::Attribute::ReadNone);
2085
7.17k
      break;
2086
69.5k
    }
2087
69.5k
    case ABIArgInfo::Ignore:
2088
12.0k
    case ABIArgInfo::Expand:
2089
12.0k
    case ABIArgInfo::CoerceAndExpand:
2090
12.0k
      break;
2091
12.0k
2092
12.0k
    case ABIArgInfo::InAlloca:
2093
290
      // inalloca disables readnone and readonly.
2094
290
      FuncAttrs.removeAttribute(llvm::Attribute::ReadOnly)
2095
290
          .removeAttribute(llvm::Attribute::ReadNone);
2096
290
      continue;
2097
2.43M
    }
2098
2.43M
2099
2.43M
    if (const auto *RefTy = ParamType->getAs<ReferenceType>()) {
2100
238k
      QualType PTy = RefTy->getPointeeType();
2101
238k
      if (!PTy->isIncompleteType() && 
PTy->isConstantSizeType()238k
)
2102
238k
        Attrs.addDereferenceableAttr(getContext().getTypeSizeInChars(PTy)
2103
238k
                                       .getQuantity());
2104
527
      else if (getContext().getTargetAddressSpace(PTy) == 0 &&
2105
527
               
!CodeGenOpts.NullPointerIsValid523
)
2106
514
        Attrs.addAttribute(llvm::Attribute::NonNull);
2107
238k
    }
2108
2.43M
2109
2.43M
    switch (FI.getExtParameterInfo(ArgNo).getABI()) {
2110
2.43M
    case ParameterABI::Ordinary:
2111
2.43M
      break;
2112
2.43M
2113
2.43M
    case ParameterABI::SwiftIndirectResult: {
2114
48
      // Add 'sret' if we haven't already used it for something, but
2115
48
      // only if the result is void.
2116
48
      if (!hasUsedSRet && 
RetTy->isVoidType()24
) {
2117
8
        Attrs.addAttribute(llvm::Attribute::StructRet);
2118
8
        hasUsedSRet = true;
2119
8
      }
2120
48
2121
48
      // Add 'noalias' in either case.
2122
48
      Attrs.addAttribute(llvm::Attribute::NoAlias);
2123
48
2124
48
      // Add 'dereferenceable' and 'alignment'.
2125
48
      auto PTy = ParamType->getPointeeType();
2126
48
      if (!PTy->isIncompleteType() && PTy->isConstantSizeType()) {
2127
48
        auto info = getContext().getTypeInfoInChars(PTy);
2128
48
        Attrs.addDereferenceableAttr(info.first.getQuantity());
2129
48
        Attrs.addAttribute(llvm::Attribute::getWithAlignment(getLLVMContext(),
2130
48
                                                 info.second.getQuantity()));
2131
48
      }
2132
48
      break;
2133
2.43M
    }
2134
2.43M
2135
2.43M
    case ParameterABI::SwiftErrorResult:
2136
24
      Attrs.addAttribute(llvm::Attribute::SwiftError);
2137
24
      break;
2138
2.43M
2139
2.43M
    case ParameterABI::SwiftContext:
2140
40
      Attrs.addAttribute(llvm::Attribute::SwiftSelf);
2141
40
      break;
2142
2.43M
    }
2143
2.43M
2144
2.43M
    if (FI.getExtParameterInfo(ArgNo).isNoEscape())
2145
162
      Attrs.addAttribute(llvm::Attribute::NoCapture);
2146
2.43M
2147
2.43M
    if (Attrs.hasAttributes()) {
2148
311k
      unsigned FirstIRArg, NumIRArgs;
2149
311k
      std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
2150
622k
      for (unsigned i = 0; i < NumIRArgs; 
i++311k
)
2151
311k
        ArgAttrs[FirstIRArg + i] =
2152
311k
            llvm::AttributeSet::get(getLLVMContext(), Attrs);
2153
311k
    }
2154
2.43M
  }
2155
1.36M
  assert(ArgNo == FI.arg_size());
2156
1.36M
2157
1.36M
  AttrList = llvm::AttributeList::get(
2158
1.36M
      getLLVMContext(), llvm::AttributeSet::get(getLLVMContext(), FuncAttrs),
2159
1.36M
      llvm::AttributeSet::get(getLLVMContext(), RetAttrs), ArgAttrs);
2160
1.36M
}
2161
2162
/// An argument came in as a promoted argument; demote it back to its
2163
/// declared type.
2164
static llvm::Value *emitArgumentDemotion(CodeGenFunction &CGF,
2165
                                         const VarDecl *var,
2166
47
                                         llvm::Value *value) {
2167
47
  llvm::Type *varType = CGF.ConvertType(var->getType());
2168
47
2169
47
  // This can happen with promotions that actually don't change the
2170
47
  // underlying type, like the enum promotions.
2171
47
  if (value->getType() == varType) 
return value11
;
2172
36
2173
36
  assert((varType->isIntegerTy() || varType->isFloatingPointTy())
2174
36
         && "unexpected promotion type");
2175
36
2176
36
  if (isa<llvm::IntegerType>(varType))
2177
21
    return CGF.Builder.CreateTrunc(value, varType, "arg.unpromote");
2178
15
2179
15
  return CGF.Builder.CreateFPCast(value, varType, "arg.unpromote");
2180
15
}
2181
2182
/// Returns the attribute (either parameter attribute, or function
2183
/// attribute), which declares argument ArgNo to be non-null.
2184
static const NonNullAttr *getNonNullAttr(const Decl *FD, const ParmVarDecl *PVD,
2185
410k
                                         QualType ArgType, unsigned ArgNo) {
2186
410k
  // FIXME: __attribute__((nonnull)) can also be applied to:
2187
410k
  //   - references to pointers, where the pointee is known to be
2188
410k
  //     nonnull (apparently a Clang extension)
2189
410k
  //   - transparent unions containing pointers
2190
410k
  // In the former case, LLVM IR cannot represent the constraint. In
2191
410k
  // the latter case, we have no guarantee that the transparent union
2192
410k
  // is in fact passed as a pointer.
2193
410k
  if (!ArgType->isAnyPointerType() && 
!ArgType->isBlockPointerType()282k
)
2194
282k
    return nullptr;
2195
128k
  // First, check attribute on parameter itself.
2196
128k
  if (PVD) {
2197
128k
    if (auto ParmNNAttr = PVD->getAttr<NonNullAttr>())
2198
8
      return ParmNNAttr;
2199
128k
  }
2200
128k
  // Check function attributes.
2201
128k
  if (!FD)
2202
277
    return nullptr;
2203
127k
  for (const auto *NNAttr : FD->specific_attrs<NonNullAttr>()) {
2204
102
    if (NNAttr->isNonNull(ArgNo))
2205
56
      return NNAttr;
2206
102
  }
2207
127k
  
return nullptr127k
;
2208
127k
}
2209
2210
namespace {
2211
  struct CopyBackSwiftError final : EHScopeStack::Cleanup {
2212
    Address Temp;
2213
    Address Arg;
2214
16
    CopyBackSwiftError(Address temp, Address arg) : Temp(temp), Arg(arg) {}
2215
16
    void Emit(CodeGenFunction &CGF, Flags flags) override {
2216
16
      llvm::Value *errorValue = CGF.Builder.CreateLoad(Temp);
2217
16
      CGF.Builder.CreateStore(errorValue, Arg);
2218
16
    }
2219
  };
2220
}
2221
2222
void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
2223
                                         llvm::Function *Fn,
2224
386k
                                         const FunctionArgList &Args) {
2225
386k
  if (CurCodeDecl && 
CurCodeDecl->hasAttr<NakedAttr>()378k
)
2226
7
    // Naked functions don't have prologues.
2227
7
    return;
2228
386k
2229
386k
  // If this is an implicit-return-zero function, go ahead and
2230
386k
  // initialize the return value.  TODO: it might be nice to have
2231
386k
  // a more general mechanism for this that didn't require synthesized
2232
386k
  // return statements.
2233
386k
  if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(CurCodeDecl)) {
2234
356k
    if (FD->hasImplicitReturnZero()) {
2235
2.71k
      QualType RetTy = FD->getReturnType().getUnqualifiedType();
2236
2.71k
      llvm::Type* LLVMTy = CGM.getTypes().ConvertType(RetTy);
2237
2.71k
      llvm::Constant* Zero = llvm::Constant::getNullValue(LLVMTy);
2238
2.71k
      Builder.CreateStore(Zero, ReturnValue);
2239
2.71k
    }
2240
356k
  }
2241
386k
2242
386k
  // FIXME: We no longer need the types from FunctionArgList; lift up and
2243
386k
  // simplify.
2244
386k
2245
386k
  ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), FI);
2246
386k
  // Flattened function arguments.
2247
386k
  SmallVector<llvm::Value *, 16> FnArgs;
2248
386k
  FnArgs.reserve(IRFunctionArgs.totalIRArgs());
2249
658k
  for (auto &Arg : Fn->args()) {
2250
658k
    FnArgs.push_back(&Arg);
2251
658k
  }
2252
386k
  assert(FnArgs.size() == IRFunctionArgs.totalIRArgs());
2253
386k
2254
386k
  // If we're using inalloca, all the memory arguments are GEPs off of the last
2255
386k
  // parameter, which is a pointer to the complete memory area.
2256
386k
  Address ArgStruct = Address::invalid();
2257
386k
  if (IRFunctionArgs.hasInallocaArg()) {
2258
35
    ArgStruct = Address(FnArgs[IRFunctionArgs.getInallocaArgNo()],
2259
35
                        FI.getArgStructAlignment());
2260
35
2261
35
    assert(ArgStruct.getType() == FI.getArgStruct()->getPointerTo());
2262
35
  }
2263
386k
2264
386k
  // Name the struct return parameter.
2265
386k
  if (IRFunctionArgs.hasSRetArg()) {
2266
4.25k
    auto AI = cast<llvm::Argument>(FnArgs[IRFunctionArgs.getSRetArgNo()]);
2267
4.25k
    AI->setName("agg.result");
2268
4.25k
    AI->addAttr(llvm::Attribute::NoAlias);
2269
4.25k
  }
2270
386k
2271
386k
  // Track if we received the parameter as a pointer (indirect, byval, or
2272
386k
  // inalloca).  If already have a pointer, EmitParmDecl doesn't need to copy it
2273
386k
  // into a local alloca for us.
2274
386k
  SmallVector<ParamValue, 16> ArgVals;
2275
386k
  ArgVals.reserve(Args.size());
2276
386k
2277
386k
  // Create a pointer value for every parameter declaration.  This usually
2278
386k
  // entails copying one or more LLVM IR arguments into an alloca.  Don't push
2279
386k
  // any cleanups or do anything that might unwind.  We do that separately, so
2280
386k
  // we can push the cleanups in the correct order for the ABI.
2281
386k
  assert(FI.arg_size() == Args.size() &&
2282
386k
         "Mismatch between function signature & arguments.");
2283
386k
  unsigned ArgNo = 0;
2284
386k
  CGFunctionInfo::const_arg_iterator info_it = FI.arg_begin();
2285
386k
  for (FunctionArgList::const_iterator i = Args.begin(), e = Args.end();
2286
1.04M
       i != e; 
++i, ++info_it, ++ArgNo657k
) {
2287
657k
    const VarDecl *Arg = *i;
2288
657k
    const ABIArgInfo &ArgI = info_it->info;
2289
657k
2290
657k
    bool isPromoted =
2291
657k
      isa<ParmVarDecl>(Arg) && 
cast<ParmVarDecl>(Arg)->isKNRPromoted()427k
;
2292
657k
    // We are converting from ABIArgInfo type to VarDecl type directly, unless
2293
657k
    // the parameter is promoted. In this case we convert to
2294
657k
    // CGFunctionInfo::ArgInfo type with subsequent argument demotion.
2295
657k
    QualType Ty = isPromoted ? 
info_it->type47
:
Arg->getType()657k
;
2296
657k
    assert(hasScalarEvaluationKind(Ty) ==
2297
657k
           hasScalarEvaluationKind(Arg->getType()));
2298
657k
2299
657k
    unsigned FirstIRArg, NumIRArgs;
2300
657k
    std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
2301
657k
2302
657k
    switch (ArgI.getKind()) {
2303
657k
    case ABIArgInfo::InAlloca: {
2304
61
      assert(NumIRArgs == 0);
2305
61
      auto FieldIndex = ArgI.getInAllocaFieldIndex();
2306
61
      Address V =
2307
61
          Builder.CreateStructGEP(ArgStruct, FieldIndex, Arg->getName());
2308
61
      ArgVals.push_back(ParamValue::forIndirect(V));
2309
61
      break;
2310
657k
    }
2311
657k
2312
657k
    case ABIArgInfo::Indirect: {
2313
2.66k
      assert(NumIRArgs == 1);
2314
2.66k
      Address ParamAddr = Address(FnArgs[FirstIRArg], ArgI.getIndirectAlign());
2315
2.66k
2316
2.66k
      if (!hasScalarEvaluationKind(Ty)) {
2317
2.49k
        // Aggregates and complex variables are accessed by reference.  All we
2318
2.49k
        // need to do is realign the value, if requested.
2319
2.49k
        Address V = ParamAddr;
2320
2.49k
        if (ArgI.getIndirectRealign()) {
2321
29
          Address AlignedTemp = CreateMemTemp(Ty, "coerce");
2322
29
2323
29
          // Copy from the incoming argument pointer to the temporary with the
2324
29
          // appropriate alignment.
2325
29
          //
2326
29
          // FIXME: We should have a common utility for generating an aggregate
2327
29
          // copy.
2328
29
          CharUnits Size = getContext().getTypeSizeInChars(Ty);
2329
29
          auto SizeVal = llvm::ConstantInt::get(IntPtrTy, Size.getQuantity());
2330
29
          Address Dst = Builder.CreateBitCast(AlignedTemp, Int8PtrTy);
2331
29
          Address Src = Builder.CreateBitCast(ParamAddr, Int8PtrTy);
2332
29
          Builder.CreateMemCpy(Dst, Src, SizeVal, false);
2333
29
          V = AlignedTemp;
2334
29
        }
2335
2.49k
        ArgVals.push_back(ParamValue::forIndirect(V));
2336
2.49k
      } else {
2337
171
        // Load scalar value from indirect argument.
2338
171
        llvm::Value *V =
2339
171
            EmitLoadOfScalar(ParamAddr, false, Ty, Arg->getBeginLoc());
2340
171
2341
171
        if (isPromoted)
2342
0
          V = emitArgumentDemotion(*this, Arg, V);
2343
171
        ArgVals.push_back(ParamValue::forDirect(V));
2344
171
      }
2345
2.66k
      break;
2346
657k
    }
2347
657k
2348
657k
    case ABIArgInfo::Extend:
2349
648k
    case ABIArgInfo::Direct: {
2350
648k
2351
648k
      // If we have the trivial case, handle it with no muss and fuss.
2352
648k
      if (!isa<llvm::StructType>(ArgI.getCoerceToType()) &&
2353
648k
          
ArgI.getCoerceToType() == ConvertType(Ty)647k
&&
2354
648k
          
ArgI.getDirectOffset() == 0641k
) {
2355
641k
        assert(NumIRArgs == 1);
2356
641k
        llvm::Value *V = FnArgs[FirstIRArg];
2357
641k
        auto AI = cast<llvm::Argument>(V);
2358
641k
2359
641k
        if (const ParmVarDecl *PVD = dyn_cast<ParmVarDecl>(Arg)) {
2360
410k
          if (getNonNullAttr(CurCodeDecl, PVD, PVD->getType(),
2361
410k
                             PVD->getFunctionScopeIndex()) &&
2362
410k
              
!CGM.getCodeGenOpts().NullPointerIsValid49
)
2363
40
            AI->addAttr(llvm::Attribute::NonNull);
2364
410k
2365
410k
          QualType OTy = PVD->getOriginalType();
2366
410k
          if (const auto *ArrTy =
2367
1.95k
              getContext().getAsConstantArrayType(OTy)) {
2368
1.95k
            // A C99 array parameter declaration with the static keyword also
2369
1.95k
            // indicates dereferenceability, and if the size is constant we can
2370
1.95k
            // use the dereferenceable attribute (which requires the size in
2371
1.95k
            // bytes).
2372
1.95k
            if (ArrTy->getSizeModifier() == ArrayType::Static) {
2373
4
              QualType ETy = ArrTy->getElementType();
2374
4
              uint64_t ArrSize = ArrTy->getSize().getZExtValue();
2375
4
              if (!ETy->isIncompleteType() && ETy->isConstantSizeType() &&
2376
4
                  ArrSize) {
2377
4
                llvm::AttrBuilder Attrs;
2378
4
                Attrs.addDereferenceableAttr(
2379
4
                  getContext().getTypeSizeInChars(ETy).getQuantity()*ArrSize);
2380
4
                AI->addAttrs(Attrs);
2381
4
              } else 
if (0
getContext().getTargetAddressSpace(ETy) == 00
&&
2382
0
                         !CGM.getCodeGenOpts().NullPointerIsValid) {
2383
0
                AI->addAttr(llvm::Attribute::NonNull);
2384
0
              }
2385
4
            }
2386
408k
          } else if (const auto *ArrTy =
2387
71
                     getContext().getAsVariableArrayType(OTy)) {
2388
71
            // For C99 VLAs with the static keyword, we don't know the size so
2389
71
            // we can't use the dereferenceable attribute, but in addrspace(0)
2390
71
            // we know that it must be nonnull.
2391
71
            if (ArrTy->getSizeModifier() == VariableArrayType::Static &&
2392
71
                
!getContext().getTargetAddressSpace(ArrTy->getElementType())12
&&
2393
71
                
!CGM.getCodeGenOpts().NullPointerIsValid12
)
2394
11
              AI->addAttr(llvm::Attribute::NonNull);
2395
71
          }
2396
410k
2397
410k
          const auto *AVAttr = PVD->getAttr<AlignValueAttr>();
2398
410k
          if (!AVAttr)
2399
410k
            if (const auto *TOTy = dyn_cast<TypedefType>(OTy))
2400
147k
              AVAttr = TOTy->getDecl()->getAttr<AlignValueAttr>();
2401
410k
          if (AVAttr && 
!SanOpts.has(SanitizerKind::Alignment)13
) {
2402
10
            // If alignment-assumption sanitizer is enabled, we do *not* add
2403
10
            // alignment attribute here, but emit normal alignment assumption,
2404
10
            // so the UBSAN check could function.
2405
10
            llvm::Value *AlignmentValue =
2406
10
              EmitScalarExpr(AVAttr->getAlignment());
2407
10
            llvm::ConstantInt *AlignmentCI =
2408
10
              cast<llvm::ConstantInt>(AlignmentValue);
2409
10
            unsigned Alignment = std::min((unsigned)AlignmentCI->getZExtValue(),
2410
10
                                          +llvm::Value::MaximumAlignment);
2411
10
            AI->addAttrs(llvm::AttrBuilder().addAlignmentAttr(Alignment));
2412
10
          }
2413
410k
        }
2414
641k
2415
641k
        if (Arg->getType().isRestrictQualified())
2416
18.6k
          AI->addAttr(llvm::Attribute::NoAlias);
2417
641k
2418
641k
        // LLVM expects swifterror parameters to be used in very restricted
2419
641k
        // ways.  Copy the value into a less-restricted temporary.
2420
641k
        if (FI.getExtParameterInfo(ArgNo).getABI()
2421
641k
              == ParameterABI::SwiftErrorResult) {
2422
16
          QualType pointeeTy = Ty->getPointeeType();
2423
16
          assert(pointeeTy->isPointerType());
2424
16
          Address temp =
2425
16
            CreateMemTemp(pointeeTy, getPointerAlign(), "swifterror.temp");
2426
16
          Address arg = Address(V, getContext().getTypeAlignInChars(pointeeTy));
2427
16
          llvm::Value *incomingErrorValue = Builder.CreateLoad(arg);
2428
16
          Builder.CreateStore(incomingErrorValue, temp);
2429
16
          V = temp.getPointer();
2430
16
2431
16
          // Push a cleanup to copy the value back at the end of the function.
2432
16
          // The convention does not guarantee that the value will be written
2433
16
          // back if the function exits with an unwind exception.
2434
16
          EHStack.pushCleanup<CopyBackSwiftError>(NormalCleanup, temp, arg);
2435
16
        }
2436
641k
2437
641k
        // Ensure the argument is the correct type.
2438
641k
        if (V->getType() != ArgI.getCoerceToType())
2439
0
          V = Builder.CreateBitCast(V, ArgI.getCoerceToType());
2440
641k
2441
641k
        if (isPromoted)
2442
47
          V = emitArgumentDemotion(*this, Arg, V);
2443
641k
2444
641k
        // Because of merging of function types from multiple decls it is
2445
641k
        // possible for the type of an argument to not match the corresponding
2446
641k
        // type in the function type. Since we are codegening the callee
2447
641k
        // in here, add a cast to the argument type.
2448
641k
        llvm::Type *LTy = ConvertType(Arg->getType());
2449
641k
        if (V->getType() != LTy)
2450
0
          V = Builder.CreateBitCast(V, LTy);
2451
641k
2452
641k
        ArgVals.push_back(ParamValue::forDirect(V));
2453
641k
        break;
2454
641k
      }
2455
7.83k
2456
7.83k
      Address Alloca = CreateMemTemp(Ty, getContext().getDeclAlign(Arg),
2457
7.83k
                                     Arg->getName());
2458
7.83k
2459
7.83k
      // Pointer to store into.
2460
7.83k
      Address Ptr = emitAddressAtOffset(*this, Alloca, ArgI);
2461
7.83k
2462
7.83k
      // Fast-isel and the optimizer generally like scalar values better than
2463
7.83k
      // FCAs, so we flatten them if this is safe to do for this argument.
2464
7.83k
      llvm::StructType *STy = dyn_cast<llvm::StructType>(ArgI.getCoerceToType());
2465
7.83k
      if (ArgI.isDirect() && ArgI.getCanBeFlattened() && 
STy7.61k
&&
2466
7.83k
          
STy->getNumElements() > 11.06k
) {
2467
1.03k
        uint64_t SrcSize = CGM.getDataLayout().getTypeAllocSize(STy);
2468
1.03k
        llvm::Type *DstTy = Ptr.getElementType();
2469
1.03k
        uint64_t DstSize = CGM.getDataLayout().getTypeAllocSize(DstTy);
2470
1.03k
2471
1.03k
        Address AddrToStoreInto = Address::invalid();
2472
1.03k
        if (SrcSize <= DstSize) {
2473
1.00k
          AddrToStoreInto = Builder.CreateElementBitCast(Ptr, STy);
2474
1.00k
        } else {
2475
27
          AddrToStoreInto =
2476
27
            CreateTempAlloca(STy, Alloca.getAlignment(), "coerce");
2477
27
        }
2478
1.03k
2479
1.03k
        assert(STy->getNumElements() == NumIRArgs);
2480
3.21k
        for (unsigned i = 0, e = STy->getNumElements(); i != e; 
++i2.17k
) {
2481
2.17k
          auto AI = FnArgs[FirstIRArg + i];
2482
2.17k
          AI->setName(Arg->getName() + ".coerce" + Twine(i));
2483
2.17k
          Address EltPtr = Builder.CreateStructGEP(AddrToStoreInto, i);
2484
2.17k
          Builder.CreateStore(AI, EltPtr);
2485
2.17k
        }
2486
1.03k
2487
1.03k
        if (SrcSize > DstSize) {
2488
27
          Builder.CreateMemCpy(Ptr, AddrToStoreInto, DstSize);
2489
27
        }
2490
1.03k
2491
6.79k
      } else {
2492
6.79k
        // Simple case, just do a coerced store of the argument into the alloca.
2493
6.79k
        assert(NumIRArgs == 1);
2494
6.79k
        auto AI = FnArgs[FirstIRArg];
2495
6.79k
        AI->setName(Arg->getName() + ".coerce");
2496
6.79k
        CreateCoercedStore(AI, Ptr, /*DstIsVolatile=*/false, *this);
2497
6.79k
      }
2498
7.83k
2499
7.83k
      // Match to what EmitParmDecl is expecting for this type.
2500
7.83k
      if (CodeGenFunction::hasScalarEvaluationKind(Ty)) {
2501
1.68k
        llvm::Value *V =
2502
1.68k
            EmitLoadOfScalar(Alloca, false, Ty, Arg->getBeginLoc());
2503
1.68k
        if (isPromoted)
2504
0
          V = emitArgumentDemotion(*this, Arg, V);
2505
1.68k
        ArgVals.push_back(ParamValue::forDirect(V));
2506
6.14k
      } else {
2507
6.14k
        ArgVals.push_back(ParamValue::forIndirect(Alloca));
2508
6.14k
      }
2509
7.83k
      break;
2510
7.83k
    }
2511
7.83k
2512
7.83k
    case ABIArgInfo::CoerceAndExpand: {
2513
540
      // Reconstruct into a temporary.
2514
540
      Address alloca = CreateMemTemp(Ty, getContext().getDeclAlign(Arg));
2515
540
      ArgVals.push_back(ParamValue::forIndirect(alloca));
2516
540
2517
540
      auto coercionType = ArgI.getCoerceAndExpandType();
2518
540
      alloca = Builder.CreateElementBitCast(alloca, coercionType);
2519
540
2520
540
      unsigned argIndex = FirstIRArg;
2521
1.68k
      for (unsigned i = 0, e = coercionType->getNumElements(); i != e; 
++i1.14k
) {
2522
1.14k
        llvm::Type *eltType = coercionType->getElementType(i);
2523
1.14k
        if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType))
2524
7
          continue;
2525
1.13k
2526
1.13k
        auto eltAddr = Builder.CreateStructGEP(alloca, i);
2527
1.13k
        auto elt = FnArgs[argIndex++];
2528
1.13k
        Builder.CreateStore(elt, eltAddr);
2529
1.13k
      }
2530
540
      assert(argIndex == FirstIRArg + NumIRArgs);
2531
540
      break;
2532
7.83k
    }
2533
7.83k
2534
7.83k
    case ABIArgInfo::Expand: {
2535
97
      // If this structure was expanded into multiple arguments then
2536
97
      // we need to create a temporary and reconstruct it from the
2537
97
      // arguments.
2538
97
      Address Alloca = CreateMemTemp(Ty, getContext().getDeclAlign(Arg));
2539
97
      LValue LV = MakeAddrLValue(Alloca, Ty);
2540
97
      ArgVals.push_back(ParamValue::forIndirect(Alloca));
2541
97
2542
97
      auto FnArgIter = FnArgs.begin() + FirstIRArg;
2543
97
      ExpandTypeFromArgs(Ty, LV, FnArgIter);
2544
97
      assert(FnArgIter == FnArgs.begin() + FirstIRArg + NumIRArgs);
2545
333
      for (unsigned i = 0, e = NumIRArgs; i != e; 
++i236
) {
2546
236
        auto AI = FnArgs[FirstIRArg + i];
2547
236
        AI->setName(Arg->getName() + "." + Twine(i));
2548
236
      }
2549
97
      break;
2550
7.83k
    }
2551
7.83k
2552
7.83k
    case ABIArgInfo::Ignore:
2553
5.29k
      assert(NumIRArgs == 0);
2554
5.29k
      // Initialize the local variable appropriately.
2555
5.29k
      if (!hasScalarEvaluationKind(Ty)) {
2556
5.29k
        ArgVals.push_back(ParamValue::forIndirect(CreateMemTemp(Ty)));
2557
5.29k
      } else {
2558
0
        llvm::Value *U = llvm::UndefValue::get(ConvertType(Arg->getType()));
2559
0
        ArgVals.push_back(ParamValue::forDirect(U));
2560
0
      }
2561
5.29k
      break;
2562
657k
    }
2563
657k
  }
2564
386k
2565
386k
  if (getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) {
2566
29.2k
    for (int I = Args.size() - 1; I >= 0; 
--I16.9k
)
2567
16.9k
      EmitParmDecl(*Args[I], ArgVals[I], I + 1);
2568
374k
  } else {
2569
1.01M
    for (unsigned I = 0, E = Args.size(); I != E; 
++I640k
)
2570
640k
      EmitParmDecl(*Args[I], ArgVals[I], I + 1);
2571
374k
  }
2572
386k
}
2573
2574
8
static void eraseUnusedBitCasts(llvm::Instruction *insn) {
2575
12
  while (insn->use_empty()) {
2576
12
    llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(insn);
2577
12
    if (!bitcast) 
return8
;
2578
4
2579
4
    // This is "safe" because we would have used a ConstantExpr otherwise.
2580
4
    insn = cast<llvm::Instruction>(bitcast->getOperand(0));
2581
4
    bitcast->eraseFromParent();
2582
4
  }
2583
8
}
2584
2585
/// Try to emit a fused autorelease of a return result.
2586
static llvm::Value *tryEmitFusedAutoreleaseOfResult(CodeGenFunction &CGF,
2587
65
                                                    llvm::Value *result) {
2588
65
  // We must be immediately followed the cast.
2589
65
  llvm::BasicBlock *BB = CGF.Builder.GetInsertBlock();
2590
65
  if (BB->empty()) 
return nullptr0
;
2591
65
  if (&BB->back() != result) 
return nullptr13
;
2592
52
2593
52
  llvm::Type *resultType = result->getType();
2594
52
2595
52
  // result is in a BasicBlock and is therefore an Instruction.
2596
52
  llvm::Instruction *generator = cast<llvm::Instruction>(result);
2597
52
2598
52
  SmallVector<llvm::Instruction *, 4> InstsToKill;
2599
52
2600
52
  // Look for:
2601
52
  //  %generator = bitcast %type1* %generator2 to %type2*
2602
81
  while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(generator)) {
2603
29
    // We would have emitted this as a constant if the operand weren't
2604
29
    // an Instruction.
2605
29
    generator = cast<llvm::Instruction>(bitcast->getOperand(0));
2606
29
2607
29
    // Require the generator to be immediately followed by the cast.
2608
29
    if (generator->getNextNode() != bitcast)
2609
0
      return nullptr;
2610
29
2611
29
    InstsToKill.push_back(bitcast);
2612
29
  }
2613
52
2614
52
  // Look for:
2615
52
  //   %generator = call i8* @objc_retain(i8* %originalResult)
2616
52
  // or
2617
52
  //   %generator = call i8* @objc_retainAutoreleasedReturnValue(i8* %originalResult)
2618
52
  llvm::CallInst *call = dyn_cast<llvm::CallInst>(generator);
2619
52
  if (!call) 
return nullptr5
;
2620
47
2621
47
  bool doRetainAutorelease;
2622
47
2623
47
  if (call->getCalledValue() == CGF.CGM.getObjCEntrypoints().objc_retain) {
2624
6
    doRetainAutorelease = true;
2625
41
  } else if (call->getCalledValue() == CGF.CGM.getObjCEntrypoints()
2626
41
                                          .objc_retainAutoreleasedReturnValue) {
2627
18
    doRetainAutorelease = false;
2628
18
2629
18
    // If we emitted an assembly marker for this call (and the
2630
18
    // ARCEntrypoints field should have been set if so), go looking
2631
18
    // for that call.  If we can't find it, we can't do this
2632
18
    // optimization.  But it should always be the immediately previous
2633
18
    // instruction, unless we needed bitcasts around the call.
2634
18
    if (CGF.CGM.getObjCEntrypoints().retainAutoreleasedReturnValueMarker) {
2635
12
      llvm::Instruction *prev = call->getPrevNode();
2636
12
      assert(prev);
2637
12
      if (isa<llvm::BitCastInst>(prev)) {
2638
8
        prev = prev->getPrevNode();
2639
8
        assert(prev);
2640
8
      }
2641
12
      assert(isa<llvm::CallInst>(prev));
2642
12
      assert(cast<llvm::CallInst>(prev)->getCalledValue() ==
2643
12
               CGF.CGM.getObjCEntrypoints().retainAutoreleasedReturnValueMarker);
2644
12
      InstsToKill.push_back(prev);
2645
12
    }
2646
23
  } else {
2647
23
    return nullptr;
2648
23
  }
2649
24
2650
24
  result = call->getArgOperand(0);
2651
24
  InstsToKill.push_back(call);
2652
24
2653
24
  // Keep killing bitcasts, for sanity.  Note that we no longer care
2654
24
  // about precise ordering as long as there's exactly one use.
2655
38
  while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(result)) {
2656
14
    if (!bitcast->hasOneUse()) 
break0
;
2657
14
    InstsToKill.push_back(bitcast);
2658
14
    result = bitcast->getOperand(0);
2659
14
  }
2660
24
2661
24
  // Delete all the unnecessary instructions, from latest to earliest.
2662
24
  for (auto *I : InstsToKill)
2663
69
    I->eraseFromParent();
2664
24
2665
24
  // Do the fused retain/autorelease if we were asked to.
2666
24
  if (doRetainAutorelease)
2667
6
    result = CGF.EmitARCRetainAutoreleaseReturnValue(result);
2668
24
2669
24
  // Cast back to the result type.
2670
24
  return CGF.Builder.CreateBitCast(result, resultType);
2671
24
}
2672
2673
/// If this is a +1 of the value of an immutable 'self', remove it.
2674
static llvm::Value *tryRemoveRetainOfSelf(CodeGenFunction &CGF,
2675
89
                                          llvm::Value *result) {
2676
89
  // This is only applicable to a method with an immutable 'self'.
2677
89
  const ObjCMethodDecl *method =
2678
89
    dyn_cast_or_null<ObjCMethodDecl>(CGF.CurCodeDecl);
2679
89
  if (!method) 
return nullptr72
;
2680
17
  const VarDecl *self = method->getSelfDecl();
2681
17
  if (!self->getType().isConstQualified()) 
return nullptr0
;
2682
17
2683
17
  // Look for a retain call.
2684
17
  llvm::CallInst *retainCall =
2685
17
    dyn_cast<llvm::CallInst>(result->stripPointerCasts());
2686
17
  if (!retainCall ||
2687
17
      
retainCall->getCalledValue() != CGF.CGM.getObjCEntrypoints().objc_retain14
)
2688
10
    return nullptr;
2689
7
2690
7
  // Look for an ordinary load of 'self'.
2691
7
  llvm::Value *retainedValue = retainCall->getArgOperand(0);
2692
7
  llvm::LoadInst *load =
2693
7
    dyn_cast<llvm::LoadInst>(retainedValue->stripPointerCasts());
2694
7
  if (!load || load->isAtomic() || load->isVolatile() ||
2695
7
      load->getPointerOperand() != CGF.GetAddrOfLocalVar(self).getPointer())
2696
3
    return nullptr;
2697
4
2698
4
  // Okay!  Burn it all down.  This relies for correctness on the
2699
4
  // assumption that the retain is emitted as part of the return and
2700
4
  // that thereafter everything is used "linearly".
2701
4
  llvm::Type *resultType = result->getType();
2702
4
  eraseUnusedBitCasts(cast<llvm::Instruction>(result));
2703
4
  assert(retainCall->use_empty());
2704
4
  retainCall->eraseFromParent();
2705
4
  eraseUnusedBitCasts(cast<llvm::Instruction>(retainedValue));
2706
4
2707
4
  return CGF.Builder.CreateBitCast(load, resultType);
2708
4
}
2709
2710
/// Emit an ARC autorelease of the result of a function.
2711
///
2712
/// \return the value to actually return from the function
2713
static llvm::Value *emitAutoreleaseOfResult(CodeGenFunction &CGF,
2714
89
                                            llvm::Value *result) {
2715
89
  // If we're returning 'self', kill the initial retain.  This is a
2716
89
  // heuristic attempt to "encourage correctness" in the really unfortunate
2717
89
  // case where we have a return of self during a dealloc and we desperately
2718
89
  // need to avoid the possible autorelease.
2719
89
  if (llvm::Value *self = tryRemoveRetainOfSelf(CGF, result))
2720
4
    return self;
2721
85
2722
85
  // At -O0, try to emit a fused retain/autorelease.
2723
85
  if (CGF.shouldUseFusedARCCalls())
2724
65
    if (llvm::Value *fused = tryEmitFusedAutoreleaseOfResult(CGF, result))
2725
24
      return fused;
2726
61
2727
61
  return CGF.EmitARCAutoreleaseReturnValue(result);
2728
61
}
2729
2730
/// Heuristically search for a dominating store to the return-value slot.
2731
219k
static llvm::StoreInst *findDominatingStoreToReturnValue(CodeGenFunction &CGF) {
2732
219k
  // Check if a User is a store which pointerOperand is the ReturnValue.
2733
219k
  // We are looking for stores to the ReturnValue, not for stores of the
2734
219k
  // ReturnValue to some other location.
2735
219k
  auto GetStoreIfValid = [&CGF](llvm::User *U) -> llvm::StoreInst * {
2736
210k
    auto *SI = dyn_cast<llvm::StoreInst>(U);
2737
210k
    if (!SI || 
SI->getPointerOperand() != CGF.ReturnValue.getPointer()195k
)
2738
15.2k
      return nullptr;
2739
195k
    // These aren't actually possible for non-coerced returns, and we
2740
195k
    // only care about non-coerced returns on this code path.
2741
195k
    assert(!SI->isAtomic() && !SI->isVolatile());
2742
195k
    return SI;
2743
195k
  };
2744
219k
  // If there are multiple uses of the return-value slot, just check
2745
219k
  // for something immediately preceding the IP.  Sometimes this can
2746
219k
  // happen with how we generate implicit-returns; it can also happen
2747
219k
  // with noreturn cleanups.
2748
219k
  if (!CGF.ReturnValue.getPointer()->hasOneUse()) {
2749
24.9k
    llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock();
2750
24.9k
    if (IP->empty()) 
return nullptr8.67k
;
2751
16.2k
    llvm::Instruction *I = &IP->back();
2752
16.2k
2753
16.2k
    // Skip lifetime markers
2754
16.2k
    for (llvm::BasicBlock::reverse_iterator II = IP->rbegin(),
2755
16.2k
                                            IE = IP->rend();
2756
54.2k
         II != IE; 
++II38.0k
) {
2757
44.5k
      if (llvm::IntrinsicInst *Intrinsic =
2758
38.6k
              dyn_cast<llvm::IntrinsicInst>(&*II)) {
2759
38.6k
        if (Intrinsic->getIntrinsicID() == llvm::Intrinsic::lifetime_end) {
2760
38.4k
          const llvm::Value *CastAddr = Intrinsic->getArgOperand(1);
2761
38.4k
          ++II;
2762
38.4k
          if (II == IE)
2763
127
            break;
2764
38.2k
          if (isa<llvm::BitCastInst>(&*II) && 
(CastAddr == &*II)38.0k
)
2765
38.0k
            continue;
2766
6.39k
        }
2767
38.6k
      }
2768
6.39k
      I = &*II;
2769
6.39k
      break;
2770
6.39k
    }
2771
16.2k
2772
16.2k
    return GetStoreIfValid(I);
2773
16.2k
  }
2774
194k
2775
194k
  llvm::StoreInst *store =
2776
194k
      GetStoreIfValid(CGF.ReturnValue.getPointer()->user_back());
2777
194k
  if (!store) 
return nullptr1.17k
;
2778
193k
2779
193k
  // Now do a first-and-dirty dominance check: just walk up the
2780
193k
  // single-predecessors chain from the current insertion point.
2781
193k
  llvm::BasicBlock *StoreBB = store->getParent();
2782
193k
  llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock();
2783
199k
  while (IP != StoreBB) {
2784
10.5k
    if (!(IP = IP->getSinglePredecessor()))
2785
4.23k
      return nullptr;
2786
10.5k
  }
2787
193k
2788
193k
  // Okay, the store's basic block dominates the insertion point; we
2789
193k
  // can do our thing.
2790
193k
  
return store188k
;
2791
193k
}
2792
2793
void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI,
2794
                                         bool EmitRetDbgLoc,
2795
386k
                                         SourceLocation EndLoc) {
2796
386k
  if (FI.isNoReturn()) {
2797
488
    // Noreturn functions don't return.
2798
488
    EmitUnreachable(EndLoc);
2799
488
    return;
2800
488
  }
2801
386k
2802
386k
  if (CurCodeDecl && 
CurCodeDecl->hasAttr<NakedAttr>()377k
) {
2803
7
    // Naked functions don't have epilogues.
2804
7
    Builder.CreateUnreachable();
2805
7
    return;
2806
7
  }
2807
386k
2808
386k
  // Functions with no result always return void.
2809
386k
  if (!ReturnValue.isValid()) {
2810
155k
    Builder.CreateRetVoid();
2811
155k
    return;
2812
155k
  }
2813
230k
2814
230k
  llvm::DebugLoc RetDbgLoc;
2815
230k
  llvm::Value *RV = nullptr;
2816
230k
  QualType RetTy = FI.getReturnType();
2817
230k
  const ABIArgInfo &RetAI = FI.getReturnInfo();
2818
230k
2819
230k
  switch (RetAI.getKind()) {
2820
230k
  case ABIArgInfo::InAlloca:
2821
5
    // Aggregrates get evaluated directly into the destination.  Sometimes we
2822
5
    // need to return the sret value in a register, though.
2823
5
    assert(hasAggregateEvaluationKind(RetTy));
2824
5
    if (RetAI.getInAllocaSRet()) {
2825
5
      llvm::Function::arg_iterator EI = CurFn->arg_end();
2826
5
      --EI;
2827
5
      llvm::Value *ArgStruct = &*EI;
2828
5
      llvm::Value *SRet = Builder.CreateStructGEP(
2829
5
          nullptr, ArgStruct, RetAI.getInAllocaFieldIndex());
2830
5
      RV = Builder.CreateAlignedLoad(SRet, getPointerAlign(), "sret");
2831
5
    }
2832
5
    break;
2833
230k
2834
230k
  case ABIArgInfo::Indirect: {
2835
4.25k
    auto AI = CurFn->arg_begin();
2836
4.25k
    if (RetAI.isSRetAfterThis())
2837
44
      ++AI;
2838
4.25k
    switch (getEvaluationKind(RetTy)) {
2839
4.25k
    case TEK_Complex: {
2840
252
      ComplexPairTy RT =
2841
252
        EmitLoadOfComplex(MakeAddrLValue(ReturnValue, RetTy), EndLoc);
2842
252
      EmitStoreOfComplex(RT, MakeNaturalAlignAddrLValue(&*AI, RetTy),
2843
252
                         /*isInit*/ true);
2844
252
      break;
2845
4.25k
    }
2846
4.25k
    case TEK_Aggregate:
2847
3.84k
      // Do nothing; aggregrates get evaluated directly into the destination.
2848
3.84k
      break;
2849
4.25k
    case TEK_Scalar:
2850
158
      EmitStoreOfScalar(Builder.CreateLoad(ReturnValue),
2851
158
                        MakeNaturalAlignAddrLValue(&*AI, RetTy),
2852
158
                        /*isInit*/ true);
2853
158
      break;
2854
4.25k
    }
2855
4.25k
    break;
2856
4.25k
  }
2857
4.25k
2858
224k
  case ABIArgInfo::Extend:
2859
224k
  case ABIArgInfo::Direct:
2860
224k
    if (RetAI.getCoerceToType() == ConvertType(RetTy) &&
2861
224k
        
RetAI.getDirectOffset() == 0219k
) {
2862
219k
      // The internal return value temp always will have pointer-to-return-type
2863
219k
      // type, just do a load.
2864
219k
2865
219k
      // If there is a dominating store to ReturnValue, we can elide
2866
219k
      // the load, zap the store, and usually zap the alloca.
2867
219k
      if (llvm::StoreInst *SI =
2868
191k
              findDominatingStoreToReturnValue(*this)) {
2869
191k
        // Reuse the debug location from the store unless there is
2870
191k
        // cleanup code to be emitted between the store and return
2871
191k
        // instruction.
2872
191k
        if (EmitRetDbgLoc && 
!AutoreleaseResult190k
)
2873
189k
          RetDbgLoc = SI->getDebugLoc();
2874
191k
        // Get the stored value and nuke the now-dead store.
2875
191k
        RV = SI->getValueOperand();
2876
191k
        SI->eraseFromParent();
2877
191k
2878
191k
      // Otherwise, we have to do a simple load.
2879
191k
      } else {
2880
28.1k
        RV = Builder.CreateLoad(ReturnValue);
2881
28.1k
      }
2882
219k
    } else {
2883
5.63k
      // If the value is offset in memory, apply the offset now.
2884
5.63k
      Address V = emitAddressAtOffset(*this, ReturnValue, RetAI);
2885
5.63k
2886
5.63k
      RV = CreateCoercedLoad(V, RetAI.getCoerceToType(), *this);
2887
5.63k
    }
2888
224k
2889
224k
    // In ARC, end functions that return a retainable type with a call
2890
224k
    // to objc_autoreleaseReturnValue.
2891
224k
    if (AutoreleaseResult) {
2892
#ifndef NDEBUG
2893
      // Type::isObjCRetainabletype has to be called on a QualType that hasn't
2894
      // been stripped of the typedefs, so we cannot use RetTy here. Get the
2895
      // original return type of FunctionDecl, CurCodeDecl, and BlockDecl from
2896
      // CurCodeDecl or BlockInfo.
2897
      QualType RT;
2898
2899
      if (auto *FD = dyn_cast<FunctionDecl>(CurCodeDecl))
2900
        RT = FD->getReturnType();
2901
      else if (auto *MD = dyn_cast<ObjCMethodDecl>(CurCodeDecl))
2902
        RT = MD->getReturnType();
2903
      else if (isa<BlockDecl>(CurCodeDecl))
2904
        RT = BlockInfo->BlockExpression->getFunctionType()->getReturnType();
2905
      else
2906
        llvm_unreachable("Unexpected function/method type");
2907
2908
      assert(getLangOpts().ObjCAutoRefCount &&
2909
             !FI.isReturnsRetained() &&
2910
             RT->isObjCRetainableType());
2911
#endif
2912
      RV = emitAutoreleaseOfResult(*this, RV);
2913
89
    }
2914
224k
2915
224k
    break;
2916
224k
2917
224k
  case ABIArgInfo::Ignore:
2918
580
    break;
2919
224k
2920
224k
  case ABIArgInfo::CoerceAndExpand: {
2921
539
    auto coercionType = RetAI.getCoerceAndExpandType();
2922
539
2923
539
    // Load all of the coerced elements out into results.
2924
539
    llvm::SmallVector<llvm::Value*, 4> results;
2925
539
    Address addr = Builder.CreateElementBitCast(ReturnValue, coercionType);
2926
1.68k
    for (unsigned i = 0, e = coercionType->getNumElements(); i != e; 
++i1.14k
) {
2927
1.14k
      auto coercedEltType = coercionType->getElementType(i);
2928
1.14k
      if (ABIArgInfo::isPaddingForCoerceAndExpand(coercedEltType))
2929
7
        continue;
2930
1.13k
2931
1.13k
      auto eltAddr = Builder.CreateStructGEP(addr, i);
2932
1.13k
      auto elt = Builder.CreateLoad(eltAddr);
2933
1.13k
      results.push_back(elt);
2934
1.13k
    }
2935
539
2936
539
    // If we have one result, it's the single direct result type.
2937
539
    if (results.size() == 1) {
2938
180
      RV = results[0];
2939
180
2940
180
    // Otherwise, we need to make a first-class aggregate.
2941
359
    } else {
2942
359
      // Construct a return type that lacks padding elements.
2943
359
      llvm::Type *returnType = RetAI.getUnpaddedCoerceAndExpandType();
2944
359
2945
359
      RV = llvm::UndefValue::get(returnType);
2946
1.31k
      for (unsigned i = 0, e = results.size(); i != e; 
++i956
) {
2947
956
        RV = Builder.CreateInsertValue(RV, results[i], i);
2948
956
      }
2949
359
    }
2950
539
    break;
2951
224k
  }
2952
224k
2953
224k
  case ABIArgInfo::Expand:
2954
0
    llvm_unreachable("Invalid ABI kind for return argument");
2955
230k
  }
2956
230k
2957
230k
  llvm::Instruction *Ret;
2958
230k
  if (RV) {
2959
225k
    EmitReturnValueCheck(RV);
2960
225k
    Ret = Builder.CreateRet(RV);
2961
225k
  } else {
2962
4.83k
    Ret = Builder.CreateRetVoid();
2963
4.83k
  }
2964
230k
2965
230k
  if (RetDbgLoc)
2966
43.1k
    Ret->setDebugLoc(std::move(RetDbgLoc));
2967
230k
}
2968
2969
225k
void CodeGenFunction::EmitReturnValueCheck(llvm::Value *RV) {
2970
225k
  // A current decl may not be available when emitting vtable thunks.
2971
225k
  if (!CurCodeDecl)
2972
1.51k
    return;
2973
223k
2974
223k
  ReturnsNonNullAttr *RetNNAttr = nullptr;
2975
223k
  if (SanOpts.has(SanitizerKind::ReturnsNonnullAttribute))
2976
38
    RetNNAttr = CurCodeDecl->getAttr<ReturnsNonNullAttr>();
2977
223k
2978
223k
  if (!RetNNAttr && 
!requiresReturnValueNullabilityCheck()223k
)
2979
223k
    return;
2980
4
2981
4
  // Prefer the returns_nonnull attribute if it's present.
2982
4
  SourceLocation AttrLoc;
2983
4
  SanitizerMask CheckKind;
2984
4
  SanitizerHandler Handler;
2985
4
  if (RetNNAttr) {
2986
3
    assert(!requiresReturnValueNullabilityCheck() &&
2987
3
           "Cannot check nullability and the nonnull attribute");
2988
3
    AttrLoc = RetNNAttr->getLocation();
2989
3
    CheckKind = SanitizerKind::ReturnsNonnullAttribute;
2990
3
    Handler = SanitizerHandler::NonnullReturn;
2991
3
  } else {
2992
1
    if (auto *DD = dyn_cast<DeclaratorDecl>(CurCodeDecl))
2993
0
      if (auto *TSI = DD->getTypeSourceInfo())
2994
0
        if (auto FTL = TSI->getTypeLoc().castAs<FunctionTypeLoc>())
2995
0
          AttrLoc = FTL.getReturnLoc().findNullabilityLoc();
2996
1
    CheckKind = SanitizerKind::NullabilityReturn;
2997
1
    Handler = SanitizerHandler::NullabilityReturn;
2998
1
  }
2999
4
3000
4
  SanitizerScope SanScope(this);
3001
4
3002
4
  // Make sure the "return" source location is valid. If we're checking a
3003
4
  // nullability annotation, make sure the preconditions for the check are met.
3004
4
  llvm::BasicBlock *Check = createBasicBlock("nullcheck");
3005
4
  llvm::BasicBlock *NoCheck = createBasicBlock("no.nullcheck");
3006
4
  llvm::Value *SLocPtr = Builder.CreateLoad(ReturnLocation, "return.sloc.load");
3007
4
  llvm::Value *CanNullCheck = Builder.CreateIsNotNull(SLocPtr);
3008
4
  if (requiresReturnValueNullabilityCheck())
3009
1
    CanNullCheck =
3010
1
        Builder.CreateAnd(CanNullCheck, RetValNullabilityPrecondition);
3011
4
  Builder.CreateCondBr(CanNullCheck, Check, NoCheck);
3012
4
  EmitBlock(Check);
3013
4
3014
4
  // Now do the null check.
3015
4
  llvm::Value *Cond = Builder.CreateIsNotNull(RV);
3016
4
  llvm::Constant *StaticData[] = {EmitCheckSourceLocation(AttrLoc)};
3017
4
  llvm::Value *DynamicData[] = {SLocPtr};
3018
4
  EmitCheck(std::make_pair(Cond, CheckKind), Handler, StaticData, DynamicData);
3019
4
3020
4
  EmitBlock(NoCheck);
3021
4
3022
#ifndef NDEBUG
3023
  // The return location should not be used after the check has been emitted.
3024
  ReturnLocation = Address::invalid();
3025
#endif
3026
}
3027
3028
29.2k
static bool isInAllocaArgument(CGCXXABI &ABI, QualType type) {
3029
29.2k
  const CXXRecordDecl *RD = type->getAsCXXRecordDecl();
3030
29.2k
  return RD && 
ABI.getRecordArgABI(RD) == CGCXXABI::RAA_DirectInMemory1.36k
;
3031
29.2k
}
3032
3033
static AggValueSlot createPlaceholderSlot(CodeGenFunction &CGF,
3034
93
                                          QualType Ty) {
3035
93
  // FIXME: Generate IR in one pass, rather than going back and fixing up these
3036
93
  // placeholders.
3037
93
  llvm::Type *IRTy = CGF.ConvertTypeForMem(Ty);
3038
93
  llvm::Type *IRPtrTy = IRTy->getPointerTo();
3039
93
  llvm::Value *Placeholder = llvm::UndefValue::get(IRPtrTy->getPointerTo());
3040
93
3041
93
  // FIXME: When we generate this IR in one pass, we shouldn't need
3042
93
  // this win32-specific alignment hack.
3043
93
  CharUnits Align = CharUnits::fromQuantity(4);
3044
93
  Placeholder = CGF.Builder.CreateAlignedLoad(IRPtrTy, Placeholder, Align);
3045
93
3046
93
  return AggValueSlot::forAddr(Address(Placeholder, Align),
3047
93
                               Ty.getQualifiers(),
3048
93
                               AggValueSlot::IsNotDestructed,
3049
93
                               AggValueSlot::DoesNotNeedGCBarriers,
3050
93
                               AggValueSlot::IsNotAliased,
3051
93
                               AggValueSlot::DoesNotOverlap);
3052
93
}
3053
3054
void CodeGenFunction::EmitDelegateCallArg(CallArgList &args,
3055
                                          const VarDecl *param,
3056
23.1k
                                          SourceLocation loc) {
3057
23.1k
  // StartFunction converted the ABI-lowered parameter(s) into a
3058
23.1k
  // local alloca.  We need to turn that into an r-value suitable
3059
23.1k
  // for EmitCall.
3060
23.1k
  Address local = GetAddrOfLocalVar(param);
3061
23.1k
3062
23.1k
  QualType type = param->getType();
3063
23.1k
3064
23.1k
  if (isInAllocaArgument(CGM.getCXXABI(), type)) {
3065
1
    CGM.ErrorUnsupported(param, "forwarded non-trivially copyable parameter");
3066
1
  }
3067
23.1k
3068
23.1k
  // GetAddrOfLocalVar returns a pointer-to-pointer for references,
3069
23.1k
  // but the argument needs to be the original pointer.
3070
23.1k
  if (type->isReferenceType()) {
3071
10.5k
    args.add(RValue::get(Builder.CreateLoad(local)), type);
3072
10.5k
3073
10.5k
  // In ARC, move out of consumed arguments so that the release cleanup
3074
10.5k
  // entered by StartFunction doesn't cause an over-release.  This isn't
3075
10.5k
  // optimal -O0 code generation, but it should get cleaned up when
3076
10.5k
  // optimization is enabled.  This also assumes that delegate calls are
3077
10.5k
  // performed exactly once for a set of arguments, but that should be safe.
3078
12.6k
  } else if (getLangOpts().ObjCAutoRefCount &&
3079
12.6k
             
param->hasAttr<NSConsumedAttr>()9
&&
3080
12.6k
             
type->isObjCRetainableType()2
) {
3081
2
    llvm::Value *ptr = Builder.CreateLoad(local);
3082
2
    auto null =
3083
2
      llvm::ConstantPointerNull::get(cast<llvm::PointerType>(ptr->getType()));
3084
2
    Builder.CreateStore(null, local);
3085
2
    args.add(RValue::get(ptr), type);
3086
2
3087
2
  // For the most part, we just need to load the alloca, except that
3088
2
  // aggregate r-values are actually pointers to temporaries.
3089
12.6k
  } else {
3090
12.6k
    args.add(convertTempToRValue(local, type, loc), type);
3091
12.6k
  }
3092
23.1k
3093
23.1k
  // Deactivate the cleanup for the callee-destructed param that was pushed.
3094
23.1k
  if (hasAggregateEvaluationKind(type) && 
!CurFuncIsThunk1.09k
&&
3095
23.1k
      
type->getAs<RecordType>()->getDecl()->isParamDestroyedInCallee()1.08k
&&
3096
23.1k
      
type.isDestructedType()8
) {
3097
6
    EHScopeStack::stable_iterator cleanup =
3098
6
        CalleeDestructedParamCleanups.lookup(cast<ParmVarDecl>(param));
3099
6
    assert(cleanup.isValid() &&
3100
6
           "cleanup for callee-destructed param not recorded");
3101
6
    // This unreachable is a temporary marker which will be removed later.
3102
6
    llvm::Instruction *isActive = Builder.CreateUnreachable();
3103
6
    args.addArgCleanupDeactivation(cleanup, isActive);
3104
6
  }
3105
23.1k
}
3106
3107
26
static bool isProvablyNull(llvm::Value *addr) {
3108
26
  return isa<llvm::ConstantPointerNull>(addr);
3109
26
}
3110
3111
/// Emit the actual writing-back of a writeback.
3112
static void emitWriteback(CodeGenFunction &CGF,
3113
26
                          const CallArgList::Writeback &writeback) {
3114
26
  const LValue &srcLV = writeback.Source;
3115
26
  Address srcAddr = srcLV.getAddress();
3116
26
  assert(!isProvablyNull(srcAddr.getPointer()) &&
3117
26
         "shouldn't have writeback for provably null argument");
3118
26
3119
26
  llvm::BasicBlock *contBB = nullptr;
3120
26
3121
26
  // If the argument wasn't provably non-null, we need to null check
3122
26
  // before doing the store.
3123
26
  bool provablyNonNull = llvm::isKnownNonZero(srcAddr.getPointer(),
3124
26
                                              CGF.CGM.getDataLayout());
3125
26
  if (!provablyNonNull) {
3126
5
    llvm::BasicBlock *writebackBB = CGF.createBasicBlock("icr.writeback");
3127
5
    contBB = CGF.createBasicBlock("icr.done");
3128
5
3129
5
    llvm::Value *isNull =
3130
5
      CGF.Builder.CreateIsNull(srcAddr.getPointer(), "icr.isnull");
3131
5
    CGF.Builder.CreateCondBr(isNull, contBB, writebackBB);
3132
5
    CGF.EmitBlock(writebackBB);
3133
5
  }
3134
26
3135
26
  // Load the value to writeback.
3136
26
  llvm::Value *value = CGF.Builder.CreateLoad(writeback.Temporary);
3137
26
3138
26
  // Cast it back, in case we're writing an id to a Foo* or something.
3139
26
  value = CGF.Builder.CreateBitCast(value, srcAddr.getElementType(),
3140
26
                                    "icr.writeback-cast");
3141
26
3142
26
  // Perform the writeback.
3143
26
3144
26
  // If we have a "to use" value, it's something we need to emit a use
3145
26
  // of.  This has to be carefully threaded in: if it's done after the
3146
26
  // release it's potentially undefined behavior (and the optimizer
3147
26
  // will ignore it), and if it happens before the retain then the
3148
26
  // optimizer could move the release there.
3149
26
  if (writeback.ToUse) {
3150
7
    assert(srcLV.getObjCLifetime() == Qualifiers::OCL_Strong);
3151
7
3152
7
    // Retain the new value.  No need to block-copy here:  the block's
3153
7
    // being passed up the stack.
3154
7
    value = CGF.EmitARCRetainNonBlock(value);
3155
7
3156
7
    // Emit the intrinsic use here.
3157
7
    CGF.EmitARCIntrinsicUse(writeback.ToUse);
3158
7
3159
7
    // Load the old value (primitively).
3160
7
    llvm::Value *oldValue = CGF.EmitLoadOfScalar(srcLV, SourceLocation());
3161
7
3162
7
    // Put the new value in place (primitively).
3163
7
    CGF.EmitStoreOfScalar(value, srcLV, /*init*/ false);
3164
7
3165
7
    // Release the old value.
3166
7
    CGF.EmitARCRelease(oldValue, srcLV.isARCPreciseLifetime());
3167
7
3168
7
  // Otherwise, we can just do a normal lvalue store.
3169
19
  } else {
3170
19
    CGF.EmitStoreThroughLValue(RValue::get(value), srcLV);
3171
19
  }
3172
26
3173
26
  // Jump to the continuation block.
3174
26
  if (!provablyNonNull)
3175
5
    CGF.EmitBlock(contBB);
3176
26
}
3177
3178
static void emitWritebacks(CodeGenFunction &CGF,
3179
26
                           const CallArgList &args) {
3180
26
  for (const auto &I : args.writebacks())
3181
26
    emitWriteback(CGF, I);
3182
26
}
3183
3184
static void deactivateArgCleanupsBeforeCall(CodeGenFunction &CGF,
3185
133
                                            const CallArgList &CallArgs) {
3186
133
  ArrayRef<CallArgList::CallArgCleanup> Cleanups =
3187
133
    CallArgs.getCleanupsToDeactivate();
3188
133
  // Iterate in reverse to increase the likelihood of popping the cleanup.
3189
171
  for (const auto &I : llvm::reverse(Cleanups)) {
3190
171
    CGF.DeactivateCleanupBlock(I.Cleanup, I.IsActiveIP);
3191
171
    I.IsActiveIP->eraseFromParent();
3192
171
  }
3193
133
}
3194
3195
26
static const Expr *maybeGetUnaryAddrOfOperand(const Expr *E) {
3196
26
  if (const UnaryOperator *uop = dyn_cast<UnaryOperator>(E->IgnoreParens()))
3197
21
    if (uop->getOpcode() == UO_AddrOf)
3198
21
      return uop->getSubExpr();
3199
5
  return nullptr;
3200
5
}
3201
3202
/// Emit an argument that's being passed call-by-writeback.  That is,
3203
/// we are passing the address of an __autoreleased temporary; it
3204
/// might be copy-initialized with the current value of the given
3205
/// address, but it will definitely be copied out of after the call.
3206
static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args,
3207
26
                             const ObjCIndirectCopyRestoreExpr *CRE) {
3208
26
  LValue srcLV;
3209
26
3210
26
  // Make an optimistic effort to emit the address as an l-value.
3211
26
  // This can fail if the argument expression is more complicated.
3212
26
  if (const Expr *lvExpr = maybeGetUnaryAddrOfOperand(CRE->getSubExpr())) {
3213
21
    srcLV = CGF.EmitLValue(lvExpr);
3214
21
3215
21
  // Otherwise, just emit it as a scalar.
3216
21
  } else {
3217
5
    Address srcAddr = CGF.EmitPointerWithAlignment(CRE->getSubExpr());
3218
5
3219
5
    QualType srcAddrType =
3220
5
      CRE->getSubExpr()->getType()->castAs<PointerType>()->getPointeeType();
3221
5
    srcLV = CGF.MakeAddrLValue(srcAddr, srcAddrType);
3222
5
  }
3223
26
  Address srcAddr = srcLV.getAddress();
3224
26
3225
26
  // The dest and src types don't necessarily match in LLVM terms
3226
26
  // because of the crazy ObjC compatibility rules.
3227
26
3228
26
  llvm::PointerType *destType =
3229
26
    cast<llvm::PointerType>(CGF.ConvertType(CRE->getType()));
3230
26
3231
26
  // If the address is a constant null, just pass the appropriate null.
3232
26
  if (isProvablyNull(srcAddr.getPointer())) {
3233
0
    args.add(RValue::get(llvm::ConstantPointerNull::get(destType)),
3234
0
             CRE->getType());
3235
0
    return;
3236
0
  }
3237
26
3238
26
  // Create the temporary.
3239
26
  Address temp = CGF.CreateTempAlloca(destType->getElementType(),
3240
26
                                      CGF.getPointerAlign(),
3241
26
                                      "icr.temp");
3242
26
  // Loading an l-value can introduce a cleanup if the l-value is __weak,
3243
26
  // and that cleanup will be conditional if we can't prove that the l-value
3244
26
  // isn't null, so we need to register a dominating point so that the cleanups
3245
26
  // system will make valid IR.
3246
26
  CodeGenFunction::ConditionalEvaluation condEval(CGF);
3247
26
3248
26
  // Zero-initialize it if we're not doing a copy-initialization.
3249
26
  bool shouldCopy = CRE->shouldCopy();
3250
26
  if (!shouldCopy) {
3251
4
    llvm::Value *null =
3252
4
      llvm::ConstantPointerNull::get(
3253
4
        cast<llvm::PointerType>(destType->getElementType()));
3254
4
    CGF.Builder.CreateStore(null, temp);
3255
4
  }
3256
26
3257
26
  llvm::BasicBlock *contBB = nullptr;
3258
26
  llvm::BasicBlock *originBB = nullptr;
3259
26
3260
26
  // If the address is *not* known to be non-null, we need to switch.
3261
26
  llvm::Value *finalArgument;
3262
26
3263
26
  bool provablyNonNull = llvm::isKnownNonZero(srcAddr.getPointer(),
3264
26
                                              CGF.CGM.getDataLayout());
3265
26
  if (provablyNonNull) {
3266
21
    finalArgument = temp.getPointer();
3267
21
  } else {
3268
5
    llvm::Value *isNull =
3269
5
      CGF.Builder.CreateIsNull(srcAddr.getPointer(), "icr.isnull");
3270
5
3271
5
    finalArgument = CGF.Builder.CreateSelect(isNull,
3272
5
                                   llvm::ConstantPointerNull::get(destType),
3273
5
                                             temp.getPointer(), "icr.argument");
3274
5
3275
5
    // If we need to copy, then the load has to be conditional, which
3276
5
    // means we need control flow.
3277
5
    if (shouldCopy) {
3278
5
      originBB = CGF.Builder.GetInsertBlock();
3279
5
      contBB = CGF.createBasicBlock("icr.cont");
3280
5
      llvm::BasicBlock *copyBB = CGF.createBasicBlock("icr.copy");
3281
5
      CGF.Builder.CreateCondBr(isNull, contBB, copyBB);
3282
5
      CGF.EmitBlock(copyBB);
3283
5
      condEval.begin(CGF);
3284
5
    }
3285
5
  }
3286
26
3287
26
  llvm::Value *valueToUse = nullptr;
3288
26
3289
26
  // Perform a copy if necessary.
3290
26
  if (shouldCopy) {
3291
22
    RValue srcRV = CGF.EmitLoadOfLValue(srcLV, SourceLocation());
3292
22
    assert(srcRV.isScalar());
3293
22
3294
22
    llvm::Value *src = srcRV.getScalarVal();
3295
22
    src = CGF.Builder.CreateBitCast(src, destType->getElementType(),
3296
22
                                    "icr.cast");
3297
22
3298
22
    // Use an ordinary store, not a store-to-lvalue.
3299
22
    CGF.Builder.CreateStore(src, temp);
3300
22
3301
22
    // If optimization is enabled, and the value was held in a
3302
22
    // __strong variable, we need to tell the optimizer that this
3303
22
    // value has to stay alive until we're doing the store back.
3304
22
    // This is because the temporary is effectively unretained,
3305
22
    // and so otherwise we can violate the high-level semantics.
3306
22
    if (CGF.CGM.getCodeGenOpts().OptimizationLevel != 0 &&
3307
22
        
srcLV.getObjCLifetime() == Qualifiers::OCL_Strong9
) {
3308
7
      valueToUse = src;
3309
7
    }
3310
22
  }
3311
26
3312
26
  // Finish the control flow if we needed it.
3313
26
  if (shouldCopy && 
!provablyNonNull22
) {
3314
5
    llvm::BasicBlock *copyBB = CGF.Builder.GetInsertBlock();
3315
5
    CGF.EmitBlock(contBB);
3316
5
3317
5
    // Make a phi for the value to intrinsically use.
3318
5
    if (valueToUse) {
3319
2
      llvm::PHINode *phiToUse = CGF.Builder.CreatePHI(valueToUse->getType(), 2,
3320
2
                                                      "icr.to-use");
3321
2
      phiToUse->addIncoming(valueToUse, copyBB);
3322
2
      phiToUse->addIncoming(llvm::UndefValue::get(valueToUse->getType()),
3323
2
                            originBB);
3324
2
      valueToUse = phiToUse;
3325
2
    }
3326
5
3327
5
    condEval.end(CGF);
3328
5
  }
3329
26
3330
26
  args.addWriteback(srcLV, temp, valueToUse);
3331
26
  args.add(RValue::get(finalArgument), CRE->getType());
3332
26
}
3333
3334
66
void CallArgList::allocateArgumentMemory(CodeGenFunction &CGF) {
3335
66
  assert(!StackBase);
3336
66
3337
66
  // Save the stack.
3338
66
  llvm::Function *F = CGF.CGM.getIntrinsic(llvm::Intrinsic::stacksave);
3339
66
  StackBase = CGF.Builder.CreateCall(F, {}, "inalloca.save");
3340
66
}
3341
3342
881k
void CallArgList::freeArgumentMemory(CodeGenFunction &CGF) const {
3343
881k
  if (StackBase) {
3344
66
    // Restore the stack after the call.
3345
66
    llvm::Function *F = CGF.CGM.getIntrinsic(llvm::Intrinsic::stackrestore);
3346
66
    CGF.Builder.CreateCall(F, StackBase);
3347
66
  }
3348
881k
}
3349
3350
void CodeGenFunction::EmitNonNullArgCheck(RValue RV, QualType ArgType,
3351
                                          SourceLocation ArgLoc,
3352
                                          AbstractCallee AC,
3353
1.23M
                                          unsigned ParmNum) {
3354
1.23M
  if (!AC.getDecl() || 
!(1.22M
SanOpts.has(SanitizerKind::NonnullAttribute)1.22M
||
3355
1.22M
                         
SanOpts.has(SanitizerKind::NullabilityArg)1.22M
))
3356
1.23M
    return;
3357
22
3358
22
  // The param decl may be missing in a variadic function.
3359
22
  auto PVD = ParmNum < AC.getNumParams() ? 
AC.getParamDecl(ParmNum)19
:
nullptr3
;
3360
22
  unsigned ArgNo = PVD ? 
PVD->getFunctionScopeIndex()19
:
ParmNum3
;
3361
22
3362
22
  // Prefer the nonnull attribute if it's present.
3363
22
  const NonNullAttr *NNAttr = nullptr;
3364
22
  if (SanOpts.has(SanitizerKind::NonnullAttribute))
3365
22
    NNAttr = getNonNullAttr(AC.getDecl(), PVD, ArgType, ArgNo);
3366
22
3367
22
  bool CanCheckNullability = false;
3368
22
  if (SanOpts.has(SanitizerKind::NullabilityArg) && 
!NNAttr0
&&
PVD0
) {
3369
0
    auto Nullability = PVD->getType()->getNullability(getContext());
3370
0
    CanCheckNullability = Nullability &&
3371
0
                          *Nullability == NullabilityKind::NonNull &&
3372
0
                          PVD->getTypeSourceInfo();
3373
0
  }
3374
22
3375
22
  if (!NNAttr && 
!CanCheckNullability7
)
3376
7
    return;
3377
15
3378
15
  SourceLocation AttrLoc;
3379
15
  SanitizerMask CheckKind;
3380
15
  SanitizerHandler Handler;
3381
15
  if (NNAttr) {
3382
15
    AttrLoc = NNAttr->getLocation();
3383
15
    CheckKind = SanitizerKind::NonnullAttribute;
3384
15
    Handler = SanitizerHandler::NonnullArg;
3385
15
  } else {
3386
0
    AttrLoc = PVD->getTypeSourceInfo()->getTypeLoc().findNullabilityLoc();
3387
0
    CheckKind = SanitizerKind::NullabilityArg;
3388
0
    Handler = SanitizerHandler::NullabilityArg;
3389
0
  }
3390
15
3391
15
  SanitizerScope SanScope(this);
3392
15
  assert(RV.isScalar());
3393
15
  llvm::Value *V = RV.getScalarVal();
3394
15
  llvm::Value *Cond =
3395
15
      Builder.CreateICmpNE(V, llvm::Constant::getNullValue(V->getType()));
3396
15
  llvm::Constant *StaticData[] = {
3397
15
      EmitCheckSourceLocation(ArgLoc), EmitCheckSourceLocation(AttrLoc),
3398
15
      llvm::ConstantInt::get(Int32Ty, ArgNo + 1),
3399
15
  };
3400
15
  EmitCheck(std::make_pair(Cond, CheckKind), Handler, StaticData, None);
3401
15
}
3402
3403
void CodeGenFunction::EmitCallArgs(
3404
    CallArgList &Args, ArrayRef<QualType> ArgTypes,
3405
    llvm::iterator_range<CallExpr::const_arg_iterator> ArgRange,
3406
825k
    AbstractCallee AC, unsigned ParamsToSkip, EvaluationOrder Order) {
3407
825k
  assert((int)ArgTypes.size() == (ArgRange.end() - ArgRange.begin()));
3408
825k
3409
825k
  // We *have* to evaluate arguments from right to left in the MS C++ ABI,
3410
825k
  // because arguments are destroyed left to right in the callee. As a special
3411
825k
  // case, there are certain language constructs that require left-to-right
3412
825k
  // evaluation, and in those cases we consider the evaluation order requirement
3413
825k
  // to trump the "destruction order is reverse construction order" guarantee.
3414
825k
  bool LeftToRight =
3415
825k
      CGM.getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()
3416
825k
          ? 
Order == EvaluationOrder::ForceLeftToRight6.64k
3417
825k
          : 
Order != EvaluationOrder::ForceRightToLeft819k
;
3418
825k
3419
825k
  auto MaybeEmitImplicitObjectSize = [&](unsigned I, const Expr *Arg,
3420
1.23M
                                         RValue EmittedArg) {
3421
1.23M
    if (!AC.hasFunctionDecl() || 
I >= AC.getNumParams()1.21M
)
3422
89.4k
      return;
3423
1.14M
    auto *PS = AC.getParamDecl(I)->getAttr<PassObjectSizeAttr>();
3424
1.14M
    if (PS == nullptr)
3425
1.14M
      return;
3426
107
3427
107
    const auto &Context = getContext();
3428
107
    auto SizeTy = Context.getSizeType();
3429
107
    auto T = Builder.getIntNTy(Context.getTypeSize(SizeTy));
3430
107
    assert(EmittedArg.getScalarVal() && "We emitted nothing for the arg?");
3431
107
    llvm::Value *V = evaluateOrEmitBuiltinObjectSize(Arg, PS->getType(), T,
3432
107
                                                     EmittedArg.getScalarVal(),
3433
107
                                                     PS->isDynamic());
3434
107
    Args.add(RValue::get(V), SizeTy);
3435
107
    // If we're emitting args in reverse, be sure to do so with
3436
107
    // pass_object_size, as well.
3437
107
    if (!LeftToRight)
3438
9
      std::swap(Args.back(), *(&Args.back() - 1));
3439
107
  };
3440
825k
3441
825k
  // Insert a stack save if we're going to need any inalloca args.
3442
825k
  bool HasInAllocaArgs = false;
3443
825k
  if (CGM.getTarget().getCXXABI().isMicrosoft()) {
3444
6.64k
    for (ArrayRef<QualType>::iterator I = ArgTypes.begin(), E = ArgTypes.end();
3445
12.7k
         I != E && 
!HasInAllocaArgs6.12k
;
++I6.10k
)
3446
6.10k
      HasInAllocaArgs = isInAllocaArgument(CGM.getCXXABI(), *I);
3447
6.64k
    if (HasInAllocaArgs) {
3448
66
      assert(getTarget().getTriple().getArch() == llvm::Triple::x86);
3449
66
      Args.allocateArgumentMemory(*this);
3450
66
    }
3451
6.64k
  }
3452
825k
3453
825k
  // Evaluate each argument in the appropriate order.
3454
825k
  size_t CallArgsStart = Args.size();
3455
2.05M
  for (unsigned I = 0, E = ArgTypes.size(); I != E; 
++I1.23M
) {
3456
1.23M
    unsigned Idx = LeftToRight ? 
I1.21M
:
E - I - 114.9k
;
3457
1.23M
    CallExpr::const_arg_iterator Arg = ArgRange.begin() + Idx;
3458
1.23M
    unsigned InitialArgSize = Args.size();
3459
1.23M
    // If *Arg is an ObjCIndirectCopyRestoreExpr, check that either the types of
3460
1.23M
    // the argument and parameter match or the objc method is parameterized.
3461
1.23M
    assert((!isa<ObjCIndirectCopyRestoreExpr>(*Arg) ||
3462
1.23M
            getContext().hasSameUnqualifiedType((*Arg)->getType(),
3463
1.23M
                                                ArgTypes[Idx]) ||
3464
1.23M
            (isa<ObjCMethodDecl>(AC.getDecl()) &&
3465
1.23M
             isObjCMethodWithTypeParams(cast<ObjCMethodDecl>(AC.getDecl())))) &&
3466
1.23M
           "Argument and parameter types don't match");
3467
1.23M
    EmitCallArg(Args, *Arg, ArgTypes[Idx]);
3468
1.23M
    // In particular, we depend on it being the last arg in Args, and the
3469
1.23M
    // objectsize bits depend on there only being one arg if !LeftToRight.
3470
1.23M
    assert(InitialArgSize + 1 == Args.size() &&
3471
1.23M
           "The code below depends on only adding one arg per EmitCallArg");
3472
1.23M
    (void)InitialArgSize;
3473
1.23M
    // Since pointer argument are never emitted as LValue, it is safe to emit
3474
1.23M
    // non-null argument check for r-value only.
3475
1.23M
    if (!Args.back().hasLValue()) {
3476
1.23M
      RValue RVArg = Args.back().getKnownRValue();
3477
1.23M
      EmitNonNullArgCheck(RVArg, ArgTypes[Idx], (*Arg)->getExprLoc(), AC,
3478
1.23M
                          ParamsToSkip + Idx);
3479
1.23M
      // @llvm.objectsize should never have side-effects and shouldn't need
3480
1.23M
      // destruction/cleanups, so we can safely "emit" it after its arg,
3481
1.23M
      // regardless of right-to-leftness
3482
1.23M
      MaybeEmitImplicitObjectSize(Idx, *Arg, RVArg);
3483
1.23M
    }
3484
1.23M
  }
3485
825k
3486
825k
  if (!LeftToRight) {
3487
15.3k
    // Un-reverse the arguments we just evaluated so they match up with the LLVM
3488
15.3k
    // IR function.
3489
15.3k
    std::reverse(Args.begin() + CallArgsStart, Args.end());
3490
15.3k
  }
3491
825k
}
3492
3493
namespace {
3494
3495
struct DestroyUnpassedArg final : EHScopeStack::Cleanup {
3496
  DestroyUnpassedArg(Address Addr, QualType Ty)
3497
161
      : Addr(Addr), Ty(Ty) {}
3498
3499
  Address Addr;
3500
  QualType Ty;
3501
3502
18
  void Emit(CodeGenFunction &CGF, Flags flags) override {
3503
18
    QualType::DestructionKind DtorKind = Ty.isDestructedType();
3504
18
    if (DtorKind == QualType::DK_cxx_destructor) {
3505
16
      const CXXDestructorDecl *Dtor = Ty->getAsCXXRecordDecl()->getDestructor();
3506
16
      assert(!Dtor->isTrivial());
3507
16
      CGF.EmitCXXDestructorCall(Dtor, Dtor_Complete, /*for vbase*/ false,
3508
16
                                /*Delegating=*/false, Addr, Ty);
3509
16
    } else {
3510
2
      CGF.callCStructDestructor(CGF.MakeAddrLValue(Addr, Ty));
3511
2
    }
3512
18
  }
3513
};
3514
3515
struct DisableDebugLocationUpdates {
3516
  CodeGenFunction &CGF;
3517
  bool disabledDebugInfo;
3518
1.23M
  DisableDebugLocationUpdates(CodeGenFunction &CGF, const Expr *E) : CGF(CGF) {
3519
1.23M
    if ((disabledDebugInfo = isa<CXXDefaultArgExpr>(E) && 
CGF.getDebugInfo()13.1k
))
3520
8.77k
      CGF.disableDebugInfo();
3521
1.23M
  }
3522
1.23M
  ~DisableDebugLocationUpdates() {
3523
1.23M
    if (disabledDebugInfo)
3524
8.77k
      CGF.enableDebugInfo();
3525
1.23M
  }
3526
};
3527
3528
} // end anonymous namespace
3529
3530
5.05k
RValue CallArg::getRValue(CodeGenFunction &CGF) const {
3531
5.05k
  if (!HasLV)
3532
5.05k
    return RV;
3533
0
  LValue Copy = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty), Ty);
3534
0
  CGF.EmitAggregateCopy(Copy, LV, Ty, AggValueSlot::DoesNotOverlap,
3535
0
                        LV.isVolatile());
3536
0
  IsUsed = true;
3537
0
  return RValue::getAggregate(Copy.getAddress());
3538
0
}
3539
3540
1.69k
void CallArg::copyInto(CodeGenFunction &CGF, Address Addr) const {
3541
1.69k
  LValue Dst = CGF.MakeAddrLValue(Addr, Ty);
3542
1.69k
  if (!HasLV && 
RV.isScalar()1.49k
)
3543
1.12k
    CGF.EmitStoreOfScalar(RV.getScalarVal(), Dst, /*isInit=*/true);
3544
571
  else if (!HasLV && 
RV.isComplex()371
)
3545
371
    CGF.EmitStoreOfComplex(RV.getComplexVal(), Dst, /*init=*/true);
3546
200
  else {
3547
200
    auto Addr = HasLV ? LV.getAddress() : 
RV.getAggregateAddress()0
;
3548
200
    LValue SrcLV = CGF.MakeAddrLValue(Addr, Ty);
3549
200
    // We assume that call args are never copied into subobjects.
3550
200
    CGF.EmitAggregateCopy(Dst, SrcLV, Ty, AggValueSlot::DoesNotOverlap,
3551
200
                          HasLV ? LV.isVolatileQualified()
3552
200
                                : 
RV.isVolatileQualified()0
);
3553
200
  }
3554
1.69k
  IsUsed = true;
3555
1.69k
}
3556
3557
void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E,
3558
1.23M
                                  QualType type) {
3559
1.23M
  DisableDebugLocationUpdates Dis(*this, E);
3560
1.23M
  if (const ObjCIndirectCopyRestoreExpr *CRE
3561
26
        = dyn_cast<ObjCIndirectCopyRestoreExpr>(E)) {
3562
26
    assert(getLangOpts().ObjCAutoRefCount);
3563
26
    return emitWritebackArg(*this, args, CRE);
3564
26
  }
3565
1.23M
3566
1.23M
  assert(type->isReferenceType() == E->isGLValue() &&
3567
1.23M
         "reference binding to unmaterialized r-value!");
3568
1.23M
3569
1.23M
  if (E->isGLValue()) {
3570
144k
    assert(E->getObjectKind() == OK_Ordinary);
3571
144k
    return args.add(EmitReferenceBindingToExpr(E), type);
3572
144k
  }
3573
1.08M
3574
1.08M
  bool HasAggregateEvalKind = hasAggregateEvaluationKind(type);
3575
1.08M
3576
1.08M
  // In the Microsoft C++ ABI, aggregate arguments are destructed by the callee.
3577
1.08M
  // However, we still have to push an EH-only cleanup in case we unwind before
3578
1.08M
  // we make it to the call.
3579
1.08M
  if (HasAggregateEvalKind &&
3580
1.08M
      
type->getAs<RecordType>()->getDecl()->isParamDestroyedInCallee()17.3k
) {
3581
327
    // If we're using inalloca, use the argument memory.  Otherwise, use a
3582
327
    // temporary.
3583
327
    AggValueSlot Slot;
3584
327
    if (args.isUsingInAlloca())
3585
93
      Slot = createPlaceholderSlot(*this, type);
3586
234
    else
3587
234
      Slot = CreateAggTemp(type, "agg.tmp");
3588
327
3589
327
    bool DestroyedInCallee = true, NeedsEHCleanup = true;
3590
327
    if (const auto *RD = type->getAsCXXRecordDecl())
3591
317
      DestroyedInCallee = RD->hasNonTrivialDestructor();
3592
10
    else
3593
10
      NeedsEHCleanup = needsEHCleanup(type.isDestructedType());
3594
327
3595
327
    if (DestroyedInCallee)
3596
171
      Slot.setExternallyDestructed();
3597
327
3598
327
    EmitAggExpr(E, Slot);
3599
327
    RValue RV = Slot.asRValue();
3600
327
    args.add(RV, type);
3601
327
3602
327
    if (DestroyedInCallee && 
NeedsEHCleanup171
) {
3603
165
      // Create a no-op GEP between the placeholder and the cleanup so we can
3604
165
      // RAUW it successfully.  It also serves as a marker of the first
3605
165
      // instruction where the cleanup is active.
3606
165
      pushFullExprCleanup<DestroyUnpassedArg>(EHCleanup, Slot.getAddress(),
3607
165
                                              type);
3608
165
      // This unreachable is a temporary marker which will be removed later.
3609
165
      llvm::Instruction *IsActive = Builder.CreateUnreachable();
3610
165
      args.addArgCleanupDeactivation(EHStack.getInnermostEHScope(), IsActive);
3611
165
    }
3612
327
    return;
3613
327
  }
3614
1.08M
3615
1.08M
  if (HasAggregateEvalKind && 
isa<ImplicitCastExpr>(E)17.0k
&&
3616
1.08M
      
cast<CastExpr>(E)->getCastKind() == CK_LValueToRValue933
) {
3617
915
    LValue L = EmitLValue(cast<CastExpr>(E)->getSubExpr());
3618
915
    assert(L.isSimple());
3619
915
    args.addUncopiedAggregate(L, type);
3620
915
    return;
3621
915
  }
3622
1.08M
3623
1.08M
  args.add(EmitAnyExprToTemp(E), type);
3624
1.08M
}
3625
3626
71.6k
QualType CodeGenFunction::getVarArgType(const Expr *Arg) {
3627
71.6k
  // System headers on Windows define NULL to 0 instead of 0LL on Win64. MSVC
3628
71.6k
  // implicitly widens null pointer constants that are arguments to varargs
3629
71.6k
  // functions to pointer-sized ints.
3630
71.6k
  if (!getTarget().getTriple().isOSWindows())
3631
71.6k
    return Arg->getType();
3632
67
3633
67
  if (Arg->getType()->isIntegerType() &&
3634
67
      getContext().getTypeSize(Arg->getType()) <
3635
44
          getContext().getTargetInfo().getPointerWidth(0) &&
3636
67
      Arg->isNullPointerConstant(getContext(),
3637
20
                                 Expr::NPC_ValueDependentIsNotNull)) {
3638
1
    return getContext().getIntPtrType();
3639
1
  }
3640
66
3641
66
  return Arg->getType();
3642
66
}
3643
3644
// In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
3645
// optimizer it can aggressively ignore unwind edges.
3646
void
3647
1.54k
CodeGenFunction::AddObjCARCExceptionMetadata(llvm::Instruction *Inst) {
3648
1.54k
  if (CGM.getCodeGenOpts().OptimizationLevel != 0 &&
3649
1.54k
      
!CGM.getCodeGenOpts().ObjCAutoRefCountExceptions394
)
3650
306
    Inst->setMetadata("clang.arc.no_objc_arc_exceptions",
3651
306
                      CGM.getNoObjCARCExceptionsMetadata());
3652
1.54k
}
3653
3654
/// Emits a call to the given no-arguments nounwind runtime function.
3655
llvm::CallInst *
3656
CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
3657
429
                                         const llvm::Twine &name) {
3658
429
  return EmitNounwindRuntimeCall(callee, None, name);
3659
429
}
3660
3661
/// Emits a call to the given nounwind runtime function.
3662
llvm::CallInst *
3663
CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
3664
                                         ArrayRef<llvm::Value *> args,
3665
27.4k
                                         const llvm::Twine &name) {
3666
27.4k
  llvm::CallInst *call = EmitRuntimeCall(callee, args, name);
3667
27.4k
  call->setDoesNotThrow();
3668
27.4k
  return call;
3669
27.4k
}
3670
3671
/// Emits a simple call (never an invoke) to the given no-arguments
3672
/// runtime function.
3673
llvm::CallInst *CodeGenFunction::EmitRuntimeCall(llvm::FunctionCallee callee,
3674
6.47k
                                                 const llvm::Twine &name) {
3675
6.47k
  return EmitRuntimeCall(callee, None, name);
3676
6.47k
}
3677
3678
// Calls which may throw must have operand bundles indicating which funclet
3679
// they are nested within.
3680
SmallVector<llvm::OperandBundleDef, 1>
3681
984k
CodeGenFunction::getBundlesForFunclet(llvm::Value *Callee) {
3682
984k
  SmallVector<llvm::OperandBundleDef, 1> BundleList;
3683
984k
  // There is no need for a funclet operand bundle if we aren't inside a
3684
984k
  // funclet.
3685
984k
  if (!CurrentFuncletPad)
3686
984k
    return BundleList;
3687
329
3688
329
  // Skip intrinsics which cannot throw.
3689
329
  auto *CalleeFn = dyn_cast<llvm::Function>(Callee->stripPointerCasts());
3690
329
  if (CalleeFn && 
CalleeFn->isIntrinsic()323
&&
CalleeFn->doesNotThrow()14
)
3691
2
    return BundleList;
3692
327
3693
327
  BundleList.emplace_back("funclet", CurrentFuncletPad);
3694
327
  return BundleList;
3695
327
}
3696
3697
/// Emits a simple call (never an invoke) to the given runtime function.
3698
llvm::CallInst *CodeGenFunction::EmitRuntimeCall(llvm::FunctionCallee callee,
3699
                                                 ArrayRef<llvm::Value *> args,
3700
69.6k
                                                 const llvm::Twine &name) {
3701
69.6k
  llvm::CallInst *call = Builder.CreateCall(
3702
69.6k
      callee, args, getBundlesForFunclet(callee.getCallee()), name);
3703
69.6k
  call->setCallingConv(getRuntimeCC());
3704
69.6k
  return call;
3705
69.6k
}
3706
3707
/// Emits a call or invoke to the given noreturn runtime function.
3708
void CodeGenFunction::EmitNoreturnRuntimeCallOrInvoke(
3709
594
    llvm::FunctionCallee callee, ArrayRef<llvm::Value *> args) {
3710
594
  SmallVector<llvm::OperandBundleDef, 1> BundleList =
3711
594
      getBundlesForFunclet(callee.getCallee());
3712
594
3713
594
  if (getInvokeDest()) {
3714
184
    llvm::InvokeInst *invoke =
3715
184
      Builder.CreateInvoke(callee,
3716
184
                           getUnreachableBlock(),
3717
184
                           getInvokeDest(),
3718
184
                           args,
3719
184
                           BundleList);
3720
184
    invoke->setDoesNotReturn();
3721
184
    invoke->setCallingConv(getRuntimeCC());
3722
410
  } else {
3723
410
    llvm::CallInst *call = Builder.CreateCall(callee, args, BundleList);
3724
410
    call->setDoesNotReturn();
3725
410
    call->setCallingConv(getRuntimeCC());
3726
410
    Builder.CreateUnreachable();
3727
410
  }
3728
594
}
3729
3730
/// Emits a call or invoke instruction to the given nullary runtime function.
3731
llvm::CallBase *
3732
CodeGenFunction::EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
3733
677
                                         const Twine &name) {
3734
677
  return EmitRuntimeCallOrInvoke(callee, None, name);
3735
677
}
3736
3737
/// Emits a call or invoke instruction to the given runtime function.
3738
llvm::CallBase *
3739
CodeGenFunction::EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
3740
                                         ArrayRef<llvm::Value *> args,
3741
837
                                         const Twine &name) {
3742
837
  llvm::CallBase *call = EmitCallOrInvoke(callee, args, name);
3743
837
  call->setCallingConv(getRuntimeCC());
3744
837
  return call;
3745
837
}
3746
3747
/// Emits a call or invoke instruction to the given function, depending
3748
/// on the current state of the EH stack.
3749
llvm::CallBase *CodeGenFunction::EmitCallOrInvoke(llvm::FunctionCallee Callee,
3750
                                                  ArrayRef<llvm::Value *> Args,
3751
1.00k
                                                  const Twine &Name) {
3752
1.00k
  llvm::BasicBlock *InvokeDest = getInvokeDest();
3753
1.00k
  SmallVector<llvm::OperandBundleDef, 1> BundleList =
3754
1.00k
      getBundlesForFunclet(Callee.getCallee());
3755
1.00k
3756
1.00k
  llvm::CallBase *Inst;
3757
1.00k
  if (!InvokeDest)
3758
643
    Inst = Builder.CreateCall(Callee, Args, BundleList, Name);
3759
364
  else {
3760
364
    llvm::BasicBlock *ContBB = createBasicBlock("invoke.cont");
3761
364
    Inst = Builder.CreateInvoke(Callee, ContBB, InvokeDest, Args, BundleList,
3762
364
                                Name);
3763
364
    EmitBlock(ContBB);
3764
364
  }
3765
1.00k
3766
1.00k
  // In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
3767
1.00k
  // optimizer it can aggressively ignore unwind edges.
3768
1.00k
  if (CGM.getLangOpts().ObjCAutoRefCount)
3769
3
    AddObjCARCExceptionMetadata(Inst);
3770
1.00k
3771
1.00k
  return Inst;
3772
1.00k
}
3773
3774
void CodeGenFunction::deferPlaceholderReplacement(llvm::Instruction *Old,
3775
94
                                                  llvm::Value *New) {
3776
94
  DeferredReplacements.push_back(std::make_pair(Old, New));
3777
94
}
3778
3779
RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
3780
                                 const CGCallee &Callee,
3781
                                 ReturnValueSlot ReturnValue,
3782
                                 const CallArgList &CallArgs,
3783
                                 llvm::CallBase **callOrInvoke,
3784
904k
                                 SourceLocation Loc) {
3785
904k
  // FIXME: We no longer need the types from CallArgs; lift up and simplify.
3786
904k
3787
904k
  assert(Callee.isOrdinary() || Callee.isVirtual());
3788
904k
3789
904k
  // Handle struct-return functions by passing a pointer to the
3790
904k
  // location that we would like to return into.
3791
904k
  QualType RetTy = CallInfo.getReturnType();
3792
904k
  const ABIArgInfo &RetAI = CallInfo.getReturnInfo();
3793
904k
3794
904k
  llvm::FunctionType *IRFuncTy = getTypes().GetFunctionType(CallInfo);
3795
904k
3796
904k
  const Decl *TargetDecl = Callee.getAbstractInfo().getCalleeDecl().getDecl();
3797
904k
  if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(TargetDecl))
3798
896k
    // We can only guarantee that a function is called from the correct
3799
896k
    // context/function based on the appropriate target attributes,
3800
896k
    // so only check in the case where we have both always_inline and target
3801
896k
    // since otherwise we could be making a conditional call after a check for
3802
896k
    // the proper cpu features (and it won't cause code generation issues due to
3803
896k
    // function based code generation).
3804
896k
    if (TargetDecl->hasAttr<AlwaysInlineAttr>() &&
3805
896k
        
TargetDecl->hasAttr<TargetAttr>()58.8k
)
3806
11.7k
      checkTargetFeatures(Loc, FD);
3807
904k
3808
#ifndef NDEBUG
3809
  if (!(CallInfo.isVariadic() && CallInfo.getArgStruct())) {
3810
    // For an inalloca varargs function, we don't expect CallInfo to match the
3811
    // function pointer's type, because the inalloca struct a will have extra
3812
    // fields in it for the varargs parameters.  Code later in this function
3813
    // bitcasts the function pointer to the type derived from CallInfo.
3814
    //
3815
    // In other cases, we assert that the types match up (until pointers stop
3816
    // having pointee types).
3817
    llvm::Type *TypeFromVal;
3818
    if (Callee.isVirtual())
3819
      TypeFromVal = Callee.getVirtualFunctionType();
3820
    else
3821
      TypeFromVal =
3822
          Callee.getFunctionPointer()->getType()->getPointerElementType();
3823
    assert(IRFuncTy == TypeFromVal);
3824
  }
3825
#endif
3826
3827
904k
  // 1. Set up the arguments.
3828
904k
3829
904k
  // If we're using inalloca, insert the allocation after the stack save.
3830
904k
  // FIXME: Do this earlier rather than hacking it in here!
3831
904k
  Address ArgMemory = Address::invalid();
3832
904k
  if (llvm::StructType *ArgStruct = CallInfo.getArgStruct()) {
3833
67
    const llvm::DataLayout &DL = CGM.getDataLayout();
3834
67
    llvm::Instruction *IP = CallArgs.getStackBase();
3835
67
    llvm::AllocaInst *AI;
3836
67
    if (IP) {
3837
66
      IP = IP->getNextNode();
3838
66
      AI = new llvm::AllocaInst(ArgStruct, DL.getAllocaAddrSpace(),
3839
66
                                "argmem", IP);
3840
66
    } else {
3841
1
      AI = CreateTempAlloca(ArgStruct, "argmem");
3842
1
    }
3843
67
    auto Align = CallInfo.getArgStructAlignment();
3844
67
    AI->setAlignment(Align.getQuantity());
3845
67
    AI->setUsedWithInAlloca(true);
3846
67
    assert(AI->isUsedWithInAlloca() && !AI->isStaticAlloca());
3847
67
    ArgMemory = Address(AI, Align);
3848
67
  }
3849
904k
3850
904k
  ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), CallInfo);
3851
904k
  SmallVector<llvm::Value *, 16> IRCallArgs(IRFunctionArgs.totalIRArgs());
3852
904k
3853
904k
  // If the call returns a temporary with struct return, create a temporary
3854
904k
  // alloca to hold the result, unless one is given to us.
3855
904k
  Address SRetPtr = Address::invalid();
3856
904k
  Address SRetAlloca = Address::invalid();
3857
904k
  llvm::Value *UnusedReturnSizePtr = nullptr;
3858
904k
  if (RetAI.isIndirect() || 
RetAI.isInAlloca()898k
||
RetAI.isCoerceAndExpand()897k
) {
3859
7.43k
    if (!ReturnValue.isNull()) {
3860
7.28k
      SRetPtr = ReturnValue.getValue();
3861
7.28k
    } else {
3862
155
      SRetPtr = CreateMemTemp(RetTy, "tmp", &SRetAlloca);
3863
155
      if (HaveInsertPoint() && ReturnValue.isUnused()) {
3864
82
        uint64_t size =
3865
82
            CGM.getDataLayout().getTypeAllocSize(ConvertTypeForMem(RetTy));
3866
82
        UnusedReturnSizePtr = EmitLifetimeStart(size, SRetAlloca.getPointer());
3867
82
      }
3868
155
    }
3869
7.43k
    if (IRFunctionArgs.hasSRetArg()) {
3870
6.99k
      IRCallArgs[IRFunctionArgs.getSRetArgNo()] = SRetPtr.getPointer();
3871
6.99k
    } else 
if (446
RetAI.isInAlloca()446
) {
3872
4
      Address Addr =
3873
4
          Builder.CreateStructGEP(ArgMemory, RetAI.getInAllocaFieldIndex());
3874
4
      Builder.CreateStore(SRetPtr.getPointer(), Addr);
3875
4
    }
3876
7.43k
  }
3877
904k
3878
904k
  Address swiftErrorTemp = Address::invalid();
3879
904k
  Address swiftErrorArg = Address::invalid();
3880
904k
3881
904k
  // Translate all of the arguments as necessary to match the IR lowering.
3882
904k
  assert(CallInfo.arg_size() == CallArgs.size() &&
3883
904k
         "Mismatch between function signature & arguments.");
3884
904k
  unsigned ArgNo = 0;
3885
904k
  CGFunctionInfo::const_arg_iterator info_it = CallInfo.arg_begin();
3886
904k
  for (CallArgList::const_iterator I = CallArgs.begin(), E = CallArgs.end();
3887
2.54M
       I != E; 
++I, ++info_it, ++ArgNo1.64M
) {
3888
1.64M
    const ABIArgInfo &ArgInfo = info_it->info;
3889
1.64M
3890
1.64M
    // Insert a padding argument to ensure proper alignment.
3891
1.64M
    if (IRFunctionArgs.hasPaddingArg(ArgNo))
3892
17
      IRCallArgs[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
3893
17
          llvm::UndefValue::get(ArgInfo.getPaddingType());
3894
1.64M
3895
1.64M
    unsigned FirstIRArg, NumIRArgs;
3896
1.64M
    std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
3897
1.64M
3898
1.64M
    switch (ArgInfo.getKind()) {
3899
1.64M
    case ABIArgInfo::InAlloca: {
3900
145
      assert(NumIRArgs == 0);
3901
145
      assert(getTarget().getTriple().getArch() == llvm::Triple::x86);
3902
145
      if (I->isAggregate()) {
3903
94
        // Replace the placeholder with the appropriate argument slot GEP.
3904
94
        Address Addr = I->hasLValue()
3905
94
                           ? 
I->getKnownLValue().getAddress()0
3906
94
                           : I->getKnownRValue().getAggregateAddress();
3907
94
        llvm::Instruction *Placeholder =
3908
94
            cast<llvm::Instruction>(Addr.getPointer());
3909
94
        CGBuilderTy::InsertPoint IP = Builder.saveIP();
3910
94
        Builder.SetInsertPoint(Placeholder);
3911
94
        Addr =
3912
94
            Builder.CreateStructGEP(ArgMemory, ArgInfo.getInAllocaFieldIndex());
3913
94
        Builder.restoreIP(IP);
3914
94
        deferPlaceholderReplacement(Placeholder, Addr.getPointer());
3915
94
      } else {
3916
51
        // Store the RValue into the argument struct.
3917
51
        Address Addr =
3918
51
            Builder.CreateStructGEP(ArgMemory, ArgInfo.getInAllocaFieldIndex());
3919
51
        unsigned AS = Addr.getType()->getPointerAddressSpace();
3920
51
        llvm::Type *MemType = ConvertTypeForMem(I->Ty)->getPointerTo(AS);
3921
51
        // There are some cases where a trivial bitcast is not avoidable.  The
3922
51
        // definition of a type later in a translation unit may change it's type
3923
51
        // from {}* to (%struct.foo*)*.
3924
51
        if (Addr.getType() != MemType)
3925
1
          Addr = Builder.CreateBitCast(Addr, MemType);
3926
51
        I->copyInto(*this, Addr);
3927
51
      }
3928
145
      break;
3929
1.64M
    }
3930
1.64M
3931
1.64M
    case ABIArgInfo::Indirect: {
3932
3.88k
      assert(NumIRArgs == 1);
3933
3.88k
      if (!I->isAggregate()) {
3934
178
        // Make a temporary alloca to pass the argument.
3935
178
        Address Addr = CreateMemTempWithoutCast(
3936
178
            I->Ty, ArgInfo.getIndirectAlign(), "indirect-arg-temp");
3937
178
        IRCallArgs[FirstIRArg] = Addr.getPointer();
3938
178
3939
178
        I->copyInto(*this, Addr);
3940
3.70k
      } else {
3941
3.70k
        // We want to avoid creating an unnecessary temporary+copy here;
3942
3.70k
        // however, we need one in three cases:
3943
3.70k
        // 1. If the argument is not byval, and we are required to copy the
3944
3.70k
        //    source.  (This case doesn't occur on any common architecture.)
3945
3.70k
        // 2. If the argument is byval, RV is not sufficiently aligned, and
3946
3.70k
        //    we cannot force it to be sufficiently aligned.
3947
3.70k
        // 3. If the argument is byval, but RV is not located in default
3948
3.70k
        //    or alloca address space.
3949
3.70k
        Address Addr = I->hasLValue()
3950
3.70k
                           ? 
I->getKnownLValue().getAddress()276
3951
3.70k
                           : 
I->getKnownRValue().getAggregateAddress()3.43k
;
3952
3.70k
        llvm::Value *V = Addr.getPointer();
3953
3.70k
        CharUnits Align = ArgInfo.getIndirectAlign();
3954
3.70k
        const llvm::DataLayout *TD = &CGM.getDataLayout();
3955
3.70k
3956
3.70k
        assert((FirstIRArg >= IRFuncTy->getNumParams() ||
3957
3.70k
                IRFuncTy->getParamType(FirstIRArg)->getPointerAddressSpace() ==
3958
3.70k
                    TD->getAllocaAddrSpace()) &&
3959
3.70k
               "indirect argument must be in alloca address space");
3960
3.70k
3961
3.70k
        bool NeedCopy = false;
3962
3.70k
3963
3.70k
        if (Addr.getAlignment() < Align &&
3964
3.70k
            llvm::getOrEnforceKnownAlignment(V, Align.getQuantity(), *TD) <
3965
49
                Align.getQuantity()) {
3966
2
          NeedCopy = true;
3967
3.70k
        } else if (I->hasLValue()) {
3968
274
          auto LV = I->getKnownLValue();
3969
274
          auto AS = LV.getAddressSpace();
3970
274
3971
274
          if ((!ArgInfo.getIndirectByVal() &&
3972
274
               (LV.getAlignment() >=
3973
185
                getContext().getTypeAlignInChars(I->Ty)))) {
3974
185
            NeedCopy = true;
3975
185
          }
3976
274
          if (!getLangOpts().OpenCL) {
3977
244
            if ((ArgInfo.getIndirectByVal() &&
3978
244
                
(59
AS != LangAS::Default59
&&
3979
59
                 
AS != CGM.getASTAllocaAddressSpace()0
))) {
3980
0
              NeedCopy = true;
3981
0
            }
3982
244
          }
3983
30
          // For OpenCL even if RV is located in default or alloca address space
3984
30
          // we don't want to perform address space cast for it.
3985
30
          else if ((ArgInfo.getIndirectByVal() &&
3986
30
                    Addr.getType()->getAddressSpace() != IRFuncTy->
3987
30
                      getParamType(FirstIRArg)->getPointerAddressSpace())) {
3988
13
            NeedCopy = true;
3989
13
          }
3990
274
        }
3991
3.70k
3992
3.70k
        if (NeedCopy) {
3993
200
          // Create an aligned temporary, and copy to it.
3994
200
          Address AI = CreateMemTempWithoutCast(
3995
200
              I->Ty, ArgInfo.getIndirectAlign(), "byval-temp");
3996
200
          IRCallArgs[FirstIRArg] = AI.getPointer();
3997
200
          I->copyInto(*this, AI);
3998
3.50k
        } else {
3999
3.50k
          // Skip the extra memcpy call.
4000
3.50k
          auto *T = V->getType()->getPointerElementType()->getPointerTo(
4001
3.50k
              CGM.getDataLayout().getAllocaAddrSpace());
4002
3.50k
          IRCallArgs[FirstIRArg] = getTargetHooks().performAddrSpaceCast(
4003
3.50k
              *this, V, LangAS::Default, CGM.getASTAllocaAddressSpace(), T,
4004
3.50k
              true);
4005
3.50k
        }
4006
3.70k
      }
4007
3.88k
      break;
4008
1.64M
    }
4009
1.64M
4010
1.64M
    case ABIArgInfo::Ignore:
4011
5.53k
      assert(NumIRArgs == 0);
4012
5.53k
      break;
4013
1.64M
4014
1.64M
    case ABIArgInfo::Extend:
4015
1.63M
    case ABIArgInfo::Direct: {
4016
1.63M
      if (!isa<llvm::StructType>(ArgInfo.getCoerceToType()) &&
4017
1.63M
          
ArgInfo.getCoerceToType() == ConvertType(info_it->type)1.63M
&&
4018
1.63M
          
ArgInfo.getDirectOffset() == 01.62M
) {
4019
1.62M
        assert(NumIRArgs == 1);
4020
1.62M
        llvm::Value *V;
4021
1.62M
        if (!I->isAggregate())
4022
1.62M
          V = I->getKnownRValue().getScalarVal();
4023
0
        else
4024
0
          V = Builder.CreateLoad(
4025
0
              I->hasLValue() ? I->getKnownLValue().getAddress()
4026
0
                             : I->getKnownRValue().getAggregateAddress());
4027
1.62M
4028
1.62M
        // Implement swifterror by copying into a new swifterror argument.
4029
1.62M
        // We'll write back in the normal path out of the call.
4030
1.62M
        if (CallInfo.getExtParameterInfo(ArgNo).getABI()
4031
1.62M
              == ParameterABI::SwiftErrorResult) {
4032
8
          assert(!swiftErrorTemp.isValid() && "multiple swifterror args");
4033
8
4034
8
          QualType pointeeTy = I->Ty->getPointeeType();
4035
8
          swiftErrorArg =
4036
8
            Address(V, getContext().getTypeAlignInChars(pointeeTy));
4037
8
4038
8
          swiftErrorTemp =
4039
8
            CreateMemTemp(pointeeTy, getPointerAlign(), "swifterror.temp");
4040
8
          V = swiftErrorTemp.getPointer();
4041
8
          cast<llvm::AllocaInst>(V)->setSwiftError(true);
4042
8
4043
8
          llvm::Value *errorValue = Builder.CreateLoad(swiftErrorArg);
4044
8
          Builder.CreateStore(errorValue, swiftErrorTemp);
4045
8
        }
4046
1.62M
4047
1.62M
        // We might have to widen integers, but we should never truncate.
4048
1.62M
        if (ArgInfo.getCoerceToType() != V->getType() &&
4049
1.62M
            
V->getType()->isIntegerTy()383
)
4050
1
          V = Builder.CreateZExt(V, ArgInfo.getCoerceToType());
4051
1.62M
4052
1.62M
        // If the argument doesn't match, perform a bitcast to coerce it.  This
4053
1.62M
        // can happen due to trivial type mismatches.
4054
1.62M
        if (FirstIRArg < IRFuncTy->getNumParams() &&
4055
1.62M
            
V->getType() != IRFuncTy->getParamType(FirstIRArg)1.55M
)
4056
382
          V = Builder.CreateBitCast(V, IRFuncTy->getParamType(FirstIRArg));
4057
1.62M
4058
1.62M
        IRCallArgs[FirstIRArg] = V;
4059
1.62M
        break;
4060
1.62M
      }
4061
9.95k
4062
9.95k
      // FIXME: Avoid the conversion through memory if possible.
4063
9.95k
      Address Src = Address::invalid();
4064
9.95k
      if (!I->isAggregate()) {
4065
1.26k
        Src = CreateMemTemp(I->Ty, "coerce");
4066
1.26k
        I->copyInto(*this, Src);
4067
8.69k
      } else {
4068
8.69k
        Src = I->hasLValue() ? 
I->getKnownLValue().getAddress()620
4069
8.69k
                             : 
I->getKnownRValue().getAggregateAddress()8.07k
;
4070
8.69k
      }
4071
9.95k
4072
9.95k
      // If the value is offset in memory, apply the offset now.
4073
9.95k
      Src = emitAddressAtOffset(*this, Src, ArgInfo);
4074
9.95k
4075
9.95k
      // Fast-isel and the optimizer generally like scalar values better than
4076
9.95k
      // FCAs, so we flatten them if this is safe to do for this argument.
4077
9.95k
      llvm::StructType *STy =
4078
9.95k
            dyn_cast<llvm::StructType>(ArgInfo.getCoerceToType());
4079
9.95k
      if (STy && 
ArgInfo.isDirect()1.29k
&&
ArgInfo.getCanBeFlattened()1.29k
) {
4080
1.27k
        llvm::Type *SrcTy = Src.getType()->getElementType();
4081
1.27k
        uint64_t SrcSize = CGM.getDataLayout().getTypeAllocSize(SrcTy);
4082
1.27k
        uint64_t DstSize = CGM.getDataLayout().getTypeAllocSize(STy);
4083
1.27k
4084
1.27k
        // If the source type is smaller than the destination type of the
4085
1.27k
        // coerce-to logic, copy the source value into a temp alloca the size
4086
1.27k
        // of the destination type to allow loading all of it. The bits past
4087
1.27k
        // the source value are left undef.
4088
1.27k
        if (SrcSize < DstSize) {
4089
44
          Address TempAlloca
4090
44
            = CreateTempAlloca(STy, Src.getAlignment(),
4091
44
                               Src.getName() + ".coerce");
4092
44
          Builder.CreateMemCpy(TempAlloca, Src, SrcSize);
4093
44
          Src = TempAlloca;
4094
1.23k
        } else {
4095
1.23k
          Src = Builder.CreateBitCast(Src,
4096
1.23k
                                      STy->getPointerTo(Src.getAddressSpace()));
4097
1.23k
        }
4098
1.27k
4099
1.27k
        assert(NumIRArgs == STy->getNumElements());
4100
3.83k
        for (unsigned i = 0, e = STy->getNumElements(); i != e; 
++i2.55k
) {
4101
2.55k
          Address EltPtr = Builder.CreateStructGEP(Src, i);
4102
2.55k
          llvm::Value *LI = Builder.CreateLoad(EltPtr);
4103
2.55k
          IRCallArgs[FirstIRArg + i] = LI;
4104
2.55k
        }
4105
8.68k
      } else {
4106
8.68k
        // In the simple case, just pass the coerced loaded value.
4107
8.68k
        assert(NumIRArgs == 1);
4108
8.68k
        IRCallArgs[FirstIRArg] =
4109
8.68k
          CreateCoercedLoad(Src, ArgInfo.getCoerceToType(), *this);
4110
8.68k
      }
4111
9.95k
4112
9.95k
      break;
4113
9.95k
    }
4114
9.95k
4115
9.95k
    case ABIArgInfo::CoerceAndExpand: {
4116
442
      auto coercionType = ArgInfo.getCoerceAndExpandType();
4117
442
      auto layout = CGM.getDataLayout().getStructLayout(coercionType);
4118
442
4119
442
      llvm::Value *tempSize = nullptr;
4120
442
      Address addr = Address::invalid();
4121
442
      Address AllocaAddr = Address::invalid();
4122
442
      if (I->isAggregate()) {
4123
414
        addr = I->hasLValue() ? 
I->getKnownLValue().getAddress()0
4124
414
                              : I->getKnownRValue().getAggregateAddress();
4125
414
4126
414
      } else {
4127
28
        RValue RV = I->getKnownRValue();
4128
28
        assert(RV.isScalar()); // complex should always just be direct
4129
28
4130
28
        llvm::Type *scalarType = RV.getScalarVal()->getType();
4131
28
        auto scalarSize = CGM.getDataLayout().getTypeAllocSize(scalarType);
4132
28
        auto scalarAlign = CGM.getDataLayout().getPrefTypeAlignment(scalarType);
4133
28
4134
28
        // Materialize to a temporary.
4135
28
        addr = CreateTempAlloca(RV.getScalarVal()->getType(),
4136
28
                                CharUnits::fromQuantity(std::max(
4137
28
                                    layout->getAlignment(), scalarAlign)),
4138
28
                                "tmp",
4139
28
                                /*ArraySize=*/nullptr, &AllocaAddr);
4140
28
        tempSize = EmitLifetimeStart(scalarSize, AllocaAddr.getPointer());
4141
28
4142
28
        Builder.CreateStore(RV.getScalarVal(), addr);
4143
28
      }
4144
442
4145
442
      addr = Builder.CreateElementBitCast(addr, coercionType);
4146
442
4147
442
      unsigned IRArgPos = FirstIRArg;
4148
1.41k
      for (unsigned i = 0, e = coercionType->getNumElements(); i != e; 
++i973
) {
4149
973
        llvm::Type *eltType = coercionType->getElementType(i);
4150
973
        if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType)) 
continue7
;
4151
966
        Address eltAddr = Builder.CreateStructGEP(addr, i);
4152
966
        llvm::Value *elt = Builder.CreateLoad(eltAddr);
4153
966
        IRCallArgs[IRArgPos++] = elt;
4154
966
      }
4155
442
      assert(IRArgPos == FirstIRArg + NumIRArgs);
4156
442
4157
442
      if (tempSize) {
4158
0
        EmitLifetimeEnd(tempSize, AllocaAddr.getPointer());
4159
0
      }
4160
442
4161
442
      break;
4162
9.95k
    }
4163
9.95k
4164
9.95k
    case ABIArgInfo::Expand:
4165
27
      unsigned IRArgPos = FirstIRArg;
4166
27
      ExpandTypeToArgs(I->Ty, *I, IRFuncTy, IRCallArgs, IRArgPos);
4167
27
      assert(IRArgPos == FirstIRArg + NumIRArgs);
4168
27
      break;
4169
1.64M
    }
4170
1.64M
  }
4171
904k
4172
904k
  const CGCallee &ConcreteCallee = Callee.prepareConcreteCallee(*this);
4173
904k
  llvm::Value *CalleePtr = ConcreteCallee.getFunctionPointer();
4174
904k
4175
904k
  // If we're using inalloca, set up that argument.
4176
904k
  if (ArgMemory.isValid()) {
4177
67
    llvm::Value *Arg = ArgMemory.getPointer();
4178
67
    if (CallInfo.isVariadic()) {
4179
10
      // When passing non-POD arguments by value to variadic functions, we will
4180
10
      // end up with a variadic prototype and an inalloca call site.  In such
4181
10
      // cases, we can't do any parameter mismatch checks.  Give up and bitcast
4182
10
      // the callee.
4183
10
      unsigned CalleeAS = CalleePtr->getType()->getPointerAddressSpace();
4184
10
      CalleePtr =
4185
10
          Builder.CreateBitCast(CalleePtr, IRFuncTy->getPointerTo(CalleeAS));
4186
57
    } else {
4187
57
      llvm::Type *LastParamTy =
4188
57
          IRFuncTy->getParamType(IRFuncTy->getNumParams() - 1);
4189
57
      if (Arg->getType() != LastParamTy) {
4190
#ifndef NDEBUG
4191
        // Assert that these structs have equivalent element types.
4192
        llvm::StructType *FullTy = CallInfo.getArgStruct();
4193
        llvm::StructType *DeclaredTy = cast<llvm::StructType>(
4194
            cast<llvm::PointerType>(LastParamTy)->getElementType());
4195
        assert(DeclaredTy->getNumElements() == FullTy->getNumElements());
4196
        for (llvm::StructType::element_iterator DI = DeclaredTy->element_begin(),
4197
                                                DE = DeclaredTy->element_end(),
4198
                                                FI = FullTy->element_begin();
4199
             DI != DE; ++DI, ++FI)
4200
          assert(*DI == *FI);
4201
#endif
4202
        Arg = Builder.CreateBitCast(Arg, LastParamTy);
4203
0
      }
4204
57
    }
4205
67
    assert(IRFunctionArgs.hasInallocaArg());
4206
67
    IRCallArgs[IRFunctionArgs.getInallocaArgNo()] = Arg;
4207
67
  }
4208
904k
4209
904k
  // 2. Prepare the function pointer.
4210
904k
4211
904k
  // If the callee is a bitcast of a non-variadic function to have a
4212
904k
  // variadic function pointer type, check to see if we can remove the
4213
904k
  // bitcast.  This comes up with unprototyped functions.
4214
904k
  //
4215
904k
  // This makes the IR nicer, but more importantly it ensures that we
4216
904k
  // can inline the function at -O0 if it is marked always_inline.
4217
904k
  auto simplifyVariadicCallee = [](llvm::FunctionType *CalleeFT,
4218
904k
                                   llvm::Value *Ptr) -> llvm::Function * {
4219
904k
    if (!CalleeFT->isVarArg())
4220
853k
      return nullptr;
4221
51.8k
4222
51.8k
    // Get underlying value if it's a bitcast
4223
51.8k
    if (llvm::ConstantExpr *CE = dyn_cast<llvm::ConstantExpr>(Ptr)) {
4224
192
      if (CE->getOpcode() == llvm::Instruction::BitCast)
4225
192
        Ptr = CE->getOperand(0);
4226
192
    }
4227
51.8k
4228
51.8k
    llvm::Function *OrigFn = dyn_cast<llvm::Function>(Ptr);
4229
51.8k
    if (!OrigFn)
4230
128
      return nullptr;
4231
51.7k
4232
51.7k
    llvm::FunctionType *OrigFT = OrigFn->getFunctionType();
4233
51.7k
4234
51.7k
    // If the original type is variadic, or if any of the component types
4235
51.7k
    // disagree, we cannot remove the cast.
4236
51.7k
    if (OrigFT->isVarArg() ||
4237
51.7k
        
OrigFT->getNumParams() != CalleeFT->getNumParams()124
||
4238
51.7k
        
OrigFT->getReturnType() != CalleeFT->getReturnType()123
)
4239
51.5k
      return nullptr;
4240
123
4241
126
    
for (unsigned i = 0, e = OrigFT->getNumParams(); 123
i != e;
++i3
)
4242
3
      if (OrigFT->getParamType(i) != CalleeFT->getParamType(i))
4243
0
        return nullptr;
4244
123
4245
123
    return OrigFn;
4246
123
  };
4247
904k
4248
904k
  if (llvm::Function *OrigFn = simplifyVariadicCallee(IRFuncTy, CalleePtr)) {
4249
123
    CalleePtr = OrigFn;
4250
123
    IRFuncTy = OrigFn->getFunctionType();
4251
123
  }
4252
904k
4253
904k
  // 3. Perform the actual call.
4254
904k
4255
904k
  // Deactivate any cleanups that we're supposed to do immediately before
4256
904k
  // the call.
4257
904k
  if (!CallArgs.getCleanupsToDeactivate().empty())
4258
133
    deactivateArgCleanupsBeforeCall(*this, CallArgs);
4259
904k
4260
904k
  // Assert that the arguments we computed match up.  The IR verifier
4261
904k
  // will catch this, but this is a common enough source of problems
4262
904k
  // during IRGen changes that it's way better for debugging to catch
4263
904k
  // it ourselves here.
4264
#ifndef NDEBUG
4265
  assert(IRCallArgs.size() == IRFuncTy->getNumParams() || IRFuncTy->isVarArg());
4266
  for (unsigned i = 0; i < IRCallArgs.size(); ++i) {
4267
    // Inalloca argument can have different type.
4268
    if (IRFunctionArgs.hasInallocaArg() &&
4269
        i == IRFunctionArgs.getInallocaArgNo())
4270
      continue;
4271
    if (i < IRFuncTy->getNumParams())
4272
      assert(IRCallArgs[i]->getType() == IRFuncTy->getParamType(i));
4273
  }
4274
#endif
4275
4276
904k
  // Update the largest vector width if any arguments have vector types.
4277
2.55M
  for (unsigned i = 0; i < IRCallArgs.size(); 
++i1.64M
) {
4278
1.64M
    if (auto *VT = dyn_cast<llvm::VectorType>(IRCallArgs[i]->getType()))
4279
60.8k
      LargestVectorWidth = std::max(LargestVectorWidth,
4280
60.8k
                                    VT->getPrimitiveSizeInBits());
4281
1.64M
  }
4282
904k
4283
904k
  // Compute the calling convention and attributes.
4284
904k
  unsigned CallingConv;
4285
904k
  llvm::AttributeList Attrs;
4286
904k
  CGM.ConstructAttributeList(CalleePtr->getName(), CallInfo,
4287
904k
                             Callee.getAbstractInfo(), Attrs, CallingConv,
4288
904k
                             /*AttrOnCallSite=*/true);
4289
904k
4290
904k
  // Apply some call-site-specific attributes.
4291
904k
  // TODO: work this into building the attribute set.
4292
904k
4293
904k
  // Apply always_inline to all calls within flatten functions.
4294
904k
  // FIXME: should this really take priority over __try, below?
4295
904k
  if (CurCodeDecl && 
CurCodeDecl->hasAttr<FlattenAttr>()904k
&&
4296
904k
      
!(3
TargetDecl3
&&
TargetDecl->hasAttr<NoInlineAttr>()3
)) {
4297
2
    Attrs =
4298
2
        Attrs.addAttribute(getLLVMContext(), llvm::AttributeList::FunctionIndex,
4299
2
                           llvm::Attribute::AlwaysInline);
4300
2
  }
4301
904k
4302
904k
  // Disable inlining inside SEH __try blocks.
4303
904k
  if (isSEHTryScope()) {
4304
99
    Attrs =
4305
99
        Attrs.addAttribute(getLLVMContext(), llvm::AttributeList::FunctionIndex,
4306
99
                           llvm::Attribute::NoInline);
4307
99
  }
4308
904k
4309
904k
  // Decide whether to use a call or an invoke.
4310
904k
  bool CannotThrow;
4311
904k
  if (currentFunctionUsesSEHTry()) {
4312
240
    // SEH cares about asynchronous exceptions, so everything can "throw."
4313
240
    CannotThrow = false;
4314
904k
  } else if (isCleanupPadScope() &&
4315
904k
             
EHPersonality::get(*this).isMSVCXXPersonality()78
) {
4316
64
    // The MSVC++ personality will implicitly terminate the program if an
4317
64
    // exception is thrown during a cleanup outside of a try/catch.
4318
64
    // We don't need to model anything in IR to get this behavior.
4319
64
    CannotThrow = true;
4320
904k
  } else {
4321
904k
    // Otherwise, nounwind call sites will never throw.
4322
904k
    CannotThrow = Attrs.hasAttribute(llvm::AttributeList::FunctionIndex,
4323
904k
                                     llvm::Attribute::NoUnwind);
4324
904k
  }
4325
904k
4326
904k
  // If we made a temporary, be sure to clean up after ourselves. Note that we
4327
904k
  // can't depend on being inside of an ExprWithCleanups, so we need to manually
4328
904k
  // pop this cleanup later on. Being eager about this is OK, since this
4329
904k
  // temporary is 'invisible' outside of the callee.
4330
904k
  if (UnusedReturnSizePtr)
4331
43
    pushFullExprCleanup<CallLifetimeEnd>(NormalEHLifetimeMarker, SRetAlloca,
4332
43
                                         UnusedReturnSizePtr);
4333
904k
4334
904k
  llvm::BasicBlock *InvokeDest = CannotThrow ? 
nullptr177k
:
getInvokeDest()727k
;
4335
904k
4336
904k
  SmallVector<llvm::OperandBundleDef, 1> BundleList =
4337
904k
      getBundlesForFunclet(CalleePtr);
4338
904k
4339
904k
  // Emit the actual call/invoke instruction.
4340
904k
  llvm::CallBase *CI;
4341
904k
  if (!InvokeDest) {
4342
867k
    CI = Builder.CreateCall(IRFuncTy, CalleePtr, IRCallArgs, BundleList);
4343
867k
  } else {
4344
37.4k
    llvm::BasicBlock *Cont = createBasicBlock("invoke.cont");
4345
37.4k
    CI = Builder.CreateInvoke(IRFuncTy, CalleePtr, Cont, InvokeDest, IRCallArgs,
4346
37.4k
                              BundleList);
4347
37.4k
    EmitBlock(Cont);
4348
37.4k
  }
4349
904k
  if (callOrInvoke)
4350
528k
    *callOrInvoke = CI;
4351
904k
4352
904k
  // Apply the attributes and calling convention.
4353
904k
  CI->setAttributes(Attrs);
4354
904k
  CI->setCallingConv(static_cast<llvm::CallingConv::ID>(CallingConv));
4355
904k
4356
904k
  // Apply various metadata.
4357
904k
4358
904k
  if (!CI->getType()->isVoidTy())
4359
613k
    CI->setName("call");
4360
904k
4361
904k
  // Update largest vector width from the return type.
4362
904k
  if (auto *VT = dyn_cast<llvm::VectorType>(CI->getType()))
4363
43.3k
    LargestVectorWidth = std::max(LargestVectorWidth,
4364
43.3k
                                  VT->getPrimitiveSizeInBits());
4365
904k
4366
904k
  // Insert instrumentation or attach profile metadata at indirect call sites.
4367
904k
  // For more details, see the comment before the definition of
4368
904k
  // IPVK_IndirectCallTarget in InstrProfData.inc.
4369
904k
  if (!CI->getCalledFunction())
4370
21.3k
    PGO.valueProfile(Builder, llvm::IPVK_IndirectCallTarget,
4371
21.3k
                     CI, CalleePtr);
4372
904k
4373
904k
  // In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
4374
904k
  // optimizer it can aggressively ignore unwind edges.
4375
904k
  if (CGM.getLangOpts().ObjCAutoRefCount)
4376
1.54k
    AddObjCARCExceptionMetadata(CI);
4377
904k
4378
904k
  // Suppress tail calls if requested.
4379
904k
  if (llvm::CallInst *Call = dyn_cast<llvm::CallInst>(CI)) {
4380
867k
    if (TargetDecl && 
TargetDecl->hasAttr<NotTailCalledAttr>()861k
)
4381
147
      Call->setTailCallKind(llvm::CallInst::TCK_NoTail);
4382
867k
  }
4383
904k
4384
904k
  // Add metadata for calls to MSAllocator functions
4385
904k
  if (getDebugInfo() && 
TargetDecl353k
&&
4386
904k
      
TargetDecl->hasAttr<MSAllocatorAttr>()352k
)
4387
3
    getDebugInfo()->addHeapAllocSiteMetadata(CI, RetTy, Loc);
4388
904k
4389
904k
  // 4. Finish the call.
4390
904k
4391
904k
  // If the call doesn't return, finish the basic block and clear the
4392
904k
  // insertion point; this allows the rest of IRGen to discard
4393
904k
  // unreachable code.
4394
904k
  if (CI->doesNotReturn()) {
4395
23.1k
    if (UnusedReturnSizePtr)
4396
0
      PopCleanupBlock();
4397
23.1k
4398
23.1k
    // Strip away the noreturn attribute to better diagnose unreachable UB.
4399
23.1k
    if (SanOpts.has(SanitizerKind::Unreachable)) {
4400
8
      // Also remove from function since CallBase::hasFnAttr additionally checks
4401
8
      // attributes of the called function.
4402
8
      if (auto *F = CI->getCalledFunction())
4403
7
        F->removeFnAttr(llvm::Attribute::NoReturn);
4404
8
      CI->removeAttribute(llvm::AttributeList::FunctionIndex,
4405
8
                          llvm::Attribute::NoReturn);
4406
8
4407
8
      // Avoid incompatibility with ASan which relies on the `noreturn`
4408
8
      // attribute to insert handler calls.
4409
8
      if (SanOpts.hasOneOf(SanitizerKind::Address |
4410
8
                           SanitizerKind::KernelAddress)) {
4411
2
        SanitizerScope SanScope(this);
4412
2
        llvm::IRBuilder<>::InsertPointGuard IPGuard(Builder);
4413
2
        Builder.SetInsertPoint(CI);
4414
2
        auto *FnType = llvm::FunctionType::get(CGM.VoidTy, /*isVarArg=*/false);
4415
2
        llvm::FunctionCallee Fn =
4416
2
            CGM.CreateRuntimeFunction(FnType, "__asan_handle_no_return");
4417
2
        EmitNounwindRuntimeCall(Fn);
4418
2
      }
4419
8
    }
4420
23.1k
4421
23.1k
    EmitUnreachable(Loc);
4422
23.1k
    Builder.ClearInsertionPoint();
4423
23.1k
4424
23.1k
    // FIXME: For now, emit a dummy basic block because expr emitters in
4425
23.1k
    // generally are not ready to handle emitting expressions at unreachable
4426
23.1k
    // points.
4427
23.1k
    EnsureInsertPoint();
4428
23.1k
4429
23.1k
    // Return a reasonable RValue.
4430
23.1k
    return GetUndefRValue(RetTy);
4431
23.1k
  }
4432
881k
4433
881k
  // Perform the swifterror writeback.
4434
881k
  if (swiftErrorTemp.isValid()) {
4435
8
    llvm::Value *errorResult = Builder.CreateLoad(swiftErrorTemp);
4436
8
    Builder.CreateStore(errorResult, swiftErrorArg);
4437
8
  }
4438
881k
4439
881k
  // Emit any call-associated writebacks immediately.  Arguably this
4440
881k
  // should happen after any return-value munging.
4441
881k
  if (CallArgs.hasWritebacks())
4442
26
    emitWritebacks(*this, CallArgs);
4443
881k
4444
881k
  // The stack cleanup for inalloca arguments has to run out of the normal
4445
881k
  // lexical order, so deactivate it and run it manually here.
4446
881k
  CallArgs.freeArgumentMemory(*this);
4447
881k
4448
881k
  // Extract the return value.
4449
881k
  RValue Ret = [&] {
4450
881k
    switch (RetAI.getKind()) {
4451
881k
    case ABIArgInfo::CoerceAndExpand: {
4452
442
      auto coercionType = RetAI.getCoerceAndExpandType();
4453
442
4454
442
      Address addr = SRetPtr;
4455
442
      addr = Builder.CreateElementBitCast(addr, coercionType);
4456
442
4457
442
      assert(CI->getType() == RetAI.getUnpaddedCoerceAndExpandType());
4458
442
      bool requiresExtract = isa<llvm::StructType>(CI->getType());
4459
442
4460
442
      unsigned unpaddedIndex = 0;
4461
1.41k
      for (unsigned i = 0, e = coercionType->getNumElements(); i != e; 
++i973
) {
4462
973
        llvm::Type *eltType = coercionType->getElementType(i);
4463
973
        if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType)) 
continue7
;
4464
966
        Address eltAddr = Builder.CreateStructGEP(addr, i);
4465
966
        llvm::Value *elt = CI;
4466
966
        if (requiresExtract)
4467
810
          elt = Builder.CreateExtractValue(elt, unpaddedIndex++);
4468
966
        else
4469
966
          assert(unpaddedIndex == 0);
4470
966
        Builder.CreateStore(elt, eltAddr);
4471
966
      }
4472
442
      // FALLTHROUGH
4473
442
      LLVM_FALLTHROUGH;
4474
442
    }
4475
442
4476
7.43k
    case ABIArgInfo::InAlloca:
4477
7.43k
    case ABIArgInfo::Indirect: {
4478
7.43k
      RValue ret = convertTempToRValue(SRetPtr, RetTy, SourceLocation());
4479
7.43k
      if (UnusedReturnSizePtr)
4480
43
        PopCleanupBlock();
4481
7.43k
      return ret;
4482
7.43k
    }
4483
7.43k
4484
261k
    case ABIArgInfo::Ignore:
4485
261k
      // If we are ignoring an argument that had a result, make sure to
4486
261k
      // construct the appropriate return value for our caller.
4487
261k
      return GetUndefRValue(RetTy);
4488
7.43k
4489
612k
    case ABIArgInfo::Extend:
4490
612k
    case ABIArgInfo::Direct: {
4491
612k
      llvm::Type *RetIRTy = ConvertType(RetTy);
4492
612k
      if (RetAI.getCoerceToType() == RetIRTy && 
RetAI.getDirectOffset() == 0603k
) {
4493
603k
        switch (getEvaluationKind(RetTy)) {
4494
603k
        case TEK_Complex: {
4495
250
          llvm::Value *Real = Builder.CreateExtractValue(CI, 0);
4496
250
          llvm::Value *Imag = Builder.CreateExtractValue(CI, 1);
4497
250
          return RValue::getComplex(std::make_pair(Real, Imag));
4498
603k
        }
4499
603k
        case TEK_Aggregate: {
4500
3.10k
          Address DestPtr = ReturnValue.getValue();
4501
3.10k
          bool DestIsVolatile = ReturnValue.isVolatile();
4502
3.10k
4503
3.10k
          if (!DestPtr.isValid()) {
4504
6
            DestPtr = CreateMemTemp(RetTy, "agg.tmp");
4505
6
            DestIsVolatile = false;
4506
6
          }
4507
3.10k
          BuildAggStore(*this, CI, DestPtr, DestIsVolatile);
4508
3.10k
          return RValue::getAggregate(DestPtr);
4509
603k
        }
4510
603k
        case TEK_Scalar: {
4511
600k
          // If the argument doesn't match, perform a bitcast to coerce it.  This
4512
600k
          // can happen due to trivial type mismatches.
4513
600k
          llvm::Value *V = CI;
4514
600k
          if (V->getType() != RetIRTy)
4515
0
            V = Builder.CreateBitCast(V, RetIRTy);
4516
600k
          return RValue::get(V);
4517
0
        }
4518
0
        }
4519
0
        llvm_unreachable("bad evaluation kind");
4520
0
      }
4521
9.04k
4522
9.04k
      Address DestPtr = ReturnValue.getValue();
4523
9.04k
      bool DestIsVolatile = ReturnValue.isVolatile();
4524
9.04k
4525
9.04k
      if (!DestPtr.isValid()) {
4526
1.22k
        DestPtr = CreateMemTemp(RetTy, "coerce");
4527
1.22k
        DestIsVolatile = false;
4528
1.22k
      }
4529
9.04k
4530
9.04k
      // If the value is offset in memory, apply the offset now.
4531
9.04k
      Address StorePtr = emitAddressAtOffset(*this, DestPtr, RetAI);
4532
9.04k
      CreateCoercedStore(CI, StorePtr, DestIsVolatile, *this);
4533
9.04k
4534
9.04k
      return convertTempToRValue(DestPtr, RetTy, SourceLocation());
4535
9.04k
    }
4536
9.04k
4537
9.04k
    case ABIArgInfo::Expand:
4538
0
      llvm_unreachable("Invalid ABI kind for return argument");
4539
0
    }
4540
0
4541
0
    llvm_unreachable("Unhandled ABIArgInfo::Kind");
4542
0
  } ();
4543
881k
4544
881k
  // Emit the assume_aligned check on the return value.
4545
881k
  if (Ret.isScalar() && 
TargetDecl861k
) {
4546
855k
    if (const auto *AA = TargetDecl->getAttr<AssumeAlignedAttr>()) {
4547
10
      llvm::Value *OffsetValue = nullptr;
4548
10
      if (const auto *Offset = AA->getOffset())
4549
5
        OffsetValue = EmitScalarExpr(Offset);
4550
10
4551
10
      llvm::Value *Alignment = EmitScalarExpr(AA->getAlignment());
4552
10
      llvm::ConstantInt *AlignmentCI = cast<llvm::ConstantInt>(Alignment);
4553
10
      EmitAlignmentAssumption(Ret.getScalarVal(), RetTy, Loc, AA->getLocation(),
4554
10
                              AlignmentCI->getZExtValue(), OffsetValue);
4555
855k
    } else if (const auto *AA = TargetDecl->getAttr<AllocAlignAttr>()) {
4556
14
      llvm::Value *AlignmentVal = CallArgs[AA->getParamIndex().getLLVMIndex()]
4557
14
                                      .getRValue(*this)
4558
14
                                      .getScalarVal();
4559
14
      EmitAlignmentAssumption(Ret.getScalarVal(), RetTy, Loc, AA->getLocation(),
4560
14
                              AlignmentVal);
4561
14
    }
4562
855k
  }
4563
881k
4564
881k
  return Ret;
4565
881k
}
4566
4567
904k
CGCallee CGCallee::prepareConcreteCallee(CodeGenFunction &CGF) const {
4568
904k
  if (isVirtual()) {
4569
10.6k
    const CallExpr *CE = getVirtualCallExpr();
4570
10.6k
    return CGF.CGM.getCXXABI().getVirtualFunctionPointer(
4571
10.6k
        CGF, getVirtualMethodDecl(), getThisAddress(), getVirtualFunctionType(),
4572
10.6k
        CE ? 
CE->getBeginLoc()10.3k
:
SourceLocation()271
);
4573
10.6k
  }
4574
894k
4575
894k
  return *this;
4576
894k
}
4577
4578
/* VarArg handling */
4579
4580
1.11k
Address CodeGenFunction::EmitVAArg(VAArgExpr *VE, Address &VAListAddr) {
4581
1.11k
  VAListAddr = VE->isMicrosoftABI()
4582
1.11k
                 ? 
EmitMSVAListRef(VE->getSubExpr())9
4583
1.11k
                 : 
EmitVAListRef(VE->getSubExpr())1.10k
;
4584
1.11k
  QualType Ty = VE->getType();
4585
1.11k
  if (VE->isMicrosoftABI())
4586
9
    return CGM.getTypes().getABIInfo().EmitMSVAArg(*this, VAListAddr, Ty);
4587
1.10k
  return CGM.getTypes().getABIInfo().EmitVAArg(*this, VAListAddr, Ty);
4588
1.10k
}