Coverage Report

Created: 2022-05-17 06:19

/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/CodeGen/CGCall.cpp
Line
Count
Source (jump to first uncovered line)
1
//===--- CGCall.cpp - Encapsulate calling convention details --------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// These classes wrap the information about a call or function
10
// definition used to handle ABI compliancy.
11
//
12
//===----------------------------------------------------------------------===//
13
14
#include "CGCall.h"
15
#include "ABIInfo.h"
16
#include "CGBlocks.h"
17
#include "CGCXXABI.h"
18
#include "CGCleanup.h"
19
#include "CGRecordLayout.h"
20
#include "CodeGenFunction.h"
21
#include "CodeGenModule.h"
22
#include "TargetInfo.h"
23
#include "clang/AST/Attr.h"
24
#include "clang/AST/Decl.h"
25
#include "clang/AST/DeclCXX.h"
26
#include "clang/AST/DeclObjC.h"
27
#include "clang/Basic/CodeGenOptions.h"
28
#include "clang/Basic/TargetBuiltins.h"
29
#include "clang/Basic/TargetInfo.h"
30
#include "clang/CodeGen/CGFunctionInfo.h"
31
#include "clang/CodeGen/SwiftCallingConv.h"
32
#include "llvm/ADT/StringExtras.h"
33
#include "llvm/Analysis/ValueTracking.h"
34
#include "llvm/IR/Assumptions.h"
35
#include "llvm/IR/Attributes.h"
36
#include "llvm/IR/CallingConv.h"
37
#include "llvm/IR/DataLayout.h"
38
#include "llvm/IR/InlineAsm.h"
39
#include "llvm/IR/IntrinsicInst.h"
40
#include "llvm/IR/Intrinsics.h"
41
#include "llvm/IR/Type.h"
42
#include "llvm/Transforms/Utils/Local.h"
43
using namespace clang;
44
using namespace CodeGen;
45
46
/***/
47
48
238k
unsigned CodeGenTypes::ClangCallConvToLLVMCallConv(CallingConv CC) {
49
238k
  switch (CC) {
50
233k
  default: return llvm::CallingConv::C;
51
56
  case CC_X86StdCall: return llvm::CallingConv::X86_StdCall;
52
74
  case CC_X86FastCall: return llvm::CallingConv::X86_FastCall;
53
108
  case CC_X86RegCall: return llvm::CallingConv::X86_RegCall;
54
2.67k
  case CC_X86ThisCall: return llvm::CallingConv::X86_ThisCall;
55
19
  case CC_Win64: return llvm::CallingConv::Win64;
56
14
  case CC_X86_64SysV: return llvm::CallingConv::X86_64_SysV;
57
5
  case CC_AAPCS: return llvm::CallingConv::ARM_AAPCS;
58
11
  case CC_AAPCS_VFP: return llvm::CallingConv::ARM_AAPCS_VFP;
59
3
  case CC_IntelOclBicc: return llvm::CallingConv::Intel_OCL_BI;
60
  // TODO: Add support for __pascal to LLVM.
61
3
  case CC_X86Pascal: return llvm::CallingConv::C;
62
  // TODO: Add support for __vectorcall to LLVM.
63
112
  case CC_X86VectorCall: return llvm::CallingConv::X86_VectorCall;
64
2
  case CC_AArch64VectorCall: return llvm::CallingConv::AArch64_VectorCall;
65
2
  case CC_AArch64SVEPCS: return llvm::CallingConv::AArch64_SVE_VectorCall;
66
431
  case CC_SpirFunction: return llvm::CallingConv::SPIR_FUNC;
67
381
  case CC_OpenCLKernel: return CGM.getTargetCodeGenInfo().getOpenCLKernelCallingConv();
68
10
  case CC_PreserveMost: return llvm::CallingConv::PreserveMost;
69
6
  case CC_PreserveAll: return llvm::CallingConv::PreserveAll;
70
1.14k
  case CC_Swift: return llvm::CallingConv::Swift;
71
73
  case CC_SwiftAsync: return llvm::CallingConv::SwiftTail;
72
238k
  }
73
238k
}
74
75
/// Derives the 'this' type for codegen purposes, i.e. ignoring method CVR
76
/// qualification. Either or both of RD and MD may be null. A null RD indicates
77
/// that there is no meaningful 'this' type, and a null MD can occur when
78
/// calling a method pointer.
79
CanQualType CodeGenTypes::DeriveThisType(const CXXRecordDecl *RD,
80
622k
                                         const CXXMethodDecl *MD) {
81
622k
  QualType RecTy;
82
622k
  if (RD)
83
621k
    RecTy = Context.getTagDeclType(RD)->getCanonicalTypeInternal();
84
688
  else
85
688
    RecTy = Context.VoidTy;
86
87
622k
  if (MD)
88
621k
    RecTy = Context.getAddrSpaceQualType(RecTy, MD->getMethodQualifiers().getAddressSpace());
89
622k
  return Context.getPointerType(CanQualType::CreateUnsafe(RecTy));
90
622k
}
91
92
/// Returns the canonical formal type of the given C++ method.
93
577k
static CanQual<FunctionProtoType> GetFormalType(const CXXMethodDecl *MD) {
94
577k
  return MD->getType()->getCanonicalTypeUnqualified()
95
577k
           .getAs<FunctionProtoType>();
96
577k
}
97
98
/// Returns the "extra-canonicalized" return type, which discards
99
/// qualifiers on the return type.  Codegen doesn't care about them,
100
/// and it makes ABI code a little easier to be able to assume that
101
/// all parameter and return types are top-level unqualified.
102
318k
static CanQualType GetReturnType(QualType RetTy) {
103
318k
  return RetTy->getCanonicalTypeUnqualified().getUnqualifiedType();
104
318k
}
105
106
/// Arrange the argument and result information for a value of the given
107
/// unprototyped freestanding function type.
108
const CGFunctionInfo &
109
389
CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionNoProtoType> FTNP) {
110
  // When translating an unprototyped function type, always use a
111
  // variadic type.
112
389
  return arrangeLLVMFunctionInfo(FTNP->getReturnType().getUnqualifiedType(),
113
389
                                 /*instanceMethod=*/false,
114
389
                                 /*chainCall=*/false, None,
115
389
                                 FTNP->getExtInfo(), {}, RequiredArgs(0));
116
389
}
117
118
static void addExtParameterInfosForCall(
119
         llvm::SmallVectorImpl<FunctionProtoType::ExtParameterInfo> &paramInfos,
120
                                        const FunctionProtoType *proto,
121
                                        unsigned prefixArgs,
122
1.24k
                                        unsigned totalArgs) {
123
1.24k
  assert(proto->hasExtParameterInfos());
124
0
  assert(paramInfos.size() <= prefixArgs);
125
0
  assert(proto->getNumParams() + prefixArgs <= totalArgs);
126
127
0
  paramInfos.reserve(totalArgs);
128
129
  // Add default infos for any prefix args that don't already have infos.
130
1.24k
  paramInfos.resize(prefixArgs);
131
132
  // Add infos for the prototype.
133
2.06k
  for (const auto &ParamInfo : proto->getExtParameterInfos()) {
134
2.06k
    paramInfos.push_back(ParamInfo);
135
    // pass_object_size params have no parameter info.
136
2.06k
    if (ParamInfo.hasPassObjectSize())
137
447
      paramInfos.emplace_back();
138
2.06k
  }
139
140
1.24k
  assert(paramInfos.size() <= totalArgs &&
141
1.24k
         "Did we forget to insert pass_object_size args?");
142
  // Add default infos for the variadic and/or suffix arguments.
143
0
  paramInfos.resize(totalArgs);
144
1.24k
}
145
146
/// Adds the formal parameters in FPT to the given prefix. If any parameter in
147
/// FPT has pass_object_size attrs, then we'll add parameters for those, too.
148
static void appendParameterTypes(const CodeGenTypes &CGT,
149
                                 SmallVectorImpl<CanQualType> &prefix,
150
              SmallVectorImpl<FunctionProtoType::ExtParameterInfo> &paramInfos,
151
1.00M
                                 CanQual<FunctionProtoType> FPT) {
152
  // Fast path: don't touch param info if we don't need to.
153
1.00M
  if (!FPT->hasExtParameterInfos()) {
154
1.00M
    assert(paramInfos.empty() &&
155
1.00M
           "We have paramInfos, but the prototype doesn't?");
156
0
    prefix.append(FPT->param_type_begin(), FPT->param_type_end());
157
1.00M
    return;
158
1.00M
  }
159
160
825
  unsigned PrefixSize = prefix.size();
161
  // In the vast majority of cases, we'll have precisely FPT->getNumParams()
162
  // parameters; the only thing that can change this is the presence of
163
  // pass_object_size. So, we preallocate for the common case.
164
825
  prefix.reserve(prefix.size() + FPT->getNumParams());
165
166
825
  auto ExtInfos = FPT->getExtParameterInfos();
167
825
  assert(ExtInfos.size() == FPT->getNumParams());
168
2.24k
  for (unsigned I = 0, E = FPT->getNumParams(); I != E; 
++I1.42k
) {
169
1.42k
    prefix.push_back(FPT->getParamType(I));
170
1.42k
    if (ExtInfos[I].hasPassObjectSize())
171
299
      prefix.push_back(CGT.getContext().getSizeType());
172
1.42k
  }
173
174
825
  addExtParameterInfosForCall(paramInfos, FPT.getTypePtr(), PrefixSize,
175
825
                              prefix.size());
176
825
}
177
178
/// Arrange the LLVM function layout for a value of the given function
179
/// type, on top of any implicit parameters already stored.
180
static const CGFunctionInfo &
181
arrangeLLVMFunctionInfo(CodeGenTypes &CGT, bool instanceMethod,
182
                        SmallVectorImpl<CanQualType> &prefix,
183
707k
                        CanQual<FunctionProtoType> FTP) {
184
707k
  SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos;
185
707k
  RequiredArgs Required = RequiredArgs::forPrototypePlus(FTP, prefix.size());
186
  // FIXME: Kill copy.
187
707k
  appendParameterTypes(CGT, prefix, paramInfos, FTP);
188
707k
  CanQualType resultType = FTP->getReturnType().getUnqualifiedType();
189
190
707k
  return CGT.arrangeLLVMFunctionInfo(resultType, instanceMethod,
191
707k
                                     /*chainCall=*/false, prefix,
192
707k
                                     FTP->getExtInfo(), paramInfos,
193
707k
                                     Required);
194
707k
}
195
196
/// Arrange the argument and result information for a value of the
197
/// given freestanding function type.
198
const CGFunctionInfo &
199
485k
CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionProtoType> FTP) {
200
485k
  SmallVector<CanQualType, 16> argTypes;
201
485k
  return ::arrangeLLVMFunctionInfo(*this, /*instanceMethod=*/false, argTypes,
202
485k
                                   FTP);
203
485k
}
204
205
static CallingConv getCallingConventionForDecl(const ObjCMethodDecl *D,
206
27.6k
                                               bool IsWindows) {
207
  // Set the appropriate calling convention for the Function.
208
27.6k
  if (D->hasAttr<StdCallAttr>())
209
2
    return CC_X86StdCall;
210
211
27.6k
  if (D->hasAttr<FastCallAttr>())
212
0
    return CC_X86FastCall;
213
214
27.6k
  if (D->hasAttr<RegCallAttr>())
215
0
    return CC_X86RegCall;
216
217
27.6k
  if (D->hasAttr<ThisCallAttr>())
218
0
    return CC_X86ThisCall;
219
220
27.6k
  if (D->hasAttr<VectorCallAttr>())
221
0
    return CC_X86VectorCall;
222
223
27.6k
  if (D->hasAttr<PascalAttr>())
224
0
    return CC_X86Pascal;
225
226
27.6k
  if (PcsAttr *PCS = D->getAttr<PcsAttr>())
227
0
    return (PCS->getPCS() == PcsAttr::AAPCS ? CC_AAPCS : CC_AAPCS_VFP);
228
229
27.6k
  if (D->hasAttr<AArch64VectorPcsAttr>())
230
0
    return CC_AArch64VectorCall;
231
232
27.6k
  if (D->hasAttr<AArch64SVEPcsAttr>())
233
0
    return CC_AArch64SVEPCS;
234
235
27.6k
  if (D->hasAttr<IntelOclBiccAttr>())
236
0
    return CC_IntelOclBicc;
237
238
27.6k
  if (D->hasAttr<MSABIAttr>())
239
2
    return IsWindows ? 
CC_C0
: CC_Win64;
240
241
27.6k
  if (D->hasAttr<SysVABIAttr>())
242
0
    return IsWindows ? CC_X86_64SysV : CC_C;
243
244
27.6k
  if (D->hasAttr<PreserveMostAttr>())
245
0
    return CC_PreserveMost;
246
247
27.6k
  if (D->hasAttr<PreserveAllAttr>())
248
0
    return CC_PreserveAll;
249
250
27.6k
  return CC_C;
251
27.6k
}
252
253
/// Arrange the argument and result information for a call to an
254
/// unknown C++ non-static member function of the given abstract type.
255
/// (A null RD means we don't have any meaningful "this" argument type,
256
///  so fall back to a generic pointer type).
257
/// The member function must be an ordinary function, i.e. not a
258
/// constructor or destructor.
259
const CGFunctionInfo &
260
CodeGenTypes::arrangeCXXMethodType(const CXXRecordDecl *RD,
261
                                   const FunctionProtoType *FTP,
262
221k
                                   const CXXMethodDecl *MD) {
263
221k
  SmallVector<CanQualType, 16> argTypes;
264
265
  // Add the 'this' pointer.
266
221k
  argTypes.push_back(DeriveThisType(RD, MD));
267
268
221k
  return ::arrangeLLVMFunctionInfo(
269
221k
      *this, true, argTypes,
270
221k
      FTP->getCanonicalTypeUnqualified().getAs<FunctionProtoType>());
271
221k
}
272
273
/// Set calling convention for CUDA/HIP kernel.
274
static void setCUDAKernelCallingConvention(CanQualType &FTy, CodeGenModule &CGM,
275
644k
                                           const FunctionDecl *FD) {
276
644k
  if (FD->hasAttr<CUDAGlobalAttr>()) {
277
404
    const FunctionType *FT = FTy->getAs<FunctionType>();
278
404
    CGM.getTargetCodeGenInfo().setCUDAKernelCallingConvention(FT);
279
404
    FTy = FT->getCanonicalTypeUnqualified();
280
404
  }
281
644k
}
282
283
/// Arrange the argument and result information for a declaration or
284
/// definition of the given C++ non-static member function.  The
285
/// member function must be an ordinary function, i.e. not a
286
/// constructor or destructor.
287
const CGFunctionInfo &
288
227k
CodeGenTypes::arrangeCXXMethodDeclaration(const CXXMethodDecl *MD) {
289
227k
  assert(!isa<CXXConstructorDecl>(MD) && "wrong method for constructors!");
290
0
  assert(!isa<CXXDestructorDecl>(MD) && "wrong method for destructors!");
291
292
0
  CanQualType FT = GetFormalType(MD).getAs<Type>();
293
227k
  setCUDAKernelCallingConvention(FT, CGM, MD);
294
227k
  auto prototype = FT.getAs<FunctionProtoType>();
295
296
227k
  if (MD->isInstance()) {
297
    // The abstract case is perfectly fine.
298
221k
    const CXXRecordDecl *ThisType = TheCXXABI.getThisArgumentTypeForMethod(MD);
299
221k
    return arrangeCXXMethodType(ThisType, prototype.getTypePtr(), MD);
300
221k
  }
301
302
5.93k
  return arrangeFreeFunctionType(prototype);
303
227k
}
304
305
bool CodeGenTypes::inheritingCtorHasParams(
306
1.18k
    const InheritedConstructor &Inherited, CXXCtorType Type) {
307
  // Parameters are unnecessary if we're constructing a base class subobject
308
  // and the inherited constructor lives in a virtual base.
309
1.18k
  return Type == Ctor_Complete ||
310
1.18k
         
!Inherited.getShadowDecl()->constructsVirtualBase()804
||
311
1.18k
         
!Target.getCXXABI().hasConstructorVariants()74
;
312
1.18k
}
313
314
const CGFunctionInfo &
315
295k
CodeGenTypes::arrangeCXXStructorDeclaration(GlobalDecl GD) {
316
295k
  auto *MD = cast<CXXMethodDecl>(GD.getDecl());
317
318
295k
  SmallVector<CanQualType, 16> argTypes;
319
295k
  SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos;
320
295k
  argTypes.push_back(DeriveThisType(MD->getParent(), MD));
321
322
295k
  bool PassParams = true;
323
324
295k
  if (auto *CD = dyn_cast<CXXConstructorDecl>(MD)) {
325
    // A base class inheriting constructor doesn't get forwarded arguments
326
    // needed to construct a virtual base (or base class thereof).
327
175k
    if (auto Inherited = CD->getInheritedConstructor())
328
726
      PassParams = inheritingCtorHasParams(Inherited, GD.getCtorType());
329
175k
  }
330
331
295k
  CanQual<FunctionProtoType> FTP = GetFormalType(MD);
332
333
  // Add the formal parameters.
334
295k
  if (PassParams)
335
295k
    appendParameterTypes(*this, argTypes, paramInfos, FTP);
336
337
295k
  CGCXXABI::AddedStructorArgCounts AddedArgs =
338
295k
      TheCXXABI.buildStructorSignature(GD, argTypes);
339
295k
  if (!paramInfos.empty()) {
340
    // Note: prefix implies after the first param.
341
175
    if (AddedArgs.Prefix)
342
2
      paramInfos.insert(paramInfos.begin() + 1, AddedArgs.Prefix,
343
2
                        FunctionProtoType::ExtParameterInfo{});
344
175
    if (AddedArgs.Suffix)
345
20
      paramInfos.append(AddedArgs.Suffix,
346
20
                        FunctionProtoType::ExtParameterInfo{});
347
175
  }
348
349
295k
  RequiredArgs required =
350
295k
      (PassParams && 
MD->isVariadic()295k
?
RequiredArgs(argTypes.size())88
351
295k
                                      : 
RequiredArgs::All294k
);
352
353
295k
  FunctionType::ExtInfo extInfo = FTP->getExtInfo();
354
295k
  CanQualType resultType = TheCXXABI.HasThisReturn(GD)
355
295k
                               ? 
argTypes.front()7.75k
356
295k
                               : 
TheCXXABI.hasMostDerivedReturn(GD)287k
357
287k
                                     ? 
CGM.getContext().VoidPtrTy1.43k
358
287k
                                     : 
Context.VoidTy285k
;
359
295k
  return arrangeLLVMFunctionInfo(resultType, /*instanceMethod=*/true,
360
295k
                                 /*chainCall=*/false, argTypes, extInfo,
361
295k
                                 paramInfos, required);
362
295k
}
363
364
static SmallVector<CanQualType, 16>
365
77.1k
getArgTypesForCall(ASTContext &ctx, const CallArgList &args) {
366
77.1k
  SmallVector<CanQualType, 16> argTypes;
367
77.1k
  for (auto &arg : args)
368
108k
    argTypes.push_back(ctx.getCanonicalParamType(arg.Ty));
369
77.1k
  return argTypes;
370
77.1k
}
371
372
static SmallVector<CanQualType, 16>
373
30.8k
getArgTypesForDeclaration(ASTContext &ctx, const FunctionArgList &args) {
374
30.8k
  SmallVector<CanQualType, 16> argTypes;
375
30.8k
  for (auto &arg : args)
376
82.7k
    argTypes.push_back(ctx.getCanonicalParamType(arg->getType()));
377
30.8k
  return argTypes;
378
30.8k
}
379
380
static llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16>
381
getExtParameterInfosForCall(const FunctionProtoType *proto,
382
76.6k
                            unsigned prefixArgs, unsigned totalArgs) {
383
76.6k
  llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16> result;
384
76.6k
  if (proto->hasExtParameterInfos()) {
385
41
    addExtParameterInfosForCall(result, proto, prefixArgs, totalArgs);
386
41
  }
387
76.6k
  return result;
388
76.6k
}
389
390
/// Arrange a call to a C++ method, passing the given arguments.
391
///
392
/// ExtraPrefixArgs is the number of ABI-specific args passed after the `this`
393
/// parameter.
394
/// ExtraSuffixArgs is the number of ABI-specific args passed at the end of
395
/// args.
396
/// PassProtoArgs indicates whether `args` has args for the parameters in the
397
/// given CXXConstructorDecl.
398
const CGFunctionInfo &
399
CodeGenTypes::arrangeCXXConstructorCall(const CallArgList &args,
400
                                        const CXXConstructorDecl *D,
401
                                        CXXCtorType CtorKind,
402
                                        unsigned ExtraPrefixArgs,
403
                                        unsigned ExtraSuffixArgs,
404
54.3k
                                        bool PassProtoArgs) {
405
  // FIXME: Kill copy.
406
54.3k
  SmallVector<CanQualType, 16> ArgTypes;
407
54.3k
  for (const auto &Arg : args)
408
106k
    ArgTypes.push_back(Context.getCanonicalParamType(Arg.Ty));
409
410
  // +1 for implicit this, which should always be args[0].
411
54.3k
  unsigned TotalPrefixArgs = 1 + ExtraPrefixArgs;
412
413
54.3k
  CanQual<FunctionProtoType> FPT = GetFormalType(D);
414
54.3k
  RequiredArgs Required = PassProtoArgs
415
54.3k
                              ? RequiredArgs::forPrototypePlus(
416
54.3k
                                    FPT, TotalPrefixArgs + ExtraSuffixArgs)
417
54.3k
                              : 
RequiredArgs::All10
;
418
419
54.3k
  GlobalDecl GD(D, CtorKind);
420
54.3k
  CanQualType ResultType = TheCXXABI.HasThisReturn(GD)
421
54.3k
                               ? 
ArgTypes.front()2.37k
422
54.3k
                               : 
TheCXXABI.hasMostDerivedReturn(GD)51.9k
423
51.9k
                                     ? 
CGM.getContext().VoidPtrTy0
424
51.9k
                                     : Context.VoidTy;
425
426
54.3k
  FunctionType::ExtInfo Info = FPT->getExtInfo();
427
54.3k
  llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16> ParamInfos;
428
  // If the prototype args are elided, we should only have ABI-specific args,
429
  // which never have param info.
430
54.3k
  if (PassProtoArgs && 
FPT->hasExtParameterInfos()54.3k
) {
431
    // ABI-specific suffix arguments are treated the same as variadic arguments.
432
70
    addExtParameterInfosForCall(ParamInfos, FPT.getTypePtr(), TotalPrefixArgs,
433
70
                                ArgTypes.size());
434
70
  }
435
54.3k
  return arrangeLLVMFunctionInfo(ResultType, /*instanceMethod=*/true,
436
54.3k
                                 /*chainCall=*/false, ArgTypes, Info,
437
54.3k
                                 ParamInfos, Required);
438
54.3k
}
439
440
/// Arrange the argument and result information for the declaration or
441
/// definition of the given function.
442
const CGFunctionInfo &
443
517k
CodeGenTypes::arrangeFunctionDeclaration(const FunctionDecl *FD) {
444
517k
  if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(FD))
445
113k
    if (MD->isInstance())
446
100k
      return arrangeCXXMethodDeclaration(MD);
447
448
416k
  CanQualType FTy = FD->getType()->getCanonicalTypeUnqualified();
449
450
416k
  assert(isa<FunctionType>(FTy));
451
0
  setCUDAKernelCallingConvention(FTy, CGM, FD);
452
453
  // When declaring a function without a prototype, always use a
454
  // non-variadic type.
455
416k
  if (CanQual<FunctionNoProtoType> noProto = FTy.getAs<FunctionNoProtoType>()) {
456
4.28k
    return arrangeLLVMFunctionInfo(
457
4.28k
        noProto->getReturnType(), /*instanceMethod=*/false,
458
4.28k
        /*chainCall=*/false, None, noProto->getExtInfo(), {},RequiredArgs::All);
459
4.28k
  }
460
461
412k
  return arrangeFreeFunctionType(FTy.castAs<FunctionProtoType>());
462
416k
}
463
464
/// Arrange the argument and result information for the declaration or
465
/// definition of an Objective-C method.
466
const CGFunctionInfo &
467
5.21k
CodeGenTypes::arrangeObjCMethodDeclaration(const ObjCMethodDecl *MD) {
468
  // It happens that this is the same as a call with no optional
469
  // arguments, except also using the formal 'self' type.
470
5.21k
  return arrangeObjCMessageSendSignature(MD, MD->getSelfDecl()->getType());
471
5.21k
}
472
473
/// Arrange the argument and result information for the function type
474
/// through which to perform a send to the given Objective-C method,
475
/// using the given receiver type.  The receiver type is not always
476
/// the 'self' type of the method or even an Objective-C pointer type.
477
/// This is *not* the right method for actually performing such a
478
/// message send, due to the possibility of optional arguments.
479
const CGFunctionInfo &
480
CodeGenTypes::arrangeObjCMessageSendSignature(const ObjCMethodDecl *MD,
481
16.3k
                                              QualType receiverType) {
482
16.3k
  SmallVector<CanQualType, 16> argTys;
483
16.3k
  SmallVector<FunctionProtoType::ExtParameterInfo, 4> extParamInfos(2);
484
16.3k
  argTys.push_back(Context.getCanonicalParamType(receiverType));
485
16.3k
  argTys.push_back(Context.getCanonicalParamType(Context.getObjCSelType()));
486
  // FIXME: Kill copy?
487
16.3k
  for (const auto *I : MD->parameters()) {
488
13.5k
    argTys.push_back(Context.getCanonicalParamType(I->getType()));
489
13.5k
    auto extParamInfo = FunctionProtoType::ExtParameterInfo().withIsNoEscape(
490
13.5k
        I->hasAttr<NoEscapeAttr>());
491
13.5k
    extParamInfos.push_back(extParamInfo);
492
13.5k
  }
493
494
16.3k
  FunctionType::ExtInfo einfo;
495
16.3k
  bool IsWindows = getContext().getTargetInfo().getTriple().isOSWindows();
496
16.3k
  einfo = einfo.withCallingConv(getCallingConventionForDecl(MD, IsWindows));
497
498
16.3k
  if (getContext().getLangOpts().ObjCAutoRefCount &&
499
16.3k
      
MD->hasAttr<NSReturnsRetainedAttr>()880
)
500
114
    einfo = einfo.withProducesResult(true);
501
502
16.3k
  RequiredArgs required =
503
16.3k
    (MD->isVariadic() ? 
RequiredArgs(argTys.size())874
:
RequiredArgs::All15.4k
);
504
505
16.3k
  return arrangeLLVMFunctionInfo(
506
16.3k
      GetReturnType(MD->getReturnType()), /*instanceMethod=*/false,
507
16.3k
      /*chainCall=*/false, argTys, einfo, extParamInfos, required);
508
16.3k
}
509
510
const CGFunctionInfo &
511
CodeGenTypes::arrangeUnprototypedObjCMessageSend(QualType returnType,
512
800
                                                 const CallArgList &args) {
513
800
  auto argTypes = getArgTypesForCall(Context, args);
514
800
  FunctionType::ExtInfo einfo;
515
516
800
  return arrangeLLVMFunctionInfo(
517
800
      GetReturnType(returnType), /*instanceMethod=*/false,
518
800
      /*chainCall=*/false, argTypes, einfo, {}, RequiredArgs::All);
519
800
}
520
521
const CGFunctionInfo &
522
581k
CodeGenTypes::arrangeGlobalDeclaration(GlobalDecl GD) {
523
  // FIXME: Do we need to handle ObjCMethodDecl?
524
581k
  const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
525
526
581k
  if (isa<CXXConstructorDecl>(GD.getDecl()) ||
527
581k
      
isa<CXXDestructorDecl>(GD.getDecl())539k
)
528
64.1k
    return arrangeCXXStructorDeclaration(GD);
529
530
517k
  return arrangeFunctionDeclaration(FD);
531
581k
}
532
533
/// Arrange a thunk that takes 'this' as the first parameter followed by
534
/// varargs.  Return a void pointer, regardless of the actual return type.
535
/// The body of the thunk will end in a musttail call to a function of the
536
/// correct type, and the caller will bitcast the function to the correct
537
/// prototype.
538
const CGFunctionInfo &
539
55
CodeGenTypes::arrangeUnprototypedMustTailThunk(const CXXMethodDecl *MD) {
540
55
  assert(MD->isVirtual() && "only methods have thunks");
541
0
  CanQual<FunctionProtoType> FTP = GetFormalType(MD);
542
55
  CanQualType ArgTys[] = {DeriveThisType(MD->getParent(), MD)};
543
55
  return arrangeLLVMFunctionInfo(Context.VoidTy, /*instanceMethod=*/false,
544
55
                                 /*chainCall=*/false, ArgTys,
545
55
                                 FTP->getExtInfo(), {}, RequiredArgs(1));
546
55
}
547
548
const CGFunctionInfo &
549
CodeGenTypes::arrangeMSCtorClosure(const CXXConstructorDecl *CD,
550
34
                                   CXXCtorType CT) {
551
34
  assert(CT == Ctor_CopyingClosure || CT == Ctor_DefaultClosure);
552
553
0
  CanQual<FunctionProtoType> FTP = GetFormalType(CD);
554
34
  SmallVector<CanQualType, 2> ArgTys;
555
34
  const CXXRecordDecl *RD = CD->getParent();
556
34
  ArgTys.push_back(DeriveThisType(RD, CD));
557
34
  if (CT == Ctor_CopyingClosure)
558
3
    ArgTys.push_back(*FTP->param_type_begin());
559
34
  if (RD->getNumVBases() > 0)
560
0
    ArgTys.push_back(Context.IntTy);
561
34
  CallingConv CC = Context.getDefaultCallingConvention(
562
34
      /*IsVariadic=*/false, /*IsCXXMethod=*/true);
563
34
  return arrangeLLVMFunctionInfo(Context.VoidTy, /*instanceMethod=*/true,
564
34
                                 /*chainCall=*/false, ArgTys,
565
34
                                 FunctionType::ExtInfo(CC), {},
566
34
                                 RequiredArgs::All);
567
34
}
568
569
/// Arrange a call as unto a free function, except possibly with an
570
/// additional number of formal parameters considered required.
571
static const CGFunctionInfo &
572
arrangeFreeFunctionLikeCall(CodeGenTypes &CGT,
573
                            CodeGenModule &CGM,
574
                            const CallArgList &args,
575
                            const FunctionType *fnType,
576
                            unsigned numExtraRequiredArgs,
577
193k
                            bool chainCall) {
578
193k
  assert(args.size() >= numExtraRequiredArgs);
579
580
0
  llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos;
581
582
  // In most cases, there are no optional arguments.
583
193k
  RequiredArgs required = RequiredArgs::All;
584
585
  // If we have a variadic prototype, the required arguments are the
586
  // extra prefix plus the arguments in the prototype.
587
193k
  if (const FunctionProtoType *proto = dyn_cast<FunctionProtoType>(fnType)) {
588
193k
    if (proto->isVariadic())
589
76.2k
      required = RequiredArgs::forPrototypePlus(proto, numExtraRequiredArgs);
590
591
193k
    if (proto->hasExtParameterInfos())
592
305
      addExtParameterInfosForCall(paramInfos, proto, numExtraRequiredArgs,
593
305
                                  args.size());
594
595
  // If we don't have a prototype at all, but we're supposed to
596
  // explicitly use the variadic convention for unprototyped calls,
597
  // treat all of the arguments as required but preserve the nominal
598
  // possibility of variadics.
599
193k
  } else 
if (526
CGM.getTargetCodeGenInfo()
600
526
                .isNoProtoCallVariadic(args,
601
526
                                       cast<FunctionNoProtoType>(fnType))) {
602
423
    required = RequiredArgs(args.size());
603
423
  }
604
605
  // FIXME: Kill copy.
606
193k
  SmallVector<CanQualType, 16> argTypes;
607
193k
  for (const auto &arg : args)
608
336k
    argTypes.push_back(CGT.getContext().getCanonicalParamType(arg.Ty));
609
193k
  return CGT.arrangeLLVMFunctionInfo(GetReturnType(fnType->getReturnType()),
610
193k
                                     /*instanceMethod=*/false, chainCall,
611
193k
                                     argTypes, fnType->getExtInfo(), paramInfos,
612
193k
                                     required);
613
193k
}
614
615
/// Figure out the rules for calling a function with the given formal
616
/// type using the given arguments.  The arguments are necessary
617
/// because the function might be unprototyped, in which case it's
618
/// target-dependent in crazy ways.
619
const CGFunctionInfo &
620
CodeGenTypes::arrangeFreeFunctionCall(const CallArgList &args,
621
                                      const FunctionType *fnType,
622
193k
                                      bool chainCall) {
623
193k
  return arrangeFreeFunctionLikeCall(*this, CGM, args, fnType,
624
193k
                                     chainCall ? 
18
:
0193k
, chainCall);
625
193k
}
626
627
/// A block function is essentially a free function with an
628
/// extra implicit argument.
629
const CGFunctionInfo &
630
CodeGenTypes::arrangeBlockFunctionCall(const CallArgList &args,
631
620
                                       const FunctionType *fnType) {
632
620
  return arrangeFreeFunctionLikeCall(*this, CGM, args, fnType, 1,
633
620
                                     /*chainCall=*/false);
634
620
}
635
636
const CGFunctionInfo &
637
CodeGenTypes::arrangeBlockFunctionDeclaration(const FunctionProtoType *proto,
638
1.17k
                                              const FunctionArgList &params) {
639
1.17k
  auto paramInfos = getExtParameterInfosForCall(proto, 1, params.size());
640
1.17k
  auto argTypes = getArgTypesForDeclaration(Context, params);
641
642
1.17k
  return arrangeLLVMFunctionInfo(GetReturnType(proto->getReturnType()),
643
1.17k
                                 /*instanceMethod*/ false, /*chainCall*/ false,
644
1.17k
                                 argTypes, proto->getExtInfo(), paramInfos,
645
1.17k
                                 RequiredArgs::forPrototypePlus(proto, 1));
646
1.17k
}
647
648
const CGFunctionInfo &
649
CodeGenTypes::arrangeBuiltinFunctionCall(QualType resultType,
650
1.00k
                                         const CallArgList &args) {
651
  // FIXME: Kill copy.
652
1.00k
  SmallVector<CanQualType, 16> argTypes;
653
1.00k
  for (const auto &Arg : args)
654
3.68k
    argTypes.push_back(Context.getCanonicalParamType(Arg.Ty));
655
1.00k
  return arrangeLLVMFunctionInfo(
656
1.00k
      GetReturnType(resultType), /*instanceMethod=*/false,
657
1.00k
      /*chainCall=*/false, argTypes, FunctionType::ExtInfo(),
658
1.00k
      /*paramInfos=*/ {}, RequiredArgs::All);
659
1.00k
}
660
661
const CGFunctionInfo &
662
CodeGenTypes::arrangeBuiltinFunctionDeclaration(QualType resultType,
663
29.7k
                                                const FunctionArgList &args) {
664
29.7k
  auto argTypes = getArgTypesForDeclaration(Context, args);
665
666
29.7k
  return arrangeLLVMFunctionInfo(
667
29.7k
      GetReturnType(resultType), /*instanceMethod=*/false, /*chainCall=*/false,
668
29.7k
      argTypes, FunctionType::ExtInfo(), {}, RequiredArgs::All);
669
29.7k
}
670
671
const CGFunctionInfo &
672
CodeGenTypes::arrangeBuiltinFunctionDeclaration(CanQualType resultType,
673
374
                                              ArrayRef<CanQualType> argTypes) {
674
374
  return arrangeLLVMFunctionInfo(
675
374
      resultType, /*instanceMethod=*/false, /*chainCall=*/false,
676
374
      argTypes, FunctionType::ExtInfo(), {}, RequiredArgs::All);
677
374
}
678
679
/// Arrange a call to a C++ method, passing the given arguments.
680
///
681
/// numPrefixArgs is the number of ABI-specific prefix arguments we have. It
682
/// does not count `this`.
683
const CGFunctionInfo &
684
CodeGenTypes::arrangeCXXMethodCall(const CallArgList &args,
685
                                   const FunctionProtoType *proto,
686
                                   RequiredArgs required,
687
75.4k
                                   unsigned numPrefixArgs) {
688
75.4k
  assert(numPrefixArgs + 1 <= args.size() &&
689
75.4k
         "Emitting a call with less args than the required prefix?");
690
  // Add one to account for `this`. It's a bit awkward here, but we don't count
691
  // `this` in similar places elsewhere.
692
0
  auto paramInfos =
693
75.4k
    getExtParameterInfosForCall(proto, numPrefixArgs + 1, args.size());
694
695
  // FIXME: Kill copy.
696
75.4k
  auto argTypes = getArgTypesForCall(Context, args);
697
698
75.4k
  FunctionType::ExtInfo info = proto->getExtInfo();
699
75.4k
  return arrangeLLVMFunctionInfo(
700
75.4k
      GetReturnType(proto->getReturnType()), /*instanceMethod=*/true,
701
75.4k
      /*chainCall=*/false, argTypes, info, paramInfos, required);
702
75.4k
}
703
704
17.6k
const CGFunctionInfo &CodeGenTypes::arrangeNullaryFunction() {
705
17.6k
  return arrangeLLVMFunctionInfo(
706
17.6k
      getContext().VoidTy, /*instanceMethod=*/false, /*chainCall=*/false,
707
17.6k
      None, FunctionType::ExtInfo(), {}, RequiredArgs::All);
708
17.6k
}
709
710
const CGFunctionInfo &
711
CodeGenTypes::arrangeCall(const CGFunctionInfo &signature,
712
11.1k
                          const CallArgList &args) {
713
11.1k
  assert(signature.arg_size() <= args.size());
714
11.1k
  if (signature.arg_size() == args.size())
715
10.2k
    return signature;
716
717
856
  SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos;
718
856
  auto sigParamInfos = signature.getExtParameterInfos();
719
856
  if (!sigParamInfos.empty()) {
720
856
    paramInfos.append(sigParamInfos.begin(), sigParamInfos.end());
721
856
    paramInfos.resize(args.size());
722
856
  }
723
724
856
  auto argTypes = getArgTypesForCall(Context, args);
725
726
856
  assert(signature.getRequiredArgs().allowsOptionalArgs());
727
0
  return arrangeLLVMFunctionInfo(signature.getReturnType(),
728
856
                                 signature.isInstanceMethod(),
729
856
                                 signature.isChainCall(),
730
856
                                 argTypes,
731
856
                                 signature.getExtInfo(),
732
856
                                 paramInfos,
733
856
                                 signature.getRequiredArgs());
734
11.1k
}
735
736
namespace clang {
737
namespace CodeGen {
738
void computeSPIRKernelABIInfo(CodeGenModule &CGM, CGFunctionInfo &FI);
739
}
740
}
741
742
/// Arrange the argument and result information for an abstract value
743
/// of a given function type.  This is the method which all of the
744
/// above functions ultimately defer to.
745
const CGFunctionInfo &
746
CodeGenTypes::arrangeLLVMFunctionInfo(CanQualType resultType,
747
                                      bool instanceMethod,
748
                                      bool chainCall,
749
                                      ArrayRef<CanQualType> argTypes,
750
                                      FunctionType::ExtInfo info,
751
                     ArrayRef<FunctionProtoType::ExtParameterInfo> paramInfos,
752
1.39M
                                      RequiredArgs required) {
753
1.39M
  assert(llvm::all_of(argTypes,
754
1.39M
                      [](CanQualType T) { return T.isCanonicalAsParam(); }));
755
756
  // Lookup or create unique function info.
757
0
  llvm::FoldingSetNodeID ID;
758
1.39M
  CGFunctionInfo::Profile(ID, instanceMethod, chainCall, info, paramInfos,
759
1.39M
                          required, resultType, argTypes);
760
761
1.39M
  void *insertPos = nullptr;
762
1.39M
  CGFunctionInfo *FI = FunctionInfos.FindNodeOrInsertPos(ID, insertPos);
763
1.39M
  if (FI)
764
1.16M
    return *FI;
765
766
238k
  unsigned CC = ClangCallConvToLLVMCallConv(info.getCC());
767
768
  // Construct the function info.  We co-allocate the ArgInfos.
769
238k
  FI = CGFunctionInfo::create(CC, instanceMethod, chainCall, info,
770
238k
                              paramInfos, resultType, argTypes, required);
771
238k
  FunctionInfos.InsertNode(FI, insertPos);
772
773
238k
  bool inserted = FunctionsBeingProcessed.insert(FI).second;
774
238k
  (void)inserted;
775
238k
  assert(inserted && "Recursively being processed?");
776
777
  // Compute ABI information.
778
238k
  if (CC == llvm::CallingConv::SPIR_KERNEL) {
779
    // Force target independent argument handling for the host visible
780
    // kernel functions.
781
179
    computeSPIRKernelABIInfo(CGM, *FI);
782
237k
  } else if (info.getCC() == CC_Swift || 
info.getCC() == CC_SwiftAsync236k
) {
783
1.21k
    swiftcall::computeABIInfo(CGM, *FI);
784
236k
  } else {
785
236k
    getABIInfo().computeInfo(*FI);
786
236k
  }
787
788
  // Loop over all of the computed argument and return value info.  If any of
789
  // them are direct or extend without a specified coerce type, specify the
790
  // default now.
791
238k
  ABIArgInfo &retInfo = FI->getReturnInfo();
792
238k
  if (retInfo.canHaveCoerceToType() && 
retInfo.getCoerceToType() == nullptr136k
)
793
41.9k
    retInfo.setCoerceToType(ConvertType(FI->getReturnType()));
794
795
238k
  for (auto &I : FI->arguments())
796
434k
    if (I.info.canHaveCoerceToType() && 
I.info.getCoerceToType() == nullptr426k
)
797
141k
      I.info.setCoerceToType(ConvertType(I.type));
798
799
238k
  bool erased = FunctionsBeingProcessed.erase(FI); (void)erased;
800
238k
  assert(erased && "Not in set?");
801
802
0
  return *FI;
803
1.39M
}
804
805
CGFunctionInfo *CGFunctionInfo::create(unsigned llvmCC,
806
                                       bool instanceMethod,
807
                                       bool chainCall,
808
                                       const FunctionType::ExtInfo &info,
809
                                       ArrayRef<ExtParameterInfo> paramInfos,
810
                                       CanQualType resultType,
811
                                       ArrayRef<CanQualType> argTypes,
812
238k
                                       RequiredArgs required) {
813
238k
  assert(paramInfos.empty() || paramInfos.size() == argTypes.size());
814
0
  assert(!required.allowsOptionalArgs() ||
815
238k
         required.getNumRequiredArgs() <= argTypes.size());
816
817
0
  void *buffer =
818
238k
    operator new(totalSizeToAlloc<ArgInfo,             ExtParameterInfo>(
819
238k
                                  argTypes.size() + 1, paramInfos.size()));
820
821
238k
  CGFunctionInfo *FI = new(buffer) CGFunctionInfo();
822
238k
  FI->CallingConvention = llvmCC;
823
238k
  FI->EffectiveCallingConvention = llvmCC;
824
238k
  FI->ASTCallingConvention = info.getCC();
825
238k
  FI->InstanceMethod = instanceMethod;
826
238k
  FI->ChainCall = chainCall;
827
238k
  FI->CmseNSCall = info.getCmseNSCall();
828
238k
  FI->NoReturn = info.getNoReturn();
829
238k
  FI->ReturnsRetained = info.getProducesResult();
830
238k
  FI->NoCallerSavedRegs = info.getNoCallerSavedRegs();
831
238k
  FI->NoCfCheck = info.getNoCfCheck();
832
238k
  FI->Required = required;
833
238k
  FI->HasRegParm = info.getHasRegParm();
834
238k
  FI->RegParm = info.getRegParm();
835
238k
  FI->ArgStruct = nullptr;
836
238k
  FI->ArgStructAlign = 0;
837
238k
  FI->NumArgs = argTypes.size();
838
238k
  FI->HasExtParameterInfos = !paramInfos.empty();
839
238k
  FI->getArgsBuffer()[0].type = resultType;
840
238k
  FI->MaxVectorWidth = 0;
841
672k
  for (unsigned i = 0, e = argTypes.size(); i != e; 
++i434k
)
842
434k
    FI->getArgsBuffer()[i + 1].type = argTypes[i];
843
258k
  for (unsigned i = 0, e = paramInfos.size(); i != e; 
++i20.1k
)
844
20.1k
    FI->getExtParameterInfosBuffer()[i] = paramInfos[i];
845
238k
  return FI;
846
238k
}
847
848
/***/
849
850
namespace {
851
// ABIArgInfo::Expand implementation.
852
853
// Specifies the way QualType passed as ABIArgInfo::Expand is expanded.
854
struct TypeExpansion {
855
  enum TypeExpansionKind {
856
    // Elements of constant arrays are expanded recursively.
857
    TEK_ConstantArray,
858
    // Record fields are expanded recursively (but if record is a union, only
859
    // the field with the largest size is expanded).
860
    TEK_Record,
861
    // For complex types, real and imaginary parts are expanded recursively.
862
    TEK_Complex,
863
    // All other types are not expandable.
864
    TEK_None
865
  };
866
867
  const TypeExpansionKind Kind;
868
869
2.36k
  TypeExpansion(TypeExpansionKind K) : Kind(K) {}
870
2.36k
  virtual ~TypeExpansion() {}
871
};
872
873
struct ConstantArrayExpansion : TypeExpansion {
874
  QualType EltTy;
875
  uint64_t NumElts;
876
877
  ConstantArrayExpansion(QualType EltTy, uint64_t NumElts)
878
40
      : TypeExpansion(TEK_ConstantArray), EltTy(EltTy), NumElts(NumElts) {}
879
2.36k
  static bool classof(const TypeExpansion *TE) {
880
2.36k
    return TE->Kind == TEK_ConstantArray;
881
2.36k
  }
882
};
883
884
struct RecordExpansion : TypeExpansion {
885
  SmallVector<const CXXBaseSpecifier *, 1> Bases;
886
887
  SmallVector<const FieldDecl *, 1> Fields;
888
889
  RecordExpansion(SmallVector<const CXXBaseSpecifier *, 1> &&Bases,
890
                  SmallVector<const FieldDecl *, 1> &&Fields)
891
      : TypeExpansion(TEK_Record), Bases(std::move(Bases)),
892
761
        Fields(std::move(Fields)) {}
893
2.32k
  static bool classof(const TypeExpansion *TE) {
894
2.32k
    return TE->Kind == TEK_Record;
895
2.32k
  }
896
};
897
898
struct ComplexExpansion : TypeExpansion {
899
  QualType EltTy;
900
901
46
  ComplexExpansion(QualType EltTy) : TypeExpansion(TEK_Complex), EltTy(EltTy) {}
902
1.56k
  static bool classof(const TypeExpansion *TE) {
903
1.56k
    return TE->Kind == TEK_Complex;
904
1.56k
  }
905
};
906
907
struct NoExpansion : TypeExpansion {
908
1.51k
  NoExpansion() : TypeExpansion(TEK_None) {}
909
1.51k
  static bool classof(const TypeExpansion *TE) {
910
1.51k
    return TE->Kind == TEK_None;
911
1.51k
  }
912
};
913
}  // namespace
914
915
static std::unique_ptr<TypeExpansion>
916
2.36k
getTypeExpansion(QualType Ty, const ASTContext &Context) {
917
2.36k
  if (const ConstantArrayType *AT = Context.getAsConstantArrayType(Ty)) {
918
40
    return std::make_unique<ConstantArrayExpansion>(
919
40
        AT->getElementType(), AT->getSize().getZExtValue());
920
40
  }
921
2.32k
  if (const RecordType *RT = Ty->getAs<RecordType>()) {
922
761
    SmallVector<const CXXBaseSpecifier *, 1> Bases;
923
761
    SmallVector<const FieldDecl *, 1> Fields;
924
761
    const RecordDecl *RD = RT->getDecl();
925
761
    assert(!RD->hasFlexibleArrayMember() &&
926
761
           "Cannot expand structure with flexible array.");
927
761
    if (RD->isUnion()) {
928
      // Unions can be here only in degenerative cases - all the fields are same
929
      // after flattening. Thus we have to use the "largest" field.
930
5
      const FieldDecl *LargestFD = nullptr;
931
5
      CharUnits UnionSize = CharUnits::Zero();
932
933
10
      for (const auto *FD : RD->fields()) {
934
10
        if (FD->isZeroLengthBitField(Context))
935
0
          continue;
936
10
        assert(!FD->isBitField() &&
937
10
               "Cannot expand structure with bit-field members.");
938
0
        CharUnits FieldSize = Context.getTypeSizeInChars(FD->getType());
939
10
        if (UnionSize < FieldSize) {
940
5
          UnionSize = FieldSize;
941
5
          LargestFD = FD;
942
5
        }
943
10
      }
944
5
      if (LargestFD)
945
5
        Fields.push_back(LargestFD);
946
756
    } else {
947
756
      if (const auto *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
948
241
        assert(!CXXRD->isDynamicClass() &&
949
241
               "cannot expand vtable pointers in dynamic classes");
950
0
        llvm::append_range(Bases, llvm::make_pointer_range(CXXRD->bases()));
951
241
      }
952
953
1.47k
      for (const auto *FD : RD->fields()) {
954
1.47k
        if (FD->isZeroLengthBitField(Context))
955
0
          continue;
956
1.47k
        assert(!FD->isBitField() &&
957
1.47k
               "Cannot expand structure with bit-field members.");
958
0
        Fields.push_back(FD);
959
1.47k
      }
960
756
    }
961
0
    return std::make_unique<RecordExpansion>(std::move(Bases),
962
761
                                              std::move(Fields));
963
761
  }
964
1.56k
  if (const ComplexType *CT = Ty->getAs<ComplexType>()) {
965
46
    return std::make_unique<ComplexExpansion>(CT->getElementType());
966
46
  }
967
1.51k
  return std::make_unique<NoExpansion>();
968
1.56k
}
969
970
1.38k
static int getExpansionSize(QualType Ty, const ASTContext &Context) {
971
1.38k
  auto Exp = getTypeExpansion(Ty, Context);
972
1.38k
  if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
973
24
    return CAExp->NumElts * getExpansionSize(CAExp->EltTy, Context);
974
24
  }
975
1.36k
  if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
976
456
    int Res = 0;
977
456
    for (auto BS : RExp->Bases)
978
3
      Res += getExpansionSize(BS->getType(), Context);
979
456
    for (auto FD : RExp->Fields)
980
887
      Res += getExpansionSize(FD->getType(), Context);
981
456
    return Res;
982
456
  }
983
908
  if (isa<ComplexExpansion>(Exp.get()))
984
28
    return 2;
985
880
  assert(isa<NoExpansion>(Exp.get()));
986
0
  return 1;
987
908
}
988
989
void
990
CodeGenTypes::getExpandedTypes(QualType Ty,
991
546
                               SmallVectorImpl<llvm::Type *>::iterator &TI) {
992
546
  auto Exp = getTypeExpansion(Ty, Context);
993
546
  if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
994
40
    for (int i = 0, n = CAExp->NumElts; i < n; 
i++32
) {
995
32
      getExpandedTypes(CAExp->EltTy, TI);
996
32
    }
997
538
  } else if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
998
174
    for (auto BS : RExp->Bases)
999
1
      getExpandedTypes(BS->getType(), TI);
1000
174
    for (auto FD : RExp->Fields)
1001
332
      getExpandedTypes(FD->getType(), TI);
1002
364
  } else if (auto CExp = dyn_cast<ComplexExpansion>(Exp.get())) {
1003
11
    llvm::Type *EltTy = ConvertType(CExp->EltTy);
1004
11
    *TI++ = EltTy;
1005
11
    *TI++ = EltTy;
1006
353
  } else {
1007
353
    assert(isa<NoExpansion>(Exp.get()));
1008
0
    *TI++ = ConvertType(Ty);
1009
353
  }
1010
546
}
1011
1012
static void forConstantArrayExpansion(CodeGenFunction &CGF,
1013
                                      ConstantArrayExpansion *CAE,
1014
                                      Address BaseAddr,
1015
8
                                      llvm::function_ref<void(Address)> Fn) {
1016
8
  CharUnits EltSize = CGF.getContext().getTypeSizeInChars(CAE->EltTy);
1017
8
  CharUnits EltAlign =
1018
8
    BaseAddr.getAlignment().alignmentOfArrayElement(EltSize);
1019
8
  llvm::Type *EltTy = CGF.ConvertTypeForMem(CAE->EltTy);
1020
1021
40
  for (int i = 0, n = CAE->NumElts; i < n; 
i++32
) {
1022
32
    llvm::Value *EltAddr = CGF.Builder.CreateConstGEP2_32(
1023
32
        BaseAddr.getElementType(), BaseAddr.getPointer(), 0, i);
1024
32
    Fn(Address(EltAddr, EltTy, EltAlign));
1025
32
  }
1026
8
}
1027
1028
void CodeGenFunction::ExpandTypeFromArgs(QualType Ty, LValue LV,
1029
349
                                         llvm::Function::arg_iterator &AI) {
1030
349
  assert(LV.isSimple() &&
1031
349
         "Unexpected non-simple lvalue during struct expansion.");
1032
1033
0
  auto Exp = getTypeExpansion(Ty, getContext());
1034
349
  if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
1035
8
    forConstantArrayExpansion(
1036
32
        *this, CAExp, LV.getAddress(*this), [&](Address EltAddr) {
1037
32
          LValue LV = MakeAddrLValue(EltAddr, CAExp->EltTy);
1038
32
          ExpandTypeFromArgs(CAExp->EltTy, LV, AI);
1039
32
        });
1040
341
  } else if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
1041
101
    Address This = LV.getAddress(*this);
1042
101
    for (const CXXBaseSpecifier *BS : RExp->Bases) {
1043
      // Perform a single step derived-to-base conversion.
1044
1
      Address Base =
1045
1
          GetAddressOfBaseClass(This, Ty->getAsCXXRecordDecl(), &BS, &BS + 1,
1046
1
                                /*NullCheckValue=*/false, SourceLocation());
1047
1
      LValue SubLV = MakeAddrLValue(Base, BS->getType());
1048
1049
      // Recurse onto bases.
1050
1
      ExpandTypeFromArgs(BS->getType(), SubLV, AI);
1051
1
    }
1052
214
    for (auto FD : RExp->Fields) {
1053
      // FIXME: What are the right qualifiers here?
1054
214
      LValue SubLV = EmitLValueForFieldInitialization(LV, FD);
1055
214
      ExpandTypeFromArgs(FD->getType(), SubLV, AI);
1056
214
    }
1057
240
  } else if (isa<ComplexExpansion>(Exp.get())) {
1058
3
    auto realValue = &*AI++;
1059
3
    auto imagValue = &*AI++;
1060
3
    EmitStoreOfComplex(ComplexPairTy(realValue, imagValue), LV, /*init*/ true);
1061
237
  } else {
1062
    // Call EmitStoreOfScalar except when the lvalue is a bitfield to emit a
1063
    // primitive store.
1064
237
    assert(isa<NoExpansion>(Exp.get()));
1065
0
    llvm::Value *Arg = &*AI++;
1066
237
    if (LV.isBitField()) {
1067
0
      EmitStoreThroughLValue(RValue::get(Arg), LV);
1068
237
    } else {
1069
      // TODO: currently there are some places are inconsistent in what LLVM
1070
      // pointer type they use (see D118744). Once clang uses opaque pointers
1071
      // all LLVM pointer types will be the same and we can remove this check.
1072
237
      if (Arg->getType()->isPointerTy()) {
1073
6
        Address Addr = LV.getAddress(*this);
1074
6
        Arg = Builder.CreateBitCast(Arg, Addr.getElementType());
1075
6
      }
1076
237
      EmitStoreOfScalar(Arg, LV);
1077
237
    }
1078
237
  }
1079
349
}
1080
1081
void CodeGenFunction::ExpandTypeToArgs(
1082
    QualType Ty, CallArg Arg, llvm::FunctionType *IRFuncTy,
1083
81
    SmallVectorImpl<llvm::Value *> &IRCallArgs, unsigned &IRCallArgPos) {
1084
81
  auto Exp = getTypeExpansion(Ty, getContext());
1085
81
  if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
1086
0
    Address Addr = Arg.hasLValue() ? Arg.getKnownLValue().getAddress(*this)
1087
0
                                   : Arg.getKnownRValue().getAggregateAddress();
1088
0
    forConstantArrayExpansion(
1089
0
        *this, CAExp, Addr, [&](Address EltAddr) {
1090
0
          CallArg EltArg = CallArg(
1091
0
              convertTempToRValue(EltAddr, CAExp->EltTy, SourceLocation()),
1092
0
              CAExp->EltTy);
1093
0
          ExpandTypeToArgs(CAExp->EltTy, EltArg, IRFuncTy, IRCallArgs,
1094
0
                           IRCallArgPos);
1095
0
        });
1096
81
  } else if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
1097
30
    Address This = Arg.hasLValue() ? 
Arg.getKnownLValue().getAddress(*this)12
1098
30
                                   : 
Arg.getKnownRValue().getAggregateAddress()18
;
1099
30
    for (const CXXBaseSpecifier *BS : RExp->Bases) {
1100
      // Perform a single step derived-to-base conversion.
1101
0
      Address Base =
1102
0
          GetAddressOfBaseClass(This, Ty->getAsCXXRecordDecl(), &BS, &BS + 1,
1103
0
                                /*NullCheckValue=*/false, SourceLocation());
1104
0
      CallArg BaseArg = CallArg(RValue::getAggregate(Base), BS->getType());
1105
1106
      // Recurse onto bases.
1107
0
      ExpandTypeToArgs(BS->getType(), BaseArg, IRFuncTy, IRCallArgs,
1108
0
                       IRCallArgPos);
1109
0
    }
1110
1111
30
    LValue LV = MakeAddrLValue(This, Ty);
1112
48
    for (auto FD : RExp->Fields) {
1113
48
      CallArg FldArg =
1114
48
          CallArg(EmitRValueForField(LV, FD, SourceLocation()), FD->getType());
1115
48
      ExpandTypeToArgs(FD->getType(), FldArg, IRFuncTy, IRCallArgs,
1116
48
                       IRCallArgPos);
1117
48
    }
1118
51
  } else if (isa<ComplexExpansion>(Exp.get())) {
1119
4
    ComplexPairTy CV = Arg.getKnownRValue().getComplexVal();
1120
4
    IRCallArgs[IRCallArgPos++] = CV.first;
1121
4
    IRCallArgs[IRCallArgPos++] = CV.second;
1122
47
  } else {
1123
47
    assert(isa<NoExpansion>(Exp.get()));
1124
0
    auto RV = Arg.getKnownRValue();
1125
47
    assert(RV.isScalar() &&
1126
47
           "Unexpected non-scalar rvalue during struct expansion.");
1127
1128
    // Insert a bitcast as needed.
1129
0
    llvm::Value *V = RV.getScalarVal();
1130
47
    if (IRCallArgPos < IRFuncTy->getNumParams() &&
1131
47
        V->getType() != IRFuncTy->getParamType(IRCallArgPos))
1132
1
      V = Builder.CreateBitCast(V, IRFuncTy->getParamType(IRCallArgPos));
1133
1134
47
    IRCallArgs[IRCallArgPos++] = V;
1135
47
  }
1136
81
}
1137
1138
/// Create a temporary allocation for the purposes of coercion.
1139
static Address CreateTempAllocaForCoercion(CodeGenFunction &CGF, llvm::Type *Ty,
1140
                                           CharUnits MinAlign,
1141
247
                                           const Twine &Name = "tmp") {
1142
  // Don't use an alignment that's worse than what LLVM would prefer.
1143
247
  auto PrefAlign = CGF.CGM.getDataLayout().getPrefTypeAlignment(Ty);
1144
247
  CharUnits Align = std::max(MinAlign, CharUnits::fromQuantity(PrefAlign));
1145
1146
247
  return CGF.CreateTempAlloca(Ty, Align, Name + ".coerce");
1147
247
}
1148
1149
/// EnterStructPointerForCoercedAccess - Given a struct pointer that we are
1150
/// accessing some number of bytes out of it, try to gep into the struct to get
1151
/// at its inner goodness.  Dive as deep as possible without entering an element
1152
/// with an in-memory size smaller than DstSize.
1153
static Address
1154
EnterStructPointerForCoercedAccess(Address SrcPtr,
1155
                                   llvm::StructType *SrcSTy,
1156
18.9k
                                   uint64_t DstSize, CodeGenFunction &CGF) {
1157
  // We can't dive into a zero-element struct.
1158
18.9k
  if (SrcSTy->getNumElements() == 0) 
return SrcPtr0
;
1159
1160
18.9k
  llvm::Type *FirstElt = SrcSTy->getElementType(0);
1161
1162
  // If the first elt is at least as large as what we're looking for, or if the
1163
  // first element is the same size as the whole struct, we can enter it. The
1164
  // comparison must be made on the store size and not the alloca size. Using
1165
  // the alloca size may overstate the size of the load.
1166
18.9k
  uint64_t FirstEltSize =
1167
18.9k
    CGF.CGM.getDataLayout().getTypeStoreSize(FirstElt);
1168
18.9k
  if (FirstEltSize < DstSize &&
1169
18.9k
      
FirstEltSize < CGF.CGM.getDataLayout().getTypeStoreSize(SrcSTy)5.68k
)
1170
5.54k
    return SrcPtr;
1171
1172
  // GEP into the first element.
1173
13.4k
  SrcPtr = CGF.Builder.CreateStructGEP(SrcPtr, 0, "coerce.dive");
1174
1175
  // If the first element is a struct, recurse.
1176
13.4k
  llvm::Type *SrcTy = SrcPtr.getElementType();
1177
13.4k
  if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy))
1178
3.16k
    return EnterStructPointerForCoercedAccess(SrcPtr, SrcSTy, DstSize, CGF);
1179
1180
10.2k
  return SrcPtr;
1181
13.4k
}
1182
1183
/// CoerceIntOrPtrToIntOrPtr - Convert a value Val to the specific Ty where both
1184
/// are either integers or pointers.  This does a truncation of the value if it
1185
/// is too large or a zero extension if it is too small.
1186
///
1187
/// This behaves as if the value were coerced through memory, so on big-endian
1188
/// targets the high bits are preserved in a truncation, while little-endian
1189
/// targets preserve the low bits.
1190
static llvm::Value *CoerceIntOrPtrToIntOrPtr(llvm::Value *Val,
1191
                                             llvm::Type *Ty,
1192
8.98k
                                             CodeGenFunction &CGF) {
1193
8.98k
  if (Val->getType() == Ty)
1194
8.57k
    return Val;
1195
1196
418
  if (isa<llvm::PointerType>(Val->getType())) {
1197
    // If this is Pointer->Pointer avoid conversion to and from int.
1198
302
    if (isa<llvm::PointerType>(Ty))
1199
91
      return CGF.Builder.CreateBitCast(Val, Ty, "coerce.val");
1200
1201
    // Convert the pointer to an integer so we can play with its width.
1202
211
    Val = CGF.Builder.CreatePtrToInt(Val, CGF.IntPtrTy, "coerce.val.pi");
1203
211
  }
1204
1205
327
  llvm::Type *DestIntTy = Ty;
1206
327
  if (isa<llvm::PointerType>(DestIntTy))
1207
31
    DestIntTy = CGF.IntPtrTy;
1208
1209
327
  if (Val->getType() != DestIntTy) {
1210
85
    const llvm::DataLayout &DL = CGF.CGM.getDataLayout();
1211
85
    if (DL.isBigEndian()) {
1212
      // Preserve the high bits on big-endian targets.
1213
      // That is what memory coercion does.
1214
30
      uint64_t SrcSize = DL.getTypeSizeInBits(Val->getType());
1215
30
      uint64_t DstSize = DL.getTypeSizeInBits(DestIntTy);
1216
1217
30
      if (SrcSize > DstSize) {
1218
11
        Val = CGF.Builder.CreateLShr(Val, SrcSize - DstSize, "coerce.highbits");
1219
11
        Val = CGF.Builder.CreateTrunc(Val, DestIntTy, "coerce.val.ii");
1220
19
      } else {
1221
19
        Val = CGF.Builder.CreateZExt(Val, DestIntTy, "coerce.val.ii");
1222
19
        Val = CGF.Builder.CreateShl(Val, DstSize - SrcSize, "coerce.highbits");
1223
19
      }
1224
55
    } else {
1225
      // Little-endian targets preserve the low bits. No shifts required.
1226
55
      Val = CGF.Builder.CreateIntCast(Val, DestIntTy, false, "coerce.val.ii");
1227
55
    }
1228
85
  }
1229
1230
327
  if (isa<llvm::PointerType>(Ty))
1231
31
    Val = CGF.Builder.CreateIntToPtr(Val, Ty, "coerce.val.ip");
1232
327
  return Val;
1233
418
}
1234
1235
1236
1237
/// CreateCoercedLoad - Create a load from \arg SrcPtr interpreted as
1238
/// a pointer to an object of type \arg Ty, known to be aligned to
1239
/// \arg SrcAlign bytes.
1240
///
1241
/// This safely handles the case when the src type is smaller than the
1242
/// destination type; in this situation the values of bits which not
1243
/// present in the src are undefined.
1244
static llvm::Value *CreateCoercedLoad(Address Src, llvm::Type *Ty,
1245
9.18k
                                      CodeGenFunction &CGF) {
1246
9.18k
  llvm::Type *SrcTy = Src.getElementType();
1247
1248
  // If SrcTy and Ty are the same, just do a load.
1249
9.18k
  if (SrcTy == Ty)
1250
31
    return CGF.Builder.CreateLoad(Src);
1251
1252
9.15k
  llvm::TypeSize DstSize = CGF.CGM.getDataLayout().getTypeAllocSize(Ty);
1253
1254
9.15k
  if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy)) {
1255
7.01k
    Src = EnterStructPointerForCoercedAccess(Src, SrcSTy,
1256
7.01k
                                             DstSize.getFixedSize(), CGF);
1257
7.01k
    SrcTy = Src.getElementType();
1258
7.01k
  }
1259
1260
9.15k
  llvm::TypeSize SrcSize = CGF.CGM.getDataLayout().getTypeAllocSize(SrcTy);
1261
1262
  // If the source and destination are integer or pointer types, just do an
1263
  // extension or truncation to the desired type.
1264
9.15k
  if ((isa<llvm::IntegerType>(Ty) || 
isa<llvm::PointerType>(Ty)6.21k
) &&
1265
9.15k
      
(5.89k
isa<llvm::IntegerType>(SrcTy)5.89k
||
isa<llvm::PointerType>(SrcTy)4.67k
)) {
1266
4.38k
    llvm::Value *Load = CGF.Builder.CreateLoad(Src);
1267
4.38k
    return CoerceIntOrPtrToIntOrPtr(Load, Ty, CGF);
1268
4.38k
  }
1269
1270
  // If load is legal, just bitcast the src pointer.
1271
4.76k
  if (!SrcSize.isScalable() && !DstSize.isScalable() &&
1272
4.76k
      
SrcSize.getFixedSize() >= DstSize.getFixedSize()4.74k
) {
1273
    // Generally SrcSize is never greater than DstSize, since this means we are
1274
    // losing bits. However, this can happen in cases where the structure has
1275
    // additional padding, for example due to a user specified alignment.
1276
    //
1277
    // FIXME: Assert that we aren't truncating non-padding bits when have access
1278
    // to that information.
1279
4.59k
    Src = CGF.Builder.CreateElementBitCast(Src, Ty);
1280
4.59k
    return CGF.Builder.CreateLoad(Src);
1281
4.59k
  }
1282
1283
  // If coercing a fixed vector to a scalable vector for ABI compatibility, and
1284
  // the types match, use the llvm.experimental.vector.insert intrinsic to
1285
  // perform the conversion.
1286
172
  if (auto *ScalableDst = dyn_cast<llvm::ScalableVectorType>(Ty)) {
1287
26
    if (auto *FixedSrc = dyn_cast<llvm::FixedVectorType>(SrcTy)) {
1288
      // If we are casting a fixed i8 vector to a scalable 16 x i1 predicate
1289
      // vector, use a vector insert and bitcast the result.
1290
26
      bool NeedsBitcast = false;
1291
26
      auto PredType =
1292
26
          llvm::ScalableVectorType::get(CGF.Builder.getInt1Ty(), 16);
1293
26
      llvm::Type *OrigType = Ty;
1294
26
      if (ScalableDst == PredType &&
1295
26
          
FixedSrc->getElementType() == CGF.Builder.getInt8Ty()5
) {
1296
5
        ScalableDst = llvm::ScalableVectorType::get(CGF.Builder.getInt8Ty(), 2);
1297
5
        NeedsBitcast = true;
1298
5
      }
1299
26
      if (ScalableDst->getElementType() == FixedSrc->getElementType()) {
1300
26
        auto *Load = CGF.Builder.CreateLoad(Src);
1301
26
        auto *UndefVec = llvm::UndefValue::get(ScalableDst);
1302
26
        auto *Zero = llvm::Constant::getNullValue(CGF.CGM.Int64Ty);
1303
26
        llvm::Value *Result = CGF.Builder.CreateInsertVector(
1304
26
            ScalableDst, UndefVec, Load, Zero, "castScalableSve");
1305
26
        if (NeedsBitcast)
1306
5
          Result = CGF.Builder.CreateBitCast(Result, OrigType);
1307
26
        return Result;
1308
26
      }
1309
26
    }
1310
26
  }
1311
1312
  // Otherwise do coercion through memory. This is stupid, but simple.
1313
146
  Address Tmp =
1314
146
      CreateTempAllocaForCoercion(CGF, Ty, Src.getAlignment(), Src.getName());
1315
146
  CGF.Builder.CreateMemCpy(
1316
146
      Tmp.getPointer(), Tmp.getAlignment().getAsAlign(), Src.getPointer(),
1317
146
      Src.getAlignment().getAsAlign(),
1318
146
      llvm::ConstantInt::get(CGF.IntPtrTy, SrcSize.getKnownMinSize()));
1319
146
  return CGF.Builder.CreateLoad(Tmp);
1320
172
}
1321
1322
// Function to store a first-class aggregate into memory.  We prefer to
1323
// store the elements rather than the aggregate to be more friendly to
1324
// fast-isel.
1325
// FIXME: Do we need to recurse here?
1326
void CodeGenFunction::EmitAggregateStore(llvm::Value *Val, Address Dest,
1327
5.95k
                                         bool DestIsVolatile) {
1328
  // Prefer scalar stores to first-class aggregate stores.
1329
5.95k
  if (llvm::StructType *STy = dyn_cast<llvm::StructType>(Val->getType())) {
1330
5.08k
    for (unsigned i = 0, e = STy->getNumElements(); i != e; 
++i3.37k
) {
1331
3.37k
      Address EltPtr = Builder.CreateStructGEP(Dest, i);
1332
3.37k
      llvm::Value *Elt = Builder.CreateExtractValue(Val, i);
1333
3.37k
      Builder.CreateStore(Elt, EltPtr, DestIsVolatile);
1334
3.37k
    }
1335
4.23k
  } else {
1336
4.23k
    Builder.CreateStore(Val, Dest, DestIsVolatile);
1337
4.23k
  }
1338
5.95k
}
1339
1340
/// CreateCoercedStore - Create a store to \arg DstPtr from \arg Src,
1341
/// where the source and destination may have different types.  The
1342
/// destination is known to be aligned to \arg DstAlign bytes.
1343
///
1344
/// This safely handles the case when the src type is larger than the
1345
/// destination type; the upper bits of the src will be lost.
1346
static void CreateCoercedStore(llvm::Value *Src,
1347
                               Address Dst,
1348
                               bool DstIsVolatile,
1349
10.6k
                               CodeGenFunction &CGF) {
1350
10.6k
  llvm::Type *SrcTy = Src->getType();
1351
10.6k
  llvm::Type *DstTy = Dst.getElementType();
1352
10.6k
  if (SrcTy == DstTy) {
1353
144
    CGF.Builder.CreateStore(Src, Dst, DstIsVolatile);
1354
144
    return;
1355
144
  }
1356
1357
10.5k
  llvm::TypeSize SrcSize = CGF.CGM.getDataLayout().getTypeAllocSize(SrcTy);
1358
1359
10.5k
  if (llvm::StructType *DstSTy = dyn_cast<llvm::StructType>(DstTy)) {
1360
8.81k
    Dst = EnterStructPointerForCoercedAccess(Dst, DstSTy,
1361
8.81k
                                             SrcSize.getFixedSize(), CGF);
1362
8.81k
    DstTy = Dst.getElementType();
1363
8.81k
  }
1364
1365
10.5k
  llvm::PointerType *SrcPtrTy = llvm::dyn_cast<llvm::PointerType>(SrcTy);
1366
10.5k
  llvm::PointerType *DstPtrTy = llvm::dyn_cast<llvm::PointerType>(DstTy);
1367
10.5k
  if (SrcPtrTy && 
DstPtrTy3.12k
&&
1368
10.5k
      
SrcPtrTy->getAddressSpace() != DstPtrTy->getAddressSpace()3.12k
) {
1369
22
    Src = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Src, DstTy);
1370
22
    CGF.Builder.CreateStore(Src, Dst, DstIsVolatile);
1371
22
    return;
1372
22
  }
1373
1374
  // If the source and destination are integer or pointer types, just do an
1375
  // extension or truncation to the desired type.
1376
10.5k
  if ((isa<llvm::IntegerType>(SrcTy) || 
isa<llvm::PointerType>(SrcTy)7.37k
) &&
1377
10.5k
      
(6.25k
isa<llvm::IntegerType>(DstTy)6.25k
||
isa<llvm::PointerType>(DstTy)4.78k
)) {
1378
4.60k
    Src = CoerceIntOrPtrToIntOrPtr(Src, DstTy, CGF);
1379
4.60k
    CGF.Builder.CreateStore(Src, Dst, DstIsVolatile);
1380
4.60k
    return;
1381
4.60k
  }
1382
1383
5.92k
  llvm::TypeSize DstSize = CGF.CGM.getDataLayout().getTypeAllocSize(DstTy);
1384
1385
  // If store is legal, just bitcast the src pointer.
1386
5.92k
  if (isa<llvm::ScalableVectorType>(SrcTy) ||
1387
5.92k
      
isa<llvm::ScalableVectorType>(DstTy)5.92k
||
1388
5.92k
      
SrcSize.getFixedSize() <= DstSize.getFixedSize()5.92k
) {
1389
5.82k
    Dst = CGF.Builder.CreateElementBitCast(Dst, SrcTy);
1390
5.82k
    CGF.EmitAggregateStore(Src, Dst, DstIsVolatile);
1391
5.82k
  } else {
1392
    // Otherwise do coercion through memory. This is stupid, but
1393
    // simple.
1394
1395
    // Generally SrcSize is never greater than DstSize, since this means we are
1396
    // losing bits. However, this can happen in cases where the structure has
1397
    // additional padding, for example due to a user specified alignment.
1398
    //
1399
    // FIXME: Assert that we aren't truncating non-padding bits when have access
1400
    // to that information.
1401
101
    Address Tmp = CreateTempAllocaForCoercion(CGF, SrcTy, Dst.getAlignment());
1402
101
    CGF.Builder.CreateStore(Src, Tmp);
1403
101
    CGF.Builder.CreateMemCpy(
1404
101
        Dst.getPointer(), Dst.getAlignment().getAsAlign(), Tmp.getPointer(),
1405
101
        Tmp.getAlignment().getAsAlign(),
1406
101
        llvm::ConstantInt::get(CGF.IntPtrTy, DstSize.getFixedSize()));
1407
101
  }
1408
5.92k
}
1409
1410
static Address emitAddressAtOffset(CodeGenFunction &CGF, Address addr,
1411
22.1k
                                   const ABIArgInfo &info) {
1412
22.1k
  if (unsigned offset = info.getDirectOffset()) {
1413
3
    addr = CGF.Builder.CreateElementBitCast(addr, CGF.Int8Ty);
1414
3
    addr = CGF.Builder.CreateConstInBoundsByteGEP(addr,
1415
3
                                             CharUnits::fromQuantity(offset));
1416
3
    addr = CGF.Builder.CreateElementBitCast(addr, info.getCoerceToType());
1417
3
  }
1418
22.1k
  return addr;
1419
22.1k
}
1420
1421
namespace {
1422
1423
/// Encapsulates information about the way function arguments from
1424
/// CGFunctionInfo should be passed to actual LLVM IR function.
1425
class ClangToLLVMArgMapping {
1426
  static const unsigned InvalidIndex = ~0U;
1427
  unsigned InallocaArgNo;
1428
  unsigned SRetArgNo;
1429
  unsigned TotalIRArgs;
1430
1431
  /// Arguments of LLVM IR function corresponding to single Clang argument.
1432
  struct IRArgs {
1433
    unsigned PaddingArgIndex;
1434
    // Argument is expanded to IR arguments at positions
1435
    // [FirstArgIndex, FirstArgIndex + NumberOfArgs).
1436
    unsigned FirstArgIndex;
1437
    unsigned NumberOfArgs;
1438
1439
    IRArgs()
1440
        : PaddingArgIndex(InvalidIndex), FirstArgIndex(InvalidIndex),
1441
2.48M
          NumberOfArgs(0) {}
1442
  };
1443
1444
  SmallVector<IRArgs, 8> ArgInfo;
1445
1446
public:
1447
  ClangToLLVMArgMapping(const ASTContext &Context, const CGFunctionInfo &FI,
1448
                        bool OnlyRequiredArgs = false)
1449
      : InallocaArgNo(InvalidIndex), SRetArgNo(InvalidIndex), TotalIRArgs(0),
1450
2.48M
        ArgInfo(OnlyRequiredArgs ? FI.getNumRequiredArgs() : FI.arg_size()) {
1451
2.48M
    construct(Context, FI, OnlyRequiredArgs);
1452
2.48M
  }
1453
1454
3.00M
  bool hasInallocaArg() const { return InallocaArgNo != InvalidIndex; }
1455
580
  unsigned getInallocaArgNo() const {
1456
580
    assert(hasInallocaArg());
1457
0
    return InallocaArgNo;
1458
580
  }
1459
1460
2.14M
  bool hasSRetArg() const { return SRetArgNo != InvalidIndex; }
1461
22.2k
  unsigned getSRetArgNo() const {
1462
22.2k
    assert(hasSRetArg());
1463
0
    return SRetArgNo;
1464
22.2k
  }
1465
1466
2.48M
  unsigned totalIRArgs() const { return TotalIRArgs; }
1467
1468
3.49M
  bool hasPaddingArg(unsigned ArgNo) const {
1469
3.49M
    assert(ArgNo < ArgInfo.size());
1470
0
    return ArgInfo[ArgNo].PaddingArgIndex != InvalidIndex;
1471
3.49M
  }
1472
69
  unsigned getPaddingArgNo(unsigned ArgNo) const {
1473
69
    assert(hasPaddingArg(ArgNo));
1474
0
    return ArgInfo[ArgNo].PaddingArgIndex;
1475
69
  }
1476
1477
  /// Returns index of first IR argument corresponding to ArgNo, and their
1478
  /// quantity.
1479
3.97M
  std::pair<unsigned, unsigned> getIRArgs(unsigned ArgNo) const {
1480
3.97M
    assert(ArgNo < ArgInfo.size());
1481
0
    return std::make_pair(ArgInfo[ArgNo].FirstArgIndex,
1482
3.97M
                          ArgInfo[ArgNo].NumberOfArgs);
1483
3.97M
  }
1484
1485
private:
1486
  void construct(const ASTContext &Context, const CGFunctionInfo &FI,
1487
                 bool OnlyRequiredArgs);
1488
};
1489
1490
void ClangToLLVMArgMapping::construct(const ASTContext &Context,
1491
                                      const CGFunctionInfo &FI,
1492
2.48M
                                      bool OnlyRequiredArgs) {
1493
2.48M
  unsigned IRArgNo = 0;
1494
2.48M
  bool SwapThisWithSRet = false;
1495
2.48M
  const ABIArgInfo &RetAI = FI.getReturnInfo();
1496
1497
2.48M
  if (RetAI.getKind() == ABIArgInfo::Indirect) {
1498
22.2k
    SwapThisWithSRet = RetAI.isSRetAfterThis();
1499
22.2k
    SRetArgNo = SwapThisWithSRet ? 
1455
:
IRArgNo++21.8k
;
1500
22.2k
  }
1501
1502
2.48M
  unsigned ArgNo = 0;
1503
2.48M
  unsigned NumArgs = OnlyRequiredArgs ? 
FI.getNumRequiredArgs()1.08M
:
FI.arg_size()1.39M
;
1504
6.50M
  for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(); ArgNo < NumArgs;
1505
4.02M
       ++I, ++ArgNo) {
1506
4.02M
    assert(I != FI.arg_end());
1507
0
    QualType ArgType = I->type;
1508
4.02M
    const ABIArgInfo &AI = I->info;
1509
    // Collect data about IR arguments corresponding to Clang argument ArgNo.
1510
4.02M
    auto &IRArgs = ArgInfo[ArgNo];
1511
1512
4.02M
    if (AI.getPaddingType())
1513
124
      IRArgs.PaddingArgIndex = IRArgNo++;
1514
1515
4.02M
    switch (AI.getKind()) {
1516
94.4k
    case ABIArgInfo::Extend:
1517
3.94M
    case ABIArgInfo::Direct: {
1518
      // FIXME: handle sseregparm someday...
1519
3.94M
      llvm::StructType *STy = dyn_cast<llvm::StructType>(AI.getCoerceToType());
1520
3.94M
      if (AI.isDirect() && 
AI.getCanBeFlattened()3.85M
&&
STy3.84M
) {
1521
8.64k
        IRArgs.NumberOfArgs = STy->getNumElements();
1522
3.93M
      } else {
1523
3.93M
        IRArgs.NumberOfArgs = 1;
1524
3.93M
      }
1525
3.94M
      break;
1526
94.4k
    }
1527
17.5k
    case ABIArgInfo::Indirect:
1528
17.6k
    case ABIArgInfo::IndirectAliased:
1529
17.6k
      IRArgs.NumberOfArgs = 1;
1530
17.6k
      break;
1531
54.7k
    case ABIArgInfo::Ignore:
1532
55.6k
    case ABIArgInfo::InAlloca:
1533
      // ignore and inalloca doesn't have matching LLVM parameters.
1534
55.6k
      IRArgs.NumberOfArgs = 0;
1535
55.6k
      break;
1536
3.47k
    case ABIArgInfo::CoerceAndExpand:
1537
3.47k
      IRArgs.NumberOfArgs = AI.getCoerceAndExpandTypeSequence().size();
1538
3.47k
      break;
1539
474
    case ABIArgInfo::Expand:
1540
474
      IRArgs.NumberOfArgs = getExpansionSize(ArgType, Context);
1541
474
      break;
1542
4.02M
    }
1543
1544
4.02M
    if (IRArgs.NumberOfArgs > 0) {
1545
3.96M
      IRArgs.FirstArgIndex = IRArgNo;
1546
3.96M
      IRArgNo += IRArgs.NumberOfArgs;
1547
3.96M
    }
1548
1549
    // Skip over the sret parameter when it comes second.  We already handled it
1550
    // above.
1551
4.02M
    if (IRArgNo == 1 && 
SwapThisWithSRet2.29M
)
1552
455
      IRArgNo++;
1553
4.02M
  }
1554
2.48M
  assert(ArgNo == ArgInfo.size());
1555
1556
2.48M
  if (FI.usesInAlloca())
1557
477
    InallocaArgNo = IRArgNo++;
1558
1559
2.48M
  TotalIRArgs = IRArgNo;
1560
2.48M
}
1561
}  // namespace
1562
1563
/***/
1564
1565
20.1k
bool CodeGenModule::ReturnTypeUsesSRet(const CGFunctionInfo &FI) {
1566
20.1k
  const auto &RI = FI.getReturnInfo();
1567
20.1k
  return RI.isIndirect() || 
(20.0k
RI.isInAlloca()20.0k
&&
RI.getInAllocaSRet()1
);
1568
20.1k
}
1569
1570
12.9k
bool CodeGenModule::ReturnSlotInterferesWithArgs(const CGFunctionInfo &FI) {
1571
12.9k
  return ReturnTypeUsesSRet(FI) &&
1572
12.9k
         
getTargetCodeGenInfo().doesReturnSlotInterfereWithArgs()44
;
1573
12.9k
}
1574
1575
11.7k
bool CodeGenModule::ReturnTypeUsesFPRet(QualType ResultType) {
1576
11.7k
  if (const BuiltinType *BT = ResultType->getAs<BuiltinType>()) {
1577
4.92k
    switch (BT->getKind()) {
1578
4.86k
    default:
1579
4.86k
      return false;
1580
18
    case BuiltinType::Float:
1581
18
      return getTarget().useObjCFPRetForRealType(FloatModeKind::Float);
1582
43
    case BuiltinType::Double:
1583
43
      return getTarget().useObjCFPRetForRealType(FloatModeKind::Double);
1584
4
    case BuiltinType::LongDouble:
1585
4
      return getTarget().useObjCFPRetForRealType(FloatModeKind::LongDouble);
1586
4.92k
    }
1587
4.92k
  }
1588
1589
6.85k
  return false;
1590
11.7k
}
1591
1592
11.7k
bool CodeGenModule::ReturnTypeUsesFP2Ret(QualType ResultType) {
1593
11.7k
  if (const ComplexType *CT = ResultType->getAs<ComplexType>()) {
1594
14
    if (const BuiltinType *BT = CT->getElementType()->getAs<BuiltinType>()) {
1595
14
      if (BT->getKind() == BuiltinType::LongDouble)
1596
2
        return getTarget().useObjCFP2RetForComplexLongDouble();
1597
14
    }
1598
14
  }
1599
1600
11.7k
  return false;
1601
11.7k
}
1602
1603
5.50k
llvm::FunctionType *CodeGenTypes::GetFunctionType(GlobalDecl GD) {
1604
5.50k
  const CGFunctionInfo &FI = arrangeGlobalDeclaration(GD);
1605
5.50k
  return GetFunctionType(FI);
1606
5.50k
}
1607
1608
llvm::FunctionType *
1609
1.08M
CodeGenTypes::GetFunctionType(const CGFunctionInfo &FI) {
1610
1611
1.08M
  bool Inserted = FunctionsBeingProcessed.insert(&FI).second;
1612
1.08M
  (void)Inserted;
1613
1.08M
  assert(Inserted && "Recursively being processed?");
1614
1615
0
  llvm::Type *resultType = nullptr;
1616
1.08M
  const ABIArgInfo &retAI = FI.getReturnInfo();
1617
1.08M
  switch (retAI.getKind()) {
1618
0
  case ABIArgInfo::Expand:
1619
0
  case ABIArgInfo::IndirectAliased:
1620
0
    llvm_unreachable("Invalid ABI kind for return argument");
1621
1622
35.0k
  case ABIArgInfo::Extend:
1623
567k
  case ABIArgInfo::Direct:
1624
567k
    resultType = retAI.getCoerceToType();
1625
567k
    break;
1626
1627
18
  case ABIArgInfo::InAlloca:
1628
18
    if (retAI.getInAllocaSRet()) {
1629
      // sret things on win32 aren't void, they return the sret pointer.
1630
18
      QualType ret = FI.getReturnType();
1631
18
      llvm::Type *ty = ConvertType(ret);
1632
18
      unsigned addressSpace = Context.getTargetAddressSpace(ret);
1633
18
      resultType = llvm::PointerType::get(ty, addressSpace);
1634
18
    } else {
1635
0
      resultType = llvm::Type::getVoidTy(getLLVMContext());
1636
0
    }
1637
18
    break;
1638
1639
9.13k
  case ABIArgInfo::Indirect:
1640
517k
  case ABIArgInfo::Ignore:
1641
517k
    resultType = llvm::Type::getVoidTy(getLLVMContext());
1642
517k
    break;
1643
1644
1.42k
  case ABIArgInfo::CoerceAndExpand:
1645
1.42k
    resultType = retAI.getUnpaddedCoerceAndExpandType();
1646
1.42k
    break;
1647
1.08M
  }
1648
1649
1.08M
  ClangToLLVMArgMapping IRFunctionArgs(getContext(), FI, true);
1650
1.08M
  SmallVector<llvm::Type*, 8> ArgTypes(IRFunctionArgs.totalIRArgs());
1651
1652
  // Add type for sret argument.
1653
1.08M
  if (IRFunctionArgs.hasSRetArg()) {
1654
9.13k
    QualType Ret = FI.getReturnType();
1655
9.13k
    llvm::Type *Ty = ConvertType(Ret);
1656
9.13k
    unsigned AddressSpace = Context.getTargetAddressSpace(Ret);
1657
9.13k
    ArgTypes[IRFunctionArgs.getSRetArgNo()] =
1658
9.13k
        llvm::PointerType::get(Ty, AddressSpace);
1659
9.13k
  }
1660
1661
  // Add type for inalloca argument.
1662
1.08M
  if (IRFunctionArgs.hasInallocaArg()) {
1663
200
    auto ArgStruct = FI.getArgStruct();
1664
200
    assert(ArgStruct);
1665
0
    ArgTypes[IRFunctionArgs.getInallocaArgNo()] = ArgStruct->getPointerTo();
1666
200
  }
1667
1668
  // Add in all of the required arguments.
1669
0
  unsigned ArgNo = 0;
1670
1.08M
  CGFunctionInfo::const_arg_iterator it = FI.arg_begin(),
1671
1.08M
                                     ie = it + FI.getNumRequiredArgs();
1672
2.78M
  for (; it != ie; 
++it, ++ArgNo1.69M
) {
1673
1.69M
    const ABIArgInfo &ArgInfo = it->info;
1674
1675
    // Insert a padding type to ensure proper alignment.
1676
1.69M
    if (IRFunctionArgs.hasPaddingArg(ArgNo))
1677
47
      ArgTypes[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
1678
47
          ArgInfo.getPaddingType();
1679
1680
1.69M
    unsigned FirstIRArg, NumIRArgs;
1681
1.69M
    std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
1682
1683
1.69M
    switch (ArgInfo.getKind()) {
1684
27.0k
    case ABIArgInfo::Ignore:
1685
27.3k
    case ABIArgInfo::InAlloca:
1686
27.3k
      assert(NumIRArgs == 0);
1687
0
      break;
1688
1689
7.91k
    case ABIArgInfo::Indirect: {
1690
7.91k
      assert(NumIRArgs == 1);
1691
      // indirect arguments are always on the stack, which is alloca addr space.
1692
0
      llvm::Type *LTy = ConvertTypeForMem(it->type);
1693
7.91k
      ArgTypes[FirstIRArg] = LTy->getPointerTo(
1694
7.91k
          CGM.getDataLayout().getAllocaAddrSpace());
1695
7.91k
      break;
1696
27.0k
    }
1697
29
    case ABIArgInfo::IndirectAliased: {
1698
29
      assert(NumIRArgs == 1);
1699
0
      llvm::Type *LTy = ConvertTypeForMem(it->type);
1700
29
      ArgTypes[FirstIRArg] = LTy->getPointerTo(ArgInfo.getIndirectAddrSpace());
1701
29
      break;
1702
27.0k
    }
1703
39.7k
    case ABIArgInfo::Extend:
1704
1.65M
    case ABIArgInfo::Direct: {
1705
      // Fast-isel and the optimizer generally like scalar values better than
1706
      // FCAs, so we flatten them if this is safe to do for this argument.
1707
1.65M
      llvm::Type *argType = ArgInfo.getCoerceToType();
1708
1.65M
      llvm::StructType *st = dyn_cast<llvm::StructType>(argType);
1709
1.65M
      if (st && 
ArgInfo.isDirect()3.96k
&&
ArgInfo.getCanBeFlattened()3.96k
) {
1710
3.76k
        assert(NumIRArgs == st->getNumElements());
1711
11.3k
        for (unsigned i = 0, e = st->getNumElements(); i != e; 
++i7.59k
)
1712
7.59k
          ArgTypes[FirstIRArg + i] = st->getElementType(i);
1713
1.65M
      } else {
1714
1.65M
        assert(NumIRArgs == 1);
1715
0
        ArgTypes[FirstIRArg] = argType;
1716
1.65M
      }
1717
0
      break;
1718
39.7k
    }
1719
1720
1.45k
    case ABIArgInfo::CoerceAndExpand: {
1721
1.45k
      auto ArgTypesIter = ArgTypes.begin() + FirstIRArg;
1722
3.12k
      for (auto EltTy : ArgInfo.getCoerceAndExpandTypeSequence()) {
1723
3.12k
        *ArgTypesIter++ = EltTy;
1724
3.12k
      }
1725
1.45k
      assert(ArgTypesIter == ArgTypes.begin() + FirstIRArg + NumIRArgs);
1726
0
      break;
1727
39.7k
    }
1728
1729
181
    case ABIArgInfo::Expand:
1730
181
      auto ArgTypesIter = ArgTypes.begin() + FirstIRArg;
1731
181
      getExpandedTypes(it->type, ArgTypesIter);
1732
181
      assert(ArgTypesIter == ArgTypes.begin() + FirstIRArg + NumIRArgs);
1733
0
      break;
1734
1.69M
    }
1735
1.69M
  }
1736
1737
1.08M
  bool Erased = FunctionsBeingProcessed.erase(&FI); (void)Erased;
1738
1.08M
  assert(Erased && "Not in set?");
1739
1740
0
  return llvm::FunctionType::get(resultType, ArgTypes, FI.isVariadic());
1741
1.08M
}
1742
1743
5.49k
llvm::Type *CodeGenTypes::GetFunctionTypeForVTable(GlobalDecl GD) {
1744
5.49k
  const CXXMethodDecl *MD = cast<CXXMethodDecl>(GD.getDecl());
1745
5.49k
  const FunctionProtoType *FPT = MD->getType()->getAs<FunctionProtoType>();
1746
1747
5.49k
  if (!isFuncTypeConvertible(FPT))
1748
39
    return llvm::StructType::get(getLLVMContext());
1749
1750
5.45k
  return GetFunctionType(GD);
1751
5.49k
}
1752
1753
static void AddAttributesFromFunctionProtoType(ASTContext &Ctx,
1754
                                               llvm::AttrBuilder &FuncAttrs,
1755
1.36M
                                               const FunctionProtoType *FPT) {
1756
1.36M
  if (!FPT)
1757
711k
    return;
1758
1759
657k
  if (!isUnresolvedExceptionSpec(FPT->getExceptionSpecType()) &&
1760
657k
      FPT->isNothrow())
1761
192k
    FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
1762
657k
}
1763
1764
static void AddAttributesFromAssumes(llvm::AttrBuilder &FuncAttrs,
1765
717k
                                     const Decl *Callee) {
1766
717k
  if (!Callee)
1767
31.8k
    return;
1768
1769
685k
  SmallVector<StringRef, 4> Attrs;
1770
1771
685k
  for (const AssumptionAttr *AA : Callee->specific_attrs<AssumptionAttr>())
1772
481
    AA->getAssumption().split(Attrs, ",");
1773
1774
685k
  if (!Attrs.empty())
1775
198
    FuncAttrs.addAttribute(llvm::AssumptionAttrKey,
1776
198
                           llvm::join(Attrs.begin(), Attrs.end(), ","));
1777
685k
}
1778
1779
bool CodeGenModule::MayDropFunctionReturn(const ASTContext &Context,
1780
48
                                          QualType ReturnType) {
1781
  // We can't just discard the return value for a record type with a
1782
  // complex destructor or a non-trivially copyable type.
1783
48
  if (const RecordType *RT =
1784
48
          ReturnType.getCanonicalType()->getAs<RecordType>()) {
1785
24
    if (const auto *ClassDecl = dyn_cast<CXXRecordDecl>(RT->getDecl()))
1786
24
      return ClassDecl->hasTrivialDestructor();
1787
24
  }
1788
24
  return ReturnType.isTriviallyCopyableType(Context);
1789
48
}
1790
1791
void CodeGenModule::getDefaultFunctionAttributes(StringRef Name,
1792
                                                 bool HasOptnone,
1793
                                                 bool AttrOnCallSite,
1794
717k
                                               llvm::AttrBuilder &FuncAttrs) {
1795
  // OptimizeNoneAttr takes precedence over -Os or -Oz. No warning needed.
1796
717k
  if (!HasOptnone) {
1797
717k
    if (CodeGenOpts.OptimizeSize)
1798
1.07k
      FuncAttrs.addAttribute(llvm::Attribute::OptimizeForSize);
1799
717k
    if (CodeGenOpts.OptimizeSize == 2)
1800
717
      FuncAttrs.addAttribute(llvm::Attribute::MinSize);
1801
717k
  }
1802
1803
717k
  if (CodeGenOpts.DisableRedZone)
1804
25
    FuncAttrs.addAttribute(llvm::Attribute::NoRedZone);
1805
717k
  if (CodeGenOpts.IndirectTlsSegRefs)
1806
1
    FuncAttrs.addAttribute("indirect-tls-seg-refs");
1807
717k
  if (CodeGenOpts.NoImplicitFloat)
1808
16
    FuncAttrs.addAttribute(llvm::Attribute::NoImplicitFloat);
1809
1810
717k
  if (AttrOnCallSite) {
1811
    // Attributes that should go on the call site only.
1812
    // FIXME: Look for 'BuiltinAttr' on the function rather than re-checking
1813
    // the -fno-builtin-foo list.
1814
366k
    if (!CodeGenOpts.SimplifyLibCalls || 
LangOpts.isNoBuiltinFunc(Name)174k
)
1815
192k
      FuncAttrs.addAttribute(llvm::Attribute::NoBuiltin);
1816
366k
    if (!CodeGenOpts.TrapFuncName.empty())
1817
0
      FuncAttrs.addAttribute("trap-func-name", CodeGenOpts.TrapFuncName);
1818
366k
  } else {
1819
350k
    StringRef FpKind;
1820
350k
    switch (CodeGenOpts.getFramePointer()) {
1821
214k
    case CodeGenOptions::FramePointerKind::None:
1822
214k
      FpKind = "none";
1823
214k
      break;
1824
165
    case CodeGenOptions::FramePointerKind::NonLeaf:
1825
165
      FpKind = "non-leaf";
1826
165
      break;
1827
136k
    case CodeGenOptions::FramePointerKind::All:
1828
136k
      FpKind = "all";
1829
136k
      break;
1830
350k
    }
1831
350k
    FuncAttrs.addAttribute("frame-pointer", FpKind);
1832
1833
350k
    if (CodeGenOpts.LessPreciseFPMAD)
1834
6
      FuncAttrs.addAttribute("less-precise-fpmad", "true");
1835
1836
350k
    if (CodeGenOpts.NullPointerIsValid)
1837
238
      FuncAttrs.addAttribute(llvm::Attribute::NullPointerIsValid);
1838
1839
350k
    if (CodeGenOpts.FPDenormalMode != llvm::DenormalMode::getIEEE())
1840
10
      FuncAttrs.addAttribute("denormal-fp-math",
1841
10
                             CodeGenOpts.FPDenormalMode.str());
1842
350k
    if (CodeGenOpts.FP32DenormalMode != CodeGenOpts.FPDenormalMode) {
1843
16
      FuncAttrs.addAttribute(
1844
16
          "denormal-fp-math-f32",
1845
16
          CodeGenOpts.FP32DenormalMode.str());
1846
16
    }
1847
1848
350k
    if (LangOpts.getFPExceptionMode() == LangOptions::FPE_Ignore)
1849
349k
      FuncAttrs.addAttribute("no-trapping-math", "true");
1850
1851
    // TODO: Are these all needed?
1852
    // unsafe/inf/nan/nsz are handled by instruction-level FastMathFlags.
1853
350k
    if (LangOpts.NoHonorInfs)
1854
265
      FuncAttrs.addAttribute("no-infs-fp-math", "true");
1855
350k
    if (LangOpts.NoHonorNaNs)
1856
265
      FuncAttrs.addAttribute("no-nans-fp-math", "true");
1857
350k
    if (LangOpts.ApproxFunc)
1858
253
      FuncAttrs.addAttribute("approx-func-fp-math", "true");
1859
350k
    if (LangOpts.UnsafeFPMath)
1860
252
      FuncAttrs.addAttribute("unsafe-fp-math", "true");
1861
350k
    if (CodeGenOpts.SoftFloat)
1862
61
      FuncAttrs.addAttribute("use-soft-float", "true");
1863
350k
    FuncAttrs.addAttribute("stack-protector-buffer-size",
1864
350k
                           llvm::utostr(CodeGenOpts.SSPBufferSize));
1865
350k
    if (LangOpts.NoSignedZero)
1866
258
      FuncAttrs.addAttribute("no-signed-zeros-fp-math", "true");
1867
1868
    // TODO: Reciprocal estimate codegen options should apply to instructions?
1869
350k
    const std::vector<std::string> &Recips = CodeGenOpts.Reciprocals;
1870
350k
    if (!Recips.empty())
1871
1
      FuncAttrs.addAttribute("reciprocal-estimates",
1872
1
                             llvm::join(Recips, ","));
1873
1874
350k
    if (!CodeGenOpts.PreferVectorWidth.empty() &&
1875
350k
        
CodeGenOpts.PreferVectorWidth != "none"3
)
1876
2
      FuncAttrs.addAttribute("prefer-vector-width",
1877
2
                             CodeGenOpts.PreferVectorWidth);
1878
1879
350k
    if (CodeGenOpts.StackRealignment)
1880
1
      FuncAttrs.addAttribute("stackrealign");
1881
350k
    if (CodeGenOpts.Backchain)
1882
2
      FuncAttrs.addAttribute("backchain");
1883
350k
    if (CodeGenOpts.EnableSegmentedStacks)
1884
6
      FuncAttrs.addAttribute("split-stack");
1885
1886
350k
    if (CodeGenOpts.SpeculativeLoadHardening)
1887
9
      FuncAttrs.addAttribute(llvm::Attribute::SpeculativeLoadHardening);
1888
1889
    // Add zero-call-used-regs attribute.
1890
350k
    switch (CodeGenOpts.getZeroCallUsedRegs()) {
1891
350k
    case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::Skip:
1892
350k
      FuncAttrs.removeAttribute("zero-call-used-regs");
1893
350k
      break;
1894
10
    case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedGPRArg:
1895
10
      FuncAttrs.addAttribute("zero-call-used-regs", "used-gpr-arg");
1896
10
      break;
1897
10
    case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedGPR:
1898
10
      FuncAttrs.addAttribute("zero-call-used-regs", "used-gpr");
1899
10
      break;
1900
10
    case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedArg:
1901
10
      FuncAttrs.addAttribute("zero-call-used-regs", "used-arg");
1902
10
      break;
1903
10
    case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::Used:
1904
10
      FuncAttrs.addAttribute("zero-call-used-regs", "used");
1905
10
      break;
1906
10
    case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllGPRArg:
1907
10
      FuncAttrs.addAttribute("zero-call-used-regs", "all-gpr-arg");
1908
10
      break;
1909
10
    case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllGPR:
1910
10
      FuncAttrs.addAttribute("zero-call-used-regs", "all-gpr");
1911
10
      break;
1912
10
    case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllArg:
1913
10
      FuncAttrs.addAttribute("zero-call-used-regs", "all-arg");
1914
10
      break;
1915
10
    case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::All:
1916
10
      FuncAttrs.addAttribute("zero-call-used-regs", "all");
1917
10
      break;
1918
350k
    }
1919
350k
  }
1920
1921
717k
  if (getLangOpts().assumeFunctionsAreConvergent()) {
1922
    // Conservatively, mark all functions and calls in CUDA and OpenCL as
1923
    // convergent (meaning, they may call an intrinsically convergent op, such
1924
    // as __syncthreads() / barrier(), and so can't have certain optimizations
1925
    // applied around them).  LLVM will remove this attribute where it safely
1926
    // can.
1927
132k
    FuncAttrs.addAttribute(llvm::Attribute::Convergent);
1928
132k
  }
1929
1930
717k
  if (getLangOpts().CUDA && 
getLangOpts().CUDAIsDevice1.28k
) {
1931
    // Exceptions aren't supported in CUDA device code.
1932
465
    FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
1933
465
  }
1934
1935
717k
  for (StringRef Attr : CodeGenOpts.DefaultFunctionAttrs) {
1936
32
    StringRef Var, Value;
1937
32
    std::tie(Var, Value) = Attr.split('=');
1938
32
    FuncAttrs.addAttribute(Var, Value);
1939
32
  }
1940
717k
}
1941
1942
3
void CodeGenModule::addDefaultFunctionDefinitionAttributes(llvm::Function &F) {
1943
3
  llvm::AttrBuilder FuncAttrs(F.getContext());
1944
3
  getDefaultFunctionAttributes(F.getName(), F.hasOptNone(),
1945
3
                               /* AttrOnCallSite = */ false, FuncAttrs);
1946
  // TODO: call GetCPUAndFeaturesAttributes?
1947
3
  F.addFnAttrs(FuncAttrs);
1948
3
}
1949
1950
void CodeGenModule::addDefaultFunctionDefinitionAttributes(
1951
0
                                                   llvm::AttrBuilder &attrs) {
1952
0
  getDefaultFunctionAttributes(/*function name*/ "", /*optnone*/ false,
1953
0
                               /*for call*/ false, attrs);
1954
0
  GetCPUAndFeaturesAttributes(GlobalDecl(), attrs);
1955
0
}
1956
1957
static void addNoBuiltinAttributes(llvm::AttrBuilder &FuncAttrs,
1958
                                   const LangOptions &LangOpts,
1959
717k
                                   const NoBuiltinAttr *NBA = nullptr) {
1960
717k
  auto AddNoBuiltinAttr = [&FuncAttrs](StringRef BuiltinName) {
1961
2.40k
    SmallString<32> AttributeName;
1962
2.40k
    AttributeName += "no-builtin-";
1963
2.40k
    AttributeName += BuiltinName;
1964
2.40k
    FuncAttrs.addAttribute(AttributeName);
1965
2.40k
  };
1966
1967
  // First, handle the language options passed through -fno-builtin.
1968
717k
  if (LangOpts.NoBuiltin) {
1969
    // -fno-builtin disables them all.
1970
444k
    FuncAttrs.addAttribute("no-builtins");
1971
444k
    return;
1972
444k
  }
1973
1974
  // Then, add attributes for builtins specified through -fno-builtin-<name>.
1975
273k
  llvm::for_each(LangOpts.NoBuiltinFuncs, AddNoBuiltinAttr);
1976
1977
  // Now, let's check the __attribute__((no_builtin("...")) attribute added to
1978
  // the source.
1979
273k
  if (!NBA)
1980
273k
    return;
1981
1982
  // If there is a wildcard in the builtin names specified through the
1983
  // attribute, disable them all.
1984
11
  if (llvm::is_contained(NBA->builtinNames(), "*")) {
1985
1
    FuncAttrs.addAttribute("no-builtins");
1986
1
    return;
1987
1
  }
1988
1989
  // And last, add the rest of the builtin names.
1990
10
  llvm::for_each(NBA->builtinNames(), AddNoBuiltinAttr);
1991
10
}
1992
1993
static bool DetermineNoUndef(QualType QTy, CodeGenTypes &Types,
1994
                             const llvm::DataLayout &DL, const ABIArgInfo &AI,
1995
1.20M
                             bool CheckCoerce = true) {
1996
1.20M
  llvm::Type *Ty = Types.ConvertTypeForMem(QTy);
1997
1.20M
  if (AI.getKind() == ABIArgInfo::Indirect)
1998
4.18k
    return true;
1999
1.20M
  if (AI.getKind() == ABIArgInfo::Extend)
2000
43.0k
    return true;
2001
1.16M
  if (!DL.typeSizeEqualsStoreSize(Ty))
2002
    // TODO: This will result in a modest amount of values not marked noundef
2003
    // when they could be. We care about values that *invisibly* contain undef
2004
    // bits from the perspective of LLVM IR.
2005
43
    return false;
2006
1.16M
  if (CheckCoerce && 
AI.canHaveCoerceToType()1.06M
) {
2007
1.04M
    llvm::Type *CoerceTy = AI.getCoerceToType();
2008
1.04M
    if (llvm::TypeSize::isKnownGT(DL.getTypeSizeInBits(CoerceTy),
2009
1.04M
                                  DL.getTypeSizeInBits(Ty)))
2010
      // If we're coercing to a type with a greater size than the canonical one,
2011
      // we're introducing new undef bits.
2012
      // Coercing to a type of smaller or equal size is ok, as we know that
2013
      // there's no internal padding (typeSizeEqualsStoreSize).
2014
645
      return false;
2015
1.04M
  }
2016
1.16M
  if (QTy->isBitIntType())
2017
3
    return true;
2018
1.16M
  if (QTy->isReferenceType())
2019
184k
    return true;
2020
976k
  if (QTy->isNullPtrType())
2021
198
    return false;
2022
976k
  if (QTy->isMemberPointerType())
2023
    // TODO: Some member pointers are `noundef`, but it depends on the ABI. For
2024
    // now, never mark them.
2025
279
    return false;
2026
975k
  if (QTy->isScalarType()) {
2027
776k
    if (const ComplexType *Complex = dyn_cast<ComplexType>(QTy))
2028
1.09k
      return DetermineNoUndef(Complex->getElementType(), Types, DL, AI, false);
2029
775k
    return true;
2030
776k
  }
2031
199k
  if (const VectorType *Vector = dyn_cast<VectorType>(QTy))
2032
95.3k
    return DetermineNoUndef(Vector->getElementType(), Types, DL, AI, false);
2033
104k
  if (const MatrixType *Matrix = dyn_cast<MatrixType>(QTy))
2034
340
    return DetermineNoUndef(Matrix->getElementType(), Types, DL, AI, false);
2035
103k
  if (const ArrayType *Array = dyn_cast<ArrayType>(QTy))
2036
0
    return DetermineNoUndef(Array->getElementType(), Types, DL, AI, false);
2037
2038
  // TODO: Some structs may be `noundef`, in specific situations.
2039
103k
  return false;
2040
103k
}
2041
2042
/// Construct the IR attribute list of a function or call.
2043
///
2044
/// When adding an attribute, please consider where it should be handled:
2045
///
2046
///   - getDefaultFunctionAttributes is for attributes that are essentially
2047
///     part of the global target configuration (but perhaps can be
2048
///     overridden on a per-function basis).  Adding attributes there
2049
///     will cause them to also be set in frontends that build on Clang's
2050
///     target-configuration logic, as well as for code defined in library
2051
///     modules such as CUDA's libdevice.
2052
///
2053
///   - ConstructAttributeList builds on top of getDefaultFunctionAttributes
2054
///     and adds declaration-specific, convention-specific, and
2055
///     frontend-specific logic.  The last is of particular importance:
2056
///     attributes that restrict how the frontend generates code must be
2057
///     added here rather than getDefaultFunctionAttributes.
2058
///
2059
void CodeGenModule::ConstructAttributeList(StringRef Name,
2060
                                           const CGFunctionInfo &FI,
2061
                                           CGCalleeInfo CalleeInfo,
2062
                                           llvm::AttributeList &AttrList,
2063
                                           unsigned &CallingConv,
2064
717k
                                           bool AttrOnCallSite, bool IsThunk) {
2065
717k
  llvm::AttrBuilder FuncAttrs(getLLVMContext());
2066
717k
  llvm::AttrBuilder RetAttrs(getLLVMContext());
2067
2068
  // Collect function IR attributes from the CC lowering.
2069
  // We'll collect the paramete and result attributes later.
2070
717k
  CallingConv = FI.getEffectiveCallingConvention();
2071
717k
  if (FI.isNoReturn())
2072
1.28k
    FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
2073
717k
  if (FI.isCmseNSCall())
2074
62
    FuncAttrs.addAttribute("cmse_nonsecure_call");
2075
2076
  // Collect function IR attributes from the callee prototype if we have one.
2077
717k
  AddAttributesFromFunctionProtoType(getContext(), FuncAttrs,
2078
717k
                                     CalleeInfo.getCalleeFunctionProtoType());
2079
2080
717k
  const Decl *TargetDecl = CalleeInfo.getCalleeDecl().getDecl();
2081
2082
  // Attach assumption attributes to the declaration. If this is a call
2083
  // site, attach assumptions from the caller to the call as well.
2084
717k
  AddAttributesFromAssumes(FuncAttrs, TargetDecl);
2085
2086
717k
  bool HasOptnone = false;
2087
  // The NoBuiltinAttr attached to the target FunctionDecl.
2088
717k
  const NoBuiltinAttr *NBA = nullptr;
2089
2090
  // Collect function IR attributes based on declaration-specific
2091
  // information.
2092
  // FIXME: handle sseregparm someday...
2093
717k
  if (TargetDecl) {
2094
685k
    if (TargetDecl->hasAttr<ReturnsTwiceAttr>())
2095
43
      FuncAttrs.addAttribute(llvm::Attribute::ReturnsTwice);
2096
685k
    if (TargetDecl->hasAttr<NoThrowAttr>())
2097
5.55k
      FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
2098
685k
    if (TargetDecl->hasAttr<NoReturnAttr>())
2099
0
      FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
2100
685k
    if (TargetDecl->hasAttr<ColdAttr>())
2101
387
      FuncAttrs.addAttribute(llvm::Attribute::Cold);
2102
685k
    if (TargetDecl->hasAttr<HotAttr>())
2103
3
      FuncAttrs.addAttribute(llvm::Attribute::Hot);
2104
685k
    if (TargetDecl->hasAttr<NoDuplicateAttr>())
2105
4
      FuncAttrs.addAttribute(llvm::Attribute::NoDuplicate);
2106
685k
    if (TargetDecl->hasAttr<ConvergentAttr>())
2107
3
      FuncAttrs.addAttribute(llvm::Attribute::Convergent);
2108
2109
685k
    if (const FunctionDecl *Fn = dyn_cast<FunctionDecl>(TargetDecl)) {
2110
651k
      AddAttributesFromFunctionProtoType(
2111
651k
          getContext(), FuncAttrs, Fn->getType()->getAs<FunctionProtoType>());
2112
651k
      if (AttrOnCallSite && 
Fn->isReplaceableGlobalAllocationFunction()344k
) {
2113
        // A sane operator new returns a non-aliasing pointer.
2114
3.94k
        auto Kind = Fn->getDeclName().getCXXOverloadedOperator();
2115
3.94k
        if (getCodeGenOpts().AssumeSaneOperatorNew &&
2116
3.94k
            
(3.93k
Kind == OO_New3.93k
||
Kind == OO_Array_New2.62k
))
2117
1.60k
          RetAttrs.addAttribute(llvm::Attribute::NoAlias);
2118
3.94k
      }
2119
651k
      const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Fn);
2120
651k
      const bool IsVirtualCall = MD && 
MD->isVirtual()288k
;
2121
      // Don't use [[noreturn]], _Noreturn or [[no_builtin]] for a call to a
2122
      // virtual function. These attributes are not inherited by overloads.
2123
651k
      if (!(AttrOnCallSite && 
IsVirtualCall344k
)) {
2124
646k
        if (Fn->isNoReturn())
2125
3.61k
          FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
2126
646k
        NBA = Fn->getAttr<NoBuiltinAttr>();
2127
646k
      }
2128
      // Only place nomerge attribute on call sites, never functions. This
2129
      // allows it to work on indirect virtual function calls.
2130
651k
      if (AttrOnCallSite && 
TargetDecl->hasAttr<NoMergeAttr>()344k
)
2131
12
        FuncAttrs.addAttribute(llvm::Attribute::NoMerge);
2132
651k
    }
2133
2134
    // 'const', 'pure' and 'noalias' attributed functions are also nounwind.
2135
685k
    if (TargetDecl->hasAttr<ConstAttr>()) {
2136
1.79k
      FuncAttrs.addAttribute(llvm::Attribute::ReadNone);
2137
1.79k
      FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
2138
      // gcc specifies that 'const' functions have greater restrictions than
2139
      // 'pure' functions, so they also cannot have infinite loops.
2140
1.79k
      FuncAttrs.addAttribute(llvm::Attribute::WillReturn);
2141
683k
    } else if (TargetDecl->hasAttr<PureAttr>()) {
2142
211
      FuncAttrs.addAttribute(llvm::Attribute::ReadOnly);
2143
211
      FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
2144
      // gcc specifies that 'pure' functions cannot have infinite loops.
2145
211
      FuncAttrs.addAttribute(llvm::Attribute::WillReturn);
2146
683k
    } else if (TargetDecl->hasAttr<NoAliasAttr>()) {
2147
2
      FuncAttrs.addAttribute(llvm::Attribute::ArgMemOnly);
2148
2
      FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
2149
2
    }
2150
685k
    if (TargetDecl->hasAttr<RestrictAttr>())
2151
56
      RetAttrs.addAttribute(llvm::Attribute::NoAlias);
2152
685k
    if (TargetDecl->hasAttr<ReturnsNonNullAttr>() &&
2153
685k
        
!CodeGenOpts.NullPointerIsValid2.49k
)
2154
2.49k
      RetAttrs.addAttribute(llvm::Attribute::NonNull);
2155
685k
    if (TargetDecl->hasAttr<AnyX86NoCallerSavedRegistersAttr>())
2156
14
      FuncAttrs.addAttribute("no_caller_saved_registers");
2157
685k
    if (TargetDecl->hasAttr<AnyX86NoCfCheckAttr>())
2158
2
      FuncAttrs.addAttribute(llvm::Attribute::NoCfCheck);
2159
685k
    if (TargetDecl->hasAttr<LeafAttr>())
2160
1
      FuncAttrs.addAttribute(llvm::Attribute::NoCallback);
2161
2162
685k
    HasOptnone = TargetDecl->hasAttr<OptimizeNoneAttr>();
2163
685k
    if (auto *AllocSize = TargetDecl->getAttr<AllocSizeAttr>()) {
2164
2.85k
      Optional<unsigned> NumElemsParam;
2165
2.85k
      if (AllocSize->getNumElemsParam().isValid())
2166
144
        NumElemsParam = AllocSize->getNumElemsParam().getLLVMIndex();
2167
2.85k
      FuncAttrs.addAllocSizeAttr(AllocSize->getElemSizeParam().getLLVMIndex(),
2168
2.85k
                                 NumElemsParam);
2169
2.85k
    }
2170
2171
685k
    if (TargetDecl->hasAttr<OpenCLKernelAttr>()) {
2172
383
      if (getLangOpts().OpenCLVersion <= 120) {
2173
        // OpenCL v1.2 Work groups are always uniform
2174
277
        FuncAttrs.addAttribute("uniform-work-group-size", "true");
2175
277
      } else {
2176
        // OpenCL v2.0 Work groups may be whether uniform or not.
2177
        // '-cl-uniform-work-group-size' compile option gets a hint
2178
        // to the compiler that the global work-size be a multiple of
2179
        // the work-group size specified to clEnqueueNDRangeKernel
2180
        // (i.e. work groups are uniform).
2181
106
        FuncAttrs.addAttribute("uniform-work-group-size",
2182
106
                               llvm::toStringRef(CodeGenOpts.UniformWGSize));
2183
106
      }
2184
383
    }
2185
685k
  }
2186
2187
  // Attach "no-builtins" attributes to:
2188
  // * call sites: both `nobuiltin` and "no-builtins" or "no-builtin-<name>".
2189
  // * definitions: "no-builtins" or "no-builtin-<name>" only.
2190
  // The attributes can come from:
2191
  // * LangOpts: -ffreestanding, -fno-builtin, -fno-builtin-<name>
2192
  // * FunctionDecl attributes: __attribute__((no_builtin(...)))
2193
717k
  addNoBuiltinAttributes(FuncAttrs, getLangOpts(), NBA);
2194
2195
  // Collect function IR attributes based on global settiings.
2196
717k
  getDefaultFunctionAttributes(Name, HasOptnone, AttrOnCallSite, FuncAttrs);
2197
2198
  // Override some default IR attributes based on declaration-specific
2199
  // information.
2200
717k
  if (TargetDecl) {
2201
685k
    if (TargetDecl->hasAttr<NoSpeculativeLoadHardeningAttr>())
2202
13
      FuncAttrs.removeAttribute(llvm::Attribute::SpeculativeLoadHardening);
2203
685k
    if (TargetDecl->hasAttr<SpeculativeLoadHardeningAttr>())
2204
13
      FuncAttrs.addAttribute(llvm::Attribute::SpeculativeLoadHardening);
2205
685k
    if (TargetDecl->hasAttr<NoSplitStackAttr>())
2206
8
      FuncAttrs.removeAttribute("split-stack");
2207
685k
    if (TargetDecl->hasAttr<ZeroCallUsedRegsAttr>()) {
2208
      // A function "__attribute__((...))" overrides the command-line flag.
2209
81
      auto Kind =
2210
81
          TargetDecl->getAttr<ZeroCallUsedRegsAttr>()->getZeroCallUsedRegs();
2211
81
      FuncAttrs.removeAttribute("zero-call-used-regs");
2212
81
      FuncAttrs.addAttribute(
2213
81
          "zero-call-used-regs",
2214
81
          ZeroCallUsedRegsAttr::ConvertZeroCallUsedRegsKindToStr(Kind));
2215
81
    }
2216
2217
    // Add NonLazyBind attribute to function declarations when -fno-plt
2218
    // is used.
2219
    // FIXME: what if we just haven't processed the function definition
2220
    // yet, or if it's an external definition like C99 inline?
2221
685k
    if (CodeGenOpts.NoPLT) {
2222
45
      if (auto *Fn = dyn_cast<FunctionDecl>(TargetDecl)) {
2223
45
        if (!Fn->isDefined() && 
!AttrOnCallSite22
) {
2224
12
          FuncAttrs.addAttribute(llvm::Attribute::NonLazyBind);
2225
12
        }
2226
45
      }
2227
45
    }
2228
685k
  }
2229
2230
  // Add "sample-profile-suffix-elision-policy" attribute for internal linkage
2231
  // functions with -funique-internal-linkage-names.
2232
717k
  if (TargetDecl && 
CodeGenOpts.UniqueInternalLinkageNames685k
) {
2233
47
    if (isa<FunctionDecl>(TargetDecl)) {
2234
47
      if (this->getFunctionLinkage(CalleeInfo.getCalleeDecl()) ==
2235
47
          llvm::GlobalValue::InternalLinkage)
2236
34
        FuncAttrs.addAttribute("sample-profile-suffix-elision-policy",
2237
34
                               "selected");
2238
47
    }
2239
47
  }
2240
2241
  // Collect non-call-site function IR attributes from declaration-specific
2242
  // information.
2243
717k
  if (!AttrOnCallSite) {
2244
350k
    if (TargetDecl && 
TargetDecl->hasAttr<CmseNSEntryAttr>()335k
)
2245
94
      FuncAttrs.addAttribute("cmse_nonsecure_entry");
2246
2247
    // Whether tail calls are enabled.
2248
350k
    auto shouldDisableTailCalls = [&] {
2249
      // Should this be honored in getDefaultFunctionAttributes?
2250
350k
      if (CodeGenOpts.DisableTailCalls)
2251
3
        return true;
2252
2253
350k
      if (!TargetDecl)
2254
14.9k
        return false;
2255
2256
335k
      if (TargetDecl->hasAttr<DisableTailCallsAttr>() ||
2257
335k
          
TargetDecl->hasAttr<AnyX86InterruptAttr>()335k
)
2258
147
        return true;
2259
2260
335k
      if (CodeGenOpts.NoEscapingBlockTailCalls) {
2261
17
        if (const auto *BD = dyn_cast<BlockDecl>(TargetDecl))
2262
8
          if (!BD->doesNotEscape())
2263
4
            return true;
2264
17
      }
2265
2266
335k
      return false;
2267
335k
    };
2268
350k
    if (shouldDisableTailCalls())
2269
154
      FuncAttrs.addAttribute("disable-tail-calls", "true");
2270
2271
    // CPU/feature overrides.  addDefaultFunctionDefinitionAttributes
2272
    // handles these separately to set them based on the global defaults.
2273
350k
    GetCPUAndFeaturesAttributes(CalleeInfo.getCalleeDecl(), FuncAttrs);
2274
350k
  }
2275
2276
  // Collect attributes from arguments and return values.
2277
717k
  ClangToLLVMArgMapping IRFunctionArgs(getContext(), FI);
2278
2279
717k
  QualType RetTy = FI.getReturnType();
2280
717k
  const ABIArgInfo &RetAI = FI.getReturnInfo();
2281
717k
  const llvm::DataLayout &DL = getDataLayout();
2282
2283
  // C++ explicitly makes returning undefined values UB. C's rule only applies
2284
  // to used values, so we never mark them noundef for now.
2285
717k
  bool HasStrictReturn = getLangOpts().CPlusPlus;
2286
717k
  if (TargetDecl && 
HasStrictReturn685k
) {
2287
554k
    if (const FunctionDecl *FDecl = dyn_cast<FunctionDecl>(TargetDecl))
2288
523k
      HasStrictReturn &= !FDecl->isExternC();
2289
30.6k
    else if (const VarDecl *VDecl = dyn_cast<VarDecl>(TargetDecl))
2290
      // Function pointer
2291
4.98k
      HasStrictReturn &= !VDecl->isExternC();
2292
554k
  }
2293
2294
  // We don't want to be too aggressive with the return checking, unless
2295
  // it's explicit in the code opts or we're using an appropriate sanitizer.
2296
  // Try to respect what the programmer intended.
2297
717k
  HasStrictReturn &= getCodeGenOpts().StrictReturn ||
2298
717k
                     
!MayDropFunctionReturn(getContext(), RetTy)27
||
2299
717k
                     
getLangOpts().Sanitize.has(SanitizerKind::Memory)21
||
2300
717k
                     
getLangOpts().Sanitize.has(SanitizerKind::Return)21
;
2301
2302
  // Determine if the return type could be partially undef
2303
717k
  if (CodeGenOpts.EnableNoundefAttrs && 
HasStrictReturn576k
) {
2304
415k
    if (!RetTy->isVoidType() && 
RetAI.getKind() != ABIArgInfo::Indirect171k
&&
2305
415k
        
DetermineNoUndef(RetTy, getTypes(), DL, RetAI)169k
)
2306
149k
      RetAttrs.addAttribute(llvm::Attribute::NoUndef);
2307
415k
  }
2308
2309
717k
  switch (RetAI.getKind()) {
2310
21.6k
  case ABIArgInfo::Extend:
2311
21.6k
    if (RetAI.isSignExt())
2312
8.18k
      RetAttrs.addAttribute(llvm::Attribute::SExt);
2313
13.5k
    else
2314
13.5k
      RetAttrs.addAttribute(llvm::Attribute::ZExt);
2315
21.6k
    LLVM_FALLTHROUGH;
2316
398k
  case ABIArgInfo::Direct:
2317
398k
    if (RetAI.getInReg())
2318
16
      RetAttrs.addAttribute(llvm::Attribute::InReg);
2319
398k
    break;
2320
311k
  case ABIArgInfo::Ignore:
2321
311k
    break;
2322
2323
13
  case ABIArgInfo::InAlloca:
2324
6.79k
  case ABIArgInfo::Indirect: {
2325
    // inalloca and sret disable readnone and readonly
2326
6.79k
    FuncAttrs.removeAttribute(llvm::Attribute::ReadOnly)
2327
6.79k
      .removeAttribute(llvm::Attribute::ReadNone);
2328
6.79k
    break;
2329
13
  }
2330
2331
983
  case ABIArgInfo::CoerceAndExpand:
2332
983
    break;
2333
2334
0
  case ABIArgInfo::Expand:
2335
0
  case ABIArgInfo::IndirectAliased:
2336
0
    llvm_unreachable("Invalid ABI kind for return argument");
2337
717k
  }
2338
2339
717k
  if (!IsThunk) {
2340
    // FIXME: fix this properly, https://reviews.llvm.org/D100388
2341
716k
    if (const auto *RefTy = RetTy->getAs<ReferenceType>()) {
2342
67.0k
      QualType PTy = RefTy->getPointeeType();
2343
67.0k
      if (!PTy->isIncompleteType() && 
PTy->isConstantSizeType()67.0k
)
2344
67.0k
        RetAttrs.addDereferenceableAttr(
2345
67.0k
            getMinimumObjectSize(PTy).getQuantity());
2346
67.0k
      if (getContext().getTargetAddressSpace(PTy) == 0 &&
2347
67.0k
          
!CodeGenOpts.NullPointerIsValid67.0k
)
2348
67.0k
        RetAttrs.addAttribute(llvm::Attribute::NonNull);
2349
67.0k
      if (PTy->isObjectType()) {
2350
66.0k
        llvm::Align Alignment =
2351
66.0k
            getNaturalPointeeTypeAlignment(RetTy).getAsAlign();
2352
66.0k
        RetAttrs.addAlignmentAttr(Alignment);
2353
66.0k
      }
2354
67.0k
    }
2355
716k
  }
2356
2357
717k
  bool hasUsedSRet = false;
2358
717k
  SmallVector<llvm::AttributeSet, 4> ArgAttrs(IRFunctionArgs.totalIRArgs());
2359
2360
  // Attach attributes to sret.
2361
717k
  if (IRFunctionArgs.hasSRetArg()) {
2362
6.77k
    llvm::AttrBuilder SRETAttrs(getLLVMContext());
2363
6.77k
    SRETAttrs.addStructRetAttr(getTypes().ConvertTypeForMem(RetTy));
2364
6.77k
    hasUsedSRet = true;
2365
6.77k
    if (RetAI.getInReg())
2366
64
      SRETAttrs.addAttribute(llvm::Attribute::InReg);
2367
6.77k
    SRETAttrs.addAlignmentAttr(RetAI.getIndirectAlign().getQuantity());
2368
6.77k
    ArgAttrs[IRFunctionArgs.getSRetArgNo()] =
2369
6.77k
        llvm::AttributeSet::get(getLLVMContext(), SRETAttrs);
2370
6.77k
  }
2371
2372
  // Attach attributes to inalloca argument.
2373
717k
  if (IRFunctionArgs.hasInallocaArg()) {
2374
162
    llvm::AttrBuilder Attrs(getLLVMContext());
2375
162
    Attrs.addInAllocaAttr(FI.getArgStruct());
2376
162
    ArgAttrs[IRFunctionArgs.getInallocaArgNo()] =
2377
162
        llvm::AttributeSet::get(getLLVMContext(), Attrs);
2378
162
  }
2379
2380
  // Apply `nonnull`, `dereferencable(N)` and `align N` to the `this` argument,
2381
  // unless this is a thunk function.
2382
  // FIXME: fix this properly, https://reviews.llvm.org/D100388
2383
717k
  if (FI.isInstanceMethod() && 
!IRFunctionArgs.hasInallocaArg()272k
&&
2384
717k
      
!FI.arg_begin()->type->isVoidPointerType()272k
&&
!IsThunk271k
) {
2385
271k
    auto IRArgs = IRFunctionArgs.getIRArgs(0);
2386
2387
271k
    assert(IRArgs.second == 1 && "Expected only a single `this` pointer.");
2388
2389
0
    llvm::AttrBuilder Attrs(getLLVMContext());
2390
2391
271k
    QualType ThisTy =
2392
271k
        FI.arg_begin()->type.castAs<PointerType>()->getPointeeType();
2393
2394
271k
    if (!CodeGenOpts.NullPointerIsValid &&
2395
271k
        
getContext().getTargetAddressSpace(FI.arg_begin()->type) == 0270k
) {
2396
270k
      Attrs.addAttribute(llvm::Attribute::NonNull);
2397
270k
      Attrs.addDereferenceableAttr(getMinimumObjectSize(ThisTy).getQuantity());
2398
270k
    } else {
2399
      // FIXME dereferenceable should be correct here, regardless of
2400
      // NullPointerIsValid. However, dereferenceable currently does not always
2401
      // respect NullPointerIsValid and may imply nonnull and break the program.
2402
      // See https://reviews.llvm.org/D66618 for discussions.
2403
322
      Attrs.addDereferenceableOrNullAttr(
2404
322
          getMinimumObjectSize(
2405
322
              FI.arg_begin()->type.castAs<PointerType>()->getPointeeType())
2406
322
              .getQuantity());
2407
322
    }
2408
2409
271k
    llvm::Align Alignment =
2410
271k
        getNaturalTypeAlignment(ThisTy, /*BaseInfo=*/nullptr,
2411
271k
                                /*TBAAInfo=*/nullptr, /*forPointeeType=*/true)
2412
271k
            .getAsAlign();
2413
271k
    Attrs.addAlignmentAttr(Alignment);
2414
2415
271k
    ArgAttrs[IRArgs.first] = llvm::AttributeSet::get(getLLVMContext(), Attrs);
2416
271k
  }
2417
2418
0
  unsigned ArgNo = 0;
2419
717k
  for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(),
2420
717k
                                          E = FI.arg_end();
2421
1.90M
       I != E; 
++I, ++ArgNo1.18M
) {
2422
1.18M
    QualType ParamType = I->type;
2423
1.18M
    const ABIArgInfo &AI = I->info;
2424
1.18M
    llvm::AttrBuilder Attrs(getLLVMContext());
2425
2426
    // Add attribute for padding argument, if necessary.
2427
1.18M
    if (IRFunctionArgs.hasPaddingArg(ArgNo)) {
2428
47
      if (AI.getPaddingInReg()) {
2429
5
        ArgAttrs[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
2430
5
            llvm::AttributeSet::get(
2431
5
                getLLVMContext(),
2432
5
                llvm::AttrBuilder(getLLVMContext()).addAttribute(llvm::Attribute::InReg));
2433
5
      }
2434
47
    }
2435
2436
    // Decide whether the argument we're handling could be partially undef
2437
1.18M
    if (CodeGenOpts.EnableNoundefAttrs &&
2438
1.18M
        
DetermineNoUndef(ParamType, getTypes(), DL, AI)942k
) {
2439
857k
      Attrs.addAttribute(llvm::Attribute::NoUndef);
2440
857k
    }
2441
2442
    // 'restrict' -> 'noalias' is done in EmitFunctionProlog when we
2443
    // have the corresponding parameter variable.  It doesn't make
2444
    // sense to do it here because parameters are so messed up.
2445
1.18M
    switch (AI.getKind()) {
2446
27.5k
    case ABIArgInfo::Extend:
2447
27.5k
      if (AI.isSignExt())
2448
11.5k
        Attrs.addAttribute(llvm::Attribute::SExt);
2449
16.0k
      else
2450
16.0k
        Attrs.addAttribute(llvm::Attribute::ZExt);
2451
27.5k
      LLVM_FALLTHROUGH;
2452
1.16M
    case ABIArgInfo::Direct:
2453
1.16M
      if (ArgNo == 0 && 
FI.isChainCall()644k
)
2454
8
        Attrs.addAttribute(llvm::Attribute::Nest);
2455
1.16M
      else if (AI.getInReg())
2456
580
        Attrs.addAttribute(llvm::Attribute::InReg);
2457
1.16M
      Attrs.addStackAlignmentAttr(llvm::MaybeAlign(AI.getDirectAlign()));
2458
1.16M
      break;
2459
2460
5.08k
    case ABIArgInfo::Indirect: {
2461
5.08k
      if (AI.getInReg())
2462
24
        Attrs.addAttribute(llvm::Attribute::InReg);
2463
2464
5.08k
      if (AI.getIndirectByVal())
2465
1.89k
        Attrs.addByValAttr(getTypes().ConvertTypeForMem(ParamType));
2466
2467
5.08k
      auto *Decl = ParamType->getAsRecordDecl();
2468
5.08k
      if (CodeGenOpts.PassByValueIsNoAlias && 
Decl7
&&
2469
5.08k
          
Decl->getArgPassingRestrictions() == RecordDecl::APK_CanPassInRegs7
)
2470
        // When calling the function, the pointer passed in will be the only
2471
        // reference to the underlying object. Mark it accordingly.
2472
5
        Attrs.addAttribute(llvm::Attribute::NoAlias);
2473
2474
      // TODO: We could add the byref attribute if not byval, but it would
2475
      // require updating many testcases.
2476
2477
5.08k
      CharUnits Align = AI.getIndirectAlign();
2478
2479
      // In a byval argument, it is important that the required
2480
      // alignment of the type is honored, as LLVM might be creating a
2481
      // *new* stack object, and needs to know what alignment to give
2482
      // it. (Sometimes it can deduce a sensible alignment on its own,
2483
      // but not if clang decides it must emit a packed struct, or the
2484
      // user specifies increased alignment requirements.)
2485
      //
2486
      // This is different from indirect *not* byval, where the object
2487
      // exists already, and the align attribute is purely
2488
      // informative.
2489
5.08k
      assert(!Align.isZero());
2490
2491
      // For now, only add this when we have a byval argument.
2492
      // TODO: be less lazy about updating test cases.
2493
5.08k
      if (AI.getIndirectByVal())
2494
1.89k
        Attrs.addAlignmentAttr(Align.getQuantity());
2495
2496
      // byval disables readnone and readonly.
2497
5.08k
      FuncAttrs.removeAttribute(llvm::Attribute::ReadOnly)
2498
5.08k
        .removeAttribute(llvm::Attribute::ReadNone);
2499
2500
5.08k
      break;
2501
27.5k
    }
2502
14
    case ABIArgInfo::IndirectAliased: {
2503
14
      CharUnits Align = AI.getIndirectAlign();
2504
14
      Attrs.addByRefAttr(getTypes().ConvertTypeForMem(ParamType));
2505
14
      Attrs.addAlignmentAttr(Align.getQuantity());
2506
14
      break;
2507
27.5k
    }
2508
13.9k
    case ABIArgInfo::Ignore:
2509
14.1k
    case ABIArgInfo::Expand:
2510
15.1k
    case ABIArgInfo::CoerceAndExpand:
2511
15.1k
      break;
2512
2513
321
    case ABIArgInfo::InAlloca:
2514
      // inalloca disables readnone and readonly.
2515
321
      FuncAttrs.removeAttribute(llvm::Attribute::ReadOnly)
2516
321
          .removeAttribute(llvm::Attribute::ReadNone);
2517
321
      continue;
2518
1.18M
    }
2519
2520
1.18M
    if (const auto *RefTy = ParamType->getAs<ReferenceType>()) {
2521
136k
      QualType PTy = RefTy->getPointeeType();
2522
136k
      if (!PTy->isIncompleteType() && 
PTy->isConstantSizeType()135k
)
2523
135k
        Attrs.addDereferenceableAttr(
2524
135k
            getMinimumObjectSize(PTy).getQuantity());
2525
136k
      if (getContext().getTargetAddressSpace(PTy) == 0 &&
2526
136k
          
!CodeGenOpts.NullPointerIsValid135k
)
2527
135k
        Attrs.addAttribute(llvm::Attribute::NonNull);
2528
136k
      if (PTy->isObjectType()) {
2529
132k
        llvm::Align Alignment =
2530
132k
            getNaturalPointeeTypeAlignment(ParamType).getAsAlign();
2531
132k
        Attrs.addAlignmentAttr(Alignment);
2532
132k
      }
2533
136k
    }
2534
2535
    // From OpenCL spec v3.0.10 section 6.3.5 Alignment of Types:
2536
    // > For arguments to a __kernel function declared to be a pointer to a
2537
    // > data type, the OpenCL compiler can assume that the pointee is always
2538
    // > appropriately aligned as required by the data type.
2539
1.18M
    if (TargetDecl && 
TargetDecl->hasAttr<OpenCLKernelAttr>()1.11M
&&
2540
1.18M
        
ParamType->isPointerType()312
) {
2541
166
      QualType PTy = ParamType->getPointeeType();
2542
166
      if (!PTy->isIncompleteType() && 
PTy->isConstantSizeType()165
) {
2543
165
        llvm::Align Alignment =
2544
165
            getNaturalPointeeTypeAlignment(ParamType).getAsAlign();
2545
165
        Attrs.addAlignmentAttr(Alignment);
2546
165
      }
2547
166
    }
2548
2549
1.18M
    switch (FI.getExtParameterInfo(ArgNo).getABI()) {
2550
1.18M
    case ParameterABI::Ordinary:
2551
1.18M
      break;
2552
2553
48
    case ParameterABI::SwiftIndirectResult: {
2554
      // Add 'sret' if we haven't already used it for something, but
2555
      // only if the result is void.
2556
48
      if (!hasUsedSRet && 
RetTy->isVoidType()24
) {
2557
8
        Attrs.addStructRetAttr(getTypes().ConvertTypeForMem(ParamType));
2558
8
        hasUsedSRet = true;
2559
8
      }
2560
2561
      // Add 'noalias' in either case.
2562
48
      Attrs.addAttribute(llvm::Attribute::NoAlias);
2563
2564
      // Add 'dereferenceable' and 'alignment'.
2565
48
      auto PTy = ParamType->getPointeeType();
2566
48
      if (!PTy->isIncompleteType() && PTy->isConstantSizeType()) {
2567
48
        auto info = getContext().getTypeInfoInChars(PTy);
2568
48
        Attrs.addDereferenceableAttr(info.Width.getQuantity());
2569
48
        Attrs.addAlignmentAttr(info.Align.getAsAlign());
2570
48
      }
2571
48
      break;
2572
0
    }
2573
2574
24
    case ParameterABI::SwiftErrorResult:
2575
24
      Attrs.addAttribute(llvm::Attribute::SwiftError);
2576
24
      break;
2577
2578
40
    case ParameterABI::SwiftContext:
2579
40
      Attrs.addAttribute(llvm::Attribute::SwiftSelf);
2580
40
      break;
2581
2582
325
    case ParameterABI::SwiftAsyncContext:
2583
325
      Attrs.addAttribute(llvm::Attribute::SwiftAsync);
2584
325
      break;
2585
1.18M
    }
2586
2587
1.18M
    if (FI.getExtParameterInfo(ArgNo).isNoEscape())
2588
62
      Attrs.addAttribute(llvm::Attribute::NoCapture);
2589
2590
1.18M
    if (Attrs.hasAttributes()) {
2591
869k
      unsigned FirstIRArg, NumIRArgs;
2592
869k
      std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
2593
1.74M
      for (unsigned i = 0; i < NumIRArgs; 
i++870k
)
2594
870k
        ArgAttrs[FirstIRArg + i] = ArgAttrs[FirstIRArg + i].addAttributes(
2595
870k
            getLLVMContext(), llvm::AttributeSet::get(getLLVMContext(), Attrs));
2596
869k
    }
2597
1.18M
  }
2598
717k
  assert(ArgNo == FI.arg_size());
2599
2600
0
  AttrList = llvm::AttributeList::get(
2601
717k
      getLLVMContext(), llvm::AttributeSet::get(getLLVMContext(), FuncAttrs),
2602
717k
      llvm::AttributeSet::get(getLLVMContext(), RetAttrs), ArgAttrs);
2603
717k
}
2604
2605
/// An argument came in as a promoted argument; demote it back to its
2606
/// declared type.
2607
static llvm::Value *emitArgumentDemotion(CodeGenFunction &CGF,
2608
                                         const VarDecl *var,
2609
14
                                         llvm::Value *value) {
2610
14
  llvm::Type *varType = CGF.ConvertType(var->getType());
2611
2612
  // This can happen with promotions that actually don't change the
2613
  // underlying type, like the enum promotions.
2614
14
  if (value->getType() == varType) 
return value0
;
2615
2616
14
  assert((varType->isIntegerTy() || varType->isFloatingPointTy())
2617
14
         && "unexpected promotion type");
2618
2619
14
  if (isa<llvm::IntegerType>(varType))
2620
8
    return CGF.Builder.CreateTrunc(value, varType, "arg.unpromote");
2621
2622
6
  return CGF.Builder.CreateFPCast(value, varType, "arg.unpromote");
2623
14
}
2624
2625
/// Returns the attribute (either parameter attribute, or function
2626
/// attribute), which declares argument ArgNo to be non-null.
2627
static const NonNullAttr *getNonNullAttr(const Decl *FD, const ParmVarDecl *PVD,
2628
114k
                                         QualType ArgType, unsigned ArgNo) {
2629
  // FIXME: __attribute__((nonnull)) can also be applied to:
2630
  //   - references to pointers, where the pointee is known to be
2631
  //     nonnull (apparently a Clang extension)
2632
  //   - transparent unions containing pointers
2633
  // In the former case, LLVM IR cannot represent the constraint. In
2634
  // the latter case, we have no guarantee that the transparent union
2635
  // is in fact passed as a pointer.
2636
114k
  if (!ArgType->isAnyPointerType() && 
!ArgType->isBlockPointerType()47.7k
)
2637
47.6k
    return nullptr;
2638
  // First, check attribute on parameter itself.
2639
66.4k
  if (PVD) {
2640
66.4k
    if (auto ParmNNAttr = PVD->getAttr<NonNullAttr>())
2641
10
      return ParmNNAttr;
2642
66.4k
  }
2643
  // Check function attributes.
2644
66.4k
  if (!FD)
2645
5
    return nullptr;
2646
66.4k
  for (const auto *NNAttr : FD->specific_attrs<NonNullAttr>()) {
2647
32
    if (NNAttr->isNonNull(ArgNo))
2648
28
      return NNAttr;
2649
32
  }
2650
66.3k
  return nullptr;
2651
66.4k
}
2652
2653
namespace {
2654
  struct CopyBackSwiftError final : EHScopeStack::Cleanup {
2655
    Address Temp;
2656
    Address Arg;
2657
16
    CopyBackSwiftError(Address temp, Address arg) : Temp(temp), Arg(arg) {}
2658
16
    void Emit(CodeGenFunction &CGF, Flags flags) override {
2659
16
      llvm::Value *errorValue = CGF.Builder.CreateLoad(Temp);
2660
16
      CGF.Builder.CreateStore(errorValue, Arg);
2661
16
    }
2662
  };
2663
}
2664
2665
void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
2666
                                         llvm::Function *Fn,
2667
314k
                                         const FunctionArgList &Args) {
2668
314k
  if (CurCodeDecl && 
CurCodeDecl->hasAttr<NakedAttr>()305k
)
2669
    // Naked functions don't have prologues.
2670
10
    return;
2671
2672
  // If this is an implicit-return-zero function, go ahead and
2673
  // initialize the return value.  TODO: it might be nice to have
2674
  // a more general mechanism for this that didn't require synthesized
2675
  // return statements.
2676
314k
  if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(CurCodeDecl)) {
2677
270k
    if (FD->hasImplicitReturnZero()) {
2678
5.66k
      QualType RetTy = FD->getReturnType().getUnqualifiedType();
2679
5.66k
      llvm::Type* LLVMTy = CGM.getTypes().ConvertType(RetTy);
2680
5.66k
      llvm::Constant* Zero = llvm::Constant::getNullValue(LLVMTy);
2681
5.66k
      Builder.CreateStore(Zero, ReturnValue);
2682
5.66k
    }
2683
270k
  }
2684
2685
  // FIXME: We no longer need the types from FunctionArgList; lift up and
2686
  // simplify.
2687
2688
314k
  ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), FI);
2689
314k
  assert(Fn->arg_size() == IRFunctionArgs.totalIRArgs());
2690
2691
  // If we're using inalloca, all the memory arguments are GEPs off of the last
2692
  // parameter, which is a pointer to the complete memory area.
2693
0
  Address ArgStruct = Address::invalid();
2694
314k
  if (IRFunctionArgs.hasInallocaArg()) {
2695
41
    ArgStruct = Address(Fn->getArg(IRFunctionArgs.getInallocaArgNo()),
2696
41
                        FI.getArgStruct(), FI.getArgStructAlignment());
2697
2698
41
    assert(ArgStruct.getType() == FI.getArgStruct()->getPointerTo());
2699
41
  }
2700
2701
  // Name the struct return parameter.
2702
314k
  if (IRFunctionArgs.hasSRetArg()) {
2703
4.14k
    auto AI = Fn->getArg(IRFunctionArgs.getSRetArgNo());
2704
4.14k
    AI->setName("agg.result");
2705
4.14k
    AI->addAttr(llvm::Attribute::NoAlias);
2706
4.14k
  }
2707
2708
  // Track if we received the parameter as a pointer (indirect, byval, or
2709
  // inalloca).  If already have a pointer, EmitParmDecl doesn't need to copy it
2710
  // into a local alloca for us.
2711
314k
  SmallVector<ParamValue, 16> ArgVals;
2712
314k
  ArgVals.reserve(Args.size());
2713
2714
  // Create a pointer value for every parameter declaration.  This usually
2715
  // entails copying one or more LLVM IR arguments into an alloca.  Don't push
2716
  // any cleanups or do anything that might unwind.  We do that separately, so
2717
  // we can push the cleanups in the correct order for the ABI.
2718
314k
  assert(FI.arg_size() == Args.size() &&
2719
314k
         "Mismatch between function signature & arguments.");
2720
0
  unsigned ArgNo = 0;
2721
314k
  CGFunctionInfo::const_arg_iterator info_it = FI.arg_begin();
2722
314k
  for (FunctionArgList::const_iterator i = Args.begin(), e = Args.end();
2723
841k
       i != e; 
++i, ++info_it, ++ArgNo526k
) {
2724
526k
    const VarDecl *Arg = *i;
2725
526k
    const ABIArgInfo &ArgI = info_it->info;
2726
2727
526k
    bool isPromoted =
2728
526k
      isa<ParmVarDecl>(Arg) && 
cast<ParmVarDecl>(Arg)->isKNRPromoted()335k
;
2729
    // We are converting from ABIArgInfo type to VarDecl type directly, unless
2730
    // the parameter is promoted. In this case we convert to
2731
    // CGFunctionInfo::ArgInfo type with subsequent argument demotion.
2732
526k
    QualType Ty = isPromoted ? 
info_it->type14
:
Arg->getType()526k
;
2733
526k
    assert(hasScalarEvaluationKind(Ty) ==
2734
526k
           hasScalarEvaluationKind(Arg->getType()));
2735
2736
0
    unsigned FirstIRArg, NumIRArgs;
2737
526k
    std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
2738
2739
526k
    switch (ArgI.getKind()) {
2740
73
    case ABIArgInfo::InAlloca: {
2741
73
      assert(NumIRArgs == 0);
2742
0
      auto FieldIndex = ArgI.getInAllocaFieldIndex();
2743
73
      Address V =
2744
73
          Builder.CreateStructGEP(ArgStruct, FieldIndex, Arg->getName());
2745
73
      if (ArgI.getInAllocaIndirect())
2746
6
        V = Address(Builder.CreateLoad(V), ConvertTypeForMem(Ty),
2747
6
                    getContext().getTypeAlignInChars(Ty));
2748
73
      ArgVals.push_back(ParamValue::forIndirect(V));
2749
73
      break;
2750
0
    }
2751
2752
2.48k
    case ABIArgInfo::Indirect:
2753
2.50k
    case ABIArgInfo::IndirectAliased: {
2754
2.50k
      assert(NumIRArgs == 1);
2755
0
      Address ParamAddr = Address(Fn->getArg(FirstIRArg), ConvertTypeForMem(Ty),
2756
2.50k
                                  ArgI.getIndirectAlign());
2757
2758
2.50k
      if (!hasScalarEvaluationKind(Ty)) {
2759
        // Aggregates and complex variables are accessed by reference. All we
2760
        // need to do is realign the value, if requested. Also, if the address
2761
        // may be aliased, copy it to ensure that the parameter variable is
2762
        // mutable and has a unique adress, as C requires.
2763
2.16k
        Address V = ParamAddr;
2764
2.16k
        if (ArgI.getIndirectRealign() || 
ArgI.isIndirectAliased()2.13k
) {
2765
42
          Address AlignedTemp = CreateMemTemp(Ty, "coerce");
2766
2767
          // Copy from the incoming argument pointer to the temporary with the
2768
          // appropriate alignment.
2769
          //
2770
          // FIXME: We should have a common utility for generating an aggregate
2771
          // copy.
2772
42
          CharUnits Size = getContext().getTypeSizeInChars(Ty);
2773
42
          Builder.CreateMemCpy(
2774
42
              AlignedTemp.getPointer(), AlignedTemp.getAlignment().getAsAlign(),
2775
42
              ParamAddr.getPointer(), ParamAddr.getAlignment().getAsAlign(),
2776
42
              llvm::ConstantInt::get(IntPtrTy, Size.getQuantity()));
2777
42
          V = AlignedTemp;
2778
42
        }
2779
2.16k
        ArgVals.push_back(ParamValue::forIndirect(V));
2780
2.16k
      } else {
2781
        // Load scalar value from indirect argument.
2782
338
        llvm::Value *V =
2783
338
            EmitLoadOfScalar(ParamAddr, false, Ty, Arg->getBeginLoc());
2784
2785
338
        if (isPromoted)
2786
0
          V = emitArgumentDemotion(*this, Arg, V);
2787
338
        ArgVals.push_back(ParamValue::forDirect(V));
2788
338
      }
2789
2.50k
      break;
2790
2.48k
    }
2791
2792
17.7k
    case ABIArgInfo::Extend:
2793
516k
    case ABIArgInfo::Direct: {
2794
516k
      auto AI = Fn->getArg(FirstIRArg);
2795
516k
      llvm::Type *LTy = ConvertType(Arg->getType());
2796
2797
      // Prepare parameter attributes. So far, only attributes for pointer
2798
      // parameters are prepared. See
2799
      // http://llvm.org/docs/LangRef.html#paramattrs.
2800
516k
      if (ArgI.getDirectOffset() == 0 && 
LTy->isPointerTy()516k
&&
2801
516k
          
ArgI.getCoerceToType()->isPointerTy()277k
) {
2802
277k
        assert(NumIRArgs == 1);
2803
2804
277k
        if (const ParmVarDecl *PVD = dyn_cast<ParmVarDecl>(Arg)) {
2805
          // Set `nonnull` attribute if any.
2806
114k
          if (getNonNullAttr(CurCodeDecl, PVD, PVD->getType(),
2807
114k
                             PVD->getFunctionScopeIndex()) &&
2808
114k
              
!CGM.getCodeGenOpts().NullPointerIsValid22
)
2809
13
            AI->addAttr(llvm::Attribute::NonNull);
2810
2811
114k
          QualType OTy = PVD->getOriginalType();
2812
114k
          if (const auto *ArrTy =
2813
114k
              getContext().getAsConstantArrayType(OTy)) {
2814
            // A C99 array parameter declaration with the static keyword also
2815
            // indicates dereferenceability, and if the size is constant we can
2816
            // use the dereferenceable attribute (which requires the size in
2817
            // bytes).
2818
578
            if (ArrTy->getSizeModifier() == ArrayType::Static) {
2819
5
              QualType ETy = ArrTy->getElementType();
2820
5
              llvm::Align Alignment =
2821
5
                  CGM.getNaturalTypeAlignment(ETy).getAsAlign();
2822
5
              AI->addAttrs(llvm::AttrBuilder(getLLVMContext()).addAlignmentAttr(Alignment));
2823
5
              uint64_t ArrSize = ArrTy->getSize().getZExtValue();
2824
5
              if (!ETy->isIncompleteType() && ETy->isConstantSizeType() &&
2825
5
                  ArrSize) {
2826
3
                llvm::AttrBuilder Attrs(getLLVMContext());
2827
3
                Attrs.addDereferenceableAttr(
2828
3
                    getContext().getTypeSizeInChars(ETy).getQuantity() *
2829
3
                    ArrSize);
2830
3
                AI->addAttrs(Attrs);
2831
3
              } else 
if (2
getContext().getTargetInfo().getNullPointerValue(
2832
2
                             ETy.getAddressSpace()) == 0 &&
2833
2
                         !CGM.getCodeGenOpts().NullPointerIsValid) {
2834
1
                AI->addAttr(llvm::Attribute::NonNull);
2835
1
              }
2836
5
            }
2837
113k
          } else if (const auto *ArrTy =
2838
113k
                     getContext().getAsVariableArrayType(OTy)) {
2839
            // For C99 VLAs with the static keyword, we don't know the size so
2840
            // we can't use the dereferenceable attribute, but in addrspace(0)
2841
            // we know that it must be nonnull.
2842
102
            if (ArrTy->getSizeModifier() == VariableArrayType::Static) {
2843
2
              QualType ETy = ArrTy->getElementType();
2844
2
              llvm::Align Alignment =
2845
2
                  CGM.getNaturalTypeAlignment(ETy).getAsAlign();
2846
2
              AI->addAttrs(llvm::AttrBuilder(getLLVMContext()).addAlignmentAttr(Alignment));
2847
2
              if (!getContext().getTargetAddressSpace(ETy) &&
2848
2
                  !CGM.getCodeGenOpts().NullPointerIsValid)
2849
1
                AI->addAttr(llvm::Attribute::NonNull);
2850
2
            }
2851
102
          }
2852
2853
          // Set `align` attribute if any.
2854
114k
          const auto *AVAttr = PVD->getAttr<AlignValueAttr>();
2855
114k
          if (!AVAttr)
2856
114k
            if (const auto *TOTy = dyn_cast<TypedefType>(OTy))
2857
6.99k
              AVAttr = TOTy->getDecl()->getAttr<AlignValueAttr>();
2858
114k
          if (AVAttr && 
!SanOpts.has(SanitizerKind::Alignment)11
) {
2859
            // If alignment-assumption sanitizer is enabled, we do *not* add
2860
            // alignment attribute here, but emit normal alignment assumption,
2861
            // so the UBSAN check could function.
2862
8
            llvm::ConstantInt *AlignmentCI =
2863
8
                cast<llvm::ConstantInt>(EmitScalarExpr(AVAttr->getAlignment()));
2864
8
            uint64_t AlignmentInt =
2865
8
                AlignmentCI->getLimitedValue(llvm::Value::MaximumAlignment);
2866
8
            if (AI->getParamAlign().valueOrOne() < AlignmentInt) {
2867
8
              AI->removeAttr(llvm::Attribute::AttrKind::Alignment);
2868
8
              AI->addAttrs(llvm::AttrBuilder(getLLVMContext()).addAlignmentAttr(
2869
8
                  llvm::Align(AlignmentInt)));
2870
8
            }
2871
8
          }
2872
114k
        }
2873
2874
        // Set 'noalias' if an argument type has the `restrict` qualifier.
2875
277k
        if (Arg->getType().isRestrictQualified())
2876
32.0k
          AI->addAttr(llvm::Attribute::NoAlias);
2877
277k
      }
2878
2879
      // Prepare the argument value. If we have the trivial case, handle it
2880
      // with no muss and fuss.
2881
516k
      if (!isa<llvm::StructType>(ArgI.getCoerceToType()) &&
2882
516k
          
ArgI.getCoerceToType() == ConvertType(Ty)515k
&&
2883
516k
          
ArgI.getDirectOffset() == 0510k
) {
2884
510k
        assert(NumIRArgs == 1);
2885
2886
        // LLVM expects swifterror parameters to be used in very restricted
2887
        // ways.  Copy the value into a less-restricted temporary.
2888
0
        llvm::Value *V = AI;
2889
510k
        if (FI.getExtParameterInfo(ArgNo).getABI()
2890
510k
              == ParameterABI::SwiftErrorResult) {
2891
16
          QualType pointeeTy = Ty->getPointeeType();
2892
16
          assert(pointeeTy->isPointerType());
2893
0
          Address temp =
2894
16
            CreateMemTemp(pointeeTy, getPointerAlign(), "swifterror.temp");
2895
16
          Address arg(V, ConvertTypeForMem(pointeeTy),
2896
16
                      getContext().getTypeAlignInChars(pointeeTy));
2897
16
          llvm::Value *incomingErrorValue = Builder.CreateLoad(arg);
2898
16
          Builder.CreateStore(incomingErrorValue, temp);
2899
16
          V = temp.getPointer();
2900
2901
          // Push a cleanup to copy the value back at the end of the function.
2902
          // The convention does not guarantee that the value will be written
2903
          // back if the function exits with an unwind exception.
2904
16
          EHStack.pushCleanup<CopyBackSwiftError>(NormalCleanup, temp, arg);
2905
16
        }
2906
2907
        // Ensure the argument is the correct type.
2908
510k
        if (V->getType() != ArgI.getCoerceToType())
2909
0
          V = Builder.CreateBitCast(V, ArgI.getCoerceToType());
2910
2911
510k
        if (isPromoted)
2912
14
          V = emitArgumentDemotion(*this, Arg, V);
2913
2914
        // Because of merging of function types from multiple decls it is
2915
        // possible for the type of an argument to not match the corresponding
2916
        // type in the function type. Since we are codegening the callee
2917
        // in here, add a cast to the argument type.
2918
510k
        llvm::Type *LTy = ConvertType(Arg->getType());
2919
510k
        if (V->getType() != LTy)
2920
0
          V = Builder.CreateBitCast(V, LTy);
2921
2922
510k
        ArgVals.push_back(ParamValue::forDirect(V));
2923
510k
        break;
2924
510k
      }
2925
2926
      // VLST arguments are coerced to VLATs at the function boundary for
2927
      // ABI consistency. If this is a VLST that was coerced to
2928
      // a VLAT at the function boundary and the types match up, use
2929
      // llvm.experimental.vector.extract to convert back to the original
2930
      // VLST.
2931
5.89k
      if (auto *VecTyTo = dyn_cast<llvm::FixedVectorType>(ConvertType(Ty))) {
2932
1.04k
        llvm::Value *Coerced = Fn->getArg(FirstIRArg);
2933
1.04k
        if (auto *VecTyFrom =
2934
1.04k
                dyn_cast<llvm::ScalableVectorType>(Coerced->getType())) {
2935
          // If we are casting a scalable 16 x i1 predicate vector to a fixed i8
2936
          // vector, bitcast the source and use a vector extract.
2937
36
          auto PredType =
2938
36
              llvm::ScalableVectorType::get(Builder.getInt1Ty(), 16);
2939
36
          if (VecTyFrom == PredType &&
2940
36
              
VecTyTo->getElementType() == Builder.getInt8Ty()4
) {
2941
4
            VecTyFrom = llvm::ScalableVectorType::get(Builder.getInt8Ty(), 2);
2942
4
            Coerced = Builder.CreateBitCast(Coerced, VecTyFrom);
2943
4
          }
2944
36
          if (VecTyFrom->getElementType() == VecTyTo->getElementType()) {
2945
36
            llvm::Value *Zero = llvm::Constant::getNullValue(CGM.Int64Ty);
2946
2947
36
            assert(NumIRArgs == 1);
2948
0
            Coerced->setName(Arg->getName() + ".coerce");
2949
36
            ArgVals.push_back(ParamValue::forDirect(Builder.CreateExtractVector(
2950
36
                VecTyTo, Coerced, Zero, "castFixedSve")));
2951
36
            break;
2952
36
          }
2953
36
        }
2954
1.04k
      }
2955
2956
5.86k
      Address Alloca = CreateMemTemp(Ty, getContext().getDeclAlign(Arg),
2957
5.86k
                                     Arg->getName());
2958
2959
      // Pointer to store into.
2960
5.86k
      Address Ptr = emitAddressAtOffset(*this, Alloca, ArgI);
2961
2962
      // Fast-isel and the optimizer generally like scalar values better than
2963
      // FCAs, so we flatten them if this is safe to do for this argument.
2964
5.86k
      llvm::StructType *STy = dyn_cast<llvm::StructType>(ArgI.getCoerceToType());
2965
5.86k
      if (ArgI.isDirect() && ArgI.getCanBeFlattened() && 
STy5.59k
&&
2966
5.86k
          
STy->getNumElements() > 11.00k
) {
2967
977
        uint64_t SrcSize = CGM.getDataLayout().getTypeAllocSize(STy);
2968
977
        llvm::Type *DstTy = Ptr.getElementType();
2969
977
        uint64_t DstSize = CGM.getDataLayout().getTypeAllocSize(DstTy);
2970
2971
977
        Address AddrToStoreInto = Address::invalid();
2972
977
        if (SrcSize <= DstSize) {
2973
888
          AddrToStoreInto = Builder.CreateElementBitCast(Ptr, STy);
2974
888
        } else {
2975
89
          AddrToStoreInto =
2976
89
            CreateTempAlloca(STy, Alloca.getAlignment(), "coerce");
2977
89
        }
2978
2979
977
        assert(STy->getNumElements() == NumIRArgs);
2980
2.99k
        for (unsigned i = 0, e = STy->getNumElements(); i != e; 
++i2.01k
) {
2981
2.01k
          auto AI = Fn->getArg(FirstIRArg + i);
2982
2.01k
          AI->setName(Arg->getName() + ".coerce" + Twine(i));
2983
2.01k
          Address EltPtr = Builder.CreateStructGEP(AddrToStoreInto, i);
2984
2.01k
          Builder.CreateStore(AI, EltPtr);
2985
2.01k
        }
2986
2987
977
        if (SrcSize > DstSize) {
2988
89
          Builder.CreateMemCpy(Ptr, AddrToStoreInto, DstSize);
2989
89
        }
2990
2991
4.88k
      } else {
2992
        // Simple case, just do a coerced store of the argument into the alloca.
2993
4.88k
        assert(NumIRArgs == 1);
2994
0
        auto AI = Fn->getArg(FirstIRArg);
2995
4.88k
        AI->setName(Arg->getName() + ".coerce");
2996
4.88k
        CreateCoercedStore(AI, Ptr, /*DstIsVolatile=*/false, *this);
2997
4.88k
      }
2998
2999
      // Match to what EmitParmDecl is expecting for this type.
3000
5.86k
      if (CodeGenFunction::hasScalarEvaluationKind(Ty)) {
3001
1.24k
        llvm::Value *V =
3002
1.24k
            EmitLoadOfScalar(Alloca, false, Ty, Arg->getBeginLoc());
3003
1.24k
        if (isPromoted)
3004
0
          V = emitArgumentDemotion(*this, Arg, V);
3005
1.24k
        ArgVals.push_back(ParamValue::forDirect(V));
3006
4.61k
      } else {
3007
4.61k
        ArgVals.push_back(ParamValue::forIndirect(Alloca));
3008
4.61k
      }
3009
5.86k
      break;
3010
5.89k
    }
3011
3012
571
    case ABIArgInfo::CoerceAndExpand: {
3013
      // Reconstruct into a temporary.
3014
571
      Address alloca = CreateMemTemp(Ty, getContext().getDeclAlign(Arg));
3015
571
      ArgVals.push_back(ParamValue::forIndirect(alloca));
3016
3017
571
      auto coercionType = ArgI.getCoerceAndExpandType();
3018
571
      alloca = Builder.CreateElementBitCast(alloca, coercionType);
3019
3020
571
      unsigned argIndex = FirstIRArg;
3021
1.78k
      for (unsigned i = 0, e = coercionType->getNumElements(); i != e; 
++i1.21k
) {
3022
1.21k
        llvm::Type *eltType = coercionType->getElementType(i);
3023
1.21k
        if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType))
3024
13
          continue;
3025
3026
1.19k
        auto eltAddr = Builder.CreateStructGEP(alloca, i);
3027
1.19k
        auto elt = Fn->getArg(argIndex++);
3028
1.19k
        Builder.CreateStore(elt, eltAddr);
3029
1.19k
      }
3030
571
      assert(argIndex == FirstIRArg + NumIRArgs);
3031
0
      break;
3032
5.89k
    }
3033
3034
102
    case ABIArgInfo::Expand: {
3035
      // If this structure was expanded into multiple arguments then
3036
      // we need to create a temporary and reconstruct it from the
3037
      // arguments.
3038
102
      Address Alloca = CreateMemTemp(Ty, getContext().getDeclAlign(Arg));
3039
102
      LValue LV = MakeAddrLValue(Alloca, Ty);
3040
102
      ArgVals.push_back(ParamValue::forIndirect(Alloca));
3041
3042
102
      auto FnArgIter = Fn->arg_begin() + FirstIRArg;
3043
102
      ExpandTypeFromArgs(Ty, LV, FnArgIter);
3044
102
      assert(FnArgIter == Fn->arg_begin() + FirstIRArg + NumIRArgs);
3045
345
      for (unsigned i = 0, e = NumIRArgs; i != e; 
++i243
) {
3046
243
        auto AI = Fn->getArg(FirstIRArg + i);
3047
243
        AI->setName(Arg->getName() + "." + Twine(i));
3048
243
      }
3049
102
      break;
3050
5.89k
    }
3051
3052
6.79k
    case ABIArgInfo::Ignore:
3053
6.79k
      assert(NumIRArgs == 0);
3054
      // Initialize the local variable appropriately.
3055
6.79k
      if (!hasScalarEvaluationKind(Ty)) {
3056
6.79k
        ArgVals.push_back(ParamValue::forIndirect(CreateMemTemp(Ty)));
3057
6.79k
      } else {
3058
0
        llvm::Value *U = llvm::UndefValue::get(ConvertType(Arg->getType()));
3059
0
        ArgVals.push_back(ParamValue::forDirect(U));
3060
0
      }
3061
6.79k
      break;
3062
526k
    }
3063
526k
  }
3064
3065
314k
  if (getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) {
3066
31.7k
    for (int I = Args.size() - 1; I >= 0; 
--I18.2k
)
3067
18.2k
      EmitParmDecl(*Args[I], ArgVals[I], I + 1);
3068
301k
  } else {
3069
809k
    for (unsigned I = 0, E = Args.size(); I != E; 
++I508k
)
3070
508k
      EmitParmDecl(*Args[I], ArgVals[I], I + 1);
3071
301k
  }
3072
314k
}
3073
3074
8
static void eraseUnusedBitCasts(llvm::Instruction *insn) {
3075
12
  while (insn->use_empty()) {
3076
12
    llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(insn);
3077
12
    if (!bitcast) 
return8
;
3078
3079
    // This is "safe" because we would have used a ConstantExpr otherwise.
3080
4
    insn = cast<llvm::Instruction>(bitcast->getOperand(0));
3081
4
    bitcast->eraseFromParent();
3082
4
  }
3083
8
}
3084
3085
/// Try to emit a fused autorelease of a return result.
3086
static llvm::Value *tryEmitFusedAutoreleaseOfResult(CodeGenFunction &CGF,
3087
87
                                                    llvm::Value *result) {
3088
  // We must be immediately followed the cast.
3089
87
  llvm::BasicBlock *BB = CGF.Builder.GetInsertBlock();
3090
87
  if (BB->empty()) 
return nullptr0
;
3091
87
  if (&BB->back() != result) 
return nullptr24
;
3092
3093
63
  llvm::Type *resultType = result->getType();
3094
3095
  // result is in a BasicBlock and is therefore an Instruction.
3096
63
  llvm::Instruction *generator = cast<llvm::Instruction>(result);
3097
3098
63
  SmallVector<llvm::Instruction *, 4> InstsToKill;
3099
3100
  // Look for:
3101
  //  %generator = bitcast %type1* %generator2 to %type2*
3102
94
  while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(generator)) {
3103
    // We would have emitted this as a constant if the operand weren't
3104
    // an Instruction.
3105
31
    generator = cast<llvm::Instruction>(bitcast->getOperand(0));
3106
3107
    // Require the generator to be immediately followed by the cast.
3108
31
    if (generator->getNextNode() != bitcast)
3109
0
      return nullptr;
3110
3111
31
    InstsToKill.push_back(bitcast);
3112
31
  }
3113
3114
  // Look for:
3115
  //   %generator = call i8* @objc_retain(i8* %originalResult)
3116
  // or
3117
  //   %generator = call i8* @objc_retainAutoreleasedReturnValue(i8* %originalResult)
3118
63
  llvm::CallInst *call = dyn_cast<llvm::CallInst>(generator);
3119
63
  if (!call) 
return nullptr13
;
3120
3121
50
  bool doRetainAutorelease;
3122
3123
50
  if (call->getCalledOperand() == CGF.CGM.getObjCEntrypoints().objc_retain) {
3124
8
    doRetainAutorelease = true;
3125
42
  } else if (call->getCalledOperand() ==
3126
42
             CGF.CGM.getObjCEntrypoints().objc_retainAutoreleasedReturnValue) {
3127
19
    doRetainAutorelease = false;
3128
3129
    // If we emitted an assembly marker for this call (and the
3130
    // ARCEntrypoints field should have been set if so), go looking
3131
    // for that call.  If we can't find it, we can't do this
3132
    // optimization.  But it should always be the immediately previous
3133
    // instruction, unless we needed bitcasts around the call.
3134
19
    if (CGF.CGM.getObjCEntrypoints().retainAutoreleasedReturnValueMarker) {
3135
12
      llvm::Instruction *prev = call->getPrevNode();
3136
12
      assert(prev);
3137
12
      if (isa<llvm::BitCastInst>(prev)) {
3138
8
        prev = prev->getPrevNode();
3139
8
        assert(prev);
3140
8
      }
3141
0
      assert(isa<llvm::CallInst>(prev));
3142
0
      assert(cast<llvm::CallInst>(prev)->getCalledOperand() ==
3143
12
             CGF.CGM.getObjCEntrypoints().retainAutoreleasedReturnValueMarker);
3144
0
      InstsToKill.push_back(prev);
3145
12
    }
3146
23
  } else {
3147
23
    return nullptr;
3148
23
  }
3149
3150
27
  result = call->getArgOperand(0);
3151
27
  InstsToKill.push_back(call);
3152
3153
  // Keep killing bitcasts, for sanity.  Note that we no longer care
3154
  // about precise ordering as long as there's exactly one use.
3155
42
  while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(result)) {
3156
15
    if (!bitcast->hasOneUse()) 
break0
;
3157
15
    InstsToKill.push_back(bitcast);
3158
15
    result = bitcast->getOperand(0);
3159
15
  }
3160
3161
  // Delete all the unnecessary instructions, from latest to earliest.
3162
27
  for (auto *I : InstsToKill)
3163
75
    I->eraseFromParent();
3164
3165
  // Do the fused retain/autorelease if we were asked to.
3166
27
  if (doRetainAutorelease)
3167
8
    result = CGF.EmitARCRetainAutoreleaseReturnValue(result);
3168
3169
  // Cast back to the result type.
3170
27
  return CGF.Builder.CreateBitCast(result, resultType);
3171
50
}
3172
3173
/// If this is a +1 of the value of an immutable 'self', remove it.
3174
static llvm::Value *tryRemoveRetainOfSelf(CodeGenFunction &CGF,
3175
122
                                          llvm::Value *result) {
3176
  // This is only applicable to a method with an immutable 'self'.
3177
122
  const ObjCMethodDecl *method =
3178
122
    dyn_cast_or_null<ObjCMethodDecl>(CGF.CurCodeDecl);
3179
122
  if (!method) 
return nullptr103
;
3180
19
  const VarDecl *self = method->getSelfDecl();
3181
19
  if (!self->getType().isConstQualified()) 
return nullptr0
;
3182
3183
  // Look for a retain call.
3184
19
  llvm::CallInst *retainCall =
3185
19
    dyn_cast<llvm::CallInst>(result->stripPointerCasts());
3186
19
  if (!retainCall || retainCall->getCalledOperand() !=
3187
15
                         CGF.CGM.getObjCEntrypoints().objc_retain)
3188
12
    return nullptr;
3189
3190
  // Look for an ordinary load of 'self'.
3191
7
  llvm::Value *retainedValue = retainCall->getArgOperand(0);
3192
7
  llvm::LoadInst *load =
3193
7
    dyn_cast<llvm::LoadInst>(retainedValue->stripPointerCasts());
3194
7
  if (!load || load->isAtomic() || load->isVolatile() ||
3195
7
      load->getPointerOperand() != CGF.GetAddrOfLocalVar(self).getPointer())
3196
3
    return nullptr;
3197
3198
  // Okay!  Burn it all down.  This relies for correctness on the
3199
  // assumption that the retain is emitted as part of the return and
3200
  // that thereafter everything is used "linearly".
3201
4
  llvm::Type *resultType = result->getType();
3202
4
  eraseUnusedBitCasts(cast<llvm::Instruction>(result));
3203
4
  assert(retainCall->use_empty());
3204
0
  retainCall->eraseFromParent();
3205
4
  eraseUnusedBitCasts(cast<llvm::Instruction>(retainedValue));
3206
3207
4
  return CGF.Builder.CreateBitCast(load, resultType);
3208
7
}
3209
3210
/// Emit an ARC autorelease of the result of a function.
3211
///
3212
/// \return the value to actually return from the function
3213
static llvm::Value *emitAutoreleaseOfResult(CodeGenFunction &CGF,
3214
122
                                            llvm::Value *result) {
3215
  // If we're returning 'self', kill the initial retain.  This is a
3216
  // heuristic attempt to "encourage correctness" in the really unfortunate
3217
  // case where we have a return of self during a dealloc and we desperately
3218
  // need to avoid the possible autorelease.
3219
122
  if (llvm::Value *self = tryRemoveRetainOfSelf(CGF, result))
3220
4
    return self;
3221
3222
  // At -O0, try to emit a fused retain/autorelease.
3223
118
  if (CGF.shouldUseFusedARCCalls())
3224
87
    if (llvm::Value *fused = tryEmitFusedAutoreleaseOfResult(CGF, result))
3225
27
      return fused;
3226
3227
91
  return CGF.EmitARCAutoreleaseReturnValue(result);
3228
118
}
3229
3230
/// Heuristically search for a dominating store to the return-value slot.
3231
144k
static llvm::StoreInst *findDominatingStoreToReturnValue(CodeGenFunction &CGF) {
3232
  // Check if a User is a store which pointerOperand is the ReturnValue.
3233
  // We are looking for stores to the ReturnValue, not for stores of the
3234
  // ReturnValue to some other location.
3235
144k
  auto GetStoreIfValid = [&CGF](llvm::User *U) -> llvm::StoreInst * {
3236
142k
    auto *SI = dyn_cast<llvm::StoreInst>(U);
3237
142k
    if (!SI || 
SI->getPointerOperand() != CGF.ReturnValue.getPointer()140k
||
3238
142k
        
SI->getValueOperand()->getType() != CGF.ReturnValue.getElementType()139k
)
3239
2.18k
      return nullptr;
3240
    // These aren't actually possible for non-coerced returns, and we
3241
    // only care about non-coerced returns on this code path.
3242
139k
    assert(!SI->isAtomic() && !SI->isVolatile());
3243
0
    return SI;
3244
142k
  };
3245
  // If there are multiple uses of the return-value slot, just check
3246
  // for something immediately preceding the IP.  Sometimes this can
3247
  // happen with how we generate implicit-returns; it can also happen
3248
  // with noreturn cleanups.
3249
144k
  if (!CGF.ReturnValue.getPointer()->hasOneUse()) {
3250
7.38k
    llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock();
3251
7.38k
    if (IP->empty()) 
return nullptr1.88k
;
3252
3253
    // Look at directly preceding instruction, skipping bitcasts and lifetime
3254
    // markers.
3255
5.70k
    
for (llvm::Instruction &I : make_range(IP->rbegin(), IP->rend()))5.49k
{
3256
5.70k
      if (isa<llvm::BitCastInst>(&I))
3257
152
        continue;
3258
5.55k
      if (auto *II = dyn_cast<llvm::IntrinsicInst>(&I))
3259
350
        if (II->getIntrinsicID() == llvm::Intrinsic::lifetime_end)
3260
95
          continue;
3261
3262
5.45k
      return GetStoreIfValid(&I);
3263
5.55k
    }
3264
41
    return nullptr;
3265
5.49k
  }
3266
3267
136k
  llvm::StoreInst *store =
3268
136k
      GetStoreIfValid(CGF.ReturnValue.getPointer()->user_back());
3269
136k
  if (!store) 
return nullptr512
;
3270
3271
  // Now do a first-and-dirty dominance check: just walk up the
3272
  // single-predecessors chain from the current insertion point.
3273
136k
  llvm::BasicBlock *StoreBB = store->getParent();
3274
136k
  llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock();
3275
136k
  while (IP != StoreBB) {
3276
2.00k
    if (!(IP = IP->getSinglePredecessor()))
3277
1.26k
      return nullptr;
3278
2.00k
  }
3279
3280
  // Okay, the store's basic block dominates the insertion point; we
3281
  // can do our thing.
3282
134k
  return store;
3283
136k
}
3284
3285
// Helper functions for EmitCMSEClearRecord
3286
3287
// Set the bits corresponding to a field having width `BitWidth` and located at
3288
// offset `BitOffset` (from the least significant bit) within a storage unit of
3289
// `Bits.size()` bytes. Each element of `Bits` corresponds to one target byte.
3290
// Use little-endian layout, i.e.`Bits[0]` is the LSB.
3291
static void setBitRange(SmallVectorImpl<uint64_t> &Bits, int BitOffset,
3292
125
                        int BitWidth, int CharWidth) {
3293
125
  assert(CharWidth <= 64);
3294
0
  assert(static_cast<unsigned>(BitWidth) <= Bits.size() * CharWidth);
3295
3296
0
  int Pos = 0;
3297
125
  if (BitOffset >= CharWidth) {
3298
39
    Pos += BitOffset / CharWidth;
3299
39
    BitOffset = BitOffset % CharWidth;
3300
39
  }
3301
3302
125
  const uint64_t Used = (uint64_t(1) << CharWidth) - 1;
3303
125
  if (BitOffset + BitWidth >= CharWidth) {
3304
65
    Bits[Pos++] |= (Used << BitOffset) & Used;
3305
65
    BitWidth -= CharWidth - BitOffset;
3306
65
    BitOffset = 0;
3307
65
  }
3308
3309
143
  while (BitWidth >= CharWidth) {
3310
18
    Bits[Pos++] = Used;
3311
18
    BitWidth -= CharWidth;
3312
18
  }
3313
3314
125
  if (BitWidth > 0)
3315
92
    Bits[Pos++] |= (Used >> (CharWidth - BitWidth)) << BitOffset;
3316
125
}
3317
3318
// Set the bits corresponding to a field having width `BitWidth` and located at
3319
// offset `BitOffset` (from the least significant bit) within a storage unit of
3320
// `StorageSize` bytes, located at `StorageOffset` in `Bits`. Each element of
3321
// `Bits` corresponds to one target byte. Use target endian layout.
3322
static void setBitRange(SmallVectorImpl<uint64_t> &Bits, int StorageOffset,
3323
                        int StorageSize, int BitOffset, int BitWidth,
3324
125
                        int CharWidth, bool BigEndian) {
3325
3326
125
  SmallVector<uint64_t, 8> TmpBits(StorageSize);
3327
125
  setBitRange(TmpBits, BitOffset, BitWidth, CharWidth);
3328
3329
125
  if (BigEndian)
3330
50
    std::reverse(TmpBits.begin(), TmpBits.end());
3331
3332
125
  for (uint64_t V : TmpBits)
3333
350
    Bits[StorageOffset++] |= V;
3334
125
}
3335
3336
static void setUsedBits(CodeGenModule &, QualType, int,
3337
                        SmallVectorImpl<uint64_t> &);
3338
3339
// Set the bits in `Bits`, which correspond to the value representations of
3340
// the actual members of the record type `RTy`. Note that this function does
3341
// not handle base classes, virtual tables, etc, since they cannot happen in
3342
// CMSE function arguments or return. The bit mask corresponds to the target
3343
// memory layout, i.e. it's endian dependent.
3344
static void setUsedBits(CodeGenModule &CGM, const RecordType *RTy, int Offset,
3345
153
                        SmallVectorImpl<uint64_t> &Bits) {
3346
153
  ASTContext &Context = CGM.getContext();
3347
153
  int CharWidth = Context.getCharWidth();
3348
153
  const RecordDecl *RD = RTy->getDecl()->getDefinition();
3349
153
  const ASTRecordLayout &ASTLayout = Context.getASTRecordLayout(RD);
3350
153
  const CGRecordLayout &Layout = CGM.getTypes().getCGRecordLayout(RD);
3351
3352
153
  int Idx = 0;
3353
566
  for (auto I = RD->field_begin(), E = RD->field_end(); I != E; 
++I, ++Idx413
) {
3354
413
    const FieldDecl *F = *I;
3355
3356
413
    if (F->isUnnamedBitfield() || 
F->isZeroLengthBitField(Context)283
||
3357
413
        
F->getType()->isIncompleteArrayType()283
)
3358
140
      continue;
3359
3360
273
    if (F->isBitField()) {
3361
125
      const CGBitFieldInfo &BFI = Layout.getBitFieldInfo(F);
3362
125
      setBitRange(Bits, Offset + BFI.StorageOffset.getQuantity(),
3363
125
                  BFI.StorageSize / CharWidth, BFI.Offset,
3364
125
                  BFI.Size, CharWidth,
3365
125
                  CGM.getDataLayout().isBigEndian());
3366
125
      continue;
3367
125
    }
3368
3369
148
    setUsedBits(CGM, F->getType(),
3370
148
                Offset + ASTLayout.getFieldOffset(Idx) / CharWidth, Bits);
3371
148
  }
3372
153
}
3373
3374
// Set the bits in `Bits`, which correspond to the value representations of
3375
// the elements of an array type `ATy`.
3376
static void setUsedBits(CodeGenModule &CGM, const ConstantArrayType *ATy,
3377
34
                        int Offset, SmallVectorImpl<uint64_t> &Bits) {
3378
34
  const ASTContext &Context = CGM.getContext();
3379
3380
34
  QualType ETy = Context.getBaseElementType(ATy);
3381
34
  int Size = Context.getTypeSizeInChars(ETy).getQuantity();
3382
34
  SmallVector<uint64_t, 4> TmpBits(Size);
3383
34
  setUsedBits(CGM, ETy, 0, TmpBits);
3384
3385
142
  for (int I = 0, N = Context.getConstantArrayElementCount(ATy); I < N; 
++I108
) {
3386
108
    auto Src = TmpBits.begin();
3387
108
    auto Dst = Bits.begin() + Offset + I * Size;
3388
270
    for (int J = 0; J < Size; 
++J162
)
3389
162
      *Dst++ |= *Src++;
3390
108
  }
3391
34
}
3392
3393
// Set the bits in `Bits`, which correspond to the value representations of
3394
// the type `QTy`.
3395
static void setUsedBits(CodeGenModule &CGM, QualType QTy, int Offset,
3396
182
                        SmallVectorImpl<uint64_t> &Bits) {
3397
182
  if (const auto *RTy = QTy->getAs<RecordType>())
3398
30
    return setUsedBits(CGM, RTy, Offset, Bits);
3399
3400
152
  ASTContext &Context = CGM.getContext();
3401
152
  if (const auto *ATy = Context.getAsConstantArrayType(QTy))
3402
34
    return setUsedBits(CGM, ATy, Offset, Bits);
3403
3404
118
  int Size = Context.getTypeSizeInChars(QTy).getQuantity();
3405
118
  if (Size <= 0)
3406
0
    return;
3407
3408
118
  std::fill_n(Bits.begin() + Offset, Size,
3409
118
              (uint64_t(1) << Context.getCharWidth()) - 1);
3410
118
}
3411
3412
static uint64_t buildMultiCharMask(const SmallVectorImpl<uint64_t> &Bits,
3413
                                   int Pos, int Size, int CharWidth,
3414
152
                                   bool BigEndian) {
3415
152
  assert(Size > 0);
3416
0
  uint64_t Mask = 0;
3417
152
  if (BigEndian) {
3418
318
    for (auto P = Bits.begin() + Pos, E = Bits.begin() + Pos + Size; P != E;
3419
256
         ++P)
3420
256
      Mask = (Mask << CharWidth) | *P;
3421
90
  } else {
3422
90
    auto P = Bits.begin() + Pos + Size, End = Bits.begin() + Pos;
3423
90
    do
3424
372
      Mask = (Mask << CharWidth) | *--P;
3425
372
    while (P != End);
3426
90
  }
3427
152
  return Mask;
3428
152
}
3429
3430
// Emit code to clear the bits in a record, which aren't a part of any user
3431
// declared member, when the record is a function return.
3432
llvm::Value *CodeGenFunction::EmitCMSEClearRecord(llvm::Value *Src,
3433
                                                  llvm::IntegerType *ITy,
3434
84
                                                  QualType QTy) {
3435
84
  assert(Src->getType() == ITy);
3436
0
  assert(ITy->getScalarSizeInBits() <= 64);
3437
3438
0
  const llvm::DataLayout &DataLayout = CGM.getDataLayout();
3439
84
  int Size = DataLayout.getTypeStoreSize(ITy);
3440
84
  SmallVector<uint64_t, 4> Bits(Size);
3441
84
  setUsedBits(CGM, QTy->castAs<RecordType>(), 0, Bits);
3442
3443
84
  int CharWidth = CGM.getContext().getCharWidth();
3444
84
  uint64_t Mask =
3445
84
      buildMultiCharMask(Bits, 0, Size, CharWidth, DataLayout.isBigEndian());
3446
3447
84
  return Builder.CreateAnd(Src, Mask, "cmse.clear");
3448
84
}
3449
3450
// Emit code to clear the bits in a record, which aren't a part of any user
3451
// declared member, when the record is a function argument.
3452
llvm::Value *CodeGenFunction::EmitCMSEClearRecord(llvm::Value *Src,
3453
                                                  llvm::ArrayType *ATy,
3454
39
                                                  QualType QTy) {
3455
39
  const llvm::DataLayout &DataLayout = CGM.getDataLayout();
3456
39
  int Size = DataLayout.getTypeStoreSize(ATy);
3457
39
  SmallVector<uint64_t, 16> Bits(Size);
3458
39
  setUsedBits(CGM, QTy->castAs<RecordType>(), 0, Bits);
3459
3460
  // Clear each element of the LLVM array.
3461
39
  int CharWidth = CGM.getContext().getCharWidth();
3462
39
  int CharsPerElt =
3463
39
      ATy->getArrayElementType()->getScalarSizeInBits() / CharWidth;
3464
39
  int MaskIndex = 0;
3465
39
  llvm::Value *R = llvm::UndefValue::get(ATy);
3466
107
  for (int I = 0, N = ATy->getArrayNumElements(); I != N; 
++I68
) {
3467
68
    uint64_t Mask = buildMultiCharMask(Bits, MaskIndex, CharsPerElt, CharWidth,
3468
68
                                       DataLayout.isBigEndian());
3469
68
    MaskIndex += CharsPerElt;
3470
68
    llvm::Value *T0 = Builder.CreateExtractValue(Src, I);
3471
68
    llvm::Value *T1 = Builder.CreateAnd(T0, Mask, "cmse.clear");
3472
68
    R = Builder.CreateInsertValue(R, T1, I);
3473
68
  }
3474
3475
39
  return R;
3476
39
}
3477
3478
void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI,
3479
                                         bool EmitRetDbgLoc,
3480
314k
                                         SourceLocation EndLoc) {
3481
314k
  if (FI.isNoReturn()) {
3482
    // Noreturn functions don't return.
3483
12
    EmitUnreachable(EndLoc);
3484
12
    return;
3485
12
  }
3486
3487
314k
  if (CurCodeDecl && 
CurCodeDecl->hasAttr<NakedAttr>()305k
) {
3488
    // Naked functions don't have epilogues.
3489
10
    Builder.CreateUnreachable();
3490
10
    return;
3491
10
  }
3492
3493
  // Functions with no result always return void.
3494
314k
  if (!ReturnValue.isValid()) {
3495
159k
    Builder.CreateRetVoid();
3496
159k
    return;
3497
159k
  }
3498
3499
154k
  llvm::DebugLoc RetDbgLoc;
3500
154k
  llvm::Value *RV = nullptr;
3501
154k
  QualType RetTy = FI.getReturnType();
3502
154k
  const ABIArgInfo &RetAI = FI.getReturnInfo();
3503
3504
154k
  switch (RetAI.getKind()) {
3505
5
  case ABIArgInfo::InAlloca:
3506
    // Aggregrates get evaluated directly into the destination.  Sometimes we
3507
    // need to return the sret value in a register, though.
3508
5
    assert(hasAggregateEvaluationKind(RetTy));
3509
5
    if (RetAI.getInAllocaSRet()) {
3510
5
      llvm::Function::arg_iterator EI = CurFn->arg_end();
3511
5
      --EI;
3512
5
      llvm::Value *ArgStruct = &*EI;
3513
5
      llvm::Value *SRet = Builder.CreateStructGEP(
3514
5
          FI.getArgStruct(), ArgStruct, RetAI.getInAllocaFieldIndex());
3515
5
      llvm::Type *Ty =
3516
5
          cast<llvm::GetElementPtrInst>(SRet)->getResultElementType();
3517
5
      RV = Builder.CreateAlignedLoad(Ty, SRet, getPointerAlign(), "sret");
3518
5
    }
3519
5
    break;
3520
3521
4.14k
  case ABIArgInfo::Indirect: {
3522
4.14k
    auto AI = CurFn->arg_begin();
3523
4.14k
    if (RetAI.isSRetAfterThis())
3524
54
      ++AI;
3525
4.14k
    switch (getEvaluationKind(RetTy)) {
3526
292
    case TEK_Complex: {
3527
292
      ComplexPairTy RT =
3528
292
        EmitLoadOfComplex(MakeAddrLValue(ReturnValue, RetTy), EndLoc);
3529
292
      EmitStoreOfComplex(RT, MakeNaturalAlignAddrLValue(&*AI, RetTy),
3530
292
                         /*isInit*/ true);
3531
292
      break;
3532
0
    }
3533
2.33k
    case TEK_Aggregate:
3534
      // Do nothing; aggregrates get evaluated directly into the destination.
3535
2.33k
      break;
3536
1.51k
    case TEK_Scalar: {
3537
1.51k
      LValueBaseInfo BaseInfo;
3538
1.51k
      TBAAAccessInfo TBAAInfo;
3539
1.51k
      CharUnits Alignment =
3540
1.51k
          CGM.getNaturalTypeAlignment(RetTy, &BaseInfo, &TBAAInfo);
3541
1.51k
      Address ArgAddr(&*AI, ConvertType(RetTy), Alignment);
3542
1.51k
      LValue ArgVal =
3543
1.51k
          LValue::MakeAddr(ArgAddr, RetTy, getContext(), BaseInfo, TBAAInfo);
3544
1.51k
      EmitStoreOfScalar(
3545
1.51k
          Builder.CreateLoad(ReturnValue), ArgVal, /*isInit*/ true);
3546
1.51k
      break;
3547
0
    }
3548
4.14k
    }
3549
4.14k
    break;
3550
4.14k
  }
3551
3552
11.2k
  case ABIArgInfo::Extend:
3553
149k
  case ABIArgInfo::Direct:
3554
149k
    if (RetAI.getCoerceToType() == ConvertType(RetTy) &&
3555
149k
        
RetAI.getDirectOffset() == 0144k
) {
3556
      // The internal return value temp always will have pointer-to-return-type
3557
      // type, just do a load.
3558
3559
      // If there is a dominating store to ReturnValue, we can elide
3560
      // the load, zap the store, and usually zap the alloca.
3561
144k
      if (llvm::StoreInst *SI =
3562
144k
              findDominatingStoreToReturnValue(*this)) {
3563
        // Reuse the debug location from the store unless there is
3564
        // cleanup code to be emitted between the store and return
3565
        // instruction.
3566
138k
        if (EmitRetDbgLoc && 
!AutoreleaseResult137k
)
3567
137k
          RetDbgLoc = SI->getDebugLoc();
3568
        // Get the stored value and nuke the now-dead store.
3569
138k
        RV = SI->getValueOperand();
3570
138k
        SI->eraseFromParent();
3571
3572
      // Otherwise, we have to do a simple load.
3573
138k
      } else {
3574
5.38k
        RV = Builder.CreateLoad(ReturnValue);
3575
5.38k
      }
3576
144k
    } else {
3577
      // If the value is offset in memory, apply the offset now.
3578
5.79k
      Address V = emitAddressAtOffset(*this, ReturnValue, RetAI);
3579
3580
5.79k
      RV = CreateCoercedLoad(V, RetAI.getCoerceToType(), *this);
3581
5.79k
    }
3582
3583
    // In ARC, end functions that return a retainable type with a call
3584
    // to objc_autoreleaseReturnValue.
3585
149k
    if (AutoreleaseResult) {
3586
122
#ifndef NDEBUG
3587
      // Type::isObjCRetainabletype has to be called on a QualType that hasn't
3588
      // been stripped of the typedefs, so we cannot use RetTy here. Get the
3589
      // original return type of FunctionDecl, CurCodeDecl, and BlockDecl from
3590
      // CurCodeDecl or BlockInfo.
3591
122
      QualType RT;
3592
3593
122
      if (auto *FD = dyn_cast<FunctionDecl>(CurCodeDecl))
3594
74
        RT = FD->getReturnType();
3595
48
      else if (auto *MD = dyn_cast<ObjCMethodDecl>(CurCodeDecl))
3596
19
        RT = MD->getReturnType();
3597
29
      else if (isa<BlockDecl>(CurCodeDecl))
3598
29
        RT = BlockInfo->BlockExpression->getFunctionType()->getReturnType();
3599
0
      else
3600
0
        llvm_unreachable("Unexpected function/method type");
3601
3602
122
      assert(getLangOpts().ObjCAutoRefCount &&
3603
122
             !FI.isReturnsRetained() &&
3604
122
             RT->isObjCRetainableType());
3605
0
#endif
3606
0
      RV = emitAutoreleaseOfResult(*this, RV);
3607
122
    }
3608
3609
0
    break;
3610
3611
409
  case ABIArgInfo::Ignore:
3612
409
    break;
3613
3614
541
  case ABIArgInfo::CoerceAndExpand: {
3615
541
    auto coercionType = RetAI.getCoerceAndExpandType();
3616
3617
    // Load all of the coerced elements out into results.
3618
541
    llvm::SmallVector<llvm::Value*, 4> results;
3619
541
    Address addr = Builder.CreateElementBitCast(ReturnValue, coercionType);
3620
1.68k
    for (unsigned i = 0, e = coercionType->getNumElements(); i != e; 
++i1.14k
) {
3621
1.14k
      auto coercedEltType = coercionType->getElementType(i);
3622
1.14k
      if (ABIArgInfo::isPaddingForCoerceAndExpand(coercedEltType))
3623
7
        continue;
3624
3625
1.14k
      auto eltAddr = Builder.CreateStructGEP(addr, i);
3626
1.14k
      auto elt = Builder.CreateLoad(eltAddr);
3627
1.14k
      results.push_back(elt);
3628
1.14k
    }
3629
3630
    // If we have one result, it's the single direct result type.
3631
541
    if (results.size() == 1) {
3632
180
      RV = results[0];
3633
3634
    // Otherwise, we need to make a first-class aggregate.
3635
361
    } else {
3636
      // Construct a return type that lacks padding elements.
3637
361
      llvm::Type *returnType = RetAI.getUnpaddedCoerceAndExpandType();
3638
3639
361
      RV = llvm::UndefValue::get(returnType);
3640
1.32k
      for (unsigned i = 0, e = results.size(); i != e; 
++i960
) {
3641
960
        RV = Builder.CreateInsertValue(RV, results[i], i);
3642
960
      }
3643
361
    }
3644
541
    break;
3645
11.2k
  }
3646
0
  case ABIArgInfo::Expand:
3647
0
  case ABIArgInfo::IndirectAliased:
3648
0
    llvm_unreachable("Invalid ABI kind for return argument");
3649
154k
  }
3650
3651
154k
  llvm::Instruction *Ret;
3652
154k
  if (RV) {
3653
150k
    if (CurFuncDecl && 
CurFuncDecl->hasAttr<CmseNSEntryAttr>()149k
) {
3654
      // For certain return types, clear padding bits, as they may reveal
3655
      // sensitive information.
3656
      // Small struct/union types are passed as integers.
3657
90
      auto *ITy = dyn_cast<llvm::IntegerType>(RV->getType());
3658
90
      if (ITy != nullptr && 
isa<RecordType>(RetTy.getCanonicalType())84
)
3659
84
        RV = EmitCMSEClearRecord(RV, ITy, RetTy);
3660
90
    }
3661
150k
    EmitReturnValueCheck(RV);
3662
150k
    Ret = Builder.CreateRet(RV);
3663
150k
  } else {
3664
4.55k
    Ret = Builder.CreateRetVoid();
3665
4.55k
  }
3666
3667
154k
  if (RetDbgLoc)
3668
44.1k
    Ret->setDebugLoc(std::move(RetDbgLoc));
3669
154k
}
3670
3671
150k
void CodeGenFunction::EmitReturnValueCheck(llvm::Value *RV) {
3672
  // A current decl may not be available when emitting vtable thunks.
3673
150k
  if (!CurCodeDecl)
3674
1.27k
    return;
3675
3676
  // If the return block isn't reachable, neither is this check, so don't emit
3677
  // it.
3678
149k
  if (ReturnBlock.isValid() && 
ReturnBlock.getBlock()->use_empty()1.93k
)
3679
233
    return;
3680
3681
148k
  ReturnsNonNullAttr *RetNNAttr = nullptr;
3682
148k
  if (SanOpts.has(SanitizerKind::ReturnsNonnullAttribute))
3683
45
    RetNNAttr = CurCodeDecl->getAttr<ReturnsNonNullAttr>();
3684
3685
148k
  if (!RetNNAttr && 
!requiresReturnValueNullabilityCheck()148k
)
3686
148k
    return;
3687
3688
  // Prefer the returns_nonnull attribute if it's present.
3689
18
  SourceLocation AttrLoc;
3690
18
  SanitizerMask CheckKind;
3691
18
  SanitizerHandler Handler;
3692
18
  if (RetNNAttr) {
3693
5
    assert(!requiresReturnValueNullabilityCheck() &&
3694
5
           "Cannot check nullability and the nonnull attribute");
3695
0
    AttrLoc = RetNNAttr->getLocation();
3696
5
    CheckKind = SanitizerKind::ReturnsNonnullAttribute;
3697
5
    Handler = SanitizerHandler::NonnullReturn;
3698
13
  } else {
3699
13
    if (auto *DD = dyn_cast<DeclaratorDecl>(CurCodeDecl))
3700
7
      if (auto *TSI = DD->getTypeSourceInfo())
3701
7
        if (auto FTL = TSI->getTypeLoc().getAsAdjusted<FunctionTypeLoc>())
3702
7
          AttrLoc = FTL.getReturnLoc().findNullabilityLoc();
3703
13
    CheckKind = SanitizerKind::NullabilityReturn;
3704
13
    Handler = SanitizerHandler::NullabilityReturn;
3705
13
  }
3706
3707
0
  SanitizerScope SanScope(this);
3708
3709
  // Make sure the "return" source location is valid. If we're checking a
3710
  // nullability annotation, make sure the preconditions for the check are met.
3711
18
  llvm::BasicBlock *Check = createBasicBlock("nullcheck");
3712
18
  llvm::BasicBlock *NoCheck = createBasicBlock("no.nullcheck");
3713
18
  llvm::Value *SLocPtr = Builder.CreateLoad(ReturnLocation, "return.sloc.load");
3714
18
  llvm::Value *CanNullCheck = Builder.CreateIsNotNull(SLocPtr);
3715
18
  if (requiresReturnValueNullabilityCheck())
3716
13
    CanNullCheck =
3717
13
        Builder.CreateAnd(CanNullCheck, RetValNullabilityPrecondition);
3718
18
  Builder.CreateCondBr(CanNullCheck, Check, NoCheck);
3719
18
  EmitBlock(Check);
3720
3721
  // Now do the null check.
3722
18
  llvm::Value *Cond = Builder.CreateIsNotNull(RV);
3723
18
  llvm::Constant *StaticData[] = {EmitCheckSourceLocation(AttrLoc)};
3724
18
  llvm::Value *DynamicData[] = {SLocPtr};
3725
18
  EmitCheck(std::make_pair(Cond, CheckKind), Handler, StaticData, DynamicData);
3726
3727
18
  EmitBlock(NoCheck);
3728
3729
18
#ifndef NDEBUG
3730
  // The return location should not be used after the check has been emitted.
3731
18
  ReturnLocation = Address::invalid();
3732
18
#endif
3733
18
}
3734
3735
24.6k
static bool isInAllocaArgument(CGCXXABI &ABI, QualType type) {
3736
24.6k
  const CXXRecordDecl *RD = type->getAsCXXRecordDecl();
3737
24.6k
  return RD && 
ABI.getRecordArgABI(RD) == CGCXXABI::RAA_DirectInMemory2.62k
;
3738
24.6k
}
3739
3740
static AggValueSlot createPlaceholderSlot(CodeGenFunction &CGF,
3741
106
                                          QualType Ty) {
3742
  // FIXME: Generate IR in one pass, rather than going back and fixing up these
3743
  // placeholders.
3744
106
  llvm::Type *IRTy = CGF.ConvertTypeForMem(Ty);
3745
106
  llvm::Type *IRPtrTy = IRTy->getPointerTo();
3746
106
  llvm::Value *Placeholder = llvm::UndefValue::get(IRPtrTy->getPointerTo());
3747
3748
  // FIXME: When we generate this IR in one pass, we shouldn't need
3749
  // this win32-specific alignment hack.
3750
106
  CharUnits Align = CharUnits::fromQuantity(4);
3751
106
  Placeholder = CGF.Builder.CreateAlignedLoad(IRPtrTy, Placeholder, Align);
3752
3753
106
  return AggValueSlot::forAddr(Address(Placeholder, IRTy, Align),
3754
106
                               Ty.getQualifiers(),
3755
106
                               AggValueSlot::IsNotDestructed,
3756
106
                               AggValueSlot::DoesNotNeedGCBarriers,
3757
106
                               AggValueSlot::IsNotAliased,
3758
106
                               AggValueSlot::DoesNotOverlap);
3759
106
}
3760
3761
void CodeGenFunction::EmitDelegateCallArg(CallArgList &args,
3762
                                          const VarDecl *param,
3763
17.9k
                                          SourceLocation loc) {
3764
  // StartFunction converted the ABI-lowered parameter(s) into a
3765
  // local alloca.  We need to turn that into an r-value suitable
3766
  // for EmitCall.
3767
17.9k
  Address local = GetAddrOfLocalVar(param);
3768
3769
17.9k
  QualType type = param->getType();
3770
3771
17.9k
  if (isInAllocaArgument(CGM.getCXXABI(), type)) {
3772
1
    CGM.ErrorUnsupported(param, "forwarded non-trivially copyable parameter");
3773
1
  }
3774
3775
  // GetAddrOfLocalVar returns a pointer-to-pointer for references,
3776
  // but the argument needs to be the original pointer.
3777
17.9k
  if (type->isReferenceType()) {
3778
9.27k
    args.add(RValue::get(Builder.CreateLoad(local)), type);
3779
3780
  // In ARC, move out of consumed arguments so that the release cleanup
3781
  // entered by StartFunction doesn't cause an over-release.  This isn't
3782
  // optimal -O0 code generation, but it should get cleaned up when
3783
  // optimization is enabled.  This also assumes that delegate calls are
3784
  // performed exactly once for a set of arguments, but that should be safe.
3785
9.27k
  } else 
if (8.67k
getLangOpts().ObjCAutoRefCount8.67k
&&
3786
8.67k
             
param->hasAttr<NSConsumedAttr>()9
&&
3787
8.67k
             
type->isObjCRetainableType()2
) {
3788
2
    llvm::Value *ptr = Builder.CreateLoad(local);
3789
2
    auto null =
3790
2
      llvm::ConstantPointerNull::get(cast<llvm::PointerType>(ptr->getType()));
3791
2
    Builder.CreateStore(null, local);
3792
2
    args.add(RValue::get(ptr), type);
3793
3794
  // For the most part, we just need to load the alloca, except that
3795
  // aggregate r-values are actually pointers to temporaries.
3796
8.66k
  } else {
3797
8.66k
    args.add(convertTempToRValue(local, type, loc), type);
3798
8.66k
  }
3799
3800
  // Deactivate the cleanup for the callee-destructed param that was pushed.
3801
17.9k
  if (type->isRecordType() && 
!CurFuncIsThunk2.29k
&&
3802
17.9k
      
type->castAs<RecordType>()->getDecl()->isParamDestroyedInCallee()2.26k
&&
3803
17.9k
      
param->needsDestruction(getContext())8
) {
3804
6
    EHScopeStack::stable_iterator cleanup =
3805
6
        CalleeDestructedParamCleanups.lookup(cast<ParmVarDecl>(param));
3806
6
    assert(cleanup.isValid() &&
3807
6
           "cleanup for callee-destructed param not recorded");
3808
    // This unreachable is a temporary marker which will be removed later.
3809
0
    llvm::Instruction *isActive = Builder.CreateUnreachable();
3810
6
    args.addArgCleanupDeactivation(cleanup, isActive);
3811
6
  }
3812
17.9k
}
3813
3814
54
static bool isProvablyNull(llvm::Value *addr) {
3815
54
  return isa<llvm::ConstantPointerNull>(addr);
3816
54
}
3817
3818
/// Emit the actual writing-back of a writeback.
3819
static void emitWriteback(CodeGenFunction &CGF,
3820
27
                          const CallArgList::Writeback &writeback) {
3821
27
  const LValue &srcLV = writeback.Source;
3822
27
  Address srcAddr = srcLV.getAddress(CGF);
3823
27
  assert(!isProvablyNull(srcAddr.getPointer()) &&
3824
27
         "shouldn't have writeback for provably null argument");
3825
3826
0
  llvm::BasicBlock *contBB = nullptr;
3827
3828
  // If the argument wasn't provably non-null, we need to null check
3829
  // before doing the store.
3830
27
  bool provablyNonNull = llvm::isKnownNonZero(srcAddr.getPointer(),
3831
27
                                              CGF.CGM.getDataLayout());
3832
27
  if (!provablyNonNull) {
3833
5
    llvm::BasicBlock *writebackBB = CGF.createBasicBlock("icr.writeback");
3834
5
    contBB = CGF.createBasicBlock("icr.done");
3835
3836
5
    llvm::Value *isNull =
3837
5
      CGF.Builder.CreateIsNull(srcAddr.getPointer(), "icr.isnull");
3838
5
    CGF.Builder.CreateCondBr(isNull, contBB, writebackBB);
3839
5
    CGF.EmitBlock(writebackBB);
3840
5
  }
3841
3842
  // Load the value to writeback.
3843
27
  llvm::Value *value = CGF.Builder.CreateLoad(writeback.Temporary);
3844
3845
  // Cast it back, in case we're writing an id to a Foo* or something.
3846
27
  value = CGF.Builder.CreateBitCast(value, srcAddr.getElementType(),
3847
27
                                    "icr.writeback-cast");
3848
3849
  // Perform the writeback.
3850
3851
  // If we have a "to use" value, it's something we need to emit a use
3852
  // of.  This has to be carefully threaded in: if it's done after the
3853
  // release it's potentially undefined behavior (and the optimizer
3854
  // will ignore it), and if it happens before the retain then the
3855
  // optimizer could move the release there.
3856
27
  if (writeback.ToUse) {
3857
8
    assert(srcLV.getObjCLifetime() == Qualifiers::OCL_Strong);
3858
3859
    // Retain the new value.  No need to block-copy here:  the block's
3860
    // being passed up the stack.
3861
0
    value = CGF.EmitARCRetainNonBlock(value);
3862
3863
    // Emit the intrinsic use here.
3864
8
    CGF.EmitARCIntrinsicUse(writeback.ToUse);
3865
3866
    // Load the old value (primitively).
3867
8
    llvm::Value *oldValue = CGF.EmitLoadOfScalar(srcLV, SourceLocation());
3868
3869
    // Put the new value in place (primitively).
3870
8
    CGF.EmitStoreOfScalar(value, srcLV, /*init*/ false);
3871
3872
    // Release the old value.
3873
8
    CGF.EmitARCRelease(oldValue, srcLV.isARCPreciseLifetime());
3874
3875
  // Otherwise, we can just do a normal lvalue store.
3876
19
  } else {
3877
19
    CGF.EmitStoreThroughLValue(RValue::get(value), srcLV);
3878
19
  }
3879
3880
  // Jump to the continuation block.
3881
27
  if (!provablyNonNull)
3882
5
    CGF.EmitBlock(contBB);
3883
27
}
3884
3885
static void emitWritebacks(CodeGenFunction &CGF,
3886
27
                           const CallArgList &args) {
3887
27
  for (const auto &I : args.writebacks())
3888
27
    emitWriteback(CGF, I);
3889
27
}
3890
3891
static void deactivateArgCleanupsBeforeCall(CodeGenFunction &CGF,
3892
149
                                            const CallArgList &CallArgs) {
3893
149
  ArrayRef<CallArgList::CallArgCleanup> Cleanups =
3894
149
    CallArgs.getCleanupsToDeactivate();
3895
  // Iterate in reverse to increase the likelihood of popping the cleanup.
3896
192
  for (const auto &I : llvm::reverse(Cleanups)) {
3897
192
    CGF.DeactivateCleanupBlock(I.Cleanup, I.IsActiveIP);
3898
192
    I.IsActiveIP->eraseFromParent();
3899
192
  }
3900
149
}
3901
3902
27
static const Expr *maybeGetUnaryAddrOfOperand(const Expr *E) {
3903
27
  if (const UnaryOperator *uop = dyn_cast<UnaryOperator>(E->IgnoreParens()))
3904
22
    if (uop->getOpcode() == UO_AddrOf)
3905
22
      return uop->getSubExpr();
3906
5
  return nullptr;
3907
27
}
3908
3909
/// Emit an argument that's being passed call-by-writeback.  That is,
3910
/// we are passing the address of an __autoreleased temporary; it
3911
/// might be copy-initialized with the current value of the given
3912
/// address, but it will definitely be copied out of after the call.
3913
static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args,
3914
27
                             const ObjCIndirectCopyRestoreExpr *CRE) {
3915
27
  LValue srcLV;
3916
3917
  // Make an optimistic effort to emit the address as an l-value.
3918
  // This can fail if the argument expression is more complicated.
3919
27
  if (const Expr *lvExpr = maybeGetUnaryAddrOfOperand(CRE->getSubExpr())) {
3920
22
    srcLV = CGF.EmitLValue(lvExpr);
3921
3922
  // Otherwise, just emit it as a scalar.
3923
22
  } else {
3924
5
    Address srcAddr = CGF.EmitPointerWithAlignment(CRE->getSubExpr());
3925
3926
5
    QualType srcAddrType =
3927
5
      CRE->getSubExpr()->getType()->castAs<PointerType>()->getPointeeType();
3928
5
    srcLV = CGF.MakeAddrLValue(srcAddr, srcAddrType);
3929
5
  }
3930
27
  Address srcAddr = srcLV.getAddress(CGF);
3931
3932
  // The dest and src types don't necessarily match in LLVM terms
3933
  // because of the crazy ObjC compatibility rules.
3934
3935
27
  llvm::PointerType *destType =
3936
27
      cast<llvm::PointerType>(CGF.ConvertType(CRE->getType()));
3937
27
  llvm::Type *destElemType =
3938
27
      CGF.ConvertTypeForMem(CRE->getType()->getPointeeType());
3939
3940
  // If the address is a constant null, just pass the appropriate null.
3941
27
  if (isProvablyNull(srcAddr.getPointer())) {
3942
0
    args.add(RValue::get(llvm::ConstantPointerNull::get(destType)),
3943
0
             CRE->getType());
3944
0
    return;
3945
0
  }
3946
3947
  // Create the temporary.
3948
27
  Address temp =
3949
27
      CGF.CreateTempAlloca(destElemType, CGF.getPointerAlign(), "icr.temp");
3950
  // Loading an l-value can introduce a cleanup if the l-value is __weak,
3951
  // and that cleanup will be conditional if we can't prove that the l-value
3952
  // isn't null, so we need to register a dominating point so that the cleanups
3953
  // system will make valid IR.
3954
27
  CodeGenFunction::ConditionalEvaluation condEval(CGF);
3955
3956
  // Zero-initialize it if we're not doing a copy-initialization.
3957
27
  bool shouldCopy = CRE->shouldCopy();
3958
27
  if (!shouldCopy) {
3959
4
    llvm::Value *null =
3960
4
        llvm::ConstantPointerNull::get(cast<llvm::PointerType>(destElemType));
3961
4
    CGF.Builder.CreateStore(null, temp);
3962
4
  }
3963
3964
27
  llvm::BasicBlock *contBB = nullptr;
3965
27
  llvm::BasicBlock *originBB = nullptr;
3966
3967
  // If the address is *not* known to be non-null, we need to switch.
3968
27
  llvm::Value *finalArgument;
3969
3970
27
  bool provablyNonNull = llvm::isKnownNonZero(srcAddr.getPointer(),
3971
27
                                              CGF.CGM.getDataLayout());
3972
27
  if (provablyNonNull) {
3973
22
    finalArgument = temp.getPointer();
3974
22
  } else {
3975
5
    llvm::Value *isNull =
3976
5
      CGF.Builder.CreateIsNull(srcAddr.getPointer(), "icr.isnull");
3977
3978
5
    finalArgument = CGF.Builder.CreateSelect(isNull,
3979
5
                                   llvm::ConstantPointerNull::get(destType),
3980
5
                                             temp.getPointer(), "icr.argument");
3981
3982
    // If we need to copy, then the load has to be conditional, which
3983
    // means we need control flow.
3984
5
    if (shouldCopy) {
3985
5
      originBB = CGF.Builder.GetInsertBlock();
3986
5
      contBB = CGF.createBasicBlock("icr.cont");
3987
5
      llvm::BasicBlock *copyBB = CGF.createBasicBlock("icr.copy");
3988
5
      CGF.Builder.CreateCondBr(isNull, contBB, copyBB);
3989
5
      CGF.EmitBlock(copyBB);
3990
5
      condEval.begin(CGF);
3991
5
    }
3992
5
  }
3993
3994
27
  llvm::Value *valueToUse = nullptr;
3995
3996
  // Perform a copy if necessary.
3997
27
  if (shouldCopy) {
3998
23
    RValue srcRV = CGF.EmitLoadOfLValue(srcLV, SourceLocation());
3999
23
    assert(srcRV.isScalar());
4000
4001
0
    llvm::Value *src = srcRV.getScalarVal();
4002
23
    src = CGF.Builder.CreateBitCast(src, destElemType, "icr.cast");
4003
4004
    // Use an ordinary store, not a store-to-lvalue.
4005
23
    CGF.Builder.CreateStore(src, temp);
4006
4007
    // If optimization is enabled, and the value was held in a
4008
    // __strong variable, we need to tell the optimizer that this
4009
    // value has to stay alive until we're doing the store back.
4010
    // This is because the temporary is effectively unretained,
4011
    // and so otherwise we can violate the high-level semantics.
4012
23
    if (CGF.CGM.getCodeGenOpts().OptimizationLevel != 0 &&
4013
23
        
srcLV.getObjCLifetime() == Qualifiers::OCL_Strong10
) {
4014
8
      valueToUse = src;
4015
8
    }
4016
23
  }
4017
4018
  // Finish the control flow if we needed it.
4019
27
  if (shouldCopy && 
!provablyNonNull23
) {
4020
5
    llvm::BasicBlock *copyBB = CGF.Builder.GetInsertBlock();
4021
5
    CGF.EmitBlock(contBB);
4022
4023
    // Make a phi for the value to intrinsically use.
4024
5
    if (valueToUse) {
4025
2
      llvm::PHINode *phiToUse = CGF.Builder.CreatePHI(valueToUse->getType(), 2,
4026
2
                                                      "icr.to-use");
4027
2
      phiToUse->addIncoming(valueToUse, copyBB);
4028
2
      phiToUse->addIncoming(llvm::UndefValue::get(valueToUse->getType()),
4029
2
                            originBB);
4030
2
      valueToUse = phiToUse;
4031
2
    }
4032
4033
5
    condEval.end(CGF);
4034
5
  }
4035
4036
27
  args.addWriteback(srcLV, temp, valueToUse);
4037
27
  args.add(RValue::get(finalArgument), CRE->getType());
4038
27
}
4039
4040
73
void CallArgList::allocateArgumentMemory(CodeGenFunction &CGF) {
4041
73
  assert(!StackBase);
4042
4043
  // Save the stack.
4044
0
  llvm::Function *F = CGF.CGM.getIntrinsic(llvm::Intrinsic::stacksave);
4045
73
  StackBase = CGF.Builder.CreateCall(F, {}, "inalloca.save");
4046
73
}
4047
4048
364k
void CallArgList::freeArgumentMemory(CodeGenFunction &CGF) const {
4049
364k
  if (StackBase) {
4050
    // Restore the stack after the call.
4051
73
    llvm::Function *F = CGF.CGM.getIntrinsic(llvm::Intrinsic::stackrestore);
4052
73
    CGF.Builder.CreateCall(F, StackBase);
4053
73
  }
4054
364k
}
4055
4056
void CodeGenFunction::EmitNonNullArgCheck(RValue RV, QualType ArgType,
4057
                                          SourceLocation ArgLoc,
4058
                                          AbstractCallee AC,
4059
402k
                                          unsigned ParmNum) {
4060
402k
  if (!AC.getDecl() || 
!(383k
SanOpts.has(SanitizerKind::NonnullAttribute)383k
||
4061
383k
                         
SanOpts.has(SanitizerKind::NullabilityArg)383k
))
4062
402k
    return;
4063
4064
  // The param decl may be missing in a variadic function.
4065
77
  auto PVD = ParmNum < AC.getNumParams() ? 
AC.getParamDecl(ParmNum)66
:
nullptr11
;
4066
77
  unsigned ArgNo = PVD ? 
PVD->getFunctionScopeIndex()66
:
ParmNum11
;
4067
4068
  // Prefer the nonnull attribute if it's present.
4069
77
  const NonNullAttr *NNAttr = nullptr;
4070
77
  if (SanOpts.has(SanitizerKind::NonnullAttribute))
4071
25
    NNAttr = getNonNullAttr(AC.getDecl(), PVD, ArgType, ArgNo);
4072
4073
77
  bool CanCheckNullability = false;
4074
77
  if (SanOpts.has(SanitizerKind::NullabilityArg) && 
!NNAttr53
&&
PVD52
) {
4075
44
    auto Nullability = PVD->getType()->getNullability(getContext());
4076
44
    CanCheckNullability = Nullability &&
4077
44
                          
*Nullability == NullabilityKind::NonNull20
&&
4078
44
                          
PVD->getTypeSourceInfo()18
;
4079
44
  }
4080
4081
77
  if (!NNAttr && 
!CanCheckNullability61
)
4082
43
    return;
4083
4084
34
  SourceLocation AttrLoc;
4085
34
  SanitizerMask CheckKind;
4086
34
  SanitizerHandler Handler;
4087
34
  if (NNAttr) {
4088
16
    AttrLoc = NNAttr->getLocation();
4089
16
    CheckKind = SanitizerKind::NonnullAttribute;
4090
16
    Handler = SanitizerHandler::NonnullArg;
4091
18
  } else {
4092
18
    AttrLoc = PVD->getTypeSourceInfo()->getTypeLoc().findNullabilityLoc();
4093
18
    CheckKind = SanitizerKind::NullabilityArg;
4094
18
    Handler = SanitizerHandler::NullabilityArg;
4095
18
  }
4096
4097
34
  SanitizerScope SanScope(this);
4098
34
  llvm::Value *Cond = EmitNonNullRValueCheck(RV, ArgType);
4099
34
  llvm::Constant *StaticData[] = {
4100
34
      EmitCheckSourceLocation(ArgLoc), EmitCheckSourceLocation(AttrLoc),
4101
34
      llvm::ConstantInt::get(Int32Ty, ArgNo + 1),
4102
34
  };
4103
34
  EmitCheck(std::make_pair(Cond, CheckKind), Handler, StaticData, None);
4104
34
}
4105
4106
// Check if the call is going to use the inalloca convention. This needs to
4107
// agree with CGFunctionInfo::usesInAlloca. The CGFunctionInfo is arranged
4108
// later, so we can't check it directly.
4109
static bool hasInAllocaArgs(CodeGenModule &CGM, CallingConv ExplicitCC,
4110
316k
                            ArrayRef<QualType> ArgTypes) {
4111
  // The Swift calling conventions don't go through the target-specific
4112
  // argument classification, they never use inalloca.
4113
  // TODO: Consider limiting inalloca use to only calling conventions supported
4114
  // by MSVC.
4115
316k
  if (ExplicitCC == CC_Swift || 
ExplicitCC == CC_SwiftAsync315k
)
4116
1.28k
    return false;
4117
315k
  if (!CGM.getTarget().getCXXABI().isMicrosoft())
4118
308k
    return false;
4119
7.34k
  return llvm::any_of(ArgTypes, [&](QualType Ty) {
4120
6.72k
    return isInAllocaArgument(CGM.getCXXABI(), Ty);
4121
6.72k
  });
4122
315k
}
4123
4124
#ifndef NDEBUG
4125
// Determine whether the given argument is an Objective-C method
4126
// that may have type parameters in its signature.
4127
11.3k
static bool isObjCMethodWithTypeParams(const ObjCMethodDecl *method) {
4128
11.3k
  const DeclContext *dc = method->getDeclContext();
4129
11.3k
  if (const ObjCInterfaceDecl *classDecl = dyn_cast<ObjCInterfaceDecl>(dc)) {
4130
7.25k
    return classDecl->getTypeParamListAsWritten();
4131
7.25k
  }
4132
4133
4.07k
  if (const ObjCCategoryDecl *catDecl = dyn_cast<ObjCCategoryDecl>(dc)) {
4134
3.37k
    return catDecl->getTypeParamList();
4135
3.37k
  }
4136
4137
700
  return false;
4138
4.07k
}
4139
#endif
4140
4141
/// EmitCallArgs - Emit call arguments for a function.
4142
void CodeGenFunction::EmitCallArgs(
4143
    CallArgList &Args, PrototypeWrapper Prototype,
4144
    llvm::iterator_range<CallExpr::const_arg_iterator> ArgRange,
4145
316k
    AbstractCallee AC, unsigned ParamsToSkip, EvaluationOrder Order) {
4146
316k
  SmallVector<QualType, 16> ArgTypes;
4147
4148
316k
  assert((ParamsToSkip == 0 || Prototype.P) &&
4149
316k
         "Can't skip parameters if type info is not provided");
4150
4151
  // This variable only captures *explicitly* written conventions, not those
4152
  // applied by default via command line flags or target defaults, such as
4153
  // thiscall, aapcs, stdcall via -mrtd, etc. Computing that correctly would
4154
  // require knowing if this is a C++ instance method or being able to see
4155
  // unprototyped FunctionTypes.
4156
0
  CallingConv ExplicitCC = CC_C;
4157
4158
  // First, if a prototype was provided, use those argument types.
4159
316k
  bool IsVariadic = false;
4160
316k
  if (Prototype.P) {
4161
315k
    const auto *MD = Prototype.P.dyn_cast<const ObjCMethodDecl *>();
4162
315k
    if (MD) {
4163
11.3k
      IsVariadic = MD->isVariadic();
4164
11.3k
      ExplicitCC = getCallingConventionForDecl(
4165
11.3k
          MD, CGM.getTarget().getTriple().isOSWindows());
4166
11.3k
      ArgTypes.assign(MD->param_type_begin() + ParamsToSkip,
4167
11.3k
                      MD->param_type_end());
4168
304k
    } else {
4169
304k
      const auto *FPT = Prototype.P.get<const FunctionProtoType *>();
4170
304k
      IsVariadic = FPT->isVariadic();
4171
304k
      ExplicitCC = FPT->getExtInfo().getCC();
4172
304k
      ArgTypes.assign(FPT->param_type_begin() + ParamsToSkip,
4173
304k
                      FPT->param_type_end());
4174
304k
    }
4175
4176
315k
#ifndef NDEBUG
4177
    // Check that the prototyped types match the argument expression types.
4178
315k
    bool isGenericMethod = MD && 
isObjCMethodWithTypeParams(MD)11.3k
;
4179
315k
    CallExpr::const_arg_iterator Arg = ArgRange.begin();
4180
315k
    for (QualType Ty : ArgTypes) {
4181
313k
      assert(Arg != ArgRange.end() && "Running over edge of argument list!");
4182
0
      assert(
4183
313k
          (isGenericMethod || Ty->isVariablyModifiedType() ||
4184
313k
           Ty.getNonReferenceType()->isObjCRetainableType() ||
4185
313k
           getContext()
4186
313k
                   .getCanonicalType(Ty.getNonReferenceType())
4187
313k
                   .getTypePtr() ==
4188
313k
               getContext().getCanonicalType((*Arg)->getType()).getTypePtr()) &&
4189
313k
          "type mismatch in call argument!");
4190
0
      ++Arg;
4191
313k
    }
4192
4193
    // Either we've emitted all the call args, or we have a call to variadic
4194
    // function.
4195
315k
    assert((Arg == ArgRange.end() || IsVariadic) &&
4196
315k
           "Extra arguments in non-variadic function!");
4197
315k
#endif
4198
315k
  }
4199
4200
  // If we still have any arguments, emit them using the type of the argument.
4201
0
  for (auto *A : llvm::drop_begin(ArgRange, ArgTypes.size()))
4202
89.7k
    ArgTypes.push_back(IsVariadic ? 
getVarArgType(A)88.9k
:
A->getType()803
);
4203
316k
  assert((int)ArgTypes.size() == (ArgRange.end() - ArgRange.begin()));
4204
4205
  // We must evaluate arguments from right to left in the MS C++ ABI,
4206
  // because arguments are destroyed left to right in the callee. As a special
4207
  // case, there are certain language constructs that require left-to-right
4208
  // evaluation, and in those cases we consider the evaluation order requirement
4209
  // to trump the "destruction order is reverse construction order" guarantee.
4210
0
  bool LeftToRight =
4211
316k
      CGM.getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()
4212
316k
          ? 
Order == EvaluationOrder::ForceLeftToRight7.36k
4213
316k
          : 
Order != EvaluationOrder::ForceRightToLeft309k
;
4214
4215
316k
  auto MaybeEmitImplicitObjectSize = [&](unsigned I, const Expr *Arg,
4216
402k
                                         RValue EmittedArg) {
4217
402k
    if (!AC.hasFunctionDecl() || 
I >= AC.getNumParams()371k
)
4218
117k
      return;
4219
284k
    auto *PS = AC.getParamDecl(I)->getAttr<PassObjectSizeAttr>();
4220
284k
    if (PS == nullptr)
4221
284k
      return;
4222
4223
107
    const auto &Context = getContext();
4224
107
    auto SizeTy = Context.getSizeType();
4225
107
    auto T = Builder.getIntNTy(Context.getTypeSize(SizeTy));
4226
107
    assert(EmittedArg.getScalarVal() && "We emitted nothing for the arg?");
4227
0
    llvm::Value *V = evaluateOrEmitBuiltinObjectSize(Arg, PS->getType(), T,
4228
107
                                                     EmittedArg.getScalarVal(),
4229
107
                                                     PS->isDynamic());
4230
107
    Args.add(RValue::get(V), SizeTy);
4231
    // If we're emitting args in reverse, be sure to do so with
4232
    // pass_object_size, as well.
4233
107
    if (!LeftToRight)
4234
9
      std::swap(Args.back(), *(&Args.back() - 1));
4235
107
  };
4236
4237
  // Insert a stack save if we're going to need any inalloca args.
4238
316k
  if (hasInAllocaArgs(CGM, ExplicitCC, ArgTypes)) {
4239
73
    assert(getTarget().getTriple().getArch() == llvm::Triple::x86 &&
4240
73
           "inalloca only supported on x86");
4241
0
    Args.allocateArgumentMemory(*this);
4242
73
  }
4243
4244
  // Evaluate each argument in the appropriate order.
4245
0
  size_t CallArgsStart = Args.size();
4246
719k
  for (unsigned I = 0, E = ArgTypes.size(); I != E; 
++I403k
) {
4247
403k
    unsigned Idx = LeftToRight ? 
I395k
:
E - I - 17.67k
;
4248
403k
    CallExpr::const_arg_iterator Arg = ArgRange.begin() + Idx;
4249
403k
    unsigned InitialArgSize = Args.size();
4250
    // If *Arg is an ObjCIndirectCopyRestoreExpr, check that either the types of
4251
    // the argument and parameter match or the objc method is parameterized.
4252
403k
    assert((!isa<ObjCIndirectCopyRestoreExpr>(*Arg) ||
4253
403k
            getContext().hasSameUnqualifiedType((*Arg)->getType(),
4254
403k
                                                ArgTypes[Idx]) ||
4255
403k
            (isa<ObjCMethodDecl>(AC.getDecl()) &&
4256
403k
             isObjCMethodWithTypeParams(cast<ObjCMethodDecl>(AC.getDecl())))) &&
4257
403k
           "Argument and parameter types don't match");
4258
0
    EmitCallArg(Args, *Arg, ArgTypes[Idx]);
4259
    // In particular, we depend on it being the last arg in Args, and the
4260
    // objectsize bits depend on there only being one arg if !LeftToRight.
4261
403k
    assert(InitialArgSize + 1 == Args.size() &&
4262
403k
           "The code below depends on only adding one arg per EmitCallArg");
4263
0
    (void)InitialArgSize;
4264
    // Since pointer argument are never emitted as LValue, it is safe to emit
4265
    // non-null argument check for r-value only.
4266
403k
    if (!Args.back().hasLValue()) {
4267
402k
      RValue RVArg = Args.back().getKnownRValue();
4268
402k
      EmitNonNullArgCheck(RVArg, ArgTypes[Idx], (*Arg)->getExprLoc(), AC,
4269
402k
                          ParamsToSkip + Idx);
4270
      // @llvm.objectsize should never have side-effects and shouldn't need
4271
      // destruction/cleanups, so we can safely "emit" it after its arg,
4272
      // regardless of right-to-leftness
4273
402k
      MaybeEmitImplicitObjectSize(Idx, *Arg, RVArg);
4274
402k
    }
4275
403k
  }
4276
4277
316k
  if (!LeftToRight) {
4278
    // Un-reverse the arguments we just evaluated so they match up with the LLVM
4279
    // IR function.
4280
8.23k
    std::reverse(Args.begin() + CallArgsStart, Args.end());
4281
8.23k
  }
4282
316k
}
4283
4284
namespace {
4285
4286
struct DestroyUnpassedArg final : EHScopeStack::Cleanup {
4287
  DestroyUnpassedArg(Address Addr, QualType Ty)
4288
182
      : Addr(Addr), Ty(Ty) {}
4289
4290
  Address Addr;
4291
  QualType Ty;
4292
4293
20
  void Emit(CodeGenFunction &CGF, Flags flags) override {
4294
20
    QualType::DestructionKind DtorKind = Ty.isDestructedType();
4295
20
    if (DtorKind == QualType::DK_cxx_destructor) {
4296
18
      const CXXDestructorDecl *Dtor = Ty->getAsCXXRecordDecl()->getDestructor();
4297
18
      assert(!Dtor->isTrivial());
4298
0
      CGF.EmitCXXDestructorCall(Dtor, Dtor_Complete, /*for vbase*/ false,
4299
18
                                /*Delegating=*/false, Addr, Ty);
4300
18
    } else {
4301
2
      CGF.callCStructDestructor(CGF.MakeAddrLValue(Addr, Ty));
4302
2
    }
4303
20
  }
4304
};
4305
4306
struct DisableDebugLocationUpdates {
4307
  CodeGenFunction &CGF;
4308
  bool disabledDebugInfo;
4309
403k
  DisableDebugLocationUpdates(CodeGenFunction &CGF, const Expr *E) : CGF(CGF) {
4310
403k
    if ((disabledDebugInfo = isa<CXXDefaultArgExpr>(E) && 
CGF.getDebugInfo()4.11k
))
4311
2.80k
      CGF.disableDebugInfo();
4312
403k
  }
4313
403k
  ~DisableDebugLocationUpdates() {
4314
403k
    if (disabledDebugInfo)
4315
2.80k
      CGF.enableDebugInfo();
4316
403k
  }
4317
};
4318
4319
} // end anonymous namespace
4320
4321
233
RValue CallArg::getRValue(CodeGenFunction &CGF) const {
4322
233
  if (!HasLV)
4323
233
    return RV;
4324
0
  LValue Copy = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty), Ty);
4325
0
  CGF.EmitAggregateCopy(Copy, LV, Ty, AggValueSlot::DoesNotOverlap,
4326
0
                        LV.isVolatile());
4327
0
  IsUsed = true;
4328
0
  return RValue::getAggregate(Copy.getAddress(CGF));
4329
233
}
4330
4331
1.21k
void CallArg::copyInto(CodeGenFunction &CGF, Address Addr) const {
4332
1.21k
  LValue Dst = CGF.MakeAddrLValue(Addr, Ty);
4333
1.21k
  if (!HasLV && 
RV.isScalar()1.15k
)
4334
818
    CGF.EmitStoreOfScalar(RV.getScalarVal(), Dst, /*isInit=*/true);
4335
401
  else if (!HasLV && 
RV.isComplex()338
)
4336
338
    CGF.EmitStoreOfComplex(RV.getComplexVal(), Dst, /*init=*/true);
4337
63
  else {
4338
63
    auto Addr = HasLV ? LV.getAddress(CGF) : 
RV.getAggregateAddress()0
;
4339
63
    LValue SrcLV = CGF.MakeAddrLValue(Addr, Ty);
4340
    // We assume that call args are never copied into subobjects.
4341
63
    CGF.EmitAggregateCopy(Dst, SrcLV, Ty, AggValueSlot::DoesNotOverlap,
4342
63
                          HasLV ? LV.isVolatileQualified()
4343
63
                                : 
RV.isVolatileQualified()0
);
4344
63
  }
4345
1.21k
  IsUsed = true;
4346
1.21k
}
4347
4348
void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E,
4349
403k
                                  QualType type) {
4350
403k
  DisableDebugLocationUpdates Dis(*this, E);
4351
403k
  if (const ObjCIndirectCopyRestoreExpr *CRE
4352
403k
        = dyn_cast<ObjCIndirectCopyRestoreExpr>(E)) {
4353
27
    assert(getLangOpts().ObjCAutoRefCount);
4354
0
    return emitWritebackArg(*this, args, CRE);
4355
27
  }
4356
4357
403k
  assert(type->isReferenceType() == E->isGLValue() &&
4358
403k
         "reference binding to unmaterialized r-value!");
4359
4360
403k
  if (E->isGLValue()) {
4361
63.6k
    assert(E->getObjectKind() == OK_Ordinary);
4362
0
    return args.add(EmitReferenceBindingToExpr(E), type);
4363
63.6k
  }
4364
4365
339k
  bool HasAggregateEvalKind = hasAggregateEvaluationKind(type);
4366
4367
  // In the Microsoft C++ ABI, aggregate arguments are destructed by the callee.
4368
  // However, we still have to push an EH-only cleanup in case we unwind before
4369
  // we make it to the call.
4370
339k
  if (type->isRecordType() &&
4371
339k
      
type->castAs<RecordType>()->getDecl()->isParamDestroyedInCallee()10.7k
) {
4372
    // If we're using inalloca, use the argument memory.  Otherwise, use a
4373
    // temporary.
4374
417
    AggValueSlot Slot = args.isUsingInAlloca()
4375
417
        ? 
createPlaceholderSlot(*this, type)106
:
CreateAggTemp(type, "agg.tmp")311
;
4376
4377
417
    bool DestroyedInCallee = true, NeedsEHCleanup = true;
4378
417
    if (const auto *RD = type->getAsCXXRecordDecl())
4379
395
      DestroyedInCallee = RD->hasNonTrivialDestructor();
4380
22
    else
4381
22
      NeedsEHCleanup = needsEHCleanup(type.isDestructedType());
4382
4383
417
    if (DestroyedInCallee)
4384
204
      Slot.setExternallyDestructed();
4385
4386
417
    EmitAggExpr(E, Slot);
4387
417
    RValue RV = Slot.asRValue();
4388
417
    args.add(RV, type);
4389
4390
417
    if (DestroyedInCallee && 
NeedsEHCleanup204
) {
4391
      // Create a no-op GEP between the placeholder and the cleanup so we can
4392
      // RAUW it successfully.  It also serves as a marker of the first
4393
      // instruction where the cleanup is active.
4394
186
      pushFullExprCleanup<DestroyUnpassedArg>(EHCleanup, Slot.getAddress(),
4395
186
                                              type);
4396
      // This unreachable is a temporary marker which will be removed later.
4397
186
      llvm::Instruction *IsActive = Builder.CreateUnreachable();
4398
186
      args.addArgCleanupDeactivation(EHStack.stable_begin(), IsActive);
4399
186
    }
4400
417
    return;
4401
417
  }
4402
4403
339k
  if (HasAggregateEvalKind && 
isa<ImplicitCastExpr>(E)10.3k
&&
4404
339k
      
cast<CastExpr>(E)->getCastKind() == CK_LValueToRValue865
) {
4405
800
    LValue L = EmitLValue(cast<CastExpr>(E)->getSubExpr());
4406
800
    assert(L.isSimple());
4407
0
    args.addUncopiedAggregate(L, type);
4408
800
    return;
4409
800
  }
4410
4411
338k
  args.add(EmitAnyExprToTemp(E), type);
4412
338k
}
4413
4414
88.9k
QualType CodeGenFunction::getVarArgType(const Expr *Arg) {
4415
  // System headers on Windows define NULL to 0 instead of 0LL on Win64. MSVC
4416
  // implicitly widens null pointer constants that are arguments to varargs
4417
  // functions to pointer-sized ints.
4418
88.9k
  if (!getTarget().getTriple().isOSWindows())
4419
88.8k
    return Arg->getType();
4420
4421
94
  if (Arg->getType()->isIntegerType() &&
4422
94
      getContext().getTypeSize(Arg->getType()) <
4423
53
          getContext().getTargetInfo().getPointerWidth(0) &&
4424
94
      Arg->isNullPointerConstant(getContext(),
4425
29
                                 Expr::NPC_ValueDependentIsNotNull)) {
4426
1
    return getContext().getIntPtrType();
4427
1
  }
4428
4429
93
  return Arg->getType();
4430
94
}
4431
4432
// In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
4433
// optimizer it can aggressively ignore unwind edges.
4434
void
4435
1.87k
CodeGenFunction::AddObjCARCExceptionMetadata(llvm::Instruction *Inst) {
4436
1.87k
  if (CGM.getCodeGenOpts().OptimizationLevel != 0 &&
4437
1.87k
      
!CGM.getCodeGenOpts().ObjCAutoRefCountExceptions468
)
4438
378
    Inst->setMetadata("clang.arc.no_objc_arc_exceptions",
4439
378
                      CGM.getNoObjCARCExceptionsMetadata());
4440
1.87k
}
4441
4442
/// Emits a call to the given no-arguments nounwind runtime function.
4443
llvm::CallInst *
4444
CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
4445
373
                                         const llvm::Twine &name) {
4446
373
  return EmitNounwindRuntimeCall(callee, None, name);
4447
373
}
4448
4449
/// Emits a call to the given nounwind runtime function.
4450
llvm::CallInst *
4451
CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
4452
                                         ArrayRef<llvm::Value *> args,
4453
32.0k
                                         const llvm::Twine &name) {
4454
32.0k
  llvm::CallInst *call = EmitRuntimeCall(callee, args, name);
4455
32.0k
  call->setDoesNotThrow();
4456
32.0k
  return call;
4457
32.0k
}
4458
4459
/// Emits a simple call (never an invoke) to the given no-arguments
4460
/// runtime function.
4461
llvm::CallInst *CodeGenFunction::EmitRuntimeCall(llvm::FunctionCallee callee,
4462
5.60k
                                                 const llvm::Twine &name) {
4463
5.60k
  return EmitRuntimeCall(callee, None, name);
4464
5.60k
}
4465
4466
// Calls which may throw must have operand bundles indicating which funclet
4467
// they are nested within.
4468
SmallVector<llvm::OperandBundleDef, 1>
4469
464k
CodeGenFunction::getBundlesForFunclet(llvm::Value *Callee) {
4470
464k
  SmallVector<llvm::OperandBundleDef, 1> BundleList;
4471
  // There is no need for a funclet operand bundle if we aren't inside a
4472
  // funclet.
4473
464k
  if (!CurrentFuncletPad)
4474
464k
    return BundleList;
4475
4476
  // Skip intrinsics which cannot throw.
4477
243
  auto *CalleeFn = dyn_cast<llvm::Function>(Callee->stripPointerCasts());
4478
243
  if (CalleeFn && 
CalleeFn->isIntrinsic()237
&&
CalleeFn->doesNotThrow()2
)
4479
2
    return BundleList;
4480
4481
241
  BundleList.emplace_back("funclet", CurrentFuncletPad);
4482
241
  return BundleList;
4483
243
}
4484
4485
/// Emits a simple call (never an invoke) to the given runtime function.
4486
llvm::CallInst *CodeGenFunction::EmitRuntimeCall(llvm::FunctionCallee callee,
4487
                                                 ArrayRef<llvm::Value *> args,
4488
93.3k
                                                 const llvm::Twine &name) {
4489
93.3k
  llvm::CallInst *call = Builder.CreateCall(
4490
93.3k
      callee, args, getBundlesForFunclet(callee.getCallee()), name);
4491
93.3k
  call->setCallingConv(getRuntimeCC());
4492
93.3k
  return call;
4493
93.3k
}
4494
4495
/// Emits a call or invoke to the given noreturn runtime function.
4496
void CodeGenFunction::EmitNoreturnRuntimeCallOrInvoke(
4497
734
    llvm::FunctionCallee callee, ArrayRef<llvm::Value *> args) {
4498
734
  SmallVector<llvm::OperandBundleDef, 1> BundleList =
4499
734
      getBundlesForFunclet(callee.getCallee());
4500
4501
734
  if (getInvokeDest()) {
4502
134
    llvm::InvokeInst *invoke =
4503
134
      Builder.CreateInvoke(callee,
4504
134
                           getUnreachableBlock(),
4505
134
                           getInvokeDest(),
4506
134
                           args,
4507
134
                           BundleList);
4508
134
    invoke->setDoesNotReturn();
4509
134
    invoke->setCallingConv(getRuntimeCC());
4510
600
  } else {
4511
600
    llvm::CallInst *call = Builder.CreateCall(callee, args, BundleList);
4512
600
    call->setDoesNotReturn();
4513
600
    call->setCallingConv(getRuntimeCC());
4514
600
    Builder.CreateUnreachable();
4515
600
  }
4516
734
}
4517
4518
/// Emits a call or invoke instruction to the given nullary runtime function.
4519
llvm::CallBase *
4520
CodeGenFunction::EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
4521
367
                                         const Twine &name) {
4522
367
  return EmitRuntimeCallOrInvoke(callee, None, name);
4523
367
}
4524
4525
/// Emits a call or invoke instruction to the given runtime function.
4526
llvm::CallBase *
4527
CodeGenFunction::EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
4528
                                         ArrayRef<llvm::Value *> args,
4529
623
                                         const Twine &name) {
4530
623
  llvm::CallBase *call = EmitCallOrInvoke(callee, args, name);
4531
623
  call->setCallingConv(getRuntimeCC());
4532
623
  return call;
4533
623
}
4534
4535
/// Emits a call or invoke instruction to the given function, depending
4536
/// on the current state of the EH stack.
4537
llvm::CallBase *CodeGenFunction::EmitCallOrInvoke(llvm::FunctionCallee Callee,
4538
                                                  ArrayRef<llvm::Value *> Args,
4539
2.18k
                                                  const Twine &Name) {
4540
2.18k
  llvm::BasicBlock *InvokeDest = getInvokeDest();
4541
2.18k
  SmallVector<llvm::OperandBundleDef, 1> BundleList =
4542
2.18k
      getBundlesForFunclet(Callee.getCallee());
4543
4544
2.18k
  llvm::CallBase *Inst;
4545
2.18k
  if (!InvokeDest)
4546
1.91k
    Inst = Builder.CreateCall(Callee, Args, BundleList, Name);
4547
267
  else {
4548
267
    llvm::BasicBlock *ContBB = createBasicBlock("invoke.cont");
4549
267
    Inst = Builder.CreateInvoke(Callee, ContBB, InvokeDest, Args, BundleList,
4550
267
                                Name);
4551
267
    EmitBlock(ContBB);
4552
267
  }
4553
4554
  // In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
4555
  // optimizer it can aggressively ignore unwind edges.
4556
2.18k
  if (CGM.getLangOpts().ObjCAutoRefCount)
4557
8
    AddObjCARCExceptionMetadata(Inst);
4558
4559
2.18k
  return Inst;
4560
2.18k
}
4561
4562
void CodeGenFunction::deferPlaceholderReplacement(llvm::Instruction *Old,
4563
107
                                                  llvm::Value *New) {
4564
107
  DeferredReplacements.push_back(
4565
107
      std::make_pair(llvm::WeakTrackingVH(Old), New));
4566
107
}
4567
4568
namespace {
4569
4570
/// Specify given \p NewAlign as the alignment of return value attribute. If
4571
/// such attribute already exists, re-set it to the maximal one of two options.
4572
LLVM_NODISCARD llvm::AttributeList
4573
maybeRaiseRetAlignmentAttribute(llvm::LLVMContext &Ctx,
4574
                                const llvm::AttributeList &Attrs,
4575
32
                                llvm::Align NewAlign) {
4576
32
  llvm::Align CurAlign = Attrs.getRetAlignment().valueOrOne();
4577
32
  if (CurAlign >= NewAlign)
4578
2
    return Attrs;
4579
30
  llvm::Attribute AlignAttr = llvm::Attribute::getWithAlignment(Ctx, NewAlign);
4580
30
  return Attrs.removeRetAttribute(Ctx, llvm::Attribute::AttrKind::Alignment)
4581
30
      .addRetAttribute(Ctx, AlignAttr);
4582
32
}
4583
4584
template <typename AlignedAttrTy> class AbstractAssumeAlignedAttrEmitter {
4585
protected:
4586
  CodeGenFunction &CGF;
4587
4588
  /// We do nothing if this is, or becomes, nullptr.
4589
  const AlignedAttrTy *AA = nullptr;
4590
4591
  llvm::Value *Alignment = nullptr;      // May or may not be a constant.
4592
  llvm::ConstantInt *OffsetCI = nullptr; // Constant, hopefully zero.
4593
4594
  AbstractAssumeAlignedAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl)
4595
733k
      : CGF(CGF_) {
4596
733k
    if (!FuncDecl)
4597
33.8k
      return;
4598
699k
    AA = FuncDecl->getAttr<AlignedAttrTy>();
4599
699k
  }
CGCall.cpp:(anonymous namespace)::AbstractAssumeAlignedAttrEmitter<clang::AssumeAlignedAttr>::AbstractAssumeAlignedAttrEmitter(clang::CodeGen::CodeGenFunction&, clang::Decl const*)
Line
Count
Source
4595
366k
      : CGF(CGF_) {
4596
366k
    if (!FuncDecl)
4597
16.9k
      return;
4598
349k
    AA = FuncDecl->getAttr<AlignedAttrTy>();
4599
349k
  }
CGCall.cpp:(anonymous namespace)::AbstractAssumeAlignedAttrEmitter<clang::AllocAlignAttr>::AbstractAssumeAlignedAttrEmitter(clang::CodeGen::CodeGenFunction&, clang::Decl const*)
Line
Count
Source
4595
366k
      : CGF(CGF_) {
4596
366k
    if (!FuncDecl)
4597
16.9k
      return;
4598
349k
    AA = FuncDecl->getAttr<AlignedAttrTy>();
4599
349k
  }
4600
4601
public:
4602
  /// If we can, materialize the alignment as an attribute on return value.
4603
  LLVM_NODISCARD llvm::AttributeList
4604
733k
  TryEmitAsCallSiteAttribute(const llvm::AttributeList &Attrs) {
4605
733k
    if (!AA || 
OffsetCI63
||
CGF.SanOpts.has(SanitizerKind::Alignment)58
)
4606
733k
      return Attrs;
4607
49
    const auto *AlignmentCI = dyn_cast<llvm::ConstantInt>(Alignment);
4608
49
    if (!AlignmentCI)
4609
16
      return Attrs;
4610
    // We may legitimately have non-power-of-2 alignment here.
4611
    // If so, this is UB land, emit it via `@llvm.assume` instead.
4612
33
    if (!AlignmentCI->getValue().isPowerOf2())
4613
1
      return Attrs;
4614
32
    llvm::AttributeList NewAttrs = maybeRaiseRetAlignmentAttribute(
4615
32
        CGF.getLLVMContext(), Attrs,
4616
32
        llvm::Align(
4617
32
            AlignmentCI->getLimitedValue(llvm::Value::MaximumAlignment)));
4618
32
    AA = nullptr; // We're done. Disallow doing anything else.
4619
32
    return NewAttrs;
4620
33
  }
CGCall.cpp:(anonymous namespace)::AbstractAssumeAlignedAttrEmitter<clang::AssumeAlignedAttr>::TryEmitAsCallSiteAttribute(llvm::AttributeList const&)
Line
Count
Source
4604
366k
  TryEmitAsCallSiteAttribute(const llvm::AttributeList &Attrs) {
4605
366k
    if (!AA || 
OffsetCI14
||
CGF.SanOpts.has(SanitizerKind::Alignment)9
)
4606
366k
      return Attrs;
4607
6
    const auto *AlignmentCI = dyn_cast<llvm::ConstantInt>(Alignment);
4608
6
    if (!AlignmentCI)
4609
0
      return Attrs;
4610
    // We may legitimately have non-power-of-2 alignment here.
4611
    // If so, this is UB land, emit it via `@llvm.assume` instead.
4612
6
    if (!AlignmentCI->getValue().isPowerOf2())
4613
0
      return Attrs;
4614
6
    llvm::AttributeList NewAttrs = maybeRaiseRetAlignmentAttribute(
4615
6
        CGF.getLLVMContext(), Attrs,
4616
6
        llvm::Align(
4617
6
            AlignmentCI->getLimitedValue(llvm::Value::MaximumAlignment)));
4618
6
    AA = nullptr; // We're done. Disallow doing anything else.
4619
6
    return NewAttrs;
4620
6
  }
CGCall.cpp:(anonymous namespace)::AbstractAssumeAlignedAttrEmitter<clang::AllocAlignAttr>::TryEmitAsCallSiteAttribute(llvm::AttributeList const&)
Line
Count
Source
4604
366k
  TryEmitAsCallSiteAttribute(const llvm::AttributeList &Attrs) {
4605
366k
    if (!AA || 
OffsetCI49
||
CGF.SanOpts.has(SanitizerKind::Alignment)49
)
4606
366k
      return Attrs;
4607
43
    const auto *AlignmentCI = dyn_cast<llvm::ConstantInt>(Alignment);
4608
43
    if (!AlignmentCI)
4609
16
      return Attrs;
4610
    // We may legitimately have non-power-of-2 alignment here.
4611
    // If so, this is UB land, emit it via `@llvm.assume` instead.
4612
27
    if (!AlignmentCI->getValue().isPowerOf2())
4613
1
      return Attrs;
4614
26
    llvm::AttributeList NewAttrs = maybeRaiseRetAlignmentAttribute(
4615
26
        CGF.getLLVMContext(), Attrs,
4616
26
        llvm::Align(
4617
26
            AlignmentCI->getLimitedValue(llvm::Value::MaximumAlignment)));
4618
26
    AA = nullptr; // We're done. Disallow doing anything else.
4619
26
    return NewAttrs;
4620
27
  }
4621
4622
  /// Emit alignment assumption.
4623
  /// This is a general fallback that we take if either there is an offset,
4624
  /// or the alignment is variable or we are sanitizing for alignment.
4625
679k
  void EmitAsAnAssumption(SourceLocation Loc, QualType RetTy, RValue &Ret) {
4626
679k
    if (!AA)
4627
679k
      return;
4628
31
    CGF.emitAlignmentAssumption(Ret.getScalarVal(), RetTy, Loc,
4629
31
                                AA->getLocation(), Alignment, OffsetCI);
4630
31
    AA = nullptr; // We're done. Disallow doing anything else.
4631
31
  }
CGCall.cpp:(anonymous namespace)::AbstractAssumeAlignedAttrEmitter<clang::AssumeAlignedAttr>::EmitAsAnAssumption(clang::SourceLocation, clang::QualType, clang::CodeGen::RValue&)
Line
Count
Source
4625
339k
  void EmitAsAnAssumption(SourceLocation Loc, QualType RetTy, RValue &Ret) {
4626
339k
    if (!AA)
4627
339k
      return;
4628
8
    CGF.emitAlignmentAssumption(Ret.getScalarVal(), RetTy, Loc,
4629
8
                                AA->getLocation(), Alignment, OffsetCI);
4630
8
    AA = nullptr; // We're done. Disallow doing anything else.
4631
8
  }
CGCall.cpp:(anonymous namespace)::AbstractAssumeAlignedAttrEmitter<clang::AllocAlignAttr>::EmitAsAnAssumption(clang::SourceLocation, clang::QualType, clang::CodeGen::RValue&)
Line
Count
Source
4625
339k
  void EmitAsAnAssumption(SourceLocation Loc, QualType RetTy, RValue &Ret) {
4626
339k
    if (!AA)
4627
339k
      return;
4628
23
    CGF.emitAlignmentAssumption(Ret.getScalarVal(), RetTy, Loc,
4629
23
                                AA->getLocation(), Alignment, OffsetCI);
4630
23
    AA = nullptr; // We're done. Disallow doing anything else.
4631
23
  }
4632
};
4633
4634
/// Helper data structure to emit `AssumeAlignedAttr`.
4635
class AssumeAlignedAttrEmitter final
4636
    : public AbstractAssumeAlignedAttrEmitter<AssumeAlignedAttr> {
4637
public:
4638
  AssumeAlignedAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl)
4639
366k
      : AbstractAssumeAlignedAttrEmitter(CGF_, FuncDecl) {
4640
366k
    if (!AA)
4641
366k
      return;
4642
    // It is guaranteed that the alignment/offset are constants.
4643
14
    Alignment = cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AA->getAlignment()));
4644
14
    if (Expr *Offset = AA->getOffset()) {
4645
5
      OffsetCI = cast<llvm::ConstantInt>(CGF.EmitScalarExpr(Offset));
4646
5
      if (OffsetCI->isNullValue()) // Canonicalize zero offset to no offset.
4647
0
        OffsetCI = nullptr;
4648
5
    }
4649
14
  }
4650
};
4651
4652
/// Helper data structure to emit `AllocAlignAttr`.
4653
class AllocAlignAttrEmitter final
4654
    : public AbstractAssumeAlignedAttrEmitter<AllocAlignAttr> {
4655
public:
4656
  AllocAlignAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl,
4657
                        const CallArgList &CallArgs)
4658
366k
      : AbstractAssumeAlignedAttrEmitter(CGF_, FuncDecl) {
4659
366k
    if (!AA)
4660
366k
      return;
4661
    // Alignment may or may not be a constant, and that is okay.
4662
49
    Alignment = CallArgs[AA->getParamIndex().getLLVMIndex()]
4663
49
                    .getRValue(CGF)
4664
49
                    .getScalarVal();
4665
49
  }
4666
};
4667
4668
} // namespace
4669
4670
985k
static unsigned getMaxVectorWidth(const llvm::Type *Ty) {
4671
985k
  if (auto *VT = dyn_cast<llvm::VectorType>(Ty))
4672
47.2k
    return VT->getPrimitiveSizeInBits().getKnownMinSize();
4673
938k
  if (auto *AT = dyn_cast<llvm::ArrayType>(Ty))
4674
530
    return getMaxVectorWidth(AT->getElementType());
4675
4676
937k
  unsigned MaxVectorWidth = 0;
4677
937k
  if (auto *ST = dyn_cast<llvm::StructType>(Ty))
4678
2.37k
    for (auto *I : ST->elements())
4679
4.91k
      MaxVectorWidth = std::max(MaxVectorWidth, getMaxVectorWidth(I));
4680
937k
  return MaxVectorWidth;
4681
938k
}
4682
4683
RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
4684
                                 const CGCallee &Callee,
4685
                                 ReturnValueSlot ReturnValue,
4686
                                 const CallArgList &CallArgs,
4687
                                 llvm::CallBase **callOrInvoke, bool IsMustTail,
4688
366k
                                 SourceLocation Loc) {
4689
  // FIXME: We no longer need the types from CallArgs; lift up and simplify.
4690
4691
366k
  assert(Callee.isOrdinary() || Callee.isVirtual());
4692
4693
  // Handle struct-return functions by passing a pointer to the
4694
  // location that we would like to return into.
4695
0
  QualType RetTy = CallInfo.getReturnType();
4696
366k
  const ABIArgInfo &RetAI = CallInfo.getReturnInfo();
4697
4698
366k
  llvm::FunctionType *IRFuncTy = getTypes().GetFunctionType(CallInfo);
4699
4700
366k
  const Decl *TargetDecl = Callee.getAbstractInfo().getCalleeDecl().getDecl();
4701
366k
  if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(TargetDecl)) {
4702
    // We can only guarantee that a function is called from the correct
4703
    // context/function based on the appropriate target attributes,
4704
    // so only check in the case where we have both always_inline and target
4705
    // since otherwise we could be making a conditional call after a check for
4706
    // the proper cpu features (and it won't cause code generation issues due to
4707
    // function based code generation).
4708
344k
    if (TargetDecl->hasAttr<AlwaysInlineAttr>() &&
4709
344k
        
TargetDecl->hasAttr<TargetAttr>()21.9k
)
4710
14.8k
      checkTargetFeatures(Loc, FD);
4711
4712
    // Some architectures (such as x86-64) have the ABI changed based on
4713
    // attribute-target/features. Give them a chance to diagnose.
4714
344k
    CGM.getTargetCodeGenInfo().checkFunctionCallABI(
4715
344k
        CGM, Loc, dyn_cast_or_null<FunctionDecl>(CurCodeDecl), FD, CallArgs);
4716
344k
  }
4717
4718
366k
#ifndef NDEBUG
4719
366k
  if (!(CallInfo.isVariadic() && 
CallInfo.getArgStruct()77.5k
)) {
4720
    // For an inalloca varargs function, we don't expect CallInfo to match the
4721
    // function pointer's type, because the inalloca struct a will have extra
4722
    // fields in it for the varargs parameters.  Code later in this function
4723
    // bitcasts the function pointer to the type derived from CallInfo.
4724
    //
4725
    // In other cases, we assert that the types match up (until pointers stop
4726
    // having pointee types).
4727
366k
    if (Callee.isVirtual())
4728
957
      assert(IRFuncTy == Callee.getVirtualFunctionType());
4729
365k
    else {
4730
365k
      llvm::PointerType *PtrTy =
4731
365k
          llvm::cast<llvm::PointerType>(Callee.getFunctionPointer()->getType());
4732
365k
      assert(PtrTy->isOpaqueOrPointeeTypeMatches(IRFuncTy));
4733
365k
    }
4734
366k
  }
4735
0
#endif
4736
4737
  // 1. Set up the arguments.
4738
4739
  // If we're using inalloca, insert the allocation after the stack save.
4740
  // FIXME: Do this earlier rather than hacking it in here!
4741
0
  Address ArgMemory = Address::invalid();
4742
366k
  if (llvm::StructType *ArgStruct = CallInfo.getArgStruct()) {
4743
74
    const llvm::DataLayout &DL = CGM.getDataLayout();
4744
74
    llvm::Instruction *IP = CallArgs.getStackBase();
4745
74
    llvm::AllocaInst *AI;
4746
74
    if (IP) {
4747
73
      IP = IP->getNextNode();
4748
73
      AI = new llvm::AllocaInst(ArgStruct, DL.getAllocaAddrSpace(),
4749
73
                                "argmem", IP);
4750
73
    } else {
4751
1
      AI = CreateTempAlloca(ArgStruct, "argmem");
4752
1
    }
4753
74
    auto Align = CallInfo.getArgStructAlignment();
4754
74
    AI->setAlignment(Align.getAsAlign());
4755
74
    AI->setUsedWithInAlloca(true);
4756
74
    assert(AI->isUsedWithInAlloca() && !AI->isStaticAlloca());
4757
0
    ArgMemory = Address(AI, ArgStruct, Align);
4758
74
  }
4759
4760
0
  ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), CallInfo);
4761
366k
  SmallVector<llvm::Value *, 16> IRCallArgs(IRFunctionArgs.totalIRArgs());
4762
4763
  // If the call returns a temporary with struct return, create a temporary
4764
  // alloca to hold the result, unless one is given to us.
4765
366k
  Address SRetPtr = Address::invalid();
4766
366k
  Address SRetAlloca = Address::invalid();
4767
366k
  llvm::Value *UnusedReturnSizePtr = nullptr;
4768
366k
  if (RetAI.isIndirect() || 
RetAI.isInAlloca()364k
||
RetAI.isCoerceAndExpand()364k
) {
4769
2.66k
    if (!ReturnValue.isNull()) {
4770
1.87k
      SRetPtr = ReturnValue.getValue();
4771
1.87k
    } else {
4772
791
      SRetPtr = CreateMemTemp(RetTy, "tmp", &SRetAlloca);
4773
791
      if (HaveInsertPoint() && ReturnValue.isUnused()) {
4774
130
        llvm::TypeSize size =
4775
130
            CGM.getDataLayout().getTypeAllocSize(ConvertTypeForMem(RetTy));
4776
130
        UnusedReturnSizePtr = EmitLifetimeStart(size, SRetAlloca.getPointer());
4777
130
      }
4778
791
    }
4779
2.66k
    if (IRFunctionArgs.hasSRetArg()) {
4780
2.21k
      IRCallArgs[IRFunctionArgs.getSRetArgNo()] = SRetPtr.getPointer();
4781
2.21k
    } else 
if (446
RetAI.isInAlloca()446
) {
4782
4
      Address Addr =
4783
4
          Builder.CreateStructGEP(ArgMemory, RetAI.getInAllocaFieldIndex());
4784
4
      Builder.CreateStore(SRetPtr.getPointer(), Addr);
4785
4
    }
4786
2.66k
  }
4787
4788
366k
  Address swiftErrorTemp = Address::invalid();
4789
366k
  Address swiftErrorArg = Address::invalid();
4790
4791
  // When passing arguments using temporary allocas, we need to add the
4792
  // appropriate lifetime markers. This vector keeps track of all the lifetime
4793
  // markers that need to be ended right after the call.
4794
366k
  SmallVector<CallLifetimeEnd, 2> CallLifetimeEndAfterCall;
4795
4796
  // Translate all of the arguments as necessary to match the IR lowering.
4797
366k
  assert(CallInfo.arg_size() == CallArgs.size() &&
4798
366k
         "Mismatch between function signature & arguments.");
4799
0
  unsigned ArgNo = 0;
4800
366k
  CGFunctionInfo::const_arg_iterator info_it = CallInfo.arg_begin();
4801
366k
  for (CallArgList::const_iterator I = CallArgs.begin(), E = CallArgs.end();
4802
983k
       I != E; 
++I, ++info_it, ++ArgNo616k
) {
4803
616k
    const ABIArgInfo &ArgInfo = info_it->info;
4804
4805
    // Insert a padding argument to ensure proper alignment.
4806
616k
    if (IRFunctionArgs.hasPaddingArg(ArgNo))
4807
17
      IRCallArgs[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
4808
17
          llvm::UndefValue::get(ArgInfo.getPaddingType());
4809
4810
616k
    unsigned FirstIRArg, NumIRArgs;
4811
616k
    std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
4812
4813
616k
    switch (ArgInfo.getKind()) {
4814
160
    case ABIArgInfo::InAlloca: {
4815
160
      assert(NumIRArgs == 0);
4816
0
      assert(getTarget().getTriple().getArch() == llvm::Triple::x86);
4817
160
      if (I->isAggregate()) {
4818
107
        Address Addr = I->hasLValue()
4819
107
                           ? 
I->getKnownLValue().getAddress(*this)0
4820
107
                           : I->getKnownRValue().getAggregateAddress();
4821
107
        llvm::Instruction *Placeholder =
4822
107
            cast<llvm::Instruction>(Addr.getPointer());
4823
4824
107
        if (!ArgInfo.getInAllocaIndirect()) {
4825
          // Replace the placeholder with the appropriate argument slot GEP.
4826
105
          CGBuilderTy::InsertPoint IP = Builder.saveIP();
4827
105
          Builder.SetInsertPoint(Placeholder);
4828
105
          Addr = Builder.CreateStructGEP(ArgMemory,
4829
105
                                         ArgInfo.getInAllocaFieldIndex());
4830
105
          Builder.restoreIP(IP);
4831
105
        } else {
4832
          // For indirect things such as overaligned structs, replace the
4833
          // placeholder with a regular aggregate temporary alloca. Store the
4834
          // address of this alloca into the struct.
4835
2
          Addr = CreateMemTemp(info_it->type, "inalloca.indirect.tmp");
4836
2
          Address ArgSlot = Builder.CreateStructGEP(
4837
2
              ArgMemory, ArgInfo.getInAllocaFieldIndex());
4838
2
          Builder.CreateStore(Addr.getPointer(), ArgSlot);
4839
2
        }
4840
107
        deferPlaceholderReplacement(Placeholder, Addr.getPointer());
4841
107
      } else 
if (53
ArgInfo.getInAllocaIndirect()53
) {
4842
        // Make a temporary alloca and store the address of it into the argument
4843
        // struct.
4844
2
        Address Addr = CreateMemTempWithoutCast(
4845
2
            I->Ty, getContext().getTypeAlignInChars(I->Ty),
4846
2
            "indirect-arg-temp");
4847
2
        I->copyInto(*this, Addr);
4848
2
        Address ArgSlot =
4849
2
            Builder.CreateStructGEP(ArgMemory, ArgInfo.getInAllocaFieldIndex());
4850
2
        Builder.CreateStore(Addr.getPointer(), ArgSlot);
4851
51
      } else {
4852
        // Store the RValue into the argument struct.
4853
51
        Address Addr =
4854
51
            Builder.CreateStructGEP(ArgMemory, ArgInfo.getInAllocaFieldIndex());
4855
        // There are some cases where a trivial bitcast is not avoidable.  The
4856
        // definition of a type later in a translation unit may change it's type
4857
        // from {}* to (%struct.foo*)*.
4858
51
        Addr = Builder.CreateElementBitCast(Addr, ConvertTypeForMem(I->Ty));
4859
51
        I->copyInto(*this, Addr);
4860
51
      }
4861
160
      break;
4862
0
    }
4863
4864
2.07k
    case ABIArgInfo::Indirect:
4865
2.07k
    case ABIArgInfo::IndirectAliased: {
4866
2.07k
      assert(NumIRArgs == 1);
4867
2.07k
      if (!I->isAggregate()) {
4868
        // Make a temporary alloca to pass the argument.
4869
232
        Address Addr = CreateMemTempWithoutCast(
4870
232
            I->Ty, ArgInfo.getIndirectAlign(), "indirect-arg-temp");
4871
232
        IRCallArgs[FirstIRArg] = Addr.getPointer();
4872
4873
232
        I->copyInto(*this, Addr);
4874
1.84k
      } else {
4875
        // We want to avoid creating an unnecessary temporary+copy here;
4876
        // however, we need one in three cases:
4877
        // 1. If the argument is not byval, and we are required to copy the
4878
        //    source.  (This case doesn't occur on any common architecture.)
4879
        // 2. If the argument is byval, RV is not sufficiently aligned, and
4880
        //    we cannot force it to be sufficiently aligned.
4881
        // 3. If the argument is byval, but RV is not located in default
4882
        //    or alloca address space.
4883
1.84k
        Address Addr = I->hasLValue()
4884
1.84k
                           ? 
I->getKnownLValue().getAddress(*this)313
4885
1.84k
                           : 
I->getKnownRValue().getAggregateAddress()1.53k
;
4886
1.84k
        llvm::Value *V = Addr.getPointer();
4887
1.84k
        CharUnits Align = ArgInfo.getIndirectAlign();
4888
1.84k
        const llvm::DataLayout *TD = &CGM.getDataLayout();
4889
4890
1.84k
        assert((FirstIRArg >= IRFuncTy->getNumParams() ||
4891
1.84k
                IRFuncTy->getParamType(FirstIRArg)->getPointerAddressSpace() ==
4892
1.84k
                    TD->getAllocaAddrSpace()) &&
4893
1.84k
               "indirect argument must be in alloca address space");
4894
4895
0
        bool NeedCopy = false;
4896
4897
1.84k
        if (Addr.getAlignment() < Align &&
4898
1.84k
            llvm::getOrEnforceKnownAlignment(V, Align.getAsAlign(), *TD) <
4899
88
                Align.getAsAlign()) {
4900
2
          NeedCopy = true;
4901
1.84k
        } else if (I->hasLValue()) {
4902
311
          auto LV = I->getKnownLValue();
4903
311
          auto AS = LV.getAddressSpace();
4904
4905
311
          if (!ArgInfo.getIndirectByVal() ||
4906
311
              
(LV.getAlignment() < getContext().getTypeAlignInChars(I->Ty))268
) {
4907
43
            NeedCopy = true;
4908
43
          }
4909
311
          if (!getLangOpts().OpenCL) {
4910
270
            if ((ArgInfo.getIndirectByVal() &&
4911
270
                
(227
AS != LangAS::Default227
&&
4912
227
                 
AS != CGM.getASTAllocaAddressSpace()0
))) {
4913
0
              NeedCopy = true;
4914
0
            }
4915
270
          }
4916
          // For OpenCL even if RV is located in default or alloca address space
4917
          // we don't want to perform address space cast for it.
4918
41
          else if ((ArgInfo.getIndirectByVal() &&
4919
41
                    Addr.getType()->getAddressSpace() != IRFuncTy->
4920
41
                      getParamType(FirstIRArg)->getPointerAddressSpace())) {
4921
18
            NeedCopy = true;
4922
18
          }
4923
311
        }
4924
4925
1.84k
        if (NeedCopy) {
4926
          // Create an aligned temporary, and copy to it.
4927
63
          Address AI = CreateMemTempWithoutCast(
4928
63
              I->Ty, ArgInfo.getIndirectAlign(), "byval-temp");
4929
63
          IRCallArgs[FirstIRArg] = AI.getPointer();
4930
4931
          // Emit lifetime markers for the temporary alloca.
4932
63
          llvm::TypeSize ByvalTempElementSize =
4933
63
              CGM.getDataLayout().getTypeAllocSize(AI.getElementType());
4934
63
          llvm::Value *LifetimeSize =
4935
63
              EmitLifetimeStart(ByvalTempElementSize, AI.getPointer());
4936
4937
          // Add cleanup code to emit the end lifetime marker after the call.
4938
63
          if (LifetimeSize) // In case we disabled lifetime markers.
4939
2
            CallLifetimeEndAfterCall.emplace_back(AI, LifetimeSize);
4940
4941
          // Generate the copy.
4942
63
          I->copyInto(*this, AI);
4943
1.78k
        } else {
4944
          // Skip the extra memcpy call.
4945
1.78k
          auto *T = llvm::PointerType::getWithSamePointeeType(
4946
1.78k
              cast<llvm::PointerType>(V->getType()),
4947
1.78k
              CGM.getDataLayout().getAllocaAddrSpace());
4948
1.78k
          IRCallArgs[FirstIRArg] = getTargetHooks().performAddrSpaceCast(
4949
1.78k
              *this, V, LangAS::Default, CGM.getASTAllocaAddressSpace(), T,
4950
1.78k
              true);
4951
1.78k
        }
4952
1.84k
      }
4953
0
      break;
4954
2.07k
    }
4955
4956
6.99k
    case ABIArgInfo::Ignore:
4957
6.99k
      assert(NumIRArgs == 0);
4958
0
      break;
4959
4960
9.45k
    case ABIArgInfo::Extend:
4961
606k
    case ABIArgInfo::Direct: {
4962
606k
      if (!isa<llvm::StructType>(ArgInfo.getCoerceToType()) &&
4963
606k
          
ArgInfo.getCoerceToType() == ConvertType(info_it->type)605k
&&
4964
606k
          
ArgInfo.getDirectOffset() == 0602k
) {
4965
602k
        assert(NumIRArgs == 1);
4966
0
        llvm::Value *V;
4967
602k
        if (!I->isAggregate())
4968
602k
          V = I->getKnownRValue().getScalarVal();
4969
0
        else
4970
0
          V = Builder.CreateLoad(
4971
0
              I->hasLValue() ? I->getKnownLValue().getAddress(*this)
4972
0
                             : I->getKnownRValue().getAggregateAddress());
4973
4974
        // Implement swifterror by copying into a new swifterror argument.
4975
        // We'll write back in the normal path out of the call.
4976
602k
        if (CallInfo.getExtParameterInfo(ArgNo).getABI()
4977
602k
              == ParameterABI::SwiftErrorResult) {
4978
8
          assert(!swiftErrorTemp.isValid() && "multiple swifterror args");
4979
4980
0
          QualType pointeeTy = I->Ty->getPointeeType();
4981
8
          swiftErrorArg = Address(V, ConvertTypeForMem(pointeeTy),
4982
8
                                  getContext().getTypeAlignInChars(pointeeTy));
4983
4984
8
          swiftErrorTemp =
4985
8
            CreateMemTemp(pointeeTy, getPointerAlign(), "swifterror.temp");
4986
8
          V = swiftErrorTemp.getPointer();
4987
8
          cast<llvm::AllocaInst>(V)->setSwiftError(true);
4988
4989
8
          llvm::Value *errorValue = Builder.CreateLoad(swiftErrorArg);
4990
8
          Builder.CreateStore(errorValue, swiftErrorTemp);
4991
8
        }
4992
4993
        // We might have to widen integers, but we should never truncate.
4994
602k
        if (ArgInfo.getCoerceToType() != V->getType() &&
4995
602k
            
V->getType()->isIntegerTy()227
)
4996
3
          V = Builder.CreateZExt(V, ArgInfo.getCoerceToType());
4997
4998
        // If the argument doesn't match, perform a bitcast to coerce it.  This
4999
        // can happen due to trivial type mismatches.
5000
602k
        if (FirstIRArg < IRFuncTy->getNumParams() &&
5001
602k
            
V->getType() != IRFuncTy->getParamType(FirstIRArg)513k
)
5002
224
          V = Builder.CreateBitCast(V, IRFuncTy->getParamType(FirstIRArg));
5003
5004
602k
        IRCallArgs[FirstIRArg] = V;
5005
602k
        break;
5006
602k
      }
5007
5008
      // FIXME: Avoid the conversion through memory if possible.
5009
4.65k
      Address Src = Address::invalid();
5010
4.65k
      if (!I->isAggregate()) {
5011
871
        Src = CreateMemTemp(I->Ty, "coerce");
5012
871
        I->copyInto(*this, Src);
5013
3.78k
      } else {
5014
3.78k
        Src = I->hasLValue() ? 
I->getKnownLValue().getAddress(*this)468
5015
3.78k
                             : 
I->getKnownRValue().getAggregateAddress()3.31k
;
5016
3.78k
      }
5017
5018
      // If the value is offset in memory, apply the offset now.
5019
4.65k
      Src = emitAddressAtOffset(*this, Src, ArgInfo);
5020
5021
      // Fast-isel and the optimizer generally like scalar values better than
5022
      // FCAs, so we flatten them if this is safe to do for this argument.
5023
4.65k
      llvm::StructType *STy =
5024
4.65k
            dyn_cast<llvm::StructType>(ArgInfo.getCoerceToType());
5025
4.65k
      if (STy && 
ArgInfo.isDirect()1.29k
&&
ArgInfo.getCanBeFlattened()1.29k
) {
5026
1.26k
        llvm::Type *SrcTy = Src.getElementType();
5027
1.26k
        uint64_t SrcSize = CGM.getDataLayout().getTypeAllocSize(SrcTy);
5028
1.26k
        uint64_t DstSize = CGM.getDataLayout().getTypeAllocSize(STy);
5029
5030
        // If the source type is smaller than the destination type of the
5031
        // coerce-to logic, copy the source value into a temp alloca the size
5032
        // of the destination type to allow loading all of it. The bits past
5033
        // the source value are left undef.
5034
1.26k
        if (SrcSize < DstSize) {
5035
184
          Address TempAlloca
5036
184
            = CreateTempAlloca(STy, Src.getAlignment(),
5037
184
                               Src.getName() + ".coerce");
5038
184
          Builder.CreateMemCpy(TempAlloca, Src, SrcSize);
5039
184
          Src = TempAlloca;
5040
1.08k
        } else {
5041
1.08k
          Src = Builder.CreateElementBitCast(Src, STy);
5042
1.08k
        }
5043
5044
1.26k
        assert(NumIRArgs == STy->getNumElements());
5045
3.80k
        for (unsigned i = 0, e = STy->getNumElements(); i != e; 
++i2.53k
) {
5046
2.53k
          Address EltPtr = Builder.CreateStructGEP(Src, i);
5047
2.53k
          llvm::Value *LI = Builder.CreateLoad(EltPtr);
5048
2.53k
          IRCallArgs[FirstIRArg + i] = LI;
5049
2.53k
        }
5050
3.38k
      } else {
5051
        // In the simple case, just pass the coerced loaded value.
5052
3.38k
        assert(NumIRArgs == 1);
5053
0
        llvm::Value *Load =
5054
3.38k
            CreateCoercedLoad(Src, ArgInfo.getCoerceToType(), *this);
5055
5056
3.38k
        if (CallInfo.isCmseNSCall()) {
5057
          // For certain parameter types, clear padding bits, as they may reveal
5058
          // sensitive information.
5059
          // Small struct/union types are passed as integer arrays.
5060
40
          auto *ATy = dyn_cast<llvm::ArrayType>(Load->getType());
5061
40
          if (ATy != nullptr && 
isa<RecordType>(I->Ty.getCanonicalType())39
)
5062
39
            Load = EmitCMSEClearRecord(Load, ATy, I->Ty);
5063
40
        }
5064
3.38k
        IRCallArgs[FirstIRArg] = Load;
5065
3.38k
      }
5066
5067
0
      break;
5068
606k
    }
5069
5070
442
    case ABIArgInfo::CoerceAndExpand: {
5071
442
      auto coercionType = ArgInfo.getCoerceAndExpandType();