Coverage Report

Created: 2022-07-16 07:03

/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/CodeGen/CGCall.cpp
</
Line
Count
Source (jump to first uncovered line)
1
//===--- CGCall.cpp - Encapsulate calling convention details --------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// These classes wrap the information about a call or function
10
// definition used to handle ABI compliancy.
11
//
12
//===----------------------------------------------------------------------===//
13
14
#include "CGCall.h"
15
#include "ABIInfo.h"
16
#include "CGBlocks.h"
17
#include "CGCXXABI.h"
18
#include "CGCleanup.h"
19
#include "CGRecordLayout.h"
20
#include "CodeGenFunction.h"
21
#include "CodeGenModule.h"
22
#include "TargetInfo.h"
23
#include "clang/AST/Attr.h"
24
#include "clang/AST/Decl.h"
25
#include "clang/AST/DeclCXX.h"
26
#include "clang/AST/DeclObjC.h"
27
#include "clang/Basic/CodeGenOptions.h"
28
#include "clang/Basic/TargetBuiltins.h"
29
#include "clang/Basic/TargetInfo.h"
30
#include "clang/CodeGen/CGFunctionInfo.h"
31
#include "clang/CodeGen/SwiftCallingConv.h"
32
#include "llvm/ADT/StringExtras.h"
33
#include "llvm/Analysis/ValueTracking.h"
34
#include "llvm/IR/Assumptions.h"
35
#include "llvm/IR/Attributes.h"
36
#include "llvm/IR/CallingConv.h"
37
#include "llvm/IR/DataLayout.h"
38
#include "llvm/IR/InlineAsm.h"
39
#include "llvm/IR/IntrinsicInst.h"
40
#include "llvm/IR/Intrinsics.h"
41
#include "llvm/IR/Type.h"
42
#include "llvm/Transforms/Utils/Local.h"
43
using namespace clang;
44
using namespace CodeGen;
45
46
/***/
47
48
237k
unsigned CodeGenTypes::ClangCallConvToLLVMCallConv(CallingConv CC) {
49
237k
  switch (CC) {
50
232k
  default: return llvm::CallingConv::C;
51
56
  case CC_X86StdCall: return llvm::CallingConv::X86_StdCall;
52
74
  case CC_X86FastCall: return llvm::CallingConv::X86_FastCall;
53
108
  case CC_X86RegCall: return llvm::CallingConv::X86_RegCall;
54
2.67k
  case CC_X86ThisCall: return llvm::CallingConv::X86_ThisCall;
55
19
  case CC_Win64: return llvm::CallingConv::Win64;
56
14
  case CC_X86_64SysV: return llvm::CallingConv::X86_64_SysV;
57
5
  case CC_AAPCS: return llvm::CallingConv::ARM_AAPCS;
58
11
  case CC_AAPCS_VFP: return llvm::CallingConv::ARM_AAPCS_VFP;
59
3
  case CC_IntelOclBicc: return llvm::CallingConv::Intel_OCL_BI;
60
  // TODO: Add support for __pascal to LLVM.
61
3
  case CC_X86Pascal: return llvm::CallingConv::C;
62
  // TODO: Add support for __vectorcall to LLVM.
63
112
  case CC_X86VectorCall: return llvm::CallingConv::X86_VectorCall;
64
2
  case CC_AArch64VectorCall: return llvm::CallingConv::AArch64_VectorCall;
65
2
  case CC_AArch64SVEPCS: return llvm::CallingConv::AArch64_SVE_VectorCall;
66
7
  case CC_AMDGPUKernelCall: return llvm::CallingConv::AMDGPU_KERNEL;
67
432
  case CC_SpirFunction: return llvm::CallingConv::SPIR_FUNC;
68
520
  case CC_OpenCLKernel: return CGM.getTargetCodeGenInfo().getOpenCLKernelCallingConv();
69
10
  case CC_PreserveMost: return llvm::CallingConv::PreserveMost;
70
6
  case CC_PreserveAll: return llvm::CallingConv::PreserveAll;
71
1.14k
  case CC_Swift: return llvm::CallingConv::Swift;
72
73
  case CC_SwiftAsync: return llvm::CallingConv::SwiftTail;
73
237k
  }
74
237k
}
75
76
/// Derives the 'this' type for codegen purposes, i.e. ignoring method CVR
77
/// qualification. Either or both of RD and MD may be null. A null RD indicates
78
/// that there is no meaningful 'this' type, and a null MD can occur when
79
/// calling a method pointer.
80
CanQualType CodeGenTypes::DeriveThisType(const CXXRecordDecl *RD,
81
630k
                                         const CXXMethodDecl *MD) {
82
630k
  QualType RecTy;
83
630k
  if (RD)
84
629k
    RecTy = Context.getTagDeclType(RD)->getCanonicalTypeInternal();
85
688
  else
86
688
    RecTy = Context.VoidTy;
87
88
630k
  if (MD)
89
630k
    RecTy = Context.getAddrSpaceQualType(RecTy, MD->getMethodQualifiers().getAddressSpace());
90
630k
  return Context.getPointerType(CanQualType::CreateUnsafe(RecTy));
91
630k
}
92
93
/// Returns the canonical formal type of the given C++ method.
94
585k
static CanQual<FunctionProtoType> GetFormalType(const CXXMethodDecl *MD) {
95
585k
  return MD->getType()->getCanonicalTypeUnqualified()
96
585k
           .getAs<FunctionProtoType>();
97
585k
}
98
99
/// Returns the "extra-canonicalized" return type, which discards
100
/// qualifiers on the return type.  Codegen doesn't care about them,
101
/// and it makes ABI code a little easier to be able to assume that
102
/// all parameter and return types are top-level unqualified.
103
309k
static CanQualType GetReturnType(QualType RetTy) {
104
309k
  return RetTy->getCanonicalTypeUnqualified().getUnqualifiedType();
105
309k
}
106
107
/// Arrange the argument and result information for a value of the given
108
/// unprototyped freestanding function type.
109
const CGFunctionInfo &
110
394
CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionNoProtoType> FTNP) {
111
  // When translating an unprototyped function type, always use a
112
  // variadic type.
113
394
  return arrangeLLVMFunctionInfo(FTNP->getReturnType().getUnqualifiedType(),
114
394
                                 /*instanceMethod=*/false,
115
394
                                 /*chainCall=*/false, None,
116
394
                                 FTNP->getExtInfo(), {}, RequiredArgs(0));
117
394
}
118
119
static void addExtParameterInfosForCall(
120
         llvm::SmallVectorImpl<FunctionProtoType::ExtParameterInfo> &paramInfos,
121
                                        const FunctionProtoType *proto,
122
                                        unsigned prefixArgs,
123
1.24k
                                        unsigned totalArgs) {
124
1.24k
  assert(proto->hasExtParameterInfos());
125
0
  assert(paramInfos.size() <= prefixArgs);
126
0
  assert(proto->getNumParams() + prefixArgs <= totalArgs);
127
128
0
  paramInfos.reserve(totalArgs);
129
130
  // Add default infos for any prefix args that don't already have infos.
131
1.24k
  paramInfos.resize(prefixArgs);
132
133
  // Add infos for the prototype.
134
2.06k
  for (const auto &ParamInfo : proto->getExtParameterInfos()) {
135
2.06k
    paramInfos.push_back(ParamInfo);
136
    // pass_object_size params have no parameter info.
137
2.06k
    if (ParamInfo.hasPassObjectSize())
138
447
      paramInfos.emplace_back();
139
2.06k
  }
140
141
1.24k
  assert(paramInfos.size() <= totalArgs &&
142
1.24k
         "Did we forget to insert pass_object_size args?");
143
  // Add default infos for the variadic and/or suffix arguments.
144
0
  paramInfos.resize(totalArgs);
145
1.24k
}
146
147
/// Adds the formal parameters in FPT to the given prefix. If any parameter in
148
/// FPT has pass_object_size attrs, then we'll add parameters for those, too.
149
static void appendParameterTypes(const CodeGenTypes &CGT,
150
                                 SmallVectorImpl<CanQualType> &prefix,
151
              SmallVectorImpl<FunctionProtoType::ExtParameterInfo> &paramInfos,
152
996k
                                 CanQual<FunctionProtoType> FPT) {
153
  // Fast path: don't touch param info if we don't need to.
154
996k
  if (!FPT->hasExtParameterInfos()) {
155
996k
    assert(paramInfos.empty() &&
156
996k
           "We have paramInfos, but the prototype doesn't?");
157
0
    prefix.append(FPT->param_type_begin(), FPT->param_type_end());
158
996k
    return;
159
996k
  }
160
161
825
  unsigned PrefixSize = prefix.size();
162
  // In the vast majority of cases, we'll have precisely FPT->getNumParams()
163
  // parameters; the only thing that can change this is the presence of
164
  // pass_object_size. So, we preallocate for the common case.
165
825
  prefix.reserve(prefix.size() + FPT->getNumParams());
166
167
825
  auto ExtInfos = FPT->getExtParameterInfos();
168
825
  assert(ExtInfos.size() == FPT->getNumParams());
169
2.24k
  for (unsigned I = 0, E = FPT->getNumParams(); I != E; 
++I1.42k
) {
170
1.42k
    prefix.push_back(FPT->getParamType(I));
171
1.42k
    if (ExtInfos[I].hasPassObjectSize())
172
299
      prefix.push_back(CGT.getContext().getSizeType());
173
1.42k
  }
174
175
825
  addExtParameterInfosForCall(paramInfos, FPT.getTypePtr(), PrefixSize,
176
825
                              prefix.size());
177
825
}
178
179
/// Arrange the LLVM function layout for a value of the given function
180
/// type, on top of any implicit parameters already stored.
181
static const CGFunctionInfo &
182
arrangeLLVMFunctionInfo(CodeGenTypes &CGT, bool instanceMethod,
183
                        SmallVectorImpl<CanQualType> &prefix,
184
697k
                        CanQual<FunctionProtoType> FTP) {
185
697k
  SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos;
186
697k
  RequiredArgs Required = RequiredArgs::forPrototypePlus(FTP, prefix.size());
187
  // FIXME: Kill copy.
188
697k
  appendParameterTypes(CGT, prefix, paramInfos, FTP);
189
697k
  CanQualType resultType = FTP->getReturnType().getUnqualifiedType();
190
191
697k
  return CGT.arrangeLLVMFunctionInfo(resultType, instanceMethod,
192
697k
                                     /*chainCall=*/false, prefix,
193
697k
                                     FTP->getExtInfo(), paramInfos,
194
697k
                                     Required);
195
697k
}
196
197
/// Arrange the argument and result information for a value of the
198
/// given freestanding function type.
199
const CGFunctionInfo &
200
472k
CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionProtoType> FTP) {
201
472k
  SmallVector<CanQualType, 16> argTypes;
202
472k
  return ::arrangeLLVMFunctionInfo(*this, /*instanceMethod=*/false, argTypes,
203
472k
                                   FTP);
204
472k
}
205
206
static CallingConv getCallingConventionForDecl(const ObjCMethodDecl *D,
207
27.6k
                                               bool IsWindows) {
208
  // Set the appropriate calling convention for the Function.
209
27.6k
  if (D->hasAttr<StdCallAttr>())
210
2
    return CC_X86StdCall;
211
212
27.6k
  if (D->hasAttr<FastCallAttr>())
213
0
    return CC_X86FastCall;
214
215
27.6k
  if (D->hasAttr<RegCallAttr>())
216
0
    return CC_X86RegCall;
217
218
27.6k
  if (D->hasAttr<ThisCallAttr>())
219
0
    return CC_X86ThisCall;
220
221
27.6k
  if (D->hasAttr<VectorCallAttr>())
222
0
    return CC_X86VectorCall;
223
224
27.6k
  if (D->hasAttr<PascalAttr>())
225
0
    return CC_X86Pascal;
226
227
27.6k
  if (PcsAttr *PCS = D->getAttr<PcsAttr>())
228
0
    return (PCS->getPCS() == PcsAttr::AAPCS ? CC_AAPCS : CC_AAPCS_VFP);
229
230
27.6k
  if (D->hasAttr<AArch64VectorPcsAttr>())
231
0
    return CC_AArch64VectorCall;
232
233
27.6k
  if (D->hasAttr<AArch64SVEPcsAttr>())
234
0
    return CC_AArch64SVEPCS;
235
236
27.6k
  if (D->hasAttr<AMDGPUKernelCallAttr>())
237
0
    return CC_AMDGPUKernelCall;
238
239
27.6k
  if (D->hasAttr<IntelOclBiccAttr>())
240
0
    return CC_IntelOclBicc;
241
242
27.6k
  if (D->hasAttr<MSABIAttr>())
243
2
    return IsWindows ? 
CC_C0
: CC_Win64;
244
245
27.6k
  if (D->hasAttr<SysVABIAttr>())
246
0
    return IsWindows ? CC_X86_64SysV : CC_C;
247
248
27.6k
  if (D->hasAttr<PreserveMostAttr>())
249
0
    return CC_PreserveMost;
250
251
27.6k
  if (D->hasAttr<PreserveAllAttr>())
252
0
    return CC_PreserveAll;
253
254
27.6k
  return CC_C;
255
27.6k
}
256
257
/// Arrange the argument and result information for a call to an
258
/// unknown C++ non-static member function of the given abstract type.
259
/// (A null RD means we don't have any meaningful "this" argument type,
260
///  so fall back to a generic pointer type).
261
/// The member function must be an ordinary function, i.e. not a
262
/// constructor or destructor.
263
const CGFunctionInfo &
264
CodeGenTypes::arrangeCXXMethodType(const CXXRecordDecl *RD,
265
                                   const FunctionProtoType *FTP,
266
225k
                                   const CXXMethodDecl *MD) {
267
225k
  SmallVector<CanQualType, 16> argTypes;
268
269
  // Add the 'this' pointer.
270
225k
  argTypes.push_back(DeriveThisType(RD, MD));
271
272
225k
  return ::arrangeLLVMFunctionInfo(
273
225k
      *this, true, argTypes,
274
225k
      FTP->getCanonicalTypeUnqualified().getAs<FunctionProtoType>());
275
225k
}
276
277
/// Set calling convention for CUDA/HIP kernel.
278
static void setCUDAKernelCallingConvention(CanQualType &FTy, CodeGenModule &CGM,
279
639k
                                           const FunctionDecl *FD) {
280
639k
  if (FD->hasAttr<CUDAGlobalAttr>()) {
281
519
    const FunctionType *FT = FTy->getAs<FunctionType>();
282
519
    CGM.getTargetCodeGenInfo().setCUDAKernelCallingConvention(FT);
283
519
    FTy = FT->getCanonicalTypeUnqualified();
284
519
  }
285
639k
}
286
287
/// Arrange the argument and result information for a declaration or
288
/// definition of the given C++ non-static member function.  The
289
/// member function must be an ordinary function, i.e. not a
290
/// constructor or destructor.
291
const CGFunctionInfo &
292
231k
CodeGenTypes::arrangeCXXMethodDeclaration(const CXXMethodDecl *MD) {
293
231k
  assert(!isa<CXXConstructorDecl>(MD) && "wrong method for constructors!");
294
0
  assert(!isa<CXXDestructorDecl>(MD) && "wrong method for destructors!");
295
296
0
  CanQualType FT = GetFormalType(MD).getAs<Type>();
297
231k
  setCUDAKernelCallingConvention(FT, CGM, MD);
298
231k
  auto prototype = FT.getAs<FunctionProtoType>();
299
300
231k
  if (MD->isInstance()) {
301
    // The abstract case is perfectly fine.
302
225k
    const CXXRecordDecl *ThisType = TheCXXABI.getThisArgumentTypeForMethod(MD);
303
225k
    return arrangeCXXMethodType(ThisType, prototype.getTypePtr(), MD);
304
225k
  }
305
306
6.20k
  return arrangeFreeFunctionType(prototype);
307
231k
}
308
309
bool CodeGenTypes::inheritingCtorHasParams(
310
1.18k
    const InheritedConstructor &Inherited, CXXCtorType Type) {
311
  // Parameters are unnecessary if we're constructing a base class subobject
312
  // and the inherited constructor lives in a virtual base.
313
1.18k
  return Type == Ctor_Complete ||
314
1.18k
         
!Inherited.getShadowDecl()->constructsVirtualBase()804
||
315
1.18k
         
!Target.getCXXABI().hasConstructorVariants()74
;
316
1.18k
}
317
318
const CGFunctionInfo &
319
298k
CodeGenTypes::arrangeCXXStructorDeclaration(GlobalDecl GD) {
320
298k
  auto *MD = cast<CXXMethodDecl>(GD.getDecl());
321
322
298k
  SmallVector<CanQualType, 16> argTypes;
323
298k
  SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos;
324
298k
  argTypes.push_back(DeriveThisType(MD->getParent(), MD));
325
326
298k
  bool PassParams = true;
327
328
298k
  if (auto *CD = dyn_cast<CXXConstructorDecl>(MD)) {
329
    // A base class inheriting constructor doesn't get forwarded arguments
330
    // needed to construct a virtual base (or base class thereof).
331
178k
    if (auto Inherited = CD->getInheritedConstructor())
332
726
      PassParams = inheritingCtorHasParams(Inherited, GD.getCtorType());
333
178k
  }
334
335
298k
  CanQual<FunctionProtoType> FTP = GetFormalType(MD);
336
337
  // Add the formal parameters.
338
298k
  if (PassParams)
339
298k
    appendParameterTypes(*this, argTypes, paramInfos, FTP);
340
341
298k
  CGCXXABI::AddedStructorArgCounts AddedArgs =
342
298k
      TheCXXABI.buildStructorSignature(GD, argTypes);
343
298k
  if (!paramInfos.empty()) {
344
    // Note: prefix implies after the first param.
345
175
    if (AddedArgs.Prefix)
346
2
      paramInfos.insert(paramInfos.begin() + 1, AddedArgs.Prefix,
347
2
                        FunctionProtoType::ExtParameterInfo{});
348
175
    if (AddedArgs.Suffix)
349
20
      paramInfos.append(AddedArgs.Suffix,
350
20
                        FunctionProtoType::ExtParameterInfo{});
351
175
  }
352
353
298k
  RequiredArgs required =
354
298k
      (PassParams && 
MD->isVariadic()298k
?
RequiredArgs(argTypes.size())88
355
298k
                                      : 
RequiredArgs::All298k
);
356
357
298k
  FunctionType::ExtInfo extInfo = FTP->getExtInfo();
358
298k
  CanQualType resultType = TheCXXABI.HasThisReturn(GD)
359
298k
                               ? 
argTypes.front()7.75k
360
298k
                               : 
TheCXXABI.hasMostDerivedReturn(GD)291k
361
291k
                                     ? 
CGM.getContext().VoidPtrTy1.43k
362
291k
                                     : 
Context.VoidTy289k
;
363
298k
  return arrangeLLVMFunctionInfo(resultType, /*instanceMethod=*/true,
364
298k
                                 /*chainCall=*/false, argTypes, extInfo,
365
298k
                                 paramInfos, required);
366
298k
}
367
368
static SmallVector<CanQualType, 16>
369
78.5k
getArgTypesForCall(ASTContext &ctx, const CallArgList &args) {
370
78.5k
  SmallVector<CanQualType, 16> argTypes;
371
78.5k
  for (auto &arg : args)
372
110k
    argTypes.push_back(ctx.getCanonicalParamType(arg.Ty));
373
78.5k
  return argTypes;
374
78.5k
}
375
376
static SmallVector<CanQualType, 16>
377
30.9k
getArgTypesForDeclaration(ASTContext &ctx, const FunctionArgList &args) {
378
30.9k
  SmallVector<CanQualType, 16> argTypes;
379
30.9k
  for (auto &arg : args)
380
83.0k
    argTypes.push_back(ctx.getCanonicalParamType(arg->getType()));
381
30.9k
  return argTypes;
382
30.9k
}
383
384
static llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16>
385
getExtParameterInfosForCall(const FunctionProtoType *proto,
386
78.0k
                            unsigned prefixArgs, unsigned totalArgs) {
387
78.0k
  llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16> result;
388
78.0k
  if (proto->hasExtParameterInfos()) {
389
41
    addExtParameterInfosForCall(result, proto, prefixArgs, totalArgs);
390
41
  }
391
78.0k
  return result;
392
78.0k
}
393
394
/// Arrange a call to a C++ method, passing the given arguments.
395
///
396
/// ExtraPrefixArgs is the number of ABI-specific args passed after the `this`
397
/// parameter.
398
/// ExtraSuffixArgs is the number of ABI-specific args passed at the end of
399
/// args.
400
/// PassProtoArgs indicates whether `args` has args for the parameters in the
401
/// given CXXConstructorDecl.
402
const CGFunctionInfo &
403
CodeGenTypes::arrangeCXXConstructorCall(const CallArgList &args,
404
                                        const CXXConstructorDecl *D,
405
                                        CXXCtorType CtorKind,
406
                                        unsigned ExtraPrefixArgs,
407
                                        unsigned ExtraSuffixArgs,
408
55.3k
                                        bool PassProtoArgs) {
409
  // FIXME: Kill copy.
410
55.3k
  SmallVector<CanQualType, 16> ArgTypes;
411
55.3k
  for (const auto &Arg : args)
412
109k
    ArgTypes.push_back(Context.getCanonicalParamType(Arg.Ty));
413
414
  // +1 for implicit this, which should always be args[0].
415
55.3k
  unsigned TotalPrefixArgs = 1 + ExtraPrefixArgs;
416
417
55.3k
  CanQual<FunctionProtoType> FPT = GetFormalType(D);
418
55.3k
  RequiredArgs Required = PassProtoArgs
419
55.3k
                              ? RequiredArgs::forPrototypePlus(
420
55.3k
                                    FPT, TotalPrefixArgs + ExtraSuffixArgs)
421
55.3k
                              : 
RequiredArgs::All10
;
422
423
55.3k
  GlobalDecl GD(D, CtorKind);
424
55.3k
  CanQualType ResultType = TheCXXABI.HasThisReturn(GD)
425
55.3k
                               ? 
ArgTypes.front()2.37k
426
55.3k
                               : 
TheCXXABI.hasMostDerivedReturn(GD)52.9k
427
52.9k
                                     ? 
CGM.getContext().VoidPtrTy0
428
52.9k
                                     : Context.VoidTy;
429
430
55.3k
  FunctionType::ExtInfo Info = FPT->getExtInfo();
431
55.3k
  llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16> ParamInfos;
432
  // If the prototype args are elided, we should only have ABI-specific args,
433
  // which never have param info.
434
55.3k
  if (PassProtoArgs && 
FPT->hasExtParameterInfos()55.3k
) {
435
    // ABI-specific suffix arguments are treated the same as variadic arguments.
436
70
    addExtParameterInfosForCall(ParamInfos, FPT.getTypePtr(), TotalPrefixArgs,
437
70
                                ArgTypes.size());
438
70
  }
439
55.3k
  return arrangeLLVMFunctionInfo(ResultType, /*instanceMethod=*/true,
440
55.3k
                                 /*chainCall=*/false, ArgTypes, Info,
441
55.3k
                                 ParamInfos, Required);
442
55.3k
}
443
444
/// Arrange the argument and result information for the declaration or
445
/// definition of the given function.
446
const CGFunctionInfo &
447
510k
CodeGenTypes::arrangeFunctionDeclaration(const FunctionDecl *FD) {
448
510k
  if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(FD))
449
115k
    if (MD->isInstance())
450
101k
      return arrangeCXXMethodDeclaration(MD);
451
452
408k
  CanQualType FTy = FD->getType()->getCanonicalTypeUnqualified();
453
454
408k
  assert(isa<FunctionType>(FTy));
455
0
  setCUDAKernelCallingConvention(FTy, CGM, FD);
456
457
  // When declaring a function without a prototype, always use a
458
  // non-variadic type.
459
408k
  if (CanQual<FunctionNoProtoType> noProto = FTy.getAs<FunctionNoProtoType>()) {
460
4.39k
    return arrangeLLVMFunctionInfo(
461
4.39k
        noProto->getReturnType(), /*instanceMethod=*/false,
462
4.39k
        /*chainCall=*/false, None, noProto->getExtInfo(), {},RequiredArgs::All);
463
4.39k
  }
464
465
403k
  return arrangeFreeFunctionType(FTy.castAs<FunctionProtoType>());
466
408k
}
467
468
/// Arrange the argument and result information for the declaration or
469
/// definition of an Objective-C method.
470
const CGFunctionInfo &
471
5.21k
CodeGenTypes::arrangeObjCMethodDeclaration(const ObjCMethodDecl *MD) {
472
  // It happens that this is the same as a call with no optional
473
  // arguments, except also using the formal 'self' type.
474
5.21k
  return arrangeObjCMessageSendSignature(MD, MD->getSelfDecl()->getType());
475
5.21k
}
476
477
/// Arrange the argument and result information for the function type
478
/// through which to perform a send to the given Objective-C method,
479
/// using the given receiver type.  The receiver type is not always
480
/// the 'self' type of the method or even an Objective-C pointer type.
481
/// This is *not* the right method for actually performing such a
482
/// message send, due to the possibility of optional arguments.
483
const CGFunctionInfo &
484
CodeGenTypes::arrangeObjCMessageSendSignature(const ObjCMethodDecl *MD,
485
16.3k
                                              QualType receiverType) {
486
16.3k
  SmallVector<CanQualType, 16> argTys;
487
16.3k
  SmallVector<FunctionProtoType::ExtParameterInfo, 4> extParamInfos(2);
488
16.3k
  argTys.push_back(Context.getCanonicalParamType(receiverType));
489
16.3k
  argTys.push_back(Context.getCanonicalParamType(Context.getObjCSelType()));
490
  // FIXME: Kill copy?
491
16.3k
  for (const auto *I : MD->parameters()) {
492
13.5k
    argTys.push_back(Context.getCanonicalParamType(I->getType()));
493
13.5k
    auto extParamInfo = FunctionProtoType::ExtParameterInfo().withIsNoEscape(
494
13.5k
        I->hasAttr<NoEscapeAttr>());
495
13.5k
    extParamInfos.push_back(extParamInfo);
496
13.5k
  }
497
498
16.3k
  FunctionType::ExtInfo einfo;
499
16.3k
  bool IsWindows = getContext().getTargetInfo().getTriple().isOSWindows();
500
16.3k
  einfo = einfo.withCallingConv(getCallingConventionForDecl(MD, IsWindows));
501
502
16.3k
  if (getContext().getLangOpts().ObjCAutoRefCount &&
503
16.3k
      
MD->hasAttr<NSReturnsRetainedAttr>()880
)
504
114
    einfo = einfo.withProducesResult(true);
505
506
16.3k
  RequiredArgs required =
507
16.3k
    (MD->isVariadic() ? 
RequiredArgs(argTys.size())874
:
RequiredArgs::All15.4k
);
508
509
16.3k
  return arrangeLLVMFunctionInfo(
510
16.3k
      GetReturnType(MD->getReturnType()), /*instanceMethod=*/false,
511
16.3k
      /*chainCall=*/false, argTys, einfo, extParamInfos, required);
512
16.3k
}
513
514
const CGFunctionInfo &
515
CodeGenTypes::arrangeUnprototypedObjCMessageSend(QualType returnType,
516
810
                                                 const CallArgList &args) {
517
810
  auto argTypes = getArgTypesForCall(Context, args);
518
810
  FunctionType::ExtInfo einfo;
519
520
810
  return arrangeLLVMFunctionInfo(
521
810
      GetReturnType(returnType), /*instanceMethod=*/false,
522
810
      /*chainCall=*/false, argTypes, einfo, {}, RequiredArgs::All);
523
810
}
524
525
const CGFunctionInfo &
526
575k
CodeGenTypes::arrangeGlobalDeclaration(GlobalDecl GD) {
527
  // FIXME: Do we need to handle ObjCMethodDecl?
528
575k
  const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
529
530
575k
  if (isa<CXXConstructorDecl>(GD.getDecl()) ||
531
575k
      
isa<CXXDestructorDecl>(GD.getDecl())532k
)
532
65.1k
    return arrangeCXXStructorDeclaration(GD);
533
534
510k
  return arrangeFunctionDeclaration(FD);
535
575k
}
536
537
/// Arrange a thunk that takes 'this' as the first parameter followed by
538
/// varargs.  Return a void pointer, regardless of the actual return type.
539
/// The body of the thunk will end in a musttail call to a function of the
540
/// correct type, and the caller will bitcast the function to the correct
541
/// prototype.
542
const CGFunctionInfo &
543
55
CodeGenTypes::arrangeUnprototypedMustTailThunk(const CXXMethodDecl *MD) {
544
55
  assert(MD->isVirtual() && "only methods have thunks");
545
0
  CanQual<FunctionProtoType> FTP = GetFormalType(MD);
546
55
  CanQualType ArgTys[] = {DeriveThisType(MD->getParent(), MD)};
547
55
  return arrangeLLVMFunctionInfo(Context.VoidTy, /*instanceMethod=*/false,
548
55
                                 /*chainCall=*/false, ArgTys,
549
55
                                 FTP->getExtInfo(), {}, RequiredArgs(1));
550
55
}
551
552
const CGFunctionInfo &
553
CodeGenTypes::arrangeMSCtorClosure(const CXXConstructorDecl *CD,
554
34
                                   CXXCtorType CT) {
555
34
  assert(CT == Ctor_CopyingClosure || CT == Ctor_DefaultClosure);
556
557
0
  CanQual<FunctionProtoType> FTP = GetFormalType(CD);
558
34
  SmallVector<CanQualType, 2> ArgTys;
559
34
  const CXXRecordDecl *RD = CD->getParent();
560
34
  ArgTys.push_back(DeriveThisType(RD, CD));
561
34
  if (CT == Ctor_CopyingClosure)
562
3
    ArgTys.push_back(*FTP->param_type_begin());
563
34
  if (RD->getNumVBases() > 0)
564
0
    ArgTys.push_back(Context.IntTy);
565
34
  CallingConv CC = Context.getDefaultCallingConvention(
566
34
      /*IsVariadic=*/false, /*IsCXXMethod=*/true);
567
34
  return arrangeLLVMFunctionInfo(Context.VoidTy, /*instanceMethod=*/true,
568
34
                                 /*chainCall=*/false, ArgTys,
569
34
                                 FunctionType::ExtInfo(CC), {},
570
34
                                 RequiredArgs::All);
571
34
}
572
573
/// Arrange a call as unto a free function, except possibly with an
574
/// additional number of formal parameters considered required.
575
static const CGFunctionInfo &
576
arrangeFreeFunctionLikeCall(CodeGenTypes &CGT,
577
                            CodeGenModule &CGM,
578
                            const CallArgList &args,
579
                            const FunctionType *fnType,
580
                            unsigned numExtraRequiredArgs,
581
183k
                            bool chainCall) {
582
183k
  assert(args.size() >= numExtraRequiredArgs);
583
584
0
  llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos;
585
586
  // In most cases, there are no optional arguments.
587
183k
  RequiredArgs required = RequiredArgs::All;
588
589
  // If we have a variadic prototype, the required arguments are the
590
  // extra prefix plus the arguments in the prototype.
591
183k
  if (const FunctionProtoType *proto = dyn_cast<FunctionProtoType>(fnType)) {
592
183k
    if (proto->isVariadic())
593
77.5k
      required = RequiredArgs::forPrototypePlus(proto, numExtraRequiredArgs);
594
595
183k
    if (proto->hasExtParameterInfos())
596
305
      addExtParameterInfosForCall(paramInfos, proto, numExtraRequiredArgs,
597
305
                                  args.size());
598
599
  // If we don't have a prototype at all, but we're supposed to
600
  // explicitly use the variadic convention for unprototyped calls,
601
  // treat all of the arguments as required but preserve the nominal
602
  // possibility of variadics.
603
183k
  } else 
if (531
CGM.getTargetCodeGenInfo()
604
531
                .isNoProtoCallVariadic(args,
605
531
                                       cast<FunctionNoProtoType>(fnType))) {
606
428
    required = RequiredArgs(args.size());
607
428
  }
608
609
  // FIXME: Kill copy.
610
183k
  SmallVector<CanQualType, 16> argTypes;
611
183k
  for (const auto &arg : args)
612
328k
    argTypes.push_back(CGT.getContext().getCanonicalParamType(arg.Ty));
613
183k
  return CGT.arrangeLLVMFunctionInfo(GetReturnType(fnType->getReturnType()),
614
183k
                                     /*instanceMethod=*/false, chainCall,
615
183k
                                     argTypes, fnType->getExtInfo(), paramInfos,
616
183k
                                     required);
617
183k
}
618
619
/// Figure out the rules for calling a function with the given formal
620
/// type using the given arguments.  The arguments are necessary
621
/// because the function might be unprototyped, in which case it's
622
/// target-dependent in crazy ways.
623
const CGFunctionInfo &
624
CodeGenTypes::arrangeFreeFunctionCall(const CallArgList &args,
625
                                      const FunctionType *fnType,
626
183k
                                      bool chainCall) {
627
183k
  return arrangeFreeFunctionLikeCall(*this, CGM, args, fnType,
628
183k
                                     chainCall ? 
18
:
0183k
, chainCall);
629
183k
}
630
631
/// A block function is essentially a free function with an
632
/// extra implicit argument.
633
const CGFunctionInfo &
634
CodeGenTypes::arrangeBlockFunctionCall(const CallArgList &args,
635
621
                                       const FunctionType *fnType) {
636
621
  return arrangeFreeFunctionLikeCall(*this, CGM, args, fnType, 1,
637
621
                                     /*chainCall=*/false);
638
621
}
639
640
const CGFunctionInfo &
641
CodeGenTypes::arrangeBlockFunctionDeclaration(const FunctionProtoType *proto,
642
1.17k
                                              const FunctionArgList &params) {
643
1.17k
  auto paramInfos = getExtParameterInfosForCall(proto, 1, params.size());
644
1.17k
  auto argTypes = getArgTypesForDeclaration(Context, params);
645
646
1.17k
  return arrangeLLVMFunctionInfo(GetReturnType(proto->getReturnType()),
647
1.17k
                                 /*instanceMethod*/ false, /*chainCall*/ false,
648
1.17k
                                 argTypes, proto->getExtInfo(), paramInfos,
649
1.17k
                                 RequiredArgs::forPrototypePlus(proto, 1));
650
1.17k
}
651
652
const CGFunctionInfo &
653
CodeGenTypes::arrangeBuiltinFunctionCall(QualType resultType,
654
1.00k
                                         const CallArgList &args) {
655
  // FIXME: Kill copy.
656
1.00k
  SmallVector<CanQualType, 16> argTypes;
657
1.00k
  for (const auto &Arg : args)
658
3.68k
    argTypes.push_back(Context.getCanonicalParamType(Arg.Ty));
659
1.00k
  return arrangeLLVMFunctionInfo(
660
1.00k
      GetReturnType(resultType), /*instanceMethod=*/false,
661
1.00k
      /*chainCall=*/false, argTypes, FunctionType::ExtInfo(),
662
1.00k
      /*paramInfos=*/ {}, RequiredArgs::All);
663
1.00k
}
664
665
const CGFunctionInfo &
666
CodeGenTypes::arrangeBuiltinFunctionDeclaration(QualType resultType,
667
29.7k
                                                const FunctionArgList &args) {
668
29.7k
  auto argTypes = getArgTypesForDeclaration(Context, args);
669
670
29.7k
  return arrangeLLVMFunctionInfo(
671
29.7k
      GetReturnType(resultType), /*instanceMethod=*/false, /*chainCall=*/false,
672
29.7k
      argTypes, FunctionType::ExtInfo(), {}, RequiredArgs::All);
673
29.7k
}
674
675
const CGFunctionInfo &
676
CodeGenTypes::arrangeBuiltinFunctionDeclaration(CanQualType resultType,
677
374
                                              ArrayRef<CanQualType> argTypes) {
678
374
  return arrangeLLVMFunctionInfo(
679
374
      resultType, /*instanceMethod=*/false, /*chainCall=*/false,
680
374
      argTypes, FunctionType::ExtInfo(), {}, RequiredArgs::All);
681
374
}
682
683
/// Arrange a call to a C++ method, passing the given arguments.
684
///
685
/// numPrefixArgs is the number of ABI-specific prefix arguments we have. It
686
/// does not count `this`.
687
const CGFunctionInfo &
688
CodeGenTypes::arrangeCXXMethodCall(const CallArgList &args,
689
                                   const FunctionProtoType *proto,
690
                                   RequiredArgs required,
691
76.8k
                                   unsigned numPrefixArgs) {
692
76.8k
  assert(numPrefixArgs + 1 <= args.size() &&
693
76.8k
         "Emitting a call with less args than the required prefix?");
694
  // Add one to account for `this`. It's a bit awkward here, but we don't count
695
  // `this` in similar places elsewhere.
696
0
  auto paramInfos =
697
76.8k
    getExtParameterInfosForCall(proto, numPrefixArgs + 1, args.size());
698
699
  // FIXME: Kill copy.
700
76.8k
  auto argTypes = getArgTypesForCall(Context, args);
701
702
76.8k
  FunctionType::ExtInfo info = proto->getExtInfo();
703
76.8k
  return arrangeLLVMFunctionInfo(
704
76.8k
      GetReturnType(proto->getReturnType()), /*instanceMethod=*/true,
705
76.8k
      /*chainCall=*/false, argTypes, info, paramInfos, required);
706
76.8k
}
707
708
17.7k
const CGFunctionInfo &CodeGenTypes::arrangeNullaryFunction() {
709
17.7k
  return arrangeLLVMFunctionInfo(
710
17.7k
      getContext().VoidTy, /*instanceMethod=*/false, /*chainCall=*/false,
711
17.7k
      None, FunctionType::ExtInfo(), {}, RequiredArgs::All);
712
17.7k
}
713
714
const CGFunctionInfo &
715
CodeGenTypes::arrangeCall(const CGFunctionInfo &signature,
716
11.1k
                          const CallArgList &args) {
717
11.1k
  assert(signature.arg_size() <= args.size());
718
11.1k
  if (signature.arg_size() == args.size())
719
10.2k
    return signature;
720
721
856
  SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos;
722
856
  auto sigParamInfos = signature.getExtParameterInfos();
723
856
  if (!sigParamInfos.empty()) {
724
856
    paramInfos.append(sigParamInfos.begin(), sigParamInfos.end());
725
856
    paramInfos.resize(args.size());
726
856
  }
727
728
856
  auto argTypes = getArgTypesForCall(Context, args);
729
730
856
  assert(signature.getRequiredArgs().allowsOptionalArgs());
731
0
  return arrangeLLVMFunctionInfo(signature.getReturnType(),
732
856
                                 signature.isInstanceMethod(),
733
856
                                 signature.isChainCall(),
734
856
                                 argTypes,
735
856
                                 signature.getExtInfo(),
736
856
                                 paramInfos,
737
856
                                 signature.getRequiredArgs());
738
11.1k
}
739
740
namespace clang {
741
namespace CodeGen {
742
void computeSPIRKernelABIInfo(CodeGenModule &CGM, CGFunctionInfo &FI);
743
}
744
}
745
746
/// Arrange the argument and result information for an abstract value
747
/// of a given function type.  This is the method which all of the
748
/// above functions ultimately defer to.
749
const CGFunctionInfo &
750
CodeGenTypes::arrangeLLVMFunctionInfo(CanQualType resultType,
751
                                      bool instanceMethod,
752
                                      bool chainCall,
753
                                      ArrayRef<CanQualType> argTypes,
754
                                      FunctionType::ExtInfo info,
755
                     ArrayRef<FunctionProtoType::ExtParameterInfo> paramInfos,
756
1.38M
                                      RequiredArgs required) {
757
1.38M
  assert(llvm::all_of(argTypes,
758
1.38M
                      [](CanQualType T) { return T.isCanonicalAsParam(); }));
759
760
  // Lookup or create unique function info.
761
0
  llvm::FoldingSetNodeID ID;
762
1.38M
  CGFunctionInfo::Profile(ID, instanceMethod, chainCall, info, paramInfos,
763
1.38M
                          required, resultType, argTypes);
764
765
1.38M
  void *insertPos = nullptr;
766
1.38M
  CGFunctionInfo *FI = FunctionInfos.FindNodeOrInsertPos(ID, insertPos);
767
1.38M
  if (FI)
768
1.14M
    return *FI;
769
770
237k
  unsigned CC = ClangCallConvToLLVMCallConv(info.getCC());
771
772
  // Construct the function info.  We co-allocate the ArgInfos.
773
237k
  FI = CGFunctionInfo::create(CC, instanceMethod, chainCall, info,
774
237k
                              paramInfos, resultType, argTypes, required);
775
237k
  FunctionInfos.InsertNode(FI, insertPos);
776
777
237k
  bool inserted = FunctionsBeingProcessed.insert(FI).second;
778
237k
  (void)inserted;
779
237k
  assert(inserted && "Recursively being processed?");
780
781
  // Compute ABI information.
782
237k
  if (CC == llvm::CallingConv::SPIR_KERNEL) {
783
    // Force target independent argument handling for the host visible
784
    // kernel functions.
785
179
    computeSPIRKernelABIInfo(CGM, *FI);
786
237k
  } else if (info.getCC() == CC_Swift || 
info.getCC() == CC_SwiftAsync236k
) {
787
1.21k
    swiftcall::computeABIInfo(CGM, *FI);
788
236k
  } else {
789
236k
    getABIInfo().computeInfo(*FI);
790
236k
  }
791
792
  // Loop over all of the computed argument and return value info.  If any of
793
  // them are direct or extend without a specified coerce type, specify the
794
  // default now.
795
237k
  ABIArgInfo &retInfo = FI->getReturnInfo();
796
237k
  if (retInfo.canHaveCoerceToType() && 
retInfo.getCoerceToType() == nullptr134k
)
797
42.2k
    retInfo.setCoerceToType(ConvertType(FI->getReturnType()));
798
799
237k
  for (auto &I : FI->arguments())
800
437k
    if (I.info.canHaveCoerceToType() && 
I.info.getCoerceToType() == nullptr429k
)
801
142k
      I.info.setCoerceToType(ConvertType(I.type));
802
803
237k
  bool erased = FunctionsBeingProcessed.erase(FI); (void)erased;
804
237k
  assert(erased && "Not in set?");
805
806
0
  return *FI;
807
1.38M
}
808
809
CGFunctionInfo *CGFunctionInfo::create(unsigned llvmCC,
810
                                       bool instanceMethod,
811
                                       bool chainCall,
812
                                       const FunctionType::ExtInfo &info,
813
                                       ArrayRef<ExtParameterInfo> paramInfos,
814
                                       CanQualType resultType,
815
                                       ArrayRef<CanQualType> argTypes,
816
237k
                                       RequiredArgs required) {
817
237k
  assert(paramInfos.empty() || paramInfos.size() == argTypes.size());
818
0
  assert(!required.allowsOptionalArgs() ||
819
237k
         required.getNumRequiredArgs() <= argTypes.size());
820
821
0
  void *buffer =
822
237k
    operator new(totalSizeToAlloc<ArgInfo,             ExtParameterInfo>(
823
237k
                                  argTypes.size() + 1, paramInfos.size()));
824
825
237k
  CGFunctionInfo *FI = new(buffer) CGFunctionInfo();
826
237k
  FI->CallingConvention = llvmCC;
827
237k
  FI->EffectiveCallingConvention = llvmCC;
828
237k
  FI->ASTCallingConvention = info.getCC();
829
237k
  FI->InstanceMethod = instanceMethod;
830
237k
  FI->ChainCall = chainCall;
831
237k
  FI->CmseNSCall = info.getCmseNSCall();
832
237k
  FI->NoReturn = info.getNoReturn();
833
237k
  FI->ReturnsRetained = info.getProducesResult();
834
237k
  FI->NoCallerSavedRegs = info.getNoCallerSavedRegs();
835
237k
  FI->NoCfCheck = info.getNoCfCheck();
836
237k
  FI->Required = required;
837
237k
  FI->HasRegParm = info.getHasRegParm();
838
237k
  FI->RegParm = info.getRegParm();
839
237k
  FI->ArgStruct = nullptr;
840
237k
  FI->ArgStructAlign = 0;
841
237k
  FI->NumArgs = argTypes.size();
842
237k
  FI->HasExtParameterInfos = !paramInfos.empty();
843
237k
  FI->getArgsBuffer()[0].type = resultType;
844
237k
  FI->MaxVectorWidth = 0;
845
674k
  for (unsigned i = 0, e = argTypes.size(); i != e; 
++i437k
)
846
437k
    FI->getArgsBuffer()[i + 1].type = argTypes[i];
847
257k
  for (unsigned i = 0, e = paramInfos.size(); i != e; 
++i20.1k
)
848
20.1k
    FI->getExtParameterInfosBuffer()[i] = paramInfos[i];
849
237k
  return FI;
850
237k
}
851
852
/***/
853
854
namespace {
855
// ABIArgInfo::Expand implementation.
856
857
// Specifies the way QualType passed as ABIArgInfo::Expand is expanded.
858
struct TypeExpansion {
859
  enum TypeExpansionKind {
860
    // Elements of constant arrays are expanded recursively.
861
    TEK_ConstantArray,
862
    // Record fields are expanded recursively (but if record is a union, only
863
    // the field with the largest size is expanded).
864
    TEK_Record,
865
    // For complex types, real and imaginary parts are expanded recursively.
866
    TEK_Complex,
867
    // All other types are not expandable.
868
    TEK_None
869
  };
870
871
  const TypeExpansionKind Kind;
872
873
2.36k
  TypeExpansion(TypeExpansionKind K) : Kind(K) {}
874
2.36k
  virtual ~TypeExpansion() {}
875
};
876
877
struct ConstantArrayExpansion : TypeExpansion {
878
  QualType EltTy;
879
  uint64_t NumElts;
880
881
  ConstantArrayExpansion(QualType EltTy, uint64_t NumElts)
882
40
      : TypeExpansion(TEK_ConstantArray), EltTy(EltTy), NumElts(NumElts) {}
883
2.36k
  static bool classof(const TypeExpansion *TE) {
884
2.36k
    return TE->Kind == TEK_ConstantArray;
885
2.36k
  }
886
};
887
888
struct RecordExpansion : TypeExpansion {
889
  SmallVector<const CXXBaseSpecifier *, 1> Bases;
890
891
  SmallVector<const FieldDecl *, 1> Fields;
892
893
  RecordExpansion(SmallVector<const CXXBaseSpecifier *, 1> &&Bases,
894
                  SmallVector<const FieldDecl *, 1> &&Fields)
895
      : TypeExpansion(TEK_Record), Bases(std::move(Bases)),
896
761
        Fields(std::move(Fields)) {}
897
2.32k
  static bool classof(const TypeExpansion *TE) {
898
2.32k
    return TE->Kind == TEK_Record;
899
2.32k
  }
900
};
901
902
struct ComplexExpansion : TypeExpansion {
903
  QualType EltTy;
904
905
46
  ComplexExpansion(QualType EltTy) : TypeExpansion(TEK_Complex), EltTy(EltTy) {}
906
1.56k
  static bool classof(const TypeExpansion *TE) {
907
1.56k
    return TE->Kind == TEK_Complex;
908
1.56k
  }
909
};
910
911
struct NoExpansion : TypeExpansion {
912
1.51k
  NoExpansion() : TypeExpansion(TEK_None) {}
913
1.51k
  static bool classof(const TypeExpansion *TE) {
914
1.51k
    return TE->Kind == TEK_None;
915
1.51k
  }
916
};
917
}  // namespace
918
919
static std::unique_ptr<TypeExpansion>
920
2.36k
getTypeExpansion(QualType Ty, const ASTContext &Context) {
921
2.36k
  if (const ConstantArrayType *AT = Context.getAsConstantArrayType(Ty)) {
922
40
    return std::make_unique<ConstantArrayExpansion>(
923
40
        AT->getElementType(), AT->getSize().getZExtValue());
924
40
  }
925
2.32k
  if (const RecordType *RT = Ty->getAs<RecordType>()) {
926
761
    SmallVector<const CXXBaseSpecifier *, 1> Bases;
927
761
    SmallVector<const FieldDecl *, 1> Fields;
928
761
    const RecordDecl *RD = RT->getDecl();
929
761
    assert(!RD->hasFlexibleArrayMember() &&
930
761
           "Cannot expand structure with flexible array.");
931
761
    if (RD->isUnion()) {
932
      // Unions can be here only in degenerative cases - all the fields are same
933
      // after flattening. Thus we have to use the "largest" field.
934
5
      const FieldDecl *LargestFD = nullptr;
935
5
      CharUnits UnionSize = CharUnits::Zero();
936
937
10
      for (const auto *FD : RD->fields()) {
938
10
        if (FD->isZeroLengthBitField(Context))
939
0
          continue;
940
10
        assert(!FD->isBitField() &&
941
10
               "Cannot expand structure with bit-field members.");
942
0
        CharUnits FieldSize = Context.getTypeSizeInChars(FD->getType());
943
10
        if (UnionSize < FieldSize) {
944
5
          UnionSize = FieldSize;
945
5
          LargestFD = FD;
946
5
        }
947
10
      }
948
5
      if (LargestFD)
949
5
        Fields.push_back(LargestFD);
950
756
    } else {
951
756
      if (const auto *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
952
241
        assert(!CXXRD->isDynamicClass() &&
953
241
               "cannot expand vtable pointers in dynamic classes");
954
0
        llvm::append_range(Bases, llvm::make_pointer_range(CXXRD->bases()));
955
241
      }
956
957
1.47k
      for (const auto *FD : RD->fields()) {
958
1.47k
        if (FD->isZeroLengthBitField(Context))
959
0
          continue;
960
1.47k
        assert(!FD->isBitField() &&
961
1.47k
               "Cannot expand structure with bit-field members.");
962
0
        Fields.push_back(FD);
963
1.47k
      }
964
756
    }
965
0
    return std::make_unique<RecordExpansion>(std::move(Bases),
966
761
                                              std::move(Fields));
967
761
  }
968
1.56k
  if (const ComplexType *CT = Ty->getAs<ComplexType>()) {
969
46
    return std::make_unique<ComplexExpansion>(CT->getElementType());
970
46
  }
971
1.51k
  return std::make_unique<NoExpansion>();
972
1.56k
}
973
974
1.38k
static int getExpansionSize(QualType Ty, const ASTContext &Context) {
975
1.38k
  auto Exp = getTypeExpansion(Ty, Context);
976
1.38k
  if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
977
24
    return CAExp->NumElts * getExpansionSize(CAExp->EltTy, Context);
978
24
  }
979
1.36k
  if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
980
456
    int Res = 0;
981
456
    for (auto BS : RExp->Bases)
982
3
      Res += getExpansionSize(BS->getType(), Context);
983
456
    for (auto FD : RExp->Fields)
984
887
      Res += getExpansionSize(FD->getType(), Context);
985
456
    return Res;
986
456
  }
987
908
  if (isa<ComplexExpansion>(Exp.get()))
988
28
    return 2;
989
880
  assert(isa<NoExpansion>(Exp.get()));
990
0
  return 1;
991
908
}
992
993
void
994
CodeGenTypes::getExpandedTypes(QualType Ty,
995
546
                               SmallVectorImpl<llvm::Type *>::iterator &TI) {
996
546
  auto Exp = getTypeExpansion(Ty, Context);
997
546
  if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
998
40
    for (int i = 0, n = CAExp->NumElts; i < n; 
i++32
) {
999
32
      getExpandedTypes(CAExp->EltTy, TI);
1000
32
    }
1001
538
  } else if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
1002
174
    for (auto BS : RExp->Bases)
1003
1
      getExpandedTypes(BS->getType(), TI);
1004
174
    for (auto FD : RExp->Fields)
1005
332
      getExpandedTypes(FD->getType(), TI);
1006
364
  } else if (auto CExp = dyn_cast<ComplexExpansion>(Exp.get())) {
1007
11
    llvm::Type *EltTy = ConvertType(CExp->EltTy);
1008
11
    *TI++ = EltTy;
1009
11
    *TI++ = EltTy;
1010
353
  } else {
1011
353
    assert(isa<NoExpansion>(Exp.get()));
1012
0
    *TI++ = ConvertType(Ty);
1013
353
  }
1014
546
}
1015
1016
static void forConstantArrayExpansion(CodeGenFunction &CGF,
1017
                                      ConstantArrayExpansion *CAE,
1018
                                      Address BaseAddr,
1019
8
                                      llvm::function_ref<void(Address)> Fn) {
1020
8
  CharUnits EltSize = CGF.getContext().getTypeSizeInChars(CAE->EltTy);
1021
8
  CharUnits EltAlign =
1022
8
    BaseAddr.getAlignment().alignmentOfArrayElement(EltSize);
1023
8
  llvm::Type *EltTy = CGF.ConvertTypeForMem(CAE->EltTy);
1024
1025
40
  for (int i = 0, n = CAE->NumElts; i < n; 
i++32
) {
1026
32
    llvm::Value *EltAddr = CGF.Builder.CreateConstGEP2_32(
1027
32
        BaseAddr.getElementType(), BaseAddr.getPointer(), 0, i);
1028
32
    Fn(Address(EltAddr, EltTy, EltAlign));
1029
32
  }
1030
8
}
1031
1032
void CodeGenFunction::ExpandTypeFromArgs(QualType Ty, LValue LV,
1033
349
                                         llvm::Function::arg_iterator &AI) {
1034
349
  assert(LV.isSimple() &&
1035
349
         "Unexpected non-simple lvalue during struct expansion.");
1036
1037
0
  auto Exp = getTypeExpansion(Ty, getContext());
1038
349
  if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
1039
8
    forConstantArrayExpansion(
1040
32
        *this, CAExp, LV.getAddress(*this), [&](Address EltAddr) {
1041
32
          LValue LV = MakeAddrLValue(EltAddr, CAExp->EltTy);
1042
32
          ExpandTypeFromArgs(CAExp->EltTy, LV, AI);
1043
32
        });
1044
341
  } else if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
1045
101
    Address This = LV.getAddress(*this);
1046
101
    for (const CXXBaseSpecifier *BS : RExp->Bases) {
1047
      // Perform a single step derived-to-base conversion.
1048
1
      Address Base =
1049
1
          GetAddressOfBaseClass(This, Ty->getAsCXXRecordDecl(), &BS, &BS + 1,
1050
1
                                /*NullCheckValue=*/false, SourceLocation());
1051
1
      LValue SubLV = MakeAddrLValue(Base, BS->getType());
1052
1053
      // Recurse onto bases.
1054
1
      ExpandTypeFromArgs(BS->getType(), SubLV, AI);
1055
1
    }
1056
214
    for (auto FD : RExp->Fields) {
1057
      // FIXME: What are the right qualifiers here?
1058
214
      LValue SubLV = EmitLValueForFieldInitialization(LV, FD);
1059
214
      ExpandTypeFromArgs(FD->getType(), SubLV, AI);
1060
214
    }
1061
240
  } else if (isa<ComplexExpansion>(Exp.get())) {
1062
3
    auto realValue = &*AI++;
1063
3
    auto imagValue = &*AI++;
1064
3
    EmitStoreOfComplex(ComplexPairTy(realValue, imagValue), LV, /*init*/ true);
1065
237
  } else {
1066
    // Call EmitStoreOfScalar except when the lvalue is a bitfield to emit a
1067
    // primitive store.
1068
237
    assert(isa<NoExpansion>(Exp.get()));
1069
0
    llvm::Value *Arg = &*AI++;
1070
237
    if (LV.isBitField()) {
1071
0
      EmitStoreThroughLValue(RValue::get(Arg), LV);
1072
237
    } else {
1073
      // TODO: currently there are some places are inconsistent in what LLVM
1074
      // pointer type they use (see D118744). Once clang uses opaque pointers
1075
      // all LLVM pointer types will be the same and we can remove this check.
1076
237
      if (Arg->getType()->isPointerTy()) {
1077
6
        Address Addr = LV.getAddress(*this);
1078
6
        Arg = Builder.CreateBitCast(Arg, Addr.getElementType());
1079
6
      }
1080
237
      EmitStoreOfScalar(Arg, LV);
1081
237
    }
1082
237
  }
1083
349
}
1084
1085
void CodeGenFunction::ExpandTypeToArgs(
1086
    QualType Ty, CallArg Arg, llvm::FunctionType *IRFuncTy,
1087
81
    SmallVectorImpl<llvm::Value *> &IRCallArgs, unsigned &IRCallArgPos) {
1088
81
  auto Exp = getTypeExpansion(Ty, getContext());
1089
81
  if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
1090
0
    Address Addr = Arg.hasLValue() ? Arg.getKnownLValue().getAddress(*this)
1091
0
                                   : Arg.getKnownRValue().getAggregateAddress();
1092
0
    forConstantArrayExpansion(
1093
0
        *this, CAExp, Addr, [&](Address EltAddr) {
1094
0
          CallArg EltArg = CallArg(
1095
0
              convertTempToRValue(EltAddr, CAExp->EltTy, SourceLocation()),
1096
0
              CAExp->EltTy);
1097
0
          ExpandTypeToArgs(CAExp->EltTy, EltArg, IRFuncTy, IRCallArgs,
1098
0
                           IRCallArgPos);
1099
0
        });
1100
81
  } else if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
1101
30
    Address This = Arg.hasLValue() ? 
Arg.getKnownLValue().getAddress(*this)12
1102
30
                                   : 
Arg.getKnownRValue().getAggregateAddress()18
;
1103
30
    for (const CXXBaseSpecifier *BS : RExp->Bases) {
1104
      // Perform a single step derived-to-base conversion.
1105
0
      Address Base =
1106
0
          GetAddressOfBaseClass(This, Ty->getAsCXXRecordDecl(), &BS, &BS + 1,
1107
0
                                /*NullCheckValue=*/false, SourceLocation());
1108
0
      CallArg BaseArg = CallArg(RValue::getAggregate(Base), BS->getType());
1109
1110
      // Recurse onto bases.
1111
0
      ExpandTypeToArgs(BS->getType(), BaseArg, IRFuncTy, IRCallArgs,
1112
0
                       IRCallArgPos);
1113
0
    }
1114
1115
30
    LValue LV = MakeAddrLValue(This, Ty);
1116
48
    for (auto FD : RExp->Fields) {
1117
48
      CallArg FldArg =
1118
48
          CallArg(EmitRValueForField(LV, FD, SourceLocation()), FD->getType());
1119
48
      ExpandTypeToArgs(FD->getType(), FldArg, IRFuncTy, IRCallArgs,
1120
48
                       IRCallArgPos);
1121
48
    }
1122
51
  } else if (isa<ComplexExpansion>(Exp.get())) {
1123
4
    ComplexPairTy CV = Arg.getKnownRValue().getComplexVal();
1124
4
    IRCallArgs[IRCallArgPos++] = CV.first;
1125
4
    IRCallArgs[IRCallArgPos++] = CV.second;
1126
47
  } else {
1127
47
    assert(isa<NoExpansion>(Exp.get()));
1128
0
    auto RV = Arg.getKnownRValue();
1129
47
    assert(RV.isScalar() &&
1130
47
           "Unexpected non-scalar rvalue during struct expansion.");
1131
1132
    // Insert a bitcast as needed.
1133
0
    llvm::Value *V = RV.getScalarVal();
1134
47
    if (IRCallArgPos < IRFuncTy->getNumParams() &&
1135
47
        V->getType() != IRFuncTy->getParamType(IRCallArgPos))
1136
1
      V = Builder.CreateBitCast(V, IRFuncTy->getParamType(IRCallArgPos));
1137
1138
47
    IRCallArgs[IRCallArgPos++] = V;
1139
47
  }
1140
81
}
1141
1142
/// Create a temporary allocation for the purposes of coercion.
1143
static Address CreateTempAllocaForCoercion(CodeGenFunction &CGF, llvm::Type *Ty,
1144
                                           CharUnits MinAlign,
1145
251
                                           const Twine &Name = "tmp") {
1146
  // Don't use an alignment that's worse than what LLVM would prefer.
1147
251
  auto PrefAlign = CGF.CGM.getDataLayout().getPrefTypeAlignment(Ty);
1148
251
  CharUnits Align = std::max(MinAlign, CharUnits::fromQuantity(PrefAlign));
1149
1150
251
  return CGF.CreateTempAlloca(Ty, Align, Name + ".coerce");
1151
251
}
1152
1153
/// EnterStructPointerForCoercedAccess - Given a struct pointer that we are
1154
/// accessing some number of bytes out of it, try to gep into the struct to get
1155
/// at its inner goodness.  Dive as deep as possible without entering an element
1156
/// with an in-memory size smaller than DstSize.
1157
static Address
1158
EnterStructPointerForCoercedAccess(Address SrcPtr,
1159
                                   llvm::StructType *SrcSTy,
1160
19.4k
                                   uint64_t DstSize, CodeGenFunction &CGF) {
1161
  // We can't dive into a zero-element struct.
1162
19.4k
  if (SrcSTy->getNumElements() == 0) 
return SrcPtr0
;
1163
1164
19.4k
  llvm::Type *FirstElt = SrcSTy->getElementType(0);
1165
1166
  // If the first elt is at least as large as what we're looking for, or if the
1167
  // first element is the same size as the whole struct, we can enter it. The
1168
  // comparison must be made on the store size and not the alloca size. Using
1169
  // the alloca size may overstate the size of the load.
1170
19.4k
  uint64_t FirstEltSize =
1171
19.4k
    CGF.CGM.getDataLayout().getTypeStoreSize(FirstElt);
1172
19.4k
  if (FirstEltSize < DstSize &&
1173
19.4k
      
FirstEltSize < CGF.CGM.getDataLayout().getTypeStoreSize(SrcSTy)6.30k
)
1174
6.16k
    return SrcPtr;
1175
1176
  // GEP into the first element.
1177
13.2k
  SrcPtr = CGF.Builder.CreateStructGEP(SrcPtr, 0, "coerce.dive");
1178
1179
  // If the first element is a struct, recurse.
1180
13.2k
  llvm::Type *SrcTy = SrcPtr.getElementType();
1181
13.2k
  if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy))
1182
3.25k
    return EnterStructPointerForCoercedAccess(SrcPtr, SrcSTy, DstSize, CGF);
1183
1184
10.0k
  return SrcPtr;
1185
13.2k
}
1186
1187
/// CoerceIntOrPtrToIntOrPtr - Convert a value Val to the specific Ty where both
1188
/// are either integers or pointers.  This does a truncation of the value if it
1189
/// is too large or a zero extension if it is too small.
1190
///
1191
/// This behaves as if the value were coerced through memory, so on big-endian
1192
/// targets the high bits are preserved in a truncation, while little-endian
1193
/// targets preserve the low bits.
1194
static llvm::Value *CoerceIntOrPtrToIntOrPtr(llvm::Value *Val,
1195
                                             llvm::Type *Ty,
1196
8.71k
                                             CodeGenFunction &CGF) {
1197
8.71k
  if (Val->getType() == Ty)
1198
8.29k
    return Val;
1199
1200
418
  if (isa<llvm::PointerType>(Val->getType())) {
1201
    // If this is Pointer->Pointer avoid conversion to and from int.
1202
302
    if (isa<llvm::PointerType>(Ty))
1203
91
      return CGF.Builder.CreateBitCast(Val, Ty, "coerce.val");
1204
1205
    // Convert the pointer to an integer so we can play with its width.
1206
211
    Val = CGF.Builder.CreatePtrToInt(Val, CGF.IntPtrTy, "coerce.val.pi");
1207
211
  }
1208
1209
327
  llvm::Type *DestIntTy = Ty;
1210
327
  if (isa<llvm::PointerType>(DestIntTy))
1211
31
    DestIntTy = CGF.IntPtrTy;
1212
1213
327
  if (Val->getType() != DestIntTy) {
1214
85
    const llvm::DataLayout &DL = CGF.CGM.getDataLayout();
1215
85
    if (DL.isBigEndian()) {
1216
      // Preserve the high bits on big-endian targets.
1217
      // That is what memory coercion does.
1218
30
      uint64_t SrcSize = DL.getTypeSizeInBits(Val->getType());
1219
30
      uint64_t DstSize = DL.getTypeSizeInBits(DestIntTy);
1220
1221
30
      if (SrcSize > DstSize) {
1222
11
        Val = CGF.Builder.CreateLShr(Val, SrcSize - DstSize, "coerce.highbits");
1223
11
        Val = CGF.Builder.CreateTrunc(Val, DestIntTy, "coerce.val.ii");
1224
19
      } else {
1225
19
        Val = CGF.Builder.CreateZExt(Val, DestIntTy, "coerce.val.ii");
1226
19
        Val = CGF.Builder.CreateShl(Val, DstSize - SrcSize, "coerce.highbits");
1227
19
      }
1228
55
    } else {
1229
      // Little-endian targets preserve the low bits. No shifts required.
1230
55
      Val = CGF.Builder.CreateIntCast(Val, DestIntTy, false, "coerce.val.ii");
1231
55
    }
1232
85
  }
1233
1234
327
  if (isa<llvm::PointerType>(Ty))
1235
31
    Val = CGF.Builder.CreateIntToPtr(Val, Ty, "coerce.val.ip");
1236
327
  return Val;
1237
418
}
1238
1239
1240
1241
/// CreateCoercedLoad - Create a load from \arg SrcPtr interpreted as
1242
/// a pointer to an object of type \arg Ty, known to be aligned to
1243
/// \arg SrcAlign bytes.
1244
///
1245
/// This safely handles the case when the src type is smaller than the
1246
/// destination type; in this situation the values of bits which not
1247
/// present in the src are undefined.
1248
static llvm::Value *CreateCoercedLoad(Address Src, llvm::Type *Ty,
1249
9.76k
                                      CodeGenFunction &CGF) {
1250
9.76k
  llvm::Type *SrcTy = Src.getElementType();
1251
1252
  // If SrcTy and Ty are the same, just do a load.
1253
9.76k
  if (SrcTy == Ty)
1254
31
    return CGF.Builder.CreateLoad(Src);
1255
1256
9.73k
  llvm::TypeSize DstSize = CGF.CGM.getDataLayout().getTypeAllocSize(Ty);
1257
1258
9.73k
  if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy)) {
1259
7.18k
    Src = EnterStructPointerForCoercedAccess(Src, SrcSTy,
1260
7.18k
                                             DstSize.getFixedSize(), CGF);
1261
7.18k
    SrcTy = Src.getElementType();
1262
7.18k
  }
1263
1264
9.73k
  llvm::TypeSize SrcSize = CGF.CGM.getDataLayout().getTypeAllocSize(SrcTy);
1265
1266
  // If the source and destination are integer or pointer types, just do an
1267
  // extension or truncation to the desired type.
1268
9.73k
  if ((isa<llvm::IntegerType>(Ty) || 
isa<llvm::PointerType>(Ty)6.67k
) &&
1269
9.73k
      
(5.92k
isa<llvm::IntegerType>(SrcTy)5.92k
||
isa<llvm::PointerType>(SrcTy)4.68k
)) {
1270
4.31k
    llvm::Value *Load = CGF.Builder.CreateLoad(Src);
1271
4.31k
    return CoerceIntOrPtrToIntOrPtr(Load, Ty, CGF);
1272
4.31k
  }
1273
1274
  // If load is legal, just bitcast the src pointer.
1275
5.41k
  if (!SrcSize.isScalable() && !DstSize.isScalable() &&
1276
5.41k
      
SrcSize.getFixedSize() >= DstSize.getFixedSize()5.07k
) {
1277
    // Generally SrcSize is never greater than DstSize, since this means we are
1278
    // losing bits. However, this can happen in cases where the structure has
1279
    // additional padding, for example due to a user specified alignment.
1280
    //
1281
    // FIXME: Assert that we aren't truncating non-padding bits when have access
1282
    // to that information.
1283
4.93k
    Src = CGF.Builder.CreateElementBitCast(Src, Ty);
1284
4.93k
    return CGF.Builder.CreateLoad(Src);
1285
4.93k
  }
1286
1287
  // If coercing a fixed vector to a scalable vector for ABI compatibility, and
1288
  // the types match, use the llvm.vector.insert intrinsic to perform the
1289
  // conversion.
1290
486
  if (auto *ScalableDst = dyn_cast<llvm::ScalableVectorType>(Ty)) {
1291
338
    if (auto *FixedSrc = dyn_cast<llvm::FixedVectorType>(SrcTy)) {
1292
      // If we are casting a fixed i8 vector to a scalable 16 x i1 predicate
1293
      // vector, use a vector insert and bitcast the result.
1294
338
      bool NeedsBitcast = false;
1295
338
      auto PredType =
1296
338
          llvm::ScalableVectorType::get(CGF.Builder.getInt1Ty(), 16);
1297
338
      llvm::Type *OrigType = Ty;
1298
338
      if (ScalableDst == PredType &&
1299
338
          
FixedSrc->getElementType() == CGF.Builder.getInt8Ty()15
) {
1300
15
        ScalableDst = llvm::ScalableVectorType::get(CGF.Builder.getInt8Ty(), 2);
1301
15
        NeedsBitcast = true;
1302
15
      }
1303
338
      if (ScalableDst->getElementType() == FixedSrc->getElementType()) {
1304
338
        auto *Load = CGF.Builder.CreateLoad(Src);
1305
338
        auto *UndefVec = llvm::UndefValue::get(ScalableDst);
1306
338
        auto *Zero = llvm::Constant::getNullValue(CGF.CGM.Int64Ty);
1307
338
        llvm::Value *Result = CGF.Builder.CreateInsertVector(
1308
338
            ScalableDst, UndefVec, Load, Zero, "castScalableSve");
1309
338
        if (NeedsBitcast)
1310
15
          Result = CGF.Builder.CreateBitCast(Result, OrigType);
1311
338
        return Result;
1312
338
      }
1313
338
    }
1314
338
  }
1315
1316
  // Otherwise do coercion through memory. This is stupid, but simple.
1317
148
  Address Tmp =
1318
148
      CreateTempAllocaForCoercion(CGF, Ty, Src.getAlignment(), Src.getName());
1319
148
  CGF.Builder.CreateMemCpy(
1320
148
      Tmp.getPointer(), Tmp.getAlignment().getAsAlign(), Src.getPointer(),
1321
148
      Src.getAlignment().getAsAlign(),
1322
148
      llvm::ConstantInt::get(CGF.IntPtrTy, SrcSize.getKnownMinSize()));
1323
148
  return CGF.Builder.CreateLoad(Tmp);
1324
486
}
1325
1326
// Function to store a first-class aggregate into memory.  We prefer to
1327
// store the elements rather than the aggregate to be more friendly to
1328
// fast-isel.
1329
// FIXME: Do we need to recurse here?
1330
void CodeGenFunction::EmitAggregateStore(llvm::Value *Val, Address Dest,
1331
6.45k
                                         bool DestIsVolatile) {
1332
  // Prefer scalar stores to first-class aggregate stores.
1333
6.45k
  if (llvm::StructType *STy = dyn_cast<llvm::StructType>(Val->getType())) {
1334
5.92k
    for (unsigned i = 0, e = STy->getNumElements(); i != e; 
++i3.93k
) {
1335
3.93k
      Address EltPtr = Builder.CreateStructGEP(Dest, i);
1336
3.93k
      llvm::Value *Elt = Builder.CreateExtractValue(Val, i);
1337
3.93k
      Builder.CreateStore(Elt, EltPtr, DestIsVolatile);
1338
3.93k
    }
1339
4.46k
  } else {
1340
4.46k
    Builder.CreateStore(Val, Dest, DestIsVolatile);
1341
4.46k
  }
1342
6.45k
}
1343
1344
/// CreateCoercedStore - Create a store to \arg DstPtr from \arg Src,
1345
/// where the source and destination may have different types.  The
1346
/// destination is known to be aligned to \arg DstAlign bytes.
1347
///
1348
/// This safely handles the case when the src type is larger than the
1349
/// destination type; the upper bits of the src will be lost.
1350
static void CreateCoercedStore(llvm::Value *Src,
1351
                               Address Dst,
1352
                               bool DstIsVolatile,
1353
11.0k
                               CodeGenFunction &CGF) {
1354
11.0k
  llvm::Type *SrcTy = Src->getType();
1355
11.0k
  llvm::Type *DstTy = Dst.getElementType();
1356
11.0k
  if (SrcTy == DstTy) {
1357
160
    CGF.Builder.CreateStore(Src, Dst, DstIsVolatile);
1358
160
    return;
1359
160
  }
1360
1361
10.8k
  llvm::TypeSize SrcSize = CGF.CGM.getDataLayout().getTypeAllocSize(SrcTy);
1362
1363
10.8k
  if (llvm::StructType *DstSTy = dyn_cast<llvm::StructType>(DstTy)) {
1364
9.01k
    Dst = EnterStructPointerForCoercedAccess(Dst, DstSTy,
1365
9.01k
                                             SrcSize.getFixedSize(), CGF);
1366
9.01k
    DstTy = Dst.getElementType();
1367
9.01k
  }
1368
1369
10.8k
  llvm::PointerType *SrcPtrTy = llvm::dyn_cast<llvm::PointerType>(SrcTy);
1370
10.8k
  llvm::PointerType *DstPtrTy = llvm::dyn_cast<llvm::PointerType>(DstTy);
1371
10.8k
  if (SrcPtrTy && 
DstPtrTy2.92k
&&
1372
10.8k
      
SrcPtrTy->getAddressSpace() != DstPtrTy->getAddressSpace()2.92k
) {
1373
55
    Src = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Src, DstTy);
1374
55
    CGF.Builder.CreateStore(Src, Dst, DstIsVolatile);
1375
55
    return;
1376
55
  }
1377
1378
  // If the source and destination are integer or pointer types, just do an
1379
  // extension or truncation to the desired type.
1380
10.8k
  if ((isa<llvm::IntegerType>(SrcTy) || 
isa<llvm::PointerType>(SrcTy)7.52k
) &&
1381
10.8k
      
(6.17k
isa<llvm::IntegerType>(DstTy)6.17k
||
isa<llvm::PointerType>(DstTy)4.67k
)) {
1382
4.39k
    Src = CoerceIntOrPtrToIntOrPtr(Src, DstTy, CGF);
1383
4.39k
    CGF.Builder.CreateStore(Src, Dst, DstIsVolatile);
1384
4.39k
    return;
1385
4.39k
  }
1386
1387
6.43k
  llvm::TypeSize DstSize = CGF.CGM.getDataLayout().getTypeAllocSize(DstTy);
1388
1389
  // If store is legal, just bitcast the src pointer.
1390
6.43k
  if (isa<llvm::ScalableVectorType>(SrcTy) ||
1391
6.43k
      
isa<llvm::ScalableVectorType>(DstTy)6.43k
||
1392
6.43k
      
SrcSize.getFixedSize() <= DstSize.getFixedSize()6.43k
) {
1393
6.33k
    Dst = CGF.Builder.CreateElementBitCast(Dst, SrcTy);
1394
6.33k
    CGF.EmitAggregateStore(Src, Dst, DstIsVolatile);
1395
6.33k
  } else {
1396
    // Otherwise do coercion through memory. This is stupid, but
1397
    // simple.
1398
1399
    // Generally SrcSize is never greater than DstSize, since this means we are
1400
    // losing bits. However, this can happen in cases where the structure has
1401
    // additional padding, for example due to a user specified alignment.
1402
    //
1403
    // FIXME: Assert that we aren't truncating non-padding bits when have access
1404
    // to that information.
1405
103
    Address Tmp = CreateTempAllocaForCoercion(CGF, SrcTy, Dst.getAlignment());
1406
103
    CGF.Builder.CreateStore(Src, Tmp);
1407
103
    CGF.Builder.CreateMemCpy(
1408
103
        Dst.getPointer(), Dst.getAlignment().getAsAlign(), Tmp.getPointer(),
1409
103
        Tmp.getAlignment().getAsAlign(),
1410
103
        llvm::ConstantInt::get(CGF.IntPtrTy, DstSize.getFixedSize()));
1411
103
  }
1412
6.43k
}
1413
1414
static Address emitAddressAtOffset(CodeGenFunction &CGF, Address addr,
1415
23.1k
                                   const ABIArgInfo &info) {
1416
23.1k
  if (unsigned offset = info.getDirectOffset()) {
1417
3
    addr = CGF.Builder.CreateElementBitCast(addr, CGF.Int8Ty);
1418
3
    addr = CGF.Builder.CreateConstInBoundsByteGEP(addr,
1419
3
                                             CharUnits::fromQuantity(offset));
1420
3
    addr = CGF.Builder.CreateElementBitCast(addr, info.getCoerceToType());
1421
3
  }
1422
23.1k
  return addr;
1423
23.1k
}
1424
1425
namespace {
1426
1427
/// Encapsulates information about the way function arguments from
1428
/// CGFunctionInfo should be passed to actual LLVM IR function.
1429
class ClangToLLVMArgMapping {
1430
  static const unsigned InvalidIndex = ~0U;
1431
  unsigned InallocaArgNo;
1432
  unsigned SRetArgNo;
1433
  unsigned TotalIRArgs;
1434
1435
  /// Arguments of LLVM IR function corresponding to single Clang argument.
1436
  struct IRArgs {
1437
    unsigned PaddingArgIndex;
1438
    // Argument is expanded to IR arguments at positions
1439
    // [FirstArgIndex, FirstArgIndex + NumberOfArgs).
1440
    unsigned FirstArgIndex;
1441
    unsigned NumberOfArgs;
1442
1443
    IRArgs()
1444
        : PaddingArgIndex(InvalidIndex), FirstArgIndex(InvalidIndex),
1445
2.45M
          NumberOfArgs(0) {}
1446
  };
1447
1448
  SmallVector<IRArgs, 8> ArgInfo;
1449
1450
public:
1451
  ClangToLLVMArgMapping(const ASTContext &Context, const CGFunctionInfo &FI,
1452
                        bool OnlyRequiredArgs = false)
1453
      : InallocaArgNo(InvalidIndex), SRetArgNo(InvalidIndex), TotalIRArgs(0),
1454
2.45M
        ArgInfo(OnlyRequiredArgs ? FI.getNumRequiredArgs() : FI.arg_size()) {
1455
2.45M
    construct(Context, FI, OnlyRequiredArgs);
1456
2.45M
  }
1457
1458
2.98M
  bool hasInallocaArg() const { return InallocaArgNo != InvalidIndex; }
1459
580
  unsigned getInallocaArgNo() const {
1460
580
    assert(hasInallocaArg());
1461
0
    return InallocaArgNo;
1462
580
  }
1463
1464
2.12M
  bool hasSRetArg() const { return SRetArgNo != InvalidIndex; }
1465
22.8k
  unsigned getSRetArgNo() const {
1466
22.8k
    assert(hasSRetArg());
1467
0
    return SRetArgNo;
1468
22.8k
  }
1469
1470
2.45M
  unsigned totalIRArgs() const { return TotalIRArgs; }
1471
1472
3.49M
  bool hasPaddingArg(unsigned ArgNo) const {
1473
3.49M
    assert(ArgNo < ArgInfo.size());
1474
0
    return ArgInfo[ArgNo].PaddingArgIndex != InvalidIndex;
1475
3.49M
  }
1476
69
  unsigned getPaddingArgNo(unsigned ArgNo) const {
1477
69
    assert(hasPaddingArg(ArgNo));
1478
0
    return ArgInfo[ArgNo].PaddingArgIndex;
1479
69
  }
1480
1481
  /// Returns index of first IR argument corresponding to ArgNo, and their
1482
  /// quantity.
1483
3.97M
  std::pair<unsigned, unsigned> getIRArgs(unsigned ArgNo) const {
1484
3.97M
    assert(ArgNo < ArgInfo.size());
1485
0
    return std::make_pair(ArgInfo[ArgNo].FirstArgIndex,
1486
3.97M
                          ArgInfo[ArgNo].NumberOfArgs);
1487
3.97M
  }
1488
1489
private:
1490
  void construct(const ASTContext &Context, const CGFunctionInfo &FI,
1491
                 bool OnlyRequiredArgs);
1492
};
1493
1494
void ClangToLLVMArgMapping::construct(const ASTContext &Context,
1495
                                      const CGFunctionInfo &FI,
1496
2.45M
                                      bool OnlyRequiredArgs) {
1497
2.45M
  unsigned IRArgNo = 0;
1498
2.45M
  bool SwapThisWithSRet = false;
1499
2.45M
  const ABIArgInfo &RetAI = FI.getReturnInfo();
1500
1501
2.45M
  if (RetAI.getKind() == ABIArgInfo::Indirect) {
1502
22.8k
    SwapThisWithSRet = RetAI.isSRetAfterThis();
1503
22.8k
    SRetArgNo = SwapThisWithSRet ? 
1455
:
IRArgNo++22.3k
;
1504
22.8k
  }
1505
1506
2.45M
  unsigned ArgNo = 0;
1507
2.45M
  unsigned NumArgs = OnlyRequiredArgs ? 
FI.getNumRequiredArgs()1.07M
:
FI.arg_size()1.38M
;
1508
6.47M
  for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(); ArgNo < NumArgs;
1509
4.02M
       ++I, ++ArgNo) {
1510
4.02M
    assert(I != FI.arg_end());
1511
0
    QualType ArgType = I->type;
1512
4.02M
    const ABIArgInfo &AI = I->info;
1513
    // Collect data about IR arguments corresponding to Clang argument ArgNo.
1514
4.02M
    auto &IRArgs = ArgInfo[ArgNo];
1515
1516
4.02M
    if (AI.getPaddingType())
1517
124
      IRArgs.PaddingArgIndex = IRArgNo++;
1518
1519
4.02M
    switch (AI.getKind()) {
1520
95.0k
    case ABIArgInfo::Extend:
1521
3.94M
    case ABIArgInfo::Direct: {
1522
      // FIXME: handle sseregparm someday...
1523
3.94M
      llvm::StructType *STy = dyn_cast<llvm::StructType>(AI.getCoerceToType());
1524
3.94M
      if (AI.isDirect() && 
AI.getCanBeFlattened()3.84M
&&
STy3.84M
) {
1525
8.95k
        IRArgs.NumberOfArgs = STy->getNumElements();
1526
3.93M
      } else {
1527
3.93M
        IRArgs.NumberOfArgs = 1;
1528
3.93M
      }
1529
3.94M
      break;
1530
95.0k
    }
1531
18.1k
    case ABIArgInfo::Indirect:
1532
18.2k
    case ABIArgInfo::IndirectAliased:
1533
18.2k
      IRArgs.NumberOfArgs = 1;
1534
18.2k
      break;
1535
55.5k
    case ABIArgInfo::Ignore:
1536
56.4k
    case ABIArgInfo::InAlloca:
1537
      // ignore and inalloca doesn't have matching LLVM parameters.
1538
56.4k
      IRArgs.NumberOfArgs = 0;
1539
56.4k
      break;
1540
3.47k
    case ABIArgInfo::CoerceAndExpand:
1541
3.47k
      IRArgs.NumberOfArgs = AI.getCoerceAndExpandTypeSequence().size();
1542
3.47k
      break;
1543
474
    case ABIArgInfo::Expand:
1544
474
      IRArgs.NumberOfArgs = getExpansionSize(ArgType, Context);
1545
474
      break;
1546
4.02M
    }
1547
1548
4.02M
    if (IRArgs.NumberOfArgs > 0) {
1549
3.96M
      IRArgs.FirstArgIndex = IRArgNo;
1550
3.96M
      IRArgNo += IRArgs.NumberOfArgs;
1551
3.96M
    }
1552
1553
    // Skip over the sret parameter when it comes second.  We already handled it
1554
    // above.
1555
4.02M
    if (IRArgNo == 1 && 
SwapThisWithSRet2.26M
)
1556
455
      IRArgNo++;
1557
4.02M
  }
1558
2.45M
  assert(ArgNo == ArgInfo.size());
1559
1560
2.45M
  if (FI.usesInAlloca())
1561
477
    InallocaArgNo = IRArgNo++;
1562
1563
2.45M
  TotalIRArgs = IRArgNo;
1564
2.45M
}
1565
}  // namespace
1566
1567
/***/
1568
1569
20.1k
bool CodeGenModule::ReturnTypeUsesSRet(const CGFunctionInfo &FI) {
1570
20.1k
  const auto &RI = FI.getReturnInfo();
1571
20.1k
  return RI.isIndirect() || 
(20.1k
RI.isInAlloca()20.1k
&&
RI.getInAllocaSRet()1
);
1572
20.1k
}
1573
1574
12.9k
bool CodeGenModule::ReturnSlotInterferesWithArgs(const CGFunctionInfo &FI) {
1575
12.9k
  return ReturnTypeUsesSRet(FI) &&
1576
12.9k
         
getTargetCodeGenInfo().doesReturnSlotInterfereWithArgs()44
;
1577
12.9k
}
1578
1579
11.7k
bool CodeGenModule::ReturnTypeUsesFPRet(QualType ResultType) {
1580
11.7k
  if (const BuiltinType *BT = ResultType->getAs<BuiltinType>()) {
1581
4.93k
    switch (BT->getKind()) {
1582
4.87k
    default:
1583
4.87k
      return false;
1584
18
    case BuiltinType::Float:
1585
18
      return getTarget().useObjCFPRetForRealType(FloatModeKind::Float);
1586
43
    case BuiltinType::Double:
1587
43
      return getTarget().useObjCFPRetForRealType(FloatModeKind::Double);
1588
4
    case BuiltinType::LongDouble:
1589
4
      return getTarget().useObjCFPRetForRealType(FloatModeKind::LongDouble);
1590
4.93k
    }
1591
4.93k
  }
1592
1593
6.85k
  return false;
1594
11.7k
}
1595
1596
11.7k
bool CodeGenModule::ReturnTypeUsesFP2Ret(QualType ResultType) {
1597
11.7k
  if (const ComplexType *CT = ResultType->getAs<ComplexType>()) {
1598
14
    if (const BuiltinType *BT = CT->getElementType()->getAs<BuiltinType>()) {
1599
14
      if (BT->getKind() == BuiltinType::LongDouble)
1600
2
        return getTarget().useObjCFP2RetForComplexLongDouble();
1601
14
    }
1602
14
  }
1603
1604
11.7k
  return false;
1605
11.7k
}
1606
1607
5.65k
llvm::FunctionType *CodeGenTypes::GetFunctionType(GlobalDecl GD) {
1608
5.65k
  const CGFunctionInfo &FI = arrangeGlobalDeclaration(GD);
1609
5.65k
  return GetFunctionType(FI);
1610
5.65k
}
1611
1612
llvm::FunctionType *
1613
1.07M
CodeGenTypes::GetFunctionType(const CGFunctionInfo &FI) {
1614
1615
1.07M
  bool Inserted = FunctionsBeingProcessed.insert(&FI).second;
1616
1.07M
  (void)Inserted;
1617
1.07M
  assert(Inserted && "Recursively being processed?");
1618
1619
0
  llvm::Type *resultType = nullptr;
1620
1.07M
  const ABIArgInfo &retAI = FI.getReturnInfo();
1621
1.07M
  switch (retAI.getKind()) {
1622
0
  case ABIArgInfo::Expand:
1623
0
  case ABIArgInfo::IndirectAliased:
1624
0
    llvm_unreachable("Invalid ABI kind for return argument");
1625
1626
35.5k
  case ABIArgInfo::Extend:
1627
547k
  case ABIArgInfo::Direct:
1628
547k
    resultType = retAI.getCoerceToType();
1629
547k
    break;
1630
1631
18
  case ABIArgInfo::InAlloca:
1632
18
    if (retAI.getInAllocaSRet()) {
1633
      // sret things on win32 aren't void, they return the sret pointer.
1634
18
      QualType ret = FI.getReturnType();
1635
18
      llvm::Type *ty = ConvertType(ret);
1636
18
      unsigned addressSpace = Context.getTargetAddressSpace(ret);
1637
18
      resultType = llvm::PointerType::get(ty, addressSpace);
1638
18
    } else {
1639
0
      resultType = llvm::Type::getVoidTy(getLLVMContext());
1640
0
    }
1641
18
    break;
1642
1643
9.38k
  case ABIArgInfo::Indirect:
1644
525k
  case ABIArgInfo::Ignore:
1645
525k
    resultType = llvm::Type::getVoidTy(getLLVMContext());
1646
525k
    break;
1647
1648
1.42k
  case ABIArgInfo::CoerceAndExpand:
1649
1.42k
    resultType = retAI.getUnpaddedCoerceAndExpandType();
1650
1.42k
    break;
1651
1.07M
  }
1652
1653
1.07M
  ClangToLLVMArgMapping IRFunctionArgs(getContext(), FI, true);
1654
1.07M
  SmallVector<llvm::Type*, 8> ArgTypes(IRFunctionArgs.totalIRArgs());
1655
1656
  // Add type for sret argument.
1657
1.07M
  if (IRFunctionArgs.hasSRetArg()) {
1658
9.38k
    QualType Ret = FI.getReturnType();
1659
9.38k
    llvm::Type *Ty = ConvertType(Ret);
1660
9.38k
    unsigned AddressSpace = Context.getTargetAddressSpace(Ret);
1661
9.38k
    ArgTypes[IRFunctionArgs.getSRetArgNo()] =
1662
9.38k
        llvm::PointerType::get(Ty, AddressSpace);
1663
9.38k
  }
1664
1665
  // Add type for inalloca argument.
1666
1.07M
  if (IRFunctionArgs.hasInallocaArg()) {
1667
200
    auto ArgStruct = FI.getArgStruct();
1668
200
    assert(ArgStruct);
1669
0
    ArgTypes[IRFunctionArgs.getInallocaArgNo()] = ArgStruct->getPointerTo();
1670
200
  }
1671
1672
  // Add in all of the required arguments.
1673
0
  unsigned ArgNo = 0;
1674
1.07M
  CGFunctionInfo::const_arg_iterator it = FI.arg_begin(),
1675
1.07M
                                     ie = it + FI.getNumRequiredArgs();
1676
2.76M
  for (; it != ie; 
++it, ++ArgNo1.69M
) {
1677
1.69M
    const ABIArgInfo &ArgInfo = it->info;
1678
1679
    // Insert a padding type to ensure proper alignment.
1680
1.69M
    if (IRFunctionArgs.hasPaddingArg(ArgNo))
1681
47
      ArgTypes[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
1682
47
          ArgInfo.getPaddingType();
1683
1684
1.69M
    unsigned FirstIRArg, NumIRArgs;
1685
1.69M
    std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
1686
1687
1.69M
    switch (ArgInfo.getKind()) {
1688
27.4k
    case ABIArgInfo::Ignore:
1689
27.7k
    case ABIArgInfo::InAlloca:
1690
27.7k
      assert(NumIRArgs == 0);
1691
0
      break;
1692
1693
8.02k
    case ABIArgInfo::Indirect: {
1694
8.02k
      assert(NumIRArgs == 1);
1695
      // indirect arguments are always on the stack, which is alloca addr space.
1696
0
      llvm::Type *LTy = ConvertTypeForMem(it->type);
1697
8.02k
      ArgTypes[FirstIRArg] = LTy->getPointerTo(
1698
8.02k
          CGM.getDataLayout().getAllocaAddrSpace());
1699
8.02k
      break;
1700
27.4k
    }
1701
35
    case ABIArgInfo::IndirectAliased: {
1702
35
      assert(NumIRArgs == 1);
1703
0
      llvm::Type *LTy = ConvertTypeForMem(it->type);
1704
35
      ArgTypes[FirstIRArg] = LTy->getPointerTo(ArgInfo.getIndirectAddrSpace());
1705
35
      break;
1706
27.4k
    }
1707
39.9k
    case ABIArgInfo::Extend:
1708
1.65M
    case ABIArgInfo::Direct: {
1709
      // Fast-isel and the optimizer generally like scalar values better than
1710
      // FCAs, so we flatten them if this is safe to do for this argument.
1711
1.65M
      llvm::Type *argType = ArgInfo.getCoerceToType();
1712
1.65M
      llvm::StructType *st = dyn_cast<llvm::StructType>(argType);
1713
1.65M
      if (st && 
ArgInfo.isDirect()4.11k
&&
ArgInfo.getCanBeFlattened()4.11k
) {
1714
3.89k
        assert(NumIRArgs == st->getNumElements());
1715
11.8k
        for (unsigned i = 0, e = st->getNumElements(); i != e; 
++i7.90k
)
1716
7.90k
          ArgTypes[FirstIRArg + i] = st->getElementType(i);
1717
1.65M
      } else {
1718
1.65M
        assert(NumIRArgs == 1);
1719
0
        ArgTypes[FirstIRArg] = argType;
1720
1.65M
      }
1721
0
      break;
1722
39.9k
    }
1723
1724
1.45k
    case ABIArgInfo::CoerceAndExpand: {
1725
1.45k
      auto ArgTypesIter = ArgTypes.begin() + FirstIRArg;
1726
3.12k
      for (auto EltTy : ArgInfo.getCoerceAndExpandTypeSequence()) {
1727
3.12k
        *ArgTypesIter++ = EltTy;
1728
3.12k
      }
1729
1.45k
      assert(ArgTypesIter == ArgTypes.begin() + FirstIRArg + NumIRArgs);
1730
0
      break;
1731
39.9k
    }
1732
1733
181
    case ABIArgInfo::Expand:
1734
181
      auto ArgTypesIter = ArgTypes.begin() + FirstIRArg;
1735
181
      getExpandedTypes(it->type, ArgTypesIter);
1736
181
      assert(ArgTypesIter == ArgTypes.begin() + FirstIRArg + NumIRArgs);
1737
0
      break;
1738
1.69M
    }
1739
1.69M
  }
1740
1741
1.07M
  bool Erased = FunctionsBeingProcessed.erase(&FI); (void)Erased;
1742
1.07M
  assert(Erased && "Not in set?");
1743
1744
0
  return llvm::FunctionType::get(resultType, ArgTypes, FI.isVariadic());
1745
1.07M
}
1746
1747
5.63k
llvm::Type *CodeGenTypes::GetFunctionTypeForVTable(GlobalDecl GD) {
1748
5.63k
  const CXXMethodDecl *MD = cast<CXXMethodDecl>(GD.getDecl());
1749
5.63k
  const FunctionProtoType *FPT = MD->getType()->getAs<FunctionProtoType>();
1750
1751
5.63k
  if (!isFuncTypeConvertible(FPT))
1752
39
    return llvm::StructType::get(getLLVMContext());
1753
1754
5.59k
  return GetFunctionType(GD);
1755
5.63k
}
1756
1757
static void AddAttributesFromFunctionProtoType(ASTContext &Ctx,
1758
                                               llvm::AttrBuilder &FuncAttrs,
1759
1.35M
                                               const FunctionProtoType *FPT) {
1760
1.35M
  if (!FPT)
1761
703k
    return;
1762
1763
649k
  if (!isUnresolvedExceptionSpec(FPT->getExceptionSpecType()) &&
1764
649k
      FPT->isNothrow())
1765
177k
    FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
1766
649k
}
1767
1768
static void AddAttributesFromAssumes(llvm::AttrBuilder &FuncAttrs,
1769
709k
                                     const Decl *Callee) {
1770
709k
  if (!Callee)
1771
32.0k
    return;
1772
1773
677k
  SmallVector<StringRef, 4> Attrs;
1774
1775
677k
  for (const AssumptionAttr *AA : Callee->specific_attrs<AssumptionAttr>())
1776
481
    AA->getAssumption().split(Attrs, ",");
1777
1778
677k
  if (!Attrs.empty())
1779
198
    FuncAttrs.addAttribute(llvm::AssumptionAttrKey,
1780
198
                           llvm::join(Attrs.begin(), Attrs.end(), ","));
1781
677k
}
1782
1783
bool CodeGenModule::MayDropFunctionReturn(const ASTContext &Context,
1784
48
                                          QualType ReturnType) {
1785
  // We can't just discard the return value for a record type with a
1786
  // complex destructor or a non-trivially copyable type.
1787
48
  if (const RecordType *RT =
1788
48
          ReturnType.getCanonicalType()->getAs<RecordType>()) {
1789
24
    if (const auto *ClassDecl = dyn_cast<CXXRecordDecl>(RT->getDecl()))
1790
24
      return ClassDecl->hasTrivialDestructor();
1791
24
  }
1792
24
  return ReturnType.isTriviallyCopyableType(Context);
1793
48
}
1794
1795
void CodeGenModule::getDefaultFunctionAttributes(StringRef Name,
1796
                                                 bool HasOptnone,
1797
                                                 bool AttrOnCallSite,
1798
709k
                                               llvm::AttrBuilder &FuncAttrs) {
1799
  // OptimizeNoneAttr takes precedence over -Os or -Oz. No warning needed.
1800
709k
  if (!HasOptnone) {
1801
709k
    if (CodeGenOpts.OptimizeSize)
1802
1.07k
      FuncAttrs.addAttribute(llvm::Attribute::OptimizeForSize);
1803
709k
    if (CodeGenOpts.OptimizeSize == 2)
1804
718
      FuncAttrs.addAttribute(llvm::Attribute::MinSize);
1805
709k
  }
1806
1807
709k
  if (CodeGenOpts.DisableRedZone)
1808
25
    FuncAttrs.addAttribute(llvm::Attribute::NoRedZone);
1809
709k
  if (CodeGenOpts.IndirectTlsSegRefs)
1810
1
    FuncAttrs.addAttribute("indirect-tls-seg-refs");
1811
709k
  if (CodeGenOpts.NoImplicitFloat)
1812
16
    FuncAttrs.addAttribute(llvm::Attribute::NoImplicitFloat);
1813
1814
709k
  if (AttrOnCallSite) {
1815
    // Attributes that should go on the call site only.
1816
    // FIXME: Look for 'BuiltinAttr' on the function rather than re-checking
1817
    // the -fno-builtin-foo list.
1818
359k
    if (!CodeGenOpts.SimplifyLibCalls || 
LangOpts.isNoBuiltinFunc(Name)342k
)
1819
17.1k
      FuncAttrs.addAttribute(llvm::Attribute::NoBuiltin);
1820
359k
    if (!CodeGenOpts.TrapFuncName.empty())
1821
0
      FuncAttrs.addAttribute("trap-func-name", CodeGenOpts.TrapFuncName);
1822
359k
  } else {
1823
350k
    StringRef FpKind;
1824
350k
    switch (CodeGenOpts.getFramePointer()) {
1825
217k
    case CodeGenOptions::FramePointerKind::None:
1826
217k
      FpKind = "none";
1827
217k
      break;
1828
165
    case CodeGenOptions::FramePointerKind::NonLeaf:
1829
165
      FpKind = "non-leaf";
1830
165
      break;
1831
133k
    case CodeGenOptions::FramePointerKind::All:
1832
133k
      FpKind = "all";
1833
133k
      break;
1834
350k
    }
1835
350k
    FuncAttrs.addAttribute("frame-pointer", FpKind);
1836
1837
350k
    if (CodeGenOpts.LessPreciseFPMAD)
1838
8
      FuncAttrs.addAttribute("less-precise-fpmad", "true");
1839
1840
350k
    if (CodeGenOpts.NullPointerIsValid)
1841
238
      FuncAttrs.addAttribute(llvm::Attribute::NullPointerIsValid);
1842
1843
350k
    if (CodeGenOpts.FPDenormalMode != llvm::DenormalMode::getIEEE())
1844
10
      FuncAttrs.addAttribute("denormal-fp-math",
1845
10
                             CodeGenOpts.FPDenormalMode.str());
1846
350k
    if (CodeGenOpts.FP32DenormalMode != CodeGenOpts.FPDenormalMode) {
1847
22
      FuncAttrs.addAttribute(
1848
22
          "denormal-fp-math-f32",
1849
22
          CodeGenOpts.FP32DenormalMode.str());
1850
22
    }
1851
1852
350k
    if (LangOpts.getDefaultExceptionMode() == LangOptions::FPE_Ignore)
1853
348k
      FuncAttrs.addAttribute("no-trapping-math", "true");
1854
1855
    // TODO: Are these all needed?
1856
    // unsafe/inf/nan/nsz are handled by instruction-level FastMathFlags.
1857
350k
    if (LangOpts.NoHonorInfs)
1858
267
      FuncAttrs.addAttribute("no-infs-fp-math", "true");
1859
350k
    if (LangOpts.NoHonorNaNs)
1860
273
      FuncAttrs.addAttribute("no-nans-fp-math", "true");
1861
350k
    if (LangOpts.ApproxFunc)
1862
255
      FuncAttrs.addAttribute("approx-func-fp-math", "true");
1863
350k
    if (LangOpts.UnsafeFPMath)
1864
254
      FuncAttrs.addAttribute("unsafe-fp-math", "true");
1865
350k
    if (CodeGenOpts.SoftFloat)
1866
61
      FuncAttrs.addAttribute("use-soft-float", "true");
1867
350k
    FuncAttrs.addAttribute("stack-protector-buffer-size",
1868
350k
                           llvm::utostr(CodeGenOpts.SSPBufferSize));
1869
350k
    if (LangOpts.NoSignedZero)
1870
260
      FuncAttrs.addAttribute("no-signed-zeros-fp-math", "true");
1871
1872
    // TODO: Reciprocal estimate codegen options should apply to instructions?
1873
350k
    const std::vector<std::string> &Recips = CodeGenOpts.Reciprocals;
1874
350k
    if (!Recips.empty())
1875
1
      FuncAttrs.addAttribute("reciprocal-estimates",
1876
1
                             llvm::join(Recips, ","));
1877
1878
350k
    if (!CodeGenOpts.PreferVectorWidth.empty() &&
1879
350k
        
CodeGenOpts.PreferVectorWidth != "none"3
)
1880
2
      FuncAttrs.addAttribute("prefer-vector-width",
1881
2
                             CodeGenOpts.PreferVectorWidth);
1882
1883
350k
    if (CodeGenOpts.StackRealignment)
1884
1
      FuncAttrs.addAttribute("stackrealign");
1885
350k
    if (CodeGenOpts.Backchain)
1886
2
      FuncAttrs.addAttribute("backchain");
1887
350k
    if (CodeGenOpts.EnableSegmentedStacks)
1888
6
      FuncAttrs.addAttribute("split-stack");
1889
1890
350k
    if (CodeGenOpts.SpeculativeLoadHardening)
1891
9
      FuncAttrs.addAttribute(llvm::Attribute::SpeculativeLoadHardening);
1892
1893
    // Add zero-call-used-regs attribute.
1894
350k
    switch (CodeGenOpts.getZeroCallUsedRegs()) {
1895
350k
    case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::Skip:
1896
350k
      FuncAttrs.removeAttribute("zero-call-used-regs");
1897
350k
      break;
1898
10
    case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedGPRArg:
1899
10
      FuncAttrs.addAttribute("zero-call-used-regs", "used-gpr-arg");
1900
10
      break;
1901
10
    case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedGPR:
1902
10
      FuncAttrs.addAttribute("zero-call-used-regs", "used-gpr");
1903
10
      break;
1904
10
    case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedArg:
1905
10
      FuncAttrs.addAttribute("zero-call-used-regs", "used-arg");
1906
10
      break;
1907
10
    case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::Used:
1908
10
      FuncAttrs.addAttribute("zero-call-used-regs", "used");
1909
10
      break;
1910
10
    case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllGPRArg:
1911
10
      FuncAttrs.addAttribute("zero-call-used-regs", "all-gpr-arg");
1912
10
      break;
1913
10
    case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllGPR:
1914
10
      FuncAttrs.addAttribute("zero-call-used-regs", "all-gpr");
1915
10
      break;
1916
10
    case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllArg:
1917
10
      FuncAttrs.addAttribute("zero-call-used-regs", "all-arg");
1918
10
      break;
1919
10
    case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::All:
1920
10
      FuncAttrs.addAttribute("zero-call-used-regs", "all");
1921
10
      break;
1922
350k
    }
1923
350k
  }
1924
1925
709k
  if (getLangOpts().assumeFunctionsAreConvergent()) {
1926
    // Conservatively, mark all functions and calls in CUDA and OpenCL as
1927
    // convergent (meaning, they may call an intrinsically convergent op, such
1928
    // as __syncthreads() / barrier(), and so can't have certain optimizations
1929
    // applied around them).  LLVM will remove this attribute where it safely
1930
    // can.
1931
135k
    FuncAttrs.addAttribute(llvm::Attribute::Convergent);
1932
135k
  }
1933
1934
  // TODO: NoUnwind attribute should be added for other GPU modes OpenCL, HIP,
1935
  // SYCL, OpenMP offload. AFAIK, none of them support exceptions in device
1936
  // code.
1937
709k
  if (getLangOpts().CUDA && 
getLangOpts().CUDAIsDevice1.76k
) {
1938
    // Exceptions aren't supported in CUDA device code.
1939
865
    FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
1940
865
  }
1941
1942
709k
  for (StringRef Attr : CodeGenOpts.DefaultFunctionAttrs) {
1943
32
    StringRef Var, Value;
1944
32
    std::tie(Var, Value) = Attr.split('=');
1945
32
    FuncAttrs.addAttribute(Var, Value);
1946
32
  }
1947
709k
}
1948
1949
4
void CodeGenModule::addDefaultFunctionDefinitionAttributes(llvm::Function &F) {
1950
4
  llvm::AttrBuilder FuncAttrs(F.getContext());
1951
4
  getDefaultFunctionAttributes(F.getName(), F.hasOptNone(),
1952
4
                               /* AttrOnCallSite = */ false, FuncAttrs);
1953
  // TODO: call GetCPUAndFeaturesAttributes?
1954
4
  F.addFnAttrs(FuncAttrs);
1955
4
}
1956
1957
void CodeGenModule::addDefaultFunctionDefinitionAttributes(
1958
0
                                                   llvm::AttrBuilder &attrs) {
1959
0
  getDefaultFunctionAttributes(/*function name*/ "", /*optnone*/ false,
1960
0
                               /*for call*/ false, attrs);
1961
0
  GetCPUAndFeaturesAttributes(GlobalDecl(), attrs);
1962
0
}
1963
1964
static void addNoBuiltinAttributes(llvm::AttrBuilder &FuncAttrs,
1965
                                   const LangOptions &LangOpts,
1966
709k
                                   const NoBuiltinAttr *NBA = nullptr) {
1967
709k
  auto AddNoBuiltinAttr = [&FuncAttrs](StringRef BuiltinName) {
1968
2.41k
    SmallString<32> AttributeName;
1969
2.41k
    AttributeName += "no-builtin-";
1970
2.41k
    AttributeName += BuiltinName;
1971
2.41k
    FuncAttrs.addAttribute(AttributeName);
1972
2.41k
  };
1973
1974
  // First, handle the language options passed through -fno-builtin.
1975
709k
  if (LangOpts.NoBuiltin) {
1976
    // -fno-builtin disables them all.
1977
170k
    FuncAttrs.addAttribute("no-builtins");
1978
170k
    return;
1979
170k
  }
1980
1981
  // Then, add attributes for builtins specified through -fno-builtin-<name>.
1982
539k
  llvm::for_each(LangOpts.NoBuiltinFuncs, AddNoBuiltinAttr);
1983
1984
  // Now, let's check the __attribute__((no_builtin("...")) attribute added to
1985
  // the source.
1986
539k
  if (!NBA)
1987
539k
    return;
1988
1989
  // If there is a wildcard in the builtin names specified through the
1990
  // attribute, disable them all.
1991
14
  if (llvm::is_contained(NBA->builtinNames(), "*")) {
1992
2
    FuncAttrs.addAttribute("no-builtins");
1993
2
    return;
1994
2
  }
1995
1996
  // And last, add the rest of the builtin names.
1997
12
  llvm::for_each(NBA->builtinNames(), AddNoBuiltinAttr);
1998
12
}
1999
2000
static bool DetermineNoUndef(QualType QTy, CodeGenTypes &Types,
2001
                             const llvm::DataLayout &DL, const ABIArgInfo &AI,
2002
1.19M
                             bool CheckCoerce = true) {
2003
1.19M
  llvm::Type *Ty = Types.ConvertTypeForMem(QTy);
2004
1.19M
  if (AI.getKind() == ABIArgInfo::Indirect)
2005
4.41k
    return true;
2006
1.18M
  if (AI.getKind() == ABIArgInfo::Extend)
2007
43.4k
    return true;
2008
1.14M
  if (!DL.typeSizeEqualsStoreSize(Ty))
2009
    // TODO: This will result in a modest amount of values not marked noundef
2010
    // when they could be. We care about values that *invisibly* contain undef
2011
    // bits from the perspective of LLVM IR.
2012
43
    return false;
2013
1.14M
  if (CheckCoerce && 
AI.canHaveCoerceToType()1.04M
) {
2014
1.02M
    llvm::Type *CoerceTy = AI.getCoerceToType();
2015
1.02M
    if (llvm::TypeSize::isKnownGT(DL.getTypeSizeInBits(CoerceTy),
2016
1.02M
                                  DL.getTypeSizeInBits(Ty)))
2017
      // If we're coercing to a type with a greater size than the canonical one,
2018
      // we're introducing new undef bits.
2019
      // Coercing to a type of smaller or equal size is ok, as we know that
2020
      // there's no internal padding (typeSizeEqualsStoreSize).
2021
647
      return false;
2022
1.02M
  }
2023
1.14M
  if (QTy->isBitIntType())
2024
3
    return true;
2025
1.14M
  if (QTy->isReferenceType())
2026
151k
    return true;
2027
990k
  if (QTy->isNullPtrType())
2028
198
    return false;
2029
990k
  if (QTy->isMemberPointerType())
2030
    // TODO: Some member pointers are `noundef`, but it depends on the ABI. For
2031
    // now, never mark them.
2032
281
    return false;
2033
989k
  if (QTy->isScalarType()) {
2034
788k
    if (const ComplexType *Complex = dyn_cast<ComplexType>(QTy))
2035
1.11k
      return DetermineNoUndef(Complex->getElementType(), Types, DL, AI, false);
2036
787k
    return true;
2037
788k
  }
2038
201k
  if (const VectorType *Vector = dyn_cast<VectorType>(QTy))
2039
96.4k
    return DetermineNoUndef(Vector->getElementType(), Types, DL, AI, false);
2040
104k
  if (const MatrixType *Matrix = dyn_cast<MatrixType>(QTy))
2041
340
    return DetermineNoUndef(Matrix->getElementType(), Types, DL, AI, false);
2042
104k
  if (const ArrayType *Array = dyn_cast<ArrayType>(QTy))
2043
0
    return DetermineNoUndef(Array->getElementType(), Types, DL, AI, false);
2044
2045
  // TODO: Some structs may be `noundef`, in specific situations.
2046
104k
  return false;
2047
104k
}
2048
2049
/// Construct the IR attribute list of a function or call.
2050
///
2051
/// When adding an attribute, please consider where it should be handled:
2052
///
2053
///   - getDefaultFunctionAttributes is for attributes that are essentially
2054
///     part of the global target configuration (but perhaps can be
2055
///     overridden on a per-function basis).  Adding attributes there
2056
///     will cause them to also be set in frontends that build on Clang's
2057
///     target-configuration logic, as well as for code defined in library
2058
///     modules such as CUDA's libdevice.
2059
///
2060
///   - ConstructAttributeList builds on top of getDefaultFunctionAttributes
2061
///     and adds declaration-specific, convention-specific, and
2062
///     frontend-specific logic.  The last is of particular importance:
2063
///     attributes that restrict how the frontend generates code must be
2064
///     added here rather than getDefaultFunctionAttributes.
2065
///
2066
void CodeGenModule::ConstructAttributeList(StringRef Name,
2067
                                           const CGFunctionInfo &FI,
2068
                                           CGCalleeInfo CalleeInfo,
2069
                                           llvm::AttributeList &AttrList,
2070
                                           unsigned &CallingConv,
2071
709k
                                           bool AttrOnCallSite, bool IsThunk) {
2072
709k
  llvm::AttrBuilder FuncAttrs(getLLVMContext());
2073
709k
  llvm::AttrBuilder RetAttrs(getLLVMContext());
2074
2075
  // Collect function IR attributes from the CC lowering.
2076
  // We'll collect the paramete and result attributes later.
2077
709k
  CallingConv = FI.getEffectiveCallingConvention();
2078
709k
  if (FI.isNoReturn())
2079
1.27k
    FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
2080
709k
  if (FI.isCmseNSCall())
2081
62
    FuncAttrs.addAttribute("cmse_nonsecure_call");
2082
2083
  // Collect function IR attributes from the callee prototype if we have one.
2084
709k
  AddAttributesFromFunctionProtoType(getContext(), FuncAttrs,
2085
709k
                                     CalleeInfo.getCalleeFunctionProtoType());
2086
2087
709k
  const Decl *TargetDecl = CalleeInfo.getCalleeDecl().getDecl();
2088
2089
  // Attach assumption attributes to the declaration. If this is a call
2090
  // site, attach assumptions from the caller to the call as well.
2091
709k
  AddAttributesFromAssumes(FuncAttrs, TargetDecl);
2092
2093
709k
  bool HasOptnone = false;
2094
  // The NoBuiltinAttr attached to the target FunctionDecl.
2095
709k
  const NoBuiltinAttr *NBA = nullptr;
2096
2097
  // Collect function IR attributes based on declaration-specific
2098
  // information.
2099
  // FIXME: handle sseregparm someday...
2100
709k
  if (TargetDecl) {
2101
677k
    if (TargetDecl->hasAttr<ReturnsTwiceAttr>())
2102
173
      FuncAttrs.addAttribute(llvm::Attribute::ReturnsTwice);
2103
677k
    if (TargetDecl->hasAttr<NoThrowAttr>())
2104
5.58k
      FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
2105
677k
    if (TargetDecl->hasAttr<NoReturnAttr>())
2106
0
      FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
2107
677k
    if (TargetDecl->hasAttr<ColdAttr>())
2108
387
      FuncAttrs.addAttribute(llvm::Attribute::Cold);
2109
677k
    if (TargetDecl->hasAttr<HotAttr>())
2110
3
      FuncAttrs.addAttribute(llvm::Attribute::Hot);
2111
677k
    if (TargetDecl->hasAttr<NoDuplicateAttr>())
2112
4
      FuncAttrs.addAttribute(llvm::Attribute::NoDuplicate);
2113
677k
    if (TargetDecl->hasAttr<ConvergentAttr>())
2114
3
      FuncAttrs.addAttribute(llvm::Attribute::Convergent);
2115
2116
677k
    if (const FunctionDecl *Fn = dyn_cast<FunctionDecl>(TargetDecl)) {
2117
643k
      AddAttributesFromFunctionProtoType(
2118
643k
          getContext(), FuncAttrs, Fn->getType()->getAs<FunctionProtoType>());
2119
643k
      if (AttrOnCallSite && 
Fn->isReplaceableGlobalAllocationFunction()336k
) {
2120
        // A sane operator new returns a non-aliasing pointer.
2121
4.00k
        auto Kind = Fn->getDeclName().getCXXOverloadedOperator();
2122
4.00k
        if (getCodeGenOpts().AssumeSaneOperatorNew &&
2123
4.00k
            
(4.00k
Kind == OO_New4.00k
||
Kind == OO_Array_New2.68k
))
2124
1.62k
          RetAttrs.addAttribute(llvm::Attribute::NoAlias);
2125
4.00k
      }
2126
643k
      const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Fn);
2127
643k
      const bool IsVirtualCall = MD && 
MD->isVirtual()293k
;
2128
      // Don't use [[noreturn]], _Noreturn or [[no_builtin]] for a call to a
2129
      // virtual function. These attributes are not inherited by overloads.
2130
643k
      if (!(AttrOnCallSite && 
IsVirtualCall336k
)) {
2131
638k
        if (Fn->isNoReturn())
2132
3.65k
          FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
2133
638k
        NBA = Fn->getAttr<NoBuiltinAttr>();
2134
638k
      }
2135
      // Only place nomerge attribute on call sites, never functions. This
2136
      // allows it to work on indirect virtual function calls.
2137
643k
      if (AttrOnCallSite && 
TargetDecl->hasAttr<NoMergeAttr>()336k
)
2138
12
        FuncAttrs.addAttribute(llvm::Attribute::NoMerge);
2139
643k
    }
2140
2141
    // 'const', 'pure' and 'noalias' attributed functions are also nounwind.
2142
677k
    if (TargetDecl->hasAttr<ConstAttr>()) {
2143
1.84k
      FuncAttrs.addAttribute(llvm::Attribute::ReadNone);
2144
1.84k
      FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
2145
      // gcc specifies that 'const' functions have greater restrictions than
2146
      // 'pure' functions, so they also cannot have infinite loops.
2147
1.84k
      FuncAttrs.addAttribute(llvm::Attribute::WillReturn);
2148
675k
    } else if (TargetDecl->hasAttr<PureAttr>()) {
2149
211
      FuncAttrs.addAttribute(llvm::Attribute::ReadOnly);
2150
211
      FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
2151
      // gcc specifies that 'pure' functions cannot have infinite loops.
2152
211
      FuncAttrs.addAttribute(llvm::Attribute::WillReturn);
2153
675k
    } else if (TargetDecl->hasAttr<NoAliasAttr>()) {
2154
2
      FuncAttrs.addAttribute(llvm::Attribute::ArgMemOnly);
2155
2
      FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
2156
2
    }
2157
677k
    if (TargetDecl->hasAttr<RestrictAttr>())
2158
56
      RetAttrs.addAttribute(llvm::Attribute::NoAlias);
2159
677k
    if (TargetDecl->hasAttr<ReturnsNonNullAttr>() &&
2160
677k
        
!CodeGenOpts.NullPointerIsValid2.53k
)
2161
2.53k
      RetAttrs.addAttribute(llvm::Attribute::NonNull);
2162
677k
    if (TargetDecl->hasAttr<AnyX86NoCallerSavedRegistersAttr>())
2163
14
      FuncAttrs.addAttribute("no_caller_saved_registers");
2164
677k
    if (TargetDecl->hasAttr<AnyX86NoCfCheckAttr>())
2165
2
      FuncAttrs.addAttribute(llvm::Attribute::NoCfCheck);
2166
677k
    if (TargetDecl->hasAttr<LeafAttr>())
2167
1
      FuncAttrs.addAttribute(llvm::Attribute::NoCallback);
2168
2169
677k
    HasOptnone = TargetDecl->hasAttr<OptimizeNoneAttr>();
2170
677k
    if (auto *AllocSize = TargetDecl->getAttr<AllocSizeAttr>()) {
2171
2.91k
      Optional<unsigned> NumElemsParam;
2172
2.91k
      if (AllocSize->getNumElemsParam().isValid())
2173
144
        NumElemsParam = AllocSize->getNumElemsParam().getLLVMIndex();
2174
2.91k
      FuncAttrs.addAllocSizeAttr(AllocSize->getElemSizeParam().getLLVMIndex(),
2175
2.91k
                                 NumElemsParam);
2176
2.91k
    }
2177
2178
677k
    if (TargetDecl->hasAttr<OpenCLKernelAttr>()) {
2179
544
      if (getLangOpts().OpenCLVersion <= 120) {
2180
        // OpenCL v1.2 Work groups are always uniform
2181
366
        FuncAttrs.addAttribute("uniform-work-group-size", "true");
2182
366
      } else {
2183
        // OpenCL v2.0 Work groups may be whether uniform or not.
2184
        // '-cl-uniform-work-group-size' compile option gets a hint
2185
        // to the compiler that the global work-size be a multiple of
2186
        // the work-group size specified to clEnqueueNDRangeKernel
2187
        // (i.e. work groups are uniform).
2188
178
        FuncAttrs.addAttribute("uniform-work-group-size",
2189
178
                               llvm::toStringRef(CodeGenOpts.UniformWGSize));
2190
178
      }
2191
544
    }
2192
677k
  }
2193
2194
  // Attach "no-builtins" attributes to:
2195
  // * call sites: both `nobuiltin` and "no-builtins" or "no-builtin-<name>".
2196
  // * definitions: "no-builtins" or "no-builtin-<name>" only.
2197
  // The attributes can come from:
2198
  // * LangOpts: -ffreestanding, -fno-builtin, -fno-builtin-<name>
2199
  // * FunctionDecl attributes: __attribute__((no_builtin(...)))
2200
709k
  addNoBuiltinAttributes(FuncAttrs, getLangOpts(), NBA);
2201
2202
  // Collect function IR attributes based on global settiings.
2203
709k
  getDefaultFunctionAttributes(Name, HasOptnone, AttrOnCallSite, FuncAttrs);
2204
2205
  // Override some default IR attributes based on declaration-specific
2206
  // information.
2207
709k
  if (TargetDecl) {
2208
677k
    if (TargetDecl->hasAttr<NoSpeculativeLoadHardeningAttr>())
2209
13
      FuncAttrs.removeAttribute(llvm::Attribute::SpeculativeLoadHardening);
2210
677k
    if (TargetDecl->hasAttr<SpeculativeLoadHardeningAttr>())
2211
13
      FuncAttrs.addAttribute(llvm::Attribute::SpeculativeLoadHardening);
2212
677k
    if (TargetDecl->hasAttr<NoSplitStackAttr>())
2213
8
      FuncAttrs.removeAttribute("split-stack");
2214
677k
    if (TargetDecl->hasAttr<ZeroCallUsedRegsAttr>()) {
2215
      // A function "__attribute__((...))" overrides the command-line flag.
2216
81
      auto Kind =
2217
81
          TargetDecl->getAttr<ZeroCallUsedRegsAttr>()->getZeroCallUsedRegs();
2218
81
      FuncAttrs.removeAttribute("zero-call-used-regs");
2219
81
      FuncAttrs.addAttribute(
2220
81
          "zero-call-used-regs",
2221
81
          ZeroCallUsedRegsAttr::ConvertZeroCallUsedRegsKindToStr(Kind));
2222
81
    }
2223
2224
    // Add NonLazyBind attribute to function declarations when -fno-plt
2225
    // is used.
2226
    // FIXME: what if we just haven't processed the function definition
2227
    // yet, or if it's an external definition like C99 inline?
2228
677k
    if (CodeGenOpts.NoPLT) {
2229
45
      if (auto *Fn = dyn_cast<FunctionDecl>(TargetDecl)) {
2230
45
        if (!Fn->isDefined() && 
!AttrOnCallSite22
) {
2231
12
          FuncAttrs.addAttribute(llvm::Attribute::NonLazyBind);
2232
12
        }
2233
45
      }
2234
45
    }
2235
677k
  }
2236
2237
  // Add "sample-profile-suffix-elision-policy" attribute for internal linkage
2238
  // functions with -funique-internal-linkage-names.
2239
709k
  if (TargetDecl && 
CodeGenOpts.UniqueInternalLinkageNames677k
) {
2240
47
    if (isa<FunctionDecl>(TargetDecl)) {
2241
47
      if (this->getFunctionLinkage(CalleeInfo.getCalleeDecl()) ==
2242
47
          llvm::GlobalValue::InternalLinkage)
2243
34
        FuncAttrs.addAttribute("sample-profile-suffix-elision-policy",
2244
34
                               "selected");
2245
47
    }
2246
47
  }
2247
2248
  // Collect non-call-site function IR attributes from declaration-specific
2249
  // information.
2250
709k
  if (!AttrOnCallSite) {
2251
350k
    if (TargetDecl && 
TargetDecl->hasAttr<CmseNSEntryAttr>()335k
)
2252
94
      FuncAttrs.addAttribute("cmse_nonsecure_entry");
2253
2254
    // Whether tail calls are enabled.
2255
350k
    auto shouldDisableTailCalls = [&] {
2256
      // Should this be honored in getDefaultFunctionAttributes?
2257
350k
      if (CodeGenOpts.DisableTailCalls)
2258
3
        return true;
2259
2260
350k
      if (!TargetDecl)
2261
15.0k
        return false;
2262
2263
335k
      if (TargetDecl->hasAttr<DisableTailCallsAttr>() ||
2264
335k
          
TargetDecl->hasAttr<AnyX86InterruptAttr>()335k
)
2265
147
        return true;
2266
2267
335k
      if (CodeGenOpts.NoEscapingBlockTailCalls) {
2268
17
        if (const auto *BD = dyn_cast<BlockDecl>(TargetDecl))
2269
8
          if (!BD->doesNotEscape())
2270
4
            return true;
2271
17
      }
2272
2273
335k
      return false;
2274
335k
    };
2275
350k
    if (shouldDisableTailCalls())
2276
154
      FuncAttrs.addAttribute("disable-tail-calls", "true");
2277
2278
    // CPU/feature overrides.  addDefaultFunctionDefinitionAttributes
2279
    // handles these separately to set them based on the global defaults.
2280
350k
    GetCPUAndFeaturesAttributes(CalleeInfo.getCalleeDecl(), FuncAttrs);
2281
350k
  }
2282
2283
  // Collect attributes from arguments and return values.
2284
709k
  ClangToLLVMArgMapping IRFunctionArgs(getContext(), FI);
2285
2286
709k
  QualType RetTy = FI.getReturnType();
2287
709k
  const ABIArgInfo &RetAI = FI.getReturnInfo();
2288
709k
  const llvm::DataLayout &DL = getDataLayout();
2289
2290
  // C++ explicitly makes returning undefined values UB. C's rule only applies
2291
  // to used values, so we never mark them noundef for now.
2292
709k
  bool HasStrictReturn = getLangOpts().CPlusPlus;
2293
709k
  if (TargetDecl && 
HasStrictReturn677k
) {
2294
544k
    if (const FunctionDecl *FDecl = dyn_cast<FunctionDecl>(TargetDecl))
2295
513k
      HasStrictReturn &= !FDecl->isExternC();
2296
30.7k
    else if (const VarDecl *VDecl = dyn_cast<VarDecl>(TargetDecl))
2297
      // Function pointer
2298
5.08k
      HasStrictReturn &= !VDecl->isExternC();
2299
544k
  }
2300
2301
  // We don't want to be too aggressive with the return checking, unless
2302
  // it's explicit in the code opts or we're using an appropriate sanitizer.
2303
  // Try to respect what the programmer intended.
2304
709k
  HasStrictReturn &= getCodeGenOpts().StrictReturn ||
2305
709k
                     
!MayDropFunctionReturn(getContext(), RetTy)27
||
2306
709k
                     
getLangOpts().Sanitize.has(SanitizerKind::Memory)21
||
2307
709k
                     
getLangOpts().Sanitize.has(SanitizerKind::Return)21
;
2308
2309
  // Determine if the return type could be partially undef
2310
709k
  if (CodeGenOpts.EnableNoundefAttrs && 
HasStrictReturn566k
) {
2311
403k
    if (!RetTy->isVoidType() && 
RetAI.getKind() != ABIArgInfo::Indirect156k
&&
2312
403k
        
DetermineNoUndef(RetTy, getTypes(), DL, RetAI)154k
)
2313
134k
      RetAttrs.addAttribute(llvm::Attribute::NoUndef);
2314
403k
  }
2315
2316
709k
  switch (RetAI.getKind()) {
2317
21.9k
  case ABIArgInfo::Extend:
2318
21.9k
    if (RetAI.isSignExt())
2319
8.22k
      RetAttrs.addAttribute(llvm::Attribute::SExt);
2320
13.7k
    else
2321
13.7k
      RetAttrs.addAttribute(llvm::Attribute::ZExt);
2322
21.9k
    LLVM_FALLTHROUGH;
2323
385k
  case ABIArgInfo::Direct:
2324
385k
    if (RetAI.getInReg())
2325
16
      RetAttrs.addAttribute(llvm::Attribute::InReg);
2326
385k
    break;
2327
316k
  case ABIArgInfo::Ignore:
2328
316k
    break;
2329
2330
13
  case ABIArgInfo::InAlloca:
2331
6.94k
  case ABIArgInfo::Indirect: {
2332
    // inalloca and sret disable readnone and readonly
2333
6.94k
    FuncAttrs.removeAttribute(llvm::Attribute::ReadOnly)
2334
6.94k
      .removeAttribute(llvm::Attribute::ReadNone);
2335
6.94k
    break;
2336
13
  }
2337
2338
983
  case ABIArgInfo::CoerceAndExpand:
2339
983
    break;
2340
2341
0
  case ABIArgInfo::Expand:
2342
0
  case ABIArgInfo::IndirectAliased:
2343
0
    llvm_unreachable("Invalid ABI kind for return argument");
2344
709k
  }
2345
2346
709k
  if (!IsThunk) {
2347
    // FIXME: fix this properly, https://reviews.llvm.org/D100388
2348
708k
    if (const auto *RefTy = RetTy->getAs<ReferenceType>()) {
2349
50.5k
      QualType PTy = RefTy->getPointeeType();
2350
50.5k
      if (!PTy->isIncompleteType() && 
PTy->isConstantSizeType()50.4k
)
2351
50.4k
        RetAttrs.addDereferenceableAttr(
2352
50.4k
            getMinimumObjectSize(PTy).getQuantity());
2353
50.5k
      if (getContext().getTargetAddressSpace(PTy) == 0 &&
2354
50.5k
          
!CodeGenOpts.NullPointerIsValid50.5k
)
2355
50.4k
        RetAttrs.addAttribute(llvm::Attribute::NonNull);
2356
50.5k
      if (PTy->isObjectType()) {
2357
50.5k
        llvm::Align Alignment =
2358
50.5k
            getNaturalPointeeTypeAlignment(RetTy).getAsAlign();
2359
50.5k
        RetAttrs.addAlignmentAttr(Alignment);
2360
50.5k
      }
2361
50.5k
    }
2362
708k
  }
2363
2364
709k
  bool hasUsedSRet = false;
2365
709k
  SmallVector<llvm::AttributeSet, 4> ArgAttrs(IRFunctionArgs.totalIRArgs());
2366
2367
  // Attach attributes to sret.
2368
709k
  if (IRFunctionArgs.hasSRetArg()) {
2369
6.92k
    llvm::AttrBuilder SRETAttrs(getLLVMContext());
2370
6.92k
    SRETAttrs.addStructRetAttr(getTypes().ConvertTypeForMem(RetTy));
2371
6.92k
    hasUsedSRet = true;
2372
6.92k
    if (RetAI.getInReg())
2373
64
      SRETAttrs.addAttribute(llvm::Attribute::InReg);
2374
6.92k
    SRETAttrs.addAlignmentAttr(RetAI.getIndirectAlign().getQuantity());
2375
6.92k
    ArgAttrs[IRFunctionArgs.getSRetArgNo()] =
2376
6.92k
        llvm::AttributeSet::get(getLLVMContext(), SRETAttrs);
2377
6.92k
  }
2378
2379
  // Attach attributes to inalloca argument.
2380
709k
  if (IRFunctionArgs.hasInallocaArg()) {
2381
162
    llvm::AttrBuilder Attrs(getLLVMContext());
2382
162
    Attrs.addInAllocaAttr(FI.getArgStruct());
2383
162
    ArgAttrs[IRFunctionArgs.getInallocaArgNo()] =
2384
162
        llvm::AttributeSet::get(getLLVMContext(), Attrs);
2385
162
  }
2386
2387
  // Apply `nonnull`, `dereferencable(N)` and `align N` to the `this` argument,
2388
  // unless this is a thunk function.
2389
  // FIXME: fix this properly, https://reviews.llvm.org/D100388
2390
709k
  if (FI.isInstanceMethod() && 
!IRFunctionArgs.hasInallocaArg()276k
&&
2391
709k
      
!FI.arg_begin()->type->isVoidPointerType()276k
&&
!IsThunk275k
) {
2392
275k
    auto IRArgs = IRFunctionArgs.getIRArgs(0);
2393
2394
275k
    assert(IRArgs.second == 1 && "Expected only a single `this` pointer.");
2395
2396
0
    llvm::AttrBuilder Attrs(getLLVMContext());
2397
2398
275k
    QualType ThisTy =
2399
275k
        FI.arg_begin()->type.castAs<PointerType>()->getPointeeType();
2400
2401
275k
    if (!CodeGenOpts.NullPointerIsValid &&
2402
275k
        
getContext().getTargetAddressSpace(FI.arg_begin()->type) == 0274k
) {
2403
274k
      Attrs.addAttribute(llvm::Attribute::NonNull);
2404
274k
      Attrs.addDereferenceableAttr(getMinimumObjectSize(ThisTy).getQuantity());
2405
274k
    } else {
2406
      // FIXME dereferenceable should be correct here, regardless of
2407
      // NullPointerIsValid. However, dereferenceable currently does not always
2408
      // respect NullPointerIsValid and may imply nonnull and break the program.
2409
      // See https://reviews.llvm.org/D66618 for discussions.
2410
322
      Attrs.addDereferenceableOrNullAttr(
2411
322
          getMinimumObjectSize(
2412
322
              FI.arg_begin()->type.castAs<PointerType>()->getPointeeType())
2413
322
              .getQuantity());
2414
322
    }
2415
2416
275k
    llvm::Align Alignment =
2417
275k
        getNaturalTypeAlignment(ThisTy, /*BaseInfo=*/nullptr,
2418
275k
                                /*TBAAInfo=*/nullptr, /*forPointeeType=*/true)
2419
275k
            .getAsAlign();
2420
275k
    Attrs.addAlignmentAttr(Alignment);
2421
2422
275k
    ArgAttrs[IRArgs.first] = llvm::AttributeSet::get(getLLVMContext(), Attrs);
2423
275k
  }
2424
2425
0
  unsigned ArgNo = 0;
2426
709k
  for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(),
2427
709k
                                          E = FI.arg_end();
2428
1.89M
       I != E; 
++I, ++ArgNo1.18M
) {
2429
1.18M
    QualType ParamType = I->type;
2430
1.18M
    const ABIArgInfo &AI = I->info;
2431
1.18M
    llvm::AttrBuilder Attrs(getLLVMContext());
2432
2433
    // Add attribute for padding argument, if necessary.
2434
1.18M
    if (IRFunctionArgs.hasPaddingArg(ArgNo)) {
2435
47
      if (AI.getPaddingInReg()) {
2436
5
        ArgAttrs[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
2437
5
            llvm::AttributeSet::get(
2438
5
                getLLVMContext(),
2439
5
                llvm::AttrBuilder(getLLVMContext()).addAttribute(llvm::Attribute::InReg));
2440
5
      }
2441
47
    }
2442
2443
    // Decide whether the argument we're handling could be partially undef
2444
1.18M
    if (CodeGenOpts.EnableNoundefAttrs &&
2445
1.18M
        
DetermineNoUndef(ParamType, getTypes(), DL, AI)938k
) {
2446
852k
      Attrs.addAttribute(llvm::Attribute::NoUndef);
2447
852k
    }
2448
2449
    // 'restrict' -> 'noalias' is done in EmitFunctionProlog when we
2450
    // have the corresponding parameter variable.  It doesn't make
2451
    // sense to do it here because parameters are so messed up.
2452
1.18M
    switch (AI.getKind()) {
2453
27.7k
    case ABIArgInfo::Extend:
2454
27.7k
      if (AI.isSignExt())
2455
11.6k
        Attrs.addAttribute(llvm::Attribute::SExt);
2456
16.1k
      else
2457
16.1k
        Attrs.addAttribute(llvm::Attribute::ZExt);
2458
27.7k
      LLVM_FALLTHROUGH;
2459
1.16M
    case ABIArgInfo::Direct:
2460
1.16M
      if (ArgNo == 0 && 
FI.isChainCall()635k
)
2461
8
        Attrs.addAttribute(llvm::Attribute::Nest);
2462
1.16M
      else if (AI.getInReg())
2463
580
        Attrs.addAttribute(llvm::Attribute::InReg);
2464
1.16M
      Attrs.addStackAlignmentAttr(llvm::MaybeAlign(AI.getDirectAlign()));
2465
1.16M
      break;
2466
2467
5.31k
    case ABIArgInfo::Indirect: {
2468
5.31k
      if (AI.getInReg())
2469
24
        Attrs.addAttribute(llvm::Attribute::InReg);
2470
2471
5.31k
      if (AI.getIndirectByVal())
2472
2.06k
        Attrs.addByValAttr(getTypes().ConvertTypeForMem(ParamType));
2473
2474
5.31k
      auto *Decl = ParamType->getAsRecordDecl();
2475
5.31k
      if (CodeGenOpts.PassByValueIsNoAlias && 
Decl7
&&
2476
5.31k
          
Decl->getArgPassingRestrictions() == RecordDecl::APK_CanPassInRegs7
)
2477
        // When calling the function, the pointer passed in will be the only
2478
        // reference to the underlying object. Mark it accordingly.
2479
5
        Attrs.addAttribute(llvm::Attribute::NoAlias);
2480
2481
      // TODO: We could add the byref attribute if not byval, but it would
2482
      // require updating many testcases.
2483
2484
5.31k
      CharUnits Align = AI.getIndirectAlign();
2485
2486
      // In a byval argument, it is important that the required
2487
      // alignment of the type is honored, as LLVM might be creating a
2488
      // *new* stack object, and needs to know what alignment to give
2489
      // it. (Sometimes it can deduce a sensible alignment on its own,
2490
      // but not if clang decides it must emit a packed struct, or the
2491
      // user specifies increased alignment requirements.)
2492
      //
2493
      // This is different from indirect *not* byval, where the object
2494
      // exists already, and the align attribute is purely
2495
      // informative.
2496
5.31k
      assert(!Align.isZero());
2497
2498
      // For now, only add this when we have a byval argument.
2499
      // TODO: be less lazy about updating test cases.
2500
5.31k
      if (AI.getIndirectByVal())
2501
2.06k
        Attrs.addAlignmentAttr(Align.getQuantity());
2502
2503
      // byval disables readnone and readonly.
2504
5.31k
      FuncAttrs.removeAttribute(llvm::Attribute::ReadOnly)
2505
5.31k
        .removeAttribute(llvm::Attribute::ReadNone);
2506
2507
5.31k
      break;
2508
27.7k
    }
2509
20
    case ABIArgInfo::IndirectAliased: {
2510
20
      CharUnits Align = AI.getIndirectAlign();
2511
20
      Attrs.addByRefAttr(getTypes().ConvertTypeForMem(ParamType));
2512
20
      Attrs.addAlignmentAttr(Align.getQuantity());
2513
20
      break;
2514
27.7k
    }
2515
14.1k
    case ABIArgInfo::Ignore:
2516
14.3k
    case ABIArgInfo::Expand:
2517
15.3k
    case ABIArgInfo::CoerceAndExpand:
2518
15.3k
      break;
2519
2520
321
    case ABIArgInfo::InAlloca:
2521
      // inalloca disables readnone and readonly.
2522
321
      FuncAttrs.removeAttribute(llvm::Attribute::ReadOnly)
2523
321
          .removeAttribute(llvm::Attribute::ReadNone);
2524
321
      continue;
2525
1.18M
    }
2526
2527
1.18M
    if (const auto *RefTy = ParamType->getAs<ReferenceType>()) {
2528
119k
      QualType PTy = RefTy->getPointeeType();
2529
119k
      if (!PTy->isIncompleteType() && 
PTy->isConstantSizeType()119k
)
2530
119k
        Attrs.addDereferenceableAttr(
2531
119k
            getMinimumObjectSize(PTy).getQuantity());
2532
119k
      if (getContext().getTargetAddressSpace(PTy) == 0 &&
2533
119k
          
!CodeGenOpts.NullPointerIsValid119k
)
2534
119k
        Attrs.addAttribute(llvm::Attribute::NonNull);
2535
119k
      if (PTy->isObjectType()) {
2536
116k
        llvm::Align Alignment =
2537
116k
            getNaturalPointeeTypeAlignment(ParamType).getAsAlign();
2538
116k
        Attrs.addAlignmentAttr(Alignment);
2539
116k
      }
2540
119k
    }
2541
2542
    // From OpenCL spec v3.0.10 section 6.3.5 Alignment of Types:
2543
    // > For arguments to a __kernel function declared to be a pointer to a
2544
    // > data type, the OpenCL compiler can assume that the pointee is always
2545
    // > appropriately aligned as required by the data type.
2546
1.18M
    if (TargetDecl && 
TargetDecl->hasAttr<OpenCLKernelAttr>()1.11M
&&
2547
1.18M
        
ParamType->isPointerType()661
) {
2548
327
      QualType PTy = ParamType->getPointeeType();
2549
327
      if (!PTy->isIncompleteType() && 
PTy->isConstantSizeType()326
) {
2550
326
        llvm::Align Alignment =
2551
326
            getNaturalPointeeTypeAlignment(ParamType).getAsAlign();
2552
326
        Attrs.addAlignmentAttr(Alignment);
2553
326
      }
2554
327
    }
2555
2556
1.18M
    switch (FI.getExtParameterInfo(ArgNo).getABI()) {
2557
1.18M
    case ParameterABI::Ordinary:
2558
1.18M
      break;
2559
2560
48
    case ParameterABI::SwiftIndirectResult: {
2561
      // Add 'sret' if we haven't already used it for something, but
2562
      // only if the result is void.
2563
48
      if (!hasUsedSRet && 
RetTy->isVoidType()24
) {
2564
8
        Attrs.addStructRetAttr(getTypes().ConvertTypeForMem(ParamType));
2565
8
        hasUsedSRet = true;
2566
8
      }
2567
2568
      // Add 'noalias' in either case.
2569
48
      Attrs.addAttribute(llvm::Attribute::NoAlias);
2570
2571
      // Add 'dereferenceable' and 'alignment'.
2572
48
      auto PTy = ParamType->getPointeeType();
2573
48
      if (!PTy->isIncompleteType() && PTy->isConstantSizeType()) {
2574
48
        auto info = getContext().getTypeInfoInChars(PTy);
2575
48
        Attrs.addDereferenceableAttr(info.Width.getQuantity());
2576
48
        Attrs.addAlignmentAttr(info.Align.getAsAlign());
2577
48
      }
2578
48
      break;
2579
0
    }
2580
2581
24
    case ParameterABI::SwiftErrorResult:
2582
24
      Attrs.addAttribute(llvm::Attribute::SwiftError);
2583
24
      break;
2584
2585
40
    case ParameterABI::SwiftContext:
2586
40
      Attrs.addAttribute(llvm::Attribute::SwiftSelf);
2587
40
      break;
2588
2589
325
    case ParameterABI::SwiftAsyncContext:
2590
325
      Attrs.addAttribute(llvm::Attribute::SwiftAsync);
2591
325
      break;
2592
1.18M
    }
2593
2594
1.18M
    if (FI.getExtParameterInfo(ArgNo).isNoEscape())
2595
62
      Attrs.addAttribute(llvm::Attribute::NoCapture);
2596
2597
1.18M
    if (Attrs.hasAttributes()) {
2598
864k
      unsigned FirstIRArg, NumIRArgs;
2599
864k
      std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
2600
1.73M
      for (unsigned i = 0; i < NumIRArgs; 
i++865k
)
2601
865k
        ArgAttrs[FirstIRArg + i] = ArgAttrs[FirstIRArg + i].addAttributes(
2602
865k
            getLLVMContext(), llvm::AttributeSet::get(getLLVMContext(), Attrs));
2603
864k
    }
2604
1.18M
  }
2605
709k
  assert(ArgNo == FI.arg_size());
2606
2607
0
  AttrList = llvm::AttributeList::get(
2608
709k
      getLLVMContext(), llvm::AttributeSet::get(getLLVMContext(), FuncAttrs),
2609
709k
      llvm::AttributeSet::get(getLLVMContext(), RetAttrs), ArgAttrs);
2610
709k
}
2611
2612
/// An argument came in as a promoted argument; demote it back to its
2613
/// declared type.
2614
static llvm::Value *emitArgumentDemotion(CodeGenFunction &CGF,
2615
                                         const VarDecl *var,
2616
14
                                         llvm::Value *value) {
2617
14
  llvm::Type *varType = CGF.ConvertType(var->getType());
2618
2619
  // This can happen with promotions that actually don't change the
2620
  // underlying type, like the enum promotions.
2621
14
  if (value->getType() == varType) 
return value0
;
2622
2623
14
  assert((varType->isIntegerTy() || varType->isFloatingPointTy())
2624
14
         && "unexpected promotion type");
2625
2626
14
  if (isa<llvm::IntegerType>(varType))
2627
8
    return CGF.Builder.CreateTrunc(value, varType, "arg.unpromote");
2628
2629
6
  return CGF.Builder.CreateFPCast(value, varType, "arg.unpromote");
2630
14
}
2631
2632
/// Returns the attribute (either parameter attribute, or function
2633
/// attribute), which declares argument ArgNo to be non-null.
2634
static const NonNullAttr *getNonNullAttr(const Decl *FD, const ParmVarDecl *PVD,
2635
112k
                                         QualType ArgType, unsigned ArgNo) {
2636
  // FIXME: __attribute__((nonnull)) can also be applied to:
2637
  //   - references to pointers, where the pointee is known to be
2638
  //     nonnull (apparently a Clang extension)
2639
  //   - transparent unions containing pointers
2640
  // In the former case, LLVM IR cannot represent the constraint. In
2641
  // the latter case, we have no guarantee that the transparent union
2642
  // is in fact passed as a pointer.
2643
112k
  if (!ArgType->isAnyPointerType() && 
!ArgType->isBlockPointerType()43.2k
)
2644
43.2k
    return nullptr;
2645
  // First, check attribute on parameter itself.
2646
68.9k
  if (PVD) {
2647
68.9k
    if (auto ParmNNAttr = PVD->getAttr<NonNullAttr>())
2648
10
      return ParmNNAttr;
2649
68.9k
  }
2650
  // Check function attributes.
2651
68.9k
  if (!FD)
2652
5
    return nullptr;
2653
68.9k
  for (const auto *NNAttr : FD->specific_attrs<NonNullAttr>()) {
2654
32
    if (NNAttr->isNonNull(ArgNo))
2655
28
      return NNAttr;
2656
32
  }
2657
68.8k
  return nullptr;
2658
68.9k
}
2659
2660
namespace {
2661
  struct CopyBackSwiftError final : EHScopeStack::Cleanup {
2662
    Address Temp;
2663
    Address Arg;
2664
16
    CopyBackSwiftError(Address temp, Address arg) : Temp(temp), Arg(arg) {}
2665
16
    void Emit(CodeGenFunction &CGF, Flags flags) override {
2666
16
      llvm::Value *errorValue = CGF.Builder.CreateLoad(Temp);
2667
16
      CGF.Builder.CreateStore(errorValue, Arg);
2668
16
    }
2669
  };
2670
}
2671
2672
void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
2673
                                         llvm::Function *Fn,
2674
314k
                                         const FunctionArgList &Args) {
2675
314k
  if (CurCodeDecl && 
CurCodeDecl->hasAttr<NakedAttr>()305k
)
2676
    // Naked functions don't have prologues.
2677
10
    return;
2678
2679
  // If this is an implicit-return-zero function, go ahead and
2680
  // initialize the return value.  TODO: it might be nice to have
2681
  // a more general mechanism for this that didn't require synthesized
2682
  // return statements.
2683
314k
  if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(CurCodeDecl)) {
2684
269k
    if (FD->hasImplicitReturnZero()) {
2685
5.72k
      QualType RetTy = FD->getReturnType().getUnqualifiedType();
2686
5.72k
      llvm::Type* LLVMTy = CGM.getTypes().ConvertType(RetTy);
2687
5.72k
      llvm::Constant* Zero = llvm::Constant::getNullValue(LLVMTy);
2688
5.72k
      Builder.CreateStore(Zero, ReturnValue);
2689
5.72k
    }
2690
269k
  }
2691
2692
  // FIXME: We no longer need the types from FunctionArgList; lift up and
2693
  // simplify.
2694
2695
314k
  ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), FI);
2696
314k
  assert(Fn->arg_size() == IRFunctionArgs.totalIRArgs());
2697
2698
  // If we're using inalloca, all the memory arguments are GEPs off of the last
2699
  // parameter, which is a pointer to the complete memory area.
2700
0
  Address ArgStruct = Address::invalid();
2701
314k
  if (IRFunctionArgs.hasInallocaArg()) {
2702
41
    ArgStruct = Address(Fn->getArg(IRFunctionArgs.getInallocaArgNo()),
2703
41
                        FI.getArgStruct(), FI.getArgStructAlignment());
2704
2705
41
    assert(ArgStruct.getType() == FI.getArgStruct()->getPointerTo());
2706
41
  }
2707
2708
  // Name the struct return parameter.
2709
314k
  if (IRFunctionArgs.hasSRetArg()) {
2710
4.25k
    auto AI = Fn->getArg(IRFunctionArgs.getSRetArgNo());
2711
4.25k
    AI->setName("agg.result");
2712
4.25k
    AI->addAttr(llvm::Attribute::NoAlias);
2713
4.25k
  }
2714
2715
  // Track if we received the parameter as a pointer (indirect, byval, or
2716
  // inalloca).  If already have a pointer, EmitParmDecl doesn't need to copy it
2717
  // into a local alloca for us.
2718
314k
  SmallVector<ParamValue, 16> ArgVals;
2719
314k
  ArgVals.reserve(Args.size());
2720
2721
  // Create a pointer value for every parameter declaration.  This usually
2722
  // entails copying one or more LLVM IR arguments into an alloca.  Don't push
2723
  // any cleanups or do anything that might unwind.  We do that separately, so
2724
  // we can push the cleanups in the correct order for the ABI.
2725
314k
  assert(FI.arg_size() == Args.size() &&
2726
314k
         "Mismatch between function signature & arguments.");
2727
0
  unsigned ArgNo = 0;
2728
314k
  CGFunctionInfo::const_arg_iterator info_it = FI.arg_begin();
2729
314k
  for (FunctionArgList::const_iterator i = Args.begin(), e = Args.end();
2730
843k
       i != e; 
++i, ++info_it, ++ArgNo529k
) {
2731
529k
    const VarDecl *Arg = *i;
2732
529k
    const ABIArgInfo &ArgI = info_it->info;
2733
2734
529k
    bool isPromoted =
2735
529k
      isa<ParmVarDecl>(Arg) && 
cast<ParmVarDecl>(Arg)->isKNRPromoted()336k
;
2736
    // We are converting from ABIArgInfo type to VarDecl type directly, unless
2737
    // the parameter is promoted. In this case we convert to
2738
    // CGFunctionInfo::ArgInfo type with subsequent argument demotion.
2739
529k
    QualType Ty = isPromoted ? 
info_it->type14
:
Arg->getType()529k
;
2740
529k
    assert(hasScalarEvaluationKind(Ty) ==
2741
529k
           hasScalarEvaluationKind(Arg->getType()));
2742
2743
0
    unsigned FirstIRArg, NumIRArgs;
2744
529k
    std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
2745
2746
529k
    switch (ArgI.getKind()) {
2747
73
    case ABIArgInfo::InAlloca: {
2748
73
      assert(NumIRArgs == 0);
2749
0
      auto FieldIndex = ArgI.getInAllocaFieldIndex();
2750
73
      Address V =
2751
73
          Builder.CreateStructGEP(ArgStruct, FieldIndex, Arg->getName());
2752
73
      if (ArgI.getInAllocaIndirect())
2753
6
        V = Address(Builder.CreateLoad(V), ConvertTypeForMem(Ty),
2754
6
                    getContext().getTypeAlignInChars(Ty));
2755
73
      ArgVals.push_back(ParamValue::forIndirect(V));
2756
73
      break;
2757
0
    }
2758
2759
2.60k
    case ABIArgInfo::Indirect:
2760
2.62k
    case ABIArgInfo::IndirectAliased: {
2761
2.62k
      assert(NumIRArgs == 1);
2762
0
      Address ParamAddr = Address(Fn->getArg(FirstIRArg), ConvertTypeForMem(Ty),
2763
2.62k
                                  ArgI.getIndirectAlign());
2764
2765
2.62k
      if (!hasScalarEvaluationKind(Ty)) {
2766
        // Aggregates and complex variables are accessed by reference. All we
2767
        // need to do is realign the value, if requested. Also, if the address
2768
        // may be aliased, copy it to ensure that the parameter variable is
2769
        // mutable and has a unique adress, as C requires.
2770
2.28k
        Address V = ParamAddr;
2771
2.28k
        if (ArgI.getIndirectRealign() || 
ArgI.isIndirectAliased()2.25k
) {
2772
48
          Address AlignedTemp = CreateMemTemp(Ty, "coerce");
2773
2774
          // Copy from the incoming argument pointer to the temporary with the
2775
          // appropriate alignment.
2776
          //
2777
          // FIXME: We should have a common utility for generating an aggregate
2778
          // copy.
2779
48
          CharUnits Size = getContext().getTypeSizeInChars(Ty);
2780
48
          Builder.CreateMemCpy(
2781
48
              AlignedTemp.getPointer(), AlignedTemp.getAlignment().getAsAlign(),
2782
48
              ParamAddr.getPointer(), ParamAddr.getAlignment().getAsAlign(),
2783
48
              llvm::ConstantInt::get(IntPtrTy, Size.getQuantity()));
2784
48
          V = AlignedTemp;
2785
48
        }
2786
2.28k
        ArgVals.push_back(ParamValue::forIndirect(V));
2787
2.28k
      } else {
2788
        // Load scalar value from indirect argument.
2789
339
        llvm::Value *V =
2790
339
            EmitLoadOfScalar(ParamAddr, false, Ty, Arg->getBeginLoc());
2791
2792
339
        if (isPromoted)
2793
0
          V = emitArgumentDemotion(*this, Arg, V);
2794
339
        ArgVals.push_back(ParamValue::forDirect(V));
2795
339
      }
2796
2.62k
      break;
2797
2.60k
    }
2798
2799
17.8k
    case ABIArgInfo::Extend:
2800
519k
    case ABIArgInfo::Direct: {
2801
519k
      auto AI = Fn->getArg(FirstIRArg);
2802
519k
      llvm::Type *LTy = ConvertType(Arg->getType());
2803
2804
      // Prepare parameter attributes. So far, only attributes for pointer
2805
      // parameters are prepared. See
2806
      // http://llvm.org/docs/LangRef.html#paramattrs.
2807
519k
      if (ArgI.getDirectOffset() == 0 && 
LTy->isPointerTy()519k
&&
2808
519k
          
ArgI.getCoerceToType()->isPointerTy()277k
) {
2809
277k
        assert(NumIRArgs == 1);
2810
2811
277k
        if (const ParmVarDecl *PVD = dyn_cast<ParmVarDecl>(Arg)) {
2812
          // Set `nonnull` attribute if any.
2813
112k
          if (getNonNullAttr(CurCodeDecl, PVD, PVD->getType(),
2814
112k
                             PVD->getFunctionScopeIndex()) &&
2815
112k
              
!CGM.getCodeGenOpts().NullPointerIsValid22
)
2816
13
            AI->addAttr(llvm::Attribute::NonNull);
2817
2818
112k
          QualType OTy = PVD->getOriginalType();
2819
112k
          if (const auto *ArrTy =
2820
112k
              getContext().getAsConstantArrayType(OTy)) {
2821
            // A C99 array parameter declaration with the static keyword also
2822
            // indicates dereferenceability, and if the size is constant we can
2823
            // use the dereferenceable attribute (which requires the size in
2824
            // bytes).
2825
578
            if (ArrTy->getSizeModifier() == ArrayType::Static) {
2826
5
              QualType ETy = ArrTy->getElementType();
2827
5
              llvm::Align Alignment =
2828
5
                  CGM.getNaturalTypeAlignment(ETy).getAsAlign();
2829
5
              AI->addAttrs(llvm::AttrBuilder(getLLVMContext()).addAlignmentAttr(Alignment));
2830
5
              uint64_t ArrSize = ArrTy->getSize().getZExtValue();
2831
5
              if (!ETy->isIncompleteType() && ETy->isConstantSizeType() &&
2832
5
                  ArrSize) {
2833
3
                llvm::AttrBuilder Attrs(getLLVMContext());
2834
3
                Attrs.addDereferenceableAttr(
2835
3
                    getContext().getTypeSizeInChars(ETy).getQuantity() *
2836
3
                    ArrSize);
2837
3
                AI->addAttrs(Attrs);
2838
3
              } else 
if (2
getContext().getTargetInfo().getNullPointerValue(
2839
2
                             ETy.getAddressSpace()) == 0 &&
2840
2
                         !CGM.getCodeGenOpts().NullPointerIsValid) {
2841
1
                AI->addAttr(llvm::Attribute::NonNull);
2842
1
              }
2843
5
            }
2844
111k
          } else if (const auto *ArrTy =
2845
111k
                     getContext().getAsVariableArrayType(OTy)) {
2846
            // For C99 VLAs with the static keyword, we don't know the size so
2847
            // we can't use the dereferenceable attribute, but in addrspace(0)
2848
            // we know that it must be nonnull.
2849
102
            if (ArrTy->getSizeModifier() == VariableArrayType::Static) {
2850
2
              QualType ETy = ArrTy->getElementType();
2851
2
              llvm::Align Alignment =
2852
2
                  CGM.getNaturalTypeAlignment(ETy).getAsAlign();
2853
2
              AI->addAttrs(llvm::AttrBuilder(getLLVMContext()).addAlignmentAttr(Alignment));
2854
2
              if (!getContext().getTargetAddressSpace(ETy) &&
2855
2
                  !CGM.getCodeGenOpts().NullPointerIsValid)
2856
1
                AI->addAttr(llvm::Attribute::NonNull);
2857
2
            }
2858
102
          }
2859
2860
          // Set `align` attribute if any.
2861
112k
          const auto *AVAttr = PVD->getAttr<AlignValueAttr>();
2862
112k
          if (!AVAttr)
2863
112k
            if (const auto *TOTy = dyn_cast<TypedefType>(OTy))
2864
6.96k
              AVAttr = TOTy->getDecl()->getAttr<AlignValueAttr>();
2865
112k
          if (AVAttr && 
!SanOpts.has(SanitizerKind::Alignment)11
) {
2866
            // If alignment-assumption sanitizer is enabled, we do *not* add
2867
            // alignment attribute here, but emit normal alignment assumption,
2868
            // so the UBSAN check could function.
2869
8
            llvm::ConstantInt *AlignmentCI =
2870
8
                cast<llvm::ConstantInt>(EmitScalarExpr(AVAttr->getAlignment()));
2871
8
            uint64_t AlignmentInt =
2872
8
                AlignmentCI->getLimitedValue(llvm::Value::MaximumAlignment);
2873
8
            if (AI->getParamAlign().valueOrOne() < AlignmentInt) {
2874
8
              AI->removeAttr(llvm::Attribute::AttrKind::Alignment);
2875
8
              AI->addAttrs(llvm::AttrBuilder(getLLVMContext()).addAlignmentAttr(
2876
8
                  llvm::Align(AlignmentInt)));
2877
8
            }
2878
8
          }
2879
112k
        }
2880
2881
        // Set 'noalias' if an argument type has the `restrict` qualifier.
2882
277k
        if (Arg->getType().isRestrictQualified())
2883
32.1k
          AI->addAttr(llvm::Attribute::NoAlias);
2884
277k
      }
2885
2886
      // Prepare the argument value. If we have the trivial case, handle it
2887
      // with no muss and fuss.
2888
519k
      if (!isa<llvm::StructType>(ArgI.getCoerceToType()) &&
2889
519k
          
ArgI.getCoerceToType() == ConvertType(Ty)518k
&&
2890
519k
          
ArgI.getDirectOffset() == 0512k
) {
2891
512k
        assert(NumIRArgs == 1);
2892
2893
        // LLVM expects swifterror parameters to be used in very restricted
2894
        // ways.  Copy the value into a less-restricted temporary.
2895
0
        llvm::Value *V = AI;
2896
512k
        if (FI.getExtParameterInfo(ArgNo).getABI()
2897
512k
              == ParameterABI::SwiftErrorResult) {
2898
16
          QualType pointeeTy = Ty->getPointeeType();
2899
16
          assert(pointeeTy->isPointerType());
2900
0
          Address temp =
2901
16
            CreateMemTemp(pointeeTy, getPointerAlign(), "swifterror.temp");
2902
16
          Address arg(V, ConvertTypeForMem(pointeeTy),
2903
16
                      getContext().getTypeAlignInChars(pointeeTy));
2904
16
          llvm::Value *incomingErrorValue = Builder.CreateLoad(arg);
2905
16
          Builder.CreateStore(incomingErrorValue, temp);
2906
16
          V = temp.getPointer();
2907
2908
          // Push a cleanup to copy the value back at the end of the function.
2909
          // The convention does not guarantee that the value will be written
2910
          // back if the function exits with an unwind exception.
2911
16
          EHStack.pushCleanup<CopyBackSwiftError>(NormalCleanup, temp, arg);
2912
16
        }
2913
2914
        // Ensure the argument is the correct type.
2915
512k
        if (V->getType() != ArgI.getCoerceToType())
2916
0
          V = Builder.CreateBitCast(V, ArgI.getCoerceToType());
2917
2918
512k
        if (isPromoted)
2919
14
          V = emitArgumentDemotion(*this, Arg, V);
2920
2921
        // Because of merging of function types from multiple decls it is
2922
        // possible for the type of an argument to not match the corresponding
2923
        // type in the function type. Since we are codegening the callee
2924
        // in here, add a cast to the argument type.
2925
512k
        llvm::Type *LTy = ConvertType(Arg->getType());
2926
512k
        if (V->getType() != LTy)
2927
0
          V = Builder.CreateBitCast(V, LTy);
2928
2929
512k
        ArgVals.push_back(ParamValue::forDirect(V));
2930
512k
        break;
2931
512k
      }
2932
2933
      // VLST arguments are coerced to VLATs at the function boundary for
2934
      // ABI consistency. If this is a VLST that was coerced to
2935
      // a VLAT at the function boundary and the types match up, use
2936
      // llvm.vector.extract to convert back to the original VLST.
2937
6.81k
      if (auto *VecTyTo = dyn_cast<llvm::FixedVectorType>(ConvertType(Ty))) {
2938
1.66k
        llvm::Value *Coerced = Fn->getArg(FirstIRArg);
2939
1.66k
        if (auto *VecTyFrom =
2940
1.66k
                dyn_cast<llvm::ScalableVectorType>(Coerced->getType())) {
2941
          // If we are casting a scalable 16 x i1 predicate vector to a fixed i8
2942
          // vector, bitcast the source and use a vector extract.
2943
576
          auto PredType =
2944
576
              llvm::ScalableVectorType::get(Builder.getInt1Ty(), 16);
2945
576
          if (VecTyFrom == PredType &&
2946
576
              
VecTyTo->getElementType() == Builder.getInt8Ty()23
) {
2947
23
            VecTyFrom = llvm::ScalableVectorType::get(Builder.getInt8Ty(), 2);
2948
23
            Coerced = Builder.CreateBitCast(Coerced, VecTyFrom);
2949
23
          }
2950
576
          if (VecTyFrom->getElementType() == VecTyTo->getElementType()) {
2951
576
            llvm::Value *Zero = llvm::Constant::getNullValue(CGM.Int64Ty);
2952
2953
576
            assert(NumIRArgs == 1);
2954
0
            Coerced->setName(Arg->getName() + ".coerce");
2955
576
            ArgVals.push_back(ParamValue::forDirect(Builder.CreateExtractVector(
2956
576
                VecTyTo, Coerced, Zero, "castFixedSve")));
2957
576
            break;
2958
576
          }
2959
576
        }
2960
1.66k
      }
2961
2962
6.23k
      Address Alloca = CreateMemTemp(Ty, getContext().getDeclAlign(Arg),
2963
6.23k
                                     Arg->getName());
2964
2965
      // Pointer to store into.
2966
6.23k
      Address Ptr = emitAddressAtOffset(*this, Alloca, ArgI);
2967
2968
      // Fast-isel and the optimizer generally like scalar values better than
2969
      // FCAs, so we flatten them if this is safe to do for this argument.
2970
6.23k
      llvm::StructType *STy = dyn_cast<llvm::StructType>(ArgI.getCoerceToType());
2971
6.23k
      if (ArgI.isDirect() && ArgI.getCanBeFlattened() && 
STy5.91k
&&
2972
6.23k
          
STy->getNumElements() > 11.07k
) {
2973
1.03k
        uint64_t SrcSize = CGM.getDataLayout().getTypeAllocSize(STy);
2974
1.03k
        llvm::Type *DstTy = Ptr.getElementType();
2975
1.03k
        uint64_t DstSize = CGM.getDataLayout().getTypeAllocSize(DstTy);
2976
2977
1.03k
        Address AddrToStoreInto = Address::invalid();
2978
1.03k
        if (SrcSize <= DstSize) {
2979
947
          AddrToStoreInto = Builder.CreateElementBitCast(Ptr, STy);
2980
947
        } else {
2981
89
          AddrToStoreInto =
2982
89
            CreateTempAlloca(STy, Alloca.getAlignment(), "coerce");
2983
89
        }
2984
2985
1.03k
        assert(STy->getNumElements() == NumIRArgs);
2986
3.21k
        for (unsigned i = 0, e = STy->getNumElements(); i != e; 
++i2.18k
) {
2987
2.18k
          auto AI = Fn->getArg(FirstIRArg + i);
2988
2.18k
          AI->setName(Arg->getName() + ".coerce" + Twine(i));
2989
2.18k
          Address EltPtr = Builder.CreateStructGEP(AddrToStoreInto, i);
2990
2.18k
          Builder.CreateStore(AI, EltPtr);
2991
2.18k
        }
2992
2993
1.03k
        if (SrcSize > DstSize) {
2994
89
          Builder.CreateMemCpy(Ptr, AddrToStoreInto, DstSize);
2995
89
        }
2996
2997
5.19k
      } else {
2998
        // Simple case, just do a coerced store of the argument into the alloca.
2999
5.19k
        assert(NumIRArgs == 1);
3000
0
        auto AI = Fn->getArg(FirstIRArg);
3001
5.19k
        AI->setName(Arg->getName() + ".coerce");
3002
5.19k
        CreateCoercedStore(AI, Ptr, /*DstIsVolatile=*/false, *this);
3003
5.19k
      }
3004
3005
      // Match to what EmitParmDecl is expecting for this type.
3006
6.23k
      if (CodeGenFunction::hasScalarEvaluationKind(Ty)) {
3007
1.36k
        llvm::Value *V =
3008
1.36k
            EmitLoadOfScalar(Alloca, false, Ty, Arg->getBeginLoc());
3009
1.36k
        if (isPromoted)
3010
0
          V = emitArgumentDemotion(*this, Arg, V);
3011
1.36k
        ArgVals.push_back(ParamValue::forDirect(V));
3012
4.87k
      } else {
3013
4.87k
        ArgVals.push_back(ParamValue::forIndirect(Alloca));
3014
4.87k
      }
3015
6.23k
      break;
3016
6.81k
    }
3017
3018
571
    case ABIArgInfo::CoerceAndExpand: {
3019
      // Reconstruct into a temporary.
3020
571
      Address alloca = CreateMemTemp(Ty, getContext().getDeclAlign(Arg));
3021
571
      ArgVals.push_back(ParamValue::forIndirect(alloca));
3022
3023
571
      auto coercionType = ArgI.getCoerceAndExpandType();
3024
571
      alloca = Builder.CreateElementBitCast(alloca, coercionType);
3025
3026
571
      unsigned argIndex = FirstIRArg;
3027
1.78k
      for (unsigned i = 0, e = coercionType->getNumElements(); i != e; 
++i1.21k
) {
3028
1.21k
        llvm::Type *eltType = coercionType->getElementType(i);
3029
1.21k
        if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType))
3030
13
          continue;
3031
3032
1.19k
        auto eltAddr = Builder.CreateStructGEP(alloca, i);
3033
1.19k
        auto elt = Fn->getArg(argIndex++);
3034
1.19k
        Builder.CreateStore(elt, eltAddr);
3035
1.19k
      }
3036
571
      assert(argIndex == FirstIRArg + NumIRArgs);
3037
0
      break;
3038
6.81k
    }
3039
3040
102
    case ABIArgInfo::Expand: {
3041
      // If this structure was expanded into multiple arguments then
3042
      // we need to create a temporary and reconstruct it from the
3043
      // arguments.
3044
102
      Address Alloca = CreateMemTemp(Ty, getContext().getDeclAlign(Arg));
3045
102
      LValue LV = MakeAddrLValue(Alloca, Ty);
3046
102
      ArgVals.push_back(ParamValue::forIndirect(Alloca));
3047
3048
102
      auto FnArgIter = Fn->arg_begin() + FirstIRArg;
3049
102
      ExpandTypeFromArgs(Ty, LV, FnArgIter);
3050
102
      assert(FnArgIter == Fn->arg_begin() + FirstIRArg + NumIRArgs);
3051
345
      for (unsigned i = 0, e = NumIRArgs; i != e; 
++i243
) {
3052
243
        auto AI = Fn->getArg(FirstIRArg + i);
3053
243
        AI->setName(Arg->getName() + "." + Twine(i));
3054
243
      }
3055
102
      break;
3056
6.81k
    }
3057
3058
6.89k
    case ABIArgInfo::Ignore:
3059
6.89k
      assert(NumIRArgs == 0);
3060
      // Initialize the local variable appropriately.
3061
6.89k
      if (!hasScalarEvaluationKind(Ty)) {
3062
6.89k
        ArgVals.push_back(ParamValue::forIndirect(CreateMemTemp(Ty)));
3063
6.89k
      } else {
3064
0
        llvm::Value *U = llvm::UndefValue::get(ConvertType(Arg->getType()));
3065
0
        ArgVals.push_back(ParamValue::forDirect(U));
3066
0
      }
3067
6.89k
      break;
3068
529k
    }
3069
529k
  }
3070
3071
314k
  if (getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) {
3072
31.7k
    for (int I = Args.size() - 1; I >= 0; 
--I18.2k
)
3073
18.2k
      EmitParmDecl(*Args[I], ArgVals[I], I + 1);
3074
300k
  } else {
3075
811k
    for (unsigned I = 0, E = Args.size(); I != E; 
++I511k
)
3076
511k
      EmitParmDecl(*Args[I], ArgVals[I], I + 1);
3077
300k
  }
3078
314k
}
3079
3080
8
static void eraseUnusedBitCasts(llvm::Instruction *insn) {
3081
12
  while (insn->use_empty()) {
3082
12
    llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(insn);
3083
12
    if (!bitcast) 
return8
;
3084
3085
    // This is "safe" because we would have used a ConstantExpr otherwise.
3086
4
    insn = cast<llvm::Instruction>(bitcast->getOperand(0));
3087
4
    bitcast->eraseFromParent();
3088
4
  }
3089
8
}
3090
3091
/// Try to emit a fused autorelease of a return result.
3092
static llvm::Value *tryEmitFusedAutoreleaseOfResult(CodeGenFunction &CGF,
3093
87
                                                    llvm::Value *result) {
3094
  // We must be immediately followed the cast.
3095
87
  llvm::BasicBlock *BB = CGF.Builder.GetInsertBlock();
3096
87
  if (BB->empty()) 
return nullptr0
;
3097
87
  if (&BB->back() != result) 
return nullptr24
;
3098
3099
63
  llvm::Type *resultType = result->getType();
3100
3101
  // result is in a BasicBlock and is therefore an Instruction.
3102
63
  llvm::Instruction *generator = cast<llvm::Instruction>(result);
3103
3104
63
  SmallVector<llvm::Instruction *, 4> InstsToKill;
3105
3106
  // Look for:
3107
  //  %generator = bitcast %type1* %generator2 to %type2*
3108
94
  while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(generator)) {
3109
    // We would have emitted this as a constant if the operand weren't
3110
    // an Instruction.
3111
31
    generator = cast<llvm::Instruction>(bitcast->getOperand(0));
3112
3113
    // Require the generator to be immediately followed by the cast.
3114
31
    if (generator->getNextNode() != bitcast)
3115
0
      return nullptr;
3116
3117
31
    InstsToKill.push_back(bitcast);
3118
31
  }
3119
3120
  // Look for:
3121
  //   %generator = call i8* @objc_retain(i8* %originalResult)
3122
  // or
3123
  //   %generator = call i8* @objc_retainAutoreleasedReturnValue(i8* %originalResult)
3124
63
  llvm::CallInst *call = dyn_cast<llvm::CallInst>(generator);
3125
63
  if (!call) 
return nullptr13
;
3126
3127
50
  bool doRetainAutorelease;
3128
3129
50
  if (call->getCalledOperand() == CGF.CGM.getObjCEntrypoints().objc_retain) {
3130
8
    doRetainAutorelease = true;
3131
42
  } else if (call->getCalledOperand() ==
3132
42
             CGF.CGM.getObjCEntrypoints().objc_retainAutoreleasedReturnValue) {
3133
19
    doRetainAutorelease = false;
3134
3135
    // If we emitted an assembly marker for this call (and the
3136
    // ARCEntrypoints field should have been set if so), go looking
3137
    // for that call.  If we can't find it, we can't do this
3138
    // optimization.  But it should always be the immediately previous
3139
    // instruction, unless we needed bitcasts around the call.
3140
19
    if (CGF.CGM.getObjCEntrypoints().retainAutoreleasedReturnValueMarker) {
3141
12
      llvm::Instruction *prev = call->getPrevNode();
3142
12
      assert(prev);
3143
12
      if (isa<llvm::BitCastInst>(prev)) {
3144
8
        prev = prev->getPrevNode();
3145
8
        assert(prev);
3146
8
      }
3147
0
      assert(isa<llvm::CallInst>(prev));
3148
0
      assert(cast<llvm::CallInst>(prev)->getCalledOperand() ==
3149
12
             CGF.CGM.getObjCEntrypoints().retainAutoreleasedReturnValueMarker);
3150
0
      InstsToKill.push_back(prev);
3151
12
    }
3152
23
  } else {
3153
23
    return nullptr;
3154
23
  }
3155
3156
27
  result = call->getArgOperand(0);
3157
27
  InstsToKill.push_back(call);
3158
3159
  // Keep killing bitcasts, for sanity.  Note that we no longer care
3160
  // about precise ordering as long as there's exactly one use.
3161
42
  while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(result)) {
3162
15
    if (!bitcast->hasOneUse()) 
break0
;
3163
15
    InstsToKill.push_back(bitcast);
3164
15
    result = bitcast->getOperand(0);
3165
15
  }
3166
3167
  // Delete all the unnecessary instructions, from latest to earliest.
3168
27
  for (auto *I : InstsToKill)
3169
75
    I->eraseFromParent();
3170
3171
  // Do the fused retain/autorelease if we were asked to.
3172
27
  if (doRetainAutorelease)
3173
8
    result = CGF.EmitARCRetainAutoreleaseReturnValue(result);
3174
3175
  // Cast back to the result type.
3176
27
  return CGF.Builder.CreateBitCast(result, resultType);
3177
50
}
3178
3179
/// If this is a +1 of the value of an immutable 'self', remove it.
3180
static llvm::Value *tryRemoveRetainOfSelf(CodeGenFunction &CGF,
3181
122
                                          llvm::Value *result) {
3182
  // This is only applicable to a method with an immutable 'self'.
3183
122
  const ObjCMethodDecl *method =
3184
122
    dyn_cast_or_null<ObjCMethodDecl>(CGF.CurCodeDecl);
3185
122
  if (!method) 
return nullptr103
;
3186
19
  const VarDecl *self = method->getSelfDecl();
3187
19
  if (!self->getType().isConstQualified()) 
return nullptr0
;
3188
3189
  // Look for a retain call.
3190
19
  llvm::CallInst *retainCall =
3191
19
    dyn_cast<llvm::CallInst>(result->stripPointerCasts());
3192
19
  if (!retainCall || retainCall->getCalledOperand() !=
3193
15
                         CGF.CGM.getObjCEntrypoints().objc_retain)
3194
12
    return nullptr;
3195
3196
  // Look for an ordinary load of 'self'.
3197
7
  llvm::Value *retainedValue = retainCall->getArgOperand(0);
3198
7
  llvm::LoadInst *load =
3199
7
    dyn_cast<llvm::LoadInst>(retainedValue->stripPointerCasts());
3200
7
  if (!load || load->isAtomic() || load->isVolatile() ||
3201
7
      load->getPointerOperand() != CGF.GetAddrOfLocalVar(self).getPointer())
3202
3
    return nullptr;
3203
3204
  // Okay!  Burn it all down.  This relies for correctness on the
3205
  // assumption that the retain is emitted as part of the return and
3206
  // that thereafter everything is used "linearly".
3207
4
  llvm::Type *resultType = result->getType();
3208
4
  eraseUnusedBitCasts(cast<llvm::Instruction>(result));
3209
4
  assert(retainCall->use_empty());
3210
0
  retainCall->eraseFromParent();
3211
4
  eraseUnusedBitCasts(cast<llvm::Instruction>(retainedValue));
3212
3213
4
  return CGF.Builder.CreateBitCast(load, resultType);
3214
7
}
3215
3216
/// Emit an ARC autorelease of the result of a function.
3217
///
3218
/// \return the value to actually return from the function
3219
static llvm::Value *emitAutoreleaseOfResult(CodeGenFunction &CGF,
3220
122
                                            llvm::Value *result) {
3221
  // If we're returning 'self', kill the initial retain.  This is a
3222
  // heuristic attempt to "encourage correctness" in the really unfortunate
3223
  // case where we have a return of self during a dealloc and we desperately
3224
  // need to avoid the possible autorelease.
3225
122
  if (llvm::Value *self = tryRemoveRetainOfSelf(CGF, result))
3226
4
    return self;
3227
3228
  // At -O0, try to emit a fused retain/autorelease.
3229
118
  if (CGF.shouldUseFusedARCCalls())
3230
87
    if (llvm::Value *fused = tryEmitFusedAutoreleaseOfResult(CGF, result))
3231
27
      return fused;
3232
3233
91
  return CGF.EmitARCAutoreleaseReturnValue(result);
3234
118
}
3235
3236
/// Heuristically search for a dominating store to the return-value slot.
3237
140k
static llvm::StoreInst *findDominatingStoreToReturnValue(CodeGenFunction &CGF) {
3238
  // Check if a User is a store which pointerOperand is the ReturnValue.
3239
  // We are looking for stores to the ReturnValue, not for stores of the
3240
  // ReturnValue to some other location.
3241
140k
  auto GetStoreIfValid = [&CGF](llvm::User *U) -> llvm::StoreInst * {
3242
138k
    auto *SI = dyn_cast<llvm::StoreInst>(U);
3243
138k
    if (!SI || 
SI->getPointerOperand() != CGF.ReturnValue.getPointer()136k
||
3244
138k
        
SI->getValueOperand()->getType() != CGF.ReturnValue.getElementType()135k
)
3245
2.20k
      return nullptr;
3246
    // These aren't actually possible for non-coerced returns, and we
3247
    // only care about non-coerced returns on this code path.
3248
135k
    assert(!SI->isAtomic() && !SI->isVolatile());
3249
0
    return SI;
3250
138k
  };
3251
  // If there are multiple uses of the return-value slot, just check
3252
  // for something immediately preceding the IP.  Sometimes this can
3253
  // happen with how we generate implicit-returns; it can also happen
3254
  // with noreturn cleanups.
3255
140k
  if (!CGF.ReturnValue.getPointer()->hasOneUse()) {
3256
7.48k
    llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock();
3257
7.48k
    if (IP->empty()) 
return nullptr1.93k
;
3258
3259
    // Look at directly preceding instruction, skipping bitcasts and lifetime
3260
    // markers.
3261
5.76k
    
for (llvm::Instruction &I : make_range(IP->rbegin(), IP->rend()))5.55k
{
3262
5.76k
      if (isa<llvm::BitCastInst>(&I))
3263
153
        continue;
3264
5.61k
      if (auto *II = dyn_cast<llvm::IntrinsicInst>(&I))
3265
357
        if (II->getIntrinsicID() == llvm::Intrinsic::lifetime_end)
3266
102
          continue;
3267
3268
5.51k
      return GetStoreIfValid(&I);
3269
5.61k
    }
3270
43
    return nullptr;
3271
5.55k
  }
3272
3273
132k
  llvm::StoreInst *store =
3274
132k
      GetStoreIfValid(CGF.ReturnValue.getPointer()->user_back());
3275
132k
  if (!store) 
return nullptr524
;
3276
3277
  // Now do a first-and-dirty dominance check: just walk up the
3278
  // single-predecessors chain from the current insertion point.
3279
132k
  llvm::BasicBlock *StoreBB = store->getParent();
3280
132k
  llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock();
3281
132k
  while (IP != StoreBB) {
3282
2.00k
    if (!(IP = IP->getSinglePredecessor()))
3283
1.27k
      return nullptr;
3284
2.00k
  }
3285
3286
  // Okay, the store's basic block dominates the insertion point; we
3287
  // can do our thing.
3288
130k
  return store;
3289
132k
}
3290
3291
// Helper functions for EmitCMSEClearRecord
3292
3293
// Set the bits corresponding to a field having width `BitWidth` and located at
3294
// offset `BitOffset` (from the least significant bit) within a storage unit of
3295
// `Bits.size()` bytes. Each element of `Bits` corresponds to one target byte.
3296
// Use little-endian layout, i.e.`Bits[0]` is the LSB.
3297
static void setBitRange(SmallVectorImpl<uint64_t> &Bits, int BitOffset,
3298
125
                        int BitWidth, int CharWidth) {
3299
125
  assert(CharWidth <= 64);
3300
0
  assert(static_cast<unsigned>(BitWidth) <= Bits.size() * CharWidth);
3301
3302
0
  int Pos = 0;
3303
125
  if (BitOffset >= CharWidth) {
3304
39
    Pos += BitOffset / CharWidth;
3305
39
    BitOffset = BitOffset % CharWidth;
3306
39
  }
3307
3308
125
  const uint64_t Used = (uint64_t(1) << CharWidth) - 1;
3309
125
  if (BitOffset + BitWidth >= CharWidth) {
3310
65
    Bits[Pos++] |= (Used << BitOffset) & Used;
3311
65
    BitWidth -= CharWidth - BitOffset;
3312
65
    BitOffset = 0;
3313
65
  }
3314
3315
143
  while (BitWidth >= CharWidth) {
3316
18
    Bits[Pos++] = Used;
3317
18
    BitWidth -= CharWidth;
3318
18
  }
3319
3320
125
  if (BitWidth > 0)
3321
92
    Bits[Pos++] |= (Used >> (CharWidth - BitWidth)) << BitOffset;
3322
125
}
3323
3324
// Set the bits corresponding to a field having width `BitWidth` and located at
3325
// offset `BitOffset` (from the least significant bit) within a storage unit of
3326
// `StorageSize` bytes, located at `StorageOffset` in `Bits`. Each element of
3327
// `Bits` corresponds to one target byte. Use target endian layout.
3328
static void setBitRange(SmallVectorImpl<uint64_t> &Bits, int StorageOffset,
3329
                        int StorageSize, int BitOffset, int BitWidth,
3330
125
                        int CharWidth, bool BigEndian) {
3331
3332
125
  SmallVector<uint64_t, 8> TmpBits(StorageSize);
3333
125
  setBitRange(TmpBits, BitOffset, BitWidth, CharWidth);
3334
3335
125
  if (BigEndian)
3336
50
    std::reverse(TmpBits.begin(), TmpBits.end());
3337
3338
125
  for (uint64_t V : TmpBits)
3339
350
    Bits[StorageOffset++] |= V;
3340
125
}
3341
3342
static void setUsedBits(CodeGenModule &, QualType, int,
3343
                        SmallVectorImpl<uint64_t> &);
3344
3345
// Set the bits in `Bits`, which correspond to the value representations of
3346
// the actual members of the record type `RTy`. Note that this function does
3347
// not handle base classes, virtual tables, etc, since they cannot happen in
3348
// CMSE function arguments or return. The bit mask corresponds to the target
3349
// memory layout, i.e. it's endian dependent.
3350
static void setUsedBits(CodeGenModule &CGM, const RecordType *RTy, int Offset,
3351
153
                        SmallVectorImpl<uint64_t> &Bits) {
3352
153
  ASTContext &Context = CGM.getContext();
3353
153
  int CharWidth = Context.getCharWidth();
3354
153
  const RecordDecl *RD = RTy->getDecl()->getDefinition();
3355
153
  const ASTRecordLayout &ASTLayout = Context.getASTRecordLayout(RD);
3356
153
  const CGRecordLayout &Layout = CGM.getTypes().getCGRecordLayout(RD);
3357
3358
153
  int Idx = 0;
3359
566
  for (auto I = RD->field_begin(), E = RD->field_end(); I != E; 
++I, ++Idx413
) {
3360
413
    const FieldDecl *F = *I;
3361
3362
413
    if (F->isUnnamedBitfield() || 
F->isZeroLengthBitField(Context)283
||
3363
413
        
F->getType()->isIncompleteArrayType()283
)
3364
140
      continue;
3365
3366
273
    if (F->isBitField()) {
3367
125
      const CGBitFieldInfo &BFI = Layout.getBitFieldInfo(F);
3368
125
      setBitRange(Bits, Offset + BFI.StorageOffset.getQuantity(),
3369
125
                  BFI.StorageSize / CharWidth, BFI.Offset,
3370
125
                  BFI.Size, CharWidth,
3371
125
                  CGM.getDataLayout().isBigEndian());
3372
125
      continue;
3373
125
    }
3374
3375
148
    setUsedBits(CGM, F->getType(),
3376
148
                Offset + ASTLayout.getFieldOffset(Idx) / CharWidth, Bits);
3377
148
  }
3378
153
}
3379
3380
// Set the bits in `Bits`, which correspond to the value representations of
3381
// the elements of an array type `ATy`.
3382
static void setUsedBits(CodeGenModule &CGM, const ConstantArrayType *ATy,
3383
34
                        int Offset, SmallVectorImpl<uint64_t> &Bits) {
3384
34
  const ASTContext &Context = CGM.getContext();
3385
3386
34
  QualType ETy = Context.getBaseElementType(ATy);
3387
34
  int Size = Context.getTypeSizeInChars(ETy).getQuantity();
3388
34
  SmallVector<uint64_t, 4> TmpBits(Size);
3389
34
  setUsedBits(CGM, ETy, 0, TmpBits);
3390
3391
142
  for (int I = 0, N = Context.getConstantArrayElementCount(ATy); I < N; 
++I108
) {
3392
108
    auto Src = TmpBits.begin();
3393
108
    auto Dst = Bits.begin() + Offset + I * Size;
3394
270
    for (int J = 0; J < Size; 
++J162
)
3395
162
      *Dst++ |= *Src++;
3396
108
  }
3397
34
}
3398
3399
// Set the bits in `Bits`, which correspond to the value representations of
3400
// the type `QTy`.
3401
static void setUsedBits(CodeGenModule &CGM, QualType QTy, int Offset,
3402
182
                        SmallVectorImpl<uint64_t> &Bits) {
3403
182
  if (const auto *RTy = QTy->getAs<RecordType>())
3404
30
    return setUsedBits(CGM, RTy, Offset, Bits);
3405
3406
152
  ASTContext &Context = CGM.getContext();
3407
152
  if (const auto *ATy = Context.getAsConstantArrayType(QTy))
3408
34
    return setUsedBits(CGM, ATy, Offset, Bits);
3409
3410
118
  int Size = Context.getTypeSizeInChars(QTy).getQuantity();
3411
118
  if (Size <= 0)
3412
0
    return;
3413
3414
118
  std::fill_n(Bits.begin() + Offset, Size,
3415
118
              (uint64_t(1) << Context.getCharWidth()) - 1);
3416
118
}
3417
3418
static uint64_t buildMultiCharMask(const SmallVectorImpl<uint64_t> &Bits,
3419
                                   int Pos, int Size, int CharWidth,
3420
152
                                   bool BigEndian) {
3421
152
  assert(Size > 0);
3422
0
  uint64_t Mask = 0;
3423
152
  if (BigEndian) {
3424
318
    for (auto P = Bits.begin() + Pos, E = Bits.begin() + Pos + Size; P != E;
3425
256
         ++P)
3426
256
      Mask = (Mask << CharWidth) | *P;
3427
90
  } else {
3428
90
    auto P = Bits.begin() + Pos + Size, End = Bits.begin() + Pos;
3429
90
    do
3430
372
      Mask = (Mask << CharWidth) | *--P;
3431
372
    while (P != End);
3432
90
  }
3433
152
  return Mask;
3434
152
}
3435
3436
// Emit code to clear the bits in a record, which aren't a part of any user
3437
// declared member, when the record is a function return.
3438
llvm::Value *CodeGenFunction::EmitCMSEClearRecord(llvm::Value *Src,
3439
                                                  llvm::IntegerType *ITy,
3440
84
                                                  QualType QTy) {
3441
84
  assert(Src->getType() == ITy);
3442
0
  assert(ITy->getScalarSizeInBits() <= 64);
3443
3444
0
  const llvm::DataLayout &DataLayout = CGM.getDataLayout();
3445
84
  int Size = DataLayout.getTypeStoreSize(ITy);
3446
84
  SmallVector<uint64_t, 4> Bits(Size);
3447
84
  setUsedBits(CGM, QTy->castAs<RecordType>(), 0, Bits);
3448
3449
84
  int CharWidth = CGM.getContext().getCharWidth();
3450
84
  uint64_t Mask =
3451
84
      buildMultiCharMask(Bits, 0, Size, CharWidth, DataLayout.isBigEndian());
3452
3453
84
  return Builder.CreateAnd(Src, Mask, "cmse.clear");
3454
84
}
3455
3456
// Emit code to clear the bits in a record, which aren't a part of any user
3457
// declared member, when the record is a function argument.
3458
llvm::Value *CodeGenFunction::EmitCMSEClearRecord(llvm::Value *Src,
3459
                                                  llvm::ArrayType *ATy,
3460
39
                                                  QualType QTy) {
3461
39
  const llvm::DataLayout &DataLayout = CGM.getDataLayout();
3462
39
  int Size = DataLayout.getTypeStoreSize(ATy);
3463
39
  SmallVector<uint64_t, 16> Bits(Size);
3464
39
  setUsedBits(CGM, QTy->castAs<RecordType>(), 0, Bits);
3465
3466
  // Clear each element of the LLVM array.
3467
39
  int CharWidth = CGM.getContext().getCharWidth();
3468
39
  int CharsPerElt =
3469
39
      ATy->getArrayElementType()->getScalarSizeInBits() / CharWidth;
3470
39
  int MaskIndex = 0;
3471
39
  llvm::Value *R = llvm::PoisonValue::get(ATy);
3472
107
  for (int I = 0, N = ATy->getArrayNumElements(); I != N; 
++I68
) {
3473
68
    uint64_t Mask = buildMultiCharMask(Bits, MaskIndex, CharsPerElt, CharWidth,
3474
68
                                       DataLayout.isBigEndian());
3475
68
    MaskIndex += CharsPerElt;
3476
68
    llvm::Value *T0 = Builder.CreateExtractValue(Src, I);
3477
68
    llvm::Value *T1 = Builder.CreateAnd(T0, Mask, "cmse.clear");
3478
68
    R = Builder.CreateInsertValue(R, T1, I);
3479
68
  }
3480
3481
39
  return R;
3482
39
}
3483
3484
void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI,
3485
                                         bool EmitRetDbgLoc,
3486
314k
                                         SourceLocation EndLoc) {
3487
314k
  if (FI.isNoReturn()) {
3488
    // Noreturn functions don't return.
3489
22
    EmitUnreachable(EndLoc);
3490
22
    return;
3491
22
  }
3492
3493
314k
  if (CurCodeDecl && 
CurCodeDecl->hasAttr<NakedAttr>()305k
) {
3494
    // Naked functions don't have epilogues.
3495
10
    Builder.CreateUnreachable();
3496
10
    return;
3497
10
  }
3498
3499
  // Functions with no result always return void.
3500
314k
  if (!ReturnValue.isValid()) {
3501
162k
    Builder.CreateRetVoid();
3502
162k
    return;
3503
162k
  }
3504
3505
151k
  llvm::DebugLoc RetDbgLoc;
3506
151k
  llvm::Value *RV = nullptr;
3507
151k
  QualType RetTy = FI.getReturnType();
3508
151k
  const ABIArgInfo &RetAI = FI.getReturnInfo();
3509
3510
151k
  switch (RetAI.getKind()) {
3511
5
  case ABIArgInfo::InAlloca:
3512
    // Aggregrates get evaluated directly into the destination.  Sometimes we
3513
    // need to return the sret value in a register, though.
3514
5
    assert(hasAggregateEvaluationKind(RetTy));
3515
5
    if (RetAI.getInAllocaSRet()) {
3516
5
      llvm::Function::arg_iterator EI = CurFn->arg_end();
3517
5
      --EI;
3518
5
      llvm::Value *ArgStruct = &*EI;
3519
5
      llvm::Value *SRet = Builder.CreateStructGEP(
3520
5
          FI.getArgStruct(), ArgStruct, RetAI.getInAllocaFieldIndex());
3521
5
      llvm::Type *Ty =
3522
5
          cast<llvm::GetElementPtrInst>(SRet)->getResultElementType();
3523
5
      RV = Builder.CreateAlignedLoad(Ty, SRet, getPointerAlign(), "sret");
3524
5
    }
3525
5
    break;
3526
3527
4.25k
  case ABIArgInfo::Indirect: {
3528
4.25k
    auto AI = CurFn->arg_begin();
3529
4.25k
    if (RetAI.isSRetAfterThis())
3530
54
      ++AI;
3531
4.25k
    switch (getEvaluationKind(RetTy)) {
3532
296
    case TEK_Complex: {
3533
296
      ComplexPairTy RT =
3534
296
        EmitLoadOfComplex(MakeAddrLValue(ReturnValue, RetTy), EndLoc);
3535
296
      EmitStoreOfComplex(RT, MakeNaturalAlignAddrLValue(&*AI, RetTy),
3536
296
                         /*isInit*/ true);
3537
296
      break;
3538
0
    }
3539
2.43k
    case TEK_Aggregate:
3540
      // Do nothing; aggregrates get evaluated directly into the destination.
3541
2.43k
      break;
3542
1.51k
    case TEK_Scalar: {
3543
1.51k
      LValueBaseInfo BaseInfo;
3544
1.51k
      TBAAAccessInfo TBAAInfo;
3545
1.51k
      CharUnits Alignment =
3546
1.51k
          CGM.getNaturalTypeAlignment(RetTy, &BaseInfo, &TBAAInfo);
3547
1.51k
      Address ArgAddr(&*AI, ConvertType(RetTy), Alignment);
3548
1.51k
      LValue ArgVal =
3549
1.51k
          LValue::MakeAddr(ArgAddr, RetTy, getContext(), BaseInfo, TBAAInfo);
3550
1.51k
      EmitStoreOfScalar(
3551
1.51k
          Builder.CreateLoad(ReturnValue), ArgVal, /*isInit*/ true);
3552
1.51k
      break;
3553
0
    }
3554
4.25k
    }
3555
4.25k
    break;
3556
4.25k
  }
3557
3558
11.3k
  case ABIArgInfo::Extend:
3559
146k
  case ABIArgInfo::Direct:
3560
146k
    if (RetAI.getCoerceToType() == ConvertType(RetTy) &&
3561
146k
        
RetAI.getDirectOffset() == 0140k
) {
3562
      // The internal return value temp always will have pointer-to-return-type
3563
      // type, just do a load.
3564
3565
      // If there is a dominating store to ReturnValue, we can elide
3566
      // the load, zap the store, and usually zap the alloca.
3567
140k
      if (llvm::StoreInst *SI =
3568
140k
              findDominatingStoreToReturnValue(*this)) {
3569
        // Reuse the debug location from the store unless there is
3570
        // cleanup code to be emitted between the store and return
3571
        // instruction.
3572
134k
        if (EmitRetDbgLoc && 
!AutoreleaseResult133k
)
3573
133k
          RetDbgLoc = SI->getDebugLoc();
3574
        // Get the stored value and nuke the now-dead store.
3575
134k
        RV = SI->getValueOperand();
3576
134k
        SI->eraseFromParent();
3577
3578
      // Otherwise, we have to do a simple load.
3579
134k
      } else {
3580
5.45k
        RV = Builder.CreateLoad(ReturnValue);
3581
5.45k
      }
3582
140k
    } else {
3583
      // If the value is offset in memory, apply the offset now.
3584
6.26k
      Address V = emitAddressAtOffset(*this, ReturnValue, RetAI);
3585
3586
6.26k
      RV = CreateCoercedLoad(V, RetAI.getCoerceToType(), *this);
3587
6.26k
    }
3588
3589
    // In ARC, end functions that return a retainable type with a call
3590
    // to objc_autoreleaseReturnValue.
3591
146k
    if (AutoreleaseResult) {
3592
122
#ifndef NDEBUG
3593
      // Type::isObjCRetainabletype has to be called on a QualType that hasn't
3594
      // been stripped of the typedefs, so we cannot use RetTy here. Get the
3595
      // original return type of FunctionDecl, CurCodeDecl, and BlockDecl from
3596
      // CurCodeDecl or BlockInfo.
3597
122
      QualType RT;
3598
3599
122
      if (auto *FD = dyn_cast<FunctionDecl>(CurCodeDecl))
3600
74
        RT = FD->getReturnType();
3601
48
      else if (auto *MD = dyn_cast<ObjCMethodDecl>(CurCodeDecl))
3602
19
        RT = MD->getReturnType();
3603
29
      else if (isa<BlockDecl>(CurCodeDecl))
3604
29
        RT = BlockInfo->BlockExpression->getFunctionType()->getReturnType();
3605
0
      else
3606
0
        llvm_unreachable("Unexpected function/method type");
3607
3608
122
      assert(getLangOpts().ObjCAutoRefCount &&
3609
122
             !FI.isReturnsRetained() &&
3610
122
             RT->isObjCRetainableType());
3611
0
#endif
3612
0
      RV = emitAutoreleaseOfResult(*this, RV);
3613
122
    }
3614
3615
0
    break;
3616
3617
430
  case ABIArgInfo::Ignore:
3618
430
    break;
3619
3620
541
  case ABIArgInfo::CoerceAndExpand: {
3621
541
    auto coercionType = RetAI.getCoerceAndExpandType();
3622
3623
    // Load all of the coerced elements out into results.
3624
541
    llvm::SmallVector<llvm::Value*, 4> results;
3625
541
    Address addr = Builder.CreateElementBitCast(ReturnValue, coercionType);
3626
1.68k
    for (unsigned i = 0, e = coercionType->getNumElements(); i != e; 
++i1.14k
) {
3627
1.14k
      auto coercedEltType = coercionType->getElementType(i);
3628
1.14k
      if (ABIArgInfo::isPaddingForCoerceAndExpand(coercedEltType))
3629
7
        continue;
3630
3631
1.14k
      auto eltAddr = Builder.CreateStructGEP(addr, i);
3632
1.14k
      auto elt = Builder.CreateLoad(eltAddr);
3633
1.14k
      results.push_back(elt);
3634
1.14k
    }
3635
3636
    // If we have one result, it's the single direct result type.
3637
541
    if (results.size() == 1) {
3638
180
      RV = results[0];
3639
3640
    // Otherwise, we need to make a first-class aggregate.
3641
361
    } else {
3642
      // Construct a return type that lacks padding elements.
3643
361
      llvm::Type *returnType = RetAI.getUnpaddedCoerceAndExpandType();
3644
3645
361
      RV = llvm::PoisonValue::get(returnType);
3646
1.32k
      for (unsigned i = 0, e = results.size(); i != e; 
++i960
) {
3647
960
        RV = Builder.CreateInsertValue(RV, results[i], i);
3648
960
      }
3649
361
    }
3650
541
    break;
3651
11.3k
  }
3652
0
  case ABIArgInfo::Expand:
3653
0
  case ABIArgInfo::IndirectAliased:
3654
0
    llvm_unreachable("Invalid ABI kind for return argument");
3655
151k
  }
3656
3657
151k
  llvm::Instruction *Ret;
3658
151k
  if (RV) {
3659
146k
    if (CurFuncDecl && 
CurFuncDecl->hasAttr<CmseNSEntryAttr>()145k
) {
3660
      // For certain return types, clear padding bits, as they may reveal
3661
      // sensitive information.
3662
      // Small struct/union types are passed as integers.
3663
90
      auto *ITy = dyn_cast<llvm::IntegerType>(RV->getType());
3664
90
      if (ITy != nullptr && 
isa<RecordType>(RetTy.getCanonicalType())84
)
3665
84
        RV = EmitCMSEClearRecord(RV, ITy, RetTy);
3666
90
    }
3667
146k
    EmitReturnValueCheck(RV);
3668
146k
    Ret = Builder.CreateRet(RV);
3669
146k
  } else {
3670
4.68k
    Ret = Builder.CreateRetVoid();
3671
4.68k
  }
3672
3673
151k
  if (RetDbgLoc)
3674
39.6k
    Ret->setDebugLoc(std::move(RetDbgLoc));
3675
151k
}
3676
3677
146k
void CodeGenFunction::EmitReturnValueCheck(llvm::Value *RV) {
3678
  // A current decl may not be available when emitting vtable thunks.
3679
146k
  if (!CurCodeDecl)
3680
1.28k
    return;
3681
3682
  // If the return block isn't reachable, neither is this check, so don't emit
3683
  // it.
3684
145k
  if (ReturnBlock.isValid() && 
ReturnBlock.getBlock()->use_empty()1.98k
)
3685
232
    return;
3686
3687
145k
  ReturnsNonNullAttr *RetNNAttr = nullptr;
3688
145k
  if (SanOpts.has(SanitizerKind::ReturnsNonnullAttribute))
3689
45
    RetNNAttr = CurCodeDecl->getAttr<ReturnsNonNullAttr>();
3690
3691
145k
  if (!RetNNAttr && 
!requiresReturnValueNullabilityCheck()145k
)
3692
145k
    return;
3693
3694
  // Prefer the returns_nonnull attribute if it's present.
3695
18
  SourceLocation AttrLoc;
3696
18
  SanitizerMask CheckKind;
3697
18
  SanitizerHandler Handler;
3698
18
  if (RetNNAttr) {
3699
5
    assert(!requiresReturnValueNullabilityCheck() &&
3700
5
           "Cannot check nullability and the nonnull attribute");
3701
0
    AttrLoc = RetNNAttr->getLocation();
3702
5
    CheckKind = SanitizerKind::ReturnsNonnullAttribute;
3703
5
    Handler = SanitizerHandler::NonnullReturn;
3704
13
  } else {
3705
13
    if (auto *DD = dyn_cast<DeclaratorDecl>(CurCodeDecl))
3706
7
      if (auto *TSI = DD->getTypeSourceInfo())
3707
7
        if (auto FTL = TSI->getTypeLoc().getAsAdjusted<FunctionTypeLoc>())
3708
7
          AttrLoc = FTL.getReturnLoc().findNullabilityLoc();
3709
13
    CheckKind = SanitizerKind::NullabilityReturn;
3710
13
    Handler = SanitizerHandler::NullabilityReturn;
3711
13
  }
3712
3713
0
  SanitizerScope SanScope(this);
3714
3715
  // Make sure the "return" source location is valid. If we're checking a
3716
  // nullability annotation, make sure the preconditions for the check are met.
3717
18
  llvm::BasicBlock *Check = createBasicBlock("nullcheck");
3718
18
  llvm::BasicBlock *NoCheck = createBasicBlock("no.nullcheck");
3719
18
  llvm::Value *SLocPtr = Builder.CreateLoad(ReturnLocation, "return.sloc.load");
3720
18
  llvm::Value *CanNullCheck = Builder.CreateIsNotNull(SLocPtr);
3721
18
  if (requiresReturnValueNullabilityCheck())
3722
13
    CanNullCheck =
3723
13
        Builder.CreateAnd(CanNullCheck, RetValNullabilityPrecondition);
3724
18
  Builder.CreateCondBr(CanNullCheck, Check, NoCheck);
3725
18
  EmitBlock(Check);
3726
3727
  // Now do the null check.
3728
18
  llvm::Value *Cond = Builder.CreateIsNotNull(RV);
3729
18
  llvm::Constant *StaticData[] = {EmitCheckSourceLocation(AttrLoc)};
3730
18
  llvm::Value *DynamicData[] = {SLocPtr};
3731
18
  EmitCheck(std::make_pair(Cond, CheckKind), Handler, StaticData, DynamicData);
3732
3733
18
  EmitBlock(NoCheck);
3734
3735
18
#ifndef NDEBUG
3736
  // The return location should not be used after the check has been emitted.
3737
18
  ReturnLocation = Address::invalid();
3738
18
#endif
3739
18
}
3740
3741
25.3k
static bool isInAllocaArgument(CGCXXABI &ABI, QualType type) {
3742
25.3k
  const CXXRecordDecl *RD = type->getAsCXXRecordDecl();
3743
25.3k
  return RD && 
ABI.getRecordArgABI(RD) == CGCXXABI::RAA_DirectInMemory2.68k
;
3744
25.3k
}
3745
3746
static AggValueSlot createPlaceholderSlot(CodeGenFunction &CGF,
3747
106
                                          QualType Ty) {
3748
  // FIXME: Generate IR in one pass, rather than going back and fixing up these
3749
  // placeholders.
3750
106
  llvm::Type *IRTy = CGF.ConvertTypeForMem(Ty);
3751
106
  llvm::Type *IRPtrTy = IRTy->getPointerTo();
3752
106
  llvm::Value *Placeholder = llvm::PoisonValue::get(IRPtrTy->getPointerTo());
3753
3754
  // FIXME: When we generate this IR in one pass, we shouldn't need
3755
  // this win32-specific alignment hack.
3756
106
  CharUnits Align = CharUnits::fromQuantity(4);
3757
106
  Placeholder = CGF.Builder.CreateAlignedLoad(IRPtrTy, Placeholder, Align);
3758
3759
106
  return AggValueSlot::forAddr(Address(Placeholder, IRTy, Align),
3760
106
                               Ty.getQualifiers(),
3761
106
                               AggValueSlot::IsNotDestructed,
3762
106
                               AggValueSlot::DoesNotNeedGCBarriers,
3763
106
                               AggValueSlot::IsNotAliased,
3764
106
                               AggValueSlot::DoesNotOverlap);
3765
106
}
3766
3767
void CodeGenFunction::EmitDelegateCallArg(CallArgList &args,
3768
                                          const VarDecl *param,
3769
18.5k
                                          SourceLocation loc) {
3770
  // StartFunction converted the ABI-lowered parameter(s) into a
3771
  // local alloca.  We need to turn that into an r-value suitable
3772
  // for EmitCall.
3773
18.5k
  Address local = GetAddrOfLocalVar(param);
3774
3775
18.5k
  QualType type = param->getType();
3776
3777
18.5k
  if (isInAllocaArgument(CGM.getCXXABI(), type)) {
3778
1
    CGM.ErrorUnsupported(param, "forwarded non-trivially copyable parameter");
3779
1
  }
3780
3781
  // GetAddrOfLocalVar returns a pointer-to-pointer for references,
3782
  // but the argument needs to be the original pointer.
3783
18.5k
  if (type->isReferenceType()) {
3784
9.49k
    args.add(RValue::get(Builder.CreateLoad(local)), type);
3785
3786
  // In ARC, move out of consumed arguments so that the release cleanup
3787
  // entered by StartFunction doesn't cause an over-release.  This isn't
3788
  // optimal -O0 code generation, but it should get cleaned up when
3789
  // optimization is enabled.  This also assumes that delegate calls are
3790
  // performed exactly once for a set of arguments, but that should be safe.
3791
9.49k
  } else 
if (9.07k
getLangOpts().ObjCAutoRefCount9.07k
&&
3792
9.07k
             
param->hasAttr<NSConsumedAttr>()9
&&
3793
9.07k
             
type->isObjCRetainableType()2
) {
3794
2
    llvm::Value *ptr = Builder.CreateLoad(local);
3795
2
    auto null =
3796
2
      llvm::ConstantPointerNull::get(cast<llvm::PointerType>(ptr->getType()));
3797
2
    Builder.CreateStore(null, local);
3798
2
    args.add(RValue::get(ptr), type);
3799
3800
  // For the most part, we just need to load the alloca, except that
3801
  // aggregate r-values are actually pointers to temporaries.
3802
9.07k
  } else {
3803
9.07k
    args.add(convertTempToRValue(local, type, loc), type);
3804
9.07k
  }
3805
3806
  // Deactivate the cleanup for the callee-destructed param that was pushed.
3807
18.5k
  if (type->isRecordType() && 
!CurFuncIsThunk2.35k
&&
3808
18.5k
      
type->castAs<RecordType>()->getDecl()->isParamDestroyedInCallee()2.32k
&&
3809
18.5k
      
param->needsDestruction(getContext())8
) {
3810
6
    EHScopeStack::stable_iterator cleanup =
3811
6
        CalleeDestructedParamCleanups.lookup(cast<ParmVarDecl>(param));
3812
6
    assert(cleanup.isValid() &&
3813
6
           "cleanup for callee-destructed param not recorded");
3814
    // This unreachable is a temporary marker which will be removed later.
3815
0
    llvm::Instruction *isActive = Builder.CreateUnreachable();
3816
6
    args.addArgCleanupDeactivation(cleanup, isActive);
3817
6
  }
3818
18.5k
}
3819
3820
54
static bool isProvablyNull(llvm::Value *addr) {
3821
54
  return isa<llvm::ConstantPointerNull>(addr);
3822
54
}
3823
3824
/// Emit the actual writing-back of a writeback.
3825
static void emitWriteback(CodeGenFunction &CGF,
3826
27
                          const CallArgList::Writeback &writeback) {
3827
27
  const LValue &srcLV = writeback.Source;
3828
27
  Address srcAddr = srcLV.getAddress(CGF);
3829
27
  assert(!isProvablyNull(srcAddr.getPointer()) &&
3830
27
         "shouldn't have writeback for provably null argument");
3831
3832
0
  llvm::BasicBlock *contBB = nullptr;
3833
3834
  // If the argument wasn't provably non-null, we need to null check
3835
  // before doing the store.
3836
27
  bool provablyNonNull = llvm::isKnownNonZero(srcAddr.getPointer(),
3837
27
                                              CGF.CGM.getDataLayout());
3838
27
  if (!provablyNonNull) {
3839
5
    llvm::BasicBlock *writebackBB = CGF.createBasicBlock("icr.writeback");
3840
5
    contBB = CGF.createBasicBlock("icr.done");
3841
3842
5
    llvm::Value *isNull =
3843
5
      CGF.Builder.CreateIsNull(srcAddr.getPointer(), "icr.isnull");
3844
5
    CGF.Builder.CreateCondBr(isNull, contBB, writebackBB);
3845
5
    CGF.EmitBlock(writebackBB);
3846
5
  }
3847
3848
  // Load the value to writeback.
3849
27
  llvm::Value *value = CGF.Builder.CreateLoad(writeback.Temporary);
3850
3851
  // Cast it back, in case we're writing an id to a Foo* or something.
3852
27
  value = CGF.Builder.CreateBitCast(value, srcAddr.getElementType(),
3853
27
                                    "icr.writeback-cast");
3854
3855
  // Perform the writeback.
3856
3857
  // If we have a "to use" value, it's something we need to emit a use
3858
  // of.  This has to be carefully threaded in: if it's done after the
3859
  // release it's potentially undefined behavior (and the optimizer
3860
  // will ignore it), and if it happens before the retain then the
3861
  // optimizer could move the release there.
3862
27
  if (writeback.ToUse) {
3863
8
    assert(srcLV.getObjCLifetime() == Qualifiers::OCL_Strong);
3864
3865
    // Retain the new value.  No need to block-copy here:  the block's
3866
    // being passed up the stack.
3867
0
    value = CGF.EmitARCRetainNonBlock(value);
3868
3869
    // Emit the intrinsic use here.
3870
8
    CGF.EmitARCIntrinsicUse(writeback.ToUse);
3871
3872
    // Load the old value (primitively).
3873
8
    llvm::Value *oldValue = CGF.EmitLoadOfScalar(srcLV, SourceLocation());
3874
3875
    // Put the new value in place (primitively).
3876
8
    CGF.EmitStoreOfScalar(value, srcLV, /*init*/ false);
3877
3878
    // Release the old value.
3879
8
    CGF.EmitARCRelease(oldValue, srcLV.isARCPreciseLifetime());
3880
3881
  // Otherwise, we can just do a normal lvalue store.
3882
19
  } else {
3883
19
    CGF.EmitStoreThroughLValue(RValue::get(value), srcLV);
3884
19
  }
3885
3886
  // Jump to the continuation block.
3887
27
  if (!provablyNonNull)
3888
5
    CGF.EmitBlock(contBB);
3889
27
}
3890
3891
static void emitWritebacks(CodeGenFunction &CGF,
3892
27
                           const CallArgList &args) {
3893
27
  for (const auto &I : args.writebacks())
3894
27
    emitWriteback(CGF, I);
3895
27
}
3896
3897
static void deactivateArgCleanupsBeforeCall(CodeGenFunction &CGF,
3898
149
                                            const CallArgList &CallArgs) {
3899
149
  ArrayRef<CallArgList::CallArgCleanup> Cleanups =
3900
149
    CallArgs.getCleanupsToDeactivate();
3901
  // Iterate in reverse to increase the likelihood of popping the cleanup.
3902
192
  for (const auto &I : llvm::reverse(Cleanups)) {
3903
192
    CGF.DeactivateCleanupBlock(I.Cleanup, I.IsActiveIP);
3904
192
    I.IsActiveIP->eraseFromParent();
3905
192
  }
3906
149
}
3907
3908
27
static const Expr *maybeGetUnaryAddrOfOperand(const Expr *E) {
3909
27
  if (const UnaryOperator *uop = dyn_cast<UnaryOperator>(E->IgnoreParens()))
3910
22
    if (uop->getOpcode() == UO_AddrOf)
3911
22
      return uop->getSubExpr();
3912
5
  return nullptr;
3913
27
}
3914
3915
/// Emit an argument that's being passed call-by-writeback.  That is,
3916
/// we are passing the address of an __autoreleased temporary; it
3917
/// might be copy-initialized with the current value of the given
3918
/// address, but it will definitely be copied out of after the call.
3919
static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args,
3920
27
                             const ObjCIndirectCopyRestoreExpr *CRE) {
3921
27
  LValue srcLV;
3922
3923
  // Make an optimistic effort to emit the address as an l-value.
3924
  // This can fail if the argument expression is more complicated.
3925
27
  if (const Expr *lvExpr = maybeGetUnaryAddrOfOperand(CRE->getSubExpr())) {
3926
22
    srcLV = CGF.EmitLValue(lvExpr);
3927
3928
  // Otherwise, just emit it as a scalar.
3929
22
  } else {
3930
5
    Address srcAddr = CGF.EmitPointerWithAlignment(CRE->getSubExpr());
3931
3932
5
    QualType srcAddrType =
3933
5
      CRE->getSubExpr()->getType()->castAs<PointerType>()->getPointeeType();
3934
5
    srcLV = CGF.MakeAddrLValue(srcAddr, srcAddrType);
3935
5
  }
3936
27
  Address srcAddr = srcLV.getAddress(CGF);
3937
3938
  // The dest and src types don't necessarily match in LLVM terms
3939
  // because of the crazy ObjC compatibility rules.
3940
3941
27
  llvm::PointerType *destType =
3942
27
      cast<llvm::PointerType>(CGF.ConvertType(CRE->getType()));
3943
27
  llvm::Type *destElemType =
3944
27
      CGF.ConvertTypeForMem(CRE->getType()->getPointeeType());
3945
3946
  // If the address is a constant null, just pass the appropriate null.
3947
27
  if (isProvablyNull(srcAddr.getPointer())) {
3948
0
    args.add(RValue::get(llvm::ConstantPointerNull::get(destType)),
3949
0
             CRE->getType());
3950
0
    return;
3951
0
  }
3952
3953
  // Create the temporary.
3954
27
  Address temp =
3955
27
      CGF.CreateTempAlloca(destElemType, CGF.getPointerAlign(), "icr.temp");
3956
  // Loading an l-value can introduce a cleanup if the l-value is __weak,
3957
  // and that cleanup will be conditional if we can't prove that the l-value
3958
  // isn't null, so we need to register a dominating point so that the cleanups
3959
  // system will make valid IR.
3960
27
  CodeGenFunction::ConditionalEvaluation condEval(CGF);
3961
3962
  // Zero-initialize it if we're not doing a copy-initialization.
3963
27
  bool shouldCopy = CRE->shouldCopy();
3964
27
  if (!shouldCopy) {
3965
4
    llvm::Value *null =
3966
4
        llvm::ConstantPointerNull::get(cast<llvm::PointerType>(destElemType));
3967
4
    CGF.Builder.CreateStore(null, temp);
3968
4
  }
3969
3970
27
  llvm::BasicBlock *contBB = nullptr;
3971
27
  llvm::BasicBlock *originBB = nullptr;
3972
3973
  // If the address is *not* known to be non-null, we need to switch.
3974
27
  llvm::Value *finalArgument;
3975
3976
27
  bool provablyNonNull = llvm::isKnownNonZero(srcAddr.getPointer(),
3977
27
                                              CGF.CGM.getDataLayout());
3978
27
  if (provablyNonNull) {
3979
22
    finalArgument = temp.getPointer();
3980
22
  } else {
3981
5
    llvm::Value *isNull =
3982
5
      CGF.Builder.CreateIsNull(srcAddr.getPointer(), "icr.isnull");
3983
3984
5
    finalArgument = CGF.Builder.CreateSelect(isNull,
3985
5
                                   llvm::ConstantPointerNull::get(destType),
3986
5
                                             temp.getPointer(), "icr.argument");
3987
3988
    // If we need to copy, then the load has to be conditional, which
3989
    // means we need control flow.
3990
5
    if (shouldCopy) {
3991
5
      originBB = CGF.Builder.GetInsertBlock();
3992
5
      contBB = CGF.createBasicBlock("icr.cont");
3993
5
      llvm::BasicBlock *copyBB = CGF.createBasicBlock("icr.copy");
3994
5
      CGF.Builder.CreateCondBr(isNull, contBB, copyBB);
3995
5
      CGF.EmitBlock(copyBB);
3996
5
      condEval.begin(CGF);
3997
5
    }
3998
5
  }
3999
4000
27
  llvm::Value *valueToUse = nullptr;
4001
4002
  // Perform a copy if necessary.
4003
27
  if (shouldCopy) {
4004
23
    RValue srcRV = CGF.EmitLoadOfLValue(srcLV, SourceLocation());
4005
23
    assert(srcRV.isScalar());
4006
4007
0
    llvm::Value *src = srcRV.getScalarVal();
4008
23
    src = CGF.Builder.CreateBitCast(src, destElemType, "icr.cast");
4009
4010
    // Use an ordinary store, not a store-to-lvalue.
4011
23
    CGF.Builder.CreateStore(src, temp);
4012
4013
    // If optimization is enabled, and the value was held in a
4014
    // __strong variable, we need to tell the optimizer that this
4015
    // value has to stay alive until we're doing the store back.
4016
    // This is because the temporary is effectively unretained,
4017
    // and so otherwise we can violate the high-level semantics.
4018
23
    if (CGF.CGM.getCodeGenOpts().OptimizationLevel != 0 &&
4019
23
        
srcLV.getObjCLifetime() == Qualifiers::OCL_Strong10
) {
4020
8
      valueToUse = src;
4021
8
    }
4022
23
  }
4023
4024
  // Finish the control flow if we needed it.
4025
27
  if (shouldCopy && 
!provablyNonNull23
) {
4026
5
    llvm::BasicBlock *copyBB = CGF.Builder.GetInsertBlock();
4027
5
    CGF.EmitBlock(contBB);
4028
4029
    // Make a phi for the value to intrinsically use.
4030
5
    if (valueToUse) {
4031
2
      llvm::PHINode *phiToUse = CGF.Builder.CreatePHI(valueToUse->getType(), 2,
4032
2
                                                      "icr.to-use");
4033
2
      phiToUse->addIncoming(valueToUse, copyBB);
4034
2
      phiToUse->addIncoming(llvm::UndefValue::get(valueToUse->getType()),
4035
2
                            originBB);
4036
2
      valueToUse = phiToUse;
4037
2
    }
4038
4039
5
    condEval.end(CGF);
4040
5
  }
4041
4042
27
  args.addWriteback(srcLV, temp, valueToUse);
4043
27
  args.add(RValue::get(finalArgument), CRE->getType());
4044
27
}
4045
4046
73
void CallArgList::allocateArgumentMemory(CodeGenFunction &CGF) {
4047
73
  assert(!StackBase);
4048
4049
  // Save the stack.
4050
0
  llvm::Function *F = CGF.CGM.getIntrinsic(llvm::Intrinsic::stacksave);
4051
73
  StackBase = CGF.Builder.CreateCall(F, {}, "inalloca.save");
4052
73
}
4053
4054
356k
void CallArgList::freeArgumentMemory(CodeGenFunction &CGF) const {
4055
356k
  if (StackBase) {
4056
    // Restore the stack after the call.
4057
73
    llvm::Function *F = CGF.CGM.getIntrinsic(llvm::Intrinsic::stackrestore);
4058
73
    CGF.Builder.CreateCall(F, StackBase);
4059
73
  }
4060
356k
}
4061
4062
void CodeGenFunction::EmitNonNullArgCheck(RValue RV, QualType ArgType,
4063
                                          SourceLocation ArgLoc,
4064
                                          AbstractCallee AC,
4065
396k
                                          unsigned ParmNum) {
4066
396k
  if (!AC.getDecl() || 
!(377k
SanOpts.has(SanitizerKind::NonnullAttribute)377k
||
4067
377k
                         
SanOpts.has(SanitizerKind::NullabilityArg)377k
))
4068
396k
    return;
4069
4070
  // The param decl may be missing in a variadic function.
4071
77
  auto PVD = ParmNum < AC.getNumParams() ? 
AC.getParamDecl(ParmNum)66
:
nullptr11
;
4072
77
  unsigned ArgNo = PVD ? 
PVD->getFunctionScopeIndex()66
:
ParmNum11
;
4073
4074
  // Prefer the nonnull attribute if it's present.
4075
77
  const NonNullAttr *NNAttr = nullptr;
4076
77
  if (SanOpts.has(SanitizerKind::NonnullAttribute))
4077
25
    NNAttr = getNonNullAttr(AC.getDecl(), PVD, ArgType, ArgNo);
4078
4079
77
  bool CanCheckNullability = false;
4080
77
  if (SanOpts.has(SanitizerKind::NullabilityArg) && 
!NNAttr53
&&
PVD52
) {
4081
44
    auto Nullability = PVD->getType()->getNullability(getContext());
4082
44
    CanCheckNullability = Nullability &&
4083
44
                          
*Nullability == NullabilityKind::NonNull20
&&
4084
44
                          
PVD->getTypeSourceInfo()18
;
4085
44
  }
4086
4087
77
  if (!NNAttr && 
!CanCheckNullability61
)
4088
43
    return;
4089
4090
34
  SourceLocation AttrLoc;
4091
34
  SanitizerMask CheckKind;
4092
34
  SanitizerHandler Handler;
4093
34
  if (NNAttr) {
4094
16
    AttrLoc = NNAttr->getLocation();
4095
16
    CheckKind = SanitizerKind::NonnullAttribute;
4096
16
    Handler = SanitizerHandler::NonnullArg;
4097
18
  } else {
4098
18
    AttrLoc = PVD->getTypeSourceInfo()->getTypeLoc().findNullabilityLoc();
4099
18
    CheckKind = SanitizerKind::NullabilityArg;
4100
18
    Handler = SanitizerHandler::NullabilityArg;
4101
18
  }
4102
4103
34
  SanitizerScope SanScope(this);
4104
34
  llvm::Value *Cond = EmitNonNullRValueCheck(RV, ArgType);
4105
34
  llvm::Constant *StaticData[] = {
4106
34
      EmitCheckSourceLocation(ArgLoc), EmitCheckSourceLocation(AttrLoc),
4107
34
      llvm::ConstantInt::get(Int32Ty, ArgNo + 1),
4108
34
  };
4109
34
  EmitCheck(std::make_pair(Cond, CheckKind), Handler, StaticData, None);
4110
34
}
4111
4112
// Check if the call is going to use the inalloca convention. This needs to
4113
// agree with CGFunctionInfo::usesInAlloca. The CGFunctionInfo is arranged
4114
// later, so we can't check it directly.
4115
static bool hasInAllocaArgs(CodeGenModule &CGM, CallingConv ExplicitCC,
4116
308k
                            ArrayRef<QualType> ArgTypes) {
4117
  // The Swift calling conventions don't go through the target-specific
4118
  // argument classification, they never use inalloca.
4119
  // TODO: Consider limiting inalloca use to only calling conventions supported
4120
  // by MSVC.
4121
308k
  if (ExplicitCC == CC_Swift || 
ExplicitCC == CC_SwiftAsync307k
)
4122
1.28k
    return false;
4123
307k
  if (!CGM.getTarget().getCXXABI().isMicrosoft())
4124
300k
    return false;
4125
7.34k
  return llvm::any_of(ArgTypes, [&](QualType Ty) {
4126
6.72k
    return isInAllocaArgument(CGM.getCXXABI(), Ty);
4127
6.72k
  });
4128
307k
}
4129
4130
#ifndef NDEBUG
4131
// Determine whether the given argument is an Objective-C method
4132
// that may have type parameters in its signature.
4133
11.3k
static bool isObjCMethodWithTypeParams(const ObjCMethodDecl *method) {
4134
11.3k
  const DeclContext *dc = method->getDeclContext();
4135
11.3k
  if (const ObjCInterfaceDecl *classDecl = dyn_cast<ObjCInterfaceDecl>(dc)) {
4136
7.25k
    return classDecl->getTypeParamListAsWritten();
4137
7.25k
  }
4138
4139
4.07k
  if (const ObjCCategoryDecl *catDecl = dyn_cast<ObjCCategoryDecl>(dc)) {
4140
3.37k
    return catDecl->getTypeParamList();
4141
3.37k
  }
4142
4143
700
  return false;
4144
4.07k
}
4145
#endif
4146
4147
/// EmitCallArgs - Emit call arguments for a function.
4148
void CodeGenFunction::EmitCallArgs(
4149
    CallArgList &Args, PrototypeWrapper Prototype,
4150
    llvm::iterator_range<CallExpr::const_arg_iterator> ArgRange,
4151
308k
    AbstractCallee AC, unsigned ParamsToSkip, EvaluationOrder Order) {
4152
308k
  SmallVector<QualType, 16> ArgTypes;
4153
4154
308k
  assert((ParamsToSkip == 0 || Prototype.P) &&
4155
308k
         "Can't skip parameters if type info is not provided");
4156
4157
  // This variable only captures *explicitly* written conventions, not those
4158
  // applied by default via command line flags or target defaults, such as
4159
  // thiscall, aapcs, stdcall via -mrtd, etc. Computing that correctly would
4160
  // require knowing if this is a C++ instance method or being able to see
4161
  // unprototyped FunctionTypes.
4162
0
  CallingConv ExplicitCC = CC_C;
4163
4164
  // First, if a prototype was provided, use those argument types.
4165
308k
  bool IsVariadic = false;
4166
308k
  if (Prototype.P) {
4167
307k
    const auto *MD = Prototype.P.dyn_cast<const ObjCMethodDecl *>();
4168
307k
    if (MD) {
4169
11.3k
      IsVariadic = MD->isVariadic();
4170
11.3k
      ExplicitCC = getCallingConventionForDecl(
4171
11.3k
          MD, CGM.getTarget().getTriple().isOSWindows());
4172
11.3k
      ArgTypes.assign(MD->param_type_begin() + ParamsToSkip,
4173
11.3k
                      MD->param_type_end());
4174
296k
    } else {
4175
296k
      const auto *FPT = Prototype.P.get<const FunctionProtoType *>();
4176
296k
      IsVariadic = FPT->isVariadic();
4177
296k
      ExplicitCC = FPT->getExtInfo().getCC();
4178
296k
      ArgTypes.assign(FPT->param_type_begin() + ParamsToSkip,
4179
296k
                      FPT->param_type_end());
4180
296k
    }
4181
4182
307k
#ifndef NDEBUG
4183
    // Check that the prototyped types match the argument expression types.
4184
307k
    bool isGenericMethod = MD && 
isObjCMethodWithTypeParams(MD)11.3k
;
4185
307k
    CallExpr::const_arg_iterator Arg = ArgRange.begin();
4186
307k
    for (QualType Ty : ArgTypes) {
4187
305k
      assert(Arg != ArgRange.end() && "Running over edge of argument list!");
4188
0
      assert(
4189
305k
          (isGenericMethod || Ty->isVariablyModifiedType() ||
4190
305k
           Ty.getNonReferenceType()->isObjCRetainableType() ||
4191
305k
           getContext()
4192
305k
                   .getCanonicalType(Ty.getNonReferenceType())
4193
305k
                   .getTypePtr() ==
4194
305k
               getContext().getCanonicalType((*Arg)->getType()).getTypePtr()) &&
4195
305k
          "type mismatch in call argument!");
4196
0
      ++Arg;
4197
305k
    }
4198
4199
    // Either we've emitted all the call args, or we have a call to variadic
4200
    // function.
4201
307k
    assert((Arg == ArgRange.end() || IsVariadic) &&
4202
307k
           "Extra arguments in non-variadic function!");
4203
307k
#endif
4204
307k
  }
4205
4206
  // If we still have any arguments, emit them using the type of the argument.
4207
0
  for (auto *A : llvm::drop_begin(ArgRange, ArgTypes.size()))
4208
91.2k
    ArgTypes.push_back(IsVariadic ? 
getVarArgType(A)90.4k
:
A->getType()813
);
4209
308k
  assert((int)ArgTypes.size() == (ArgRange.end() - ArgRange.begin()));
4210
4211
  // We must evaluate arguments from right to left in the MS C++ ABI,
4212
  // because arguments are destroyed left to right in the callee. As a special
4213
  // case, there are certain language constructs that require left-to-right
4214
  // evaluation, and in those cases we consider the evaluation order requirement
4215
  // to trump the "destruction order is reverse construction order" guarantee.
4216
0
  bool LeftToRight =
4217
308k
      CGM.getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()
4218
308k
          ? 
Order == EvaluationOrder::ForceLeftToRight7.36k
4219
308k
          : 
Order != EvaluationOrder::ForceRightToLeft301k
;
4220
4221
308k
  auto MaybeEmitImplicitObjectSize = [&](unsigned I, const Expr *Arg,
4222
396k
                                         RValue EmittedArg) {
4223
396k
    if (!AC.hasFunctionDecl() || 
I >= AC.getNumParams()364k
)
4224
119k
      return;
4225
276k
    auto *PS = AC.getParamDecl(I)->getAttr<PassObjectSizeAttr>();
4226
276k
    if (PS == nullptr)
4227
276k
      return;
4228
4229
107
    const auto &Context = getContext();
4230
107
    auto SizeTy = Context.getSizeType();
4231
107
    auto T = Builder.getIntNTy(Context.getTypeSize(SizeTy));
4232
107
    assert(EmittedArg.getScalarVal() && "We emitted nothing for the arg?");
4233
0
    llvm::Value *V = evaluateOrEmitBuiltinObjectSize(Arg, PS->getType(), T,
4234
107
                                                     EmittedArg.getScalarVal(),
4235
107
                                                     PS->isDynamic());
4236
107
    Args.add(RValue::get(V), SizeTy);
4237
    // If we're emitting args in reverse, be sure to do so with
4238
    // pass_object_size, as well.
4239
107
    if (!LeftToRight)
4240
9
      std::swap(Args.back(), *(&Args.back() - 1));
4241
107
  };
4242
4243
  // Insert a stack save if we're going to need any inalloca args.
4244
308k
  if (hasInAllocaArgs(CGM, ExplicitCC, ArgTypes)) {
4245
73
    assert(getTarget().getTriple().getArch() == llvm::Triple::x86 &&
4246
73
           "inalloca only supported on x86");
4247
0
    Args.allocateArgumentMemory(*this);
4248
73
  }
4249
4250
  // Evaluate each argument in the appropriate order.
4251
0
  size_t CallArgsStart = Args.size();
4252
705k
  for (unsigned I = 0, E = ArgTypes.size(); I != E; 
++I396k
) {
4253
396k
    unsigned Idx = LeftToRight ? 
I389k
:
E - I - 17.68k
;
4254
396k
    CallExpr::const_arg_iterator Arg = ArgRange.begin() + Idx;
4255
396k
    unsigned InitialArgSize = Args.size();
4256
    // If *Arg is an ObjCIndirectCopyRestoreExpr, check that either the types of
4257
    // the argument and parameter match or the objc method is parameterized.
4258
396k
    assert((!isa<ObjCIndirectCopyRestoreExpr>(*Arg) ||
4259
396k
            getContext().hasSameUnqualifiedType((*Arg)->getType(),
4260
396k
                                                ArgTypes[Idx]) ||
4261
396k
            (isa<ObjCMethodDecl>(AC.getDecl()) &&
4262
396k
             isObjCMethodWithTypeParams(cast<ObjCMethodDecl>(AC.getDecl())))) &&
4263
396k
           "Argument and parameter types don't match");
4264
0
    EmitCallArg(Args, *Arg, ArgTypes[Idx]);
4265
    // In particular, we depend on it being the last arg in Args, and the
4266
    // objectsize bits depend on there only being one arg if !LeftToRight.
4267
396k
    assert(InitialArgSize + 1 == Args.size() &&
4268
396k
           "The code below depends on only adding one arg per EmitCallArg");
4269
0
    (void)InitialArgSize;
4270
    // Since pointer argument are never emitted as LValue, it is safe to emit
4271
    // non-null argument check for r-value only.
4272
396k
    if (!Args.back().hasLValue()) {
4273
396k
      RValue RVArg = Args.back().getKnownRValue();
4274
396k
      EmitNonNullArgCheck(RVArg, ArgTypes[Idx], (*Arg)->getExprLoc(), AC,
4275
396k
                          ParamsToSkip + Idx);
4276
      // @llvm.objectsize should never have side-effects and shouldn't need
4277
      // destruction/cleanups, so we can safely "emit" it after its arg,
4278
      // regardless of right-to-leftness
4279
396k
      MaybeEmitImplicitObjectSize(Idx, *Arg, RVArg);
4280
396k
    }
4281
396k
  }
4282
4283
308k
  if (!LeftToRight) {
4284
    // Un-reverse the arguments we just evaluated so they match up with the LLVM
4285
    // IR function.
4286
8.24k
    std::reverse(Args.begin() + CallArgsStart, Args.end());
4287
8.24k
  }
4288
308k
}
4289
4290
namespace {
4291
4292
struct DestroyUnpassedArg final : EHScopeStack::Cleanup {
4293
  DestroyUnpassedArg(Address Addr, QualType Ty)
4294
182
      : Addr(Addr), Ty(Ty) {}
4295
4296
  Address Addr;
4297
  QualType Ty;
4298
4299
20
  void Emit(CodeGenFunction &CGF, Flags flags) override {
4300
20
    QualType::DestructionKind DtorKind = Ty.isDestructedType();
4301
20
    if (DtorKind == QualType::DK_cxx_destructor) {
4302
18
      const CXXDestructorDecl *Dtor = Ty->getAsCXXRecordDecl()->getDestructor();
4303
18
      assert(!Dtor->isTrivial());
4304
0
      CGF.EmitCXXDestructorCall(Dtor, Dtor_Complete, /*for vbase*/ false,
4305
18
                                /*Delegating=*/false, Addr, Ty);
4306
18
    } else {
4307
2
      CGF.callCStructDestructor(CGF.MakeAddrLValue(Addr, Ty));
4308
2
    }
4309
20
  }
4310
};
4311
4312
struct DisableDebugLocationUpdates {
4313
  CodeGenFunction &CGF;
4314
  bool disabledDebugInfo;
4315
396k
  DisableDebugLocationUpdates(CodeGenFunction &CGF, const Expr *E) : CGF(CGF) {
4316
396k
    if ((disabledDebugInfo = isa<CXXDefaultArgExpr>(E) && 
CGF.getDebugInfo()4.12k
))
4317
2.82k
      CGF.disableDebugInfo();
4318
396k
  }
4319
396k
  ~DisableDebugLocationUpdates() {
4320
396k
    if (disabledDebugInfo)
4321
2.82k
      CGF.enableDebugInfo();
4322
396k
  }
4323
};
4324
4325
} // end anonymous namespace
4326
4327
266
RValue CallArg::getRValue(CodeGenFunction &CGF) const {
4328
266
  if (!HasLV)
4329
266
    return RV;
4330
0
  LValue Copy = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty), Ty);
4331
0
  CGF.EmitAggregateCopy(Copy, LV, Ty, AggValueSlot::DoesNotOverlap,
4332
0
                        LV.isVolatile());
4333
0
  IsUsed = true;
4334
0
  return RValue::getAggregate(Copy.getAddress(CGF));
4335
266
}
4336
4337
1.26k
void CallArg::copyInto(CodeGenFunction &CGF, Address Addr) const {
4338
1.26k
  LValue Dst = CGF.MakeAddrLValue(Addr, Ty);
4339
1.26k
  if (!HasLV && 
RV.isScalar()1.20k
)
4340
862
    CGF.EmitStoreOfScalar(RV.getScalarVal(), Dst, /*isInit=*/true);
4341
401
  else if (!HasLV && 
RV.isComplex()338
)
4342
338
    CGF.EmitStoreOfComplex(RV.getComplexVal(), Dst, /*init=*/true);
4343
63
  else {
4344
63
    auto Addr = HasLV ? LV.getAddress(CGF) : 
RV.getAggregateAddress()0
;
4345
63
    LValue SrcLV = CGF.MakeAddrLValue(Addr, Ty);
4346
    // We assume that call args are never copied into subobjects.
4347
63
    CGF.EmitAggregateCopy(Dst, SrcLV, Ty, AggValueSlot::DoesNotOverlap,
4348
63
                          HasLV ? LV.isVolatileQualified()
4349
63
                                : 
RV.isVolatileQualified()0
);
4350
63
  }
4351
1.26k
  IsUsed = true;
4352
1.26k
}
4353
4354
void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E,
4355
396k
                                  QualType type) {
4356
396k
  DisableDebugLocationUpdates Dis(*this, E);
4357
396k
  if (const ObjCIndirectCopyRestoreExpr *CRE
4358
396k
        = dyn_cast<ObjCIndirectCopyRestoreExpr>(E)) {
4359
27
    assert(getLangOpts().ObjCAutoRefCount);
4360
0
    return emitWritebackArg(*this, args, CRE);
4361
27
  }
4362
4363
396k
  assert(type->isReferenceType() == E->isGLValue() &&
4364
396k
         "reference binding to unmaterialized r-value!");
4365
4366
396k
  if (E->isGLValue()) {
4367
51.2k
    assert(E->getObjectKind() == OK_Ordinary);
4368
0
    return args.add(EmitReferenceBindingToExpr(E), type);
4369
51.2k
  }
4370
4371
345k
  bool HasAggregateEvalKind = hasAggregateEvaluationKind(type);
4372
4373
  // In the Microsoft C++ ABI, aggregate arguments are destructed by the callee.
4374
  // However, we still have to push an EH-only cleanup in case we unwind before
4375
  // we make it to the call.
4376
345k
  if (type->isRecordType() &&
4377
345k
      
type->castAs<RecordType>()->getDecl()->isParamDestroyedInCallee()11.0k
) {
4378
    // If we're using inalloca, use the argument memory.  Otherwise, use a
4379
    // temporary.
4380
417
    AggValueSlot Slot = args.isUsingInAlloca()
4381
417
        ? 
createPlaceholderSlot(*this, type)106
:
CreateAggTemp(type, "agg.tmp")311
;
4382
4383
417
    bool DestroyedInCallee = true, NeedsEHCleanup = true;
4384
417
    if (const auto *RD = type->getAsCXXRecordDecl())
4385
395
      DestroyedInCallee = RD->hasNonTrivialDestructor();
4386
22
    else
4387
22
      NeedsEHCleanup = needsEHCleanup(type.isDestructedType());
4388
4389
417
    if (DestroyedInCallee)
4390
204
      Slot.setExternallyDestructed();
4391
4392
417
    EmitAggExpr(E, Slot);
4393
417
    RValue RV = Slot.asRValue();
4394
417
    args.add(RV, type);
4395
4396
417
    if (DestroyedInCallee && 
NeedsEHCleanup204
) {
4397
      // Create a no-op GEP between the placeholder and the cleanup so we can
4398
      // RAUW it successfully.  It also serves as a marker of the first
4399
      // instruction where the cleanup is active.
4400
186
      pushFullExprCleanup<DestroyUnpassedArg>(EHCleanup, Slot.getAddress(),
4401
186
                                              type);
4402
      // This unreachable is a temporary marker which will be removed later.
4403
186
      llvm::Instruction *IsActive = Builder.CreateUnreachable();
4404
186
      args.addArgCleanupDeactivation(EHStack.stable_begin(), IsActive);
4405
186
    }
4406
417
    return;
4407
417
  }
4408
4409
345k
  if (HasAggregateEvalKind && 
isa<ImplicitCastExpr>(E)10.6k
&&
4410
345k
      
cast<CastExpr>(E)->getCastKind() == CK_LValueToRValue875
) {
4411
810
    LValue L = EmitLValue(cast<CastExpr>(E)->getSubExpr());
4412
810
    assert(L.isSimple());
4413
0
    args.addUncopiedAggregate(L, type);
4414
810
    return;
4415
810
  }
4416
4417
344k
  args.add(EmitAnyExprToTemp(E), type);
4418
344k
}
4419
4420
90.4k
QualType CodeGenFunction::getVarArgType(const Expr *Arg) {
4421
  // System headers on Windows define NULL to 0 instead of 0LL on Win64. MSVC
4422
  // implicitly widens null pointer constants that are arguments to varargs
4423
  // functions to pointer-sized ints.
4424
90.4k
  if (!getTarget().getTriple().isOSWindows())
4425
90.3k
    return Arg->getType();
4426
4427
94
  if (Arg->getType()->isIntegerType() &&
4428
94
      getContext().getTypeSize(Arg->getType()) <
4429
53
          getContext().getTargetInfo().getPointerWidth(0) &&
4430
94
      Arg->isNullPointerConstant(getContext(),
4431
29
                                 Expr::NPC_ValueDependentIsNotNull)) {
4432
1
    return getContext().getIntPtrType();
4433
1
  }
4434
4435
93
  return Arg->getType();
4436
94
}
4437
4438
// In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
4439
// optimizer it can aggressively ignore unwind edges.
4440
void
4441
1.87k
CodeGenFunction::AddObjCARCExceptionMetadata(llvm::Instruction *Inst) {
4442
1.87k
  if (CGM.getCodeGenOpts().OptimizationLevel != 0 &&
4443
1.87k
      
!CGM.getCodeGenOpts().ObjCAutoRefCountExceptions468
)
4444
378
    Inst->setMetadata("clang.arc.no_objc_arc_exceptions",
4445
378
                      CGM.getNoObjCARCExceptionsMetadata());
4446
1.87k
}
4447
4448
/// Emits a call to the given no-arguments nounwind runtime function.
4449
llvm::CallInst *
4450
CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
4451
373
                                         const llvm::Twine &name) {
4452
373
  return EmitNounwindRuntimeCall(callee, None, name);
4453
373
}
4454
4455
/// Emits a call to the given nounwind runtime function.
4456
llvm::CallInst *
4457
CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
4458
                                         ArrayRef<llvm::Value *> args,
4459
32.3k
                                         const llvm::Twine &name) {
4460
32.3k
  llvm::CallInst *call = EmitRuntimeCall(callee, args, name);
4461
32.3k
  call->setDoesNotThrow();
4462
32.3k
  return call;
4463
32.3k
}
4464
4465
/// Emits a simple call (never an invoke) to the given no-arguments
4466
/// runtime function.
4467
llvm::CallInst *CodeGenFunction::EmitRuntimeCall(llvm::FunctionCallee callee,
4468
5.61k
                                                 const llvm::Twine &name) {
4469
5.61k
  return EmitRuntimeCall(callee, None, name);
4470
5.61k
}
4471
4472
// Calls which may throw must have operand bundles indicating which funclet
4473
// they are nested within.
4474
SmallVector<llvm::OperandBundleDef, 1>
4475
445k
CodeGenFunction::getBundlesForFunclet(llvm::Value *Callee) {
4476
445k
  SmallVector<llvm::OperandBundleDef, 1> BundleList;
4477
  // There is no need for a funclet operand bundle if we aren't inside a
4478
  // funclet.
4479
445k
  if (!CurrentFuncletPad)
4480
445k
    return BundleList;
4481
4482
  // Skip intrinsics which cannot throw.
4483
243
  auto *CalleeFn = dyn_cast<llvm::Function>(Callee->stripPointerCasts());
4484
243
  if (CalleeFn && 
CalleeFn->isIntrinsic()237
&&
CalleeFn->doesNotThrow()2
)
4485
2
    return BundleList;
4486
4487
241
  BundleList.emplace_back("funclet", CurrentFuncletPad);
4488
241
  return BundleList;
4489
243
}
4490
4491
/// Emits a simple call (never an invoke) to the given runtime function.
4492
llvm::CallInst *CodeGenFunction::EmitRuntimeCall(llvm::FunctionCallee callee,
4493
                                                 ArrayRef<llvm::Value *> args,
4494
81.5k
                                                 const llvm::Twine &name) {
4495
81.5k
  llvm::CallInst *call = Builder.CreateCall(
4496
81.5k
      callee, args, getBundlesForFunclet(callee.getCallee()), name);
4497
81.5k
  call->setCallingConv(getRuntimeCC());
4498
81.5k
  return call;
4499
81.5k
}
4500
4501
/// Emits a call or invoke to the given noreturn runtime function.
4502
void CodeGenFunction::EmitNoreturnRuntimeCallOrInvoke(
4503
749
    llvm::FunctionCallee callee, ArrayRef<llvm::Value *> args) {
4504
749
  SmallVector<llvm::OperandBundleDef, 1> BundleList =
4505
749
      getBundlesForFunclet(callee.getCallee());
4506
4507
749
  if (getInvokeDest()) {
4508
136
    llvm::InvokeInst *invoke =
4509
136
      Builder.CreateInvoke(callee,
4510
136
                           getUnreachableBlock(),
4511
136
                           getInvokeDest(),
4512
136
                           args,
4513
136
                           BundleList);
4514
136
    invoke->setDoesNotReturn();
4515
136
    invoke->setCallingConv(getRuntimeCC());
4516
613
  } else {
4517
613
    llvm::CallInst *call = Builder.CreateCall(callee, args, BundleList);
4518
613
    call->setDoesNotReturn();
4519
613
    call->setCallingConv(getRuntimeCC());
4520
613
    Builder.CreateUnreachable();
4521
613
  }
4522
749
}
4523
4524
/// Emits a call or invoke instruction to the given nullary runtime function.
4525
llvm::CallBase *
4526
CodeGenFunction::EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
4527
379
                                         const Twine &name) {
4528
379
  return EmitRuntimeCallOrInvoke(callee, None, name);
4529
379
}
4530
4531
/// Emits a call or invoke instruction to the given runtime function.
4532
llvm::CallBase *
4533
CodeGenFunction::EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
4534
                                         ArrayRef<llvm::Value *> args,
4535
676
                                         const Twine &name) {
4536
676
  llvm::CallBase *call = EmitCallOrInvoke(callee, args, name);
4537
676
  call->setCallingConv(getRuntimeCC());
4538
676
  return call;
4539
676
}
4540
4541
/// Emits a call or invoke instruction to the given function, depending
4542
/// on the current state of the EH stack.
4543
llvm::CallBase *CodeGenFunction::EmitCallOrInvoke(llvm::FunctionCallee Callee,
4544
                                                  ArrayRef<llvm::Value *> Args,
4545
2.23k
                                                  const Twine &Name) {
4546
2.23k
  llvm::BasicBlock *InvokeDest = getInvokeDest();
4547
2.23k
  SmallVector<llvm::OperandBundleDef, 1> BundleList =
4548
2.23k
      getBundlesForFunclet(Callee.getCallee());
4549
4550
2.23k
  llvm::CallBase *Inst;
4551
2.23k
  if (!InvokeDest)
4552
1.95k
    Inst = Builder.CreateCall(Callee, Args, BundleList, Name);
4553
275
  else {
4554
275
    llvm::BasicBlock *ContBB = createBasicBlock("invoke.cont");
4555
275
    Inst = Builder.CreateInvoke(Callee, ContBB, InvokeDest, Args, BundleList,
4556
275
                                Name);
4557
275
    EmitBlock(ContBB);
4558
275
  }
4559
4560
  // In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
4561
  // optimizer it can aggressively ignore unwind edges.
4562
2.23k
  if (CGM.getLangOpts().ObjCAutoRefCount)
4563
8
    AddObjCARCExceptionMetadata(Inst);
4564
4565
2.23k
  return Inst;
4566
2.23k
}
4567
4568
void CodeGenFunction::deferPlaceholderReplacement(llvm::Instruction *Old,
4569
107
                                                  llvm::Value *New) {
4570
107
  DeferredReplacements.push_back(
4571
107
      std::make_pair(llvm::WeakTrackingVH(Old), New));
4572
107
}
4573
4574
namespace {
4575
4576
/// Specify given \p NewAlign as the alignment of return value attribute. If
4577
/// such attribute already exists, re-set it to the maximal one of two options.
4578
LLVM_NODISCARD llvm::AttributeList
4579
maybeRaiseRetAlignmentAttribute(llvm::LLVMContext &Ctx,
4580
                                const llvm::AttributeList &Attrs,
4581
32
                                llvm::Align NewAlign) {
4582
32
  llvm::Align CurAlign = Attrs.getRetAlignment().valueOrOne();
4583
32
  if (CurAlign >= NewAlign)
4584
2
    return Attrs;
4585
30
  llvm::Attribute AlignAttr = llvm::Attribute::getWithAlignment(Ctx, NewAlign);
4586
30
  return Attrs.removeRetAttribute(Ctx, llvm::Attribute::AttrKind::Alignment)
4587
30
      .addRetAttribute(Ctx, AlignAttr);
4588
32
}
4589
4590
template <typename AlignedAttrTy> class AbstractAssumeAlignedAttrEmitter {
4591
protected:
4592
  CodeGenFunction &CGF;
4593
4594
  /// We do nothing if this is, or becomes, nullptr.
4595
  const AlignedAttrTy *AA = nullptr;
4596
4597
  llvm::Value *Alignment = nullptr;      // May or may not be a constant.
4598
  llvm::ConstantInt *OffsetCI = nullptr; // Constant, hopefully zero.
4599
4600
  AbstractAssumeAlignedAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl)
4601
718k
      : CGF(CGF_) {
4602
718k
    if (!FuncDecl)
4603
33.9k
      return;
4604
684k
    AA = FuncDecl->getAttr<AlignedAttrTy>();
4605
684k
  }
CGCall.cpp:(anonymous namespace)::AbstractAssumeAlignedAttrEmitter<clang::AssumeAlignedAttr>::AbstractAssumeAlignedAttrEmitter(clang::CodeGen::CodeGenFunction&, clang::Decl const*)
Line
Count
Source
4601
359k
      : CGF(CGF_) {
4602
359k
    if (!FuncDecl)
4603
16.9k
      return;
4604
342k
    AA = FuncDecl->getAttr<AlignedAttrTy>();
4605
342k
  }
CGCall.cpp:(anonymous namespace)::AbstractAssumeAlignedAttrEmitter<clang::AllocAlignAttr>::AbstractAssumeAlignedAttrEmitter(clang::CodeGen::CodeGenFunction&, clang::Decl const*)
Line
Count
Source
4601
359k
      : CGF(CGF_) {
4602
359k
    if (!FuncDecl)
4603
16.9k
      return;
4604
342k
    AA = FuncDecl->getAttr<AlignedAttrTy>();
4605
342k
  }
4606
4607
public:
4608
  /// If we can, materialize the alignment as an attribute on return value.
4609
  LLVM_NODISCARD llvm::AttributeList
4610
718k
  TryEmitAsCallSiteAttribute(const llvm::AttributeList &Attrs) {
4611
718k
    if (!AA || 
OffsetCI69
||
CGF.SanOpts.has(SanitizerKind::Alignment)64
)
4612
718k
      return Attrs;
4613
55
    const auto *AlignmentCI = dyn_cast<llvm::ConstantInt>(Alignment);
4614
55
    if (!AlignmentCI)
4615
22
      return Attrs;
4616
    // We may legitimately have non-power-of-2 alignment here.
4617
    // If so, this is UB land, emit it via `@llvm.assume` instead.
4618
33
    if (!AlignmentCI->getValue().isPowerOf2())
4619
1
      return Attrs;
4620
32
    llvm::AttributeList NewAttrs = maybeRaiseRetAlignmentAttribute(
4621
32
        CGF.getLLVMContext(), Attrs,
4622
32
        llvm::Align(
4623
32
            AlignmentCI->getLimitedValue(llvm::Value::MaximumAlignment)));
4624
32
    AA = nullptr; // We're done. Disallow doing anything else.
4625
32
    return NewAttrs;
4626
33
  }
CGCall.cpp:(anonymous namespace)::AbstractAssumeAlignedAttrEmitter<clang::AssumeAlignedAttr>::TryEmitAsCallSiteAttribute(llvm::AttributeList const&)
Line
Count
Source
4610
359k
  TryEmitAsCallSiteAttribute(const llvm::AttributeList &Attrs) {
4611
359k
    if (!AA || 
OffsetCI14
||
CGF.SanOpts.has(SanitizerKind::Alignment)9
)
4612
359k
      return Attrs;
4613
6
    const auto *AlignmentCI = dyn_cast<llvm::ConstantInt>(Alignment);
4614
6
    if (!AlignmentCI)
4615
0
      return Attrs;
4616
    // We may legitimately have non-power-of-2 alignment here.
4617
    // If so, this is UB land, emit it via `@llvm.assume` instead.
4618
6
    if (!AlignmentCI->getValue().isPowerOf2())
4619
0
      return Attrs;
4620
6
    llvm::AttributeList NewAttrs = maybeRaiseRetAlignmentAttribute(
4621
6
        CGF.getLLVMContext(), Attrs,
4622
6
        llvm::Align(
4623
6
            AlignmentCI->getLimitedValue(llvm::Value::MaximumAlignment)));
4624
6
    AA = nullptr; // We're done. Disallow doing anything else.
4625
6
    return NewAttrs;
4626
6
  }
CGCall.cpp:(anonymous namespace)::AbstractAssumeAlignedAttrEmitter<clang::AllocAlignAttr>::TryEmitAsCallSiteAttribute(llvm::AttributeList const&)
Line
Count
Source
4610
359k
  TryEmitAsCallSiteAttribute(const llvm::AttributeList &Attrs) {
4611
359k
    if (!AA || 
OffsetCI55
||
CGF.SanOpts.has(SanitizerKind::Alignment)55
)
4612
359k
      return Attrs;
4613
49
    const auto *AlignmentCI = dyn_cast<llvm::ConstantInt>(Alignment);
4614
49
    if (!AlignmentCI)
4615
22
      return Attrs;
4616
    // We may legitimately have non-power-of-2 alignment here.
4617
    // If so, this is UB land, emit it via `@llvm.assume` instead.
4618
27
    if (!AlignmentCI->getValue().isPowerOf2())
4619
1
      return Attrs;
4620
26
    llvm::AttributeList NewAttrs = maybeRaiseRetAlignmentAttribute(
4621
26
        CGF.getLLVMContext(), Attrs,
4622
26
        llvm::Align(
4623
26
            AlignmentCI->getLimitedValue(llvm::Value::MaximumAlignment)));
4624
26
    AA = nullptr; // We're done. Disallow doing anything else.
4625
26
    return NewAttrs;
4626
27
  }
4627
4628
  /// Emit alignment assumption.
4629
  /// This is a general fallback that we take if either there is an offset,
4630
  /// or the alignment is variable or we are sanitizing for alignment.
4631
663k
  void EmitAsAnAssumption(SourceLocation Loc, QualType RetTy, RValue &Ret) {
4632
663k
    if (!AA)
4633
663k
      return;
4634
37
    CGF.emitAlignmentAssumption(Ret.getScalarVal(), RetTy, Loc,
4635
37
                                AA->getLocation(), Alignment, OffsetCI);
4636
37
    AA = nullptr; // We're done. Disallow doing anything else.
4637
37
  }
CGCall.cpp:(anonymous namespace)::AbstractAssumeAlignedAttrEmitter<clang::AssumeAlignedAttr>::EmitAsAnAssumption(clang::SourceLocation, clang::QualType, clang::CodeGen::RValue&)
Line
Count
Source
4631
331k
  void EmitAsAnAssumption(SourceLocation Loc, QualType RetTy, RValue &Ret) {
4632
331k
    if (!AA)
4633
331k
      return;
4634
8
    CGF.emitAlignmentAssumption(Ret.getScalarVal(), RetTy, Loc,
4635
8
                                AA->getLocation(), Alignment, OffsetCI);
4636
8
    AA = nullptr; // We're done. Disallow doing anything else.
4637
8
  }
CGCall.cpp:(anonymous namespace)::AbstractAssumeAlignedAttrEmitter<clang::AllocAlignAttr>::EmitAsAnAssumption(clang::SourceLocation, clang::QualType, clang::CodeGen::RValue&)
Line
Count
Source
4631
331k
  void EmitAsAnAssumption(SourceLocation Loc, QualType RetTy, RValue &Ret) {
4632
331k
    if (!AA)
4633
331k
      return;
4634
29
    CGF.emitAlignmentAssumption(Ret.getScalarVal(), RetTy, Loc,
4635
29
                                AA->getLocation(), Alignment, OffsetCI);
4636
29
    AA = nullptr; // We're done. Disallow doing anything else.
4637
29
  }
4638
};
4639
4640
/// Helper data structure to emit `AssumeAlignedAttr`.
4641
class AssumeAlignedAttrEmitter final
4642
    : public AbstractAssumeAlignedAttrEmitter<AssumeAlignedAttr> {
4643
public:
4644
  AssumeAlignedAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl)
4645
359k
      : AbstractAssumeAlignedAttrEmitter(CGF_, FuncDecl) {
4646
359k
    if (!AA)
4647
359k
      return;
4648
    // It is guaranteed that the alignment/offset are constants.
4649
14
    Alignment = cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AA->getAlignment()));
4650
14
    if (Expr *Offset = AA->getOffset()) {
4651
5
      OffsetCI = cast<llvm::ConstantInt>(CGF.EmitScalarExpr(Offset));
4652
5
      if (OffsetCI->isNullValue()) // Canonicalize zero offset to no offset.
4653
0
        OffsetCI = nullptr;
4654
5
    }
4655
14
  }
4656
};
4657
4658
/// Helper data structure to emit `AllocAlignAttr`.
4659
class AllocAlignAttrEmitter final
4660
    : public AbstractAssumeAlignedAttrEmitter<AllocAlignAttr> {
4661
public:
4662
  AllocAlignAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl,
4663
                        const CallArgList &CallArgs)
4664
359k
      : AbstractAssumeAlignedAttrEmitter(CGF_, FuncDecl) {
4665
359k
    if (!AA)
4666
359k
      return;
4667
    // Alignment may or may not be a constant, and that is okay.
4668
55
    Alignment = CallArgs[AA->getParamIndex().getLLVMIndex()]
4669
55
                    .getRValue(CGF)
4670
55
                    .getScalarVal();
4671
55
  }
4672
};
4673
4674
} // namespace
4675
4676
975k
static unsigned getMaxVectorWidth(const llvm::Type *Ty) {
4677
975k
  if (auto *VT = dyn_cast<llvm::VectorType>(Ty))
4678
47.2k
    return VT->getPrimitiveSizeInBits().getKnownMinSize();
4679
928k
  if (auto *AT = dyn_cast<llvm::ArrayType>(Ty))
4680
534
    return getMaxVectorWidth(AT->getElementType());
4681
4682
927k
  unsigned MaxVectorWidth = 0;
4683
927k
  if (auto *ST = dyn_cast<llvm::StructType>(Ty))
4684
2.65k
    for (auto *I : ST->elements())
4685
5.46k
      MaxVectorWidth = std::max(MaxVectorWidth, getMaxVectorWidth(I));
4686
927k
  return MaxVectorWidth;
4687
928k
}
4688
4689
RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
4690
                                 const CGCallee &Callee,
4691
                                 ReturnValueSlot ReturnValue,
4692
                                 const CallArgList &CallArgs,
4693
                                 llvm::CallBase **callOrInvoke, bool IsMustTail,
4694
359k
                                 SourceLocation Loc) {
4695
  // FIXME: We no longer need the types from CallArgs; lift up and simplify.
4696
4697
359k
  assert(Callee.isOrdinary() || Callee.isVirtual());
4698
4699
  // Handle struct-return functions by passing a pointer to the
4700
  // location that we would like to return into.
4701
0
  QualType RetTy = CallInfo.getReturnType();
4702
359k
  const ABIArgInfo &RetAI = CallInfo.getReturnInfo();
4703
4704
359k
  llvm::FunctionType *IRFuncTy = getTypes().GetFunctionType(CallInfo);
4705
4706
359k
  const Decl *TargetDecl = Callee.getAbstractInfo().getCalleeDecl().getDecl();
4707
359k
  if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(TargetDecl)) {
4708
    // We can only guarantee that a function is called from the correct
4709
    // context/function based on the appropriate target attributes,
4710
    // so only check in the case where we have both always_inline and target
4711
    // since otherwise we could be making a conditional call after a check for
4712
    // the proper cpu features (and it won't cause code generation issues due to
4713
    // function based code generation).
4714
336k
    if (TargetDecl->hasAttr<AlwaysInlineAttr>() &&
4715
336k
        
TargetDecl->hasAttr<TargetAttr>()22.0k
)
4716
14.8k
      checkTargetFeatures(Loc, FD);
4717
4718
    // Some architectures (such as x86-64) have the ABI changed based on
4719
    // attribute-target/features. Give them a chance to diagnose.
4720
336k
    CGM.getTargetCodeGenInfo().checkFunctionCallABI(
4721
336k
        CGM, Loc, dyn_cast_or_null<FunctionDecl>(CurCodeDecl), FD, CallArgs);
4722
336k
  }
4723
4724
359k
#ifndef NDEBUG
4725
359k
  if (!(CallInfo.isVariadic() && 
CallInfo.getArgStruct()78.8k
)) {
4726
    // For an inalloca varargs function, we don't expect CallInfo to match the
4727
    // function pointer's type, because the inalloca struct a will have extra
4728
    // fields in it for the varargs parameters.  Code later in this function
4729
    // bitcasts the function pointer to the type derived from CallInfo.
4730
    //
4731
    // In other cases, we assert that the types match up (until pointers stop
4732
    // having pointee types).
4733
359k
    if (Callee.isVirtual())
4734
1.01k
      assert(IRFuncTy == Callee.getVirtualFunctionType());
4735
358k
    else {
4736
358k
      llvm::PointerType *PtrTy =
4737
358k
          llvm::cast<llvm::PointerType>(Callee.getFunctionPointer()->getType());
4738
358k
      assert(PtrTy->isOpaqueOrPointeeTypeMatches(IRFuncTy));
4739
358k
    }
4740
359k
  }
4741
0
#endif
4742
4743
  // 1. Set up the arguments.
4744
4745
  // If we're using inalloca, insert the allocation after the stack save.
4746
  // FIXME: Do this earlier rather than hacking it in here!
4747
0
  Address ArgMemory = Address::invalid();
4748
359k
  if (llvm::StructType *ArgStruct = CallInfo.getArgStruct()) {
4749
74
    const llvm::DataLayout &DL = CGM.getDataLayout();
4750
74
    llvm::Instruction *IP = CallArgs.getStackBase();
4751
74
    llvm::AllocaInst *AI;
4752
74
    if (IP) {
4753
73
      IP = IP->getNextNode();
4754
73
      AI = new llvm::AllocaInst(ArgStruct, DL.getAllocaAddrSpace(),
4755
73
                                "argmem", IP);
4756
73
    } else {
4757
1
      AI = CreateTempAlloca(ArgStruct, "argmem");
4758
1
    }
4759
74
    auto Align = CallInfo.getArgStructAlignment();
4760
74
    AI->setAlignment(Align.getAsAlign());
4761
74
    AI->setUsedWithInAlloca(true);
4762
74
    assert(AI->isUsedWithInAlloca() && !AI->isStaticAlloca());
4763
0
    ArgMemory = Address(AI, ArgStruct, Align);
4764
74
  }
4765
4766
0
  ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), CallInfo);
4767
359k
  SmallVector<llvm::Value *, 16> IRCallArgs(IRFunctionArgs.totalIRArgs());
4768
4769
  // If the call returns a temporary with struct return, create a temporary
4770
  // alloca to hold the result, unless one is given to us.
4771
359k
  Address SRetPtr = Address::invalid();
4772
359k
  Address SRetAlloca = Address::invalid();
4773
359k
  llvm::Value *UnusedReturnSizePtr = nullptr;
4774
359k
  if (RetAI.isIndirect() || 
RetAI.isInAlloca()356k
||
RetAI.isCoerceAndExpand()356k
) {
4775
2.71k
    if (!ReturnValue.isNull()) {
4776
1.92k
      SRetPtr = ReturnValue.getValue();
4777
1.92k
    } else {
4778
791
      SRetPtr = CreateMemTemp(RetTy, "tmp", &SRetAlloca);
4779
791
      if (HaveInsertPoint() && ReturnValue.isUnused()) {
4780
130
        llvm::TypeSize size =
4781
130
            CGM.getDataLayout().getTypeAllocSize(ConvertTypeForMem(RetTy));
4782
130
        UnusedReturnSizePtr = EmitLifetimeStart(size, SRetAlloca.getPointer());
4783
130
      }
4784
791
    }
4785
2.71k
    if (IRFunctionArgs.hasSRetArg()) {
4786
2.26k
      IRCallArgs[IRFunctionArgs.getSRetArgNo()] = SRetPtr.getPointer();
4787
2.26k
    } else 
if (446
RetAI.isInAlloca()446
) {
4788
4
      Address Addr =
4789
4
          Builder.CreateStructGEP(ArgMemory, RetAI.getInAllocaFieldIndex());
4790
4
      Builder.CreateStore(SRetPtr.getPointer(), Addr);
4791
4
    }
4792
2.71k
  }
4793
4794
359k
  Address swiftErrorTemp = Address::invalid();
4795
359k
  Address swiftErrorArg = Address::invalid();
4796
4797
  // When passing arguments using temporary allocas, we need to add the
4798
  // appropriate lifetime markers. This vector keeps track of all the lifetime
4799
  // markers that need to be ended right after the call.
4800
359k
  SmallVector<CallLifetimeEnd, 2> CallLifetimeEndAfterCall;
4801
4802
  // Translate all of the arguments as necessary to match the IR lowering.
4803
359k
  assert(CallInfo.arg_size() == CallArgs.size() &&
4804
359k
         "Mismatch between function signature & arguments.");
4805
0
  unsigned ArgNo = 0;
4806
359k
  CGFunctionInfo::const_arg_iterator info_it = CallInfo.arg_begin();
4807
359k
  for (CallArgList::const_iterator I = CallArgs.begin(), E = CallArgs.end();
4808
972k
       I != E; 
++I, ++info_it, ++ArgNo613k
) {
4809
613k
    const ABIArgInfo &ArgInfo = info_it->info;
4810
4811
    // Insert a padding argument to ensure proper alignment.
4812
613k
    if (IRFunctionArgs.hasPaddingArg(ArgNo))
4813
17
      IRCallArgs[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
4814
17
          llvm::UndefValue::get(ArgInfo.getPaddingType());
4815
4816
613k
    unsigned FirstIRArg, NumIRArgs;
4817
613k
    std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
4818
4819
613k
    switch (ArgInfo.getKind()) {
4820
160
    case ABIArgInfo::InAlloca: {
4821
160
      assert(NumIRArgs == 0);
4822
0
      assert(getTarget().getTriple().getArch() == llvm::Triple::x86);
4823
160
      if (I->isAggregate()) {
4824
107
        Address Addr = I->hasLValue()
4825
107
                           ? 
I->getKnownLValue().getAddress(*this)0
4826
107
                           : I->getKnownRValue().getAggregateAddress();
4827
107
        llvm::Instruction *Placeholder =
4828
107
            cast<llvm::Instruction>(Addr.getPointer());
4829
4830
107
        if (!ArgInfo.getInAllocaIndirect()) {
4831
          // Replace the placeholder with the appropriate argument slot GEP.
4832
105
          CGBuilderTy::InsertPoint IP = Builder.saveIP();
4833
105
          Builder.SetInsertPoint(Placeholder);
4834
105
          Addr = Builder.CreateStructGEP(ArgMemory,
4835
105
                                         ArgInfo.getInAllocaFieldIndex());
4836
105
          Builder.restoreIP(IP);
4837
105
        } else {
4838
          // For indirect things such as overaligned structs, replace the
4839
          // placeholder with a regular aggregate temporary alloca. Store the
4840
          // address of this alloca into the struct.
4841
2
          Addr = CreateMemTemp(info_it->type, "inalloca.indirect.tmp");
4842
2
          Address ArgSlot = Builder.CreateStructGEP(
4843
2
              ArgMemory, ArgInfo.getInAllocaFieldIndex());
4844
2
          Builder.CreateStore(Addr.getPointer(), ArgSlot);
4845
2
        }
4846
107
        deferPlaceholderReplacement(Placeholder, Addr.getPointer());
4847
107
      } else 
if (53
ArgInfo.getInAllocaIndirect()53
) {
4848
        // Make a temporary alloca and store the address of it into the argument
4849
        // struct.
4850
2
        Address Addr = CreateMemTempWithoutCast(
4851
2
            I->Ty, getContext().getTypeAlignInChars(I->Ty),
4852
2
            "indirect-arg-temp");
4853
2
        I->copyInto(*this, Addr);
4854
2
        Address ArgSlot =
4855
2
            Builder.CreateStructGEP(ArgMemory, ArgInfo.getInAllocaFieldIndex());
4856
2
        Builder.CreateStore(Addr.getPointer(), ArgSlot);
4857
51
      } else {
4858
        // Store the RValue into the argument struct.
4859
51
        Address Addr =
4860
51
            Builder.CreateStructGEP(ArgMemory, ArgInfo.getInAllocaFieldIndex());
4861
        // There are some cases where a trivial bitcast is not avoidable.  The
4862
        // definition of a type later in a translation unit may change it's type
4863
        // from {}* to (%struct.foo*)*.
4864
51
        Addr = Builder.CreateElementBitCast(Addr, ConvertTypeForMem(I->Ty));
4865
51
        I->copyInto(*this, Addr);
4866
51
      }
4867
160
      break;
4868
0
    }
4869
4870
2.18k
    case ABIArgInfo::Indirect:
4871
2.18k
    case ABIArgInfo::IndirectAliased: {
4872
2.18k
      assert(NumIRArgs == 1);
4873
2.18k
      if (!I->isAggregate()) {
4874
        // Make a temporary alloca to pass the argument.
4875
234
        Address Addr = CreateMemTempWithoutCast(
4876
234
            I->Ty, ArgInfo.getIndirectAlign(), "indirect-arg-temp");
4877
234
        IRCallArgs[FirstIRArg] = Addr.getPointer();
4878
4879
234
        I->copyInto(*this, Addr);
4880
1.94k
      } else {
4881
        // We want to avoid creating an unnecessary temporary+copy here;
4882
        // however, we need one in three cases:
4883
        // 1. If the argument is not byval, and we are required to copy the
4884
        //    source.  (This case doesn't occur on any common architecture.)
4885
        // 2. If the argument is byval, RV is not sufficiently aligned, and
4886
        //    we cannot force it to be sufficiently aligned.
4887
        // 3. If the argument is byval, but RV is not located in default
4888
        //    or alloca address space.
4889
1.94k
        Address Addr = I->hasLValue()
4890
1.94k
                           ? 
I->getKnownLValue().getAddress(*this)315
4891
1.94k
                           : 
I->getKnownRValue().getAggregateAddress()1.63k
;
4892
1.94k
        llvm::Value *V = Addr.getPointer();
4893
1.94k
        CharUnits Align = ArgInfo.getIndirectAlign();
4894
1.94k
        const llvm::DataLayout *TD = &CGM.getDataLayout();
4895
4896
1.94k
        assert((FirstIRArg >= IRFuncTy->getNumParams() ||
4897
1.94k
                IRFuncTy->getParamType(FirstIRArg)->getPointerAddressSpace() ==
4898
1.94k
                    TD->getAllocaAddrSpace()) &&
4899
1.94k
               "indirect argument must be in alloca address space");
4900
4901
0
        bool NeedCopy = false;
4902
4903
1.94k
        if (Addr.getAlignment() < Align &&
4904
1.94k
            llvm::getOrEnforceKnownAlignment(V, Align.getAsAlign(), *TD) <
4905
104
                Align.getAsAlign()) {
4906
2
          NeedCopy = true;
4907
1.94k
        } else if (I->hasLValue()) {
4908
313
          auto LV = I->getKnownLValue();
4909
313
          auto AS = LV.getAddressSpace();
4910
4911
313
          if (!ArgInfo.getIndirectByVal() ||
4912
313
              
(LV.getAlignment() < getContext().getTypeAlignInChars(I->Ty))270
) {
4913
43
            NeedCopy = true;
4914
43
          }
4915
313
          if (!getLangOpts().OpenCL) {
4916
272
            if ((ArgInfo.getIndirectByVal() &&
4917
272
                
(229
AS != LangAS::Default229
&&
4918
229
                 
AS != CGM.getASTAllocaAddressSpace()0
))) {
4919
0
              NeedCopy = true;
4920
0
            }
4921
272
          }
4922
          // For OpenCL even if RV is located in default or alloca address space
4923
          // we don't want to perform address space cast for it.
4924
41
          else if ((ArgInfo.getIndirectByVal() &&
4925
41
                    Addr.getType()->getAddressSpace() != IRFuncTy->
4926
41
                      getParamType(FirstIRArg)->getPointerAddressSpace())) {
4927
18
            NeedCopy = true;
4928
18
          }
4929
313
        }
4930
4931
1.94k
        if (NeedCopy) {
4932
          // Create an aligned temporary, and copy to it.
4933
63
          Address AI = CreateMemTempWithoutCast(
4934
63
              I->Ty, ArgInfo.getIndirectAlign(), "byval-temp");
4935
63
          IRCallArgs[FirstIRArg] = AI.getPointer();
4936
4937
          // Emit lifetime markers for the temporary alloca.
4938
63
          llvm::TypeSize ByvalTempElementSize =
4939
63
              CGM.getDataLayout().getTypeAllocSize(AI.getElementType());
4940
63
          llvm::Value *LifetimeSize =
4941
63
              EmitLifetimeStart(ByvalTempElementSize, AI.getPointer());
4942
4943
          // Add cleanup code to emit the end lifetime marker after the call.
4944
63
          if (LifetimeSize) // In case we disabled lifetime markers.
4945
2
            CallLifetimeEndAfterCall.emplace_back(AI, LifetimeSize);
4946
4947
          // Generate the copy.
4948
63
          I->copyInto(*this, AI);
4949
1.88k
        } else {
4950
          // Skip the extra memcpy call.
4951
1.88k
          auto *T = llvm::PointerType::getWithSamePointeeType(
4952
1.88k
              cast<llvm::PointerType>(V->getType()),
4953
1.88k
              CGM.getDataLayout().getAllocaAddrSpace());
4954
1.88k
          IRCallArgs[FirstIRArg] = getTargetHooks().performAddrSpaceCast(
4955
1.88k
              *this, V, LangAS::Default, CGM.getASTAllocaAddressSpace(), T,
4956
1.88k
              true);
4957
1.88k
        }
4958
1.94k
      }
4959
0
      break;
4960
2.18k
    }
4961
4962
7.09k
    case ABIArgInfo::Ignore:
4963
7.09k
      assert(NumIRArgs == 0);
4964
0
      break;
4965
4966
9.52k
    case ABIArgInfo::Extend:
4967
603k
    case ABIArgInfo::Direct: {
4968
603k
      if (!isa<llvm::StructType>(ArgInfo.getCoerceToType()) &&
4969
603k
          
ArgInfo.getCoerceToType() == ConvertType(info_it->type)602k
&&
4970
603k
          
ArgInfo.getDirectOffset() == 0598k
) {
4971
598k
        assert(NumIRArgs == 1);
4972
0
        llvm::Value *V;
4973
598k
        if (!I->isAggregate())
4974
598k
          V = I->getKnownRValue().getScalarVal();
4975
0
        else
4976
0
          V = Builder.CreateLoad(
4977
0
              I->hasLValue() ? I->getKnownLValue().getAddress(*this)
4978
0
                             : I->getKnownRValue().getAggregateAddress());
4979
4980
        // Implement swifterror by copying into a new swifterror argument.
4981
        // We'll write back in the normal path out of the call.
4982
598k
        if (CallInfo.getExtParameterInfo(ArgNo).getABI()
4983
598k
              == ParameterABI::SwiftErrorResult) {
4984
8
          assert(!swiftErrorTemp.isValid() && "multiple swifterror args");
4985
4986
0
          QualType pointeeTy = I->Ty->getPointeeType();
4987
8
          swiftErrorArg = Address(V, ConvertTypeForMem(pointeeTy),
4988
8
                                  getContext().getTypeAlignInChars(pointeeTy));
4989
4990
8
          swiftErrorTemp =
4991
8
            CreateMemTemp(pointeeTy, getPointerAlign(), "swifterror.temp");
4992
8
          V = swiftErrorTemp.getPointer();
4993
8
          cast<llvm::AllocaInst>(V)->setSwiftError(true);
4994
4995
8
          llvm::Value *errorValue = Builder.CreateLoad(swiftErrorArg);
4996
8
          Builder.CreateStore(errorValue, swiftErrorTemp);
4997
8
        }
4998
4999
        // We might have to widen integers, but we should never truncate.
5000
598k
        if (ArgInfo.getCoerceToType() != V->getType() &&
5001
598k
            
V->getType()->isIntegerTy()212
)
5002
3
          V = Builder.CreateZExt(V, ArgInfo.getCoerceToType());
5003
5004
        // If the argument doesn't match, perform a bitcast to coerce it.  This
5005
        // can happen due to trivial type mismatches.
5006
598k
        if (FirstIRArg < IRFuncTy->getNumParams() &&
5007
598k
            
V->getType() != IRFuncTy->getParamType(FirstIRArg)508k
)
5008
209
          V = Builder.CreateBitCast(V, IRFuncTy->getParamType(FirstIRArg));
5009
5010
598k
        IRCallArgs[FirstIRArg] = V;
5011
598k
        break;
5012
598k
      }
5013
5014
      // FIXME: Avoid the conversion through memory if possible.
5015
4.78k
      Address Src = Address::invalid();
5016
4.78k
      if (!I->isAggregate()) {
5017
913
        Src = CreateMemTemp(I->Ty, "coerce");
5018
913
        I->copyInto(*this, Src);
5019
3.87k
      } else {
5020
3.87k
        Src = I->hasLValue() ? 
I->getKnownLValue().getAddress(*this)476
5021
3.87k
                             : 
I->getKnownRValue().getAggregateAddress()3.39k
;
5022
3.87k
      }
5023
5024
      // If the value is offset in memory, apply the offset now.
5025
4.78k
      Src = emitAddressAtOffset(*this, Src, ArgInfo);
5026
5027
      // Fast-isel and the optimizer generally like scalar values better than
5028
      // FCAs, so we flatten them if this is safe to do for this argument.
5029
4.78k
      llvm::StructType *STy =
5030
4.78k
            dyn_cast<llvm::StructType>(ArgInfo.getCoerceToType());
5031
4.78k
      if (STy && 
ArgInfo.isDirect()1.32k
&&
ArgInfo.getCanBeFlattened()1.32k
) {
5032
1.29k
        llvm::Type *SrcTy = Src.getElementType();
5033
1.29k
        uint64_t SrcSize = CGM.getDataLayout().getTypeAllocSize(SrcTy);
5034
1.29k
        uint64_t DstSize = CGM.getDataLayout().getTypeAllocSize(STy);
5035
5036
        // If the source type is smaller than the destination type of the
5037
        // coerce-to logic, copy the source value into a temp alloca the size
5038
        // of the destination type to allow loading all of it. The bits past
5039
        // the source value are left undef.
5040
1.29k
        if (SrcSize < DstSize) {
5041
184
          Address TempAlloca
5042
184
            = CreateTempAlloca(STy, Src.getAlignment(),
5043
184
                               Src.getName() + ".coerce");
5044
184
          Builder.CreateMemCpy(TempAlloca, Src, SrcSize);
5045
184
          Src = TempAlloca;
5046
1.10k
        } else {
5047
1.10k
          Src = Builder.CreateElementBitCast(Src, STy);
5048
1.10k
        }
5049
5050
1.29k
        assert(NumIRArgs == STy->getNumElements());
5051
3.87k
        for (unsigned i = 0, e = STy->getNumElements(); i != e; 
++i2.58k
) {
5052
2.58k
          Address EltPtr = Builder.CreateStructGEP(Src, i);
5053
2.58k
          llvm::Value *LI = Builder.CreateLoad(EltPtr);
5054
2.58k
          IRCallArgs[FirstIRArg + i] = LI;
5055
2.58k
        }
5056
3.49k
      } else {
5057
        // In the simple case, just pass the coerced loaded value.
5058
3.49k
        assert(NumIRArgs == 1);
5059
0
        llvm::Value *Load =
5060
3.49k
            CreateCoercedLoad(Src, ArgInfo.getCoerceToType(), *this);
5061
5062
3.49k
        if (CallInfo.isCmseNSCall()) {
5063
          // For certain parameter types, clear padding bits, as they may reveal
5064
          // sensitive information.
5065
          // Small struct/union types are passed as integer arrays.
5066
40
          auto *ATy = dyn_cast<llvm::ArrayType>(Load->getType());
5067
40
          if (ATy != nullptr && 
isa<RecordType>(I->Ty.getCanonicalType())39
)
5068
39
            Load = EmitCMSEClearRecord(Load, ATy, I->Ty);
5069
40
        }
5070
3.49k
        IRCallArgs[FirstIRArg] = Load;