Coverage Report

Created: 2021-07-27 07:33

/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/CodeGen/CGCall.cpp
Line
Count
Source (jump to first uncovered line)
1
//===--- CGCall.cpp - Encapsulate calling convention details --------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// These classes wrap the information about a call or function
10
// definition used to handle ABI compliancy.
11
//
12
//===----------------------------------------------------------------------===//
13
14
#include "CGCall.h"
15
#include "ABIInfo.h"
16
#include "CGBlocks.h"
17
#include "CGCXXABI.h"
18
#include "CGCleanup.h"
19
#include "CGRecordLayout.h"
20
#include "CodeGenFunction.h"
21
#include "CodeGenModule.h"
22
#include "TargetInfo.h"
23
#include "clang/AST/Attr.h"
24
#include "clang/AST/Decl.h"
25
#include "clang/AST/DeclCXX.h"
26
#include "clang/AST/DeclObjC.h"
27
#include "clang/Basic/CodeGenOptions.h"
28
#include "clang/Basic/TargetBuiltins.h"
29
#include "clang/Basic/TargetInfo.h"
30
#include "clang/CodeGen/CGFunctionInfo.h"
31
#include "clang/CodeGen/SwiftCallingConv.h"
32
#include "llvm/ADT/StringExtras.h"
33
#include "llvm/Analysis/ValueTracking.h"
34
#include "llvm/IR/Assumptions.h"
35
#include "llvm/IR/Attributes.h"
36
#include "llvm/IR/CallingConv.h"
37
#include "llvm/IR/DataLayout.h"
38
#include "llvm/IR/InlineAsm.h"
39
#include "llvm/IR/IntrinsicInst.h"
40
#include "llvm/IR/Intrinsics.h"
41
#include "llvm/Transforms/Utils/Local.h"
42
using namespace clang;
43
using namespace CodeGen;
44
45
/***/
46
47
286k
unsigned CodeGenTypes::ClangCallConvToLLVMCallConv(CallingConv CC) {
48
286k
  switch (CC) {
49
281k
  default: return llvm::CallingConv::C;
50
56
  case CC_X86StdCall: return llvm::CallingConv::X86_StdCall;
51
77
  case CC_X86FastCall: return llvm::CallingConv::X86_FastCall;
52
102
  case CC_X86RegCall: return llvm::CallingConv::X86_RegCall;
53
2.66k
  case CC_X86ThisCall: return llvm::CallingConv::X86_ThisCall;
54
17
  case CC_Win64: return llvm::CallingConv::Win64;
55
14
  case CC_X86_64SysV: return llvm::CallingConv::X86_64_SysV;
56
5
  case CC_AAPCS: return llvm::CallingConv::ARM_AAPCS;
57
11
  case CC_AAPCS_VFP: return llvm::CallingConv::ARM_AAPCS_VFP;
58
3
  case CC_IntelOclBicc: return llvm::CallingConv::Intel_OCL_BI;
59
  // TODO: Add support for __pascal to LLVM.
60
3
  case CC_X86Pascal: return llvm::CallingConv::C;
61
  // TODO: Add support for __vectorcall to LLVM.
62
110
  case CC_X86VectorCall: return llvm::CallingConv::X86_VectorCall;
63
2
  case CC_AArch64VectorCall: return llvm::CallingConv::AArch64_VectorCall;
64
359
  case CC_SpirFunction: return llvm::CallingConv::SPIR_FUNC;
65
413
  case CC_OpenCLKernel: return CGM.getTargetCodeGenInfo().getOpenCLKernelCallingConv();
66
10
  case CC_PreserveMost: return llvm::CallingConv::PreserveMost;
67
6
  case CC_PreserveAll: return llvm::CallingConv::PreserveAll;
68
1.14k
  case CC_Swift: return llvm::CallingConv::Swift;
69
57
  case CC_SwiftAsync: return llvm::CallingConv::SwiftTail;
70
286k
  }
71
286k
}
72
73
/// Derives the 'this' type for codegen purposes, i.e. ignoring method CVR
74
/// qualification. Either or both of RD and MD may be null. A null RD indicates
75
/// that there is no meaningful 'this' type, and a null MD can occur when
76
/// calling a method pointer.
77
CanQualType CodeGenTypes::DeriveThisType(const CXXRecordDecl *RD,
78
625k
                                         const CXXMethodDecl *MD) {
79
625k
  QualType RecTy;
80
625k
  if (RD)
81
624k
    RecTy = Context.getTagDeclType(RD)->getCanonicalTypeInternal();
82
688
  else
83
688
    RecTy = Context.VoidTy;
84
85
625k
  if (MD)
86
625k
    RecTy = Context.getAddrSpaceQualType(RecTy, MD->getMethodQualifiers().getAddressSpace());
87
625k
  return Context.getPointerType(CanQualType::CreateUnsafe(RecTy));
88
625k
}
89
90
/// Returns the canonical formal type of the given C++ method.
91
582k
static CanQual<FunctionProtoType> GetFormalType(const CXXMethodDecl *MD) {
92
582k
  return MD->getType()->getCanonicalTypeUnqualified()
93
582k
           .getAs<FunctionProtoType>();
94
582k
}
95
96
/// Returns the "extra-canonicalized" return type, which discards
97
/// qualifiers on the return type.  Codegen doesn't care about them,
98
/// and it makes ABI code a little easier to be able to assume that
99
/// all parameter and return types are top-level unqualified.
100
332k
static CanQualType GetReturnType(QualType RetTy) {
101
332k
  return RetTy->getCanonicalTypeUnqualified().getUnqualifiedType();
102
332k
}
103
104
/// Arrange the argument and result information for a value of the given
105
/// unprototyped freestanding function type.
106
const CGFunctionInfo &
107
633
CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionNoProtoType> FTNP) {
108
  // When translating an unprototyped function type, always use a
109
  // variadic type.
110
633
  return arrangeLLVMFunctionInfo(FTNP->getReturnType().getUnqualifiedType(),
111
633
                                 /*instanceMethod=*/false,
112
633
                                 /*chainCall=*/false, None,
113
633
                                 FTNP->getExtInfo(), {}, RequiredArgs(0));
114
633
}
115
116
static void addExtParameterInfosForCall(
117
         llvm::SmallVectorImpl<FunctionProtoType::ExtParameterInfo> &paramInfos,
118
                                        const FunctionProtoType *proto,
119
                                        unsigned prefixArgs,
120
1.19k
                                        unsigned totalArgs) {
121
1.19k
  assert(proto->hasExtParameterInfos());
122
0
  assert(paramInfos.size() <= prefixArgs);
123
0
  assert(proto->getNumParams() + prefixArgs <= totalArgs);
124
125
0
  paramInfos.reserve(totalArgs);
126
127
  // Add default infos for any prefix args that don't already have infos.
128
1.19k
  paramInfos.resize(prefixArgs);
129
130
  // Add infos for the prototype.
131
2.01k
  for (const auto &ParamInfo : proto->getExtParameterInfos()) {
132
2.01k
    paramInfos.push_back(ParamInfo);
133
    // pass_object_size params have no parameter info.
134
2.01k
    if (ParamInfo.hasPassObjectSize())
135
447
      paramInfos.emplace_back();
136
2.01k
  }
137
138
1.19k
  assert(paramInfos.size() <= totalArgs &&
139
1.19k
         "Did we forget to insert pass_object_size args?");
140
  // Add default infos for the variadic and/or suffix arguments.
141
0
  paramInfos.resize(totalArgs);
142
1.19k
}
143
144
/// Adds the formal parameters in FPT to the given prefix. If any parameter in
145
/// FPT has pass_object_size attrs, then we'll add parameters for those, too.
146
static void appendParameterTypes(const CodeGenTypes &CGT,
147
                                 SmallVectorImpl<CanQualType> &prefix,
148
              SmallVectorImpl<FunctionProtoType::ExtParameterInfo> &paramInfos,
149
1.11M
                                 CanQual<FunctionProtoType> FPT) {
150
  // Fast path: don't touch param info if we don't need to.
151
1.11M
  if (!FPT->hasExtParameterInfos()) {
152
1.11M
    assert(paramInfos.empty() &&
153
1.11M
           "We have paramInfos, but the prototype doesn't?");
154
0
    prefix.append(FPT->param_type_begin(), FPT->param_type_end());
155
1.11M
    return;
156
1.11M
  }
157
158
779
  unsigned PrefixSize = prefix.size();
159
  // In the vast majority of cases, we'll have precisely FPT->getNumParams()
160
  // parameters; the only thing that can change this is the presence of
161
  // pass_object_size. So, we preallocate for the common case.
162
779
  prefix.reserve(prefix.size() + FPT->getNumParams());
163
164
779
  auto ExtInfos = FPT->getExtParameterInfos();
165
779
  assert(ExtInfos.size() == FPT->getNumParams());
166
2.14k
  for (unsigned I = 0, E = FPT->getNumParams(); I != E; 
++I1.37k
) {
167
1.37k
    prefix.push_back(FPT->getParamType(I));
168
1.37k
    if (ExtInfos[I].hasPassObjectSize())
169
299
      prefix.push_back(CGT.getContext().getSizeType());
170
1.37k
  }
171
172
779
  addExtParameterInfosForCall(paramInfos, FPT.getTypePtr(), PrefixSize,
173
779
                              prefix.size());
174
779
}
175
176
/// Arrange the LLVM function layout for a value of the given function
177
/// type, on top of any implicit parameters already stored.
178
static const CGFunctionInfo &
179
arrangeLLVMFunctionInfo(CodeGenTypes &CGT, bool instanceMethod,
180
                        SmallVectorImpl<CanQualType> &prefix,
181
813k
                        CanQual<FunctionProtoType> FTP) {
182
813k
  SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos;
183
813k
  RequiredArgs Required = RequiredArgs::forPrototypePlus(FTP, prefix.size());
184
  // FIXME: Kill copy.
185
813k
  appendParameterTypes(CGT, prefix, paramInfos, FTP);
186
813k
  CanQualType resultType = FTP->getReturnType().getUnqualifiedType();
187
188
813k
  return CGT.arrangeLLVMFunctionInfo(resultType, instanceMethod,
189
813k
                                     /*chainCall=*/false, prefix,
190
813k
                                     FTP->getExtInfo(), paramInfos,
191
813k
                                     Required);
192
813k
}
193
194
/// Arrange the argument and result information for a value of the
195
/// given freestanding function type.
196
const CGFunctionInfo &
197
591k
CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionProtoType> FTP) {
198
591k
  SmallVector<CanQualType, 16> argTypes;
199
591k
  return ::arrangeLLVMFunctionInfo(*this, /*instanceMethod=*/false, argTypes,
200
591k
                                   FTP);
201
591k
}
202
203
static CallingConv getCallingConventionForDecl(const ObjCMethodDecl *D,
204
28.8k
                                               bool IsWindows) {
205
  // Set the appropriate calling convention for the Function.
206
28.8k
  if (D->hasAttr<StdCallAttr>())
207
2
    return CC_X86StdCall;
208
209
28.8k
  if (D->hasAttr<FastCallAttr>())
210
0
    return CC_X86FastCall;
211
212
28.8k
  if (D->hasAttr<RegCallAttr>())
213
0
    return CC_X86RegCall;
214
215
28.8k
  if (D->hasAttr<ThisCallAttr>())
216
0
    return CC_X86ThisCall;
217
218
28.8k
  if (D->hasAttr<VectorCallAttr>())
219
0
    return CC_X86VectorCall;
220
221
28.8k
  if (D->hasAttr<PascalAttr>())
222
0
    return CC_X86Pascal;
223
224
28.8k
  if (PcsAttr *PCS = D->getAttr<PcsAttr>())
225
0
    return (PCS->getPCS() == PcsAttr::AAPCS ? CC_AAPCS : CC_AAPCS_VFP);
226
227
28.8k
  if (D->hasAttr<AArch64VectorPcsAttr>())
228
0
    return CC_AArch64VectorCall;
229
230
28.8k
  if (D->hasAttr<IntelOclBiccAttr>())
231
0
    return CC_IntelOclBicc;
232
233
28.8k
  if (D->hasAttr<MSABIAttr>())
234
2
    return IsWindows ? 
CC_C0
: CC_Win64;
235
236
28.8k
  if (D->hasAttr<SysVABIAttr>())
237
0
    return IsWindows ? CC_X86_64SysV : CC_C;
238
239
28.8k
  if (D->hasAttr<PreserveMostAttr>())
240
0
    return CC_PreserveMost;
241
242
28.8k
  if (D->hasAttr<PreserveAllAttr>())
243
0
    return CC_PreserveAll;
244
245
28.8k
  return CC_C;
246
28.8k
}
247
248
/// Arrange the argument and result information for a call to an
249
/// unknown C++ non-static member function of the given abstract type.
250
/// (A null RD means we don't have any meaningful "this" argument type,
251
///  so fall back to a generic pointer type).
252
/// The member function must be an ordinary function, i.e. not a
253
/// constructor or destructor.
254
const CGFunctionInfo &
255
CodeGenTypes::arrangeCXXMethodType(const CXXRecordDecl *RD,
256
                                   const FunctionProtoType *FTP,
257
222k
                                   const CXXMethodDecl *MD) {
258
222k
  SmallVector<CanQualType, 16> argTypes;
259
260
  // Add the 'this' pointer.
261
222k
  argTypes.push_back(DeriveThisType(RD, MD));
262
263
222k
  return ::arrangeLLVMFunctionInfo(
264
222k
      *this, true, argTypes,
265
222k
      FTP->getCanonicalTypeUnqualified().getAs<FunctionProtoType>());
266
222k
}
267
268
/// Set calling convention for CUDA/HIP kernel.
269
static void setCUDAKernelCallingConvention(CanQualType &FTy, CodeGenModule &CGM,
270
769k
                                           const FunctionDecl *FD) {
271
769k
  if (FD->hasAttr<CUDAGlobalAttr>()) {
272
394
    const FunctionType *FT = FTy->getAs<FunctionType>();
273
394
    CGM.getTargetCodeGenInfo().setCUDAKernelCallingConvention(FT);
274
394
    FTy = FT->getCanonicalTypeUnqualified();
275
394
  }
276
769k
}
277
278
/// Arrange the argument and result information for a declaration or
279
/// definition of the given C++ non-static member function.  The
280
/// member function must be an ordinary function, i.e. not a
281
/// constructor or destructor.
282
const CGFunctionInfo &
283
228k
CodeGenTypes::arrangeCXXMethodDeclaration(const CXXMethodDecl *MD) {
284
228k
  assert(!isa<CXXConstructorDecl>(MD) && "wrong method for constructors!");
285
0
  assert(!isa<CXXDestructorDecl>(MD) && "wrong method for destructors!");
286
287
0
  CanQualType FT = GetFormalType(MD).getAs<Type>();
288
228k
  setCUDAKernelCallingConvention(FT, CGM, MD);
289
228k
  auto prototype = FT.getAs<FunctionProtoType>();
290
291
228k
  if (MD->isInstance()) {
292
    // The abstract case is perfectly fine.
293
222k
    const CXXRecordDecl *ThisType = TheCXXABI.getThisArgumentTypeForMethod(MD);
294
222k
    return arrangeCXXMethodType(ThisType, prototype.getTypePtr(), MD);
295
222k
  }
296
297
6.06k
  return arrangeFreeFunctionType(prototype);
298
228k
}
299
300
bool CodeGenTypes::inheritingCtorHasParams(
301
1.18k
    const InheritedConstructor &Inherited, CXXCtorType Type) {
302
  // Parameters are unnecessary if we're constructing a base class subobject
303
  // and the inherited constructor lives in a virtual base.
304
1.18k
  return Type == Ctor_Complete ||
305
1.18k
         
!Inherited.getShadowDecl()->constructsVirtualBase()798
||
306
1.18k
         
!Target.getCXXABI().hasConstructorVariants()74
;
307
1.18k
}
308
309
const CGFunctionInfo &
310
299k
CodeGenTypes::arrangeCXXStructorDeclaration(GlobalDecl GD) {
311
299k
  auto *MD = cast<CXXMethodDecl>(GD.getDecl());
312
313
299k
  SmallVector<CanQualType, 16> argTypes;
314
299k
  SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos;
315
299k
  argTypes.push_back(DeriveThisType(MD->getParent(), MD));
316
317
299k
  bool PassParams = true;
318
319
299k
  if (auto *CD = dyn_cast<CXXConstructorDecl>(MD)) {
320
    // A base class inheriting constructor doesn't get forwarded arguments
321
    // needed to construct a virtual base (or base class thereof).
322
177k
    if (auto Inherited = CD->getInheritedConstructor())
323
722
      PassParams = inheritingCtorHasParams(Inherited, GD.getCtorType());
324
177k
  }
325
326
299k
  CanQual<FunctionProtoType> FTP = GetFormalType(MD);
327
328
  // Add the formal parameters.
329
299k
  if (PassParams)
330
299k
    appendParameterTypes(*this, argTypes, paramInfos, FTP);
331
332
299k
  CGCXXABI::AddedStructorArgCounts AddedArgs =
333
299k
      TheCXXABI.buildStructorSignature(GD, argTypes);
334
299k
  if (!paramInfos.empty()) {
335
    // Note: prefix implies after the first param.
336
175
    if (AddedArgs.Prefix)
337
2
      paramInfos.insert(paramInfos.begin() + 1, AddedArgs.Prefix,
338
2
                        FunctionProtoType::ExtParameterInfo{});
339
175
    if (AddedArgs.Suffix)
340
20
      paramInfos.append(AddedArgs.Suffix,
341
20
                        FunctionProtoType::ExtParameterInfo{});
342
175
  }
343
344
299k
  RequiredArgs required =
345
299k
      (PassParams && 
MD->isVariadic()299k
?
RequiredArgs(argTypes.size())91
346
299k
                                      : 
RequiredArgs::All299k
);
347
348
299k
  FunctionType::ExtInfo extInfo = FTP->getExtInfo();
349
299k
  CanQualType resultType = TheCXXABI.HasThisReturn(GD)
350
299k
                               ? 
argTypes.front()7.93k
351
299k
                               : 
TheCXXABI.hasMostDerivedReturn(GD)291k
352
291k
                                     ? 
CGM.getContext().VoidPtrTy1.42k
353
291k
                                     : 
Context.VoidTy290k
;
354
299k
  return arrangeLLVMFunctionInfo(resultType, /*instanceMethod=*/true,
355
299k
                                 /*chainCall=*/false, argTypes, extInfo,
356
299k
                                 paramInfos, required);
357
299k
}
358
359
static SmallVector<CanQualType, 16>
360
75.7k
getArgTypesForCall(ASTContext &ctx, const CallArgList &args) {
361
75.7k
  SmallVector<CanQualType, 16> argTypes;
362
75.7k
  for (auto &arg : args)
363
107k
    argTypes.push_back(ctx.getCanonicalParamType(arg.Ty));
364
75.7k
  return argTypes;
365
75.7k
}
366
367
static SmallVector<CanQualType, 16>
368
29.7k
getArgTypesForDeclaration(ASTContext &ctx, const FunctionArgList &args) {
369
29.7k
  SmallVector<CanQualType, 16> argTypes;
370
29.7k
  for (auto &arg : args)
371
80.6k
    argTypes.push_back(ctx.getCanonicalParamType(arg->getType()));
372
29.7k
  return argTypes;
373
29.7k
}
374
375
static llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16>
376
getExtParameterInfosForCall(const FunctionProtoType *proto,
377
75.1k
                            unsigned prefixArgs, unsigned totalArgs) {
378
75.1k
  llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16> result;
379
75.1k
  if (proto->hasExtParameterInfos()) {
380
41
    addExtParameterInfosForCall(result, proto, prefixArgs, totalArgs);
381
41
  }
382
75.1k
  return result;
383
75.1k
}
384
385
/// Arrange a call to a C++ method, passing the given arguments.
386
///
387
/// ExtraPrefixArgs is the number of ABI-specific args passed after the `this`
388
/// parameter.
389
/// ExtraSuffixArgs is the number of ABI-specific args passed at the end of
390
/// args.
391
/// PassProtoArgs indicates whether `args` has args for the parameters in the
392
/// given CXXConstructorDecl.
393
const CGFunctionInfo &
394
CodeGenTypes::arrangeCXXConstructorCall(const CallArgList &args,
395
                                        const CXXConstructorDecl *D,
396
                                        CXXCtorType CtorKind,
397
                                        unsigned ExtraPrefixArgs,
398
                                        unsigned ExtraSuffixArgs,
399
54.6k
                                        bool PassProtoArgs) {
400
  // FIXME: Kill copy.
401
54.6k
  SmallVector<CanQualType, 16> ArgTypes;
402
54.6k
  for (const auto &Arg : args)
403
107k
    ArgTypes.push_back(Context.getCanonicalParamType(Arg.Ty));
404
405
  // +1 for implicit this, which should always be args[0].
406
54.6k
  unsigned TotalPrefixArgs = 1 + ExtraPrefixArgs;
407
408
54.6k
  CanQual<FunctionProtoType> FPT = GetFormalType(D);
409
54.6k
  RequiredArgs Required = PassProtoArgs
410
54.6k
                              ? RequiredArgs::forPrototypePlus(
411
54.6k
                                    FPT, TotalPrefixArgs + ExtraSuffixArgs)
412
54.6k
                              : 
RequiredArgs::All10
;
413
414
54.6k
  GlobalDecl GD(D, CtorKind);
415
54.6k
  CanQualType ResultType = TheCXXABI.HasThisReturn(GD)
416
54.6k
                               ? 
ArgTypes.front()2.33k
417
54.6k
                               : 
TheCXXABI.hasMostDerivedReturn(GD)52.2k
418
52.2k
                                     ? 
CGM.getContext().VoidPtrTy0
419
52.2k
                                     : Context.VoidTy;
420
421
54.6k
  FunctionType::ExtInfo Info = FPT->getExtInfo();
422
54.6k
  llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16> ParamInfos;
423
  // If the prototype args are elided, we should only have ABI-specific args,
424
  // which never have param info.
425
54.6k
  if (PassProtoArgs && 
FPT->hasExtParameterInfos()54.6k
) {
426
    // ABI-specific suffix arguments are treated the same as variadic arguments.
427
70
    addExtParameterInfosForCall(ParamInfos, FPT.getTypePtr(), TotalPrefixArgs,
428
70
                                ArgTypes.size());
429
70
  }
430
54.6k
  return arrangeLLVMFunctionInfo(ResultType, /*instanceMethod=*/true,
431
54.6k
                                 /*chainCall=*/false, ArgTypes, Info,
432
54.6k
                                 ParamInfos, Required);
433
54.6k
}
434
435
/// Arrange the argument and result information for the declaration or
436
/// definition of the given function.
437
const CGFunctionInfo &
438
643k
CodeGenTypes::arrangeFunctionDeclaration(const FunctionDecl *FD) {
439
643k
  if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(FD))
440
114k
    if (MD->isInstance())
441
101k
      return arrangeCXXMethodDeclaration(MD);
442
443
541k
  CanQualType FTy = FD->getType()->getCanonicalTypeUnqualified();
444
445
541k
  assert(isa<FunctionType>(FTy));
446
0
  setCUDAKernelCallingConvention(FTy, CGM, FD);
447
448
  // When declaring a function without a prototype, always use a
449
  // non-variadic type.
450
541k
  if (CanQual<FunctionNoProtoType> noProto = FTy.getAs<FunctionNoProtoType>()) {
451
16.9k
    return arrangeLLVMFunctionInfo(
452
16.9k
        noProto->getReturnType(), /*instanceMethod=*/false,
453
16.9k
        /*chainCall=*/false, None, noProto->getExtInfo(), {},RequiredArgs::All);
454
16.9k
  }
455
456
524k
  return arrangeFreeFunctionType(FTy.castAs<FunctionProtoType>());
457
541k
}
458
459
/// Arrange the argument and result information for the declaration or
460
/// definition of an Objective-C method.
461
const CGFunctionInfo &
462
5.21k
CodeGenTypes::arrangeObjCMethodDeclaration(const ObjCMethodDecl *MD) {
463
  // It happens that this is the same as a call with no optional
464
  // arguments, except also using the formal 'self' type.
465
5.21k
  return arrangeObjCMessageSendSignature(MD, MD->getSelfDecl()->getType());
466
5.21k
}
467
468
/// Arrange the argument and result information for the function type
469
/// through which to perform a send to the given Objective-C method,
470
/// using the given receiver type.  The receiver type is not always
471
/// the 'self' type of the method or even an Objective-C pointer type.
472
/// This is *not* the right method for actually performing such a
473
/// message send, due to the possibility of optional arguments.
474
const CGFunctionInfo &
475
CodeGenTypes::arrangeObjCMessageSendSignature(const ObjCMethodDecl *MD,
476
16.7k
                                              QualType receiverType) {
477
16.7k
  SmallVector<CanQualType, 16> argTys;
478
16.7k
  SmallVector<FunctionProtoType::ExtParameterInfo, 4> extParamInfos(2);
479
16.7k
  argTys.push_back(Context.getCanonicalParamType(receiverType));
480
16.7k
  argTys.push_back(Context.getCanonicalParamType(Context.getObjCSelType()));
481
  // FIXME: Kill copy?
482
16.7k
  for (const auto *I : MD->parameters()) {
483
14.1k
    argTys.push_back(Context.getCanonicalParamType(I->getType()));
484
14.1k
    auto extParamInfo = FunctionProtoType::ExtParameterInfo().withIsNoEscape(
485
14.1k
        I->hasAttr<NoEscapeAttr>());
486
14.1k
    extParamInfos.push_back(extParamInfo);
487
14.1k
  }
488
489
16.7k
  FunctionType::ExtInfo einfo;
490
16.7k
  bool IsWindows = getContext().getTargetInfo().getTriple().isOSWindows();
491
16.7k
  einfo = einfo.withCallingConv(getCallingConventionForDecl(MD, IsWindows));
492
493
16.7k
  if (getContext().getLangOpts().ObjCAutoRefCount &&
494
16.7k
      
MD->hasAttr<NSReturnsRetainedAttr>()849
)
495
114
    einfo = einfo.withProducesResult(true);
496
497
16.7k
  RequiredArgs required =
498
16.7k
    (MD->isVariadic() ? 
RequiredArgs(argTys.size())929
:
RequiredArgs::All15.8k
);
499
500
16.7k
  return arrangeLLVMFunctionInfo(
501
16.7k
      GetReturnType(MD->getReturnType()), /*instanceMethod=*/false,
502
16.7k
      /*chainCall=*/false, argTys, einfo, extParamInfos, required);
503
16.7k
}
504
505
const CGFunctionInfo &
506
CodeGenTypes::arrangeUnprototypedObjCMessageSend(QualType returnType,
507
812
                                                 const CallArgList &args) {
508
812
  auto argTypes = getArgTypesForCall(Context, args);
509
812
  FunctionType::ExtInfo einfo;
510
511
812
  return arrangeLLVMFunctionInfo(
512
812
      GetReturnType(returnType), /*instanceMethod=*/false,
513
812
      /*chainCall=*/false, argTypes, einfo, {}, RequiredArgs::All);
514
812
}
515
516
const CGFunctionInfo &
517
708k
CodeGenTypes::arrangeGlobalDeclaration(GlobalDecl GD) {
518
  // FIXME: Do we need to handle ObjCMethodDecl?
519
708k
  const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
520
521
708k
  if (isa<CXXConstructorDecl>(GD.getDecl()) ||
522
708k
      
isa<CXXDestructorDecl>(GD.getDecl())665k
)
523
65.1k
    return arrangeCXXStructorDeclaration(GD);
524
525
643k
  return arrangeFunctionDeclaration(FD);
526
708k
}
527
528
/// Arrange a thunk that takes 'this' as the first parameter followed by
529
/// varargs.  Return a void pointer, regardless of the actual return type.
530
/// The body of the thunk will end in a musttail call to a function of the
531
/// correct type, and the caller will bitcast the function to the correct
532
/// prototype.
533
const CGFunctionInfo &
534
55
CodeGenTypes::arrangeUnprototypedMustTailThunk(const CXXMethodDecl *MD) {
535
55
  assert(MD->isVirtual() && "only methods have thunks");
536
0
  CanQual<FunctionProtoType> FTP = GetFormalType(MD);
537
55
  CanQualType ArgTys[] = {DeriveThisType(MD->getParent(), MD)};
538
55
  return arrangeLLVMFunctionInfo(Context.VoidTy, /*instanceMethod=*/false,
539
55
                                 /*chainCall=*/false, ArgTys,
540
55
                                 FTP->getExtInfo(), {}, RequiredArgs(1));
541
55
}
542
543
const CGFunctionInfo &
544
CodeGenTypes::arrangeMSCtorClosure(const CXXConstructorDecl *CD,
545
35
                                   CXXCtorType CT) {
546
35
  assert(CT == Ctor_CopyingClosure || CT == Ctor_DefaultClosure);
547
548
0
  CanQual<FunctionProtoType> FTP = GetFormalType(CD);
549
35
  SmallVector<CanQualType, 2> ArgTys;
550
35
  const CXXRecordDecl *RD = CD->getParent();
551
35
  ArgTys.push_back(DeriveThisType(RD, CD));
552
35
  if (CT == Ctor_CopyingClosure)
553
6
    ArgTys.push_back(*FTP->param_type_begin());
554
35
  if (RD->getNumVBases() > 0)
555
0
    ArgTys.push_back(Context.IntTy);
556
35
  CallingConv CC = Context.getDefaultCallingConvention(
557
35
      /*IsVariadic=*/false, /*IsCXXMethod=*/true);
558
35
  return arrangeLLVMFunctionInfo(Context.VoidTy, /*instanceMethod=*/true,
559
35
                                 /*chainCall=*/false, ArgTys,
560
35
                                 FunctionType::ExtInfo(CC), {},
561
35
                                 RequiredArgs::All);
562
35
}
563
564
/// Arrange a call as unto a free function, except possibly with an
565
/// additional number of formal parameters considered required.
566
static const CGFunctionInfo &
567
arrangeFreeFunctionLikeCall(CodeGenTypes &CGT,
568
                            CodeGenModule &CGM,
569
                            const CallArgList &args,
570
                            const FunctionType *fnType,
571
                            unsigned numExtraRequiredArgs,
572
209k
                            bool chainCall) {
573
209k
  assert(args.size() >= numExtraRequiredArgs);
574
575
0
  llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos;
576
577
  // In most cases, there are no optional arguments.
578
209k
  RequiredArgs required = RequiredArgs::All;
579
580
  // If we have a variadic prototype, the required arguments are the
581
  // extra prefix plus the arguments in the prototype.
582
209k
  if (const FunctionProtoType *proto = dyn_cast<FunctionProtoType>(fnType)) {
583
208k
    if (proto->isVariadic())
584
74.6k
      required = RequiredArgs::forPrototypePlus(proto, numExtraRequiredArgs);
585
586
208k
    if (proto->hasExtParameterInfos())
587
305
      addExtParameterInfosForCall(paramInfos, proto, numExtraRequiredArgs,
588
305
                                  args.size());
589
590
  // If we don't have a prototype at all, but we're supposed to
591
  // explicitly use the variadic convention for unprototyped calls,
592
  // treat all of the arguments as required but preserve the nominal
593
  // possibility of variadics.
594
208k
  } else 
if (1.01k
CGM.getTargetCodeGenInfo()
595
1.01k
                .isNoProtoCallVariadic(args,
596
1.01k
                                       cast<FunctionNoProtoType>(fnType))) {
597
730
    required = RequiredArgs(args.size());
598
730
  }
599
600
  // FIXME: Kill copy.
601
209k
  SmallVector<CanQualType, 16> argTypes;
602
209k
  for (const auto &arg : args)
603
371k
    argTypes.push_back(CGT.getContext().getCanonicalParamType(arg.Ty));
604
209k
  return CGT.arrangeLLVMFunctionInfo(GetReturnType(fnType->getReturnType()),
605
209k
                                     /*instanceMethod=*/false, chainCall,
606
209k
                                     argTypes, fnType->getExtInfo(), paramInfos,
607
209k
                                     required);
608
209k
}
609
610
/// Figure out the rules for calling a function with the given formal
611
/// type using the given arguments.  The arguments are necessary
612
/// because the function might be unprototyped, in which case it's
613
/// target-dependent in crazy ways.
614
const CGFunctionInfo &
615
CodeGenTypes::arrangeFreeFunctionCall(const CallArgList &args,
616
                                      const FunctionType *fnType,
617
209k
                                      bool chainCall) {
618
209k
  return arrangeFreeFunctionLikeCall(*this, CGM, args, fnType,
619
209k
                                     chainCall ? 
18
:
0209k
, chainCall);
620
209k
}
621
622
/// A block function is essentially a free function with an
623
/// extra implicit argument.
624
const CGFunctionInfo &
625
CodeGenTypes::arrangeBlockFunctionCall(const CallArgList &args,
626
583
                                       const FunctionType *fnType) {
627
583
  return arrangeFreeFunctionLikeCall(*this, CGM, args, fnType, 1,
628
583
                                     /*chainCall=*/false);
629
583
}
630
631
const CGFunctionInfo &
632
CodeGenTypes::arrangeBlockFunctionDeclaration(const FunctionProtoType *proto,
633
1.07k
                                              const FunctionArgList &params) {
634
1.07k
  auto paramInfos = getExtParameterInfosForCall(proto, 1, params.size());
635
1.07k
  auto argTypes = getArgTypesForDeclaration(Context, params);
636
637
1.07k
  return arrangeLLVMFunctionInfo(GetReturnType(proto->getReturnType()),
638
1.07k
                                 /*instanceMethod*/ false, /*chainCall*/ false,
639
1.07k
                                 argTypes, proto->getExtInfo(), paramInfos,
640
1.07k
                                 RequiredArgs::forPrototypePlus(proto, 1));
641
1.07k
}
642
643
const CGFunctionInfo &
644
CodeGenTypes::arrangeBuiltinFunctionCall(QualType resultType,
645
990
                                         const CallArgList &args) {
646
  // FIXME: Kill copy.
647
990
  SmallVector<CanQualType, 16> argTypes;
648
990
  for (const auto &Arg : args)
649
3.61k
    argTypes.push_back(Context.getCanonicalParamType(Arg.Ty));
650
990
  return arrangeLLVMFunctionInfo(
651
990
      GetReturnType(resultType), /*instanceMethod=*/false,
652
990
      /*chainCall=*/false, argTypes, FunctionType::ExtInfo(),
653
990
      /*paramInfos=*/ {}, RequiredArgs::All);
654
990
}
655
656
const CGFunctionInfo &
657
CodeGenTypes::arrangeBuiltinFunctionDeclaration(QualType resultType,
658
28.6k
                                                const FunctionArgList &args) {
659
28.6k
  auto argTypes = getArgTypesForDeclaration(Context, args);
660
661
28.6k
  return arrangeLLVMFunctionInfo(
662
28.6k
      GetReturnType(resultType), /*instanceMethod=*/false, /*chainCall=*/false,
663
28.6k
      argTypes, FunctionType::ExtInfo(), {}, RequiredArgs::All);
664
28.6k
}
665
666
const CGFunctionInfo &
667
CodeGenTypes::arrangeBuiltinFunctionDeclaration(CanQualType resultType,
668
376
                                              ArrayRef<CanQualType> argTypes) {
669
376
  return arrangeLLVMFunctionInfo(
670
376
      resultType, /*instanceMethod=*/false, /*chainCall=*/false,
671
376
      argTypes, FunctionType::ExtInfo(), {}, RequiredArgs::All);
672
376
}
673
674
/// Arrange a call to a C++ method, passing the given arguments.
675
///
676
/// numPrefixArgs is the number of ABI-specific prefix arguments we have. It
677
/// does not count `this`.
678
const CGFunctionInfo &
679
CodeGenTypes::arrangeCXXMethodCall(const CallArgList &args,
680
                                   const FunctionProtoType *proto,
681
                                   RequiredArgs required,
682
74.0k
                                   unsigned numPrefixArgs) {
683
74.0k
  assert(numPrefixArgs + 1 <= args.size() &&
684
74.0k
         "Emitting a call with less args than the required prefix?");
685
  // Add one to account for `this`. It's a bit awkward here, but we don't count
686
  // `this` in similar places elsewhere.
687
0
  auto paramInfos =
688
74.0k
    getExtParameterInfosForCall(proto, numPrefixArgs + 1, args.size());
689
690
  // FIXME: Kill copy.
691
74.0k
  auto argTypes = getArgTypesForCall(Context, args);
692
693
74.0k
  FunctionType::ExtInfo info = proto->getExtInfo();
694
74.0k
  return arrangeLLVMFunctionInfo(
695
74.0k
      GetReturnType(proto->getReturnType()), /*instanceMethod=*/true,
696
74.0k
      /*chainCall=*/false, argTypes, info, paramInfos, required);
697
74.0k
}
698
699
17.5k
const CGFunctionInfo &CodeGenTypes::arrangeNullaryFunction() {
700
17.5k
  return arrangeLLVMFunctionInfo(
701
17.5k
      getContext().VoidTy, /*instanceMethod=*/false, /*chainCall=*/false,
702
17.5k
      None, FunctionType::ExtInfo(), {}, RequiredArgs::All);
703
17.5k
}
704
705
const CGFunctionInfo &
706
CodeGenTypes::arrangeCall(const CGFunctionInfo &signature,
707
11.5k
                          const CallArgList &args) {
708
11.5k
  assert(signature.arg_size() <= args.size());
709
11.5k
  if (signature.arg_size() == args.size())
710
10.6k
    return signature;
711
712
911
  SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos;
713
911
  auto sigParamInfos = signature.getExtParameterInfos();
714
911
  if (!sigParamInfos.empty()) {
715
911
    paramInfos.append(sigParamInfos.begin(), sigParamInfos.end());
716
911
    paramInfos.resize(args.size());
717
911
  }
718
719
911
  auto argTypes = getArgTypesForCall(Context, args);
720
721
911
  assert(signature.getRequiredArgs().allowsOptionalArgs());
722
0
  return arrangeLLVMFunctionInfo(signature.getReturnType(),
723
911
                                 signature.isInstanceMethod(),
724
911
                                 signature.isChainCall(),
725
911
                                 argTypes,
726
911
                                 signature.getExtInfo(),
727
911
                                 paramInfos,
728
911
                                 signature.getRequiredArgs());
729
11.5k
}
730
731
namespace clang {
732
namespace CodeGen {
733
void computeSPIRKernelABIInfo(CodeGenModule &CGM, CGFunctionInfo &FI);
734
}
735
}
736
737
/// Arrange the argument and result information for an abstract value
738
/// of a given function type.  This is the method which all of the
739
/// above functions ultimately defer to.
740
const CGFunctionInfo &
741
CodeGenTypes::arrangeLLVMFunctionInfo(CanQualType resultType,
742
                                      bool instanceMethod,
743
                                      bool chainCall,
744
                                      ArrayRef<CanQualType> argTypes,
745
                                      FunctionType::ExtInfo info,
746
                     ArrayRef<FunctionProtoType::ExtParameterInfo> paramInfos,
747
1.53M
                                      RequiredArgs required) {
748
1.53M
  assert(llvm::all_of(argTypes,
749
1.53M
                      [](CanQualType T) { return T.isCanonicalAsParam(); }));
750
751
  // Lookup or create unique function info.
752
0
  llvm::FoldingSetNodeID ID;
753
1.53M
  CGFunctionInfo::Profile(ID, instanceMethod, chainCall, info, paramInfos,
754
1.53M
                          required, resultType, argTypes);
755
756
1.53M
  void *insertPos = nullptr;
757
1.53M
  CGFunctionInfo *FI = FunctionInfos.FindNodeOrInsertPos(ID, insertPos);
758
1.53M
  if (FI)
759
1.24M
    return *FI;
760
761
286k
  unsigned CC = ClangCallConvToLLVMCallConv(info.getCC());
762
763
  // Construct the function info.  We co-allocate the ArgInfos.
764
286k
  FI = CGFunctionInfo::create(CC, instanceMethod, chainCall, info,
765
286k
                              paramInfos, resultType, argTypes, required);
766
286k
  FunctionInfos.InsertNode(FI, insertPos);
767
768
286k
  bool inserted = FunctionsBeingProcessed.insert(FI).second;
769
286k
  (void)inserted;
770
286k
  assert(inserted && "Recursively being processed?");
771
772
  // Compute ABI information.
773
286k
  if (CC == llvm::CallingConv::SPIR_KERNEL) {
774
    // Force target independent argument handling for the host visible
775
    // kernel functions.
776
146
    computeSPIRKernelABIInfo(CGM, *FI);
777
286k
  } else if (info.getCC() == CC_Swift || 
info.getCC() == CC_SwiftAsync285k
) {
778
1.19k
    swiftcall::computeABIInfo(CGM, *FI);
779
285k
  } else {
780
285k
    getABIInfo().computeInfo(*FI);
781
285k
  }
782
783
  // Loop over all of the computed argument and return value info.  If any of
784
  // them are direct or extend without a specified coerce type, specify the
785
  // default now.
786
286k
  ABIArgInfo &retInfo = FI->getReturnInfo();
787
286k
  if (retInfo.canHaveCoerceToType() && 
retInfo.getCoerceToType() == nullptr165k
)
788
71.5k
    retInfo.setCoerceToType(ConvertType(FI->getReturnType()));
789
790
286k
  for (auto &I : FI->arguments())
791
711k
    if (I.info.canHaveCoerceToType() && 
I.info.getCoerceToType() == nullptr704k
)
792
418k
      I.info.setCoerceToType(ConvertType(I.type));
793
794
286k
  bool erased = FunctionsBeingProcessed.erase(FI); (void)erased;
795
286k
  assert(erased && "Not in set?");
796
797
0
  return *FI;
798
1.53M
}
799
800
CGFunctionInfo *CGFunctionInfo::create(unsigned llvmCC,
801
                                       bool instanceMethod,
802
                                       bool chainCall,
803
                                       const FunctionType::ExtInfo &info,
804
                                       ArrayRef<ExtParameterInfo> paramInfos,
805
                                       CanQualType resultType,
806
                                       ArrayRef<CanQualType> argTypes,
807
286k
                                       RequiredArgs required) {
808
286k
  assert(paramInfos.empty() || paramInfos.size() == argTypes.size());
809
0
  assert(!required.allowsOptionalArgs() ||
810
286k
         required.getNumRequiredArgs() <= argTypes.size());
811
812
0
  void *buffer =
813
286k
    operator new(totalSizeToAlloc<ArgInfo,             ExtParameterInfo>(
814
286k
                                  argTypes.size() + 1, paramInfos.size()));
815
816
286k
  CGFunctionInfo *FI = new(buffer) CGFunctionInfo();
817
286k
  FI->CallingConvention = llvmCC;
818
286k
  FI->EffectiveCallingConvention = llvmCC;
819
286k
  FI->ASTCallingConvention = info.getCC();
820
286k
  FI->InstanceMethod = instanceMethod;
821
286k
  FI->ChainCall = chainCall;
822
286k
  FI->CmseNSCall = info.getCmseNSCall();
823
286k
  FI->NoReturn = info.getNoReturn();
824
286k
  FI->ReturnsRetained = info.getProducesResult();
825
286k
  FI->NoCallerSavedRegs = info.getNoCallerSavedRegs();
826
286k
  FI->NoCfCheck = info.getNoCfCheck();
827
286k
  FI->Required = required;
828
286k
  FI->HasRegParm = info.getHasRegParm();
829
286k
  FI->RegParm = info.getRegParm();
830
286k
  FI->ArgStruct = nullptr;
831
286k
  FI->ArgStructAlign = 0;
832
286k
  FI->NumArgs = argTypes.size();
833
286k
  FI->HasExtParameterInfos = !paramInfos.empty();
834
286k
  FI->getArgsBuffer()[0].type = resultType;
835
998k
  for (unsigned i = 0, e = argTypes.size(); i != e; 
++i711k
)
836
711k
    FI->getArgsBuffer()[i + 1].type = argTypes[i];
837
307k
  for (unsigned i = 0, e = paramInfos.size(); i != e; 
++i20.6k
)
838
20.6k
    FI->getExtParameterInfosBuffer()[i] = paramInfos[i];
839
286k
  return FI;
840
286k
}
841
842
/***/
843
844
namespace {
845
// ABIArgInfo::Expand implementation.
846
847
// Specifies the way QualType passed as ABIArgInfo::Expand is expanded.
848
struct TypeExpansion {
849
  enum TypeExpansionKind {
850
    // Elements of constant arrays are expanded recursively.
851
    TEK_ConstantArray,
852
    // Record fields are expanded recursively (but if record is a union, only
853
    // the field with the largest size is expanded).
854
    TEK_Record,
855
    // For complex types, real and imaginary parts are expanded recursively.
856
    TEK_Complex,
857
    // All other types are not expandable.
858
    TEK_None
859
  };
860
861
  const TypeExpansionKind Kind;
862
863
2.31k
  TypeExpansion(TypeExpansionKind K) : Kind(K) {}
864
2.31k
  virtual ~TypeExpansion() {}
865
};
866
867
struct ConstantArrayExpansion : TypeExpansion {
868
  QualType EltTy;
869
  uint64_t NumElts;
870
871
  ConstantArrayExpansion(QualType EltTy, uint64_t NumElts)
872
40
      : TypeExpansion(TEK_ConstantArray), EltTy(EltTy), NumElts(NumElts) {}
873
2.35k
  static bool classof(const TypeExpansion *TE) {
874
2.35k
    return TE->Kind == TEK_ConstantArray;
875
2.35k
  }
876
};
877
878
struct RecordExpansion : TypeExpansion {
879
  SmallVector<const CXXBaseSpecifier *, 1> Bases;
880
881
  SmallVector<const FieldDecl *, 1> Fields;
882
883
  RecordExpansion(SmallVector<const CXXBaseSpecifier *, 1> &&Bases,
884
                  SmallVector<const FieldDecl *, 1> &&Fields)
885
      : TypeExpansion(TEK_Record), Bases(std::move(Bases)),
886
735
        Fields(std::move(Fields)) {}
887
3.00k
  static bool classof(const TypeExpansion *TE) {
888
3.00k
    return TE->Kind == TEK_Record;
889
3.00k
  }
890
};
891
892
struct ComplexExpansion : TypeExpansion {
893
  QualType EltTy;
894
895
46
  ComplexExpansion(QualType EltTy) : TypeExpansion(TEK_Complex), EltTy(EltTy) {}
896
1.54k
  static bool classof(const TypeExpansion *TE) {
897
1.54k
    return TE->Kind == TEK_Complex;
898
1.54k
  }
899
};
900
901
struct NoExpansion : TypeExpansion {
902
1.49k
  NoExpansion() : TypeExpansion(TEK_None) {}
903
1.49k
  static bool classof(const TypeExpansion *TE) {
904
1.49k
    return TE->Kind == TEK_None;
905
1.49k
  }
906
};
907
}  // namespace
908
909
static std::unique_ptr<TypeExpansion>
910
2.31k
getTypeExpansion(QualType Ty, const ASTContext &Context) {
911
2.31k
  if (const ConstantArrayType *AT = Context.getAsConstantArrayType(Ty)) {
912
40
    return std::make_unique<ConstantArrayExpansion>(
913
40
        AT->getElementType(), AT->getSize().getZExtValue());
914
40
  }
915
2.27k
  if (const RecordType *RT = Ty->getAs<RecordType>()) {
916
735
    SmallVector<const CXXBaseSpecifier *, 1> Bases;
917
735
    SmallVector<const FieldDecl *, 1> Fields;
918
735
    const RecordDecl *RD = RT->getDecl();
919
735
    assert(!RD->hasFlexibleArrayMember() &&
920
735
           "Cannot expand structure with flexible array.");
921
735
    if (RD->isUnion()) {
922
      // Unions can be here only in degenerative cases - all the fields are same
923
      // after flattening. Thus we have to use the "largest" field.
924
5
      const FieldDecl *LargestFD = nullptr;
925
5
      CharUnits UnionSize = CharUnits::Zero();
926
927
10
      for (const auto *FD : RD->fields()) {
928
10
        if (FD->isZeroLengthBitField(Context))
929
0
          continue;
930
10
        assert(!FD->isBitField() &&
931
10
               "Cannot expand structure with bit-field members.");
932
0
        CharUnits FieldSize = Context.getTypeSizeInChars(FD->getType());
933
10
        if (UnionSize < FieldSize) {
934
5
          UnionSize = FieldSize;
935
5
          LargestFD = FD;
936
5
        }
937
10
      }
938
5
      if (LargestFD)
939
5
        Fields.push_back(LargestFD);
940
730
    } else {
941
730
      if (const auto *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
942
215
        assert(!CXXRD->isDynamicClass() &&
943
215
               "cannot expand vtable pointers in dynamic classes");
944
0
        for (const CXXBaseSpecifier &BS : CXXRD->bases())
945
5
          Bases.push_back(&BS);
946
215
      }
947
948
1.45k
      for (const auto *FD : RD->fields()) {
949
1.45k
        if (FD->isZeroLengthBitField(Context))
950
0
          continue;
951
1.45k
        assert(!FD->isBitField() &&
952
1.45k
               "Cannot expand structure with bit-field members.");
953
0
        Fields.push_back(FD);
954
1.45k
      }
955
730
    }
956
0
    return std::make_unique<RecordExpansion>(std::move(Bases),
957
735
                                              std::move(Fields));
958
735
  }
959
1.53k
  if (const ComplexType *CT = Ty->getAs<ComplexType>()) {
960
46
    return std::make_unique<ComplexExpansion>(CT->getElementType());
961
46
  }
962
1.49k
  return std::make_unique<NoExpansion>();
963
1.53k
}
964
965
1.35k
static int getExpansionSize(QualType Ty, const ASTContext &Context) {
966
1.35k
  auto Exp = getTypeExpansion(Ty, Context);
967
1.35k
  if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
968
24
    return CAExp->NumElts * getExpansionSize(CAExp->EltTy, Context);
969
24
  }
970
1.33k
  if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
971
441
    int Res = 0;
972
441
    for (auto BS : RExp->Bases)
973
3
      Res += getExpansionSize(BS->getType(), Context);
974
441
    for (auto FD : RExp->Fields)
975
872
      Res += getExpansionSize(FD->getType(), Context);
976
441
    return Res;
977
441
  }
978
893
  if (isa<ComplexExpansion>(Exp.get()))
979
28
    return 2;
980
865
  assert(isa<NoExpansion>(Exp.get()));
981
0
  return 1;
982
893
}
983
984
void
985
CodeGenTypes::getExpandedTypes(QualType Ty,
986
530
                               SmallVectorImpl<llvm::Type *>::iterator &TI) {
987
530
  auto Exp = getTypeExpansion(Ty, Context);
988
530
  if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
989
40
    for (int i = 0, n = CAExp->NumElts; i < n; 
i++32
) {
990
32
      getExpandedTypes(CAExp->EltTy, TI);
991
32
    }
992
522
  } else if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
993
166
    for (auto BS : RExp->Bases)
994
1
      getExpandedTypes(BS->getType(), TI);
995
166
    for (auto FD : RExp->Fields)
996
324
      getExpandedTypes(FD->getType(), TI);
997
356
  } else if (auto CExp = dyn_cast<ComplexExpansion>(Exp.get())) {
998
11
    llvm::Type *EltTy = ConvertType(CExp->EltTy);
999
11
    *TI++ = EltTy;
1000
11
    *TI++ = EltTy;
1001
345
  } else {
1002
345
    assert(isa<NoExpansion>(Exp.get()));
1003
0
    *TI++ = ConvertType(Ty);
1004
345
  }
1005
530
}
1006
1007
static void forConstantArrayExpansion(CodeGenFunction &CGF,
1008
                                      ConstantArrayExpansion *CAE,
1009
                                      Address BaseAddr,
1010
8
                                      llvm::function_ref<void(Address)> Fn) {
1011
8
  CharUnits EltSize = CGF.getContext().getTypeSizeInChars(CAE->EltTy);
1012
8
  CharUnits EltAlign =
1013
8
    BaseAddr.getAlignment().alignmentOfArrayElement(EltSize);
1014
1015
40
  for (int i = 0, n = CAE->NumElts; i < n; 
i++32
) {
1016
32
    llvm::Value *EltAddr = CGF.Builder.CreateConstGEP2_32(
1017
32
        BaseAddr.getElementType(), BaseAddr.getPointer(), 0, i);
1018
32
    Fn(Address(EltAddr, EltAlign));
1019
32
  }
1020
8
}
1021
1022
void CodeGenFunction::ExpandTypeFromArgs(QualType Ty, LValue LV,
1023
347
                                         llvm::Function::arg_iterator &AI) {
1024
347
  assert(LV.isSimple() &&
1025
347
         "Unexpected non-simple lvalue during struct expansion.");
1026
1027
0
  auto Exp = getTypeExpansion(Ty, getContext());
1028
347
  if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
1029
8
    forConstantArrayExpansion(
1030
32
        *this, CAExp, LV.getAddress(*this), [&](Address EltAddr) {
1031
32
          LValue LV = MakeAddrLValue(EltAddr, CAExp->EltTy);
1032
32
          ExpandTypeFromArgs(CAExp->EltTy, LV, AI);
1033
32
        });
1034
339
  } else if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
1035
100
    Address This = LV.getAddress(*this);
1036
100
    for (const CXXBaseSpecifier *BS : RExp->Bases) {
1037
      // Perform a single step derived-to-base conversion.
1038
1
      Address Base =
1039
1
          GetAddressOfBaseClass(This, Ty->getAsCXXRecordDecl(), &BS, &BS + 1,
1040
1
                                /*NullCheckValue=*/false, SourceLocation());
1041
1
      LValue SubLV = MakeAddrLValue(Base, BS->getType());
1042
1043
      // Recurse onto bases.
1044
1
      ExpandTypeFromArgs(BS->getType(), SubLV, AI);
1045
1
    }
1046
213
    for (auto FD : RExp->Fields) {
1047
      // FIXME: What are the right qualifiers here?
1048
213
      LValue SubLV = EmitLValueForFieldInitialization(LV, FD);
1049
213
      ExpandTypeFromArgs(FD->getType(), SubLV, AI);
1050
213
    }
1051
239
  } else if (isa<ComplexExpansion>(Exp.get())) {
1052
3
    auto realValue = &*AI++;
1053
3
    auto imagValue = &*AI++;
1054
3
    EmitStoreOfComplex(ComplexPairTy(realValue, imagValue), LV, /*init*/ true);
1055
236
  } else {
1056
    // Call EmitStoreOfScalar except when the lvalue is a bitfield to emit a
1057
    // primitive store.
1058
236
    assert(isa<NoExpansion>(Exp.get()));
1059
236
    if (LV.isBitField())
1060
0
      EmitStoreThroughLValue(RValue::get(&*AI++), LV);
1061
236
    else
1062
236
      EmitStoreOfScalar(&*AI++, LV);
1063
236
  }
1064
347
}
1065
1066
void CodeGenFunction::ExpandTypeToArgs(
1067
    QualType Ty, CallArg Arg, llvm::FunctionType *IRFuncTy,
1068
77
    SmallVectorImpl<llvm::Value *> &IRCallArgs, unsigned &IRCallArgPos) {
1069
77
  auto Exp = getTypeExpansion(Ty, getContext());
1070
77
  if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
1071
0
    Address Addr = Arg.hasLValue() ? Arg.getKnownLValue().getAddress(*this)
1072
0
                                   : Arg.getKnownRValue().getAggregateAddress();
1073
0
    forConstantArrayExpansion(
1074
0
        *this, CAExp, Addr, [&](Address EltAddr) {
1075
0
          CallArg EltArg = CallArg(
1076
0
              convertTempToRValue(EltAddr, CAExp->EltTy, SourceLocation()),
1077
0
              CAExp->EltTy);
1078
0
          ExpandTypeToArgs(CAExp->EltTy, EltArg, IRFuncTy, IRCallArgs,
1079
0
                           IRCallArgPos);
1080
0
        });
1081
77
  } else if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
1082
28
    Address This = Arg.hasLValue() ? 
Arg.getKnownLValue().getAddress(*this)12
1083
28
                                   : 
Arg.getKnownRValue().getAggregateAddress()16
;
1084
28
    for (const CXXBaseSpecifier *BS : RExp->Bases) {
1085
      // Perform a single step derived-to-base conversion.
1086
0
      Address Base =
1087
0
          GetAddressOfBaseClass(This, Ty->getAsCXXRecordDecl(), &BS, &BS + 1,
1088
0
                                /*NullCheckValue=*/false, SourceLocation());
1089
0
      CallArg BaseArg = CallArg(RValue::getAggregate(Base), BS->getType());
1090
1091
      // Recurse onto bases.
1092
0
      ExpandTypeToArgs(BS->getType(), BaseArg, IRFuncTy, IRCallArgs,
1093
0
                       IRCallArgPos);
1094
0
    }
1095
1096
28
    LValue LV = MakeAddrLValue(This, Ty);
1097
46
    for (auto FD : RExp->Fields) {
1098
46
      CallArg FldArg =
1099
46
          CallArg(EmitRValueForField(LV, FD, SourceLocation()), FD->getType());
1100
46
      ExpandTypeToArgs(FD->getType(), FldArg, IRFuncTy, IRCallArgs,
1101
46
                       IRCallArgPos);
1102
46
    }
1103
49
  } else if (isa<ComplexExpansion>(Exp.get())) {
1104
4
    ComplexPairTy CV = Arg.getKnownRValue().getComplexVal();
1105
4
    IRCallArgs[IRCallArgPos++] = CV.first;
1106
4
    IRCallArgs[IRCallArgPos++] = CV.second;
1107
45
  } else {
1108
45
    assert(isa<NoExpansion>(Exp.get()));
1109
0
    auto RV = Arg.getKnownRValue();
1110
45
    assert(RV.isScalar() &&
1111
45
           "Unexpected non-scalar rvalue during struct expansion.");
1112
1113
    // Insert a bitcast as needed.
1114
0
    llvm::Value *V = RV.getScalarVal();
1115
45
    if (IRCallArgPos < IRFuncTy->getNumParams() &&
1116
45
        V->getType() != IRFuncTy->getParamType(IRCallArgPos))
1117
0
      V = Builder.CreateBitCast(V, IRFuncTy->getParamType(IRCallArgPos));
1118
1119
45
    IRCallArgs[IRCallArgPos++] = V;
1120
45
  }
1121
77
}
1122
1123
/// Create a temporary allocation for the purposes of coercion.
1124
static Address CreateTempAllocaForCoercion(CodeGenFunction &CGF, llvm::Type *Ty,
1125
                                           CharUnits MinAlign,
1126
276
                                           const Twine &Name = "tmp") {
1127
  // Don't use an alignment that's worse than what LLVM would prefer.
1128
276
  auto PrefAlign = CGF.CGM.getDataLayout().getPrefTypeAlignment(Ty);
1129
276
  CharUnits Align = std::max(MinAlign, CharUnits::fromQuantity(PrefAlign));
1130
1131
276
  return CGF.CreateTempAlloca(Ty, Align, Name + ".coerce");
1132
276
}
1133
1134
/// EnterStructPointerForCoercedAccess - Given a struct pointer that we are
1135
/// accessing some number of bytes out of it, try to gep into the struct to get
1136
/// at its inner goodness.  Dive as deep as possible without entering an element
1137
/// with an in-memory size smaller than DstSize.
1138
static Address
1139
EnterStructPointerForCoercedAccess(Address SrcPtr,
1140
                                   llvm::StructType *SrcSTy,
1141
19.0k
                                   uint64_t DstSize, CodeGenFunction &CGF) {
1142
  // We can't dive into a zero-element struct.
1143
19.0k
  if (SrcSTy->getNumElements() == 0) 
return SrcPtr0
;
1144
1145
19.0k
  llvm::Type *FirstElt = SrcSTy->getElementType(0);
1146
1147
  // If the first elt is at least as large as what we're looking for, or if the
1148
  // first element is the same size as the whole struct, we can enter it. The
1149
  // comparison must be made on the store size and not the alloca size. Using
1150
  // the alloca size may overstate the size of the load.
1151
19.0k
  uint64_t FirstEltSize =
1152
19.0k
    CGF.CGM.getDataLayout().getTypeStoreSize(FirstElt);
1153
19.0k
  if (FirstEltSize < DstSize &&
1154
19.0k
      
FirstEltSize < CGF.CGM.getDataLayout().getTypeStoreSize(SrcSTy)5.01k
)
1155
4.85k
    return SrcPtr;
1156
1157
  // GEP into the first element.
1158
14.2k
  SrcPtr = CGF.Builder.CreateStructGEP(SrcPtr, 0, "coerce.dive");
1159
1160
  // If the first element is a struct, recurse.
1161
14.2k
  llvm::Type *SrcTy = SrcPtr.getElementType();
1162
14.2k
  if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy))
1163
3.82k
    return EnterStructPointerForCoercedAccess(SrcPtr, SrcSTy, DstSize, CGF);
1164
1165
10.3k
  return SrcPtr;
1166
14.2k
}
1167
1168
/// CoerceIntOrPtrToIntOrPtr - Convert a value Val to the specific Ty where both
1169
/// are either integers or pointers.  This does a truncation of the value if it
1170
/// is too large or a zero extension if it is too small.
1171
///
1172
/// This behaves as if the value were coerced through memory, so on big-endian
1173
/// targets the high bits are preserved in a truncation, while little-endian
1174
/// targets preserve the low bits.
1175
static llvm::Value *CoerceIntOrPtrToIntOrPtr(llvm::Value *Val,
1176
                                             llvm::Type *Ty,
1177
9.12k
                                             CodeGenFunction &CGF) {
1178
9.12k
  if (Val->getType() == Ty)
1179
8.67k
    return Val;
1180
1181
451
  if (isa<llvm::PointerType>(Val->getType())) {
1182
    // If this is Pointer->Pointer avoid conversion to and from int.
1183
351
    if (isa<llvm::PointerType>(Ty))
1184
144
      return CGF.Builder.CreateBitCast(Val, Ty, "coerce.val");
1185
1186
    // Convert the pointer to an integer so we can play with its width.
1187
207
    Val = CGF.Builder.CreatePtrToInt(Val, CGF.IntPtrTy, "coerce.val.pi");
1188
207
  }
1189
1190
307
  llvm::Type *DestIntTy = Ty;
1191
307
  if (isa<llvm::PointerType>(DestIntTy))
1192
29
    DestIntTy = CGF.IntPtrTy;
1193
1194
307
  if (Val->getType() != DestIntTy) {
1195
71
    const llvm::DataLayout &DL = CGF.CGM.getDataLayout();
1196
71
    if (DL.isBigEndian()) {
1197
      // Preserve the high bits on big-endian targets.
1198
      // That is what memory coercion does.
1199
30
      uint64_t SrcSize = DL.getTypeSizeInBits(Val->getType());
1200
30
      uint64_t DstSize = DL.getTypeSizeInBits(DestIntTy);
1201
1202
30
      if (SrcSize > DstSize) {
1203
11
        Val = CGF.Builder.CreateLShr(Val, SrcSize - DstSize, "coerce.highbits");
1204
11
        Val = CGF.Builder.CreateTrunc(Val, DestIntTy, "coerce.val.ii");
1205
19
      } else {
1206
19
        Val = CGF.Builder.CreateZExt(Val, DestIntTy, "coerce.val.ii");
1207
19
        Val = CGF.Builder.CreateShl(Val, DstSize - SrcSize, "coerce.highbits");
1208
19
      }
1209
41
    } else {
1210
      // Little-endian targets preserve the low bits. No shifts required.
1211
41
      Val = CGF.Builder.CreateIntCast(Val, DestIntTy, false, "coerce.val.ii");
1212
41
    }
1213
71
  }
1214
1215
307
  if (isa<llvm::PointerType>(Ty))
1216
29
    Val = CGF.Builder.CreateIntToPtr(Val, Ty, "coerce.val.ip");
1217
307
  return Val;
1218
451
}
1219
1220
1221
1222
/// CreateCoercedLoad - Create a load from \arg SrcPtr interpreted as
1223
/// a pointer to an object of type \arg Ty, known to be aligned to
1224
/// \arg SrcAlign bytes.
1225
///
1226
/// This safely handles the case when the src type is smaller than the
1227
/// destination type; in this situation the values of bits which not
1228
/// present in the src are undefined.
1229
static llvm::Value *CreateCoercedLoad(Address Src, llvm::Type *Ty,
1230
8.12k
                                      CodeGenFunction &CGF) {
1231
8.12k
  llvm::Type *SrcTy = Src.getElementType();
1232
1233
  // If SrcTy and Ty are the same, just do a load.
1234
8.12k
  if (SrcTy == Ty)
1235
31
    return CGF.Builder.CreateLoad(Src);
1236
1237
8.08k
  llvm::TypeSize DstSize = CGF.CGM.getDataLayout().getTypeAllocSize(Ty);
1238
1239
8.08k
  if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy)) {
1240
6.71k
    Src = EnterStructPointerForCoercedAccess(Src, SrcSTy,
1241
6.71k
                                             DstSize.getFixedSize(), CGF);
1242
6.71k
    SrcTy = Src.getElementType();
1243
6.71k
  }
1244
1245
8.08k
  llvm::TypeSize SrcSize = CGF.CGM.getDataLayout().getTypeAllocSize(SrcTy);
1246
1247
  // If the source and destination are integer or pointer types, just do an
1248
  // extension or truncation to the desired type.
1249
8.08k
  if ((isa<llvm::IntegerType>(Ty) || 
isa<llvm::PointerType>(Ty)5.14k
) &&
1250
8.08k
      
(5.97k
isa<llvm::IntegerType>(SrcTy)5.97k
||
isa<llvm::PointerType>(SrcTy)4.77k
)) {
1251
4.43k
    llvm::Value *Load = CGF.Builder.CreateLoad(Src);
1252
4.43k
    return CoerceIntOrPtrToIntOrPtr(Load, Ty, CGF);
1253
4.43k
  }
1254
1255
  // If load is legal, just bitcast the src pointer.
1256
3.65k
  if (!SrcSize.isScalable() && !DstSize.isScalable() &&
1257
3.65k
      
SrcSize.getFixedSize() >= DstSize.getFixedSize()3.62k
) {
1258
    // Generally SrcSize is never greater than DstSize, since this means we are
1259
    // losing bits. However, this can happen in cases where the structure has
1260
    // additional padding, for example due to a user specified alignment.
1261
    //
1262
    // FIXME: Assert that we aren't truncating non-padding bits when have access
1263
    // to that information.
1264
3.45k
    Src = CGF.Builder.CreateBitCast(Src,
1265
3.45k
                                    Ty->getPointerTo(Src.getAddressSpace()));
1266
3.45k
    return CGF.Builder.CreateLoad(Src);
1267
3.45k
  }
1268
1269
  // If coercing a fixed vector to a scalable vector for ABI compatibility, and
1270
  // the types match, use the llvm.experimental.vector.insert intrinsic to
1271
  // perform the conversion.
1272
196
  if (auto *ScalableDst = dyn_cast<llvm::ScalableVectorType>(Ty)) {
1273
26
    if (auto *FixedSrc = dyn_cast<llvm::FixedVectorType>(SrcTy)) {
1274
26
      if (ScalableDst->getElementType() == FixedSrc->getElementType()) {
1275
21
        auto *Load = CGF.Builder.CreateLoad(Src);
1276
21
        auto *UndefVec = llvm::UndefValue::get(ScalableDst);
1277
21
        auto *Zero = llvm::Constant::getNullValue(CGF.CGM.Int64Ty);
1278
21
        return CGF.Builder.CreateInsertVector(ScalableDst, UndefVec, Load, Zero,
1279
21
                                              "castScalableSve");
1280
21
      }
1281
26
    }
1282
26
  }
1283
1284
  // Otherwise do coercion through memory. This is stupid, but simple.
1285
175
  Address Tmp =
1286
175
      CreateTempAllocaForCoercion(CGF, Ty, Src.getAlignment(), Src.getName());
1287
175
  CGF.Builder.CreateMemCpy(
1288
175
      Tmp.getPointer(), Tmp.getAlignment().getAsAlign(), Src.getPointer(),
1289
175
      Src.getAlignment().getAsAlign(),
1290
175
      llvm::ConstantInt::get(CGF.IntPtrTy, SrcSize.getKnownMinSize()));
1291
175
  return CGF.Builder.CreateLoad(Tmp);
1292
196
}
1293
1294
// Function to store a first-class aggregate into memory.  We prefer to
1295
// store the elements rather than the aggregate to be more friendly to
1296
// fast-isel.
1297
// FIXME: Do we need to recurse here?
1298
void CodeGenFunction::EmitAggregateStore(llvm::Value *Val, Address Dest,
1299
5.29k
                                         bool DestIsVolatile) {
1300
  // Prefer scalar stores to first-class aggregate stores.
1301
5.29k
  if (llvm::StructType *STy = dyn_cast<llvm::StructType>(Val->getType())) {
1302
4.29k
    for (unsigned i = 0, e = STy->getNumElements(); i != e; 
++i2.84k
) {
1303
2.84k
      Address EltPtr = Builder.CreateStructGEP(Dest, i);
1304
2.84k
      llvm::Value *Elt = Builder.CreateExtractValue(Val, i);
1305
2.84k
      Builder.CreateStore(Elt, EltPtr, DestIsVolatile);
1306
2.84k
    }
1307
3.85k
  } else {
1308
3.85k
    Builder.CreateStore(Val, Dest, DestIsVolatile);
1309
3.85k
  }
1310
5.29k
}
1311
1312
/// CreateCoercedStore - Create a store to \arg DstPtr from \arg Src,
1313
/// where the source and destination may have different types.  The
1314
/// destination is known to be aligned to \arg DstAlign bytes.
1315
///
1316
/// This safely handles the case when the src type is larger than the
1317
/// destination type; the upper bits of the src will be lost.
1318
static void CreateCoercedStore(llvm::Value *Src,
1319
                               Address Dst,
1320
                               bool DstIsVolatile,
1321
10.1k
                               CodeGenFunction &CGF) {
1322
10.1k
  llvm::Type *SrcTy = Src->getType();
1323
10.1k
  llvm::Type *DstTy = Dst.getElementType();
1324
10.1k
  if (SrcTy == DstTy) {
1325
170
    CGF.Builder.CreateStore(Src, Dst, DstIsVolatile);
1326
170
    return;
1327
170
  }
1328
1329
10.0k
  llvm::TypeSize SrcSize = CGF.CGM.getDataLayout().getTypeAllocSize(SrcTy);
1330
1331
10.0k
  if (llvm::StructType *DstSTy = dyn_cast<llvm::StructType>(DstTy)) {
1332
8.51k
    Dst = EnterStructPointerForCoercedAccess(Dst, DstSTy,
1333
8.51k
                                             SrcSize.getFixedSize(), CGF);
1334
8.51k
    DstTy = Dst.getElementType();
1335
8.51k
  }
1336
1337
10.0k
  llvm::PointerType *SrcPtrTy = llvm::dyn_cast<llvm::PointerType>(SrcTy);
1338
10.0k
  llvm::PointerType *DstPtrTy = llvm::dyn_cast<llvm::PointerType>(DstTy);
1339
10.0k
  if (SrcPtrTy && 
DstPtrTy3.27k
&&
1340
10.0k
      
SrcPtrTy->getAddressSpace() != DstPtrTy->getAddressSpace()3.27k
) {
1341
37
    Src = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Src, DstTy);
1342
37
    CGF.Builder.CreateStore(Src, Dst, DstIsVolatile);
1343
37
    return;
1344
37
  }
1345
1346
  // If the source and destination are integer or pointer types, just do an
1347
  // extension or truncation to the desired type.
1348
9.96k
  if ((isa<llvm::IntegerType>(SrcTy) || 
isa<llvm::PointerType>(SrcTy)6.88k
) &&
1349
9.96k
      
(6.31k
isa<llvm::IntegerType>(DstTy)6.31k
||
isa<llvm::PointerType>(DstTy)4.88k
)) {
1350
4.69k
    Src = CoerceIntOrPtrToIntOrPtr(Src, DstTy, CGF);
1351
4.69k
    CGF.Builder.CreateStore(Src, Dst, DstIsVolatile);
1352
4.69k
    return;
1353
4.69k
  }
1354
1355
5.27k
  llvm::TypeSize DstSize = CGF.CGM.getDataLayout().getTypeAllocSize(DstTy);
1356
1357
  // If store is legal, just bitcast the src pointer.
1358
5.27k
  if (isa<llvm::ScalableVectorType>(SrcTy) ||
1359
5.27k
      
isa<llvm::ScalableVectorType>(DstTy)5.26k
||
1360
5.27k
      
SrcSize.getFixedSize() <= DstSize.getFixedSize()5.26k
) {
1361
5.17k
    Dst = CGF.Builder.CreateElementBitCast(Dst, SrcTy);
1362
5.17k
    CGF.EmitAggregateStore(Src, Dst, DstIsVolatile);
1363
5.17k
  } else {
1364
    // Otherwise do coercion through memory. This is stupid, but
1365
    // simple.
1366
1367
    // Generally SrcSize is never greater than DstSize, since this means we are
1368
    // losing bits. However, this can happen in cases where the structure has
1369
    // additional padding, for example due to a user specified alignment.
1370
    //
1371
    // FIXME: Assert that we aren't truncating non-padding bits when have access
1372
    // to that information.
1373
101
    Address Tmp = CreateTempAllocaForCoercion(CGF, SrcTy, Dst.getAlignment());
1374
101
    CGF.Builder.CreateStore(Src, Tmp);
1375
101
    CGF.Builder.CreateMemCpy(
1376
101
        Dst.getPointer(), Dst.getAlignment().getAsAlign(), Tmp.getPointer(),
1377
101
        Tmp.getAlignment().getAsAlign(),
1378
101
        llvm::ConstantInt::get(CGF.IntPtrTy, DstSize.getFixedSize()));
1379
101
  }
1380
5.27k
}
1381
1382
static Address emitAddressAtOffset(CodeGenFunction &CGF, Address addr,
1383
20.4k
                                   const ABIArgInfo &info) {
1384
20.4k
  if (unsigned offset = info.getDirectOffset()) {
1385
3
    addr = CGF.Builder.CreateElementBitCast(addr, CGF.Int8Ty);
1386
3
    addr = CGF.Builder.CreateConstInBoundsByteGEP(addr,
1387
3
                                             CharUnits::fromQuantity(offset));
1388
3
    addr = CGF.Builder.CreateElementBitCast(addr, info.getCoerceToType());
1389
3
  }
1390
20.4k
  return addr;
1391
20.4k
}
1392
1393
namespace {
1394
1395
/// Encapsulates information about the way function arguments from
1396
/// CGFunctionInfo should be passed to actual LLVM IR function.
1397
class ClangToLLVMArgMapping {
1398
  static const unsigned InvalidIndex = ~0U;
1399
  unsigned InallocaArgNo;
1400
  unsigned SRetArgNo;
1401
  unsigned TotalIRArgs;
1402
1403
  /// Arguments of LLVM IR function corresponding to single Clang argument.
1404
  struct IRArgs {
1405
    unsigned PaddingArgIndex;
1406
    // Argument is expanded to IR arguments at positions
1407
    // [FirstArgIndex, FirstArgIndex + NumberOfArgs).
1408
    unsigned FirstArgIndex;
1409
    unsigned NumberOfArgs;
1410
1411
    IRArgs()
1412
        : PaddingArgIndex(InvalidIndex), FirstArgIndex(InvalidIndex),
1413
2.71M
          NumberOfArgs(0) {}
1414
  };
1415
1416
  SmallVector<IRArgs, 8> ArgInfo;
1417
1418
public:
1419
  ClangToLLVMArgMapping(const ASTContext &Context, const CGFunctionInfo &FI,
1420
                        bool OnlyRequiredArgs = false)
1421
      : InallocaArgNo(InvalidIndex), SRetArgNo(InvalidIndex), TotalIRArgs(0),
1422
2.71M
        ArgInfo(OnlyRequiredArgs ? FI.getNumRequiredArgs() : FI.arg_size()) {
1423
2.71M
    construct(Context, FI, OnlyRequiredArgs);
1424
2.71M
  }
1425
1426
3.24M
  bool hasInallocaArg() const { return InallocaArgNo != InvalidIndex; }
1427
568
  unsigned getInallocaArgNo() const {
1428
568
    assert(hasInallocaArg());
1429
0
    return InallocaArgNo;
1430
568
  }
1431
1432
2.34M
  bool hasSRetArg() const { return SRetArgNo != InvalidIndex; }
1433
16.1k
  unsigned getSRetArgNo() const {
1434
16.1k
    assert(hasSRetArg());
1435
0
    return SRetArgNo;
1436
16.1k
  }
1437
1438
2.71M
  unsigned totalIRArgs() const { return TotalIRArgs; }
1439
1440
4.20M
  bool hasPaddingArg(unsigned ArgNo) const {
1441
4.20M
    assert(ArgNo < ArgInfo.size());
1442
0
    return ArgInfo[ArgNo].PaddingArgIndex != InvalidIndex;
1443
4.20M
  }
1444
69
  unsigned getPaddingArgNo(unsigned ArgNo) const {
1445
69
    assert(hasPaddingArg(ArgNo));
1446
0
    return ArgInfo[ArgNo].PaddingArgIndex;
1447
69
  }
1448
1449
  /// Returns index of first IR argument corresponding to ArgNo, and their
1450
  /// quantity.
1451
3.96M
  std::pair<unsigned, unsigned> getIRArgs(unsigned ArgNo) const {
1452
3.96M
    assert(ArgNo < ArgInfo.size());
1453
0
    return std::make_pair(ArgInfo[ArgNo].FirstArgIndex,
1454
3.96M
                          ArgInfo[ArgNo].NumberOfArgs);
1455
3.96M
  }
1456
1457
private:
1458
  void construct(const ASTContext &Context, const CGFunctionInfo &FI,
1459
                 bool OnlyRequiredArgs);
1460
};
1461
1462
void ClangToLLVMArgMapping::construct(const ASTContext &Context,
1463
                                      const CGFunctionInfo &FI,
1464
2.71M
                                      bool OnlyRequiredArgs) {
1465
2.71M
  unsigned IRArgNo = 0;
1466
2.71M
  bool SwapThisWithSRet = false;
1467
2.71M
  const ABIArgInfo &RetAI = FI.getReturnInfo();
1468
1469
2.71M
  if (RetAI.getKind() == ABIArgInfo::Indirect) {
1470
16.1k
    SwapThisWithSRet = RetAI.isSRetAfterThis();
1471
16.1k
    SRetArgNo = SwapThisWithSRet ? 
1392
:
IRArgNo++15.7k
;
1472
16.1k
  }
1473
1474
2.71M
  unsigned ArgNo = 0;
1475
2.71M
  unsigned NumArgs = OnlyRequiredArgs ? 
FI.getNumRequiredArgs()1.16M
:
FI.arg_size()1.54M
;
1476
7.73M
  for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(); ArgNo < NumArgs;
1477
5.02M
       ++I, ++ArgNo) {
1478
5.02M
    assert(I != FI.arg_end());
1479
0
    QualType ArgType = I->type;
1480
5.02M
    const ABIArgInfo &AI = I->info;
1481
    // Collect data about IR arguments corresponding to Clang argument ArgNo.
1482
5.02M
    auto &IRArgs = ArgInfo[ArgNo];
1483
1484
5.02M
    if (AI.getPaddingType())
1485
124
      IRArgs.PaddingArgIndex = IRArgNo++;
1486
1487
5.02M
    switch (AI.getKind()) {
1488
119k
    case ABIArgInfo::Extend:
1489
4.94M
    case ABIArgInfo::Direct: {
1490
      // FIXME: handle sseregparm someday...
1491
4.94M
      llvm::StructType *STy = dyn_cast<llvm::StructType>(AI.getCoerceToType());
1492
4.94M
      if (AI.isDirect() && 
AI.getCanBeFlattened()4.82M
&&
STy4.82M
) {
1493
8.44k
        IRArgs.NumberOfArgs = STy->getNumElements();
1494
4.93M
      } else {
1495
4.93M
        IRArgs.NumberOfArgs = 1;
1496
4.93M
      }
1497
4.94M
      break;
1498
119k
    }
1499
16.4k
    case ABIArgInfo::Indirect:
1500
16.5k
    case ABIArgInfo::IndirectAliased:
1501
16.5k
      IRArgs.NumberOfArgs = 1;
1502
16.5k
      break;
1503
56.7k
    case ABIArgInfo::Ignore:
1504
57.6k
    case ABIArgInfo::InAlloca:
1505
      // ignore and inalloca doesn't have matching LLVM parameters.
1506
57.6k
      IRArgs.NumberOfArgs = 0;
1507
57.6k
      break;
1508
3.47k
    case ABIArgInfo::CoerceAndExpand:
1509
3.47k
      IRArgs.NumberOfArgs = AI.getCoerceAndExpandTypeSequence().size();
1510
3.47k
      break;
1511
459
    case ABIArgInfo::Expand:
1512
459
      IRArgs.NumberOfArgs = getExpansionSize(ArgType, Context);
1513
459
      break;
1514
5.02M
    }
1515
1516
5.02M
    if (IRArgs.NumberOfArgs > 0) {
1517
4.96M
      IRArgs.FirstArgIndex = IRArgNo;
1518
4.96M
      IRArgNo += IRArgs.NumberOfArgs;
1519
4.96M
    }
1520
1521
    // Skip over the sret parameter when it comes second.  We already handled it
1522
    // above.
1523
5.02M
    if (IRArgNo == 1 && 
SwapThisWithSRet2.52M
)
1524
392
      IRArgNo++;
1525
5.02M
  }
1526
2.71M
  assert(ArgNo == ArgInfo.size());
1527
1528
2.71M
  if (FI.usesInAlloca())
1529
467
    InallocaArgNo = IRArgNo++;
1530
1531
2.71M
  TotalIRArgs = IRArgNo;
1532
2.71M
}
1533
}  // namespace
1534
1535
/***/
1536
1537
20.9k
bool CodeGenModule::ReturnTypeUsesSRet(const CGFunctionInfo &FI) {
1538
20.9k
  const auto &RI = FI.getReturnInfo();
1539
20.9k
  return RI.isIndirect() || 
(20.8k
RI.isInAlloca()20.8k
&&
RI.getInAllocaSRet()1
);
1540
20.9k
}
1541
1542
13.3k
bool CodeGenModule::ReturnSlotInterferesWithArgs(const CGFunctionInfo &FI) {
1543
13.3k
  return ReturnTypeUsesSRet(FI) &&
1544
13.3k
         
getTargetCodeGenInfo().doesReturnSlotInterfereWithArgs()44
;
1545
13.3k
}
1546
1547
12.2k
bool CodeGenModule::ReturnTypeUsesFPRet(QualType ResultType) {
1548
12.2k
  if (const BuiltinType *BT = ResultType->getAs<BuiltinType>()) {
1549
5.22k
    switch (BT->getKind()) {
1550
5.15k
    default:
1551
5.15k
      return false;
1552
18
    case BuiltinType::Float:
1553
18
      return getTarget().useObjCFPRetForRealType(TargetInfo::Float);
1554
46
    case BuiltinType::Double:
1555
46
      return getTarget().useObjCFPRetForRealType(TargetInfo::Double);
1556
4
    case BuiltinType::LongDouble:
1557
4
      return getTarget().useObjCFPRetForRealType(TargetInfo::LongDouble);
1558
5.22k
    }
1559
5.22k
  }
1560
1561
7.01k
  return false;
1562
12.2k
}
1563
1564
12.2k
bool CodeGenModule::ReturnTypeUsesFP2Ret(QualType ResultType) {
1565
12.2k
  if (const ComplexType *CT = ResultType->getAs<ComplexType>()) {
1566
14
    if (const BuiltinType *BT = CT->getElementType()->getAs<BuiltinType>()) {
1567
14
      if (BT->getKind() == BuiltinType::LongDouble)
1568
2
        return getTarget().useObjCFP2RetForComplexLongDouble();
1569
14
    }
1570
14
  }
1571
1572
12.2k
  return false;
1573
12.2k
}
1574
1575
5.82k
llvm::FunctionType *CodeGenTypes::GetFunctionType(GlobalDecl GD) {
1576
5.82k
  const CGFunctionInfo &FI = arrangeGlobalDeclaration(GD);
1577
5.82k
  return GetFunctionType(FI);
1578
5.82k
}
1579
1580
llvm::FunctionType *
1581
1.16M
CodeGenTypes::GetFunctionType(const CGFunctionInfo &FI) {
1582
1583
1.16M
  bool Inserted = FunctionsBeingProcessed.insert(&FI).second;
1584
1.16M
  (void)Inserted;
1585
1.16M
  assert(Inserted && "Recursively being processed?");
1586
1587
0
  llvm::Type *resultType = nullptr;
1588
1.16M
  const ABIArgInfo &retAI = FI.getReturnInfo();
1589
1.16M
  switch (retAI.getKind()) {
1590
0
  case ABIArgInfo::Expand:
1591
0
  case ABIArgInfo::IndirectAliased:
1592
0
    llvm_unreachable("Invalid ABI kind for return argument");
1593
1594
40.8k
  case ABIArgInfo::Extend:
1595
630k
  case ABIArgInfo::Direct:
1596
630k
    resultType = retAI.getCoerceToType();
1597
630k
    break;
1598
1599
18
  case ABIArgInfo::InAlloca:
1600
18
    if (retAI.getInAllocaSRet()) {
1601
      // sret things on win32 aren't void, they return the sret pointer.
1602
18
      QualType ret = FI.getReturnType();
1603
18
      llvm::Type *ty = ConvertType(ret);
1604
18
      unsigned addressSpace = Context.getTargetAddressSpace(ret);
1605
18
      resultType = llvm::PointerType::get(ty, addressSpace);
1606
18
    } else {
1607
0
      resultType = llvm::Type::getVoidTy(getLLVMContext());
1608
0
    }
1609
18
    break;
1610
1611
6.67k
  case ABIArgInfo::Indirect:
1612
536k
  case ABIArgInfo::Ignore:
1613
536k
    resultType = llvm::Type::getVoidTy(getLLVMContext());
1614
536k
    break;
1615
1616
1.42k
  case ABIArgInfo::CoerceAndExpand:
1617
1.42k
    resultType = retAI.getUnpaddedCoerceAndExpandType();
1618
1.42k
    break;
1619
1.16M
  }
1620
1621
1.16M
  ClangToLLVMArgMapping IRFunctionArgs(getContext(), FI, true);
1622
1.16M
  SmallVector<llvm::Type*, 8> ArgTypes(IRFunctionArgs.totalIRArgs());
1623
1624
  // Add type for sret argument.
1625
1.16M
  if (IRFunctionArgs.hasSRetArg()) {
1626
6.67k
    QualType Ret = FI.getReturnType();
1627
6.67k
    llvm::Type *Ty = ConvertType(Ret);
1628
6.67k
    unsigned AddressSpace = Context.getTargetAddressSpace(Ret);
1629
6.67k
    ArgTypes[IRFunctionArgs.getSRetArgNo()] =
1630
6.67k
        llvm::PointerType::get(Ty, AddressSpace);
1631
6.67k
  }
1632
1633
  // Add type for inalloca argument.
1634
1.16M
  if (IRFunctionArgs.hasInallocaArg()) {
1635
194
    auto ArgStruct = FI.getArgStruct();
1636
194
    assert(ArgStruct);
1637
0
    ArgTypes[IRFunctionArgs.getInallocaArgNo()] = ArgStruct->getPointerTo();
1638
194
  }
1639
1640
  // Add in all of the required arguments.
1641
0
  unsigned ArgNo = 0;
1642
1.16M
  CGFunctionInfo::const_arg_iterator it = FI.arg_begin(),
1643
1.16M
                                     ie = it + FI.getNumRequiredArgs();
1644
3.20M
  for (; it != ie; 
++it, ++ArgNo2.04M
) {
1645
2.04M
    const ABIArgInfo &ArgInfo = it->info;
1646
1647
    // Insert a padding type to ensure proper alignment.
1648
2.04M
    if (IRFunctionArgs.hasPaddingArg(ArgNo))
1649
47
      ArgTypes[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
1650
47
          ArgInfo.getPaddingType();
1651
1652
2.04M
    unsigned FirstIRArg, NumIRArgs;
1653
2.04M
    std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
1654
1655
2.04M
    switch (ArgInfo.getKind()) {
1656
28.0k
    case ABIArgInfo::Ignore:
1657
28.3k
    case ABIArgInfo::InAlloca:
1658
28.3k
      assert(NumIRArgs == 0);
1659
0
      break;
1660
1661
6.90k
    case ABIArgInfo::Indirect: {
1662
6.90k
      assert(NumIRArgs == 1);
1663
      // indirect arguments are always on the stack, which is alloca addr space.
1664
0
      llvm::Type *LTy = ConvertTypeForMem(it->type);
1665
6.90k
      ArgTypes[FirstIRArg] = LTy->getPointerTo(
1666
6.90k
          CGM.getDataLayout().getAllocaAddrSpace());
1667
6.90k
      break;
1668
28.0k
    }
1669
27
    case ABIArgInfo::IndirectAliased: {
1670
27
      assert(NumIRArgs == 1);
1671
0
      llvm::Type *LTy = ConvertTypeForMem(it->type);
1672
27
      ArgTypes[FirstIRArg] = LTy->getPointerTo(ArgInfo.getIndirectAddrSpace());
1673
27
      break;
1674
28.0k
    }
1675
49.2k
    case ABIArgInfo::Extend:
1676
2.00M
    case ABIArgInfo::Direct: {
1677
      // Fast-isel and the optimizer generally like scalar values better than
1678
      // FCAs, so we flatten them if this is safe to do for this argument.
1679
2.00M
      llvm::Type *argType = ArgInfo.getCoerceToType();
1680
2.00M
      llvm::StructType *st = dyn_cast<llvm::StructType>(argType);
1681
2.00M
      if (st && 
ArgInfo.isDirect()3.86k
&&
ArgInfo.getCanBeFlattened()3.86k
) {
1682
3.63k
        assert(NumIRArgs == st->getNumElements());
1683
11.0k
        for (unsigned i = 0, e = st->getNumElements(); i != e; 
++i7.39k
)
1684
7.39k
          ArgTypes[FirstIRArg + i] = st->getElementType(i);
1685
2.00M
      } else {
1686
2.00M
        assert(NumIRArgs == 1);
1687
0
        ArgTypes[FirstIRArg] = argType;
1688
2.00M
      }
1689
0
      break;
1690
49.2k
    }
1691
1692
1.45k
    case ABIArgInfo::CoerceAndExpand: {
1693
1.45k
      auto ArgTypesIter = ArgTypes.begin() + FirstIRArg;
1694
3.12k
      for (auto EltTy : ArgInfo.getCoerceAndExpandTypeSequence()) {
1695
3.12k
        *ArgTypesIter++ = EltTy;
1696
3.12k
      }
1697
1.45k
      assert(ArgTypesIter == ArgTypes.begin() + FirstIRArg + NumIRArgs);
1698
0
      break;
1699
49.2k
    }
1700
1701
173
    case ABIArgInfo::Expand:
1702
173
      auto ArgTypesIter = ArgTypes.begin() + FirstIRArg;
1703
173
      getExpandedTypes(it->type, ArgTypesIter);
1704
173
      assert(ArgTypesIter == ArgTypes.begin() + FirstIRArg + NumIRArgs);
1705
0
      break;
1706
2.04M
    }
1707
2.04M
  }
1708
1709
1.16M
  bool Erased = FunctionsBeingProcessed.erase(&FI); (void)Erased;
1710
1.16M
  assert(Erased && "Not in set?");
1711
1712
0
  return llvm::FunctionType::get(resultType, ArgTypes, FI.isVariadic());
1713
1.16M
}
1714
1715
5.80k
llvm::Type *CodeGenTypes::GetFunctionTypeForVTable(GlobalDecl GD) {
1716
5.80k
  const CXXMethodDecl *MD = cast<CXXMethodDecl>(GD.getDecl());
1717
5.80k
  const FunctionProtoType *FPT = MD->getType()->getAs<FunctionProtoType>();
1718
1719
5.80k
  if (!isFuncTypeConvertible(FPT))
1720
39
    return llvm::StructType::get(getLLVMContext());
1721
1722
5.76k
  return GetFunctionType(GD);
1723
5.80k
}
1724
1725
static void AddAttributesFromFunctionProtoType(ASTContext &Ctx,
1726
                                               llvm::AttrBuilder &FuncAttrs,
1727
1.51M
                                               const FunctionProtoType *FPT) {
1728
1.51M
  if (!FPT)
1729
790k
    return;
1730
1731
723k
  if (!isUnresolvedExceptionSpec(FPT->getExceptionSpecType()) &&
1732
723k
      FPT->isNothrow())
1733
194k
    FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
1734
723k
}
1735
1736
bool CodeGenModule::MayDropFunctionReturn(const ASTContext &Context,
1737
48
                                          QualType ReturnType) {
1738
  // We can't just discard the return value for a record type with a
1739
  // complex destructor or a non-trivially copyable type.
1740
48
  if (const RecordType *RT =
1741
48
          ReturnType.getCanonicalType()->getAs<RecordType>()) {
1742
24
    if (const auto *ClassDecl = dyn_cast<CXXRecordDecl>(RT->getDecl()))
1743
24
      return ClassDecl->hasTrivialDestructor();
1744
24
  }
1745
24
  return ReturnType.isTriviallyCopyableType(Context);
1746
48
}
1747
1748
void CodeGenModule::getDefaultFunctionAttributes(StringRef Name,
1749
                                                 bool HasOptnone,
1750
                                                 bool AttrOnCallSite,
1751
789k
                                               llvm::AttrBuilder &FuncAttrs) {
1752
  // OptimizeNoneAttr takes precedence over -Os or -Oz. No warning needed.
1753
789k
  if (!HasOptnone) {
1754
789k
    if (CodeGenOpts.OptimizeSize)
1755
1.13k
      FuncAttrs.addAttribute(llvm::Attribute::OptimizeForSize);
1756
789k
    if (CodeGenOpts.OptimizeSize == 2)
1757
718
      FuncAttrs.addAttribute(llvm::Attribute::MinSize);
1758
789k
  }
1759
1760
789k
  if (CodeGenOpts.DisableRedZone)
1761
25
    FuncAttrs.addAttribute(llvm::Attribute::NoRedZone);
1762
789k
  if (CodeGenOpts.IndirectTlsSegRefs)
1763
1
    FuncAttrs.addAttribute("indirect-tls-seg-refs");
1764
789k
  if (CodeGenOpts.NoImplicitFloat)
1765
16
    FuncAttrs.addAttribute(llvm::Attribute::NoImplicitFloat);
1766
1767
789k
  if (AttrOnCallSite) {
1768
    // Attributes that should go on the call site only.
1769
382k
    if (!CodeGenOpts.SimplifyLibCalls || 
LangOpts.isNoBuiltinFunc(Name)191k
)
1770
190k
      FuncAttrs.addAttribute(llvm::Attribute::NoBuiltin);
1771
382k
    if (!CodeGenOpts.TrapFuncName.empty())
1772
0
      FuncAttrs.addAttribute("trap-func-name", CodeGenOpts.TrapFuncName);
1773
407k
  } else {
1774
407k
    StringRef FpKind;
1775
407k
    switch (CodeGenOpts.getFramePointer()) {
1776
267k
    case CodeGenOptions::FramePointerKind::None:
1777
267k
      FpKind = "none";
1778
267k
      break;
1779
212
    case CodeGenOptions::FramePointerKind::NonLeaf:
1780
212
      FpKind = "non-leaf";
1781
212
      break;
1782
139k
    case CodeGenOptions::FramePointerKind::All:
1783
139k
      FpKind = "all";
1784
139k
      break;
1785
407k
    }
1786
407k
    FuncAttrs.addAttribute("frame-pointer", FpKind);
1787
1788
407k
    if (CodeGenOpts.LessPreciseFPMAD)
1789
8
      FuncAttrs.addAttribute("less-precise-fpmad", "true");
1790
1791
407k
    if (CodeGenOpts.NullPointerIsValid)
1792
238
      FuncAttrs.addAttribute(llvm::Attribute::NullPointerIsValid);
1793
1794
407k
    if (CodeGenOpts.FPDenormalMode != llvm::DenormalMode::getIEEE())
1795
2
      FuncAttrs.addAttribute("denormal-fp-math",
1796
2
                             CodeGenOpts.FPDenormalMode.str());
1797
407k
    if (CodeGenOpts.FP32DenormalMode != CodeGenOpts.FPDenormalMode) {
1798
16
      FuncAttrs.addAttribute(
1799
16
          "denormal-fp-math-f32",
1800
16
          CodeGenOpts.FP32DenormalMode.str());
1801
16
    }
1802
1803
407k
    if (LangOpts.getFPExceptionMode() == LangOptions::FPE_Ignore)
1804
405k
      FuncAttrs.addAttribute("no-trapping-math", "true");
1805
1806
    // Strict (compliant) code is the default, so only add this attribute to
1807
    // indicate that we are trying to workaround a problem case.
1808
407k
    if (!CodeGenOpts.StrictFloatCastOverflow)
1809
1
      FuncAttrs.addAttribute("strict-float-cast-overflow", "false");
1810
1811
    // TODO: Are these all needed?
1812
    // unsafe/inf/nan/nsz are handled by instruction-level FastMathFlags.
1813
407k
    if (LangOpts.NoHonorInfs)
1814
269
      FuncAttrs.addAttribute("no-infs-fp-math", "true");
1815
407k
    if (LangOpts.NoHonorNaNs)
1816
289
      FuncAttrs.addAttribute("no-nans-fp-math", "true");
1817
407k
    if (LangOpts.UnsafeFPMath)
1818
256
      FuncAttrs.addAttribute("unsafe-fp-math", "true");
1819
407k
    if (CodeGenOpts.SoftFloat)
1820
63
      FuncAttrs.addAttribute("use-soft-float", "true");
1821
407k
    FuncAttrs.addAttribute("stack-protector-buffer-size",
1822
407k
                           llvm::utostr(CodeGenOpts.SSPBufferSize));
1823
407k
    if (LangOpts.NoSignedZero)
1824
262
      FuncAttrs.addAttribute("no-signed-zeros-fp-math", "true");
1825
1826
    // TODO: Reciprocal estimate codegen options should apply to instructions?
1827
407k
    const std::vector<std::string> &Recips = CodeGenOpts.Reciprocals;
1828
407k
    if (!Recips.empty())
1829
1
      FuncAttrs.addAttribute("reciprocal-estimates",
1830
1
                             llvm::join(Recips, ","));
1831
1832
407k
    if (!CodeGenOpts.PreferVectorWidth.empty() &&
1833
407k
        
CodeGenOpts.PreferVectorWidth != "none"3
)
1834
2
      FuncAttrs.addAttribute("prefer-vector-width",
1835
2
                             CodeGenOpts.PreferVectorWidth);
1836
1837
407k
    if (CodeGenOpts.StackRealignment)
1838
1
      FuncAttrs.addAttribute("stackrealign");
1839
407k
    if (CodeGenOpts.Backchain)
1840
2
      FuncAttrs.addAttribute("backchain");
1841
407k
    if (CodeGenOpts.EnableSegmentedStacks)
1842
6
      FuncAttrs.addAttribute("split-stack");
1843
1844
407k
    if (CodeGenOpts.SpeculativeLoadHardening)
1845
9
      FuncAttrs.addAttribute(llvm::Attribute::SpeculativeLoadHardening);
1846
407k
  }
1847
1848
789k
  if (getLangOpts().assumeFunctionsAreConvergent()) {
1849
    // Conservatively, mark all functions and calls in CUDA and OpenCL as
1850
    // convergent (meaning, they may call an intrinsically convergent op, such
1851
    // as __syncthreads() / barrier(), and so can't have certain optimizations
1852
    // applied around them).  LLVM will remove this attribute where it safely
1853
    // can.
1854
132k
    FuncAttrs.addAttribute(llvm::Attribute::Convergent);
1855
132k
  }
1856
1857
789k
  if (getLangOpts().CUDA && 
getLangOpts().CUDAIsDevice1.92k
) {
1858
    // Exceptions aren't supported in CUDA device code.
1859
1.21k
    FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
1860
1.21k
  }
1861
1862
789k
  for (StringRef Attr : CodeGenOpts.DefaultFunctionAttrs) {
1863
40
    StringRef Var, Value;
1864
40
    std::tie(Var, Value) = Attr.split('=');
1865
40
    FuncAttrs.addAttribute(Var, Value);
1866
40
  }
1867
789k
}
1868
1869
23
void CodeGenModule::addDefaultFunctionDefinitionAttributes(llvm::Function &F) {
1870
23
  llvm::AttrBuilder FuncAttrs;
1871
23
  getDefaultFunctionAttributes(F.getName(), F.hasOptNone(),
1872
23
                               /* AttrOnCallSite = */ false, FuncAttrs);
1873
  // TODO: call GetCPUAndFeaturesAttributes?
1874
23
  F.addAttributes(llvm::AttributeList::FunctionIndex, FuncAttrs);
1875
23
}
1876
1877
void CodeGenModule::addDefaultFunctionDefinitionAttributes(
1878
0
                                                   llvm::AttrBuilder &attrs) {
1879
0
  getDefaultFunctionAttributes(/*function name*/ "", /*optnone*/ false,
1880
0
                               /*for call*/ false, attrs);
1881
0
  GetCPUAndFeaturesAttributes(GlobalDecl(), attrs);
1882
0
}
1883
1884
static void addNoBuiltinAttributes(llvm::AttrBuilder &FuncAttrs,
1885
                                   const LangOptions &LangOpts,
1886
789k
                                   const NoBuiltinAttr *NBA = nullptr) {
1887
789k
  auto AddNoBuiltinAttr = [&FuncAttrs](StringRef BuiltinName) {
1888
2.43k
    SmallString<32> AttributeName;
1889
2.43k
    AttributeName += "no-builtin-";
1890
2.43k
    AttributeName += BuiltinName;
1891
2.43k
    FuncAttrs.addAttribute(AttributeName);
1892
2.43k
  };
1893
1894
  // First, handle the language options passed through -fno-builtin.
1895
789k
  if (LangOpts.NoBuiltin) {
1896
    // -fno-builtin disables them all.
1897
435k
    FuncAttrs.addAttribute("no-builtins");
1898
435k
    return;
1899
435k
  }
1900
1901
  // Then, add attributes for builtins specified through -fno-builtin-<name>.
1902
353k
  llvm::for_each(LangOpts.NoBuiltinFuncs, AddNoBuiltinAttr);
1903
1904
  // Now, let's check the __attribute__((no_builtin("...")) attribute added to
1905
  // the source.
1906
353k
  if (!NBA)
1907
353k
    return;
1908
1909
  // If there is a wildcard in the builtin names specified through the
1910
  // attribute, disable them all.
1911
9
  if (llvm::is_contained(NBA->builtinNames(), "*")) {
1912
1
    FuncAttrs.addAttribute("no-builtins");
1913
1
    return;
1914
1
  }
1915
1916
  // And last, add the rest of the builtin names.
1917
8
  llvm::for_each(NBA->builtinNames(), AddNoBuiltinAttr);
1918
8
}
1919
1920
static bool DetermineNoUndef(QualType QTy, CodeGenTypes &Types,
1921
                             const llvm::DataLayout &DL, const ABIArgInfo &AI,
1922
1.65M
                             bool CheckCoerce = true) {
1923
1.65M
  llvm::Type *Ty = Types.ConvertTypeForMem(QTy);
1924
1.65M
  if (AI.getKind() == ABIArgInfo::Indirect)
1925
5.05k
    return true;
1926
1.64M
  if (AI.getKind() == ABIArgInfo::Extend)
1927
35.4k
    return true;
1928
1.61M
  if (!DL.typeSizeEqualsStoreSize(Ty))
1929
    // TODO: This will result in a modest amount of values not marked noundef
1930
    // when they could be. We care about values that *invisibly* contain undef
1931
    // bits from the perspective of LLVM IR.
1932
16.8k
    return false;
1933
1.59M
  if (CheckCoerce && 
AI.canHaveCoerceToType()1.45M
) {
1934
1.43M
    llvm::Type *CoerceTy = AI.getCoerceToType();
1935
1.43M
    if (llvm::TypeSize::isKnownGT(DL.getTypeSizeInBits(CoerceTy),
1936
1.43M
                                  DL.getTypeSizeInBits(Ty)))
1937
      // If we're coercing to a type with a greater size than the canonical one,
1938
      // we're introducing new undef bits.
1939
      // Coercing to a type of smaller or equal size is ok, as we know that
1940
      // there's no internal padding (typeSizeEqualsStoreSize).
1941
587
      return false;
1942
1.43M
  }
1943
1.59M
  if (QTy->isExtIntType())
1944
48
    return true;
1945
1.59M
  if (QTy->isReferenceType())
1946
138k
    return true;
1947
1.45M
  if (QTy->isNullPtrType())
1948
234
    return false;
1949
1.45M
  if (QTy->isMemberPointerType())
1950
    // TODO: Some member pointers are `noundef`, but it depends on the ABI. For
1951
    // now, never mark them.
1952
200
    return false;
1953
1.45M
  if (QTy->isScalarType()) {
1954
1.09M
    if (const ComplexType *Complex = dyn_cast<ComplexType>(QTy))
1955
1.11k
      return DetermineNoUndef(Complex->getElementType(), Types, DL, AI, false);
1956
1.09M
    return true;
1957
1.09M
  }
1958
359k
  if (const VectorType *Vector = dyn_cast<VectorType>(QTy))
1959
139k
    return DetermineNoUndef(Vector->getElementType(), Types, DL, AI, false);
1960
219k
  if (const MatrixType *Matrix = dyn_cast<MatrixType>(QTy))
1961
227
    return DetermineNoUndef(Matrix->getElementType(), Types, DL, AI, false);
1962
219k
  if (const ArrayType *Array = dyn_cast<ArrayType>(QTy))
1963
0
    return DetermineNoUndef(Array->getElementType(), Types, DL, AI, false);
1964
1965
  // TODO: Some structs may be `noundef`, in specific situations.
1966
219k
  return false;
1967
219k
}
1968
1969
/// Construct the IR attribute list of a function or call.
1970
///
1971
/// When adding an attribute, please consider where it should be handled:
1972
///
1973
///   - getDefaultFunctionAttributes is for attributes that are essentially
1974
///     part of the global target configuration (but perhaps can be
1975
///     overridden on a per-function basis).  Adding attributes there
1976
///     will cause them to also be set in frontends that build on Clang's
1977
///     target-configuration logic, as well as for code defined in library
1978
///     modules such as CUDA's libdevice.
1979
///
1980
///   - ConstructAttributeList builds on top of getDefaultFunctionAttributes
1981
///     and adds declaration-specific, convention-specific, and
1982
///     frontend-specific logic.  The last is of particular importance:
1983
///     attributes that restrict how the frontend generates code must be
1984
///     added here rather than getDefaultFunctionAttributes.
1985
///
1986
void CodeGenModule::ConstructAttributeList(StringRef Name,
1987
                                           const CGFunctionInfo &FI,
1988
                                           CGCalleeInfo CalleeInfo,
1989
                                           llvm::AttributeList &AttrList,
1990
                                           unsigned &CallingConv,
1991
789k
                                           bool AttrOnCallSite, bool IsThunk) {
1992
789k
  llvm::AttrBuilder FuncAttrs;
1993
789k
  llvm::AttrBuilder RetAttrs;
1994
1995
  // Collect function IR attributes from the CC lowering.
1996
  // We'll collect the paramete and result attributes later.
1997
789k
  CallingConv = FI.getEffectiveCallingConvention();
1998
789k
  if (FI.isNoReturn())
1999
1.23k
    FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
2000
789k
  if (FI.isCmseNSCall())
2001
62
    FuncAttrs.addAttribute("cmse_nonsecure_call");
2002
2003
  // Collect function IR attributes from the callee prototype if we have one.
2004
789k
  AddAttributesFromFunctionProtoType(getContext(), FuncAttrs,
2005
789k
                                     CalleeInfo.getCalleeFunctionProtoType());
2006
2007
789k
  const Decl *TargetDecl = CalleeInfo.getCalleeDecl().getDecl();
2008
2009
789k
  bool HasOptnone = false;
2010
  // The NoBuiltinAttr attached to the target FunctionDecl.
2011
789k
  const NoBuiltinAttr *NBA = nullptr;
2012
2013
  // Collect function IR attributes based on declaration-specific
2014
  // information.
2015
  // FIXME: handle sseregparm someday...
2016
789k
  if (TargetDecl) {
2017
757k
    if (TargetDecl->hasAttr<ReturnsTwiceAttr>())
2018
43
      FuncAttrs.addAttribute(llvm::Attribute::ReturnsTwice);
2019
757k
    if (TargetDecl->hasAttr<NoThrowAttr>())
2020
5.92k
      FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
2021
757k
    if (TargetDecl->hasAttr<NoReturnAttr>())
2022
0
      FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
2023
757k
    if (TargetDecl->hasAttr<ColdAttr>())
2024
321
      FuncAttrs.addAttribute(llvm::Attribute::Cold);
2025
757k
    if (TargetDecl->hasAttr<HotAttr>())
2026
12
      FuncAttrs.addAttribute(llvm::Attribute::Hot);
2027
757k
    if (TargetDecl->hasAttr<NoDuplicateAttr>())
2028
6
      FuncAttrs.addAttribute(llvm::Attribute::NoDuplicate);
2029
757k
    if (TargetDecl->hasAttr<ConvergentAttr>())
2030
6
      FuncAttrs.addAttribute(llvm::Attribute::Convergent);
2031
2032
757k
    if (const FunctionDecl *Fn = dyn_cast<FunctionDecl>(TargetDecl)) {
2033
724k
      AddAttributesFromFunctionProtoType(
2034
724k
          getContext(), FuncAttrs, Fn->getType()->getAs<FunctionProtoType>());
2035
724k
      if (AttrOnCallSite && 
Fn->isReplaceableGlobalAllocationFunction()359k
) {
2036
        // A sane operator new returns a non-aliasing pointer.
2037
4.03k
        auto Kind = Fn->getDeclName().getCXXOverloadedOperator();
2038
4.03k
        if (getCodeGenOpts().AssumeSaneOperatorNew &&
2039
4.03k
            
(4.02k
Kind == OO_New4.02k
||
Kind == OO_Array_New2.77k
))
2040
1.54k
          RetAttrs.addAttribute(llvm::Attribute::NoAlias);
2041
4.03k
      }
2042
724k
      const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Fn);
2043
724k
      const bool IsVirtualCall = MD && 
MD->isVirtual()288k
;
2044
      // Don't use [[noreturn]], _Noreturn or [[no_builtin]] for a call to a
2045
      // virtual function. These attributes are not inherited by overloads.
2046
724k
      if (!(AttrOnCallSite && 
IsVirtualCall359k
)) {
2047
719k
        if (Fn->isNoReturn())
2048
3.05k
          FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
2049
719k
        NBA = Fn->getAttr<NoBuiltinAttr>();
2050
719k
      }
2051
      // Only place nomerge attribute on call sites, never functions. This
2052
      // allows it to work on indirect virtual function calls.
2053
724k
      if (AttrOnCallSite && 
TargetDecl->hasAttr<NoMergeAttr>()359k
)
2054
12
        FuncAttrs.addAttribute(llvm::Attribute::NoMerge);
2055
2056
      // Add known guaranteed alignment for allocation functions.
2057
724k
      if (unsigned BuiltinID = Fn->getBuiltinID()) {
2058
6.67k
        switch (BuiltinID) {
2059
0
        case Builtin::BIaligned_alloc:
2060
12
        case Builtin::BIcalloc:
2061
79
        case Builtin::BImalloc:
2062
79
        case Builtin::BImemalign:
2063
91
        case Builtin::BIrealloc:
2064
91
        case Builtin::BIstrdup:
2065
91
        case Builtin::BIstrndup:
2066
91
          RetAttrs.addAlignmentAttr(Context.getTargetInfo().getNewAlign() /
2067
91
                                    Context.getTargetInfo().getCharWidth());
2068
91
          break;
2069
6.58k
        default:
2070
6.58k
          break;
2071
6.67k
        }
2072
6.67k
      }
2073
724k
    }
2074
2075
    // 'const', 'pure' and 'noalias' attributed functions are also nounwind.
2076
757k
    if (TargetDecl->hasAttr<ConstAttr>()) {
2077
1.49k
      FuncAttrs.addAttribute(llvm::Attribute::ReadNone);
2078
1.49k
      FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
2079
      // gcc specifies that 'const' functions have greater restrictions than
2080
      // 'pure' functions, so they also cannot have infinite loops.
2081
1.49k
      FuncAttrs.addAttribute(llvm::Attribute::WillReturn);
2082
755k
    } else if (TargetDecl->hasAttr<PureAttr>()) {
2083
211
      FuncAttrs.addAttribute(llvm::Attribute::ReadOnly);
2084
211
      FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
2085
      // gcc specifies that 'pure' functions cannot have infinite loops.
2086
211
      FuncAttrs.addAttribute(llvm::Attribute::WillReturn);
2087
755k
    } else if (TargetDecl->hasAttr<NoAliasAttr>()) {
2088
2
      FuncAttrs.addAttribute(llvm::Attribute::ArgMemOnly);
2089
2
      FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
2090
2
    }
2091
757k
    if (TargetDecl->hasAttr<RestrictAttr>())
2092
54
      RetAttrs.addAttribute(llvm::Attribute::NoAlias);
2093
757k
    if (TargetDecl->hasAttr<ReturnsNonNullAttr>() &&
2094
757k
        
!CodeGenOpts.NullPointerIsValid2.39k
)
2095
2.39k
      RetAttrs.addAttribute(llvm::Attribute::NonNull);
2096
757k
    if (TargetDecl->hasAttr<AnyX86NoCallerSavedRegistersAttr>())
2097
14
      FuncAttrs.addAttribute("no_caller_saved_registers");
2098
757k
    if (TargetDecl->hasAttr<AnyX86NoCfCheckAttr>())
2099
8
      FuncAttrs.addAttribute(llvm::Attribute::NoCfCheck);
2100
757k
    if (TargetDecl->hasAttr<LeafAttr>())
2101
1
      FuncAttrs.addAttribute(llvm::Attribute::NoCallback);
2102
2103
757k
    HasOptnone = TargetDecl->hasAttr<OptimizeNoneAttr>();
2104
757k
    if (auto *AllocSize = TargetDecl->getAttr<AllocSizeAttr>()) {
2105
2.70k
      Optional<unsigned> NumElemsParam;
2106
2.70k
      if (AllocSize->getNumElemsParam().isValid())
2107
149
        NumElemsParam = AllocSize->getNumElemsParam().getLLVMIndex();
2108
2.70k
      FuncAttrs.addAllocSizeAttr(AllocSize->getElemSizeParam().getLLVMIndex(),
2109
2.70k
                                 NumElemsParam);
2110
2.70k
    }
2111
2112
757k
    if (TargetDecl->hasAttr<OpenCLKernelAttr>()) {
2113
501
      if (getLangOpts().OpenCLVersion <= 120) {
2114
        // OpenCL v1.2 Work groups are always uniform
2115
339
        FuncAttrs.addAttribute("uniform-work-group-size", "true");
2116
339
      } else {
2117
        // OpenCL v2.0 Work groups may be whether uniform or not.
2118
        // '-cl-uniform-work-group-size' compile option gets a hint
2119
        // to the compiler that the global work-size be a multiple of
2120
        // the work-group size specified to clEnqueueNDRangeKernel
2121
        // (i.e. work groups are uniform).
2122
162
        FuncAttrs.addAttribute("uniform-work-group-size",
2123
162
                               llvm::toStringRef(CodeGenOpts.UniformWGSize));
2124
162
      }
2125
501
    }
2126
2127
757k
    std::string AssumptionValueStr;
2128
757k
    for (AssumptionAttr *AssumptionA :
2129
757k
         TargetDecl->specific_attrs<AssumptionAttr>()) {
2130
467
      std::string AS = AssumptionA->getAssumption().str();
2131
467
      if (!AS.empty() && !AssumptionValueStr.empty())
2132
283
        AssumptionValueStr += ",";
2133
467
      AssumptionValueStr += AS;
2134
467
    }
2135
2136
757k
    if (!AssumptionValueStr.empty())
2137
184
      FuncAttrs.addAttribute(llvm::AssumptionAttrKey, AssumptionValueStr);
2138
757k
  }
2139
2140
  // Attach "no-builtins" attributes to:
2141
  // * call sites: both `nobuiltin` and "no-builtins" or "no-builtin-<name>".
2142
  // * definitions: "no-builtins" or "no-builtin-<name>" only.
2143
  // The attributes can come from:
2144
  // * LangOpts: -ffreestanding, -fno-builtin, -fno-builtin-<name>
2145
  // * FunctionDecl attributes: __attribute__((no_builtin(...)))
2146
789k
  addNoBuiltinAttributes(FuncAttrs, getLangOpts(), NBA);
2147
2148
  // Collect function IR attributes based on global settiings.
2149
789k
  getDefaultFunctionAttributes(Name, HasOptnone, AttrOnCallSite, FuncAttrs);
2150
2151
  // Override some default IR attributes based on declaration-specific
2152
  // information.
2153
789k
  if (TargetDecl) {
2154
757k
    if (TargetDecl->hasAttr<NoSpeculativeLoadHardeningAttr>())
2155
13
      FuncAttrs.removeAttribute(llvm::Attribute::SpeculativeLoadHardening);
2156
757k
    if (TargetDecl->hasAttr<SpeculativeLoadHardeningAttr>())
2157
13
      FuncAttrs.addAttribute(llvm::Attribute::SpeculativeLoadHardening);
2158
757k
    if (TargetDecl->hasAttr<NoSplitStackAttr>())
2159
8
      FuncAttrs.removeAttribute("split-stack");
2160
2161
    // Add NonLazyBind attribute to function declarations when -fno-plt
2162
    // is used.
2163
    // FIXME: what if we just haven't processed the function definition
2164
    // yet, or if it's an external definition like C99 inline?
2165
757k
    if (CodeGenOpts.NoPLT) {
2166
45
      if (auto *Fn = dyn_cast<FunctionDecl>(TargetDecl)) {
2167
45
        if (!Fn->isDefined() && 
!AttrOnCallSite22
) {
2168
12
          FuncAttrs.addAttribute(llvm::Attribute::NonLazyBind);
2169
12
        }
2170
45
      }
2171
45
    }
2172
757k
  }
2173
2174
  // Add "sample-profile-suffix-elision-policy" attribute for internal linkage
2175
  // functions with -funique-internal-linkage-names.
2176
789k
  if (TargetDecl && 
CodeGenOpts.UniqueInternalLinkageNames757k
) {
2177
47
    if (isa<FunctionDecl>(TargetDecl)) {
2178
47
      if (this->getFunctionLinkage(CalleeInfo.getCalleeDecl()) ==
2179
47
          llvm::GlobalValue::InternalLinkage)
2180
34
        FuncAttrs.addAttribute("sample-profile-suffix-elision-policy",
2181
34
                               "selected");
2182
47
    }
2183
47
  }
2184
2185
  // Collect non-call-site function IR attributes from declaration-specific
2186
  // information.
2187
789k
  if (!AttrOnCallSite) {
2188
407k
    if (TargetDecl && 
TargetDecl->hasAttr<CmseNSEntryAttr>()392k
)
2189
94
      FuncAttrs.addAttribute("cmse_nonsecure_entry");
2190
2191
    // Whether tail calls are enabled.
2192
407k
    auto shouldDisableTailCalls = [&] {
2193
      // Should this be honored in getDefaultFunctionAttributes?
2194
407k
      if (CodeGenOpts.DisableTailCalls)
2195
3
        return true;
2196
2197
407k
      if (!TargetDecl)
2198
14.8k
        return false;
2199
2200
392k
      if (TargetDecl->hasAttr<DisableTailCallsAttr>() ||
2201
392k
          
TargetDecl->hasAttr<AnyX86InterruptAttr>()392k
)
2202
147
        return true;
2203
2204
392k
      if (CodeGenOpts.NoEscapingBlockTailCalls) {
2205
17
        if (const auto *BD = dyn_cast<BlockDecl>(TargetDecl))
2206
8
          if (!BD->doesNotEscape())
2207
4
            return true;
2208
17
      }
2209
2210
392k
      return false;
2211
392k
    };
2212
407k
    if (shouldDisableTailCalls())
2213
154
      FuncAttrs.addAttribute("disable-tail-calls", "true");
2214
2215
    // CPU/feature overrides.  addDefaultFunctionDefinitionAttributes
2216
    // handles these separately to set them based on the global defaults.
2217
407k
    GetCPUAndFeaturesAttributes(CalleeInfo.getCalleeDecl(), FuncAttrs);
2218
407k
  }
2219
2220
  // Collect attributes from arguments and return values.
2221
789k
  ClangToLLVMArgMapping IRFunctionArgs(getContext(), FI);
2222
2223
789k
  QualType RetTy = FI.getReturnType();
2224
789k
  const ABIArgInfo &RetAI = FI.getReturnInfo();
2225
789k
  const llvm::DataLayout &DL = getDataLayout();
2226
2227
  // C++ explicitly makes returning undefined values UB. C's rule only applies
2228
  // to used values, so we never mark them noundef for now.
2229
789k
  bool HasStrictReturn = getLangOpts().CPlusPlus;
2230
789k
  if (TargetDecl) {
2231
757k
    if (const FunctionDecl *FDecl = dyn_cast<FunctionDecl>(TargetDecl))
2232
724k
      HasStrictReturn &= !FDecl->isExternC();
2233
32.8k
    else if (const VarDecl *VDecl = dyn_cast<VarDecl>(TargetDecl))
2234
      // Function pointer
2235
5.08k
      HasStrictReturn &= !VDecl->isExternC();
2236
757k
  }
2237
2238
  // We don't want to be too aggressive with the return checking, unless
2239
  // it's explicit in the code opts or we're using an appropriate sanitizer.
2240
  // Try to respect what the programmer intended.
2241
789k
  HasStrictReturn &= getCodeGenOpts().StrictReturn ||
2242
789k
                     
!MayDropFunctionReturn(getContext(), RetTy)27
||
2243
789k
                     
getLangOpts().Sanitize.has(SanitizerKind::Memory)21
||
2244
789k
                     
getLangOpts().Sanitize.has(SanitizerKind::Return)21
;
2245
2246
  // Determine if the return type could be partially undef
2247
789k
  if (CodeGenOpts.EnableNoundefAttrs && 
HasStrictReturn67
) {
2248
67
    if (!RetTy->isVoidType() && 
RetAI.getKind() != ABIArgInfo::Indirect39
&&
2249
67
        
DetermineNoUndef(RetTy, getTypes(), DL, RetAI)33
)
2250
19
      RetAttrs.addAttribute(llvm::Attribute::NoUndef);
2251
67
  }
2252
2253
789k
  switch (RetAI.getKind()) {
2254
25.6k
  case ABIArgInfo::Extend:
2255
25.6k
    if (RetAI.isSignExt())
2256
12.3k
      RetAttrs.addAttribute(llvm::Attribute::SExt);
2257
13.2k
    else
2258
13.2k
      RetAttrs.addAttribute(llvm::Attribute::ZExt);
2259
25.6k
    LLVM_FALLTHROUGH;
2260
451k
  case ABIArgInfo::Direct:
2261
451k
    if (RetAI.getInReg())
2262
34
      RetAttrs.addAttribute(llvm::Attribute::InReg);
2263
451k
    break;
2264
332k
  case ABIArgInfo::Ignore:
2265
332k
    break;
2266
2267
13
  case ABIArgInfo::InAlloca:
2268
4.91k
  case ABIArgInfo::Indirect: {
2269
    // inalloca and sret disable readnone and readonly
2270
4.91k
    FuncAttrs.removeAttribute(llvm::Attribute::ReadOnly)
2271
4.91k
      .removeAttribute(llvm::Attribute::ReadNone);
2272
4.91k
    break;
2273
13
  }
2274
2275
983
  case ABIArgInfo::CoerceAndExpand:
2276
983
    break;
2277
2278
0
  case ABIArgInfo::Expand:
2279
0
  case ABIArgInfo::IndirectAliased:
2280
0
    llvm_unreachable("Invalid ABI kind for return argument");
2281
789k
  }
2282
2283
789k
  if (!IsThunk) {
2284
    // FIXME: fix this properly, https://reviews.llvm.org/D100388
2285
788k
    if (const auto *RefTy = RetTy->getAs<ReferenceType>()) {
2286
68.0k
      QualType PTy = RefTy->getPointeeType();
2287
68.0k
      if (!PTy->isIncompleteType() && 
PTy->isConstantSizeType()67.9k
)
2288
67.9k
        RetAttrs.addDereferenceableAttr(
2289
67.9k
            getMinimumObjectSize(PTy).getQuantity());
2290
68.0k
      if (getContext().getTargetAddressSpace(PTy) == 0 &&
2291
68.0k
          
!CodeGenOpts.NullPointerIsValid67.9k
)
2292
67.9k
        RetAttrs.addAttribute(llvm::Attribute::NonNull);
2293
68.0k
      if (PTy->isObjectType()) {
2294
67.3k
        llvm::Align Alignment =
2295
67.3k
            getNaturalPointeeTypeAlignment(RetTy).getAsAlign();
2296
67.3k
        RetAttrs.addAlignmentAttr(Alignment);
2297
67.3k
      }
2298
68.0k
    }
2299
788k
  }
2300
2301
789k
  bool hasUsedSRet = false;
2302
789k
  SmallVector<llvm::AttributeSet, 4> ArgAttrs(IRFunctionArgs.totalIRArgs());
2303
2304
  // Attach attributes to sret.
2305
789k
  if (IRFunctionArgs.hasSRetArg()) {
2306
4.89k
    llvm::AttrBuilder SRETAttrs;
2307
4.89k
    SRETAttrs.addStructRetAttr(getTypes().ConvertTypeForMem(RetTy));
2308
4.89k
    hasUsedSRet = true;
2309
4.89k
    if (RetAI.getInReg())
2310
67
      SRETAttrs.addAttribute(llvm::Attribute::InReg);
2311
4.89k
    SRETAttrs.addAlignmentAttr(RetAI.getIndirectAlign().getQuantity());
2312
4.89k
    ArgAttrs[IRFunctionArgs.getSRetArgNo()] =
2313
4.89k
        llvm::AttributeSet::get(getLLVMContext(), SRETAttrs);
2314
4.89k
  }
2315
2316
  // Attach attributes to inalloca argument.
2317
789k
  if (IRFunctionArgs.hasInallocaArg()) {
2318
160
    llvm::AttrBuilder Attrs;
2319
160
    Attrs.addInAllocaAttr(FI.getArgStruct());
2320
160
    ArgAttrs[IRFunctionArgs.getInallocaArgNo()] =
2321
160
        llvm::AttributeSet::get(getLLVMContext(), Attrs);
2322
160
  }
2323
2324
  // Apply `nonnull`, `dereferencable(N)` and `align N` to the `this` argument,
2325
  // unless this is a thunk function.
2326
  // FIXME: fix this properly, https://reviews.llvm.org/D100388
2327
789k
  if (FI.isInstanceMethod() && 
!IRFunctionArgs.hasInallocaArg()272k
&&
2328
789k
      
!FI.arg_begin()->type->isVoidPointerType()272k
&&
!IsThunk272k
) {
2329
271k
    auto IRArgs = IRFunctionArgs.getIRArgs(0);
2330
2331
271k
    assert(IRArgs.second == 1 && "Expected only a single `this` pointer.");
2332
2333
0
    llvm::AttrBuilder Attrs;
2334
2335
271k
    QualType ThisTy =
2336
271k
        FI.arg_begin()->type.castAs<PointerType>()->getPointeeType();
2337
2338
271k
    if (!CodeGenOpts.NullPointerIsValid &&
2339
271k
        
getContext().getTargetAddressSpace(FI.arg_begin()->type) == 0271k
) {
2340
271k
      Attrs.addAttribute(llvm::Attribute::NonNull);
2341
271k
      Attrs.addDereferenceableAttr(getMinimumObjectSize(ThisTy).getQuantity());
2342
271k
    } else {
2343
      // FIXME dereferenceable should be correct here, regardless of
2344
      // NullPointerIsValid. However, dereferenceable currently does not always
2345
      // respect NullPointerIsValid and may imply nonnull and break the program.
2346
      // See https://reviews.llvm.org/D66618 for discussions.
2347
318
      Attrs.addDereferenceableOrNullAttr(
2348
318
          getMinimumObjectSize(
2349
318
              FI.arg_begin()->type.castAs<PointerType>()->getPointeeType())
2350
318
              .getQuantity());
2351
318
    }
2352
2353
271k
    llvm::Align Alignment =
2354
271k
        getNaturalTypeAlignment(ThisTy, /*BaseInfo=*/nullptr,
2355
271k
                                /*TBAAInfo=*/nullptr, /*forPointeeType=*/true)
2356
271k
            .getAsAlign();
2357
271k
    Attrs.addAlignmentAttr(Alignment);
2358
2359
271k
    ArgAttrs[IRArgs.first] = llvm::AttributeSet::get(getLLVMContext(), Attrs);
2360
271k
  }
2361
2362
0
  unsigned ArgNo = 0;
2363
789k
  for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(),
2364
789k
                                          E = FI.arg_end();
2365
2.30M
       I != E; 
++I, ++ArgNo1.51M
) {
2366
1.51M
    QualType ParamType = I->type;
2367
1.51M
    const ABIArgInfo &AI = I->info;
2368
1.51M
    llvm::AttrBuilder Attrs;
2369
2370
    // Add attribute for padding argument, if necessary.
2371
1.51M
    if (IRFunctionArgs.hasPaddingArg(ArgNo)) {
2372
47
      if (AI.getPaddingInReg()) {
2373
5
        ArgAttrs[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
2374
5
            llvm::AttributeSet::get(
2375
5
                getLLVMContext(),
2376
5
                llvm::AttrBuilder().addAttribute(llvm::Attribute::InReg));
2377
5
      }
2378
47
    }
2379
2380
    // Decide whether the argument we're handling could be partially undef
2381
1.51M
    bool ArgNoUndef = DetermineNoUndef(ParamType, getTypes(), DL, AI);
2382
1.51M
    if (CodeGenOpts.EnableNoundefAttrs && 
ArgNoUndef40
)
2383
27
      Attrs.addAttribute(llvm::Attribute::NoUndef);
2384
2385
    // 'restrict' -> 'noalias' is done in EmitFunctionProlog when we
2386
    // have the corresponding parameter variable.  It doesn't make
2387
    // sense to do it here because parameters are so messed up.
2388
1.51M
    switch (AI.getKind()) {
2389
35.4k
    case ABIArgInfo::Extend:
2390
35.4k
      if (AI.isSignExt())
2391
17.4k
        Attrs.addAttribute(llvm::Attribute::SExt);
2392
17.9k
      else
2393
17.9k
        Attrs.addAttribute(llvm::Attribute::ZExt);
2394
35.4k
      LLVM_FALLTHROUGH;
2395
1.49M
    case ABIArgInfo::Direct:
2396
1.49M
      if (ArgNo == 0 && 
FI.isChainCall()716k
)
2397
8
        Attrs.addAttribute(llvm::Attribute::Nest);
2398
1.49M
      else if (AI.getInReg())
2399
586
        Attrs.addAttribute(llvm::Attribute::InReg);
2400
1.49M
      Attrs.addStackAlignmentAttr(llvm::MaybeAlign(AI.getDirectAlign()));
2401
1.49M
      break;
2402
2403
5.05k
    case ABIArgInfo::Indirect: {
2404
5.05k
      if (AI.getInReg())
2405
24
        Attrs.addAttribute(llvm::Attribute::InReg);
2406
2407
5.05k
      if (AI.getIndirectByVal())
2408
1.99k
        Attrs.addByValAttr(getTypes().ConvertTypeForMem(ParamType));
2409
2410
5.05k
      auto *Decl = ParamType->getAsRecordDecl();
2411
5.05k
      if (CodeGenOpts.PassByValueIsNoAlias && 
Decl7
&&
2412
5.05k
          
Decl->getArgPassingRestrictions() == RecordDecl::APK_CanPassInRegs7
)
2413
        // When calling the function, the pointer passed in will be the only
2414
        // reference to the underlying object. Mark it accordingly.
2415
5
        Attrs.addAttribute(llvm::Attribute::NoAlias);
2416
2417
      // TODO: We could add the byref attribute if not byval, but it would
2418
      // require updating many testcases.
2419
2420
5.05k
      CharUnits Align = AI.getIndirectAlign();
2421
2422
      // In a byval argument, it is important that the required
2423
      // alignment of the type is honored, as LLVM might be creating a
2424
      // *new* stack object, and needs to know what alignment to give
2425
      // it. (Sometimes it can deduce a sensible alignment on its own,
2426
      // but not if clang decides it must emit a packed struct, or the
2427
      // user specifies increased alignment requirements.)
2428
      //
2429
      // This is different from indirect *not* byval, where the object
2430
      // exists already, and the align attribute is purely
2431
      // informative.
2432
5.05k
      assert(!Align.isZero());
2433
2434
      // For now, only add this when we have a byval argument.
2435
      // TODO: be less lazy about updating test cases.
2436
5.05k
      if (AI.getIndirectByVal())
2437
1.99k
        Attrs.addAlignmentAttr(Align.getQuantity());
2438
2439
      // byval disables readnone and readonly.
2440
5.05k
      FuncAttrs.removeAttribute(llvm::Attribute::ReadOnly)
2441
5.05k
        .removeAttribute(llvm::Attribute::ReadNone);
2442
2443
5.05k
      break;
2444
35.4k
    }
2445
16
    case ABIArgInfo::IndirectAliased: {
2446
16
      CharUnits Align = AI.getIndirectAlign();
2447
16
      Attrs.addByRefAttr(getTypes().ConvertTypeForMem(ParamType));
2448
16
      Attrs.addAlignmentAttr(Align.getQuantity());
2449
16
      break;
2450
35.4k
    }
2451
14.4k
    case ABIArgInfo::Ignore:
2452
14.6k
    case ABIArgInfo::Expand:
2453
15.6k
    case ABIArgInfo::CoerceAndExpand:
2454
15.6k
      break;
2455
2456
317
    case ABIArgInfo::InAlloca:
2457
      // inalloca disables readnone and readonly.
2458
317
      FuncAttrs.removeAttribute(llvm::Attribute::ReadOnly)
2459
317
          .removeAttribute(llvm::Attribute::ReadNone);
2460
317
      continue;
2461
1.51M
    }
2462
2463
1.51M
    if (const auto *RefTy = ParamType->getAs<ReferenceType>()) {
2464
138k
      QualType PTy = RefTy->getPointeeType();
2465
138k
      if (!PTy->isIncompleteType() && 
PTy->isConstantSizeType()137k
)
2466
137k
        Attrs.addDereferenceableAttr(
2467
137k
            getMinimumObjectSize(PTy).getQuantity());
2468
138k
      if (getContext().getTargetAddressSpace(PTy) == 0 &&
2469
138k
          
!CodeGenOpts.NullPointerIsValid138k
)
2470
138k
        Attrs.addAttribute(llvm::Attribute::NonNull);
2471
138k
      if (PTy->isObjectType()) {
2472
136k
        llvm::Align Alignment =
2473
136k
            getNaturalPointeeTypeAlignment(ParamType).getAsAlign();
2474
136k
        Attrs.addAlignmentAttr(Alignment);
2475
136k
      }
2476
138k
    }
2477
2478
1.51M
    switch (FI.getExtParameterInfo(ArgNo).getABI()) {
2479
1.51M
    case ParameterABI::Ordinary:
2480
1.51M
      break;
2481
2482
48
    case ParameterABI::SwiftIndirectResult: {
2483
      // Add 'sret' if we haven't already used it for something, but
2484
      // only if the result is void.
2485
48
      if (!hasUsedSRet && 
RetTy->isVoidType()24
) {
2486
8
        Attrs.addStructRetAttr(getTypes().ConvertTypeForMem(ParamType));
2487
8
        hasUsedSRet = true;
2488
8
      }
2489
2490
      // Add 'noalias' in either case.
2491
48
      Attrs.addAttribute(llvm::Attribute::NoAlias);
2492
2493
      // Add 'dereferenceable' and 'alignment'.
2494
48
      auto PTy = ParamType->getPointeeType();
2495
48
      if (!PTy->isIncompleteType() && PTy->isConstantSizeType()) {
2496
48
        auto info = getContext().getTypeInfoInChars(PTy);
2497
48
        Attrs.addDereferenceableAttr(info.Width.getQuantity());
2498
48
        Attrs.addAlignmentAttr(info.Align.getAsAlign());
2499
48
      }
2500
48
      break;
2501
0
    }
2502
2503
24
    case ParameterABI::SwiftErrorResult:
2504
24
      Attrs.addAttribute(llvm::Attribute::SwiftError);
2505
24
      break;
2506
2507
40
    case ParameterABI::SwiftContext:
2508
40
      Attrs.addAttribute(llvm::Attribute::SwiftSelf);
2509
40
      break;
2510
2511
309
    case ParameterABI::SwiftAsyncContext:
2512
309
      Attrs.addAttribute(llvm::Attribute::SwiftAsync);
2513
309
      break;
2514
1.51M
    }
2515
2516
1.51M
    if (FI.getExtParameterInfo(ArgNo).isNoEscape())
2517
62
      Attrs.addAttribute(llvm::Attribute::NoCapture);
2518
2519
1.51M
    if (Attrs.hasAttributes()) {
2520
176k
      unsigned FirstIRArg, NumIRArgs;
2521
176k
      std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
2522
353k
      for (unsigned i = 0; i < NumIRArgs; 
i++176k
)
2523
176k
        ArgAttrs[FirstIRArg + i] =
2524
176k
            llvm::AttributeSet::get(getLLVMContext(), Attrs);
2525
176k
    }
2526
1.51M
  }
2527
789k
  assert(ArgNo == FI.arg_size());
2528
2529
0
  AttrList = llvm::AttributeList::get(
2530
789k
      getLLVMContext(), llvm::AttributeSet::get(getLLVMContext(), FuncAttrs),
2531
789k
      llvm::AttributeSet::get(getLLVMContext(), RetAttrs), ArgAttrs);
2532
789k
}
2533
2534
/// An argument came in as a promoted argument; demote it back to its
2535
/// declared type.
2536
static llvm::Value *emitArgumentDemotion(CodeGenFunction &CGF,
2537
                                         const VarDecl *var,
2538
14
                                         llvm::Value *value) {
2539
14
  llvm::Type *varType = CGF.ConvertType(var->getType());
2540
2541
  // This can happen with promotions that actually don't change the
2542
  // underlying type, like the enum promotions.
2543
14
  if (value->getType() == varType) 
return value0
;
2544
2545
14
  assert((varType->isIntegerTy() || varType->isFloatingPointTy())
2546
14
         && "unexpected promotion type");
2547
2548
14
  if (isa<llvm::IntegerType>(varType))
2549
8
    return CGF.Builder.CreateTrunc(value, varType, "arg.unpromote");
2550
2551
6
  return CGF.Builder.CreateFPCast(value, varType, "arg.unpromote");
2552
14
}
2553
2554
/// Returns the attribute (either parameter attribute, or function
2555
/// attribute), which declares argument ArgNo to be non-null.
2556
static const NonNullAttr *getNonNullAttr(const Decl *FD, const ParmVarDecl *PVD,
2557
184k
                                         QualType ArgType, unsigned ArgNo) {
2558
  // FIXME: __attribute__((nonnull)) can also be applied to:
2559
  //   - references to pointers, where the pointee is known to be
2560
  //     nonnull (apparently a Clang extension)
2561
  //   - transparent unions containing pointers
2562
  // In the former case, LLVM IR cannot represent the constraint. In
2563
  // the latter case, we have no guarantee that the transparent union
2564
  // is in fact passed as a pointer.
2565
184k
  if (!ArgType->isAnyPointerType() && 
!ArgType->isBlockPointerType()48.6k
)
2566
48.6k
    return nullptr;
2567
  // First, check attribute on parameter itself.
2568
135k
  if (PVD) {
2569
135k
    if (auto ParmNNAttr = PVD->getAttr<NonNullAttr>())
2570
10
      return ParmNNAttr;
2571
135k
  }
2572
  // Check function attributes.
2573
135k
  if (!FD)
2574
5
    return nullptr;
2575
135k
  for (const auto *NNAttr : FD->specific_attrs<NonNullAttr>()) {
2576
32
    if (NNAttr->isNonNull(ArgNo))
2577
28
      return NNAttr;
2578
32
  }
2579
135k
  return nullptr;
2580
135k
}
2581
2582
namespace {
2583
  struct CopyBackSwiftError final : EHScopeStack::Cleanup {
2584
    Address Temp;
2585
    Address Arg;
2586
16
    CopyBackSwiftError(Address temp, Address arg) : Temp(temp), Arg(arg) {}
2587
16
    void Emit(CodeGenFunction &CGF, Flags flags) override {
2588
16
      llvm::Value *errorValue = CGF.Builder.CreateLoad(Temp);
2589
16
      CGF.Builder.CreateStore(errorValue, Arg);
2590
16
    }
2591
  };
2592
}
2593
2594
void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
2595
                                         llvm::Function *Fn,
2596
371k
                                         const FunctionArgList &Args) {
2597
371k
  if (CurCodeDecl && 
CurCodeDecl->hasAttr<NakedAttr>()363k
)
2598
    // Naked functions don't have prologues.
2599
7
    return;
2600
2601
  // If this is an implicit-return-zero function, go ahead and
2602
  // initialize the return value.  TODO: it might be nice to have
2603
  // a more general mechanism for this that didn't require synthesized
2604
  // return statements.
2605
371k
  if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(CurCodeDecl)) {
2606
328k
    if (FD->hasImplicitReturnZero()) {
2607
5.50k
      QualType RetTy = FD->getReturnType().getUnqualifiedType();
2608
5.50k
      llvm::Type* LLVMTy = CGM.getTypes().ConvertType(RetTy);
2609
5.50k
      llvm::Constant* Zero = llvm::Constant::getNullValue(LLVMTy);
2610
5.50k
      Builder.CreateStore(Zero, ReturnValue);
2611
5.50k
    }
2612
328k
  }
2613
2614
  // FIXME: We no longer need the types from FunctionArgList; lift up and
2615
  // simplify.
2616
2617
371k
  ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), FI);
2618
371k
  assert(Fn->arg_size() == IRFunctionArgs.totalIRArgs());
2619
2620
  // If we're using inalloca, all the memory arguments are GEPs off of the last
2621
  // parameter, which is a pointer to the complete memory area.
2622
0
  Address ArgStruct = Address::invalid();
2623
371k
  if (IRFunctionArgs.hasInallocaArg()) {
2624
41
    ArgStruct = Address(Fn->getArg(IRFunctionArgs.getInallocaArgNo()),
2625
41
                        FI.getArgStructAlignment());
2626
2627
41
    assert(ArgStruct.getType() == FI.getArgStruct()->getPointerTo());
2628
41
  }
2629
2630
  // Name the struct return parameter.
2631
371k
  if (IRFunctionArgs.hasSRetArg()) {
2632
2.91k
    auto AI = Fn->getArg(IRFunctionArgs.getSRetArgNo());
2633
2.91k
    AI->setName("agg.result");
2634
2.91k
    AI->addAttr(llvm::Attribute::NoAlias);
2635
2.91k
  }
2636
2637
  // Track if we received the parameter as a pointer (indirect, byval, or
2638
  // inalloca).  If already have a pointer, EmitParmDecl doesn't need to copy it
2639
  // into a local alloca for us.
2640
371k
  SmallVector<ParamValue, 16> ArgVals;
2641
371k
  ArgVals.reserve(Args.size());
2642
2643
  // Create a pointer value for every parameter declaration.  This usually
2644
  // entails copying one or more LLVM IR arguments into an alloca.  Don't push
2645
  // any cleanups or do anything that might unwind.  We do that separately, so
2646
  // we can push the cleanups in the correct order for the ABI.
2647
371k
  assert(FI.arg_size() == Args.size() &&
2648
371k
         "Mismatch between function signature & arguments.");
2649
0
  unsigned ArgNo = 0;
2650
371k
  CGFunctionInfo::const_arg_iterator info_it = FI.arg_begin();
2651
371k
  for (FunctionArgList::const_iterator i = Args.begin(), e = Args.end();
2652
1.19M
       i != e; 
++i, ++info_it, ++ArgNo819k
) {
2653
819k
    const VarDecl *Arg = *i;
2654
819k
    const ABIArgInfo &ArgI = info_it->info;
2655
2656
819k
    bool isPromoted =
2657
819k
      isa<ParmVarDecl>(Arg) && 
cast<ParmVarDecl>(Arg)->isKNRPromoted()627k
;
2658
    // We are converting from ABIArgInfo type to VarDecl type directly, unless
2659
    // the parameter is promoted. In this case we convert to
2660
    // CGFunctionInfo::ArgInfo type with subsequent argument demotion.
2661
819k
    QualType Ty = isPromoted ? 
info_it->type14
:
Arg->getType()819k
;
2662
819k
    assert(hasScalarEvaluationKind(Ty) ==
2663
819k
           hasScalarEvaluationKind(Arg->getType()));
2664
2665
0
    unsigned FirstIRArg, NumIRArgs;
2666
819k
    std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
2667
2668
819k
    switch (ArgI.getKind()) {
2669
73
    case ABIArgInfo::InAlloca: {
2670
73
      assert(NumIRArgs == 0);
2671
0
      auto FieldIndex = ArgI.getInAllocaFieldIndex();
2672
73
      Address V =
2673
73
          Builder.CreateStructGEP(ArgStruct, FieldIndex, Arg->getName());
2674
73
      if (ArgI.getInAllocaIndirect())
2675
6
        V = Address(Builder.CreateLoad(V),
2676
6
                    getContext().getTypeAlignInChars(Ty));
2677
73
      ArgVals.push_back(ParamValue::forIndirect(V));
2678
73
      break;
2679
0
    }
2680
2681
2.49k
    case ABIArgInfo::Indirect:
2682
2.50k
    case ABIArgInfo::IndirectAliased: {
2683
2.50k
      assert(NumIRArgs == 1);
2684
0
      Address ParamAddr =
2685
2.50k
          Address(Fn->getArg(FirstIRArg), ArgI.getIndirectAlign());
2686
2687
2.50k
      if (!hasScalarEvaluationKind(Ty)) {
2688
        // Aggregates and complex variables are accessed by reference. All we
2689
        // need to do is realign the value, if requested. Also, if the address
2690
        // may be aliased, copy it to ensure that the parameter variable is
2691
        // mutable and has a unique adress, as C requires.
2692
2.12k
        Address V = ParamAddr;
2693
2.12k
        if (ArgI.getIndirectRealign() || 
ArgI.isIndirectAliased()2.09k
) {
2694
45
          Address AlignedTemp = CreateMemTemp(Ty, "coerce");
2695
2696
          // Copy from the incoming argument pointer to the temporary with the
2697
          // appropriate alignment.
2698
          //
2699
          // FIXME: We should have a common utility for generating an aggregate
2700
          // copy.
2701
45
          CharUnits Size = getContext().getTypeSizeInChars(Ty);
2702
45
          Builder.CreateMemCpy(
2703
45
              AlignedTemp.getPointer(), AlignedTemp.getAlignment().getAsAlign(),
2704
45
              ParamAddr.getPointer(), ParamAddr.getAlignment().getAsAlign(),
2705
45
              llvm::ConstantInt::get(IntPtrTy, Size.getQuantity()));
2706
45
          V = AlignedTemp;
2707
45
        }
2708
2.12k
        ArgVals.push_back(ParamValue::forIndirect(V));
2709
2.12k
      } else {
2710
        // Load scalar value from indirect argument.
2711
387
        llvm::Value *V =
2712
387
            EmitLoadOfScalar(ParamAddr, false, Ty, Arg->getBeginLoc());
2713
2714
387
        if (isPromoted)
2715
0
          V = emitArgumentDemotion(*this, Arg, V);
2716
387
        ArgVals.push_back(ParamValue::forDirect(V));
2717
387
      }
2718
2.50k
      break;
2719
2.49k
    }
2720
2721
23.7k
    case ABIArgInfo::Extend:
2722
809k
    case ABIArgInfo::Direct: {
2723
809k
      auto AI = Fn->getArg(FirstIRArg);
2724
809k
      llvm::Type *LTy = ConvertType(Arg->getType());
2725
2726
      // Prepare parameter attributes. So far, only attributes for pointer
2727
      // parameters are prepared. See
2728
      // http://llvm.org/docs/LangRef.html#paramattrs.
2729
809k
      if (ArgI.getDirectOffset() == 0 && 
LTy->isPointerTy()809k
&&
2730
809k
          
ArgI.getCoerceToType()->isPointerTy()348k
) {
2731
348k
        assert(NumIRArgs == 1);
2732
2733
348k
        if (const ParmVarDecl *PVD = dyn_cast<ParmVarDecl>(Arg)) {
2734
          // Set `nonnull` attribute if any.
2735
184k
          if (getNonNullAttr(CurCodeDecl, PVD, PVD->getType(),
2736
184k
                             PVD->getFunctionScopeIndex()) &&
2737
184k
              
!CGM.getCodeGenOpts().NullPointerIsValid22
)
2738
13
            AI->addAttr(llvm::Attribute::NonNull);
2739
2740
184k
          QualType OTy = PVD->getOriginalType();
2741
184k
          if (const auto *ArrTy =
2742
184k
              getContext().getAsConstantArrayType(OTy)) {
2743
            // A C99 array parameter declaration with the static keyword also
2744
            // indicates dereferenceability, and if the size is constant we can
2745
            // use the dereferenceable attribute (which requires the size in
2746
            // bytes).
2747
551
            if (ArrTy->getSizeModifier() == ArrayType::Static) {
2748
6
              QualType ETy = ArrTy->getElementType();
2749
6
              llvm::Align Alignment =
2750
6
                  CGM.getNaturalTypeAlignment(ETy).getAsAlign();
2751
6
              AI->addAttrs(llvm::AttrBuilder().addAlignmentAttr(Alignment));
2752
6
              uint64_t ArrSize = ArrTy->getSize().getZExtValue();
2753
6
              if (!ETy->isIncompleteType() && ETy->isConstantSizeType() &&
2754
6
                  ArrSize) {
2755
4
                llvm::AttrBuilder Attrs;
2756
4
                Attrs.addDereferenceableAttr(
2757
4
                    getContext().getTypeSizeInChars(ETy).getQuantity() *
2758
4
                    ArrSize);
2759
4
                AI->addAttrs(Attrs);
2760
4
              } else 
if (2
getContext().getTargetInfo().getNullPointerValue(
2761
2
                             ETy.getAddressSpace()) == 0 &&
2762
2
                         !CGM.getCodeGenOpts().NullPointerIsValid) {
2763
1
                AI->addAttr(llvm::Attribute::NonNull);
2764
1
              }
2765
6
            }
2766
183k
          } else if (const auto *ArrTy =
2767
183k
                     getContext().getAsVariableArrayType(OTy)) {
2768
            // For C99 VLAs with the static keyword, we don't know the size so
2769
            // we can't use the dereferenceable attribute, but in addrspace(0)
2770
            // we know that it must be nonnull.
2771
102
            if (ArrTy->getSizeModifier() == VariableArrayType::Static) {
2772
2
              QualType ETy = ArrTy->getElementType();
2773
2
              llvm::Align Alignment =
2774
2
                  CGM.getNaturalTypeAlignment(ETy).getAsAlign();
2775
2
              AI->addAttrs(llvm::AttrBuilder().addAlignmentAttr(Alignment));
2776
2
              if (!getContext().getTargetAddressSpace(ETy) &&
2777
2
                  !CGM.getCodeGenOpts().NullPointerIsValid)
2778
1
                AI->addAttr(llvm::Attribute::NonNull);
2779
2
            }
2780
102
          }
2781
2782
          // Set `align` attribute if any.
2783
184k
          const auto *AVAttr = PVD->getAttr<AlignValueAttr>();
2784
184k
          if (!AVAttr)
2785
184k
            if (const auto *TOTy = dyn_cast<TypedefType>(OTy))
2786
6.89k
              AVAttr = TOTy->getDecl()->getAttr<AlignValueAttr>();
2787
184k
          if (AVAttr && 
!SanOpts.has(SanitizerKind::Alignment)13
) {
2788
            // If alignment-assumption sanitizer is enabled, we do *not* add
2789
            // alignment attribute here, but emit normal alignment assumption,
2790
            // so the UBSAN check could function.
2791
10
            llvm::ConstantInt *AlignmentCI =
2792
10
                cast<llvm::ConstantInt>(EmitScalarExpr(AVAttr->getAlignment()));
2793
10
            unsigned AlignmentInt =
2794
10
                AlignmentCI->getLimitedValue(llvm::Value::MaximumAlignment);
2795
10
            if (AI->getParamAlign().valueOrOne() < AlignmentInt) {
2796
10
              AI->removeAttr(llvm::Attribute::AttrKind::Alignment);
2797
10
              AI->addAttrs(llvm::AttrBuilder().addAlignmentAttr(
2798
10
                  llvm::Align(AlignmentInt)));
2799
10
            }
2800
10
          }
2801
184k
        }
2802
2803
        // Set 'noalias' if an argument type has the `restrict` qualifier.
2804
348k
        if (Arg->getType().isRestrictQualified())
2805
30.6k
          AI->addAttr(llvm::Attribute::NoAlias);
2806
348k
      }
2807
2808
      // Prepare the argument value. If we have the trivial case, handle it
2809
      // with no muss and fuss.
2810
809k
      if (!isa<llvm::StructType>(ArgI.getCoerceToType()) &&
2811
809k
          
ArgI.getCoerceToType() == ConvertType(Ty)807k
&&
2812
809k
          
ArgI.getDirectOffset() == 0803k
) {
2813
803k
        assert(NumIRArgs == 1);
2814
2815
        // LLVM expects swifterror parameters to be used in very restricted
2816
        // ways.  Copy the value into a less-restricted temporary.
2817
0
        llvm::Value *V = AI;
2818
803k
        if (FI.getExtParameterInfo(ArgNo).getABI()
2819
803k
              == ParameterABI::SwiftErrorResult) {
2820
16
          QualType pointeeTy = Ty->getPointeeType();
2821
16
          assert(pointeeTy->isPointerType());
2822
0
          Address temp =
2823
16
            CreateMemTemp(pointeeTy, getPointerAlign(), "swifterror.temp");
2824
16
          Address arg = Address(V, getContext().getTypeAlignInChars(pointeeTy));
2825
16
          llvm::Value *incomingErrorValue = Builder.CreateLoad(arg);
2826
16
          Builder.CreateStore(incomingErrorValue, temp);
2827
16
          V = temp.getPointer();
2828
2829
          // Push a cleanup to copy the value back at the end of the function.
2830
          // The convention does not guarantee that the value will be written
2831
          // back if the function exits with an unwind exception.
2832
16
          EHStack.pushCleanup<CopyBackSwiftError>(NormalCleanup, temp, arg);
2833
16
        }
2834
2835
        // Ensure the argument is the correct type.
2836
803k
        if (V->getType() != ArgI.getCoerceToType())
2837
0
          V = Builder.CreateBitCast(V, ArgI.getCoerceToType());
2838
2839
803k
        if (isPromoted)
2840
14
          V = emitArgumentDemotion(*this, Arg, V);
2841
2842
        // Because of merging of function types from multiple decls it is
2843
        // possible for the type of an argument to not match the corresponding
2844
        // type in the function type. Since we are codegening the callee
2845
        // in here, add a cast to the argument type.
2846
803k
        llvm::Type *LTy = ConvertType(Arg->getType());
2847
803k
        if (V->getType() != LTy)
2848
0
          V = Builder.CreateBitCast(V, LTy);
2849
2850
803k
        ArgVals.push_back(ParamValue::forDirect(V));
2851
803k
        break;
2852
803k
      }
2853
2854
      // VLST arguments are coerced to VLATs at the function boundary for
2855
      // ABI consistency. If this is a VLST that was coerced to
2856
      // a VLAT at the function boundary and the types match up, use
2857
      // llvm.experimental.vector.extract to convert back to the original
2858
      // VLST.
2859
6.01k
      if (auto *VecTyTo = dyn_cast<llvm::FixedVectorType>(ConvertType(Ty))) {
2860
1.01k
        auto *Coerced = Fn->getArg(FirstIRArg);
2861
1.01k
        if (auto *VecTyFrom =
2862
1.01k
                dyn_cast<llvm::ScalableVectorType>(Coerced->getType())) {
2863
35
          if (VecTyFrom->getElementType() == VecTyTo->getElementType()) {
2864
31
            llvm::Value *Zero = llvm::Constant::getNullValue(CGM.Int64Ty);
2865
2866
31
            assert(NumIRArgs == 1);
2867
0
            Coerced->setName(Arg->getName() + ".coerce");
2868
31
            ArgVals.push_back(ParamValue::forDirect(Builder.CreateExtractVector(
2869
31
                VecTyTo, Coerced, Zero, "castFixedSve")));
2870
31
            break;
2871
31
          }
2872
35
        }
2873
1.01k
      }
2874
2875
5.98k
      Address Alloca = CreateMemTemp(Ty, getContext().getDeclAlign(Arg),
2876
5.98k
                                     Arg->getName());
2877
2878
      // Pointer to store into.
2879
5.98k
      Address Ptr = emitAddressAtOffset(*this, Alloca, ArgI);
2880
2881
      // Fast-isel and the optimizer generally like scalar values better than
2882
      // FCAs, so we flatten them if this is safe to do for this argument.
2883
5.98k
      llvm::StructType *STy = dyn_cast<llvm::StructType>(ArgI.getCoerceToType());
2884
5.98k
      if (ArgI.isDirect() && ArgI.getCanBeFlattened() && 
STy5.67k
&&
2885
5.98k
          
STy->getNumElements() > 1993
) {
2886
966
        uint64_t SrcSize = CGM.getDataLayout().getTypeAllocSize(STy);
2887
966
        llvm::Type *DstTy = Ptr.getElementType();
2888
966
        uint64_t DstSize = CGM.getDataLayout().getTypeAllocSize(DstTy);
2889
2890
966
        Address AddrToStoreInto = Address::invalid();
2891
966
        if (SrcSize <= DstSize) {
2892
879
          AddrToStoreInto = Builder.CreateElementBitCast(Ptr, STy);
2893
879
        } else {
2894
87
          AddrToStoreInto =
2895
87
            CreateTempAlloca(STy, Alloca.getAlignment(), "coerce");
2896
87
        }
2897
2898
966
        assert(STy->getNumElements() == NumIRArgs);
2899
3.00k
        for (unsigned i = 0, e = STy->getNumElements(); i != e; 
++i2.04k
) {
2900
2.04k
          auto AI = Fn->getArg(FirstIRArg + i);
2901
2.04k
          AI->setName(Arg->getName() + ".coerce" + Twine(i));
2902
2.04k
          Address EltPtr = Builder.CreateStructGEP(AddrToStoreInto, i);
2903
2.04k
          Builder.CreateStore(AI, EltPtr);
2904
2.04k
        }
2905
2906
966
        if (SrcSize > DstSize) {
2907
87
          Builder.CreateMemCpy(Ptr, AddrToStoreInto, DstSize);
2908
87
        }
2909
2910
5.01k
      } else {
2911
        // Simple case, just do a coerced store of the argument into the alloca.
2912
5.01k
        assert(NumIRArgs == 1);
2913
0
        auto AI = Fn->getArg(FirstIRArg);
2914
5.01k
        AI->setName(Arg->getName() + ".coerce");
2915
5.01k
        CreateCoercedStore(AI, Ptr, /*DstIsVolatile=*/false, *this);
2916
5.01k
      }
2917
2918
      // Match to what EmitParmDecl is expecting for this type.
2919
5.98k
      if (CodeGenFunction::hasScalarEvaluationKind(Ty)) {
2920
1.28k
        llvm::Value *V =
2921
1.28k
            EmitLoadOfScalar(Alloca, false, Ty, Arg->getBeginLoc());
2922
1.28k
        if (isPromoted)
2923
0
          V = emitArgumentDemotion(*this, Arg, V);
2924
1.28k
        ArgVals.push_back(ParamValue::forDirect(V));
2925
4.69k
      } else {
2926
4.69k
        ArgVals.push_back(ParamValue::forIndirect(Alloca));
2927
4.69k
      }
2928
5.98k
      break;
2929
6.01k
    }
2930
2931
571
    case ABIArgInfo::CoerceAndExpand: {
2932
      // Reconstruct into a temporary.
2933
571
      Address alloca = CreateMemTemp(Ty, getContext().getDeclAlign(Arg));
2934
571
      ArgVals.push_back(ParamValue::forIndirect(alloca));
2935
2936
571
      auto coercionType = ArgI.getCoerceAndExpandType();
2937
571
      alloca = Builder.CreateElementBitCast(alloca, coercionType);
2938
2939
571
      unsigned argIndex = FirstIRArg;
2940
1.78k
      for (unsigned i = 0, e = coercionType->getNumElements(); i != e; 
++i1.21k
) {
2941
1.21k
        llvm::Type *eltType = coercionType->getElementType(i);
2942
1.21k
        if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType))
2943
13
          continue;
2944
2945
1.19k
        auto eltAddr = Builder.CreateStructGEP(alloca, i);
2946
1.19k
        auto elt = Fn->getArg(argIndex++);
2947
1.19k
        Builder.CreateStore(elt, eltAddr);
2948
1.19k
      }
2949
571
      assert(argIndex == FirstIRArg + NumIRArgs);
2950
0
      break;
2951
6.01k
    }
2952
2953
101
    case ABIArgInfo::Expand: {
2954
      // If this structure was expanded into multiple arguments then
2955
      // we need to create a temporary and reconstruct it from the
2956
      // arguments.
2957
101
      Address Alloca = CreateMemTemp(Ty, getContext().getDeclAlign(Arg));
2958
101
      LValue LV = MakeAddrLValue(Alloca, Ty);
2959
101
      ArgVals.push_back(ParamValue::forIndirect(Alloca));
2960
2961
101
      auto FnArgIter = Fn->arg_begin() + FirstIRArg;
2962
101
      ExpandTypeFromArgs(Ty, LV, FnArgIter);
2963
101
      assert(FnArgIter == Fn->arg_begin() + FirstIRArg + NumIRArgs);
2964
343
      for (unsigned i = 0, e = NumIRArgs; i != e; 
++i242
) {
2965
242
        auto AI = Fn->getArg(FirstIRArg + i);
2966
242
        AI->setName(Arg->getName() + "." + Twine(i));
2967
242
      }
2968
101
      break;
2969
6.01k
    }
2970
2971
7.07k
    case ABIArgInfo::Ignore:
2972
7.07k
      assert(NumIRArgs == 0);
2973
      // Initialize the local variable appropriately.
2974
7.07k
      if (!hasScalarEvaluationKind(Ty)) {
2975
7.07k
        ArgVals.push_back(ParamValue::forIndirect(CreateMemTemp(Ty)));
2976
7.07k
      } else {
2977
0
        llvm::Value *U = llvm::UndefValue::get(ConvertType(Arg->getType()));
2978
0
        ArgVals.push_back(ParamValue::forDirect(U));
2979
0
      }
2980
7.07k
      break;
2981
819k
    }
2982
819k
  }
2983
2984
371k
  if (getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) {
2985
31.3k
    for (int I = Args.size() - 1; I >= 0; 
--I18.0k
)
2986
18.0k
      EmitParmDecl(*Args[I], ArgVals[I], I + 1);
2987
358k
  } else {
2988
1.15M
    for (unsigned I = 0, E = Args.size(); I != E; 
++I801k
)
2989
801k
      EmitParmDecl(*Args[I], ArgVals[I], I + 1);
2990
358k
  }
2991
371k
}
2992
2993
8
static void eraseUnusedBitCasts(llvm::Instruction *insn) {
2994
12
  while (insn->use_empty()) {
2995
12
    llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(insn);
2996
12
    if (!bitcast) 
return8
;
2997
2998
    // This is "safe" because we would have used a ConstantExpr otherwise.
2999
4
    insn = cast<llvm::Instruction>(bitcast->getOperand(0));
3000
4
    bitcast->eraseFromParent();
3001
4
  }
3002
8
}
3003
3004
/// Try to emit a fused autorelease of a return result.
3005
static llvm::Value *tryEmitFusedAutoreleaseOfResult(CodeGenFunction &CGF,
3006
87
                                                    llvm::Value *result) {
3007
  // We must be immediately followed the cast.
3008
87
  llvm::BasicBlock *BB = CGF.Builder.GetInsertBlock();
3009
87
  if (BB->empty()) 
return nullptr0
;
3010
87
  if (&BB->back() != result) 
return nullptr24
;
3011
3012
63
  llvm::Type *resultType = result->getType();
3013
3014
  // result is in a BasicBlock and is therefore an Instruction.
3015
63
  llvm::Instruction *generator = cast<llvm::Instruction>(result);
3016
3017
63
  SmallVector<llvm::Instruction *, 4> InstsToKill;
3018
3019
  // Look for:
3020
  //  %generator = bitcast %type1* %generator2 to %type2*
3021
95
  while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(generator)) {
3022
    // We would have emitted this as a constant if the operand weren't
3023
    // an Instruction.
3024
32
    generator = cast<llvm::Instruction>(bitcast->getOperand(0));
3025
3026
    // Require the generator to be immediately followed by the cast.
3027
32
    if (generator->getNextNode() != bitcast)
3028
0
      return nullptr;
3029
3030
32
    InstsToKill.push_back(bitcast);
3031
32
  }
3032
3033
  // Look for:
3034
  //   %generator = call i8* @objc_retain(i8* %originalResult)
3035
  // or
3036
  //   %generator = call i8* @objc_retainAutoreleasedReturnValue(i8* %originalResult)
3037
63
  llvm::CallInst *call = dyn_cast<llvm::CallInst>(generator);
3038
63
  if (!call) 
return nullptr13
;
3039
3040
50
  bool doRetainAutorelease;
3041
3042
50
  if (call->getCalledOperand() == CGF.CGM.getObjCEntrypoints().objc_retain) {
3043
8
    doRetainAutorelease = true;
3044
42
  } else if (call->getCalledOperand() ==
3045
42
             CGF.CGM.getObjCEntrypoints().objc_retainAutoreleasedReturnValue) {
3046
19
    doRetainAutorelease = false;
3047
3048
    // If we emitted an assembly marker for this call (and the
3049
    // ARCEntrypoints field should have been set if so), go looking
3050
    // for that call.  If we can't find it, we can't do this
3051
    // optimization.  But it should always be the immediately previous
3052
    // instruction, unless we needed bitcasts around the call.
3053
19
    if (CGF.CGM.getObjCEntrypoints().retainAutoreleasedReturnValueMarker) {
3054
12
      llvm::Instruction *prev = call->getPrevNode();
3055
12
      assert(prev);
3056
12
      if (isa<llvm::BitCastInst>(prev)) {
3057
8
        prev = prev->getPrevNode();
3058
8
        assert(prev);
3059
8
      }
3060
0
      assert(isa<llvm::CallInst>(prev));
3061
0
      assert(cast<llvm::CallInst>(prev)->getCalledOperand() ==
3062
12
             CGF.CGM.getObjCEntrypoints().retainAutoreleasedReturnValueMarker);
3063
0
      InstsToKill.push_back(prev);
3064
12
    }
3065
23
  } else {
3066
23
    return nullptr;
3067
23
  }
3068
3069
27
  result = call->getArgOperand(0);
3070
27
  InstsToKill.push_back(call);
3071
3072
  // Keep killing bitcasts, for sanity.  Note that we no longer care
3073
  // about precise ordering as long as there's exactly one use.
3074
43
  while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(result)) {
3075
16
    if (!bitcast->hasOneUse()) 
break0
;
3076
16
    InstsToKill.push_back(bitcast);
3077
16
    result = bitcast->getOperand(0);
3078
16
  }
3079
3080
  // Delete all the unnecessary instructions, from latest to earliest.
3081
27
  for (auto *I : InstsToKill)
3082
77
    I->eraseFromParent();
3083
3084
  // Do the fused retain/autorelease if we were asked to.
3085
27
  if (doRetainAutorelease)
3086
8
    result = CGF.EmitARCRetainAutoreleaseReturnValue(result);
3087
3088
  // Cast back to the result type.
3089
27
  return CGF.Builder.CreateBitCast(result, resultType);
3090
50
}
3091
3092
/// If this is a +1 of the value of an immutable 'self', remove it.
3093
static llvm::Value *tryRemoveRetainOfSelf(CodeGenFunction &CGF,
3094
113
                                          llvm::Value *result) {
3095
  // This is only applicable to a method with an immutable 'self'.
3096
113
  const ObjCMethodDecl *method =
3097
113
    dyn_cast_or_null<ObjCMethodDecl>(CGF.CurCodeDecl);
3098
113
  if (!method) 
return nullptr94
;
3099
19
  const VarDecl *self = method->getSelfDecl();
3100
19
  if (!self->getType().isConstQualified()) 
return nullptr0
;
3101
3102
  // Look for a retain call.
3103
19
  llvm::CallInst *retainCall =
3104
19
    dyn_cast<llvm::CallInst>(result->stripPointerCasts());
3105
19
  if (!retainCall || retainCall->getCalledOperand() !=
3106
15
                         CGF.CGM.getObjCEntrypoints().objc_retain)
3107
12
    return nullptr;
3108
3109
  // Look for an ordinary load of 'self'.
3110
7
  llvm::Value *retainedValue = retainCall->getArgOperand(0);
3111
7
  llvm::LoadInst *load =
3112
7
    dyn_cast<llvm::LoadInst>(retainedValue->stripPointerCasts());
3113
7
  if (!load || load->isAtomic() || load->isVolatile() ||
3114
7
      load->getPointerOperand() != CGF.GetAddrOfLocalVar(self).getPointer())
3115
3
    return nullptr;
3116
3117
  // Okay!  Burn it all down.  This relies for correctness on the
3118
  // assumption that the retain is emitted as part of the return and
3119
  // that thereafter everything is used "linearly".
3120
4
  llvm::Type *resultType = result->getType();
3121
4
  eraseUnusedBitCasts(cast<llvm::Instruction>(result));
3122
4
  assert(retainCall->use_empty());
3123
0
  retainCall->eraseFromParent();
3124
4
  eraseUnusedBitCasts(cast<llvm::Instruction>(retainedValue));
3125
3126
4
  return CGF.Builder.CreateBitCast(load, resultType);
3127
7
}
3128
3129
/// Emit an ARC autorelease of the result of a function.
3130
///
3131
/// \return the value to actually return from the function
3132
static llvm::Value *emitAutoreleaseOfResult(CodeGenFunction &CGF,
3133
113
                                            llvm::Value *result) {
3134
  // If we're returning 'self', kill the initial retain.  This is a
3135
  // heuristic attempt to "encourage correctness" in the really unfortunate
3136
  // case where we have a return of self during a dealloc and we desperately
3137
  // need to avoid the possible autorelease.
3138
113
  if (llvm::Value *self = tryRemoveRetainOfSelf(CGF, result))
3139
4
    return self;
3140
3141
  // At -O0, try to emit a fused retain/autorelease.
3142
109
  if (CGF.shouldUseFusedARCCalls())
3143
87
    if (llvm::Value *fused = tryEmitFusedAutoreleaseOfResult(CGF, result))
3144
27
      return fused;
3145
3146
82
  return CGF.EmitARCAutoreleaseReturnValue(result);
3147
109
}
3148
3149
/// Heuristically search for a dominating store to the return-value slot.
3150
182k
static llvm::StoreInst *findDominatingStoreToReturnValue(CodeGenFunction &CGF) {
3151
  // Check if a User is a store which pointerOperand is the ReturnValue.
3152
  // We are looking for stores to the ReturnValue, not for stores of the
3153
  // ReturnValue to some other location.
3154
182k
  auto GetStoreIfValid = [&CGF](llvm::User *U) -> llvm::StoreInst * {
3155
180k
    auto *SI = dyn_cast<llvm::StoreInst>(U);
3156
180k
    if (!SI || 
SI->getPointerOperand() != CGF.ReturnValue.getPointer()179k
)
3157
2.24k
      return nullptr;
3158
    // These aren't actually possible for non-coerced returns, and we
3159
    // only care about non-coerced returns on this code path.
3160
178k
    assert(!SI->isAtomic() && !SI->isVolatile());
3161
0
    return SI;
3162
180k
  };
3163
  // If there are multiple uses of the return-value slot, just check
3164
  // for something immediately preceding the IP.  Sometimes this can
3165
  // happen with how we generate implicit-returns; it can also happen
3166
  // with noreturn cleanups.
3167
182k
  if (!CGF.ReturnValue.getPointer()->hasOneUse()) {
3168
7.34k
    llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock();
3169
7.34k
    if (IP->empty()) 
return nullptr1.93k
;
3170
5.40k
    llvm::Instruction *I = &IP->back();
3171
3172
    // Skip lifetime markers
3173
5.40k
    for (llvm::BasicBlock::reverse_iterator II = IP->rbegin(),
3174
5.40k
                                            IE = IP->rend();
3175
5.51k
         II != IE; 
++II104
) {
3176
5.47k
      if (llvm::IntrinsicInst *Intrinsic =
3177
5.47k
              dyn_cast<llvm::IntrinsicInst>(&*II)) {
3178
358
        if (Intrinsic->getIntrinsicID() == llvm::Intrinsic::lifetime_end) {
3179
104
          const llvm::Value *CastAddr = Intrinsic->getArgOperand(1);
3180
104
          ++II;
3181
104
          if (II == IE)
3182
0
            break;
3183
104
          if (isa<llvm::BitCastInst>(&*II) && (CastAddr == &*II))
3184
104
            continue;
3185
104
        }
3186
358
      }
3187
5.36k
      I = &*II;
3188
5.36k
      break;
3189
5.47k
    }
3190
3191
5.40k
    return GetStoreIfValid(I);
3192
7.34k
  }
3193
3194
175k
  llvm::StoreInst *store =
3195
175k
      GetStoreIfValid(CGF.ReturnValue.getPointer()->user_back());
3196
175k
  if (!store) 
return nullptr515
;
3197
3198
  // Now do a first-and-dirty dominance check: just walk up the
3199
  // single-predecessors chain from the current insertion point.
3200
175k
  llvm::BasicBlock *StoreBB = store->getParent();
3201
175k
  llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock();
3202
175k
  while (IP != StoreBB) {
3203
2.01k
    if (!(IP = IP->getSinglePredecessor()))
3204
1.27k
      return nullptr;
3205
2.01k
  }
3206
3207
  // Okay, the store's basic block dominates the insertion point; we
3208
  // can do our thing.
3209
173k
  return store;
3210
175k
}
3211
3212
// Helper functions for EmitCMSEClearRecord
3213
3214
// Set the bits corresponding to a field having width `BitWidth` and located at
3215
// offset `BitOffset` (from the least significant bit) within a storage unit of
3216
// `Bits.size()` bytes. Each element of `Bits` corresponds to one target byte.
3217
// Use little-endian layout, i.e.`Bits[0]` is the LSB.
3218
static void setBitRange(SmallVectorImpl<uint64_t> &Bits, int BitOffset,
3219
125
                        int BitWidth, int CharWidth) {
3220
125
  assert(CharWidth <= 64);
3221
0
  assert(static_cast<unsigned>(BitWidth) <= Bits.size() * CharWidth);
3222
3223
0
  int Pos = 0;
3224
125
  if (BitOffset >= CharWidth) {
3225
39
    Pos += BitOffset / CharWidth;
3226
39
    BitOffset = BitOffset % CharWidth;
3227
39
  }
3228
3229
125
  const uint64_t Used = (uint64_t(1) << CharWidth) - 1;
3230
125
  if (BitOffset + BitWidth >= CharWidth) {
3231
65
    Bits[Pos++] |= (Used << BitOffset) & Used;
3232
65
    BitWidth -= CharWidth - BitOffset;
3233
65
    BitOffset = 0;
3234
65
  }
3235
3236
143
  while (BitWidth >= CharWidth) {
3237
18
    Bits[Pos++] = Used;
3238
18
    BitWidth -= CharWidth;
3239
18
  }
3240
3241
125
  if (BitWidth > 0)
3242
92
    Bits[Pos++] |= (Used >> (CharWidth - BitWidth)) << BitOffset;
3243
125
}
3244
3245
// Set the bits corresponding to a field having width `BitWidth` and located at
3246
// offset `BitOffset` (from the least significant bit) within a storage unit of
3247
// `StorageSize` bytes, located at `StorageOffset` in `Bits`. Each element of
3248
// `Bits` corresponds to one target byte. Use target endian layout.
3249
static void setBitRange(SmallVectorImpl<uint64_t> &Bits, int StorageOffset,
3250
                        int StorageSize, int BitOffset, int BitWidth,
3251
125
                        int CharWidth, bool BigEndian) {
3252
3253
125
  SmallVector<uint64_t, 8> TmpBits(StorageSize);
3254
125
  setBitRange(TmpBits, BitOffset, BitWidth, CharWidth);
3255
3256
125
  if (BigEndian)
3257
50
    std::reverse(TmpBits.begin(), TmpBits.end());
3258
3259
125
  for (uint64_t V : TmpBits)
3260
350
    Bits[StorageOffset++] |= V;
3261
125
}
3262
3263
static void setUsedBits(CodeGenModule &, QualType, int,
3264
                        SmallVectorImpl<uint64_t> &);
3265
3266
// Set the bits in `Bits`, which correspond to the value representations of
3267
// the actual members of the record type `RTy`. Note that this function does
3268
// not handle base classes, virtual tables, etc, since they cannot happen in
3269
// CMSE function arguments or return. The bit mask corresponds to the target
3270
// memory layout, i.e. it's endian dependent.
3271
static void setUsedBits(CodeGenModule &CGM, const RecordType *RTy, int Offset,
3272
153
                        SmallVectorImpl<uint64_t> &Bits) {
3273
153
  ASTContext &Context = CGM.getContext();
3274
153
  int CharWidth = Context.getCharWidth();
3275
153
  const RecordDecl *RD = RTy->getDecl()->getDefinition();
3276
153
  const ASTRecordLayout &ASTLayout = Context.getASTRecordLayout(RD);
3277
153
  const CGRecordLayout &Layout = CGM.getTypes().getCGRecordLayout(RD);
3278
3279
153
  int Idx = 0;
3280
566
  for (auto I = RD->field_begin(), E = RD->field_end(); I != E; 
++I, ++Idx413
) {
3281
413
    const FieldDecl *F = *I;
3282
3283
413
    if (F->isUnnamedBitfield() || 
F->isZeroLengthBitField(Context)283
||
3284
413
        
F->getType()->isIncompleteArrayType()283
)
3285
140
      continue;
3286
3287
273
    if (F->isBitField()) {
3288
125
      const CGBitFieldInfo &BFI = Layout.getBitFieldInfo(F);
3289
125
      setBitRange(Bits, Offset + BFI.StorageOffset.getQuantity(),
3290
125
                  BFI.StorageSize / CharWidth, BFI.Offset,
3291
125
                  BFI.Size, CharWidth,
3292
125
                  CGM.getDataLayout().isBigEndian());
3293
125
      continue;
3294
125
    }
3295
3296
148
    setUsedBits(CGM, F->getType(),
3297
148
                Offset + ASTLayout.getFieldOffset(Idx) / CharWidth, Bits);
3298
148
  }
3299
153
}
3300
3301
// Set the bits in `Bits`, which correspond to the value representations of
3302
// the elements of an array type `ATy`.
3303
static void setUsedBits(CodeGenModule &CGM, const ConstantArrayType *ATy,
3304
34
                        int Offset, SmallVectorImpl<uint64_t> &Bits) {
3305
34
  const ASTContext &Context = CGM.getContext();
3306
3307
34
  QualType ETy = Context.getBaseElementType(ATy);
3308
34
  int Size = Context.getTypeSizeInChars(ETy).getQuantity();
3309
34
  SmallVector<uint64_t, 4> TmpBits(Size);
3310
34
  setUsedBits(CGM, ETy, 0, TmpBits);
3311
3312
142
  for (int I = 0, N = Context.getConstantArrayElementCount(ATy); I < N; 
++I108
) {
3313
108
    auto Src = TmpBits.begin();
3314
108
    auto Dst = Bits.begin() + Offset + I * Size;
3315
270
    for (int J = 0; J < Size; 
++J162
)
3316
162
      *Dst++ |= *Src++;
3317
108
  }
3318
34
}
3319
3320
// Set the bits in `Bits`, which correspond to the value representations of
3321
// the type `QTy`.
3322
static void setUsedBits(CodeGenModule &CGM, QualType QTy, int Offset,
3323
182
                        SmallVectorImpl<uint64_t> &Bits) {
3324
182
  if (const auto *RTy = QTy->getAs<RecordType>())
3325
30
    return setUsedBits(CGM, RTy, Offset, Bits);
3326
3327
152
  ASTContext &Context = CGM.getContext();
3328
152
  if (const auto *ATy = Context.getAsConstantArrayType(QTy))
3329
34
    return setUsedBits(CGM, ATy, Offset, Bits);
3330
3331
118
  int Size = Context.getTypeSizeInChars(QTy).getQuantity();
3332
118
  if (Size <= 0)
3333
0
    return;
3334
3335
118
  std::fill_n(Bits.begin() + Offset, Size,
3336
118
              (uint64_t(1) << Context.getCharWidth()) - 1);
3337
118
}
3338
3339
static uint64_t buildMultiCharMask(const SmallVectorImpl<uint64_t> &Bits,
3340
                                   int Pos, int Size, int CharWidth,
3341
152
                                   bool BigEndian) {
3342
152
  assert(Size > 0);
3343
0
  uint64_t Mask = 0;
3344
152
  if (BigEndian) {
3345
318
    for (auto P = Bits.begin() + Pos, E = Bits.begin() + Pos + Size; P != E;
3346
256
         ++P)
3347
256
      Mask = (Mask << CharWidth) | *P;
3348
90
  } else {
3349
90
    auto P = Bits.begin() + Pos + Size, End = Bits.begin() + Pos;
3350
90
    do
3351
372
      Mask = (Mask << CharWidth) | *--P;
3352
372
    while (P != End);
3353
90
  }
3354
152
  return Mask;
3355
152
}
3356
3357
// Emit code to clear the bits in a record, which aren't a part of any user
3358
// declared member, when the record is a function return.
3359
llvm::Value *CodeGenFunction::EmitCMSEClearRecord(llvm::Value *Src,
3360
                                                  llvm::IntegerType *ITy,
3361
84
                                                  QualType QTy) {
3362
84
  assert(Src->getType() == ITy);
3363
0
  assert(ITy->getScalarSizeInBits() <= 64);
3364
3365
0
  const llvm::DataLayout &DataLayout = CGM.getDataLayout();
3366
84
  int Size = DataLayout.getTypeStoreSize(ITy);
3367
84
  SmallVector<uint64_t, 4> Bits(Size);
3368
84
  setUsedBits(CGM, QTy->castAs<RecordType>(), 0, Bits);
3369
3370
84
  int CharWidth = CGM.getContext().getCharWidth();
3371
84
  uint64_t Mask =
3372
84
      buildMultiCharMask(Bits, 0, Size, CharWidth, DataLayout.isBigEndian());
3373
3374
84
  return Builder.CreateAnd(Src, Mask, "cmse.clear");
3375
84
}
3376
3377
// Emit code to clear the bits in a record, which aren't a part of any user
3378
// declared member, when the record is a function argument.
3379
llvm::Value *CodeGenFunction::EmitCMSEClearRecord(llvm::Value *Src,
3380
                                                  llvm::ArrayType *ATy,
3381
39
                                                  QualType QTy) {
3382
39
  const llvm::DataLayout &DataLayout = CGM.getDataLayout();
3383
39
  int Size = DataLayout.getTypeStoreSize(ATy);
3384
39
  SmallVector<uint64_t, 16> Bits(Size);
3385
39
  setUsedBits(CGM, QTy->castAs<RecordType>(), 0, Bits);
3386
3387
  // Clear each element of the LLVM array.
3388
39
  int CharWidth = CGM.getContext().getCharWidth();
3389
39
  int CharsPerElt =
3390
39
      ATy->getArrayElementType()->getScalarSizeInBits() / CharWidth;
3391
39
  int MaskIndex = 0;
3392
39
  llvm::Value *R = llvm::UndefValue::get(ATy);
3393
107
  for (int I = 0, N = ATy->getArrayNumElements(); I != N; 
++I68
) {
3394
68
    uint64_t Mask = buildMultiCharMask(Bits, MaskIndex, CharsPerElt, CharWidth,
3395
68
                                       DataLayout.isBigEndian());
3396
68
    MaskIndex += CharsPerElt;
3397
68
    llvm::Value *T0 = Builder.CreateExtractValue(Src, I);
3398
68
    llvm::Value *T1 = Builder.CreateAnd(T0, Mask, "cmse.clear");
3399
68
    R = Builder.CreateInsertValue(R, T1, I);
3400
68
  }
3401
3402
39
  return R;
3403
39
}
3404
3405
void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI,
3406
                                         bool EmitRetDbgLoc,
3407
371k
                                         SourceLocation EndLoc) {
3408
371k
  if (FI.isNoReturn()) {
3409
    // Noreturn functions don't return.
3410
27
    EmitUnreachable(EndLoc);
3411
27
    return;
3412
27
  }
3413
3414
371k
  if (CurCodeDecl && 
CurCodeDecl->hasAttr<NakedAttr>()363k
) {
3415
    // Naked functions don't have epilogues.
3416
7
    Builder.CreateUnreachable();
3417
7
    return;
3418
7
  }
3419
3420
  // Functions with no result always return void.
3421
371k
  if (!ReturnValue.isValid()) {
3422
180k
    Builder.CreateRetVoid();
3423
180k
    return;
3424
180k
  }
3425
3426
191k
  llvm::DebugLoc RetDbgLoc;
3427
191k
  llvm::Value *RV = nullptr;
3428
191k
  QualType RetTy = FI.getReturnType();
3429
191k
  const ABIArgInfo &RetAI = FI.getReturnInfo();
3430
3431
191k
  switch (RetAI.getKind()) {
3432
5
  case ABIArgInfo::InAlloca:
3433
    // Aggregrates get evaluated directly into the destination.  Sometimes we
3434
    // need to return the sret value in a register, though.
3435
5
    assert(hasAggregateEvaluationKind(RetTy));
3436
5
    if (RetAI.getInAllocaSRet()) {
3437
5
      llvm::Function::arg_iterator EI = CurFn->arg_end();
3438
5
      --EI;
3439
5
      llvm::Value *ArgStruct = &*EI;
3440
5
      llvm::Value *SRet = Builder.CreateStructGEP(
3441
5
          EI->getType()->getPointerElementType(), ArgStruct,
3442
5
          RetAI.getInAllocaFieldIndex());
3443
5
      llvm::Type *Ty =
3444
5
          cast<llvm::GetElementPtrInst>(SRet)->getResultElementType();
3445
5
      RV = Builder.CreateAlignedLoad(Ty, SRet, getPointerAlign(), "sret");
3446
5
    }
3447
5
    break;
3448
3449
2.91k
  case ABIArgInfo::Indirect: {
3450
2.91k
    auto AI = CurFn->arg_begin();
3451
2.91k
    if (RetAI.isSRetAfterThis())
3452
50
      ++AI;
3453
2.91k
    switch (getEvaluationKind(RetTy)) {
3454
285
    case TEK_Complex: {
3455
285
      ComplexPairTy RT =
3456
285
        EmitLoadOfComplex(MakeAddrLValue(ReturnValue, RetTy), EndLoc);
3457
285
      EmitStoreOfComplex(RT, MakeNaturalAlignAddrLValue(&*AI, RetTy),
3458
285
                         /*isInit*/ true);
3459
285
      break;
3460
0
    }
3461
2.34k
    case TEK_Aggregate:
3462
      // Do nothing; aggregrates get evaluated directly into the destination.
3463
2.34k
      break;
3464
288
    case TEK_Scalar:
3465
288
      EmitStoreOfScalar(Builder.CreateLoad(ReturnValue),
3466
288
                        MakeNaturalAlignAddrLValue(&*AI, RetTy),
3467
288
                        /*isInit*/ true);
3468
288
      break;
3469
2.91k
    }
3470
2.91k
    break;
3471
2.91k
  }
3472
3473
13.4k
  case ABIArgInfo::Extend:
3474
187k
  case ABIArgInfo::Direct:
3475
187k
    if (RetAI.getCoerceToType() == ConvertType(RetTy) &&
3476
187k
        
RetAI.getDirectOffset() == 0182k
) {
3477
      // The internal return value temp always will have pointer-to-return-type
3478
      // type, just do a load.
3479
3480
      // If there is a dominating store to ReturnValue, we can elide
3481
      // the load, zap the store, and usually zap the alloca.
3482
182k
      if (llvm::StoreInst *SI =
3483
182k
              findDominatingStoreToReturnValue(*this)) {
3484
        // Reuse the debug location from the store unless there is
3485
        // cleanup code to be emitted between the store and return
3486
        // instruction.
3487
177k
        if (EmitRetDbgLoc && 
!AutoreleaseResult176k
)
3488
176k
          RetDbgLoc = SI->getDebugLoc();
3489
        // Get the stored value and nuke the now-dead store.
3490
177k
        RV = SI->getValueOperand();
3491
177k
        SI->eraseFromParent();
3492
3493
      // Otherwise, we have to do a simple load.
3494
177k
      } else {
3495
5.45k
        RV = Builder.CreateLoad(ReturnValue);
3496
5.45k
      }
3497
182k
    } else {
3498
      // If the value is offset in memory, apply the offset now.
3499
4.71k
      Address V = emitAddressAtOffset(*this, ReturnValue, RetAI);
3500
3501
4.71k
      RV = CreateCoercedLoad(V, RetAI.getCoerceToType(), *this);
3502
4.71k
    }
3503
3504
    // In ARC, end functions that return a retainable type with a call
3505
    // to objc_autoreleaseReturnValue.
3506
187k
    if (AutoreleaseResult) {
3507
113
#ifndef NDEBUG
3508
      // Type::isObjCRetainabletype has to be called on a QualType that hasn't
3509
      // been stripped of the typedefs, so we cannot use RetTy here. Get the
3510
      // original return type of FunctionDecl, CurCodeDecl, and BlockDecl from
3511
      // CurCodeDecl or BlockInfo.
3512
113
      QualType RT;
3513
3514
113
      if (auto *FD = dyn_cast<FunctionDecl>(CurCodeDecl))
3515
70
        RT = FD->getReturnType();
3516
43
      else if (auto *MD = dyn_cast<ObjCMethodDecl>(CurCodeDecl))
3517
19
        RT = MD->getReturnType();
3518
24
      else if (isa<BlockDecl>(CurCodeDecl))
3519
24
        RT = BlockInfo->BlockExpression->getFunctionType()->getReturnType();
3520
0
      else
3521
0
        llvm_unreachable("Unexpected function/method type");
3522
3523
113
      assert(getLangOpts().ObjCAutoRefCount &&
3524
113
             !FI.isReturnsRetained() &&
3525
113
             RT->isObjCRetainableType());
3526
0
#endif
3527
0
      RV = emitAutoreleaseOfResult(*this, RV);
3528
113
    }
3529
3530
187k
    break;
3531
3532
187k
  case ABIArgInfo::Ignore:
3533
332
    break;
3534
3535
541
  case ABIArgInfo::CoerceAndExpand: {
3536
541
    auto coercionType = RetAI.getCoerceAndExpandType();
3537
3538
    // Load all of the coerced elements out into results.
3539
541
    llvm::SmallVector<llvm::Value*, 4> results;
3540
541
    Address addr = Builder.CreateElementBitCast(ReturnValue, coercionType);
3541
1.68k
    for (unsigned i = 0, e = coercionType->getNumElements(); i != e; 
++i1.14k
) {
3542
1.14k
      auto coercedEltType = coercionType->getElementType(i);
3543
1.14k
      if (ABIArgInfo::isPaddingForCoerceAndExpand(coercedEltType))
3544
7
        continue;
3545
3546
1.14k
      auto eltAddr = Builder.CreateStructGEP(addr, i);
3547
1.14k
      auto elt = Builder.CreateLoad(eltAddr);
3548
1.14k
      results.push_back(elt);
3549
1.14k
    }
3550
3551
    // If we have one result, it's the single direct result type.
3552
541
    if (results.size() == 1) {
3553
180
      RV = results[0];
3554
3555
    // Otherwise, we need to make a first-class aggregate.
3556
361
    } else {
3557
      // Construct a return type that lacks padding elements.
3558
361
      llvm::Type *returnType = RetAI.getUnpaddedCoerceAndExpandType();
3559
3560
361
      RV = llvm::UndefValue::get(returnType);
3561
1.32k
      for (unsigned i = 0, e = results.size(); i != e; 
++i960
) {
3562
960
        RV = Builder.CreateInsertValue(RV, results[i], i);
3563
960
      }
3564
361
    }
3565
541
    break;
3566
187k
  }
3567
0
  case ABIArgInfo::Expand:
3568
0
  case ABIArgInfo::IndirectAliased:
3569
0
    llvm_unreachable("Invalid ABI kind for return argument");
3570
191k
  }
3571
3572
191k
  llvm::Instruction *Ret;
3573
191k
  if (RV) {
3574
188k
    if (CurFuncDecl && 
CurFuncDecl->hasAttr<CmseNSEntryAttr>()186k
) {
3575
      // For certain return types, clear padding bits, as they may reveal
3576
      // sensitive information.
3577
      // Small struct/union types are passed as integers.
3578
90
      auto *ITy = dyn_cast<llvm::IntegerType>(RV->getType());
3579
90
      if (ITy != nullptr && 
isa<RecordType>(RetTy.getCanonicalType())84
)
3580
84
        RV = EmitCMSEClearRecord(RV, ITy, RetTy);
3581
90
    }
3582
188k
    EmitReturnValueCheck(RV);
3583
188k
    Ret = Builder.CreateRet(RV);
3584
188k
  } else {
3585
3.24k
    Ret = Builder.CreateRetVoid();
3586
3.24k
  }
3587
3588
191k
  if (RetDbgLoc)
3589
44.8k
    Ret->setDebugLoc(std::move(RetDbgLoc));
3590
191k
}
3591
3592
188k
void CodeGenFunction::EmitReturnValueCheck(llvm::Value *RV) {
3593
  // A current decl may not be available when emitting vtable thunks.
3594
188k
  if (!CurCodeDecl)
3595
1.22k
    return;
3596
3597
  // If the return block isn't reachable, neither is this check, so don't emit
3598
  // it.
3599
186k
  if (ReturnBlock.isValid() && 
ReturnBlock.getBlock()->use_empty()1.95k
)
3600
216
    return;
3601
3602
186k
  ReturnsNonNullAttr *RetNNAttr = nullptr;
3603
186k
  if (SanOpts.has(SanitizerKind::ReturnsNonnullAttribute))
3604
47
    RetNNAttr = CurCodeDecl->getAttr<ReturnsNonNullAttr>();
3605
3606
186k
  if (!RetNNAttr && 
!requiresReturnValueNullabilityCheck()186k
)
3607
186k
    return;
3608
3609
  // Prefer the returns_nonnull attribute if it's present.
3610
18
  SourceLocation AttrLoc;
3611
18
  SanitizerMask CheckKind;
3612
18
  SanitizerHandler Handler;
3613
18
  if (RetNNAttr) {
3614
5
    assert(!requiresReturnValueNullabilityCheck() &&
3615
5
           "Cannot check nullability and the nonnull attribute");
3616
0
    AttrLoc = RetNNAttr->getLocation();
3617
5
    CheckKind = SanitizerKind::ReturnsNonnullAttribute;
3618
5
    Handler = SanitizerHandler::NonnullReturn;
3619
13
  } else {
3620
13
    if (auto *DD = dyn_cast<DeclaratorDecl>(CurCodeDecl))
3621
7
      if (auto *TSI = DD->getTypeSourceInfo())
3622
7
        if (auto FTL = TSI->getTypeLoc().getAsAdjusted<FunctionTypeLoc>())
3623
7
          AttrLoc = FTL.getReturnLoc().findNullabilityLoc();
3624
13
    CheckKind = SanitizerKind::NullabilityReturn;
3625
13
    Handler = SanitizerHandler::NullabilityReturn;
3626
13
  }
3627
3628
0
  SanitizerScope SanScope(this);
3629
3630
  // Make sure the "return" source location is valid. If we're checking a
3631
  // nullability annotation, make sure the preconditions for the check are met.
3632
18
  llvm::BasicBlock *Check = createBasicBlock("nullcheck");
3633
18
  llvm::BasicBlock *NoCheck = createBasicBlock("no.nullcheck");
3634
18
  llvm::Value *SLocPtr = Builder.CreateLoad(ReturnLocation, "return.sloc.load");
3635
18
  llvm::Value *CanNullCheck = Builder.CreateIsNotNull(SLocPtr);
3636
18
  if (requiresReturnValueNullabilityCheck())
3637
13
    CanNullCheck =
3638
13
        Builder.CreateAnd(CanNullCheck, RetValNullabilityPrecondition);
3639
18
  Builder.CreateCondBr(CanNullCheck, Check, NoCheck);
3640
18
  EmitBlock(Check);
3641
3642
  // Now do the null check.
3643
18
  llvm::Value *Cond = Builder.CreateIsNotNull(RV);
3644
18
  llvm::Constant *StaticData[] = {EmitCheckSourceLocation(AttrLoc)};
3645
18
  llvm::Value *DynamicData[] = {SLocPtr};
3646
18
  EmitCheck(std::make_pair(Cond, CheckKind), Handler, StaticData, DynamicData);
3647
3648
18
  EmitBlock(NoCheck);
3649
3650
18
#ifndef NDEBUG
3651
  // The return location should not be used after the check has been emitted.
3652
18
  ReturnLocation = Address::invalid();
3653
18
#endif
3654
18
}
3655
3656
24.7k
static bool isInAllocaArgument(CGCXXABI &ABI, QualType type) {
3657
24.7k
  const CXXRecordDecl *RD = type->getAsCXXRecordDecl();
3658
24.7k
  return RD && 
ABI.getRecordArgABI(RD) == CGCXXABI::RAA_DirectInMemory2.79k
;
3659
24.7k
}
3660
3661
static AggValueSlot createPlaceholderSlot(CodeGenFunction &CGF,
3662
102
                                          QualType Ty) {
3663
  // FIXME: Generate IR in one pass, rather than going back and fixing up these
3664
  // placeholders.
3665
102
  llvm::Type *IRTy = CGF.ConvertTypeForMem(Ty);
3666
102
  llvm::Type *IRPtrTy = IRTy->getPointerTo();
3667
102
  llvm::Value *Placeholder = llvm::UndefValue::get(IRPtrTy->getPointerTo());
3668
3669
  // FIXME: When we generate this IR in one pass, we shouldn't need
3670
  // this win32-specific alignment hack.
3671
102
  CharUnits Align = CharUnits::fromQuantity(4);
3672
102
  Placeholder = CGF.Builder.CreateAlignedLoad(IRPtrTy, Placeholder, Align);
3673
3674
102
  return AggValueSlot::forAddr(Address(Placeholder, Align),
3675
102
                               Ty.getQualifiers(),
3676
102
                               AggValueSlot::IsNotDestructed,
3677
102
                               AggValueSlot::DoesNotNeedGCBarriers,
3678
102
                               AggValueSlot::IsNotAliased,
3679
102
                               AggValueSlot::DoesNotOverlap);
3680
102
}
3681
3682
void CodeGenFunction::EmitDelegateCallArg(CallArgList &args,
3683
                                          const VarDecl *param,
3684
18.2k
                                          SourceLocation loc) {
3685
  // StartFunction converted the ABI-lowered parameter(s) into a
3686
  // local alloca.  We need to turn that into an r-value suitable
3687
  // for EmitCall.
3688
18.2k
  Address local = GetAddrOfLocalVar(param);
3689
3690
18.2k
  QualType type = param->getType();
3691
3692
18.2k
  if (isInAllocaArgument(CGM.getCXXABI(), type)) {
3693
1
    CGM.ErrorUnsupported(param, "forwarded non-trivially copyable parameter");
3694
1
  }
3695
3696
  // GetAddrOfLocalVar returns a pointer-to-pointer for references,
3697
  // but the argument needs to be the original pointer.
3698
18.2k
  if (type->isReferenceType()) {
3699
9.46k
    args.add(RValue::get(Builder.CreateLoad(local)), type);
3700
3701
  // In ARC, move out of consumed arguments so that the release cleanup
3702
  // entered by StartFunction doesn't cause an over-release.  This isn't
3703
  // optimal -O0 code generation, but it should get cleaned up when
3704
  // optimization is enabled.  This also assumes that delegate calls are
3705
  // performed exactly once for a set of arguments, but that should be safe.
3706
9.46k
  } else 
if (8.81k
getLangOpts().ObjCAutoRefCount8.81k
&&
3707
8.81k
             
param->hasAttr<NSConsumedAttr>()9
&&
3708
8.81k
             
type->isObjCRetainableType()2
) {
3709
2
    llvm::Value *ptr = Builder.CreateLoad(local);
3710
2
    auto null =
3711
2
      llvm::ConstantPointerNull::get(cast<llvm::PointerType>(ptr->getType()));
3712
2
    Builder.CreateStore(null, local);
3713
2
    args.add(RValue::get(ptr), type);
3714
3715
  // For the most part, we just need to load the alloca, except that
3716
  // aggregate r-values are actually pointers to temporaries.
3717
8.81k
  } else {
3718
8.81k
    args.add(convertTempToRValue(local, type, loc), type);
3719
8.81k
  }
3720
3721
  // Deactivate the cleanup for the callee-destructed param that was pushed.
3722
18.2k
  if (type->isRecordType() && 
!CurFuncIsThunk2.50k
&&
3723
18.2k
      
type->castAs<RecordType>()->getDecl()->isParamDestroyedInCallee()2.47k
&&
3724
18.2k
      
param->needsDestruction(getContext())8
) {
3725
6
    EHScopeStack::stable_iterator cleanup =
3726
6
        CalleeDestructedParamCleanups.lookup(cast<ParmVarDecl>(param));
3727
6
    assert(cleanup.isValid() &&
3728
6
           "cleanup for callee-destructed param not recorded");
3729
    // This unreachable is a temporary marker which will be removed later.
3730
0
    llvm::Instruction *isActive = Builder.CreateUnreachable();
3731
6
    args.addArgCleanupDeactivation(cleanup, isActive);
3732
6
  }
3733
18.2k
}
3734
3735
52
static bool isProvablyNull(llvm::Value *addr) {
3736
52
  return isa<llvm::ConstantPointerNull>(addr);
3737
52
}
3738
3739
/// Emit the actual writing-back of a writeback.
3740
static void emitWriteback(CodeGenFunction &CGF,
3741
26
                          const CallArgList::Writeback &writeback) {
3742
26
  const LValue &srcLV = writeback.Source;
3743
26
  Address srcAddr = srcLV.getAddress(CGF);
3744
26
  assert(!isProvablyNull(srcAddr.getPointer()) &&
3745
26
         "shouldn't have writeback for provably null argument");
3746
3747
0
  llvm::BasicBlock *contBB = nullptr;
3748
3749
  // If the argument wasn't provably non-null, we need to null check
3750
  // before doing the store.
3751
26
  bool provablyNonNull = llvm::isKnownNonZero(srcAddr.getPointer(),
3752
26
                                              CGF.CGM.getDataLayout());
3753
26
  if (!provablyNonNull) {
3754
5
    llvm::BasicBlock *writebackBB = CGF.createBasicBlock("icr.writeback");
3755
5
    contBB = CGF.createBasicBlock("icr.done");
3756
3757
5
    llvm::Value *isNull =
3758
5
      CGF.Builder.CreateIsNull(srcAddr.getPointer(), "icr.isnull");
3759
5
    CGF.Builder.CreateCondBr(isNull, contBB, writebackBB);
3760
5
    CGF.EmitBlock(writebackBB);
3761
5
  }
3762
3763
  // Load the value to writeback.
3764
26
  llvm::Value *value = CGF.Builder.CreateLoad(writeback.Temporary);
3765
3766
  // Cast it back, in case we're writing an id to a Foo* or something.
3767
26
  value = CGF.Builder.CreateBitCast(value, srcAddr.getElementType(),
3768
26
                                    "icr.writeback-cast");
3769
3770
  // Perform the writeback.
3771
3772
  // If we have a "to use" value, it's something we need to emit a use
3773
  // of.  This has to be carefully threaded in: if it's done after the
3774
  // release it's potentially undefined behavior (and the optimizer
3775
  // will ignore it), and if it happens before the retain then the
3776
  // optimizer could move the release there.
3777
26
  if (writeback.ToUse) {
3778
7
    assert(srcLV.getObjCLifetime() == Qualifiers::OCL_Strong);
3779
3780
    // Retain the new value.  No need to block-copy here:  the block's
3781
    // being passed up the stack.
3782
0
    value = CGF.EmitARCRetainNonBlock(value);
3783
3784
    // Emit the intrinsic use here.
3785
7
    CGF.EmitARCIntrinsicUse(writeback.ToUse);
3786
3787
    // Load the old value (primitively).
3788
7
    llvm::Value *oldValue = CGF.EmitLoadOfScalar(srcLV, SourceLocation());
3789
3790
    // Put the new value in place (primitively).
3791
7
    CGF.EmitStoreOfScalar(value, srcLV, /*init*/ false);
3792
3793
    // Release the old value.
3794
7
    CGF.EmitARCRelease(oldValue, srcLV.isARCPreciseLifetime());
3795
3796
  // Otherwise, we can just do a normal lvalue store.
3797
19
  } else {
3798
19
    CGF.EmitStoreThroughLValue(RValue::get(value), srcLV);
3799
19
  }
3800
3801
  // Jump to the continuation block.
3802
26
  if (!provablyNonNull)
3803
5
    CGF.EmitBlock(contBB);
3804
26
}
3805
3806
static void emitWritebacks(CodeGenFunction &CGF,
3807
26
                           const CallArgList &args) {
3808
26
  for (const auto &I : args.writebacks())
3809
26
    emitWriteback(CGF, I);
3810
26
}
3811
3812
static void deactivateArgCleanupsBeforeCall(CodeGenFunction &CGF,
3813
147
                                            const CallArgList &CallArgs) {
3814
147
  ArrayRef<CallArgList::CallArgCleanup> Cleanups =
3815
147
    CallArgs.getCleanupsToDeactivate();
3816
  // Iterate in reverse to increase the likelihood of popping the cleanup.
3817
188
  for (const auto &I : llvm::reverse(Cleanups)) {
3818
188
    CGF.DeactivateCleanupBlock(I.Cleanup, I.IsActiveIP);
3819
188
    I.IsActiveIP->eraseFromParent();
3820
188
  }
3821
147
}
3822
3823
26
static const Expr *maybeGetUnaryAddrOfOperand(const Expr *E) {
3824
26
  if (const UnaryOperator *uop = dyn_cast<UnaryOperator>(E->IgnoreParens()))
3825
21
    if (uop->getOpcode() == UO_AddrOf)
3826
21
      return uop->getSubExpr();
3827
5
  return nullptr;
3828
26
}
3829
3830
/// Emit an argument that's being passed call-by-writeback.  That is,
3831
/// we are passing the address of an __autoreleased temporary; it
3832
/// might be copy-initialized with the current value of the given
3833
/// address, but it will definitely be copied out of after the call.
3834
static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args,
3835
26
                             const ObjCIndirectCopyRestoreExpr *CRE) {
3836
26
  LValue srcLV;
3837
3838
  // Make an optimistic effort to emit the address as an l-value.
3839
  // This can fail if the argument expression is more complicated.
3840
26
  if (const Expr *lvExpr = maybeGetUnaryAddrOfOperand(CRE->getSubExpr())) {
3841
21
    srcLV = CGF.EmitLValue(lvExpr);
3842
3843
  // Otherwise, just emit it as a scalar.
3844
21
  } else {
3845
5
    Address srcAddr = CGF.EmitPointerWithAlignment(CRE->getSubExpr());
3846
3847
5
    QualType srcAddrType =
3848
5
      CRE->getSubExpr()->getType()->castAs<PointerType>()->getPointeeType();
3849
5
    srcLV = CGF.MakeAddrLValue(srcAddr, srcAddrType);
3850
5
  }
3851
26
  Address srcAddr = srcLV.getAddress(CGF);
3852
3853
  // The dest and src types don't necessarily match in LLVM terms
3854
  // because of the crazy ObjC compatibility rules.
3855
3856
26
  llvm::PointerType *destType =
3857
26
    cast<llvm::PointerType>(CGF.ConvertType(CRE->getType()));
3858
3859
  // If the address is a constant null, just pass the appropriate null.
3860
26
  if (isProvablyNull(srcAddr.getPointer())) {
3861
0
    args.add(RValue::get(llvm::ConstantPointerNull::get(destType)),
3862
0
             CRE->getType());
3863
0
    return;
3864
0
  }
3865
3866
  // Create the temporary.
3867
26
  Address temp = CGF.CreateTempAlloca(destType->getElementType(),
3868
26
                                      CGF.getPointerAlign(),
3869
26
                                      "icr.temp");
3870
  // Loading an l-value can introduce a cleanup if the l-value is __weak,
3871
  // and that cleanup will be conditional if we can't prove that the l-value
3872
  // isn't null, so we need to register a dominating point so that the cleanups
3873
  // system will make valid IR.
3874
26
  CodeGenFunction::ConditionalEvaluation condEval(CGF);
3875
3876
  // Zero-initialize it if we're not doing a copy-initialization.
3877
26
  bool shouldCopy = CRE->shouldCopy();
3878
26
  if (!shouldCopy) {
3879
4
    llvm::Value *null =
3880
4
      llvm::ConstantPointerNull::get(
3881
4
        cast<llvm::PointerType>(destType->getElementType()));
3882
4
    CGF.Builder.CreateStore(null, temp);
3883
4
  }
3884
3885
26
  llvm::BasicBlock *contBB = nullptr;
3886
26
  llvm::BasicBlock *originBB = nullptr;
3887
3888
  // If the address is *not* known to be non-null, we need to switch.
3889
26
  llvm::Value *finalArgument;
3890
3891
26
  bool provablyNonNull = llvm::isKnownNonZero(srcAddr.getPointer(),
3892
26
                                              CGF.CGM.getDataLayout());
3893
26
  if (provablyNonNull) {
3894
21
    finalArgument = temp.getPointer();
3895
21
  } else {
3896
5
    llvm::Value *isNull =
3897
5
      CGF.Builder.CreateIsNull(srcAddr.getPointer(), "icr.isnull");
3898
3899
5
    finalArgument = CGF.Builder.CreateSelect(isNull,
3900
5
                                   llvm::ConstantPointerNull::get(destType),
3901
5
                                             temp.getPointer(), "icr.argument");
3902
3903
    // If we need to copy, then the load has to be conditional, which
3904
    // means we need control flow.
3905
5
    if (shouldCopy) {
3906
5
      originBB = CGF.Builder.GetInsertBlock();
3907
5
      contBB = CGF.createBasicBlock("icr.cont");
3908
5
      llvm::BasicBlock *copyBB = CGF.createBasicBlock("icr.copy");
3909
5
      CGF.Builder.CreateCondBr(isNull, contBB, copyBB);
3910
5
      CGF.EmitBlock(copyBB);
3911
5
      condEval.begin(CGF);
3912
5
    }
3913
5
  }
3914
3915
26
  llvm::Value *valueToUse = nullptr;
3916
3917
  // Perform a copy if necessary.
3918
26
  if (shouldCopy) {
3919
22
    RValue srcRV = CGF.EmitLoadOfLValue(srcLV, SourceLocation());
3920
22
    assert(srcRV.isScalar());
3921
3922
0
    llvm::Value *src = srcRV.getScalarVal();
3923
22
    src = CGF.Builder.CreateBitCast(src, destType->getElementType(),
3924
22
                                    "icr.cast");
3925
3926
    // Use an ordinary store, not a store-to-lvalue.
3927
22
    CGF.Builder.CreateStore(src, temp);
3928
3929
    // If optimization is enabled, and the value was held in a
3930
    // __strong variable, we need to tell the optimizer that this
3931
    // value has to stay alive until we're doing the store back.
3932
    // This is because the temporary is effectively unretained,
3933
    // and so otherwise we can violate the high-level semantics.
3934
22
    if (CGF.CGM.getCodeGenOpts().OptimizationLevel != 0 &&
3935
22
        
srcLV.getObjCLifetime() == Qualifiers::OCL_Strong9
) {
3936
7
      valueToUse = src;
3937
7
    }
3938
22
  }
3939
3940
  // Finish the control flow if we needed it.
3941
26
  if (shouldCopy && 
!provablyNonNull22
) {
3942
5
    llvm::BasicBlock *copyBB = CGF.Builder.GetInsertBlock();
3943
5
    CGF.EmitBlock(contBB);
3944
3945
    // Make a phi for the value to intrinsically use.
3946
5
    if (valueToUse) {
3947
2
      llvm::PHINode *phiToUse = CGF.Builder.CreatePHI(valueToUse->getType(), 2,
3948
2
                                                      "icr.to-use");
3949
2
      phiToUse->addIncoming(valueToUse, copyBB);
3950
2
      phiToUse->addIncoming(llvm::UndefValue::get(valueToUse->getType()),
3951
2
                            originBB);
3952
2
      valueToUse = phiToUse;
3953
2
    }
3954
3955
5
    condEval.end(CGF);
3956
5
  }
3957
3958
26
  args.addWriteback(srcLV, temp, valueToUse);
3959
26
  args.add(RValue::get(finalArgument), CRE->getType());
3960
26
}
3961
3962
71
void CallArgList::allocateArgumentMemory(CodeGenFunction &CGF) {
3963
71
  assert(!StackBase);
3964
3965
  // Save the stack.
3966
0
  llvm::Function *F = CGF.CGM.getIntrinsic(llvm::Intrinsic::stacksave);
3967
71
  StackBase = CGF.Builder.CreateCall(F, {}, "inalloca.save");
3968
71
}
3969
3970
379k
void CallArgList::freeArgumentMemory(CodeGenFunction &CGF) const {
3971
379k
  if (StackBase) {
3972
    // Restore the stack after the call.
3973
71
    llvm::Function *F = CGF.CGM.getIntrinsic(llvm::Intrinsic::stackrestore);
3974
71
    CGF.Builder.CreateCall(F, StackBase);
3975
71
  }
3976
379k
}
3977
3978
void CodeGenFunction::EmitNonNullArgCheck(RValue RV, QualType ArgType,
3979
                                          SourceLocation ArgLoc,
3980
                                          AbstractCallee AC,
3981
437k
                                          unsigned ParmNum) {
3982
437k
  if (!AC.getDecl() || 
!(418k
SanOpts.has(SanitizerKind::NonnullAttribute)418k
||
3983
418k
                         
SanOpts.has(SanitizerKind::NullabilityArg)418k
))
3984
437k
    return;
3985
3986
  // The param decl may be missing in a variadic function.
3987
77
  auto PVD = ParmNum < AC.getNumParams() ? 
AC.getParamDecl(ParmNum)66
:
nullptr11
;
3988
77
  unsigned ArgNo = PVD ? 
PVD->getFunctionScopeIndex()66
:
ParmNum11
;
3989
3990
  // Prefer the nonnull attribute if it's present.
3991
77
  const NonNullAttr *NNAttr = nullptr;
3992
77
  if (SanOpts.has(SanitizerKind::NonnullAttribute))
3993
25
    NNAttr = getNonNullAttr(AC.getDecl(), PVD, ArgType, ArgNo);
3994
3995
77
  bool CanCheckNullability = false;
3996
77
  if (SanOpts.has(SanitizerKind::NullabilityArg) && 
!NNAttr53
&&
PVD52
) {
3997
44
    auto Nullability = PVD->getType()->getNullability(getContext());
3998
44
    CanCheckNullability = Nullability &&
3999
44
                          
*Nullability == NullabilityKind::NonNull20
&&
4000
44
                          
PVD->getTypeSourceInfo()18
;
4001
44
  }
4002
4003
77
  if (!NNAttr && 
!CanCheckNullability61
)
4004
43
    return;
4005
4006
34
  SourceLocation AttrLoc;
4007
34
  SanitizerMask CheckKind;
4008
34
  SanitizerHandler Handler;
4009
34
  if (NNAttr) {
4010
16
    AttrLoc = NNAttr->getLocation();
4011
16
    CheckKind = SanitizerKind::NonnullAttribute;
4012
16
    Handler = SanitizerHandler::NonnullArg;
4013
18
  } else {
4014
18
    AttrLoc = PVD->getTypeSourceInfo()->getTypeLoc().findNullabilityLoc();
4015
18
    CheckKind = SanitizerKind::NullabilityArg;
4016
18
    Handler = SanitizerHandler::NullabilityArg;
4017
18
  }
4018
4019
34
  SanitizerScope SanScope(this);
4020
34
  llvm::Value *Cond = EmitNonNullRValueCheck(RV, ArgType);
4021
34
  llvm::Constant *StaticData[] = {
4022
34
      EmitCheckSourceLocation(ArgLoc), EmitCheckSourceLocation(AttrLoc),
4023
34
      llvm::ConstantInt::get(Int32Ty, ArgNo + 1),
4024
34
  };
4025
34
  EmitCheck(std::make_pair(Cond, CheckKind), Handler, StaticData, None);
4026
34
}
4027
4028
// Check if the call is going to use the inalloca convention. This needs to
4029
// agree with CGFunctionInfo::usesInAlloca. The CGFunctionInfo is arranged
4030
// later, so we can't check it directly.
4031
static bool hasInAllocaArgs(CodeGenModule &CGM, CallingConv ExplicitCC,
4032
331k
                            ArrayRef<QualType> ArgTypes) {
4033
  // The Swift calling conventions don't go through the target-specific
4034
  // argument classification, they never use inalloca.
4035
  // TODO: Consider limiting inalloca use to only calling conventions supported
4036
  // by MSVC.
4037
331k
  if (ExplicitCC == CC_Swift || 
ExplicitCC == CC_SwiftAsync330k
)
4038
1.28k
    return false;
4039
330k
  if (!CGM.getTarget().getCXXABI().isMicrosoft())
4040
323k
    return false;
4041
7.15k
  return llvm::any_of(ArgTypes, [&](QualType Ty) {
4042
6.50k
    return isInAllocaArgument(CGM.getCXXABI(), Ty);
4043
6.50k
  });
4044
330k
}
4045
4046
#ifndef NDEBUG
4047
// Determine whether the given argument is an Objective-C method
4048
// that may have type parameters in its signature.
4049
12.0k
static bool isObjCMethodWithTypeParams(const ObjCMethodDecl *method) {
4050
12.0k
  const DeclContext *dc = method->getDeclContext();
4051
12.0k
  if (const ObjCInterfaceDecl *classDecl = dyn_cast<ObjCInterfaceDecl>(dc)) {
4052
7.70k
    return classDecl->getTypeParamListAsWritten();
4053
7.70k
  }
4054
4055
4.35k
  if (const ObjCCategoryDecl *catDecl = dyn_cast<ObjCCategoryDecl>(dc)) {
4056
3.62k
    return catDecl->getTypeParamList();
4057
3.62k
  }
4058
4059
730
  return false;
4060
4.35k
}
4061
#endif
4062
4063
/// EmitCallArgs - Emit call arguments for a function.
4064
void CodeGenFunction::EmitCallArgs(
4065
    CallArgList &Args, PrototypeWrapper Prototype,
4066
    llvm::iterator_range<CallExpr::const_arg_iterator> ArgRange,
4067
331k
    AbstractCallee AC, unsigned ParamsToSkip, EvaluationOrder Order) {
4068
331k
  SmallVector<QualType, 16> ArgTypes;
4069
4070
331k
  assert((ParamsToSkip == 0 || Prototype.P) &&
4071
331k
         "Can't skip parameters if type info is not provided");
4072
4073
  // This variable only captures *explicitly* written conventions, not those
4074
  // applied by default via command line flags or target defaults, such as
4075
  // thiscall, aapcs, stdcall via -mrtd, etc. Computing that correctly would
4076
  // require knowing if this is a C++ instance method or being able to see
4077
  // unprototyped FunctionTypes.
4078
0
  CallingConv ExplicitCC = CC_C;
4079
4080
  // First, if a prototype was provided, use those argument types.
4081
331k
  bool IsVariadic = false;
4082
331k
  if (Prototype.P) {
4083
330k
    const auto *MD = Prototype.P.dyn_cast<const ObjCMethodDecl *>();
4084
330k
    if (MD) {
4085
12.0k
      IsVariadic = MD->isVariadic();
4086
12.0k
      ExplicitCC = getCallingConventionForDecl(
4087
12.0k
          MD, CGM.getTarget().getTriple().isOSWindows());
4088
12.0k
      ArgTypes.assign(MD->param_type_begin() + ParamsToSkip,
4089
12.0k
                      MD->param_type_end());
4090
318k
    } else {
4091
318k
      const auto *FPT = Prototype.P.get<const FunctionProtoType *>();
4092
318k
      IsVariadic = FPT->isVariadic();
4093
318k
      ExplicitCC = FPT->getExtInfo().getCC();
4094
318k
      ArgTypes.assign(FPT->param_type_begin() + ParamsToSkip,
4095
318k
                      FPT->param_type_end());
4096
318k
    }
4097
4098
330k
#ifndef NDEBUG
4099
    // Check that the prototyped types match the argument expression types.
4100
330k
    bool isGenericMethod = MD && 
isObjCMethodWithTypeParams(MD)12.0k
;
4101
330k
    CallExpr::const_arg_iterator Arg = ArgRange.begin();
4102
349k
    for (QualType Ty : ArgTypes) {
4103
349k
      assert(Arg != ArgRange.end() && "Running over edge of argument list!");
4104
0
      assert(
4105
349k
          (isGenericMethod || Ty->isVariablyModifiedType() ||
4106
349k
           Ty.getNonReferenceType()->isObjCRetainableType() ||
4107
349k
           getContext()
4108
349k
                   .getCanonicalType(Ty.getNonReferenceType())
4109
349k
                   .getTypePtr() ==
4110
349k
               getContext().getCanonicalType((*Arg)->getType()).getTypePtr()) &&
4111
349k
          "type mismatch in call argument!");
4112
0
      ++Arg;
4113
349k
    }
4114
4115
    // Either we've emitted all the call args, or we have a call to variadic
4116
    // function.
4117
330k
    assert((Arg == ArgRange.end() || IsVariadic) &&
4118
330k
           "Extra arguments in non-variadic function!");
4119
330k
#endif
4120
330k
  }
4121
4122
  // If we still have any arguments, emit them using the type of the argument.
4123
0
  for (auto *A : llvm::make_range(std::next(ArgRange.begin(), ArgTypes.size()),
4124
331k
                                  ArgRange.end()))
4125
88.0k
    ArgTypes.push_back(IsVariadic ? 
getVarArgType(A)87.3k
:
A->getType()730
);
4126
331k
  assert((int)ArgTypes.size() == (ArgRange.end() - ArgRange.begin()));
4127
4128
  // We must evaluate arguments from right to left in the MS C++ ABI,
4129
  // because arguments are destroyed left to right in the callee. As a special
4130
  // case, there are certain language constructs that require left-to-right
4131
  // evaluation, and in those cases we consider the evaluation order requirement
4132
  // to trump the "destruction order is reverse construction order" guarantee.
4133
0
  bool LeftToRight =
4134
331k
      CGM.getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()
4135
331k
          ? 
Order == EvaluationOrder::ForceLeftToRight7.18k
4136
331k
          : 
Order != EvaluationOrder::ForceRightToLeft324k
;
4137
4138
331k
  auto MaybeEmitImplicitObjectSize = [&](unsigned I, const Expr *Arg,
4139
436k
                                         RValue EmittedArg) {
4140
436k
    if (!AC.hasFunctionDecl() || 
I >= AC.getNumParams()405k
)
4141
116k
      return;
4142
320k
    auto *PS = AC.getParamDecl(I)->getAttr<PassObjectSizeAttr>();
4143
320k
    if (PS == nullptr)
4144
320k
      return;
4145
4146
107
    const auto &Context = getContext();
4147
107
    auto SizeTy = Context.getSizeType();
4148
107
    auto T = Builder.getIntNTy(Context.getTypeSize(SizeTy));
4149
107
    assert(EmittedArg.getScalarVal() && "We emitted nothing for the arg?");
4150
0
    llvm::Value *V = evaluateOrEmitBuiltinObjectSize(Arg, PS->getType(), T,
4151
107
                                                     EmittedArg.getScalarVal(),
4152
107
                                                     PS->isDynamic());
4153
107
    Args.add(RValue::get(V), SizeTy);
4154
    // If we're emitting args in reverse, be sure to do so with
4155
    // pass_object_size, as well.
4156
107
    if (!LeftToRight)
4157
9
      std::swap(Args.back(), *(&Args.back() - 1));
4158
107
  };
4159
4160
  // Insert a stack save if we're going to need any inalloca args.
4161
331k
  if (hasInAllocaArgs(CGM, ExplicitCC, ArgTypes)) {
4162
71
    assert(getTarget().getTriple().getArch() == llvm::Triple::x86 &&
4163
71
           "inalloca only supported on x86");
4164
0
    Args.allocateArgumentMemory(*this);
4165
71
  }
4166
4167
  // Evaluate each argument in the appropriate order.
4168
0
  size_t CallArgsStart = Args.size();
4169
769k
  for (unsigned I = 0, E = ArgTypes.size(); I != E; 
++I437k
) {
4170
437k
    unsigned Idx = LeftToRight ? 
I430k
:
E - I - 17.42k
;
4171
437k
    CallExpr::const_arg_iterator Arg = ArgRange.begin() + Idx;
4172
437k
    unsigned InitialArgSize = Args.size();
4173
    // If *Arg is an ObjCIndirectCopyRestoreExpr, check that either the types of
4174
    // the argument and parameter match or the objc method is parameterized.
4175
437k
    assert((!isa<ObjCIndirectCopyRestoreExpr>(*Arg) ||
4176
437k
            getContext().hasSameUnqualifiedType((*Arg)->getType(),
4177
437k
                                                ArgTypes[Idx]) ||
4178
437k
            (isa<ObjCMethodDecl>(AC.getDecl()) &&
4179
437k
             isObjCMethodWithTypeParams(cast<ObjCMethodDecl>(AC.getDecl())))) &&
4180
437k
           "Argument and parameter types don't match");
4181
0
    EmitCallArg(Args, *Arg, ArgTypes[Idx]);
4182
    // In particular, we depend on it being the last arg in Args, and the
4183
    // objectsize bits depend on there only being one arg if !LeftToRight.
4184
437k
    assert(InitialArgSize + 1 == Args.size() &&
4185
437k
           "The code below depends on only adding one arg per EmitCallArg");
4186
0
    (void)InitialArgSize;
4187
    // Since pointer argument are never emitted as LValue, it is safe to emit
4188
    // non-null argument check for r-value only.
4189
437k
    if (!Args.back().hasLValue()) {
4190
436k
      RValue RVArg = Args.back().getKnownRValue();
4191
436k
      EmitNonNullArgCheck(RVArg, ArgTypes[Idx], (*Arg)->getExprLoc(), AC,
4192
436k
                          ParamsToSkip + Idx);
4193
      // @llvm.objectsize should never have side-effects and shouldn't need
4194
      // destruction/cleanups, so we can safely "emit" it after its arg,
4195
      // regardless of right-to-leftness
4196
436k
      MaybeEmitImplicitObjectSize(Idx, *Arg, RVArg);
4197
436k
    }
4198
437k
  }
4199
4200
331k
  if (!LeftToRight) {
4201
    // Un-reverse the arguments we just evaluated so they match up with the LLVM
4202
    // IR function.
4203
8.02k
    std::reverse(Args.begin() + CallArgsStart, Args.end());
4204
8.02k
  }
4205
331k
}
4206
4207
namespace {
4208
4209
struct DestroyUnpassedArg final : EHScopeStack::Cleanup {
4210
  DestroyUnpassedArg(Address Addr, QualType Ty)
4211
178
      : Addr(Addr), Ty(Ty) {}
4212
4213
  Address Addr;
4214
  QualType Ty;
4215
4216
20
  void Emit(CodeGenFunction &CGF, Flags flags) override {
4217
20
    QualType::DestructionKind DtorKind = Ty.isDestructedType();
4218
20
    if (DtorKind == QualType::DK_cxx_destructor) {
4219
18
      const CXXDestructorDecl *Dtor = Ty->getAsCXXRecordDecl()->getDestructor();
4220
18
      assert(!Dtor->isTrivial());
4221
0
      CGF.EmitCXXDestructorCall(Dtor, Dtor_Complete, /*for vbase*/ false,
4222
18
                                /*Delegating=*/false, Addr, Ty);
4223
18
    } else {
4224
2
      CGF.callCStructDestructor(CGF.MakeAddrLValue(Addr, Ty));
4225
2
    }
4226
20
  }
4227
};
4228
4229
struct DisableDebugLocationUpdates {
4230
  CodeGenFunction &CGF;
4231
  bool disabledDebugInfo;
4232
437k
  DisableDebugLocationUpdates(CodeGenFunction &CGF, const Expr *E) : CGF(CGF) {
4233
437k
    if ((disabledDebugInfo = isa<CXXDefaultArgExpr>(E) && 
CGF.getDebugInfo()3.99k
))
4234
2.82k
      CGF.disableDebugInfo();
4235
437k
  }
4236
437k
  ~DisableDebugLocationUpdates() {
4237
437k
    if (disabledDebugInfo)
4238
2.82k
      CGF.enableDebugInfo();
4239
437k
  }
4240
};
4241
4242
} // end anonymous namespace
4243
4244
267
RValue CallArg::getRValue(CodeGenFunction &CGF) const {
4245
267
  if (!HasLV)
4246
267
    return RV;
4247
0
  LValue Copy = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty), Ty);
4248
0
  CGF.EmitAggregateCopy(Copy, LV, Ty, AggValueSlot::DoesNotOverlap,
4249
0
                        LV.isVolatile());
4250
0
  IsUsed = true;
4251
0
  return RValue::getAggregate(Copy.getAddress(CGF));
4252
267
}
4253
4254
1.23k
void CallArg::copyInto(CodeGenFunction &CGF, Address Addr) const {
4255
1.23k
  LValue Dst = CGF.MakeAddrLValue(Addr, Ty);
4256
1.23k
  if (!HasLV && 
RV.isScalar()1.16k
)
4257
798
    CGF.EmitStoreOfScalar(RV.getScalarVal(), Dst, /*isInit=*/true);
4258
432
  else if (!HasLV && 
RV.isComplex()369
)
4259
369
    CGF.EmitStoreOfComplex(RV.getComplexVal(), Dst, /*init=*/true);
4260
63
  else {
4261
63
    auto Addr = HasLV ? LV.getAddress(CGF) : 
RV.getAggregateAddress()0
;
4262
63
    LValue SrcLV = CGF.MakeAddrLValue(Addr, Ty);
4263
    // We assume that call args are never copied into subobjects.
4264
63
    CGF.EmitAggregateCopy(Dst, SrcLV, Ty, AggValueSlot::DoesNotOverlap,
4265
63
                          HasLV ? LV.isVolatileQualified()
4266
63
                                : 
RV.isVolatileQualified()0
);
4267
63
  }
4268
1.23k
  IsUsed = true;
4269
1.23k
}
4270
4271
void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E,
4272
437k
                                  QualType type) {
4273
437k
  DisableDebugLocationUpdates Dis(*this, E);
4274
437k
  if (const ObjCIndirectCopyRestoreExpr *CRE
4275
437k
        = dyn_cast<ObjCIndirectCopyRestoreExpr>(E)) {
4276
26
    assert(getLangOpts().ObjCAutoRefCount);
4277
0
    return emitWritebackArg(*this, args, CRE);
4278
26
  }
4279
4280
437k
  assert(type->isReferenceType() == E->isGLValue() &&
4281
437k
         "reference binding to unmaterialized r-value!");
4282
4283
437k
  if (E->isGLValue()) {
4284
64.6k
    assert(E->getObjectKind() == OK_Ordinary);
4285
0
    return args.add(EmitReferenceBindingToExpr(E), type);
4286
64.6k
  }
4287
4288
373k
  bool HasAggregateEvalKind = hasAggregateEvaluationKind(type);
4289
4290
  // In the Microsoft C++ ABI, aggregate arguments are destructed by the callee.
4291
  // However, we still have to push an EH-only cleanup in case we unwind before
4292
  // we make it to the call.
4293
373k
  if (type->isRecordType() &&
4294
373k
      
type->castAs<RecordType>()->getDecl()->isParamDestroyedInCallee()10.7k
) {
4295
    // If we're using inalloca, use the argument memory.  Otherwise, use a
4296
    // temporary.
4297
376
    AggValueSlot Slot;
4298
376
    if (args.isUsingInAlloca())
4299
102
      Slot = createPlaceholderSlot(*this, type);
4300
274
    else
4301
274
      Slot = CreateAggTemp(type, "agg.tmp");
4302
4303
376
    bool DestroyedInCallee = true, NeedsEHCleanup = true;
4304
376
    if (const auto *RD = type->getAsCXXRecordDecl())
4305
354
      DestroyedInCallee = RD->hasNonTrivialDestructor();
4306
22
    else
4307
22
      NeedsEHCleanup = needsEHCleanup(type.isDestructedType());
4308
4309
376
    if (DestroyedInCallee)
4310
200
      Slot.setExternallyDestructed();
4311
4312
376
    EmitAggExpr(E, Slot);
4313
376
    RValue RV = Slot.asRValue();
4314
376
    args.add(RV, type);
4315
4316
376
    if (DestroyedInCallee && 
NeedsEHCleanup200
) {
4317
      // Create a no-op GEP between the placeholder and the cleanup so we can
4318
      // RAUW it successfully.  It also serves as a marker of the first
4319
      // instruction where the cleanup is active.
4320
182
      pushFullExprCleanup<DestroyUnpassedArg>(EHCleanup, Slot.getAddress(),
4321
182
                                              type);
4322
      // This unreachable is a temporary marker which will be removed later.
4323
182
      llvm::Instruction *IsActive = Builder.CreateUnreachable();
4324
182
      args.addArgCleanupDeactivation(EHStack.getInnermostEHScope(), IsActive);
4325
182
    }
4326
376
    return;
4327
376
  }
4328
4329
372k
  if (HasAggregateEvalKind && 
isa<ImplicitCastExpr>(E)10.4k
&&
4330
372k
      
cast<CastExpr>(E)->getCastKind() == CK_LValueToRValue853
) {
4331
822
    LValue L = EmitLValue(cast<CastExpr>(E)->getSubExpr());
4332
822
    assert(L.isSimple());
4333
0
    args.addUncopiedAggregate(L, type);
4334
822
    return;
4335
822
  }
4336
4337
371k
  args.add(EmitAnyExprToTemp(E), type);
4338
371k
}
4339
4340
87.3k
QualType CodeGenFunction::getVarArgType(const Expr *Arg) {
4341
  // System headers on Windows define NULL to 0 instead of 0LL on Win64. MSVC
4342
  // implicitly widens null pointer constants that are arguments to varargs
4343
  // functions to pointer-sized ints.
4344
87.3k
  if (!getTarget().getTriple().isOSWindows())
4345
87.2k
    return Arg->getType();
4346
4347
94
  if (Arg->getType()->isIntegerType() &&
4348
94
      getContext().getTypeSize(Arg->getType()) <
4349
53
          getContext().getTargetInfo().getPointerWidth(0) &&
4350
94
      Arg->isNullPointerConstant(getContext(),
4351
29
                                 Expr::NPC_ValueDependentIsNotNull)) {
4352
1
    return getContext().getIntPtrType();
4353
1
  }
4354
4355
93
  return Arg->getType();
4356
94
}
4357
4358
// In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
4359
// optimizer it can aggressively ignore unwind edges.
4360
void
4361
1.75k
CodeGenFunction::AddObjCARCExceptionMetadata(llvm::Instruction *Inst) {
4362
1.75k
  if (CGM.getCodeGenOpts().OptimizationLevel != 0 &&
4363
1.75k
      
!CGM.getCodeGenOpts().ObjCAutoRefCountExceptions410
)
4364
320
    Inst->setMetadata("clang.arc.no_objc_arc_exceptions",
4365
320
                      CGM.getNoObjCARCExceptionsMetadata());
4366
1.75k
}
4367
4368
/// Emits a call to the given no-arguments nounwind runtime function.
4369
llvm::CallInst *
4370
CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
4371
508
                                         const llvm::Twine &name) {
4372
508
  return EmitNounwindRuntimeCall(callee, None, name);
4373
508
}
4374
4375
/// Emits a call to the given nounwind runtime function.
4376
llvm::CallInst *
4377
CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
4378
                                         ArrayRef<llvm::Value *> args,
4379
31.0k
                                         const llvm::Twine &name) {
4380
31.0k
  llvm::CallInst *call = EmitRuntimeCall(callee, args, name);
4381
31.0k
  call->setDoesNotThrow();
4382
31.0k
  return call;
4383
31.0k
}
4384
4385
/// Emits a simple call (never an invoke) to the given no-arguments
4386
/// runtime function.
4387
llvm::CallInst *CodeGenFunction::EmitRuntimeCall(llvm::FunctionCallee callee,
4388
5.61k
                                                 const llvm::Twine &name) {
4389
5.61k
  return EmitRuntimeCall(callee, None, name);
4390
5.61k
}
4391
4392
// Calls which may throw must have operand bundles indicating which funclet
4393
// they are nested within.
4394
SmallVector<llvm::OperandBundleDef, 1>
4395
476k
CodeGenFunction::getBundlesForFunclet(llvm::Value *Callee) {
4396
476k
  SmallVector<llvm::OperandBundleDef, 1> BundleList;
4397
  // There is no need for a funclet operand bundle if we aren't inside a
4398
  // funclet.
4399
476k
  if (!CurrentFuncletPad)
4400
475k
    return BundleList;
4401
4402
  // Skip intrinsics which cannot throw.
4403
639
  auto *CalleeFn = dyn_cast<llvm::Function>(Callee->stripPointerCasts());
4404
639
  if (CalleeFn && 
CalleeFn->isIntrinsic()633
&&
CalleeFn->doesNotThrow()38
)
4405
2
    return BundleList;
4406
4407
637
  BundleList.emplace_back("funclet", CurrentFuncletPad);
4408
637
  return BundleList;
4409
639
}
4410
4411
/// Emits a simple call (never an invoke) to the given runtime function.
4412
llvm::CallInst *CodeGenFunction::EmitRuntimeCall(llvm::FunctionCallee callee,
4413
                                                 ArrayRef<llvm::Value *> args,
4414
89.7k
                                                 const llvm::Twine &name) {
4415
89.7k
  llvm::CallInst *call = Builder.CreateCall(
4416
89.7k
      callee, args, getBundlesForFunclet(callee.getCallee()), name);
4417
89.7k
  call->setCallingConv(getRuntimeCC());
4418
89.7k
  return call;
4419
89.7k
}
4420
4421
/// Emits a call or invoke to the given noreturn runtime function.
4422
void CodeGenFunction::EmitNoreturnRuntimeCallOrInvoke(
4423
594
    llvm::FunctionCallee callee, ArrayRef<llvm::Value *> args) {
4424
594
  SmallVector<llvm::OperandBundleDef, 1> BundleList =
4425
594
      getBundlesForFunclet(callee.getCallee());
4426
4427
594
  if (getInvokeDest()) {
4428
154
    llvm::InvokeInst *invoke =
4429
154
      Builder.CreateInvoke(callee,
4430
154
                           getUnreachableBlock(),
4431
154
                           getInvokeDest(),
4432
154
                           args,
4433
154
                           BundleList);
4434
154
    invoke->setDoesNotReturn();
4435
154
    invoke->setCallingConv(getRuntimeCC());
4436
440
  } else {
4437
440
    llvm::CallInst *call = Builder.CreateCall(callee, args, BundleList);
4438
440
    call->setDoesNotReturn();
4439
440
    call->setCallingConv(getRuntimeCC());
4440
440
    Builder.CreateUnreachable();
4441
440
  }
4442
594
}
4443
4444
/// Emits a call or invoke instruction to the given nullary runtime function.
4445
llvm::CallBase *
4446
CodeGenFunction::EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
4447
406
                                         const Twine &name) {
4448
406
  return EmitRuntimeCallOrInvoke(callee, None, name);
4449
406
}
4450
4451
/// Emits a call or invoke instruction to the given runtime function.
4452
llvm::CallBase *
4453
CodeGenFunction::EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
4454
                                         ArrayRef<llvm::Value *> args,
4455
668
                                         const Twine &name) {
4456
668
  llvm::CallBase *call = EmitCallOrInvoke(callee, args, name);
4457
668
  call->setCallingConv(getRuntimeCC());
4458
668
  return call;
4459
668
}
4460
4461
/// Emits a call or invoke instruction to the given function, depending
4462
/// on the current state of the EH stack.
4463
llvm::CallBase *CodeGenFunction::EmitCallOrInvoke(llvm::FunctionCallee Callee,
4464
                                                  ArrayRef<llvm::Value *> Args,
4465
2.30k
                                                  const Twine &Name) {
4466
2.30k
  llvm::BasicBlock *InvokeDest = getInvokeDest();
4467
2.30k
  SmallVector<llvm::OperandBundleDef, 1> BundleList =
4468
2.30k
      getBundlesForFunclet(Callee.getCallee());
4469
4470
2.30k
  llvm::CallBase *Inst;
4471
2.30k
  if (!InvokeDest)
4472
2.03k
    Inst = Builder.CreateCall(Callee, Args, BundleList, Name);
4473
279
  else {
4474
279
    llvm::BasicBlock *ContBB = createBasicBlock("invoke.cont");
4475
279
    Inst = Builder.CreateInvoke(Callee, ContBB, InvokeDest, Args, BundleList,
4476
279
                                Name);
4477
279
    EmitBlock(ContBB);
4478
279
  }
4479
4480
  // In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
4481
  // optimizer it can aggressively ignore unwind edges.
4482
2.30k
  if (CGM.getLangOpts().ObjCAutoRefCount)
4483
4
    AddObjCARCExceptionMetadata(Inst);
4484
4485
2.30k
  return Inst;
4486
2.30k
}
4487
4488
void CodeGenFunction::deferPlaceholderReplacement(llvm::Instruction *Old,
4489
103
                                                  llvm::Value *New) {
4490
103
  DeferredReplacements.push_back(
4491
103
      std::make_pair(llvm::WeakTrackingVH(Old), New));
4492
103
}
4493
4494
namespace {
4495
4496
/// Specify given \p NewAlign as the alignment of return value attribute. If
4497
/// such attribute already exists, re-set it to the maximal one of two options.
4498
LLVM_NODISCARD llvm::AttributeList
4499
maybeRaiseRetAlignmentAttribute(llvm::LLVMContext &Ctx,
4500
                                const llvm::AttributeList &Attrs,
4501
28
                                llvm::Align NewAlign) {
4502
28
  llvm::Align CurAlign = Attrs.getRetAlignment().valueOrOne();
4503
28
  if (CurAlign >= NewAlign)
4504
2
    return Attrs;
4505
26
  llvm::Attribute AlignAttr = llvm::Attribute::getWithAlignment(Ctx, NewAlign);
4506
26
  return Attrs
4507
26
      .removeAttribute(Ctx, llvm::AttributeList::ReturnIndex,
4508
26
                       llvm::Attribute::AttrKind::Alignment)
4509
26
      .addAttribute(Ctx, llvm::AttributeList::ReturnIndex, AlignAttr);
4510
28
}
4511
4512
template <typename AlignedAttrTy> class AbstractAssumeAlignedAttrEmitter {
4513
protected:
4514
  CodeGenFunction &CGF;
4515
4516
  /// We do nothing if this is, or becomes, nullptr.
4517
  const AlignedAttrTy *AA = nullptr;
4518
4519
  llvm::Value *Alignment = nullptr;      // May or may not be a constant.
4520
  llvm::ConstantInt *OffsetCI = nullptr; // Constant, hopefully zero.
4521
4522
  AbstractAssumeAlignedAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl)
4523
764k
      : CGF(CGF_) {
4524
764k
    if (!FuncDecl)
4525
34.5k
      return;
4526
729k
    AA = FuncDecl->getAttr<AlignedAttrTy>();
4527
729k
  }
CGCall.cpp:(anonymous namespace)::AbstractAssumeAlignedAttrEmitter<clang::AssumeAlignedAttr>::AbstractAssumeAlignedAttrEmitter(clang::CodeGen::CodeGenFunction&, clang::Decl const*)
Line
Count
Source
4523
382k
      : CGF(CGF_) {
4524
382k
    if (!FuncDecl)
4525
17.2k
      return;
4526
364k
    AA = FuncDecl->getAttr<AlignedAttrTy>();
4527
364k
  }
CGCall.cpp:(anonymous namespace)::AbstractAssumeAlignedAttrEmitter<clang::AllocAlignAttr>::AbstractAssumeAlignedAttrEmitter(clang::CodeGen::CodeGenFunction&, clang::Decl const*)
Line
Count
Source
4523
382k
      : CGF(CGF_) {
4524
382k
    if (!FuncDecl)
4525
17.2k
      return;
4526
364k
    AA = FuncDecl->getAttr<AlignedAttrTy>();
4527
364k
  }
4528
4529
public:
4530
  /// If we can, materialize the alignment as an attribute on return value.
4531
  LLVM_NODISCARD llvm::AttributeList
4532
764k
  TryEmitAsCallSiteAttribute(const llvm::AttributeList &Attrs) {
4533
764k
    if (!AA || 
OffsetCI55
||
CGF.SanOpts.has(SanitizerKind::Alignment)50
)
4534
764k
      return Attrs;
4535
41
    const auto *AlignmentCI = dyn_cast<llvm::ConstantInt>(Alignment);
4536
41
    if (!AlignmentCI)
4537
12
      return Attrs;
4538
    // We may legitimately have non-power-of-2 alignment here.
4539
    // If so, this is UB land, emit it via `@llvm.assume` instead.
4540
29
    if (!AlignmentCI->getValue().isPowerOf2())
4541
1
      return Attrs;
4542
28
    llvm::AttributeList NewAttrs = maybeRaiseRetAlignmentAttribute(
4543
28
        CGF.getLLVMContext(), Attrs,
4544
28
        llvm::Align(
4545
28
            AlignmentCI->getLimitedValue(llvm::Value::MaximumAlignment)));
4546
28
    AA = nullptr; // We're done. Disallow doing anything else.
4547
28
    return NewAttrs;
4548
29
  }
CGCall.cpp:(anonymous namespace)::AbstractAssumeAlignedAttrEmitter<clang::AssumeAlignedAttr>::TryEmitAsCallSiteAttribute(llvm::AttributeList const&)
Line
Count
Source
4532
382k
  TryEmitAsCallSiteAttribute(const llvm::AttributeList &Attrs) {
4533
382k
    if (!AA || 
OffsetCI14
||
CGF.SanOpts.has(SanitizerKind::Alignment)9
)
4534
382k
      return Attrs;
4535
6
    const auto *AlignmentCI = dyn_cast<llvm::ConstantInt>(Alignment);
4536
6
    if (!AlignmentCI)
4537
0
      return Attrs;
4538
    // We may legitimately have non-power-of-2 alignment here.
4539
    // If so, this is UB land, emit it via `@llvm.assume` instead.
4540
6
    if (!AlignmentCI->getValue().isPowerOf2())
4541
0
      return Attrs;
4542
6
    llvm::AttributeList NewAttrs = maybeRaiseRetAlignmentAttribute(
4543
6
        CGF.getLLVMContext(), Attrs,
4544
6
        llvm::Align(
4545
6
            AlignmentCI->getLimitedValue(llvm::Value::MaximumAlignment)));
4546
6
    AA = nullptr; // We're done. Disallow doing anything else.
4547
6
    return NewAttrs;
4548
6
  }
CGCall.cpp:(anonymous namespace)::AbstractAssumeAlignedAttrEmitter<clang::AllocAlignAttr>::TryEmitAsCallSiteAttribute(llvm::AttributeList const&)
Line
Count
Source
4532
382k
  TryEmitAsCallSiteAttribute(const llvm::AttributeList &Attrs) {
4533
382k
    if (!AA || 
OffsetCI41
||
CGF.SanOpts.has(SanitizerKind::Alignment)41
)
4534
381k
      return Attrs;
4535
35
    const auto *AlignmentCI = dyn_cast<llvm::ConstantInt>(Alignment);
4536
35
    if (!AlignmentCI)
4537
12
      return Attrs;
4538
    // We may legitimately have non-power-of-2 alignment here.
4539
    // If so, this is UB land, emit it via `@llvm.assume` instead.
4540
23
    if (!AlignmentCI->getValue().isPowerOf2())
4541
1
      return Attrs;
4542
22
    llvm::AttributeList NewAttrs = maybeRaiseRetAlignmentAttribute(
4543
22
        CGF.getLLVMContext(), Attrs,
4544
22
        llvm::Align(
4545
22
            AlignmentCI->getLimitedValue(llvm::Value::MaximumAlignment)));
4546
22
    AA = nullptr; // We're done. Disallow doing anything else.
4547
22
    return NewAttrs;
4548
23
  }
4549
4550
  /// Emit alignment assumption.
4551
  /// This is a general fallback that we take if either there is an offset,
4552
  /// or the alignment is variable or we are sanitizing for alignment.
4553
710k
  void EmitAsAnAssumption(SourceLocation Loc, QualType RetTy, RValue &Ret) {
4554
710k
    if (!AA)
4555
710k
      return;
4556
27
    CGF.emitAlignmentAssumption(Ret.getScalarVal(), RetTy, Loc,
4557
27
                                AA->getLocation(), Alignment, OffsetCI);
4558
27
    AA = nullptr; // We're done. Disallow doing anything else.
4559
27
  }
CGCall.cpp:(anonymous namespace)::AbstractAssumeAlignedAttrEmitter<clang::AssumeAlignedAttr>::EmitAsAnAssumption(clang::SourceLocation, clang::QualType, clang::CodeGen::RValue&)
Line
Count
Source
4553
355k
  void EmitAsAnAssumption(SourceLocation Loc, QualType RetTy, RValue &Ret) {
4554
355k
    if (!AA)
4555
355k
      return;
4556
8
    CGF.emitAlignmentAssumption(Ret.getScalarVal(), RetTy, Loc,
4557
8
                                AA->getLocation(), Alignment, OffsetCI);
4558
8
    AA = nullptr; // We're done. Disallow doing anything else.
4559
8
  }
CGCall.cpp:(anonymous namespace)::AbstractAssumeAlignedAttrEmitter<clang::AllocAlignAttr>::EmitAsAnAssumption(clang::SourceLocation, clang::QualType, clang::CodeGen::RValue&)
Line
Count
Source
4553
355k
  void EmitAsAnAssumption(SourceLocation Loc, QualType RetTy, RValue &Ret) {
4554
355k
    if (!AA)
4555
355k
      return;
4556
19
    CGF.emitAlignmentAssumption(Ret.getScalarVal(), RetTy, Loc,
4557
19
                                AA->getLocation(), Alignment, OffsetCI);
4558
19
    AA = nullptr; // We're done. Disallow doing anything else.
4559
19
  }
4560
};
4561
4562
/// Helper data structure to emit `AssumeAlignedAttr`.
4563
class AssumeAlignedAttrEmitter final
4564
    : public AbstractAssumeAlignedAttrEmitter<AssumeAlignedAttr> {
4565
public:
4566
  AssumeAlignedAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl)
4567
382k
      : AbstractAssumeAlignedAttrEmitter(CGF_, FuncDecl) {
4568
382k
    if (!AA)
4569
382k
      return;
4570
    // It is guaranteed that the alignment/offset are constants.
4571
14
    Alignment = cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AA->getAlignment()));
4572
14
    if (Expr *Offset = AA->getOffset()) {
4573
5
      OffsetCI = cast<llvm::ConstantInt>(CGF.EmitScalarExpr(Offset));
4574
5
      if (OffsetCI->isNullValue()) // Canonicalize zero offset to no offset.
4575
0
        OffsetCI = nullptr;
4576
5
    }
4577
14
  }
4578
};
4579
4580
/// Helper data structure to emit `AllocAlignAttr`.
4581
class AllocAlignAttrEmitter final
4582
    : public AbstractAssumeAlignedAttrEmitter<AllocAlignAttr> {
4583
public:
4584
  AllocAlignAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl,
4585
                        const CallArgList &CallArgs)
4586
382k
      : AbstractAssumeAlignedAttrEmitter(CGF_, FuncDecl) {
4587
382k
    if (!AA)
4588
381k
      return;
4589
    // Alignment may or may not be a constant, and that is okay.
4590
41
    Alignment = CallArgs[AA->getParamIndex().getLLVMIndex()]
4591
41
                    .getRValue(CGF)
4592
41
                    .getScalarVal();
4593
41
  }
4594
};
4595
4596
} // namespace
4597
4598
RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
4599
                                 const CGCallee &Callee,
4600
                                 ReturnValueSlot ReturnValue,
4601
                                 const CallArgList &CallArgs,
4602
                                 llvm::CallBase **callOrInvoke, bool IsMustTail,
4603
382k
                                 SourceLocation Loc) {
4604
  // FIXME: We no longer need the types from CallArgs; lift up and simplify.
4605
4606
382k
  assert(Callee.isOrdinary() || Callee.isVirtual());
4607
4608
  // Handle struct-return functions by passing a pointer to the
4609
  // location that we would like to return into.
4610
0
  QualType RetTy = CallInfo.getReturnType();
4611
382k
  const ABIArgInfo &RetAI = CallInfo.getReturnInfo();
4612
4613
382k
  llvm::FunctionType *IRFuncTy = getTypes().GetFunctionType(CallInfo);
4614
4615
382k
  const Decl *TargetDecl = Callee.getAbstractInfo().getCalleeDecl().getDecl();
4616
382k
  if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(TargetDecl)) {
4617
    // We can only guarantee that a function is called from the correct
4618
    // context/function based on the appropriate target attributes,
4619
    // so only check in the case where we have both always_inline and target
4620
    // since otherwise we could be making a conditional call after a check for
4621
    // the proper cpu features (and it won't cause code generation issues due to
4622
    // function based code generation).
4623
359k
    if (TargetDecl->hasAttr<AlwaysInlineAttr>() &&
4624
359k
        
TargetDecl->hasAttr<TargetAttr>()39.7k
)
4625
12.2k
      checkTargetFeatures(Loc, FD);
4626
4627
    // Some architectures (such as x86-64) have the ABI changed based on
4628
    // attribute-target/features. Give them a chance to diagnose.
4629
359k
    CGM.getTargetCodeGenInfo().checkFunctionCallABI(
4630
359k
        CGM, Loc, dyn_cast_or_null<FunctionDecl>(CurCodeDecl), FD, CallArgs);
4631
359k
  }
4632
4633
382k
#ifndef NDEBUG
4634
382k
  if (!(CallInfo.isVariadic() && 
CallInfo.getArgStruct()76.3k
)) {
4635
    // For an inalloca varargs function, we don't expect CallInfo to match the
4636
    // function pointer's type, because the inalloca struct a will have extra
4637
    // fields in it for the varargs parameters.  Code later in this function
4638
    // bitcasts the function pointer to the type derived from CallInfo.
4639
    //
4640
    // In other cases, we assert that the types match up (until pointers stop
4641
    // having pointee types).
4642
382k
    llvm::Type *TypeFromVal;
4643
382k
    if (Callee.isVirtual())
4644
1.00k
      TypeFromVal = Callee.getVirtualFunctionType();
4645
381k
    else
4646
381k
      TypeFromVal =
4647
381k
          Callee.getFunctionPointer()->getType()->getPointerElementType();
4648
382k
    assert(IRFuncTy == TypeFromVal);
4649
382k
  }
4650
0
#endif
4651
4652
  // 1. Set up the arguments.
4653
4654
  // If we're using inalloca, insert the allocation after the stack save.
4655
  // FIXME: Do this earlier rather than hacking it in here!
4656
0
  Address ArgMemory = Address::invalid();
4657
382k
  if (llvm::StructType *ArgStruct = CallInfo.getArgStruct()) {
4658
72
    const llvm::DataLayout &DL = CGM.getDataLayout();
4659
72
    llvm::Instruction *IP = CallArgs.getStackBase();
4660
72
    llvm::AllocaInst *AI;
4661
72
    if (IP) {
4662
71
      IP = IP->getNextNode();
4663
71
      AI = new llvm::AllocaInst(ArgStruct, DL.getAllocaAddrSpace(),
4664
71
                                "argmem", IP);
4665
71
    } else {
4666
1
      AI = CreateTempAlloca(ArgStruct, "argmem");
4667
1
    }
4668
72
    auto Align = CallInfo.getArgStructAlignment();
4669
72
    AI->setAlignment(Align.getAsAlign());
4670
72
    AI->setUsedWithInAlloca(true);
4671
72
    assert(AI->isUsedWithInAlloca() && !AI->isStaticAlloca());
4672
0
    ArgMemory = Address(AI, Align);
4673
72
  }
4674
4675
0
  ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), CallInfo);
4676
382k
  SmallVector<llvm::Value *, 16> IRCallArgs(IRFunctionArgs.totalIRArgs());
4677
4678
  // If the call returns a temporary with struct return, create a temporary
4679
  // alloca to hold the result, unless one is given to us.
4680
382k
  Address SRetPtr = Address::invalid();
4681
382k
  Address SRetAlloca = Address::invalid();
4682
382k
  llvm::Value *UnusedReturnSizePtr = nullptr;
4683
382k
  if (RetAI.isIndirect() || 
RetAI.isInAlloca()380k
||
RetAI.isCoerceAndExpand()380k
) {
4684
2.08k
    if (!ReturnValue.isNull()) {
4685
1.85k
      SRetPtr = ReturnValue.getValue();
4686
1.85k
    } else {
4687
225
      SRetPtr = CreateMemTemp(RetTy, "tmp", &SRetAlloca);
4688
225
      if (HaveInsertPoint() && ReturnValue.isUnused()) {
4689
133
        llvm::TypeSize size =
4690
133
            CGM.getDataLayout().getTypeAllocSize(ConvertTypeForMem(RetTy));
4691
133
        UnusedReturnSizePtr = EmitLifetimeStart(size, SRetAlloca.getPointer());
4692
133
      }
4693
225
    }
4694
2.08k
    if (IRFunctionArgs.hasSRetArg()) {
4695
1.63k
      IRCallArgs[IRFunctionArgs.getSRetArgNo()] = SRetPtr.getPointer();
4696
1.63k
    } else 
if (446
RetAI.isInAlloca()446
) {
4697
4
      Address Addr =
4698
4
          Builder.CreateStructGEP(ArgMemory, RetAI.getInAllocaFieldIndex());
4699
4
      Builder.CreateStore(SRetPtr.getPointer(), Addr);
4700
4
    }
4701
2.08k
  }
4702
4703
382k
  Address swiftErrorTemp = Address::invalid();
4704
382k
  Address swiftErrorArg = Address::invalid();
4705
4706
  // When passing arguments using temporary allocas, we need to add the
4707
  // appropriate lifetime markers. This vector keeps track of all the lifetime
4708
  // markers that need to be ended right after the call.
4709
382k
  SmallVector<CallLifetimeEnd, 2> CallLifetimeEndAfterCall;
4710
4711
  // Translate all of the arguments as necessary to match the IR lowering.
4712
382k
  assert(CallInfo.arg_size() == CallArgs.size() &&
4713
382k
         "Mismatch between function signature & arguments.");
4714
0
  unsigned ArgNo = 0;
4715
382k
  CGFunctionInfo::const_arg_iterator info_it = CallInfo.arg_begin();
4716
382k
  for (CallArgList::const_iterator I = CallArgs.begin(), E = CallArgs.end();
4717
1.03M
       I != E; 
++I, ++info_it, ++ArgNo651k
) {
4718
651k
    const ABIArgInfo &ArgInfo = info_it->info;
4719
4720
    // Insert a padding argument to ensure proper alignment.
4721
651k
    if (IRFunctionArgs.hasPaddingArg(ArgNo))
4722
17
      IRCallArgs[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
4723
17
          llvm::UndefValue::get(ArgInfo.getPaddingType());
4724
4725
651k
    unsigned FirstIRArg, NumIRArgs;
4726
651k
    std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
4727
4728
651k
    switch (ArgInfo.getKind()) {
4729
156
    case ABIArgInfo::InAlloca: {
4730
156
      assert(NumIRArgs == 0);
4731
0
      assert(getTarget().getTriple().getArch() == llvm::Triple::x86);
4732
156
      if (I->isAggregate()) {
4733
103
        Address Addr = I->hasLValue()
4734
103
                           ? 
I->getKnownLValue().getAddress(*this)0
4735
103
                           : I->getKnownRValue().getAggregateAddress();
4736
103
        llvm::Instruction *Placeholder =
4737
103
            cast<llvm::Instruction>(Addr.getPointer());
4738
4739
103
        if (!ArgInfo.getInAllocaIndirect()) {
4740
          // Replace the placeholder with the appropriate argument slot GEP.
4741
101
          CGBuilderTy::InsertPoint IP = Builder.saveIP();
4742
101
          Builder.SetInsertPoint(Placeholder);
4743
101
          Addr = Builder.CreateStructGEP(ArgMemory,
4744
101
                                         ArgInfo.getInAllocaFieldIndex());
4745
101
          Builder.restoreIP(IP);
4746
101
        } else {
4747
          // For indirect things such as overaligned structs, replace the
4748
          // placeholder with a regular aggregate temporary alloca. Store the
4749
          // address of this alloca into the struct.
4750
2
          Addr = CreateMemTemp(info_it->type, "inalloca.indirect.tmp");
4751
2
          Address ArgSlot = Builder.CreateStructGEP(
4752
2
              ArgMemory, ArgInfo.getInAllocaFieldIndex());
4753
2
          Builder.CreateStore(Addr.getPointer(), ArgSlot);
4754
2
        }
4755
103
        deferPlaceholderReplacement(Placeholder, Addr.getPointer());
4756
103
      } else 
if (53
ArgInfo.getInAllocaIndirect()53
) {
4757
        // Make a temporary alloca and store the address of it into the argument
4758
        // struct.
4759
2
        Address Addr = CreateMemTempWithoutCast(
4760
2
            I->Ty, getContext().getTypeAlignInChars(I->Ty),
4761
2
            "indirect-arg-temp");
4762
2
        I->copyInto(*this, Addr);
4763
2
        Address ArgSlot =
4764
2
            Builder.CreateStructGEP(ArgMemory, ArgInfo.getInAllocaFieldIndex());
4765
2
        Builder.CreateStore(Addr.getPointer(), ArgSlot);
4766
51
      } else {
4767
        // Store the RValue into the argument struct.
4768
51
        Address Addr =
4769
51
            Builder.CreateStructGEP(ArgMemory, ArgInfo.getInAllocaFieldIndex());
4770
51
        unsigned AS = Addr.getType()->getPointerAddressSpace();
4771
51
        llvm::Type *MemType = ConvertTypeForMem(I->Ty)->getPointerTo(AS);
4772
        // There are some cases where a trivial bitcast is not avoidable.  The
4773
        // definition of a type later in a translation unit may change it's type
4774
        // from {}* to (%struct.foo*)*.
4775
51
        if (Addr.getType() != MemType)
4776
1
          Addr = Builder.CreateBitCast(Addr, MemType);
4777
51
        I->copyInto(*this, Addr);
4778
51
      }
4779
156
      break;
4780
0
    }
4781
4782
2.04k
    case ABIArgInfo::Indirect:
4783
2.04k
    case ABIArgInfo::IndirectAliased: {
4784
2.04k
      assert(NumIRArgs == 1);
4785
2.04k
      if (!I->isAggregate()) {
4786
        // Make a temporary alloca to pass the argument.
4787
230
        Address Addr = CreateMemTempWithoutCast(
4788
230
            I->Ty, ArgInfo.getIndirectAlign(), "indirect-arg-temp");
4789
230
        IRCallArgs[FirstIRArg] = Addr.getPointer();
4790
4791
230
        I->copyInto(*this, Addr);
4792
1.81k
      } else {
4793
        // We want to avoid creating an unnecessary temporary+copy here;
4794
        // however, we need one in three cases:
4795
        // 1. If the argument is not byval, and we are required to copy the
4796
        //    source.  (This case doesn't occur on any common architecture.)
4797
        // 2. If the argument is byval, RV is not sufficiently aligned, and
4798
        //    we cannot force it to be sufficiently aligned.
4799
        // 3. If the argument is byval, but RV is not located in default
4800
        //    or alloca address space.
4801
1.81k
        Address Addr = I->hasLValue()
4802
1.81k
                           ? 
I->getKnownLValue().getAddress(*this)331
4803
1.81k
                           : 
I->getKnownRValue().getAggregateAddress()1.48k
;
4804
1.81k
        llvm::Value *V = Addr.getPointer();
4805
1.81k
        CharUnits Align = ArgInfo.getIndirectAlign();
4806
1.81k
        const llvm::DataLayout *TD = &CGM.getDataLayout();
4807
4808
1.81k
        assert((FirstIRArg >= IRFuncTy->getNumParams() ||
4809
1.81k
                IRFuncTy->getParamType(FirstIRArg)->getPointerAddressSpace() ==
4810
1.81k
                    TD->getAllocaAddrSpace()) &&
4811
1.81k
               "indirect argument must be in alloca address space");
4812
4813
0
        bool NeedCopy = false;
4814
4815
1.81k
        if (Addr.getAlignment() < Align &&
4816
1.81k
            llvm::getOrEnforceKnownAlignment(V, Align.getAsAlign(), *TD) <
4817
92
                Align.getAsAlign()) {
4818
2
          NeedCopy = true;
4819
1.81k
        } else if (I->hasLValue()) {
4820
329
          auto LV = I->getKnownLValue();
4821
329
          auto AS = LV.getAddressSpace();
4822
4823
329
          if (!ArgInfo.getIndirectByVal() ||
4824
329
              
(LV.getAlignment() < getContext().getTypeAlignInChars(I->Ty))286
) {
4825
43
            NeedCopy = true;
4826
43
          }
4827
329
          if (!getLangOpts().OpenCL) {
4828
288
            if ((ArgInfo.getIndirectByVal() &&
4829
288
                
(245
AS != LangAS::Default245
&&
4830
245
                 
AS != CGM.getASTAllocaAddressSpace()0
))) {
4831
0
              NeedCopy = true;
4832
0
            }
4833
288
          }
4834
          // For OpenCL even if RV is located in default or alloca address space
4835
          // we don't want to perform address space cast for it.
4836
41
          else if ((ArgInfo.getIndirectByVal() &&
4837
41
                    Addr.getType()->getAddressSpace() != IRFuncTy->
4838
41
                      getParamType(FirstIRArg)->getPointerAddressSpace())) {
4839
18
            NeedCopy = true;
4840
18
          }
4841
329
        }
4842
4843
1.81k
        if (NeedCopy) {
4844
          // Create an aligned temporary, and copy to it.
4845
63
          Address AI = CreateMemTempWithoutCast(
4846
63
              I->Ty, ArgInfo.getIndirectAlign(), "byval-temp");
4847
63
          IRCallArgs[FirstIRArg] = AI.getPointer();
4848
4849
          // Emit lifetime markers for the temporary alloca.
4850
63
          llvm::TypeSize ByvalTempElementSize =
4851
63
              CGM.getDataLayout().getTypeAllocSize(AI.getElementType());
4852
63
          llvm::Value *LifetimeSize =
4853
63
              EmitLifetimeStart(ByvalTempElementSize, AI.getPointer());
4854
4855
          // Add cleanup code to emit the end lifetime marker after the call.
4856
63
          if (LifetimeSize) // In case we disabled lifetime markers.
4857
2
            CallLifetimeEndAfterCall.emplace_back(AI, LifetimeSize);
4858
4859
          // Generate the copy.
4860
63
          I->copyInto(*this, AI);
4861
1.74k
        } else {
4862
          // Skip the extra memcpy call.
4863
1.74k
          auto *T = V->getType()->getPointerElementType()->getPointerTo(
4864
1.74k
              CGM.getDataLayout().getAllocaAddrSpace());
4865
1.74k
          IRCallArgs[FirstIRArg] = getTargetHooks().performAddrSpaceCast(
4866
1.74k
              *this, V, LangAS::Default, CGM.getASTAllocaAddressSpace(), T,
4867
1.74k
              true);
4868
1.74k
        }
4869
1.81k
      }
4870
0
      break;
4871
2.04k
    }
4872
4873
7.22k
    case ABIArgInfo::Ignore:
4874
7.22k
      assert(NumIRArgs == 0);
4875
0
      break;
4876
4877
11.3k
    case ABIArgInfo::Extend:
4878
641k
    case ABIArgInfo::Direct: {
4879
641k
      if (!isa<llvm::StructType>(ArgInfo.getCoerceToType()) &&
4880
641k
          
ArgInfo.getCoerceToType() == ConvertType(info_it->type)640k
&&
4881
641k
          
ArgInfo.getDirectOffset() == 0636k
) {
4882
636k
        assert(NumIRArgs == 1);
4883
0
        llvm::Value *V;
4884
636k
        if (!I->isAggregate())
4885
636k
          V = I->getKnownRValue().getScalarVal();
4886
3
        else
4887
3
          V = Builder.CreateLoad(
4888
3
              I->hasLValue() ? 
I->getKnownLValue().getAddress(*this)0
4889
3
                             : I->getKnownRValue().getAggregateAddress());
4890
4891
        // Implement swifterror by copying into a new swifterror argument.
4892
        // We'll write back in the normal path out of the call.
4893
636k
        if (CallInfo.getExtParameterInfo(ArgNo).getABI()
4894
636k
              == ParameterABI::SwiftErrorResult) {
4895
8
          assert(!swiftErrorTemp.isValid() && "multiple swifterror args");
4896
4897
0
          QualType pointeeTy = I->Ty->getPointeeType();
4898
8
          swiftErrorArg =
4899
8
            Address(V, getContext().getTypeAlignInChars(pointeeTy));
4900
4901
8
          swiftErrorTemp =
4902
8
            CreateMemTemp(pointeeTy, getPointerAlign(), "swifterror.temp");
4903
8
          V = swiftErrorTemp.getPointer();
4904
8
          cast<llvm::AllocaInst>(V)->setSwiftError(true);
4905
4906
8
          llvm::Value *errorValue = Builder.CreateLoad(swiftErrorArg);
4907
8
          Builder.CreateStore(errorValue, swiftErrorTemp);
4908
8
        }
4909
4910
        // We might have to widen integers, but we should never truncate.
4911
636k
        if (ArgInfo.getCoerceToType() != V->getType() &&
4912
636k
            
V->getType()->isIntegerTy()699
)
4913
3
          V = Builder.CreateZExt(V, ArgInfo.getCoerceToType());
4914
4915
        // If the argument doesn't match, perform a bitcast to coerce it.  This
4916
        // can happen due to trivial type mismatches.
4917
636k
        if (FirstIRArg < IRFuncTy->getNumParams() &&
4918
636k
            
V->getType() != IRFuncTy->getParamType(FirstIRArg)549k
)
4919
696
          V = Builder.CreateBitCast(V, IRFuncTy->getParamType(FirstIRArg));
4920
4921
636k
        IRCallArgs[FirstIRArg] = V;
4922
636k
        break;
4923
636k
      }
4924
4925
      // FIXME: Avoid the conversion through memory if possible.
4926
4.64k
      Address Src = Address::invalid();
4927
4.64k
      if (!I->isAggregate()) {
4928
884
        Src = CreateMemTemp(I->Ty, "coerce");
4929
884
        I->copyInto(*this, Src);
4930
3.76k
      } else {
4931
3.76k
        Src = I->hasLValue() ? 
I->getKnownLValue().getAddress(*this)472
4932
3.76k
                             : 
I->getKnownRValue().getAggregateAddress()3.28k
;
4933
3.76k
      }
4934
4935
      // If the value is offset in memory, apply the offset now.
4936
4.64k
      Src = emitAddressAtOffset(*this, Src, ArgInfo);
4937
4938
      // Fast-isel and the optimizer generally like scalar values better than
4939
      // FCAs, so we flatten them if this is safe to do for this argument.
4940
4.64k
      llvm::StructType *STy =
4941
4.64k
            dyn_cast<llvm::StructType>(ArgInfo.getCoerceToType());
4942
4.64k
      if (STy && 
ArgInfo.isDirect()1.27k
&&
ArgInfo.getCanBeFlattened()1.27k
) {
4943
1.24k
        llvm::Type *SrcTy = Src.getElementType();
4944
1.24k
        uint64_t SrcSize = CGM.getDataLayout().getTypeAllocSize(SrcTy);
4945
1.24k
        uint64_t DstSize = CGM.getDataLayout().getTypeAllocSize(STy);
4946
4947
        // If the source type is smaller than the destination type of the
4948
        // coerce-to logic, copy the source value into a temp alloca the size
4949
        // of the destination type to allow loading all of it. The bits past
4950
        // the source value are left undef.
4951
1.24k
        if (SrcSize < DstSize) {
4952
154
          Address TempAlloca
4953
154
            = CreateTempAlloca(STy, Src.getAlignment(),
4954
154
                               Src.getName() + ".coerce");
4955
154
          Builder.CreateMemCpy(TempAlloca, Src, SrcSize);
4956
154
          Src = TempAlloca;
4957
1.08k
        } else {
4958
1.08k
          Src = Builder.CreateBitCast(Src,
4959
1.08k
                                      STy->getPointerTo(Src.getAddressSpace()));
4960
1.08k
        }
4961
4962
1.24k
        assert(NumIRArgs == STy->getNumElements());
4963
3.73k
        for (unsigned i = 0, e = STy->getNumElements(); i != e; 
++i2.49k
) {
4964
2.49k
          Address EltPtr = Builder.CreateStructGEP(Src, i);
4965
2.49k
          llvm::Value *LI = Builder.CreateLoad(EltPtr);
4966
2.49k
          IRCallArgs[FirstIRArg + i] = LI;
4967
2.49k
        }
4968
3.40k
      } else {
4969
        // In the simple case, just pass the coerced loaded value.
4970
3.40k
        assert(NumIRArgs == 1);
4971
0
        llvm::Value *Load =
4972
3.40k
            CreateCoercedLoad(Src, ArgInfo.getCoerceToType(), *this);
4973
4974
3.40k
        if (CallInfo.isCmseNSCall()) {
4975
          // For certain parameter types, clear padding bits, as they may reveal
4976
          // sensitive information.
4977
          // Small struct/union types are passed as integer arrays.
4978
40
          auto *ATy = dyn_cast<llvm::ArrayType>(Load->getType());
4979
40
          if (ATy != nullptr && 
isa<RecordType>(I->Ty.getCanonicalType())39
)
4980
39
            Load = EmitCMSEClearRecord(Load, ATy, I->Ty);
4981
40
        }
4982
3.40k
        IRCallArgs[FirstIRArg] = Load;
4983
3.40k
      }
4984
4985
0
      break;
4986
641k
    }
4987
4988
442
    case ABIArgInfo::CoerceAndExpand: {
4989
442
      auto coercionType = ArgInfo.getCoerceAndExpandType();
4990
442
      auto layout = CGM.getDataLayout().getStructLayout(coercionType);
4991
4992
442
      llvm::Value *tempSize = nullptr;
4993
442
      Address addr = Address::invalid();
4994
442
      Address AllocaAddr = Address::invalid();
4995
442
      if (I->isAggregate()) {
4996
414
        addr = I->hasLValue() ? 
I->getKnownLValue().getAddress(*this)0
4997
414
                              : I->getKnownRValue().getAggregateAddress();
4998
4999
414
      } else {
5000
28
        RValue RV = I->getKnownRValue();
5001
28
        assert(RV.isScalar()); // complex should always just be direct
5002
5003
0
        llvm::Type *scalarType = RV.getScalarVal()->getType();
5004
28
        auto scalarSize = CGM.getDataLayout().getTypeAllocSize(scalarType);
5005
28
        auto scalarAlign = CGM.getDataLayout().getPrefTypeAlignment(scalarType);
5006
5007
        // Materialize to a temporary.
5008
28
        addr = CreateTempAlloca(
5009
28
            RV.getScalarVal()->getType(),
5010
28
            CharUnits::fromQuantity(std::max(
5011
28
                (unsigned)layout->getAlignment().value(), scalarAlign)),
5012
28
            "tmp",
5013
28
            /*ArraySize=*/nullptr, &AllocaAddr);
5014
28
        tempSize = EmitLifetimeStart(scalarSize, AllocaAddr.getPointer());
5015
5016
28
        Builder.CreateStore(RV.getScalarVal(), addr);
5017
28
      }
5018
5019
0
      addr = Builder.CreateElementBitCast(addr, coercionType);
5020
5021
442
      unsigned IRArgPos = FirstIRArg;
5022
1.41k
      for (unsigned i = 0, e = coercionType->getNumElements(); i != e; 
++i973
) {
5023
973
        llvm::Type *eltType = coercionType->getElementType(i);
5024
973
        if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType)) 
continue7
;
5025
966
        Address eltAddr = Builder.CreateStructGEP(addr, i);
5026
966
        llvm::Value *elt = Builder.CreateLoad(eltAddr);
5027
966
        IRCallArgs[IRArgPos++] = elt;
5028
966
      }
5029
442
      assert(IRArgPos == FirstIRArg + NumIRArgs);
5030
5031
442
      if (tempSize) {
5032
0
        EmitLifetimeEnd(tempSize, AllocaAddr.getPointer());
5033
0
      }
5034
5035
442
      break;
5036
641k
    }
5037
5038
31
    case ABIArgInfo::Expand: {
5039
31
      unsigned IRArgPos = FirstIRArg;
5040
31
      ExpandTypeToArgs(I->Ty, *I, IRFuncTy, IRCallArgs, IRArgPos);
5041
31
      assert(IRArgPos == FirstIRArg + NumIRArgs);
5042
0
      break;
5043
641k
    }
5044
651k
    }
5045
651k
  }
5046
5047
382k
  const CGCallee &ConcreteCallee = Callee.prepareConcreteCallee(*this);
5048
382k
  llvm::Value *CalleePtr = ConcreteCallee.getFunctionPointer();
5049
5050
  // If we're using inalloca, set up that argument.
5051
382k
  if (ArgMemory.isValid()) {
5052
72
    llvm::Value *Arg = ArgMemory.getPointer();
5053
72
    if (CallInfo.isVariadic()) {
5054
      // When passing non-POD arguments by value to variadic functions, we will
5055
      // end up with a variadic prototype and an inalloca call site.  In such
5056
      // cases, we can't do any parameter mismatch checks.  Give up and bitcast
5057
      // the callee.
5058
10
      unsigned CalleeAS = CalleePtr->getType()->getPointerAddressSpace();
5059
10
      CalleePtr =
5060
10
          Builder.CreateBitCast(CalleePtr, IRFuncTy->getPointerTo(CalleeAS));
5061
62
    } else {
5062
62
      llvm::Type *LastParamTy =
5063
62
          IRFuncTy->getParamType(IRFuncTy->getNumParams() - 1);
5064
62
      if (Arg->getType() != LastParamTy) {
5065
0
#ifndef NDEBUG
5066
        // Assert that these structs have equivalent element types.
5067
0
        llvm::StructType *FullTy = CallInfo.getArgStruct();
5068
0
        llvm::StructType *DeclaredTy = cast<llvm::StructType>(
5069
0
            cast<llvm::PointerType>(LastParamTy)->getElementType());
5070
0
        assert(DeclaredTy->getNumElements() == FullTy->getNumElements());
5071
0
        for (llvm::StructType::element_iterator DI = DeclaredTy->element_begin(),
5072
0
                                                DE = DeclaredTy->element_end(),
5073
0
                                                FI = FullTy->element_begin();
5074
0
             DI != DE; ++DI, ++FI)
5075
0
          assert(*DI == *FI);
5076
0
#endif
5077
0
        Arg = Builder.CreateBitCast(Arg, LastParamTy