Coverage Report

Created: 2021-09-21 08:58

/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/CodeGen/CGCall.cpp
Line
Count
Source (jump to first uncovered line)
1
//===--- CGCall.cpp - Encapsulate calling convention details --------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// These classes wrap the information about a call or function
10
// definition used to handle ABI compliancy.
11
//
12
//===----------------------------------------------------------------------===//
13
14
#include "CGCall.h"
15
#include "ABIInfo.h"
16
#include "CGBlocks.h"
17
#include "CGCXXABI.h"
18
#include "CGCleanup.h"
19
#include "CGRecordLayout.h"
20
#include "CodeGenFunction.h"
21
#include "CodeGenModule.h"
22
#include "TargetInfo.h"
23
#include "clang/AST/Attr.h"
24
#include "clang/AST/Decl.h"
25
#include "clang/AST/DeclCXX.h"
26
#include "clang/AST/DeclObjC.h"
27
#include "clang/Basic/CodeGenOptions.h"
28
#include "clang/Basic/TargetBuiltins.h"
29
#include "clang/Basic/TargetInfo.h"
30
#include "clang/CodeGen/CGFunctionInfo.h"
31
#include "clang/CodeGen/SwiftCallingConv.h"
32
#include "llvm/ADT/StringExtras.h"
33
#include "llvm/Analysis/ValueTracking.h"
34
#include "llvm/IR/Assumptions.h"
35
#include "llvm/IR/Attributes.h"
36
#include "llvm/IR/CallingConv.h"
37
#include "llvm/IR/DataLayout.h"
38
#include "llvm/IR/InlineAsm.h"
39
#include "llvm/IR/IntrinsicInst.h"
40
#include "llvm/IR/Intrinsics.h"
41
#include "llvm/Transforms/Utils/Local.h"
42
using namespace clang;
43
using namespace CodeGen;
44
45
/***/
46
47
289k
unsigned CodeGenTypes::ClangCallConvToLLVMCallConv(CallingConv CC) {
48
289k
  switch (CC) {
49
284k
  default: return llvm::CallingConv::C;
50
56
  case CC_X86StdCall: return llvm::CallingConv::X86_StdCall;
51
77
  case CC_X86FastCall: return llvm::CallingConv::X86_FastCall;
52
102
  case CC_X86RegCall: return llvm::CallingConv::X86_RegCall;
53
2.67k
  case CC_X86ThisCall: return llvm::CallingConv::X86_ThisCall;
54
17
  case CC_Win64: return llvm::CallingConv::Win64;
55
14
  case CC_X86_64SysV: return llvm::CallingConv::X86_64_SysV;
56
5
  case CC_AAPCS: return llvm::CallingConv::ARM_AAPCS;
57
11
  case CC_AAPCS_VFP: return llvm::CallingConv::ARM_AAPCS_VFP;
58
3
  case CC_IntelOclBicc: return llvm::CallingConv::Intel_OCL_BI;
59
  // TODO: Add support for __pascal to LLVM.
60
3
  case CC_X86Pascal: return llvm::CallingConv::C;
61
  // TODO: Add support for __vectorcall to LLVM.
62
110
  case CC_X86VectorCall: return llvm::CallingConv::X86_VectorCall;
63
2
  case CC_AArch64VectorCall: return llvm::CallingConv::AArch64_VectorCall;
64
383
  case CC_SpirFunction: return llvm::CallingConv::SPIR_FUNC;
65
436
  case CC_OpenCLKernel: return CGM.getTargetCodeGenInfo().getOpenCLKernelCallingConv();
66
10
  case CC_PreserveMost: return llvm::CallingConv::PreserveMost;
67
6
  case CC_PreserveAll: return llvm::CallingConv::PreserveAll;
68
1.14k
  case CC_Swift: return llvm::CallingConv::Swift;
69
73
  case CC_SwiftAsync: return llvm::CallingConv::SwiftTail;
70
289k
  }
71
289k
}
72
73
/// Derives the 'this' type for codegen purposes, i.e. ignoring method CVR
74
/// qualification. Either or both of RD and MD may be null. A null RD indicates
75
/// that there is no meaningful 'this' type, and a null MD can occur when
76
/// calling a method pointer.
77
CanQualType CodeGenTypes::DeriveThisType(const CXXRecordDecl *RD,
78
629k
                                         const CXXMethodDecl *MD) {
79
629k
  QualType RecTy;
80
629k
  if (RD)
81
629k
    RecTy = Context.getTagDeclType(RD)->getCanonicalTypeInternal();
82
688
  else
83
688
    RecTy = Context.VoidTy;
84
85
629k
  if (MD)
86
629k
    RecTy = Context.getAddrSpaceQualType(RecTy, MD->getMethodQualifiers().getAddressSpace());
87
629k
  return Context.getPointerType(CanQualType::CreateUnsafe(RecTy));
88
629k
}
89
90
/// Returns the canonical formal type of the given C++ method.
91
585k
static CanQual<FunctionProtoType> GetFormalType(const CXXMethodDecl *MD) {
92
585k
  return MD->getType()->getCanonicalTypeUnqualified()
93
585k
           .getAs<FunctionProtoType>();
94
585k
}
95
96
/// Returns the "extra-canonicalized" return type, which discards
97
/// qualifiers on the return type.  Codegen doesn't care about them,
98
/// and it makes ABI code a little easier to be able to assume that
99
/// all parameter and return types are top-level unqualified.
100
338k
static CanQualType GetReturnType(QualType RetTy) {
101
338k
  return RetTy->getCanonicalTypeUnqualified().getUnqualifiedType();
102
338k
}
103
104
/// Arrange the argument and result information for a value of the given
105
/// unprototyped freestanding function type.
106
const CGFunctionInfo &
107
647
CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionNoProtoType> FTNP) {
108
  // When translating an unprototyped function type, always use a
109
  // variadic type.
110
647
  return arrangeLLVMFunctionInfo(FTNP->getReturnType().getUnqualifiedType(),
111
647
                                 /*instanceMethod=*/false,
112
647
                                 /*chainCall=*/false, None,
113
647
                                 FTNP->getExtInfo(), {}, RequiredArgs(0));
114
647
}
115
116
static void addExtParameterInfosForCall(
117
         llvm::SmallVectorImpl<FunctionProtoType::ExtParameterInfo> &paramInfos,
118
                                        const FunctionProtoType *proto,
119
                                        unsigned prefixArgs,
120
1.22k
                                        unsigned totalArgs) {
121
1.22k
  assert(proto->hasExtParameterInfos());
122
0
  assert(paramInfos.size() <= prefixArgs);
123
0
  assert(proto->getNumParams() + prefixArgs <= totalArgs);
124
125
0
  paramInfos.reserve(totalArgs);
126
127
  // Add default infos for any prefix args that don't already have infos.
128
1.22k
  paramInfos.resize(prefixArgs);
129
130
  // Add infos for the prototype.
131
2.04k
  for (const auto &ParamInfo : proto->getExtParameterInfos()) {
132
2.04k
    paramInfos.push_back(ParamInfo);
133
    // pass_object_size params have no parameter info.
134
2.04k
    if (ParamInfo.hasPassObjectSize())
135
447
      paramInfos.emplace_back();
136
2.04k
  }
137
138
1.22k
  assert(paramInfos.size() <= totalArgs &&
139
1.22k
         "Did we forget to insert pass_object_size args?");
140
  // Add default infos for the variadic and/or suffix arguments.
141
0
  paramInfos.resize(totalArgs);
142
1.22k
}
143
144
/// Adds the formal parameters in FPT to the given prefix. If any parameter in
145
/// FPT has pass_object_size attrs, then we'll add parameters for those, too.
146
static void appendParameterTypes(const CodeGenTypes &CGT,
147
                                 SmallVectorImpl<CanQualType> &prefix,
148
              SmallVectorImpl<FunctionProtoType::ExtParameterInfo> &paramInfos,
149
1.12M
                                 CanQual<FunctionProtoType> FPT) {
150
  // Fast path: don't touch param info if we don't need to.
151
1.12M
  if (!FPT->hasExtParameterInfos()) {
152
1.12M
    assert(paramInfos.empty() &&
153
1.12M
           "We have paramInfos, but the prototype doesn't?");
154
0
    prefix.append(FPT->param_type_begin(), FPT->param_type_end());
155
1.12M
    return;
156
1.12M
  }
157
158
811
  unsigned PrefixSize = prefix.size();
159
  // In the vast majority of cases, we'll have precisely FPT->getNumParams()
160
  // parameters; the only thing that can change this is the presence of
161
  // pass_object_size. So, we preallocate for the common case.
162
811
  prefix.reserve(prefix.size() + FPT->getNumParams());
163
164
811
  auto ExtInfos = FPT->getExtParameterInfos();
165
811
  assert(ExtInfos.size() == FPT->getNumParams());
166
2.21k
  for (unsigned I = 0, E = FPT->getNumParams(); I != E; 
++I1.40k
) {
167
1.40k
    prefix.push_back(FPT->getParamType(I));
168
1.40k
    if (ExtInfos[I].hasPassObjectSize())
169
299
      prefix.push_back(CGT.getContext().getSizeType());
170
1.40k
  }
171
172
811
  addExtParameterInfosForCall(paramInfos, FPT.getTypePtr(), PrefixSize,
173
811
                              prefix.size());
174
811
}
175
176
/// Arrange the LLVM function layout for a value of the given function
177
/// type, on top of any implicit parameters already stored.
178
static const CGFunctionInfo &
179
arrangeLLVMFunctionInfo(CodeGenTypes &CGT, bool instanceMethod,
180
                        SmallVectorImpl<CanQualType> &prefix,
181
825k
                        CanQual<FunctionProtoType> FTP) {
182
825k
  SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos;
183
825k
  RequiredArgs Required = RequiredArgs::forPrototypePlus(FTP, prefix.size());
184
  // FIXME: Kill copy.
185
825k
  appendParameterTypes(CGT, prefix, paramInfos, FTP);
186
825k
  CanQualType resultType = FTP->getReturnType().getUnqualifiedType();
187
188
825k
  return CGT.arrangeLLVMFunctionInfo(resultType, instanceMethod,
189
825k
                                     /*chainCall=*/false, prefix,
190
825k
                                     FTP->getExtInfo(), paramInfos,
191
825k
                                     Required);
192
825k
}
193
194
/// Arrange the argument and result information for a value of the
195
/// given freestanding function type.
196
const CGFunctionInfo &
197
601k
CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionProtoType> FTP) {
198
601k
  SmallVector<CanQualType, 16> argTypes;
199
601k
  return ::arrangeLLVMFunctionInfo(*this, /*instanceMethod=*/false, argTypes,
200
601k
                                   FTP);
201
601k
}
202
203
static CallingConv getCallingConventionForDecl(const ObjCMethodDecl *D,
204
27.6k
                                               bool IsWindows) {
205
  // Set the appropriate calling convention for the Function.
206
27.6k
  if (D->hasAttr<StdCallAttr>())
207
2
    return CC_X86StdCall;
208
209
27.6k
  if (D->hasAttr<FastCallAttr>())
210
0
    return CC_X86FastCall;
211
212
27.6k
  if (D->hasAttr<RegCallAttr>())
213
0
    return CC_X86RegCall;
214
215
27.6k
  if (D->hasAttr<ThisCallAttr>())
216
0
    return CC_X86ThisCall;
217
218
27.6k
  if (D->hasAttr<VectorCallAttr>())
219
0
    return CC_X86VectorCall;
220
221
27.6k
  if (D->hasAttr<PascalAttr>())
222
0
    return CC_X86Pascal;
223
224
27.6k
  if (PcsAttr *PCS = D->getAttr<PcsAttr>())
225
0
    return (PCS->getPCS() == PcsAttr::AAPCS ? CC_AAPCS : CC_AAPCS_VFP);
226
227
27.6k
  if (D->hasAttr<AArch64VectorPcsAttr>())
228
0
    return CC_AArch64VectorCall;
229
230
27.6k
  if (D->hasAttr<IntelOclBiccAttr>())
231
0
    return CC_IntelOclBicc;
232
233
27.6k
  if (D->hasAttr<MSABIAttr>())
234
2
    return IsWindows ? 
CC_C0
: CC_Win64;
235
236
27.6k
  if (D->hasAttr<SysVABIAttr>())
237
0
    return IsWindows ? CC_X86_64SysV : CC_C;
238
239
27.6k
  if (D->hasAttr<PreserveMostAttr>())
240
0
    return CC_PreserveMost;
241
242
27.6k
  if (D->hasAttr<PreserveAllAttr>())
243
0
    return CC_PreserveAll;
244
245
27.6k
  return CC_C;
246
27.6k
}
247
248
/// Arrange the argument and result information for a call to an
249
/// unknown C++ non-static member function of the given abstract type.
250
/// (A null RD means we don't have any meaningful "this" argument type,
251
///  so fall back to a generic pointer type).
252
/// The member function must be an ordinary function, i.e. not a
253
/// constructor or destructor.
254
const CGFunctionInfo &
255
CodeGenTypes::arrangeCXXMethodType(const CXXRecordDecl *RD,
256
                                   const FunctionProtoType *FTP,
257
224k
                                   const CXXMethodDecl *MD) {
258
224k
  SmallVector<CanQualType, 16> argTypes;
259
260
  // Add the 'this' pointer.
261
224k
  argTypes.push_back(DeriveThisType(RD, MD));
262
263
224k
  return ::arrangeLLVMFunctionInfo(
264
224k
      *this, true, argTypes,
265
224k
      FTP->getCanonicalTypeUnqualified().getAs<FunctionProtoType>());
266
224k
}
267
268
/// Set calling convention for CUDA/HIP kernel.
269
static void setCUDAKernelCallingConvention(CanQualType &FTy, CodeGenModule &CGM,
270
780k
                                           const FunctionDecl *FD) {
271
780k
  if (FD->hasAttr<CUDAGlobalAttr>()) {
272
412
    const FunctionType *FT = FTy->getAs<FunctionType>();
273
412
    CGM.getTargetCodeGenInfo().setCUDAKernelCallingConvention(FT);
274
412
    FTy = FT->getCanonicalTypeUnqualified();
275
412
  }
276
780k
}
277
278
/// Arrange the argument and result information for a declaration or
279
/// definition of the given C++ non-static member function.  The
280
/// member function must be an ordinary function, i.e. not a
281
/// constructor or destructor.
282
const CGFunctionInfo &
283
230k
CodeGenTypes::arrangeCXXMethodDeclaration(const CXXMethodDecl *MD) {
284
230k
  assert(!isa<CXXConstructorDecl>(MD) && "wrong method for constructors!");
285
0
  assert(!isa<CXXDestructorDecl>(MD) && "wrong method for destructors!");
286
287
0
  CanQualType FT = GetFormalType(MD).getAs<Type>();
288
230k
  setCUDAKernelCallingConvention(FT, CGM, MD);
289
230k
  auto prototype = FT.getAs<FunctionProtoType>();
290
291
230k
  if (MD->isInstance()) {
292
    // The abstract case is perfectly fine.
293
224k
    const CXXRecordDecl *ThisType = TheCXXABI.getThisArgumentTypeForMethod(MD);
294
224k
    return arrangeCXXMethodType(ThisType, prototype.getTypePtr(), MD);
295
224k
  }
296
297
6.07k
  return arrangeFreeFunctionType(prototype);
298
230k
}
299
300
bool CodeGenTypes::inheritingCtorHasParams(
301
1.18k
    const InheritedConstructor &Inherited, CXXCtorType Type) {
302
  // Parameters are unnecessary if we're constructing a base class subobject
303
  // and the inherited constructor lives in a virtual base.
304
1.18k
  return Type == Ctor_Complete ||
305
1.18k
         
!Inherited.getShadowDecl()->constructsVirtualBase()798
||
306
1.18k
         
!Target.getCXXABI().hasConstructorVariants()74
;
307
1.18k
}
308
309
const CGFunctionInfo &
310
300k
CodeGenTypes::arrangeCXXStructorDeclaration(GlobalDecl GD) {
311
300k
  auto *MD = cast<CXXMethodDecl>(GD.getDecl());
312
313
300k
  SmallVector<CanQualType, 16> argTypes;
314
300k
  SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos;
315
300k
  argTypes.push_back(DeriveThisType(MD->getParent(), MD));
316
317
300k
  bool PassParams = true;
318
319
300k
  if (auto *CD = dyn_cast<CXXConstructorDecl>(MD)) {
320
    // A base class inheriting constructor doesn't get forwarded arguments
321
    // needed to construct a virtual base (or base class thereof).
322
178k
    if (auto Inherited = CD->getInheritedConstructor())
323
722
      PassParams = inheritingCtorHasParams(Inherited, GD.getCtorType());
324
178k
  }
325
326
300k
  CanQual<FunctionProtoType> FTP = GetFormalType(MD);
327
328
  // Add the formal parameters.
329
300k
  if (PassParams)
330
300k
    appendParameterTypes(*this, argTypes, paramInfos, FTP);
331
332
300k
  CGCXXABI::AddedStructorArgCounts AddedArgs =
333
300k
      TheCXXABI.buildStructorSignature(GD, argTypes);
334
300k
  if (!paramInfos.empty()) {
335
    // Note: prefix implies after the first param.
336
175
    if (AddedArgs.Prefix)
337
2
      paramInfos.insert(paramInfos.begin() + 1, AddedArgs.Prefix,
338
2
                        FunctionProtoType::ExtParameterInfo{});
339
175
    if (AddedArgs.Suffix)
340
20
      paramInfos.append(AddedArgs.Suffix,
341
20
                        FunctionProtoType::ExtParameterInfo{});
342
175
  }
343
344
300k
  RequiredArgs required =
345
300k
      (PassParams && 
MD->isVariadic()300k
?
RequiredArgs(argTypes.size())91
346
300k
                                      : 
RequiredArgs::All300k
);
347
348
300k
  FunctionType::ExtInfo extInfo = FTP->getExtInfo();
349
300k
  CanQualType resultType = TheCXXABI.HasThisReturn(GD)
350
300k
                               ? 
argTypes.front()7.93k
351
300k
                               : 
TheCXXABI.hasMostDerivedReturn(GD)292k
352
292k
                                     ? 
CGM.getContext().VoidPtrTy1.42k
353
292k
                                     : 
Context.VoidTy291k
;
354
300k
  return arrangeLLVMFunctionInfo(resultType, /*instanceMethod=*/true,
355
300k
                                 /*chainCall=*/false, argTypes, extInfo,
356
300k
                                 paramInfos, required);
357
300k
}
358
359
static SmallVector<CanQualType, 16>
360
76.7k
getArgTypesForCall(ASTContext &ctx, const CallArgList &args) {
361
76.7k
  SmallVector<CanQualType, 16> argTypes;
362
76.7k
  for (auto &arg : args)
363
107k
    argTypes.push_back(ctx.getCanonicalParamType(arg.Ty));
364
76.7k
  return argTypes;
365
76.7k
}
366
367
static SmallVector<CanQualType, 16>
368
30.7k
getArgTypesForDeclaration(ASTContext &ctx, const FunctionArgList &args) {
369
30.7k
  SmallVector<CanQualType, 16> argTypes;
370
30.7k
  for (auto &arg : args)
371
82.4k
    argTypes.push_back(ctx.getCanonicalParamType(arg->getType()));
372
30.7k
  return argTypes;
373
30.7k
}
374
375
static llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16>
376
getExtParameterInfosForCall(const FunctionProtoType *proto,
377
76.1k
                            unsigned prefixArgs, unsigned totalArgs) {
378
76.1k
  llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16> result;
379
76.1k
  if (proto->hasExtParameterInfos()) {
380
41
    addExtParameterInfosForCall(result, proto, prefixArgs, totalArgs);
381
41
  }
382
76.1k
  return result;
383
76.1k
}
384
385
/// Arrange a call to a C++ method, passing the given arguments.
386
///
387
/// ExtraPrefixArgs is the number of ABI-specific args passed after the `this`
388
/// parameter.
389
/// ExtraSuffixArgs is the number of ABI-specific args passed at the end of
390
/// args.
391
/// PassProtoArgs indicates whether `args` has args for the parameters in the
392
/// given CXXConstructorDecl.
393
const CGFunctionInfo &
394
CodeGenTypes::arrangeCXXConstructorCall(const CallArgList &args,
395
                                        const CXXConstructorDecl *D,
396
                                        CXXCtorType CtorKind,
397
                                        unsigned ExtraPrefixArgs,
398
                                        unsigned ExtraSuffixArgs,
399
55.0k
                                        bool PassProtoArgs) {
400
  // FIXME: Kill copy.
401
55.0k
  SmallVector<CanQualType, 16> ArgTypes;
402
55.0k
  for (const auto &Arg : args)
403
108k
    ArgTypes.push_back(Context.getCanonicalParamType(Arg.Ty));
404
405
  // +1 for implicit this, which should always be args[0].
406
55.0k
  unsigned TotalPrefixArgs = 1 + ExtraPrefixArgs;
407
408
55.0k
  CanQual<FunctionProtoType> FPT = GetFormalType(D);
409
55.0k
  RequiredArgs Required = PassProtoArgs
410
55.0k
                              ? RequiredArgs::forPrototypePlus(
411
54.9k
                                    FPT, TotalPrefixArgs + ExtraSuffixArgs)
412
55.0k
                              : 
RequiredArgs::All10
;
413
414
55.0k
  GlobalDecl GD(D, CtorKind);
415
55.0k
  CanQualType ResultType = TheCXXABI.HasThisReturn(GD)
416
55.0k
                               ? 
ArgTypes.front()2.33k
417
55.0k
                               : 
TheCXXABI.hasMostDerivedReturn(GD)52.6k
418
52.6k
                                     ? 
CGM.getContext().VoidPtrTy0
419
52.6k
                                     : Context.VoidTy;
420
421
55.0k
  FunctionType::ExtInfo Info = FPT->getExtInfo();
422
55.0k
  llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16> ParamInfos;
423
  // If the prototype args are elided, we should only have ABI-specific args,
424
  // which never have param info.
425
55.0k
  if (PassProtoArgs && 
FPT->hasExtParameterInfos()54.9k
) {
426
    // ABI-specific suffix arguments are treated the same as variadic arguments.
427
70
    addExtParameterInfosForCall(ParamInfos, FPT.getTypePtr(), TotalPrefixArgs,
428
70
                                ArgTypes.size());
429
70
  }
430
55.0k
  return arrangeLLVMFunctionInfo(ResultType, /*instanceMethod=*/true,
431
55.0k
                                 /*chainCall=*/false, ArgTypes, Info,
432
55.0k
                                 ParamInfos, Required);
433
55.0k
}
434
435
/// Arrange the argument and result information for the declaration or
436
/// definition of the given function.
437
const CGFunctionInfo &
438
653k
CodeGenTypes::arrangeFunctionDeclaration(const FunctionDecl *FD) {
439
653k
  if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(FD))
440
115k
    if (MD->isInstance())
441
102k
      return arrangeCXXMethodDeclaration(MD);
442
443
550k
  CanQualType FTy = FD->getType()->getCanonicalTypeUnqualified();
444
445
550k
  assert(isa<FunctionType>(FTy));
446
0
  setCUDAKernelCallingConvention(FTy, CGM, FD);
447
448
  // When declaring a function without a prototype, always use a
449
  // non-variadic type.
450
550k
  if (CanQual<FunctionNoProtoType> noProto = FTy.getAs<FunctionNoProtoType>()) {
451
17.1k
    return arrangeLLVMFunctionInfo(
452
17.1k
        noProto->getReturnType(), /*instanceMethod=*/false,
453
17.1k
        /*chainCall=*/false, None, noProto->getExtInfo(), {},RequiredArgs::All);
454
17.1k
  }
455
456
533k
  return arrangeFreeFunctionType(FTy.castAs<FunctionProtoType>());
457
550k
}
458
459
/// Arrange the argument and result information for the declaration or
460
/// definition of an Objective-C method.
461
const CGFunctionInfo &
462
5.17k
CodeGenTypes::arrangeObjCMethodDeclaration(const ObjCMethodDecl *MD) {
463
  // It happens that this is the same as a call with no optional
464
  // arguments, except also using the formal 'self' type.
465
5.17k
  return arrangeObjCMessageSendSignature(MD, MD->getSelfDecl()->getType());
466
5.17k
}
467
468
/// Arrange the argument and result information for the function type
469
/// through which to perform a send to the given Objective-C method,
470
/// using the given receiver type.  The receiver type is not always
471
/// the 'self' type of the method or even an Objective-C pointer type.
472
/// This is *not* the right method for actually performing such a
473
/// message send, due to the possibility of optional arguments.
474
const CGFunctionInfo &
475
CodeGenTypes::arrangeObjCMessageSendSignature(const ObjCMethodDecl *MD,
476
16.2k
                                              QualType receiverType) {
477
16.2k
  SmallVector<CanQualType, 16> argTys;
478
16.2k
  SmallVector<FunctionProtoType::ExtParameterInfo, 4> extParamInfos(2);
479
16.2k
  argTys.push_back(Context.getCanonicalParamType(receiverType));
480
16.2k
  argTys.push_back(Context.getCanonicalParamType(Context.getObjCSelType()));
481
  // FIXME: Kill copy?
482
16.2k
  for (const auto *I : MD->parameters()) {
483
13.5k
    argTys.push_back(Context.getCanonicalParamType(I->getType()));
484
13.5k
    auto extParamInfo = FunctionProtoType::ExtParameterInfo().withIsNoEscape(
485
13.5k
        I->hasAttr<NoEscapeAttr>());
486
13.5k
    extParamInfos.push_back(extParamInfo);
487
13.5k
  }
488
489
16.2k
  FunctionType::ExtInfo einfo;
490
16.2k
  bool IsWindows = getContext().getTargetInfo().getTriple().isOSWindows();
491
16.2k
  einfo = einfo.withCallingConv(getCallingConventionForDecl(MD, IsWindows));
492
493
16.2k
  if (getContext().getLangOpts().ObjCAutoRefCount &&
494
16.2k
      
MD->hasAttr<NSReturnsRetainedAttr>()862
)
495
114
    einfo = einfo.withProducesResult(true);
496
497
16.2k
  RequiredArgs required =
498
16.2k
    (MD->isVariadic() ? 
RequiredArgs(argTys.size())876
:
RequiredArgs::All15.4k
);
499
500
16.2k
  return arrangeLLVMFunctionInfo(
501
16.2k
      GetReturnType(MD->getReturnType()), /*instanceMethod=*/false,
502
16.2k
      /*chainCall=*/false, argTys, einfo, extParamInfos, required);
503
16.2k
}
504
505
const CGFunctionInfo &
506
CodeGenTypes::arrangeUnprototypedObjCMessageSend(QualType returnType,
507
806
                                                 const CallArgList &args) {
508
806
  auto argTypes = getArgTypesForCall(Context, args);
509
806
  FunctionType::ExtInfo einfo;
510
511
806
  return arrangeLLVMFunctionInfo(
512
806
      GetReturnType(returnType), /*instanceMethod=*/false,
513
806
      /*chainCall=*/false, argTypes, einfo, {}, RequiredArgs::All);
514
806
}
515
516
const CGFunctionInfo &
517
718k
CodeGenTypes::arrangeGlobalDeclaration(GlobalDecl GD) {
518
  // FIXME: Do we need to handle ObjCMethodDecl?
519
718k
  const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
520
521
718k
  if (isa<CXXConstructorDecl>(GD.getDecl()) ||
522
718k
      
isa<CXXDestructorDecl>(GD.getDecl())675k
)
523
65.3k
    return arrangeCXXStructorDeclaration(GD);
524
525
653k
  return arrangeFunctionDeclaration(FD);
526
718k
}
527
528
/// Arrange a thunk that takes 'this' as the first parameter followed by
529
/// varargs.  Return a void pointer, regardless of the actual return type.
530
/// The body of the thunk will end in a musttail call to a function of the
531
/// correct type, and the caller will bitcast the function to the correct
532
/// prototype.
533
const CGFunctionInfo &
534
55
CodeGenTypes::arrangeUnprototypedMustTailThunk(const CXXMethodDecl *MD) {
535
55
  assert(MD->isVirtual() && "only methods have thunks");
536
0
  CanQual<FunctionProtoType> FTP = GetFormalType(MD);
537
55
  CanQualType ArgTys[] = {DeriveThisType(MD->getParent(), MD)};
538
55
  return arrangeLLVMFunctionInfo(Context.VoidTy, /*instanceMethod=*/false,
539
55
                                 /*chainCall=*/false, ArgTys,
540
55
                                 FTP->getExtInfo(), {}, RequiredArgs(1));
541
55
}
542
543
const CGFunctionInfo &
544
CodeGenTypes::arrangeMSCtorClosure(const CXXConstructorDecl *CD,
545
36
                                   CXXCtorType CT) {
546
36
  assert(CT == Ctor_CopyingClosure || CT == Ctor_DefaultClosure);
547
548
0
  CanQual<FunctionProtoType> FTP = GetFormalType(CD);
549
36
  SmallVector<CanQualType, 2> ArgTys;
550
36
  const CXXRecordDecl *RD = CD->getParent();
551
36
  ArgTys.push_back(DeriveThisType(RD, CD));
552
36
  if (CT == Ctor_CopyingClosure)
553
6
    ArgTys.push_back(*FTP->param_type_begin());
554
36
  if (RD->getNumVBases() > 0)
555
0
    ArgTys.push_back(Context.IntTy);
556
36
  CallingConv CC = Context.getDefaultCallingConvention(
557
36
      /*IsVariadic=*/false, /*IsCXXMethod=*/true);
558
36
  return arrangeLLVMFunctionInfo(Context.VoidTy, /*instanceMethod=*/true,
559
36
                                 /*chainCall=*/false, ArgTys,
560
36
                                 FunctionType::ExtInfo(CC), {},
561
36
                                 RequiredArgs::All);
562
36
}
563
564
/// Arrange a call as unto a free function, except possibly with an
565
/// additional number of formal parameters considered required.
566
static const CGFunctionInfo &
567
arrangeFreeFunctionLikeCall(CodeGenTypes &CGT,
568
                            CodeGenModule &CGM,
569
                            const CallArgList &args,
570
                            const FunctionType *fnType,
571
                            unsigned numExtraRequiredArgs,
572
214k
                            bool chainCall) {
573
214k
  assert(args.size() >= numExtraRequiredArgs);
574
575
0
  llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos;
576
577
  // In most cases, there are no optional arguments.
578
214k
  RequiredArgs required = RequiredArgs::All;
579
580
  // If we have a variadic prototype, the required arguments are the
581
  // extra prefix plus the arguments in the prototype.
582
214k
  if (const FunctionProtoType *proto = dyn_cast<FunctionProtoType>(fnType)) {
583
213k
    if (proto->isVariadic())
584
76.9k
      required = RequiredArgs::forPrototypePlus(proto, numExtraRequiredArgs);
585
586
213k
    if (proto->hasExtParameterInfos())
587
305
      addExtParameterInfosForCall(paramInfos, proto, numExtraRequiredArgs,
588
305
                                  args.size());
589
590
  // If we don't have a prototype at all, but we're supposed to
591
  // explicitly use the variadic convention for unprototyped calls,
592
  // treat all of the arguments as required but preserve the nominal
593
  // possibility of variadics.
594
213k
  } else 
if (1.04k
CGM.getTargetCodeGenInfo()
595
1.04k
                .isNoProtoCallVariadic(args,
596
1.04k
                                       cast<FunctionNoProtoType>(fnType))) {
597
745
    required = RequiredArgs(args.size());
598
745
  }
599
600
  // FIXME: Kill copy.
601
214k
  SmallVector<CanQualType, 16> argTypes;
602
214k
  for (const auto &arg : args)
603
380k
    argTypes.push_back(CGT.getContext().getCanonicalParamType(arg.Ty));
604
214k
  return CGT.arrangeLLVMFunctionInfo(GetReturnType(fnType->getReturnType()),
605
214k
                                     /*instanceMethod=*/false, chainCall,
606
214k
                                     argTypes, fnType->getExtInfo(), paramInfos,
607
214k
                                     required);
608
214k
}
609
610
/// Figure out the rules for calling a function with the given formal
611
/// type using the given arguments.  The arguments are necessary
612
/// because the function might be unprototyped, in which case it's
613
/// target-dependent in crazy ways.
614
const CGFunctionInfo &
615
CodeGenTypes::arrangeFreeFunctionCall(const CallArgList &args,
616
                                      const FunctionType *fnType,
617
214k
                                      bool chainCall) {
618
214k
  return arrangeFreeFunctionLikeCall(*this, CGM, args, fnType,
619
214k
                                     chainCall ? 
18
:
0214k
, chainCall);
620
214k
}
621
622
/// A block function is essentially a free function with an
623
/// extra implicit argument.
624
const CGFunctionInfo &
625
CodeGenTypes::arrangeBlockFunctionCall(const CallArgList &args,
626
605
                                       const FunctionType *fnType) {
627
605
  return arrangeFreeFunctionLikeCall(*this, CGM, args, fnType, 1,
628
605
                                     /*chainCall=*/false);
629
605
}
630
631
const CGFunctionInfo &
632
CodeGenTypes::arrangeBlockFunctionDeclaration(const FunctionProtoType *proto,
633
1.11k
                                              const FunctionArgList &params) {
634
1.11k
  auto paramInfos = getExtParameterInfosForCall(proto, 1, params.size());
635
1.11k
  auto argTypes = getArgTypesForDeclaration(Context, params);
636
637
1.11k
  return arrangeLLVMFunctionInfo(GetReturnType(proto->getReturnType()),
638
1.11k
                                 /*instanceMethod*/ false, /*chainCall*/ false,
639
1.11k
                                 argTypes, proto->getExtInfo(), paramInfos,
640
1.11k
                                 RequiredArgs::forPrototypePlus(proto, 1));
641
1.11k
}
642
643
const CGFunctionInfo &
644
CodeGenTypes::arrangeBuiltinFunctionCall(QualType resultType,
645
990
                                         const CallArgList &args) {
646
  // FIXME: Kill copy.
647
990
  SmallVector<CanQualType, 16> argTypes;
648
990
  for (const auto &Arg : args)
649
3.62k
    argTypes.push_back(Context.getCanonicalParamType(Arg.Ty));
650
990
  return arrangeLLVMFunctionInfo(
651
990
      GetReturnType(resultType), /*instanceMethod=*/false,
652
990
      /*chainCall=*/false, argTypes, FunctionType::ExtInfo(),
653
990
      /*paramInfos=*/ {}, RequiredArgs::All);
654
990
}
655
656
const CGFunctionInfo &
657
CodeGenTypes::arrangeBuiltinFunctionDeclaration(QualType resultType,
658
29.6k
                                                const FunctionArgList &args) {
659
29.6k
  auto argTypes = getArgTypesForDeclaration(Context, args);
660
661
29.6k
  return arrangeLLVMFunctionInfo(
662
29.6k
      GetReturnType(resultType), /*instanceMethod=*/false, /*chainCall=*/false,
663
29.6k
      argTypes, FunctionType::ExtInfo(), {}, RequiredArgs::All);
664
29.6k
}
665
666
const CGFunctionInfo &
667
CodeGenTypes::arrangeBuiltinFunctionDeclaration(CanQualType resultType,
668
376
                                              ArrayRef<CanQualType> argTypes) {
669
376
  return arrangeLLVMFunctionInfo(
670
376
      resultType, /*instanceMethod=*/false, /*chainCall=*/false,
671
376
      argTypes, FunctionType::ExtInfo(), {}, RequiredArgs::All);
672
376
}
673
674
/// Arrange a call to a C++ method, passing the given arguments.
675
///
676
/// numPrefixArgs is the number of ABI-specific prefix arguments we have. It
677
/// does not count `this`.
678
const CGFunctionInfo &
679
CodeGenTypes::arrangeCXXMethodCall(const CallArgList &args,
680
                                   const FunctionProtoType *proto,
681
                                   RequiredArgs required,
682
75.0k
                                   unsigned numPrefixArgs) {
683
75.0k
  assert(numPrefixArgs + 1 <= args.size() &&
684
75.0k
         "Emitting a call with less args than the required prefix?");
685
  // Add one to account for `this`. It's a bit awkward here, but we don't count
686
  // `this` in similar places elsewhere.
687
0
  auto paramInfos =
688
75.0k
    getExtParameterInfosForCall(proto, numPrefixArgs + 1, args.size());
689
690
  // FIXME: Kill copy.
691
75.0k
  auto argTypes = getArgTypesForCall(Context, args);
692
693
75.0k
  FunctionType::ExtInfo info = proto->getExtInfo();
694
75.0k
  return arrangeLLVMFunctionInfo(
695
75.0k
      GetReturnType(proto->getReturnType()), /*instanceMethod=*/true,
696
75.0k
      /*chainCall=*/false, argTypes, info, paramInfos, required);
697
75.0k
}
698
699
17.5k
const CGFunctionInfo &CodeGenTypes::arrangeNullaryFunction() {
700
17.5k
  return arrangeLLVMFunctionInfo(
701
17.5k
      getContext().VoidTy, /*instanceMethod=*/false, /*chainCall=*/false,
702
17.5k
      None, FunctionType::ExtInfo(), {}, RequiredArgs::All);
703
17.5k
}
704
705
const CGFunctionInfo &
706
CodeGenTypes::arrangeCall(const CGFunctionInfo &signature,
707
11.1k
                          const CallArgList &args) {
708
11.1k
  assert(signature.arg_size() <= args.size());
709
11.1k
  if (signature.arg_size() == args.size())
710
10.2k
    return signature;
711
712
858
  SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos;
713
858
  auto sigParamInfos = signature.getExtParameterInfos();
714
858
  if (!sigParamInfos.empty()) {
715
858
    paramInfos.append(sigParamInfos.begin(), sigParamInfos.end());
716
858
    paramInfos.resize(args.size());
717
858
  }
718
719
858
  auto argTypes = getArgTypesForCall(Context, args);
720
721
858
  assert(signature.getRequiredArgs().allowsOptionalArgs());
722
0
  return arrangeLLVMFunctionInfo(signature.getReturnType(),
723
858
                                 signature.isInstanceMethod(),
724
858
                                 signature.isChainCall(),
725
858
                                 argTypes,
726
858
                                 signature.getExtInfo(),
727
858
                                 paramInfos,
728
858
                                 signature.getRequiredArgs());
729
11.1k
}
730
731
namespace clang {
732
namespace CodeGen {
733
void computeSPIRKernelABIInfo(CodeGenModule &CGM, CGFunctionInfo &FI);
734
}
735
}
736
737
/// Arrange the argument and result information for an abstract value
738
/// of a given function type.  This is the method which all of the
739
/// above functions ultimately defer to.
740
const CGFunctionInfo &
741
CodeGenTypes::arrangeLLVMFunctionInfo(CanQualType resultType,
742
                                      bool instanceMethod,
743
                                      bool chainCall,
744
                                      ArrayRef<CanQualType> argTypes,
745
                                      FunctionType::ExtInfo info,
746
                     ArrayRef<FunctionProtoType::ExtParameterInfo> paramInfos,
747
1.55M
                                      RequiredArgs required) {
748
1.55M
  assert(llvm::all_of(argTypes,
749
1.55M
                      [](CanQualType T) { return T.isCanonicalAsParam(); }));
750
751
  // Lookup or create unique function info.
752
0
  llvm::FoldingSetNodeID ID;
753
1.55M
  CGFunctionInfo::Profile(ID, instanceMethod, chainCall, info, paramInfos,
754
1.55M
                          required, resultType, argTypes);
755
756
1.55M
  void *insertPos = nullptr;
757
1.55M
  CGFunctionInfo *FI = FunctionInfos.FindNodeOrInsertPos(ID, insertPos);
758
1.55M
  if (FI)
759
1.26M
    return *FI;
760
761
289k
  unsigned CC = ClangCallConvToLLVMCallConv(info.getCC());
762
763
  // Construct the function info.  We co-allocate the ArgInfos.
764
289k
  FI = CGFunctionInfo::create(CC, instanceMethod, chainCall, info,
765
289k
                              paramInfos, resultType, argTypes, required);
766
289k
  FunctionInfos.InsertNode(FI, insertPos);
767
768
289k
  bool inserted = FunctionsBeingProcessed.insert(FI).second;
769
289k
  (void)inserted;
770
289k
  assert(inserted && "Recursively being processed?");
771
772
  // Compute ABI information.
773
289k
  if (CC == llvm::CallingConv::SPIR_KERNEL) {
774
    // Force target independent argument handling for the host visible
775
    // kernel functions.
776
162
    computeSPIRKernelABIInfo(CGM, *FI);
777
289k
  } else if (info.getCC() == CC_Swift || 
info.getCC() == CC_SwiftAsync287k
) {
778
1.21k
    swiftcall::computeABIInfo(CGM, *FI);
779
287k
  } else {
780
287k
    getABIInfo().computeInfo(*FI);
781
287k
  }
782
783
  // Loop over all of the computed argument and return value info.  If any of
784
  // them are direct or extend without a specified coerce type, specify the
785
  // default now.
786
289k
  ABIArgInfo &retInfo = FI->getReturnInfo();
787
289k
  if (retInfo.canHaveCoerceToType() && 
retInfo.getCoerceToType() == nullptr167k
)
788
71.7k
    retInfo.setCoerceToType(ConvertType(FI->getReturnType()));
789
790
289k
  for (auto &I : FI->arguments())
791
716k
    if (I.info.canHaveCoerceToType() && 
I.info.getCoerceToType() == nullptr708k
)
792
418k
      I.info.setCoerceToType(ConvertType(I.type));
793
794
289k
  bool erased = FunctionsBeingProcessed.erase(FI); (void)erased;
795
289k
  assert(erased && "Not in set?");
796
797
0
  return *FI;
798
1.55M
}
799
800
CGFunctionInfo *CGFunctionInfo::create(unsigned llvmCC,
801
                                       bool instanceMethod,
802
                                       bool chainCall,
803
                                       const FunctionType::ExtInfo &info,
804
                                       ArrayRef<ExtParameterInfo> paramInfos,
805
                                       CanQualType resultType,
806
                                       ArrayRef<CanQualType> argTypes,
807
289k
                                       RequiredArgs required) {
808
289k
  assert(paramInfos.empty() || paramInfos.size() == argTypes.size());
809
0
  assert(!required.allowsOptionalArgs() ||
810
289k
         required.getNumRequiredArgs() <= argTypes.size());
811
812
0
  void *buffer =
813
289k
    operator new(totalSizeToAlloc<ArgInfo,             ExtParameterInfo>(
814
289k
                                  argTypes.size() + 1, paramInfos.size()));
815
816
289k
  CGFunctionInfo *FI = new(buffer) CGFunctionInfo();
817
289k
  FI->CallingConvention = llvmCC;
818
289k
  FI->EffectiveCallingConvention = llvmCC;
819
289k
  FI->ASTCallingConvention = info.getCC();
820
289k
  FI->InstanceMethod = instanceMethod;
821
289k
  FI->ChainCall = chainCall;
822
289k
  FI->CmseNSCall = info.getCmseNSCall();
823
289k
  FI->NoReturn = info.getNoReturn();
824
289k
  FI->ReturnsRetained = info.getProducesResult();
825
289k
  FI->NoCallerSavedRegs = info.getNoCallerSavedRegs();
826
289k
  FI->NoCfCheck = info.getNoCfCheck();
827
289k
  FI->Required = required;
828
289k
  FI->HasRegParm = info.getHasRegParm();
829
289k
  FI->RegParm = info.getRegParm();
830
289k
  FI->ArgStruct = nullptr;
831
289k
  FI->ArgStructAlign = 0;
832
289k
  FI->NumArgs = argTypes.size();
833
289k
  FI->HasExtParameterInfos = !paramInfos.empty();
834
289k
  FI->getArgsBuffer()[0].type = resultType;
835
1.00M
  for (unsigned i = 0, e = argTypes.size(); i != e; 
++i716k
)
836
716k
    FI->getArgsBuffer()[i + 1].type = argTypes[i];
837
309k
  for (unsigned i = 0, e = paramInfos.size(); i != e; 
++i20.1k
)
838
20.1k
    FI->getExtParameterInfosBuffer()[i] = paramInfos[i];
839
289k
  return FI;
840
289k
}
841
842
/***/
843
844
namespace {
845
// ABIArgInfo::Expand implementation.
846
847
// Specifies the way QualType passed as ABIArgInfo::Expand is expanded.
848
struct TypeExpansion {
849
  enum TypeExpansionKind {
850
    // Elements of constant arrays are expanded recursively.
851
    TEK_ConstantArray,
852
    // Record fields are expanded recursively (but if record is a union, only
853
    // the field with the largest size is expanded).
854
    TEK_Record,
855
    // For complex types, real and imaginary parts are expanded recursively.
856
    TEK_Complex,
857
    // All other types are not expandable.
858
    TEK_None
859
  };
860
861
  const TypeExpansionKind Kind;
862
863
2.31k
  TypeExpansion(TypeExpansionKind K) : Kind(K) {}
864
2.31k
  virtual ~TypeExpansion() {}
865
};
866
867
struct ConstantArrayExpansion : TypeExpansion {
868
  QualType EltTy;
869
  uint64_t NumElts;
870
871
  ConstantArrayExpansion(QualType EltTy, uint64_t NumElts)
872
40
      : TypeExpansion(TEK_ConstantArray), EltTy(EltTy), NumElts(NumElts) {}
873
2.35k
  static bool classof(const TypeExpansion *TE) {
874
2.35k
    return TE->Kind == TEK_ConstantArray;
875
2.35k
  }
876
};
877
878
struct RecordExpansion : TypeExpansion {
879
  SmallVector<const CXXBaseSpecifier *, 1> Bases;
880
881
  SmallVector<const FieldDecl *, 1> Fields;
882
883
  RecordExpansion(SmallVector<const CXXBaseSpecifier *, 1> &&Bases,
884
                  SmallVector<const FieldDecl *, 1> &&Fields)
885
      : TypeExpansion(TEK_Record), Bases(std::move(Bases)),
886
735
        Fields(std::move(Fields)) {}
887
3.00k
  static bool classof(const TypeExpansion *TE) {
888
3.00k
    return TE->Kind == TEK_Record;
889
3.00k
  }
890
};
891
892
struct ComplexExpansion : TypeExpansion {
893
  QualType EltTy;
894
895
46
  ComplexExpansion(QualType EltTy) : TypeExpansion(TEK_Complex), EltTy(EltTy) {}
896
1.54k
  static bool classof(const TypeExpansion *TE) {
897
1.54k
    return TE->Kind == TEK_Complex;
898
1.54k
  }
899
};
900
901
struct NoExpansion : TypeExpansion {
902
1.49k
  NoExpansion() : TypeExpansion(TEK_None) {}
903
1.49k
  static bool classof(const TypeExpansion *TE) {
904
1.49k
    return TE->Kind == TEK_None;
905
1.49k
  }
906
};
907
}  // namespace
908
909
static std::unique_ptr<TypeExpansion>
910
2.31k
getTypeExpansion(QualType Ty, const ASTContext &Context) {
911
2.31k
  if (const ConstantArrayType *AT = Context.getAsConstantArrayType(Ty)) {
912
40
    return std::make_unique<ConstantArrayExpansion>(
913
40
        AT->getElementType(), AT->getSize().getZExtValue());
914
40
  }
915
2.27k
  if (const RecordType *RT = Ty->getAs<RecordType>()) {
916
735
    SmallVector<const CXXBaseSpecifier *, 1> Bases;
917
735
    SmallVector<const FieldDecl *, 1> Fields;
918
735
    const RecordDecl *RD = RT->getDecl();
919
735
    assert(!RD->hasFlexibleArrayMember() &&
920
735
           "Cannot expand structure with flexible array.");
921
735
    if (RD->isUnion()) {
922
      // Unions can be here only in degenerative cases - all the fields are same
923
      // after flattening. Thus we have to use the "largest" field.
924
5
      const FieldDecl *LargestFD = nullptr;
925
5
      CharUnits UnionSize = CharUnits::Zero();
926
927
10
      for (const auto *FD : RD->fields()) {
928
10
        if (FD->isZeroLengthBitField(Context))
929
0
          continue;
930
10
        assert(!FD->isBitField() &&
931
10
               "Cannot expand structure with bit-field members.");
932
0
        CharUnits FieldSize = Context.getTypeSizeInChars(FD->getType());
933
10
        if (UnionSize < FieldSize) {
934
5
          UnionSize = FieldSize;
935
5
          LargestFD = FD;
936
5
        }
937
10
      }
938
5
      if (LargestFD)
939
5
        Fields.push_back(LargestFD);
940
730
    } else {
941
730
      if (const auto *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
942
215
        assert(!CXXRD->isDynamicClass() &&
943
215
               "cannot expand vtable pointers in dynamic classes");
944
0
        for (const CXXBaseSpecifier &BS : CXXRD->bases())
945
5
          Bases.push_back(&BS);
946
215
      }
947
948
1.45k
      for (const auto *FD : RD->fields()) {
949
1.45k
        if (FD->isZeroLengthBitField(Context))
950
0
          continue;
951
1.45k
        assert(!FD->isBitField() &&
952
1.45k
               "Cannot expand structure with bit-field members.");
953
0
        Fields.push_back(FD);
954
1.45k
      }
955
730
    }
956
0
    return std::make_unique<RecordExpansion>(std::move(Bases),
957
735
                                              std::move(Fields));
958
735
  }
959
1.53k
  if (const ComplexType *CT = Ty->getAs<ComplexType>()) {
960
46
    return std::make_unique<ComplexExpansion>(CT->getElementType());
961
46
  }
962
1.49k
  return std::make_unique<NoExpansion>();
963
1.53k
}
964
965
1.35k
static int getExpansionSize(QualType Ty, const ASTContext &Context) {
966
1.35k
  auto Exp = getTypeExpansion(Ty, Context);
967
1.35k
  if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
968
24
    return CAExp->NumElts * getExpansionSize(CAExp->EltTy, Context);
969
24
  }
970
1.33k
  if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
971
441
    int Res = 0;
972
441
    for (auto BS : RExp->Bases)
973
3
      Res += getExpansionSize(BS->getType(), Context);
974
441
    for (auto FD : RExp->Fields)
975
872
      Res += getExpansionSize(FD->getType(), Context);
976
441
    return Res;
977
441
  }
978
893
  if (isa<ComplexExpansion>(Exp.get()))
979
28
    return 2;
980
865
  assert(isa<NoExpansion>(Exp.get()));
981
0
  return 1;
982
893
}
983
984
void
985
CodeGenTypes::getExpandedTypes(QualType Ty,
986
530
                               SmallVectorImpl<llvm::Type *>::iterator &TI) {
987
530
  auto Exp = getTypeExpansion(Ty, Context);
988
530
  if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
989
40
    for (int i = 0, n = CAExp->NumElts; i < n; 
i++32
) {
990
32
      getExpandedTypes(CAExp->EltTy, TI);
991
32
    }
992
522
  } else if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
993
166
    for (auto BS : RExp->Bases)
994
1
      getExpandedTypes(BS->getType(), TI);
995
166
    for (auto FD : RExp->Fields)
996
324
      getExpandedTypes(FD->getType(), TI);
997
356
  } else if (auto CExp = dyn_cast<ComplexExpansion>(Exp.get())) {
998
11
    llvm::Type *EltTy = ConvertType(CExp->EltTy);
999
11
    *TI++ = EltTy;
1000
11
    *TI++ = EltTy;
1001
345
  } else {
1002
345
    assert(isa<NoExpansion>(Exp.get()));
1003
0
    *TI++ = ConvertType(Ty);
1004
345
  }
1005
530
}
1006
1007
static void forConstantArrayExpansion(CodeGenFunction &CGF,
1008
                                      ConstantArrayExpansion *CAE,
1009
                                      Address BaseAddr,
1010
8
                                      llvm::function_ref<void(Address)> Fn) {
1011
8
  CharUnits EltSize = CGF.getContext().getTypeSizeInChars(CAE->EltTy);
1012
8
  CharUnits EltAlign =
1013
8
    BaseAddr.getAlignment().alignmentOfArrayElement(EltSize);
1014
1015
40
  for (int i = 0, n = CAE->NumElts; i < n; 
i++32
) {
1016
32
    llvm::Value *EltAddr = CGF.Builder.CreateConstGEP2_32(
1017
32
        BaseAddr.getElementType(), BaseAddr.getPointer(), 0, i);
1018
32
    Fn(Address(EltAddr, EltAlign));
1019
32
  }
1020
8
}
1021
1022
void CodeGenFunction::ExpandTypeFromArgs(QualType Ty, LValue LV,
1023
347
                                         llvm::Function::arg_iterator &AI) {
1024
347
  assert(LV.isSimple() &&
1025
347
         "Unexpected non-simple lvalue during struct expansion.");
1026
1027
0
  auto Exp = getTypeExpansion(Ty, getContext());
1028
347
  if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
1029
8
    forConstantArrayExpansion(
1030
32
        *this, CAExp, LV.getAddress(*this), [&](Address EltAddr) {
1031
32
          LValue LV = MakeAddrLValue(EltAddr, CAExp->EltTy);
1032
32
          ExpandTypeFromArgs(CAExp->EltTy, LV, AI);
1033
32
        });
1034
339
  } else if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
1035
100
    Address This = LV.getAddress(*this);
1036
100
    for (const CXXBaseSpecifier *BS : RExp->Bases) {
1037
      // Perform a single step derived-to-base conversion.
1038
1
      Address Base =
1039
1
          GetAddressOfBaseClass(This, Ty->getAsCXXRecordDecl(), &BS, &BS + 1,
1040
1
                                /*NullCheckValue=*/false, SourceLocation());
1041
1
      LValue SubLV = MakeAddrLValue(Base, BS->getType());
1042
1043
      // Recurse onto bases.
1044
1
      ExpandTypeFromArgs(BS->getType(), SubLV, AI);
1045
1
    }
1046
213
    for (auto FD : RExp->Fields) {
1047
      // FIXME: What are the right qualifiers here?
1048
213
      LValue SubLV = EmitLValueForFieldInitialization(LV, FD);
1049
213
      ExpandTypeFromArgs(FD->getType(), SubLV, AI);
1050
213
    }
1051
239
  } else if (isa<ComplexExpansion>(Exp.get())) {
1052
3
    auto realValue = &*AI++;
1053
3
    auto imagValue = &*AI++;
1054
3
    EmitStoreOfComplex(ComplexPairTy(realValue, imagValue), LV, /*init*/ true);
1055
236
  } else {
1056
    // Call EmitStoreOfScalar except when the lvalue is a bitfield to emit a
1057
    // primitive store.
1058
236
    assert(isa<NoExpansion>(Exp.get()));
1059
236
    if (LV.isBitField())
1060
0
      EmitStoreThroughLValue(RValue::get(&*AI++), LV);
1061
236
    else
1062
236
      EmitStoreOfScalar(&*AI++, LV);
1063
236
  }
1064
347
}
1065
1066
void CodeGenFunction::ExpandTypeToArgs(
1067
    QualType Ty, CallArg Arg, llvm::FunctionType *IRFuncTy,
1068
77
    SmallVectorImpl<llvm::Value *> &IRCallArgs, unsigned &IRCallArgPos) {
1069
77
  auto Exp = getTypeExpansion(Ty, getContext());
1070
77
  if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
1071
0
    Address Addr = Arg.hasLValue() ? Arg.getKnownLValue().getAddress(*this)
1072
0
                                   : Arg.getKnownRValue().getAggregateAddress();
1073
0
    forConstantArrayExpansion(
1074
0
        *this, CAExp, Addr, [&](Address EltAddr) {
1075
0
          CallArg EltArg = CallArg(
1076
0
              convertTempToRValue(EltAddr, CAExp->EltTy, SourceLocation()),
1077
0
              CAExp->EltTy);
1078
0
          ExpandTypeToArgs(CAExp->EltTy, EltArg, IRFuncTy, IRCallArgs,
1079
0
                           IRCallArgPos);
1080
0
        });
1081
77
  } else if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
1082
28
    Address This = Arg.hasLValue() ? 
Arg.getKnownLValue().getAddress(*this)12
1083
28
                                   : 
Arg.getKnownRValue().getAggregateAddress()16
;
1084
28
    for (const CXXBaseSpecifier *BS : RExp->Bases) {
1085
      // Perform a single step derived-to-base conversion.
1086
0
      Address Base =
1087
0
          GetAddressOfBaseClass(This, Ty->getAsCXXRecordDecl(), &BS, &BS + 1,
1088
0
                                /*NullCheckValue=*/false, SourceLocation());
1089
0
      CallArg BaseArg = CallArg(RValue::getAggregate(Base), BS->getType());
1090
1091
      // Recurse onto bases.
1092
0
      ExpandTypeToArgs(BS->getType(), BaseArg, IRFuncTy, IRCallArgs,
1093
0
                       IRCallArgPos);
1094
0
    }
1095
1096
28
    LValue LV = MakeAddrLValue(This, Ty);
1097
46
    for (auto FD : RExp->Fields) {
1098
46
      CallArg FldArg =
1099
46
          CallArg(EmitRValueForField(LV, FD, SourceLocation()), FD->getType());
1100
46
      ExpandTypeToArgs(FD->getType(), FldArg, IRFuncTy, IRCallArgs,
1101
46
                       IRCallArgPos);
1102
46
    }
1103
49
  } else if (isa<ComplexExpansion>(Exp.get())) {
1104
4
    ComplexPairTy CV = Arg.getKnownRValue().getComplexVal();
1105
4
    IRCallArgs[IRCallArgPos++] = CV.first;
1106
4
    IRCallArgs[IRCallArgPos++] = CV.second;
1107
45
  } else {
1108
45
    assert(isa<NoExpansion>(Exp.get()));
1109
0
    auto RV = Arg.getKnownRValue();
1110
45
    assert(RV.isScalar() &&
1111
45
           "Unexpected non-scalar rvalue during struct expansion.");
1112
1113
    // Insert a bitcast as needed.
1114
0
    llvm::Value *V = RV.getScalarVal();
1115
45
    if (IRCallArgPos < IRFuncTy->getNumParams() &&
1116
45
        V->getType() != IRFuncTy->getParamType(IRCallArgPos))
1117
0
      V = Builder.CreateBitCast(V, IRFuncTy->getParamType(IRCallArgPos));
1118
1119
45
    IRCallArgs[IRCallArgPos++] = V;
1120
45
  }
1121
77
}
1122
1123
/// Create a temporary allocation for the purposes of coercion.
1124
static Address CreateTempAllocaForCoercion(CodeGenFunction &CGF, llvm::Type *Ty,
1125
                                           CharUnits MinAlign,
1126
279
                                           const Twine &Name = "tmp") {
1127
  // Don't use an alignment that's worse than what LLVM would prefer.
1128
279
  auto PrefAlign = CGF.CGM.getDataLayout().getPrefTypeAlignment(Ty);
1129
279
  CharUnits Align = std::max(MinAlign, CharUnits::fromQuantity(PrefAlign));
1130
1131
279
  return CGF.CreateTempAlloca(Ty, Align, Name + ".coerce");
1132
279
}
1133
1134
/// EnterStructPointerForCoercedAccess - Given a struct pointer that we are
1135
/// accessing some number of bytes out of it, try to gep into the struct to get
1136
/// at its inner goodness.  Dive as deep as possible without entering an element
1137
/// with an in-memory size smaller than DstSize.
1138
static Address
1139
EnterStructPointerForCoercedAccess(Address SrcPtr,
1140
                                   llvm::StructType *SrcSTy,
1141
19.2k
                                   uint64_t DstSize, CodeGenFunction &CGF) {
1142
  // We can't dive into a zero-element struct.
1143
19.2k
  if (SrcSTy->getNumElements() == 0) 
return SrcPtr0
;
1144
1145
19.2k
  llvm::Type *FirstElt = SrcSTy->getElementType(0);
1146
1147
  // If the first elt is at least as large as what we're looking for, or if the
1148
  // first element is the same size as the whole struct, we can enter it. The
1149
  // comparison must be made on the store size and not the alloca size. Using
1150
  // the alloca size may overstate the size of the load.
1151
19.2k
  uint64_t FirstEltSize =
1152
19.2k
    CGF.CGM.getDataLayout().getTypeStoreSize(FirstElt);
1153
19.2k
  if (FirstEltSize < DstSize &&
1154
19.2k
      
FirstEltSize < CGF.CGM.getDataLayout().getTypeStoreSize(SrcSTy)5.16k
)
1155
5.00k
    return SrcPtr;
1156
1157
  // GEP into the first element.
1158
14.2k
  SrcPtr = CGF.Builder.CreateStructGEP(SrcPtr, 0, "coerce.dive");
1159
1160
  // If the first element is a struct, recurse.
1161
14.2k
  llvm::Type *SrcTy = SrcPtr.getElementType();
1162
14.2k
  if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy))
1163
3.82k
    return EnterStructPointerForCoercedAccess(SrcPtr, SrcSTy, DstSize, CGF);
1164
1165
10.3k
  return SrcPtr;
1166
14.2k
}
1167
1168
/// CoerceIntOrPtrToIntOrPtr - Convert a value Val to the specific Ty where both
1169
/// are either integers or pointers.  This does a truncation of the value if it
1170
/// is too large or a zero extension if it is too small.
1171
///
1172
/// This behaves as if the value were coerced through memory, so on big-endian
1173
/// targets the high bits are preserved in a truncation, while little-endian
1174
/// targets preserve the low bits.
1175
static llvm::Value *CoerceIntOrPtrToIntOrPtr(llvm::Value *Val,
1176
                                             llvm::Type *Ty,
1177
9.13k
                                             CodeGenFunction &CGF) {
1178
9.13k
  if (Val->getType() == Ty)
1179
8.68k
    return Val;
1180
1181
451
  if (isa<llvm::PointerType>(Val->getType())) {
1182
    // If this is Pointer->Pointer avoid conversion to and from int.
1183
351
    if (isa<llvm::PointerType>(Ty))
1184
144
      return CGF.Builder.CreateBitCast(Val, Ty, "coerce.val");
1185
1186
    // Convert the pointer to an integer so we can play with its width.
1187
207
    Val = CGF.Builder.CreatePtrToInt(Val, CGF.IntPtrTy, "coerce.val.pi");
1188
207
  }
1189
1190
307
  llvm::Type *DestIntTy = Ty;
1191
307
  if (isa<llvm::PointerType>(DestIntTy))
1192
29
    DestIntTy = CGF.IntPtrTy;
1193
1194
307
  if (Val->getType() != DestIntTy) {
1195
71
    const llvm::DataLayout &DL = CGF.CGM.getDataLayout();
1196
71
    if (DL.isBigEndian()) {
1197
      // Preserve the high bits on big-endian targets.
1198
      // That is what memory coercion does.
1199
30
      uint64_t SrcSize = DL.getTypeSizeInBits(Val->getType());
1200
30
      uint64_t DstSize = DL.getTypeSizeInBits(DestIntTy);
1201
1202
30
      if (SrcSize > DstSize) {
1203
11
        Val = CGF.Builder.CreateLShr(Val, SrcSize - DstSize, "coerce.highbits");
1204
11
        Val = CGF.Builder.CreateTrunc(Val, DestIntTy, "coerce.val.ii");
1205
19
      } else {
1206
19
        Val = CGF.Builder.CreateZExt(Val, DestIntTy, "coerce.val.ii");
1207
19
        Val = CGF.Builder.CreateShl(Val, DstSize - SrcSize, "coerce.highbits");
1208
19
      }
1209
41
    } else {
1210
      // Little-endian targets preserve the low bits. No shifts required.
1211
41
      Val = CGF.Builder.CreateIntCast(Val, DestIntTy, false, "coerce.val.ii");
1212
41
    }
1213
71
  }
1214
1215
307
  if (isa<llvm::PointerType>(Ty))
1216
29
    Val = CGF.Builder.CreateIntToPtr(Val, Ty, "coerce.val.ip");
1217
307
  return Val;
1218
451
}
1219
1220
1221
1222
/// CreateCoercedLoad - Create a load from \arg SrcPtr interpreted as
1223
/// a pointer to an object of type \arg Ty, known to be aligned to
1224
/// \arg SrcAlign bytes.
1225
///
1226
/// This safely handles the case when the src type is smaller than the
1227
/// destination type; in this situation the values of bits which not
1228
/// present in the src are undefined.
1229
static llvm::Value *CreateCoercedLoad(Address Src, llvm::Type *Ty,
1230
8.19k
                                      CodeGenFunction &CGF) {
1231
8.19k
  llvm::Type *SrcTy = Src.getElementType();
1232
1233
  // If SrcTy and Ty are the same, just do a load.
1234
8.19k
  if (SrcTy == Ty)
1235
31
    return CGF.Builder.CreateLoad(Src);
1236
1237
8.16k
  llvm::TypeSize DstSize = CGF.CGM.getDataLayout().getTypeAllocSize(Ty);
1238
1239
8.16k
  if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy)) {
1240
6.79k
    Src = EnterStructPointerForCoercedAccess(Src, SrcSTy,
1241
6.79k
                                             DstSize.getFixedSize(), CGF);
1242
6.79k
    SrcTy = Src.getElementType();
1243
6.79k
  }
1244
1245
8.16k
  llvm::TypeSize SrcSize = CGF.CGM.getDataLayout().getTypeAllocSize(SrcTy);
1246
1247
  // If the source and destination are integer or pointer types, just do an
1248
  // extension or truncation to the desired type.
1249
8.16k
  if ((isa<llvm::IntegerType>(Ty) || 
isa<llvm::PointerType>(Ty)5.22k
) &&
1250
8.16k
      
(5.97k
isa<llvm::IntegerType>(SrcTy)5.97k
||
isa<llvm::PointerType>(SrcTy)4.77k
)) {
1251
4.43k
    llvm::Value *Load = CGF.Builder.CreateLoad(Src);
1252
4.43k
    return CoerceIntOrPtrToIntOrPtr(Load, Ty, CGF);
1253
4.43k
  }
1254
1255
  // If load is legal, just bitcast the src pointer.
1256
3.72k
  if (!SrcSize.isScalable() && !DstSize.isScalable() &&
1257
3.72k
      
SrcSize.getFixedSize() >= DstSize.getFixedSize()3.70k
) {
1258
    // Generally SrcSize is never greater than DstSize, since this means we are
1259
    // losing bits. However, this can happen in cases where the structure has
1260
    // additional padding, for example due to a user specified alignment.
1261
    //
1262
    // FIXME: Assert that we aren't truncating non-padding bits when have access
1263
    // to that information.
1264
3.52k
    Src = CGF.Builder.CreateBitCast(Src,
1265
3.52k
                                    Ty->getPointerTo(Src.getAddressSpace()));
1266
3.52k
    return CGF.Builder.CreateLoad(Src);
1267
3.52k
  }
1268
1269
  // If coercing a fixed vector to a scalable vector for ABI compatibility, and
1270
  // the types match, use the llvm.experimental.vector.insert intrinsic to
1271
  // perform the conversion.
1272
202
  if (auto *ScalableDst = dyn_cast<llvm::ScalableVectorType>(Ty)) {
1273
26
    if (auto *FixedSrc = dyn_cast<llvm::FixedVectorType>(SrcTy)) {
1274
      // If we are casting a fixed i8 vector to a scalable 16 x i1 predicate
1275
      // vector, use a vector insert and bitcast the result.
1276
26
      bool NeedsBitcast = false;
1277
26
      auto PredType =
1278
26
          llvm::ScalableVectorType::get(CGF.Builder.getInt1Ty(), 16);
1279
26
      llvm::Type *OrigType = Ty;
1280
26
      if (ScalableDst == PredType &&
1281
26
          
FixedSrc->getElementType() == CGF.Builder.getInt8Ty()5
) {
1282
5
        ScalableDst = llvm::ScalableVectorType::get(CGF.Builder.getInt8Ty(), 2);
1283
5
        NeedsBitcast = true;
1284
5
      }
1285
26
      if (ScalableDst->getElementType() == FixedSrc->getElementType()) {
1286
26
        auto *Load = CGF.Builder.CreateLoad(Src);
1287
26
        auto *UndefVec = llvm::UndefValue::get(ScalableDst);
1288
26
        auto *Zero = llvm::Constant::getNullValue(CGF.CGM.Int64Ty);
1289
26
        llvm::Value *Result = CGF.Builder.CreateInsertVector(
1290
26
            ScalableDst, UndefVec, Load, Zero, "castScalableSve");
1291
26
        if (NeedsBitcast)
1292
5
          Result = CGF.Builder.CreateBitCast(Result, OrigType);
1293
26
        return Result;
1294
26
      }
1295
26
    }
1296
26
  }
1297
1298
  // Otherwise do coercion through memory. This is stupid, but simple.
1299
176
  Address Tmp =
1300
176
      CreateTempAllocaForCoercion(CGF, Ty, Src.getAlignment(), Src.getName());
1301
176
  CGF.Builder.CreateMemCpy(
1302
176
      Tmp.getPointer(), Tmp.getAlignment().getAsAlign(), Src.getPointer(),
1303
176
      Src.getAlignment().getAsAlign(),
1304
176
      llvm::ConstantInt::get(CGF.IntPtrTy, SrcSize.getKnownMinSize()));
1305
176
  return CGF.Builder.CreateLoad(Tmp);
1306
202
}
1307
1308
// Function to store a first-class aggregate into memory.  We prefer to
1309
// store the elements rather than the aggregate to be more friendly to
1310
// fast-isel.
1311
// FIXME: Do we need to recurse here?
1312
void CodeGenFunction::EmitAggregateStore(llvm::Value *Val, Address Dest,
1313
5.41k
                                         bool DestIsVolatile) {
1314
  // Prefer scalar stores to first-class aggregate stores.
1315
5.41k
  if (llvm::StructType *STy = dyn_cast<llvm::StructType>(Val->getType())) {
1316
4.45k
    for (unsigned i = 0, e = STy->getNumElements(); i != e; 
++i2.95k
) {
1317
2.95k
      Address EltPtr = Builder.CreateStructGEP(Dest, i);
1318
2.95k
      llvm::Value *Elt = Builder.CreateExtractValue(Val, i);
1319
2.95k
      Builder.CreateStore(Elt, EltPtr, DestIsVolatile);
1320
2.95k
    }
1321
3.91k
  } else {
1322
3.91k
    Builder.CreateStore(Val, Dest, DestIsVolatile);
1323
3.91k
  }
1324
5.41k
}
1325
1326
/// CreateCoercedStore - Create a store to \arg DstPtr from \arg Src,
1327
/// where the source and destination may have different types.  The
1328
/// destination is known to be aligned to \arg DstAlign bytes.
1329
///
1330
/// This safely handles the case when the src type is larger than the
1331
/// destination type; the upper bits of the src will be lost.
1332
static void CreateCoercedStore(llvm::Value *Src,
1333
                               Address Dst,
1334
                               bool DstIsVolatile,
1335
10.2k
                               CodeGenFunction &CGF) {
1336
10.2k
  llvm::Type *SrcTy = Src->getType();
1337
10.2k
  llvm::Type *DstTy = Dst.getElementType();
1338
10.2k
  if (SrcTy == DstTy) {
1339
170
    CGF.Builder.CreateStore(Src, Dst, DstIsVolatile);
1340
170
    return;
1341
170
  }
1342
1343
10.0k
  llvm::TypeSize SrcSize = CGF.CGM.getDataLayout().getTypeAllocSize(SrcTy);
1344
1345
10.0k
  if (llvm::StructType *DstSTy = dyn_cast<llvm::StructType>(DstTy)) {
1346
8.59k
    Dst = EnterStructPointerForCoercedAccess(Dst, DstSTy,
1347
8.59k
                                             SrcSize.getFixedSize(), CGF);
1348
8.59k
    DstTy = Dst.getElementType();
1349
8.59k
  }
1350
1351
10.0k
  llvm::PointerType *SrcPtrTy = llvm::dyn_cast<llvm::PointerType>(SrcTy);
1352
10.0k
  llvm::PointerType *DstPtrTy = llvm::dyn_cast<llvm::PointerType>(DstTy);
1353
10.0k
  if (SrcPtrTy && 
DstPtrTy3.28k
&&
1354
10.0k
      
SrcPtrTy->getAddressSpace() != DstPtrTy->getAddressSpace()3.28k
) {
1355
49
    Src = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Src, DstTy);
1356
49
    CGF.Builder.CreateStore(Src, Dst, DstIsVolatile);
1357
49
    return;
1358
49
  }
1359
1360
  // If the source and destination are integer or pointer types, just do an
1361
  // extension or truncation to the desired type.
1362
10.0k
  if ((isa<llvm::IntegerType>(SrcTy) || 
isa<llvm::PointerType>(SrcTy)6.95k
) &&
1363
10.0k
      
(6.31k
isa<llvm::IntegerType>(DstTy)6.31k
||
isa<llvm::PointerType>(DstTy)4.88k
)) {
1364
4.69k
    Src = CoerceIntOrPtrToIntOrPtr(Src, DstTy, CGF);
1365
4.69k
    CGF.Builder.CreateStore(Src, Dst, DstIsVolatile);
1366
4.69k
    return;
1367
4.69k
  }
1368
1369
5.34k
  llvm::TypeSize DstSize = CGF.CGM.getDataLayout().getTypeAllocSize(DstTy);
1370
1371
  // If store is legal, just bitcast the src pointer.
1372
5.34k
  if (isa<llvm::ScalableVectorType>(SrcTy) ||
1373
5.34k
      
isa<llvm::ScalableVectorType>(DstTy)5.34k
||
1374
5.34k
      
SrcSize.getFixedSize() <= DstSize.getFixedSize()5.34k
) {
1375
5.24k
    Dst = CGF.Builder.CreateElementBitCast(Dst, SrcTy);
1376
5.24k
    CGF.EmitAggregateStore(Src, Dst, DstIsVolatile);
1377
5.24k
  } else {
1378
    // Otherwise do coercion through memory. This is stupid, but
1379
    // simple.
1380
1381
    // Generally SrcSize is never greater than DstSize, since this means we are
1382
    // losing bits. However, this can happen in cases where the structure has
1383
    // additional padding, for example due to a user specified alignment.
1384
    //
1385
    // FIXME: Assert that we aren't truncating non-padding bits when have access
1386
    // to that information.
1387
103
    Address Tmp = CreateTempAllocaForCoercion(CGF, SrcTy, Dst.getAlignment());
1388
103
    CGF.Builder.CreateStore(Src, Tmp);
1389
103
    CGF.Builder.CreateMemCpy(
1390
103
        Dst.getPointer(), Dst.getAlignment().getAsAlign(), Tmp.getPointer(),
1391
103
        Tmp.getAlignment().getAsAlign(),
1392
103
        llvm::ConstantInt::get(CGF.IntPtrTy, DstSize.getFixedSize()));
1393
103
  }
1394
5.34k
}
1395
1396
static Address emitAddressAtOffset(CodeGenFunction &CGF, Address addr,
1397
20.6k
                                   const ABIArgInfo &info) {
1398
20.6k
  if (unsigned offset = info.getDirectOffset()) {
1399
3
    addr = CGF.Builder.CreateElementBitCast(addr, CGF.Int8Ty);
1400
3
    addr = CGF.Builder.CreateConstInBoundsByteGEP(addr,
1401
3
                                             CharUnits::fromQuantity(offset));
1402
3
    addr = CGF.Builder.CreateElementBitCast(addr, info.getCoerceToType());
1403
3
  }
1404
20.6k
  return addr;
1405
20.6k
}
1406
1407
namespace {
1408
1409
/// Encapsulates information about the way function arguments from
1410
/// CGFunctionInfo should be passed to actual LLVM IR function.
1411
class ClangToLLVMArgMapping {
1412
  static const unsigned InvalidIndex = ~0U;
1413
  unsigned InallocaArgNo;
1414
  unsigned SRetArgNo;
1415
  unsigned TotalIRArgs;
1416
1417
  /// Arguments of LLVM IR function corresponding to single Clang argument.
1418
  struct IRArgs {
1419
    unsigned PaddingArgIndex;
1420
    // Argument is expanded to IR arguments at positions
1421
    // [FirstArgIndex, FirstArgIndex + NumberOfArgs).
1422
    unsigned FirstArgIndex;
1423
    unsigned NumberOfArgs;
1424
1425
    IRArgs()
1426
        : PaddingArgIndex(InvalidIndex), FirstArgIndex(InvalidIndex),
1427
2.75M
          NumberOfArgs(0) {}
1428
  };
1429
1430
  SmallVector<IRArgs, 8> ArgInfo;
1431
1432
public:
1433
  ClangToLLVMArgMapping(const ASTContext &Context, const CGFunctionInfo &FI,
1434
                        bool OnlyRequiredArgs = false)
1435
      : InallocaArgNo(InvalidIndex), SRetArgNo(InvalidIndex), TotalIRArgs(0),
1436
2.75M
        ArgInfo(OnlyRequiredArgs ? FI.getNumRequiredArgs() : FI.arg_size()) {
1437
2.75M
    construct(Context, FI, OnlyRequiredArgs);
1438
2.75M
  }
1439
1440
3.29M
  bool hasInallocaArg() const { return InallocaArgNo != InvalidIndex; }
1441
568
  unsigned getInallocaArgNo() const {
1442
568
    assert(hasInallocaArg());
1443
0
    return InallocaArgNo;
1444
568
  }
1445
1446
2.38M
  bool hasSRetArg() const { return SRetArgNo != InvalidIndex; }
1447
16.1k
  unsigned getSRetArgNo() const {
1448
16.1k
    assert(hasSRetArg());
1449
0
    return SRetArgNo;
1450
16.1k
  }
1451
1452
2.75M
  unsigned totalIRArgs() const { return TotalIRArgs; }
1453
1454
4.25M
  bool hasPaddingArg(unsigned ArgNo) const {
1455
4.25M
    assert(ArgNo < ArgInfo.size());
1456
0
    return ArgInfo[ArgNo].PaddingArgIndex != InvalidIndex;
1457
4.25M
  }
1458
69
  unsigned getPaddingArgNo(unsigned ArgNo) const {
1459
69
    assert(hasPaddingArg(ArgNo));
1460
0
    return ArgInfo[ArgNo].PaddingArgIndex;
1461
69
  }
1462
1463
  /// Returns index of first IR argument corresponding to ArgNo, and their
1464
  /// quantity.
1465
4.00M
  std::pair<unsigned, unsigned> getIRArgs(unsigned ArgNo) const {
1466
4.00M
    assert(ArgNo < ArgInfo.size());
1467
0
    return std::make_pair(ArgInfo[ArgNo].FirstArgIndex,
1468
4.00M
                          ArgInfo[ArgNo].NumberOfArgs);
1469
4.00M
  }
1470
1471
private:
1472
  void construct(const ASTContext &Context, const CGFunctionInfo &FI,
1473
                 bool OnlyRequiredArgs);
1474
};
1475
1476
void ClangToLLVMArgMapping::construct(const ASTContext &Context,
1477
                                      const CGFunctionInfo &FI,
1478
2.75M
                                      bool OnlyRequiredArgs) {
1479
2.75M
  unsigned IRArgNo = 0;
1480
2.75M
  bool SwapThisWithSRet = false;
1481
2.75M
  const ABIArgInfo &RetAI = FI.getReturnInfo();
1482
1483
2.75M
  if (RetAI.getKind() == ABIArgInfo::Indirect) {
1484
16.1k
    SwapThisWithSRet = RetAI.isSRetAfterThis();
1485
16.1k
    SRetArgNo = SwapThisWithSRet ? 
1392
:
IRArgNo++15.7k
;
1486
16.1k
  }
1487
1488
2.75M
  unsigned ArgNo = 0;
1489
2.75M
  unsigned NumArgs = OnlyRequiredArgs ? 
FI.getNumRequiredArgs()1.18M
:
FI.arg_size()1.56M
;
1490
7.83M
  for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(); ArgNo < NumArgs;
1491
5.08M
       ++I, ++ArgNo) {
1492
5.08M
    assert(I != FI.arg_end());
1493
0
    QualType ArgType = I->type;
1494
5.08M
    const ABIArgInfo &AI = I->info;
1495
    // Collect data about IR arguments corresponding to Clang argument ArgNo.
1496
5.08M
    auto &IRArgs = ArgInfo[ArgNo];
1497
1498
5.08M
    if (AI.getPaddingType())
1499
124
      IRArgs.PaddingArgIndex = IRArgNo++;
1500
1501
5.08M
    switch (AI.getKind()) {
1502
124k
    case ABIArgInfo::Extend:
1503
5.00M
    case ABIArgInfo::Direct: {
1504
      // FIXME: handle sseregparm someday...
1505
5.00M
      llvm::StructType *STy = dyn_cast<llvm::StructType>(AI.getCoerceToType());
1506
5.00M
      if (AI.isDirect() && 
AI.getCanBeFlattened()4.88M
&&
STy4.88M
) {
1507
8.44k
        IRArgs.NumberOfArgs = STy->getNumElements();
1508
4.99M
      } else {
1509
4.99M
        IRArgs.NumberOfArgs = 1;
1510
4.99M
      }
1511
5.00M
      break;
1512
124k
    }
1513
16.5k
    case ABIArgInfo::Indirect:
1514
16.5k
    case ABIArgInfo::IndirectAliased:
1515
16.5k
      IRArgs.NumberOfArgs = 1;
1516
16.5k
      break;
1517
56.9k
    case ABIArgInfo::Ignore:
1518
57.8k
    case ABIArgInfo::InAlloca:
1519
      // ignore and inalloca doesn't have matching LLVM parameters.
1520
57.8k
      IRArgs.NumberOfArgs = 0;
1521
57.8k
      break;
1522
3.47k
    case ABIArgInfo::CoerceAndExpand:
1523
3.47k
      IRArgs.NumberOfArgs = AI.getCoerceAndExpandTypeSequence().size();
1524
3.47k
      break;
1525
459
    case ABIArgInfo::Expand:
1526
459
      IRArgs.NumberOfArgs = getExpansionSize(ArgType, Context);
1527
459
      break;
1528
5.08M
    }
1529
1530
5.08M
    if (IRArgs.NumberOfArgs > 0) {
1531
5.02M
      IRArgs.FirstArgIndex = IRArgNo;
1532
5.02M
      IRArgNo += IRArgs.NumberOfArgs;
1533
5.02M
    }
1534
1535
    // Skip over the sret parameter when it comes second.  We already handled it
1536
    // above.
1537
5.08M
    if (IRArgNo == 1 && 
SwapThisWithSRet2.56M
)
1538
392
      IRArgNo++;
1539
5.08M
  }
1540
2.75M
  assert(ArgNo == ArgInfo.size());
1541
1542
2.75M
  if (FI.usesInAlloca())
1543
467
    InallocaArgNo = IRArgNo++;
1544
1545
2.75M
  TotalIRArgs = IRArgNo;
1546
2.75M
}
1547
}  // namespace
1548
1549
/***/
1550
1551
20.0k
bool CodeGenModule::ReturnTypeUsesSRet(const CGFunctionInfo &FI) {
1552
20.0k
  const auto &RI = FI.getReturnInfo();
1553
20.0k
  return RI.isIndirect() || 
(20.0k
RI.isInAlloca()20.0k
&&
RI.getInAllocaSRet()1
);
1554
20.0k
}
1555
1556
12.9k
bool CodeGenModule::ReturnSlotInterferesWithArgs(const CGFunctionInfo &FI) {
1557
12.9k
  return ReturnTypeUsesSRet(FI) &&
1558
12.9k
         
getTargetCodeGenInfo().doesReturnSlotInterfereWithArgs()44
;
1559
12.9k
}
1560
1561
11.7k
bool CodeGenModule::ReturnTypeUsesFPRet(QualType ResultType) {
1562
11.7k
  if (const BuiltinType *BT = ResultType->getAs<BuiltinType>()) {
1563
4.92k
    switch (BT->getKind()) {
1564
4.86k
    default:
1565
4.86k
      return false;
1566
18
    case BuiltinType::Float:
1567
18
      return getTarget().useObjCFPRetForRealType(TargetInfo::Float);
1568
43
    case BuiltinType::Double:
1569
43
      return getTarget().useObjCFPRetForRealType(TargetInfo::Double);
1570
4
    case BuiltinType::LongDouble:
1571
4
      return getTarget().useObjCFPRetForRealType(TargetInfo::LongDouble);
1572
4.92k
    }
1573
4.92k
  }
1574
1575
6.85k
  return false;
1576
11.7k
}
1577
1578
11.7k
bool CodeGenModule::ReturnTypeUsesFP2Ret(QualType ResultType) {
1579
11.7k
  if (const ComplexType *CT = ResultType->getAs<ComplexType>()) {
1580
14
    if (const BuiltinType *BT = CT->getElementType()->getAs<BuiltinType>()) {
1581
14
      if (BT->getKind() == BuiltinType::LongDouble)
1582
2
        return getTarget().useObjCFP2RetForComplexLongDouble();
1583
14
    }
1584
14
  }
1585
1586
11.7k
  return false;
1587
11.7k
}
1588
1589
5.82k
llvm::FunctionType *CodeGenTypes::GetFunctionType(GlobalDecl GD) {
1590
5.82k
  const CGFunctionInfo &FI = arrangeGlobalDeclaration(GD);
1591
5.82k
  return GetFunctionType(FI);
1592
5.82k
}
1593
1594
llvm::FunctionType *
1595
1.18M
CodeGenTypes::GetFunctionType(const CGFunctionInfo &FI) {
1596
1597
1.18M
  bool Inserted = FunctionsBeingProcessed.insert(&FI).second;
1598
1.18M
  (void)Inserted;
1599
1.18M
  assert(Inserted && "Recursively being processed?");
1600
1601
0
  llvm::Type *resultType = nullptr;
1602
1.18M
  const ABIArgInfo &retAI = FI.getReturnInfo();
1603
1.18M
  switch (retAI.getKind()) {
1604
0
  case ABIArgInfo::Expand:
1605
0
  case ABIArgInfo::IndirectAliased:
1606
0
    llvm_unreachable("Invalid ABI kind for return argument");
1607
1608
41.4k
  case ABIArgInfo::Extend:
1609
640k
  case ABIArgInfo::Direct:
1610
640k
    resultType = retAI.getCoerceToType();
1611
640k
    break;
1612
1613
18
  case ABIArgInfo::InAlloca:
1614
18
    if (retAI.getInAllocaSRet()) {
1615
      // sret things on win32 aren't void, they return the sret pointer.
1616
18
      QualType ret = FI.getReturnType();
1617
18
      llvm::Type *ty = ConvertType(ret);
1618
18
      unsigned addressSpace = Context.getTargetAddressSpace(ret);
1619
18
      resultType = llvm::PointerType::get(ty, addressSpace);
1620
18
    } else {
1621
0
      resultType = llvm::Type::getVoidTy(getLLVMContext());
1622
0
    }
1623
18
    break;
1624
1625
6.69k
  case ABIArgInfo::Indirect:
1626
541k
  case ABIArgInfo::Ignore:
1627
541k
    resultType = llvm::Type::getVoidTy(getLLVMContext());
1628
541k
    break;
1629
1630
1.42k
  case ABIArgInfo::CoerceAndExpand:
1631
1.42k
    resultType = retAI.getUnpaddedCoerceAndExpandType();
1632
1.42k
    break;
1633
1.18M
  }
1634
1635
1.18M
  ClangToLLVMArgMapping IRFunctionArgs(getContext(), FI, true);
1636
1.18M
  SmallVector<llvm::Type*, 8> ArgTypes(IRFunctionArgs.totalIRArgs());
1637
1638
  // Add type for sret argument.
1639
1.18M
  if (IRFunctionArgs.hasSRetArg()) {
1640
6.69k
    QualType Ret = FI.getReturnType();
1641
6.69k
    llvm::Type *Ty = ConvertType(Ret);
1642
6.69k
    unsigned AddressSpace = Context.getTargetAddressSpace(Ret);
1643
6.69k
    ArgTypes[IRFunctionArgs.getSRetArgNo()] =
1644
6.69k
        llvm::PointerType::get(Ty, AddressSpace);
1645
6.69k
  }
1646
1647
  // Add type for inalloca argument.
1648
1.18M
  if (IRFunctionArgs.hasInallocaArg()) {
1649
194
    auto ArgStruct = FI.getArgStruct();
1650
194
    assert(ArgStruct);
1651
0
    ArgTypes[IRFunctionArgs.getInallocaArgNo()] = ArgStruct->getPointerTo();
1652
194
  }
1653
1654
  // Add in all of the required arguments.
1655
0
  unsigned ArgNo = 0;
1656
1.18M
  CGFunctionInfo::const_arg_iterator it = FI.arg_begin(),
1657
1.18M
                                     ie = it + FI.getNumRequiredArgs();
1658
3.24M
  for (; it != ie; 
++it, ++ArgNo2.06M
) {
1659
2.06M
    const ABIArgInfo &ArgInfo = it->info;
1660
1661
    // Insert a padding type to ensure proper alignment.
1662
2.06M
    if (IRFunctionArgs.hasPaddingArg(ArgNo))
1663
47
      ArgTypes[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
1664
47
          ArgInfo.getPaddingType();
1665
1666
2.06M
    unsigned FirstIRArg, NumIRArgs;
1667
2.06M
    std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
1668
1669
2.06M
    switch (ArgInfo.getKind()) {
1670
28.1k
    case ABIArgInfo::Ignore:
1671
28.4k
    case ABIArgInfo::InAlloca:
1672
28.4k
      assert(NumIRArgs == 0);
1673
0
      break;
1674
1675
6.93k
    case ABIArgInfo::Indirect: {
1676
6.93k
      assert(NumIRArgs == 1);
1677
      // indirect arguments are always on the stack, which is alloca addr space.
1678
0
      llvm::Type *LTy = ConvertTypeForMem(it->type);
1679
6.93k
      ArgTypes[FirstIRArg] = LTy->getPointerTo(
1680
6.93k
          CGM.getDataLayout().getAllocaAddrSpace());
1681
6.93k
      break;
1682
28.1k
    }
1683
27
    case ABIArgInfo::IndirectAliased: {
1684
27
      assert(NumIRArgs == 1);
1685
0
      llvm::Type *LTy = ConvertTypeForMem(it->type);
1686
27
      ArgTypes[FirstIRArg] = LTy->getPointerTo(ArgInfo.getIndirectAddrSpace());
1687
27
      break;
1688
28.1k
    }
1689
51.1k
    case ABIArgInfo::Extend:
1690
2.02M
    case ABIArgInfo::Direct: {
1691
      // Fast-isel and the optimizer generally like scalar values better than
1692
      // FCAs, so we flatten them if this is safe to do for this argument.
1693
2.02M
      llvm::Type *argType = ArgInfo.getCoerceToType();
1694
2.02M
      llvm::StructType *st = dyn_cast<llvm::StructType>(argType);
1695
2.02M
      if (st && 
ArgInfo.isDirect()3.86k
&&
ArgInfo.getCanBeFlattened()3.86k
) {
1696
3.63k
        assert(NumIRArgs == st->getNumElements());
1697
11.0k
        for (unsigned i = 0, e = st->getNumElements(); i != e; 
++i7.39k
)
1698
7.39k
          ArgTypes[FirstIRArg + i] = st->getElementType(i);
1699
2.02M
      } else {
1700
2.02M
        assert(NumIRArgs == 1);
1701
0
        ArgTypes[FirstIRArg] = argType;
1702
2.02M
      }
1703
0
      break;
1704
51.1k
    }
1705
1706
1.45k
    case ABIArgInfo::CoerceAndExpand: {
1707
1.45k
      auto ArgTypesIter = ArgTypes.begin() + FirstIRArg;
1708
3.12k
      for (auto EltTy : ArgInfo.getCoerceAndExpandTypeSequence()) {
1709
3.12k
        *ArgTypesIter++ = EltTy;
1710
3.12k
      }
1711
1.45k
      assert(ArgTypesIter == ArgTypes.begin() + FirstIRArg + NumIRArgs);
1712
0
      break;
1713
51.1k
    }
1714
1715
173
    case ABIArgInfo::Expand:
1716
173
      auto ArgTypesIter = ArgTypes.begin() + FirstIRArg;
1717
173
      getExpandedTypes(it->type, ArgTypesIter);
1718
173
      assert(ArgTypesIter == ArgTypes.begin() + FirstIRArg + NumIRArgs);
1719
0
      break;
1720
2.06M
    }
1721
2.06M
  }
1722
1723
1.18M
  bool Erased = FunctionsBeingProcessed.erase(&FI); (void)Erased;
1724
1.18M
  assert(Erased && "Not in set?");
1725
1726
0
  return llvm::FunctionType::get(resultType, ArgTypes, FI.isVariadic());
1727
1.18M
}
1728
1729
5.80k
llvm::Type *CodeGenTypes::GetFunctionTypeForVTable(GlobalDecl GD) {
1730
5.80k
  const CXXMethodDecl *MD = cast<CXXMethodDecl>(GD.getDecl());
1731
5.80k
  const FunctionProtoType *FPT = MD->getType()->getAs<FunctionProtoType>();
1732
1733
5.80k
  if (!isFuncTypeConvertible(FPT))
1734
39
    return llvm::StructType::get(getLLVMContext());
1735
1736
5.76k
  return GetFunctionType(GD);
1737
5.80k
}
1738
1739
static void AddAttributesFromFunctionProtoType(ASTContext &Ctx,
1740
                                               llvm::AttrBuilder &FuncAttrs,
1741
1.53M
                                               const FunctionProtoType *FPT) {
1742
1.53M
  if (!FPT)
1743
802k
    return;
1744
1745
734k
  if (!isUnresolvedExceptionSpec(FPT->getExceptionSpecType()) &&
1746
734k
      FPT->isNothrow())
1747
195k
    FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
1748
734k
}
1749
1750
bool CodeGenModule::MayDropFunctionReturn(const ASTContext &Context,
1751
48
                                          QualType ReturnType) {
1752
  // We can't just discard the return value for a record type with a
1753
  // complex destructor or a non-trivially copyable type.
1754
48
  if (const RecordType *RT =
1755
48
          ReturnType.getCanonicalType()->getAs<RecordType>()) {
1756
24
    if (const auto *ClassDecl = dyn_cast<CXXRecordDecl>(RT->getDecl()))
1757
24
      return ClassDecl->hasTrivialDestructor();
1758
24
  }
1759
24
  return ReturnType.isTriviallyCopyableType(Context);
1760
48
}
1761
1762
void CodeGenModule::getDefaultFunctionAttributes(StringRef Name,
1763
                                                 bool HasOptnone,
1764
                                                 bool AttrOnCallSite,
1765
801k
                                               llvm::AttrBuilder &FuncAttrs) {
1766
  // OptimizeNoneAttr takes precedence over -Os or -Oz. No warning needed.
1767
801k
  if (!HasOptnone) {
1768
801k
    if (CodeGenOpts.OptimizeSize)
1769
1.13k
      FuncAttrs.addAttribute(llvm::Attribute::OptimizeForSize);
1770
801k
    if (CodeGenOpts.OptimizeSize == 2)
1771
718
      FuncAttrs.addAttribute(llvm::Attribute::MinSize);
1772
801k
  }
1773
1774
801k
  if (CodeGenOpts.DisableRedZone)
1775
25
    FuncAttrs.addAttribute(llvm::Attribute::NoRedZone);
1776
801k
  if (CodeGenOpts.IndirectTlsSegRefs)
1777
1
    FuncAttrs.addAttribute("indirect-tls-seg-refs");
1778
801k
  if (CodeGenOpts.NoImplicitFloat)
1779
16
    FuncAttrs.addAttribute(llvm::Attribute::NoImplicitFloat);
1780
1781
801k
  if (AttrOnCallSite) {
1782
    // Attributes that should go on the call site only.
1783
388k
    if (!CodeGenOpts.SimplifyLibCalls || 
LangOpts.isNoBuiltinFunc(Name)196k
)
1784
191k
      FuncAttrs.addAttribute(llvm::Attribute::NoBuiltin);
1785
388k
    if (!CodeGenOpts.TrapFuncName.empty())
1786
0
      FuncAttrs.addAttribute("trap-func-name", CodeGenOpts.TrapFuncName);
1787
412k
  } else {
1788
412k
    StringRef FpKind;
1789
412k
    switch (CodeGenOpts.getFramePointer()) {
1790
272k
    case CodeGenOptions::FramePointerKind::None:
1791
272k
      FpKind = "none";
1792
272k
      break;
1793
214
    case CodeGenOptions::FramePointerKind::NonLeaf:
1794
214
      FpKind = "non-leaf";
1795
214
      break;
1796
140k
    case CodeGenOptions::FramePointerKind::All:
1797
140k
      FpKind = "all";
1798
140k
      break;
1799
412k
    }
1800
412k
    FuncAttrs.addAttribute("frame-pointer", FpKind);
1801
1802
412k
    if (CodeGenOpts.LessPreciseFPMAD)
1803
8
      FuncAttrs.addAttribute("less-precise-fpmad", "true");
1804
1805
412k
    if (CodeGenOpts.NullPointerIsValid)
1806
238
      FuncAttrs.addAttribute(llvm::Attribute::NullPointerIsValid);
1807
1808
412k
    if (CodeGenOpts.FPDenormalMode != llvm::DenormalMode::getIEEE())
1809
2
      FuncAttrs.addAttribute("denormal-fp-math",
1810
2
                             CodeGenOpts.FPDenormalMode.str());
1811
412k
    if (CodeGenOpts.FP32DenormalMode != CodeGenOpts.FPDenormalMode) {
1812
16
      FuncAttrs.addAttribute(
1813
16
          "denormal-fp-math-f32",
1814
16
          CodeGenOpts.FP32DenormalMode.str());
1815
16
    }
1816
1817
412k
    if (LangOpts.getFPExceptionMode() == LangOptions::FPE_Ignore)
1818
411k
      FuncAttrs.addAttribute("no-trapping-math", "true");
1819
1820
    // Strict (compliant) code is the default, so only add this attribute to
1821
    // indicate that we are trying to workaround a problem case.
1822
412k
    if (!CodeGenOpts.StrictFloatCastOverflow)
1823
1
      FuncAttrs.addAttribute("strict-float-cast-overflow", "false");
1824
1825
    // TODO: Are these all needed?
1826
    // unsafe/inf/nan/nsz are handled by instruction-level FastMathFlags.
1827
412k
    if (LangOpts.NoHonorInfs)
1828
277
      FuncAttrs.addAttribute("no-infs-fp-math", "true");
1829
412k
    if (LangOpts.NoHonorNaNs)
1830
297
      FuncAttrs.addAttribute("no-nans-fp-math", "true");
1831
412k
    if (LangOpts.UnsafeFPMath)
1832
264
      FuncAttrs.addAttribute("unsafe-fp-math", "true");
1833
412k
    if (CodeGenOpts.SoftFloat)
1834
63
      FuncAttrs.addAttribute("use-soft-float", "true");
1835
412k
    FuncAttrs.addAttribute("stack-protector-buffer-size",
1836
412k
                           llvm::utostr(CodeGenOpts.SSPBufferSize));
1837
412k
    if (LangOpts.NoSignedZero)
1838
270
      FuncAttrs.addAttribute("no-signed-zeros-fp-math", "true");
1839
1840
    // TODO: Reciprocal estimate codegen options should apply to instructions?
1841
412k
    const std::vector<std::string> &Recips = CodeGenOpts.Reciprocals;
1842
412k
    if (!Recips.empty())
1843
1
      FuncAttrs.addAttribute("reciprocal-estimates",
1844
1
                             llvm::join(Recips, ","));
1845
1846
412k
    if (!CodeGenOpts.PreferVectorWidth.empty() &&
1847
412k
        
CodeGenOpts.PreferVectorWidth != "none"3
)
1848
2
      FuncAttrs.addAttribute("prefer-vector-width",
1849
2
                             CodeGenOpts.PreferVectorWidth);
1850
1851
412k
    if (CodeGenOpts.StackRealignment)
1852
1
      FuncAttrs.addAttribute("stackrealign");
1853
412k
    if (CodeGenOpts.Backchain)
1854
2
      FuncAttrs.addAttribute("backchain");
1855
412k
    if (CodeGenOpts.EnableSegmentedStacks)
1856
6
      FuncAttrs.addAttribute("split-stack");
1857
1858
412k
    if (CodeGenOpts.SpeculativeLoadHardening)
1859
9
      FuncAttrs.addAttribute(llvm::Attribute::SpeculativeLoadHardening);
1860
412k
  }
1861
1862
801k
  if (getLangOpts().assumeFunctionsAreConvergent()) {
1863
    // Conservatively, mark all functions and calls in CUDA and OpenCL as
1864
    // convergent (meaning, they may call an intrinsically convergent op, such
1865
    // as __syncthreads() / barrier(), and so can't have certain optimizations
1866
    // applied around them).  LLVM will remove this attribute where it safely
1867
    // can.
1868
137k
    FuncAttrs.addAttribute(llvm::Attribute::Convergent);
1869
137k
  }
1870
1871
801k
  if (getLangOpts().CUDA && 
getLangOpts().CUDAIsDevice1.93k
) {
1872
    // Exceptions aren't supported in CUDA device code.
1873
1.22k
    FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
1874
1.22k
  }
1875
1876
801k
  for (StringRef Attr : CodeGenOpts.DefaultFunctionAttrs) {
1877
42
    StringRef Var, Value;
1878
42
    std::tie(Var, Value) = Attr.split('=');
1879
42
    FuncAttrs.addAttribute(Var, Value);
1880
42
  }
1881
801k
}
1882
1883
23
void CodeGenModule::addDefaultFunctionDefinitionAttributes(llvm::Function &F) {
1884
23
  llvm::AttrBuilder FuncAttrs;
1885
23
  getDefaultFunctionAttributes(F.getName(), F.hasOptNone(),
1886
23
                               /* AttrOnCallSite = */ false, FuncAttrs);
1887
  // TODO: call GetCPUAndFeaturesAttributes?
1888
23
  F.addFnAttrs(FuncAttrs);
1889
23
}
1890
1891
void CodeGenModule::addDefaultFunctionDefinitionAttributes(
1892
0
                                                   llvm::AttrBuilder &attrs) {
1893
0
  getDefaultFunctionAttributes(/*function name*/ "", /*optnone*/ false,
1894
0
                               /*for call*/ false, attrs);
1895
0
  GetCPUAndFeaturesAttributes(GlobalDecl(), attrs);
1896
0
}
1897
1898
static void addNoBuiltinAttributes(llvm::AttrBuilder &FuncAttrs,
1899
                                   const LangOptions &LangOpts,
1900
801k
                                   const NoBuiltinAttr *NBA = nullptr) {
1901
801k
  auto AddNoBuiltinAttr = [&FuncAttrs](StringRef BuiltinName) {
1902
2.47k
    SmallString<32> AttributeName;
1903
2.47k
    AttributeName += "no-builtin-";
1904
2.47k
    AttributeName += BuiltinName;
1905
2.47k
    FuncAttrs.addAttribute(AttributeName);
1906
2.47k
  };
1907
1908
  // First, handle the language options passed through -fno-builtin.
1909
801k
  if (LangOpts.NoBuiltin) {
1910
    // -fno-builtin disables them all.
1911
442k
    FuncAttrs.addAttribute("no-builtins");
1912
442k
    return;
1913
442k
  }
1914
1915
  // Then, add attributes for builtins specified through -fno-builtin-<name>.
1916
358k
  llvm::for_each(LangOpts.NoBuiltinFuncs, AddNoBuiltinAttr);
1917
1918
  // Now, let's check the __attribute__((no_builtin("...")) attribute added to
1919
  // the source.
1920
358k
  if (!NBA)
1921
358k
    return;
1922
1923
  // If there is a wildcard in the builtin names specified through the
1924
  // attribute, disable them all.
1925
9
  if (llvm::is_contained(NBA->builtinNames(), "*")) {
1926
1
    FuncAttrs.addAttribute("no-builtins");
1927
1
    return;
1928
1
  }
1929
1930
  // And last, add the rest of the builtin names.
1931
8
  llvm::for_each(NBA->builtinNames(), AddNoBuiltinAttr);
1932
8
}
1933
1934
static bool DetermineNoUndef(QualType QTy, CodeGenTypes &Types,
1935
                             const llvm::DataLayout &DL, const ABIArgInfo &AI,
1936
1.67M
                             bool CheckCoerce = true) {
1937
1.67M
  llvm::Type *Ty = Types.ConvertTypeForMem(QTy);
1938
1.67M
  if (AI.getKind() == ABIArgInfo::Indirect)
1939
5.06k
    return true;
1940
1.67M
  if (AI.getKind() == ABIArgInfo::Extend)
1941
36.7k
    return true;
1942
1.63M
  if (!DL.typeSizeEqualsStoreSize(Ty))
1943
    // TODO: This will result in a modest amount of values not marked noundef
1944
    // when they could be. We care about values that *invisibly* contain undef
1945
    // bits from the perspective of LLVM IR.
1946
16.8k
    return false;
1947
1.61M
  if (CheckCoerce && 
AI.canHaveCoerceToType()1.47M
) {
1948
1.45M
    llvm::Type *CoerceTy = AI.getCoerceToType();
1949
1.45M
    if (llvm::TypeSize::isKnownGT(DL.getTypeSizeInBits(CoerceTy),
1950
1.45M
                                  DL.getTypeSizeInBits(Ty)))
1951
      // If we're coercing to a type with a greater size than the canonical one,
1952
      // we're introducing new undef bits.
1953
      // Coercing to a type of smaller or equal size is ok, as we know that
1954
      // there's no internal padding (typeSizeEqualsStoreSize).
1955
591
      return false;
1956
1.45M
  }
1957
1.61M
  if (QTy->isExtIntType())
1958
48
    return true;
1959
1.61M
  if (QTy->isReferenceType())
1960
139k
    return true;
1961
1.47M
  if (QTy->isNullPtrType())
1962
234
    return false;
1963
1.47M
  if (QTy->isMemberPointerType())
1964
    // TODO: Some member pointers are `noundef`, but it depends on the ABI. For
1965
    // now, never mark them.
1966
200
    return false;
1967
1.47M
  if (QTy->isScalarType()) {
1968
1.11M
    if (const ComplexType *Complex = dyn_cast<ComplexType>(QTy))
1969
1.13k
      return DetermineNoUndef(Complex->getElementType(), Types, DL, AI, false);
1970
1.11M
    return true;
1971
1.11M
  }
1972
364k
  if (const VectorType *Vector = dyn_cast<VectorType>(QTy))
1973
144k
    return DetermineNoUndef(Vector->getElementType(), Types, DL, AI, false);
1974
219k
  if (const MatrixType *Matrix = dyn_cast<MatrixType>(QTy))
1975
227
    return DetermineNoUndef(Matrix->getElementType(), Types, DL, AI, false);
1976
219k
  if (const ArrayType *Array = dyn_cast<ArrayType>(QTy))
1977
0
    return DetermineNoUndef(Array->getElementType(), Types, DL, AI, false);
1978
1979
  // TODO: Some structs may be `noundef`, in specific situations.
1980
219k
  return false;
1981
219k
}
1982
1983
/// Construct the IR attribute list of a function or call.
1984
///
1985
/// When adding an attribute, please consider where it should be handled:
1986
///
1987
///   - getDefaultFunctionAttributes is for attributes that are essentially
1988
///     part of the global target configuration (but perhaps can be
1989
///     overridden on a per-function basis).  Adding attributes there
1990
///     will cause them to also be set in frontends that build on Clang's
1991
///     target-configuration logic, as well as for code defined in library
1992
///     modules such as CUDA's libdevice.
1993
///
1994
///   - ConstructAttributeList builds on top of getDefaultFunctionAttributes
1995
///     and adds declaration-specific, convention-specific, and
1996
///     frontend-specific logic.  The last is of particular importance:
1997
///     attributes that restrict how the frontend generates code must be
1998
///     added here rather than getDefaultFunctionAttributes.
1999
///
2000
void CodeGenModule::ConstructAttributeList(StringRef Name,
2001
                                           const CGFunctionInfo &FI,
2002
                                           CGCalleeInfo CalleeInfo,
2003
                                           llvm::AttributeList &AttrList,
2004
                                           unsigned &CallingConv,
2005
801k
                                           bool AttrOnCallSite, bool IsThunk) {
2006
801k
  llvm::AttrBuilder FuncAttrs;
2007
801k
  llvm::AttrBuilder RetAttrs;
2008
2009
  // Collect function IR attributes from the CC lowering.
2010
  // We'll collect the paramete and result attributes later.
2011
801k
  CallingConv = FI.getEffectiveCallingConvention();
2012
801k
  if (FI.isNoReturn())
2013
1.31k
    FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
2014
801k
  if (FI.isCmseNSCall())
2015
62
    FuncAttrs.addAttribute("cmse_nonsecure_call");
2016
2017
  // Collect function IR attributes from the callee prototype if we have one.
2018
801k
  AddAttributesFromFunctionProtoType(getContext(), FuncAttrs,
2019
801k
                                     CalleeInfo.getCalleeFunctionProtoType());
2020
2021
801k
  const Decl *TargetDecl = CalleeInfo.getCalleeDecl().getDecl();
2022
2023
801k
  bool HasOptnone = false;
2024
  // The NoBuiltinAttr attached to the target FunctionDecl.
2025
801k
  const NoBuiltinAttr *NBA = nullptr;
2026
2027
  // Collect function IR attributes based on declaration-specific
2028
  // information.
2029
  // FIXME: handle sseregparm someday...
2030
801k
  if (TargetDecl) {
2031
769k
    if (TargetDecl->hasAttr<ReturnsTwiceAttr>())
2032
43
      FuncAttrs.addAttribute(llvm::Attribute::ReturnsTwice);
2033
769k
    if (TargetDecl->hasAttr<NoThrowAttr>())
2034
5.93k
      FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
2035
769k
    if (TargetDecl->hasAttr<NoReturnAttr>())
2036
0
      FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
2037
769k
    if (TargetDecl->hasAttr<ColdAttr>())
2038
403
      FuncAttrs.addAttribute(llvm::Attribute::Cold);
2039
769k
    if (TargetDecl->hasAttr<HotAttr>())
2040
12
      FuncAttrs.addAttribute(llvm::Attribute::Hot);
2041
769k
    if (TargetDecl->hasAttr<NoDuplicateAttr>())
2042
6
      FuncAttrs.addAttribute(llvm::Attribute::NoDuplicate);
2043
769k
    if (TargetDecl->hasAttr<ConvergentAttr>())
2044
6
      FuncAttrs.addAttribute(llvm::Attribute::Convergent);
2045
2046
769k
    if (const FunctionDecl *Fn = dyn_cast<FunctionDecl>(TargetDecl)) {
2047
735k
      AddAttributesFromFunctionProtoType(
2048
735k
          getContext(), FuncAttrs, Fn->getType()->getAs<FunctionProtoType>());
2049
735k
      if (AttrOnCallSite && 
Fn->isReplaceableGlobalAllocationFunction()366k
) {
2050
        // A sane operator new returns a non-aliasing pointer.
2051
4.04k
        auto Kind = Fn->getDeclName().getCXXOverloadedOperator();
2052
4.04k
        if (getCodeGenOpts().AssumeSaneOperatorNew &&
2053
4.04k
            
(4.03k
Kind == OO_New4.03k
||
Kind == OO_Array_New2.78k
))
2054
1.54k
          RetAttrs.addAttribute(llvm::Attribute::NoAlias);
2055
4.04k
      }
2056
735k
      const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Fn);
2057
735k
      const bool IsVirtualCall = MD && 
MD->isVirtual()290k
;
2058
      // Don't use [[noreturn]], _Noreturn or [[no_builtin]] for a call to a
2059
      // virtual function. These attributes are not inherited by overloads.
2060
735k
      if (!(AttrOnCallSite && 
IsVirtualCall366k
)) {
2061
730k
        if (Fn->isNoReturn())
2062
3.59k
          FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
2063
730k
        NBA = Fn->getAttr<NoBuiltinAttr>();
2064
730k
      }
2065
      // Only place nomerge attribute on call sites, never functions. This
2066
      // allows it to work on indirect virtual function calls.
2067
735k
      if (AttrOnCallSite && 
TargetDecl->hasAttr<NoMergeAttr>()366k
)
2068
12
        FuncAttrs.addAttribute(llvm::Attribute::NoMerge);
2069
735k
    }
2070
2071
    // 'const', 'pure' and 'noalias' attributed functions are also nounwind.
2072
769k
    if (TargetDecl->hasAttr<ConstAttr>()) {
2073
1.83k
      FuncAttrs.addAttribute(llvm::Attribute::ReadNone);
2074
1.83k
      FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
2075
      // gcc specifies that 'const' functions have greater restrictions than
2076
      // 'pure' functions, so they also cannot have infinite loops.
2077
1.83k
      FuncAttrs.addAttribute(llvm::Attribute::WillReturn);
2078
767k
    } else if (TargetDecl->hasAttr<PureAttr>()) {
2079
223
      FuncAttrs.addAttribute(llvm::Attribute::ReadOnly);
2080
223
      FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
2081
      // gcc specifies that 'pure' functions cannot have infinite loops.
2082
223
      FuncAttrs.addAttribute(llvm::Attribute::WillReturn);
2083
767k
    } else if (TargetDecl->hasAttr<NoAliasAttr>()) {
2084
2
      FuncAttrs.addAttribute(llvm::Attribute::ArgMemOnly);
2085
2
      FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
2086
2
    }
2087
769k
    if (TargetDecl->hasAttr<RestrictAttr>())
2088
54
      RetAttrs.addAttribute(llvm::Attribute::NoAlias);
2089
769k
    if (TargetDecl->hasAttr<ReturnsNonNullAttr>() &&
2090
769k
        
!CodeGenOpts.NullPointerIsValid2.40k
)
2091
2.40k
      RetAttrs.addAttribute(llvm::Attribute::NonNull);
2092
769k
    if (TargetDecl->hasAttr<AnyX86NoCallerSavedRegistersAttr>())
2093
14
      FuncAttrs.addAttribute("no_caller_saved_registers");
2094
769k
    if (TargetDecl->hasAttr<AnyX86NoCfCheckAttr>())
2095
8
      FuncAttrs.addAttribute(llvm::Attribute::NoCfCheck);
2096
769k
    if (TargetDecl->hasAttr<LeafAttr>())
2097
1
      FuncAttrs.addAttribute(llvm::Attribute::NoCallback);
2098
2099
769k
    HasOptnone = TargetDecl->hasAttr<OptimizeNoneAttr>();
2100
769k
    if (auto *AllocSize = TargetDecl->getAttr<AllocSizeAttr>()) {
2101
2.69k
      Optional<unsigned> NumElemsParam;
2102
2.69k
      if (AllocSize->getNumElemsParam().isValid())
2103
140
        NumElemsParam = AllocSize->getNumElemsParam().getLLVMIndex();
2104
2.69k
      FuncAttrs.addAllocSizeAttr(AllocSize->getElemSizeParam().getLLVMIndex(),
2105
2.69k
                                 NumElemsParam);
2106
2.69k
    }
2107
2108
769k
    if (TargetDecl->hasAttr<OpenCLKernelAttr>()) {
2109
520
      if (getLangOpts().OpenCLVersion <= 120) {
2110
        // OpenCL v1.2 Work groups are always uniform
2111
349
        FuncAttrs.addAttribute("uniform-work-group-size", "true");
2112
349
      } else {
2113
        // OpenCL v2.0 Work groups may be whether uniform or not.
2114
        // '-cl-uniform-work-group-size' compile option gets a hint
2115
        // to the compiler that the global work-size be a multiple of
2116
        // the work-group size specified to clEnqueueNDRangeKernel
2117
        // (i.e. work groups are uniform).
2118
171
        FuncAttrs.addAttribute("uniform-work-group-size",
2119
171
                               llvm::toStringRef(CodeGenOpts.UniformWGSize));
2120
171
      }
2121
520
    }
2122
2123
769k
    std::string AssumptionValueStr;
2124
769k
    for (AssumptionAttr *AssumptionA :
2125
769k
         TargetDecl->specific_attrs<AssumptionAttr>()) {
2126
1.07k
      std::string AS = AssumptionA->getAssumption().str();
2127
1.07k
      if (!AS.empty() && !AssumptionValueStr.empty())
2128
283
        AssumptionValueStr += ",";
2129
1.07k
      AssumptionValueStr += AS;
2130
1.07k
    }
2131
2132
769k
    if (!AssumptionValueStr.empty())
2133
788
      FuncAttrs.addAttribute(llvm::AssumptionAttrKey, AssumptionValueStr);
2134
769k
  }
2135
2136
  // Attach "no-builtins" attributes to:
2137
  // * call sites: both `nobuiltin` and "no-builtins" or "no-builtin-<name>".
2138
  // * definitions: "no-builtins" or "no-builtin-<name>" only.
2139
  // The attributes can come from:
2140
  // * LangOpts: -ffreestanding, -fno-builtin, -fno-builtin-<name>
2141
  // * FunctionDecl attributes: __attribute__((no_builtin(...)))
2142
801k
  addNoBuiltinAttributes(FuncAttrs, getLangOpts(), NBA);
2143
2144
  // Collect function IR attributes based on global settiings.
2145
801k
  getDefaultFunctionAttributes(Name, HasOptnone, AttrOnCallSite, FuncAttrs);
2146
2147
  // Override some default IR attributes based on declaration-specific
2148
  // information.
2149
801k
  if (TargetDecl) {
2150
769k
    if (TargetDecl->hasAttr<NoSpeculativeLoadHardeningAttr>())
2151
13
      FuncAttrs.removeAttribute(llvm::Attribute::SpeculativeLoadHardening);
2152
769k
    if (TargetDecl->hasAttr<SpeculativeLoadHardeningAttr>())
2153
13
      FuncAttrs.addAttribute(llvm::Attribute::SpeculativeLoadHardening);
2154
769k
    if (TargetDecl->hasAttr<NoSplitStackAttr>())
2155
8
      FuncAttrs.removeAttribute("split-stack");
2156
2157
    // Add NonLazyBind attribute to function declarations when -fno-plt
2158
    // is used.
2159
    // FIXME: what if we just haven't processed the function definition
2160
    // yet, or if it's an external definition like C99 inline?
2161
769k
    if (CodeGenOpts.NoPLT) {
2162
45
      if (auto *Fn = dyn_cast<FunctionDecl>(TargetDecl)) {
2163
45
        if (!Fn->isDefined() && 
!AttrOnCallSite22
) {
2164
12
          FuncAttrs.addAttribute(llvm::Attribute::NonLazyBind);
2165
12
        }
2166
45
      }
2167
45
    }
2168
769k
  }
2169
2170
  // Add "sample-profile-suffix-elision-policy" attribute for internal linkage
2171
  // functions with -funique-internal-linkage-names.
2172
801k
  if (TargetDecl && 
CodeGenOpts.UniqueInternalLinkageNames769k
) {
2173
47
    if (isa<FunctionDecl>(TargetDecl)) {
2174
47
      if (this->getFunctionLinkage(CalleeInfo.getCalleeDecl()) ==
2175
47
          llvm::GlobalValue::InternalLinkage)
2176
34
        FuncAttrs.addAttribute("sample-profile-suffix-elision-policy",
2177
34
                               "selected");
2178
47
    }
2179
47
  }
2180
2181
  // Collect non-call-site function IR attributes from declaration-specific
2182
  // information.
2183
801k
  if (!AttrOnCallSite) {
2184
412k
    if (TargetDecl && 
TargetDecl->hasAttr<CmseNSEntryAttr>()397k
)
2185
94
      FuncAttrs.addAttribute("cmse_nonsecure_entry");
2186
2187
    // Whether tail calls are enabled.
2188
412k
    auto shouldDisableTailCalls = [&] {
2189
      // Should this be honored in getDefaultFunctionAttributes?
2190
412k
      if (CodeGenOpts.DisableTailCalls)
2191
3
        return true;
2192
2193
412k
      if (!TargetDecl)
2194
14.9k
        return false;
2195
2196
397k
      if (TargetDecl->hasAttr<DisableTailCallsAttr>() ||
2197
397k
          
TargetDecl->hasAttr<AnyX86InterruptAttr>()397k
)
2198
147
        return true;
2199
2200
397k
      if (CodeGenOpts.NoEscapingBlockTailCalls) {
2201
17
        if (const auto *BD = dyn_cast<BlockDecl>(TargetDecl))
2202
8
          if (!BD->doesNotEscape())
2203
4
            return true;
2204
17
      }
2205
2206
397k
      return false;
2207
397k
    };
2208
412k
    if (shouldDisableTailCalls())
2209
154
      FuncAttrs.addAttribute("disable-tail-calls", "true");
2210
2211
    // CPU/feature overrides.  addDefaultFunctionDefinitionAttributes
2212
    // handles these separately to set them based on the global defaults.
2213
412k
    GetCPUAndFeaturesAttributes(CalleeInfo.getCalleeDecl(), FuncAttrs);
2214
412k
  }
2215
2216
  // Collect attributes from arguments and return values.
2217
801k
  ClangToLLVMArgMapping IRFunctionArgs(getContext(), FI);
2218
2219
801k
  QualType RetTy = FI.getReturnType();
2220
801k
  const ABIArgInfo &RetAI = FI.getReturnInfo();
2221
801k
  const llvm::DataLayout &DL = getDataLayout();
2222
2223
  // C++ explicitly makes returning undefined values UB. C's rule only applies
2224
  // to used values, so we never mark them noundef for now.
2225
801k
  bool HasStrictReturn = getLangOpts().CPlusPlus;
2226
801k
  if (TargetDecl && 
HasStrictReturn769k
) {
2227
558k
    if (const FunctionDecl *FDecl = dyn_cast<FunctionDecl>(TargetDecl))
2228
527k
      HasStrictReturn &= !FDecl->isExternC();
2229
30.7k
    else if (const VarDecl *VDecl = dyn_cast<VarDecl>(TargetDecl))
2230
      // Function pointer
2231
5.03k
      HasStrictReturn &= !VDecl->isExternC();
2232
558k
  }
2233
2234
  // We don't want to be too aggressive with the return checking, unless
2235
  // it's explicit in the code opts or we're using an appropriate sanitizer.
2236
  // Try to respect what the programmer intended.
2237
801k
  HasStrictReturn &= getCodeGenOpts().StrictReturn ||
2238
801k
                     
!MayDropFunctionReturn(getContext(), RetTy)27
||
2239
801k
                     
getLangOpts().Sanitize.has(SanitizerKind::Memory)21
||
2240
801k
                     
getLangOpts().Sanitize.has(SanitizerKind::Return)21
;
2241
2242
  // Determine if the return type could be partially undef
2243
801k
  if (CodeGenOpts.EnableNoundefAttrs && 
HasStrictReturn67
) {
2244
67
    if (!RetTy->isVoidType() && 
RetAI.getKind() != ABIArgInfo::Indirect39
&&
2245
67
        
DetermineNoUndef(RetTy, getTypes(), DL, RetAI)33
)
2246
19
      RetAttrs.addAttribute(llvm::Attribute::NoUndef);
2247
67
  }
2248
2249
801k
  switch (RetAI.getKind()) {
2250
26.2k
  case ABIArgInfo::Extend:
2251
26.2k
    if (RetAI.isSignExt())
2252
12.4k
      RetAttrs.addAttribute(llvm::Attribute::SExt);
2253
13.7k
    else
2254
13.7k
      RetAttrs.addAttribute(llvm::Attribute::ZExt);
2255
26.2k
    LLVM_FALLTHROUGH;
2256
459k
  case ABIArgInfo::Direct:
2257
459k
    if (RetAI.getInReg())
2258
34
      RetAttrs.addAttribute(llvm::Attribute::InReg);
2259
459k
    break;
2260
335k
  case ABIArgInfo::Ignore:
2261
335k
    break;
2262
2263
13
  case ABIArgInfo::InAlloca:
2264
4.93k
  case ABIArgInfo::Indirect: {
2265
    // inalloca and sret disable readnone and readonly
2266
4.93k
    FuncAttrs.removeAttribute(llvm::Attribute::ReadOnly)
2267
4.93k
      .removeAttribute(llvm::Attribute::ReadNone);
2268
4.93k
    break;
2269
13
  }
2270
2271
983
  case ABIArgInfo::CoerceAndExpand:
2272
983
    break;
2273
2274
0
  case ABIArgInfo::Expand:
2275
0
  case ABIArgInfo::IndirectAliased:
2276
0
    llvm_unreachable("Invalid ABI kind for return argument");
2277
801k
  }
2278
2279
801k
  if (!IsThunk) {
2280
    // FIXME: fix this properly, https://reviews.llvm.org/D100388
2281
800k
    if (const auto *RefTy = RetTy->getAs<ReferenceType>()) {
2282
68.1k
      QualType PTy = RefTy->getPointeeType();
2283
68.1k
      if (!PTy->isIncompleteType() && 
PTy->isConstantSizeType()68.1k
)
2284
68.1k
        RetAttrs.addDereferenceableAttr(
2285
68.1k
            getMinimumObjectSize(PTy).getQuantity());
2286
68.1k
      if (getContext().getTargetAddressSpace(PTy) == 0 &&
2287
68.1k
          
!CodeGenOpts.NullPointerIsValid68.1k
)
2288
68.1k
        RetAttrs.addAttribute(llvm::Attribute::NonNull);
2289
68.1k
      if (PTy->isObjectType()) {
2290
67.5k
        llvm::Align Alignment =
2291
67.5k
            getNaturalPointeeTypeAlignment(RetTy).getAsAlign();
2292
67.5k
        RetAttrs.addAlignmentAttr(Alignment);
2293
67.5k
      }
2294
68.1k
    }
2295
800k
  }
2296
2297
801k
  bool hasUsedSRet = false;
2298
801k
  SmallVector<llvm::AttributeSet, 4> ArgAttrs(IRFunctionArgs.totalIRArgs());
2299
2300
  // Attach attributes to sret.
2301
801k
  if (IRFunctionArgs.hasSRetArg()) {
2302
4.91k
    llvm::AttrBuilder SRETAttrs;
2303
4.91k
    SRETAttrs.addStructRetAttr(getTypes().ConvertTypeForMem(RetTy));
2304
4.91k
    hasUsedSRet = true;
2305
4.91k
    if (RetAI.getInReg())
2306
67
      SRETAttrs.addAttribute(llvm::Attribute::InReg);
2307
4.91k
    SRETAttrs.addAlignmentAttr(RetAI.getIndirectAlign().getQuantity());
2308
4.91k
    ArgAttrs[IRFunctionArgs.getSRetArgNo()] =
2309
4.91k
        llvm::AttributeSet::get(getLLVMContext(), SRETAttrs);
2310
4.91k
  }
2311
2312
  // Attach attributes to inalloca argument.
2313
801k
  if (IRFunctionArgs.hasInallocaArg()) {
2314
160
    llvm::AttrBuilder Attrs;
2315
160
    Attrs.addInAllocaAttr(FI.getArgStruct());
2316
160
    ArgAttrs[IRFunctionArgs.getInallocaArgNo()] =
2317
160
        llvm::AttributeSet::get(getLLVMContext(), Attrs);
2318
160
  }
2319
2320
  // Apply `nonnull`, `dereferencable(N)` and `align N` to the `this` argument,
2321
  // unless this is a thunk function.
2322
  // FIXME: fix this properly, https://reviews.llvm.org/D100388
2323
801k
  if (FI.isInstanceMethod() && 
!IRFunctionArgs.hasInallocaArg()274k
&&
2324
801k
      
!FI.arg_begin()->type->isVoidPointerType()274k
&&
!IsThunk274k
) {
2325
273k
    auto IRArgs = IRFunctionArgs.getIRArgs(0);
2326
2327
273k
    assert(IRArgs.second == 1 && "Expected only a single `this` pointer.");
2328
2329
0
    llvm::AttrBuilder Attrs;
2330
2331
273k
    QualType ThisTy =
2332
273k
        FI.arg_begin()->type.castAs<PointerType>()->getPointeeType();
2333
2334
273k
    if (!CodeGenOpts.NullPointerIsValid &&
2335
273k
        
getContext().getTargetAddressSpace(FI.arg_begin()->type) == 0273k
) {
2336
273k
      Attrs.addAttribute(llvm::Attribute::NonNull);
2337
273k
      Attrs.addDereferenceableAttr(getMinimumObjectSize(ThisTy).getQuantity());
2338
273k
    } else {
2339
      // FIXME dereferenceable should be correct here, regardless of
2340
      // NullPointerIsValid. However, dereferenceable currently does not always
2341
      // respect NullPointerIsValid and may imply nonnull and break the program.
2342
      // See https://reviews.llvm.org/D66618 for discussions.
2343
318
      Attrs.addDereferenceableOrNullAttr(
2344
318
          getMinimumObjectSize(
2345
318
              FI.arg_begin()->type.castAs<PointerType>()->getPointeeType())
2346
318
              .getQuantity());
2347
318
    }
2348
2349
273k
    llvm::Align Alignment =
2350
273k
        getNaturalTypeAlignment(ThisTy, /*BaseInfo=*/nullptr,
2351
273k
                                /*TBAAInfo=*/nullptr, /*forPointeeType=*/true)
2352
273k
            .getAsAlign();
2353
273k
    Attrs.addAlignmentAttr(Alignment);
2354
2355
273k
    ArgAttrs[IRArgs.first] = llvm::AttributeSet::get(getLLVMContext(), Attrs);
2356
273k
  }
2357
2358
0
  unsigned ArgNo = 0;
2359
801k
  for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(),
2360
801k
                                          E = FI.arg_end();
2361
2.33M
       I != E; 
++I, ++ArgNo1.53M
) {
2362
1.53M
    QualType ParamType = I->type;
2363
1.53M
    const ABIArgInfo &AI = I->info;
2364
1.53M
    llvm::AttrBuilder Attrs;
2365
2366
    // Add attribute for padding argument, if necessary.
2367
1.53M
    if (IRFunctionArgs.hasPaddingArg(ArgNo)) {
2368
47
      if (AI.getPaddingInReg()) {
2369
5
        ArgAttrs[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
2370
5
            llvm::AttributeSet::get(
2371
5
                getLLVMContext(),
2372
5
                llvm::AttrBuilder().addAttribute(llvm::Attribute::InReg));
2373
5
      }
2374
47
    }
2375
2376
    // Decide whether the argument we're handling could be partially undef
2377
1.53M
    bool ArgNoUndef = DetermineNoUndef(ParamType, getTypes(), DL, AI);
2378
1.53M
    if (CodeGenOpts.EnableNoundefAttrs && 
ArgNoUndef40
)
2379
27
      Attrs.addAttribute(llvm::Attribute::NoUndef);
2380
2381
    // 'restrict' -> 'noalias' is done in EmitFunctionProlog when we
2382
    // have the corresponding parameter variable.  It doesn't make
2383
    // sense to do it here because parameters are so messed up.
2384
1.53M
    switch (AI.getKind()) {
2385
36.7k
    case ABIArgInfo::Extend:
2386
36.7k
      if (AI.isSignExt())
2387
17.5k
        Attrs.addAttribute(llvm::Attribute::SExt);
2388
19.2k
      else
2389
19.2k
        Attrs.addAttribute(llvm::Attribute::ZExt);
2390
36.7k
      LLVM_FALLTHROUGH;
2391
1.51M
    case ABIArgInfo::Direct:
2392
1.51M
      if (ArgNo == 0 && 
FI.isChainCall()726k
)
2393
8
        Attrs.addAttribute(llvm::Attribute::Nest);
2394
1.51M
      else if (AI.getInReg())
2395
586
        Attrs.addAttribute(llvm::Attribute::InReg);
2396
1.51M
      Attrs.addStackAlignmentAttr(llvm::MaybeAlign(AI.getDirectAlign()));
2397
1.51M
      break;
2398
2399
5.06k
    case ABIArgInfo::Indirect: {
2400
5.06k
      if (AI.getInReg())
2401
24
        Attrs.addAttribute(llvm::Attribute::InReg);
2402
2403
5.06k
      if (AI.getIndirectByVal())
2404
1.99k
        Attrs.addByValAttr(getTypes().ConvertTypeForMem(ParamType));
2405
2406
5.06k
      auto *Decl = ParamType->getAsRecordDecl();
2407
5.06k
      if (CodeGenOpts.PassByValueIsNoAlias && 
Decl7
&&
2408
5.06k
          
Decl->getArgPassingRestrictions() == RecordDecl::APK_CanPassInRegs7
)
2409
        // When calling the function, the pointer passed in will be the only
2410
        // reference to the underlying object. Mark it accordingly.
2411
5
        Attrs.addAttribute(llvm::Attribute::NoAlias);
2412
2413
      // TODO: We could add the byref attribute if not byval, but it would
2414
      // require updating many testcases.
2415
2416
5.06k
      CharUnits Align = AI.getIndirectAlign();
2417
2418
      // In a byval argument, it is important that the required
2419
      // alignment of the type is honored, as LLVM might be creating a
2420
      // *new* stack object, and needs to know what alignment to give
2421
      // it. (Sometimes it can deduce a sensible alignment on its own,
2422
      // but not if clang decides it must emit a packed struct, or the
2423
      // user specifies increased alignment requirements.)
2424
      //
2425
      // This is different from indirect *not* byval, where the object
2426
      // exists already, and the align attribute is purely
2427
      // informative.
2428
5.06k
      assert(!Align.isZero());
2429
2430
      // For now, only add this when we have a byval argument.
2431
      // TODO: be less lazy about updating test cases.
2432
5.06k
      if (AI.getIndirectByVal())
2433
1.99k
        Attrs.addAlignmentAttr(Align.getQuantity());
2434
2435
      // byval disables readnone and readonly.
2436
5.06k
      FuncAttrs.removeAttribute(llvm::Attribute::ReadOnly)
2437
5.06k
        .removeAttribute(llvm::Attribute::ReadNone);
2438
2439
5.06k
      break;
2440
36.7k
    }
2441
16
    case ABIArgInfo::IndirectAliased: {
2442
16
      CharUnits Align = AI.getIndirectAlign();
2443
16
      Attrs.addByRefAttr(getTypes().ConvertTypeForMem(ParamType));
2444
16
      Attrs.addAlignmentAttr(Align.getQuantity());
2445
16
      break;
2446
36.7k
    }
2447
14.4k
    case ABIArgInfo::Ignore:
2448
14.6k
    case ABIArgInfo::Expand:
2449
15.6k
    case ABIArgInfo::CoerceAndExpand:
2450
15.6k
      break;
2451
2452
317
    case ABIArgInfo::InAlloca:
2453
      // inalloca disables readnone and readonly.
2454
317
      FuncAttrs.removeAttribute(llvm::Attribute::ReadOnly)
2455
317
          .removeAttribute(llvm::Attribute::ReadNone);
2456
317
      continue;
2457
1.53M
    }
2458
2459
1.53M
    if (const auto *RefTy = ParamType->getAs<ReferenceType>()) {
2460
139k
      QualType PTy = RefTy->getPointeeType();
2461
139k
      if (!PTy->isIncompleteType() && 
PTy->isConstantSizeType()139k
)
2462
139k
        Attrs.addDereferenceableAttr(
2463
139k
            getMinimumObjectSize(PTy).getQuantity());
2464
139k
      if (getContext().getTargetAddressSpace(PTy) == 0 &&
2465
139k
          
!CodeGenOpts.NullPointerIsValid139k
)
2466
139k
        Attrs.addAttribute(llvm::Attribute::NonNull);
2467
139k
      if (PTy->isObjectType()) {
2468
137k
        llvm::Align Alignment =
2469
137k
            getNaturalPointeeTypeAlignment(ParamType).getAsAlign();
2470
137k
        Attrs.addAlignmentAttr(Alignment);
2471
137k
      }
2472
139k
    }
2473
2474
1.53M
    switch (FI.getExtParameterInfo(ArgNo).getABI()) {
2475
1.53M
    case ParameterABI::Ordinary:
2476
1.53M
      break;
2477
2478
48
    case ParameterABI::SwiftIndirectResult: {
2479
      // Add 'sret' if we haven't already used it for something, but
2480
      // only if the result is void.
2481
48
      if (!hasUsedSRet && 
RetTy->isVoidType()24
) {
2482
8
        Attrs.addStructRetAttr(getTypes().ConvertTypeForMem(ParamType));
2483
8
        hasUsedSRet = true;
2484
8
      }
2485
2486
      // Add 'noalias' in either case.
2487
48
      Attrs.addAttribute(llvm::Attribute::NoAlias);
2488
2489
      // Add 'dereferenceable' and 'alignment'.
2490
48
      auto PTy = ParamType->getPointeeType();
2491
48
      if (!PTy->isIncompleteType() && PTy->isConstantSizeType()) {
2492
48
        auto info = getContext().getTypeInfoInChars(PTy);
2493
48
        Attrs.addDereferenceableAttr(info.Width.getQuantity());
2494
48
        Attrs.addAlignmentAttr(info.Align.getAsAlign());
2495
48
      }
2496
48
      break;
2497
0
    }
2498
2499
24
    case ParameterABI::SwiftErrorResult:
2500
24
      Attrs.addAttribute(llvm::Attribute::SwiftError);
2501
24
      break;
2502
2503
40
    case ParameterABI::SwiftContext:
2504
40
      Attrs.addAttribute(llvm::Attribute::SwiftSelf);
2505
40
      break;
2506
2507
325
    case ParameterABI::SwiftAsyncContext:
2508
325
      Attrs.addAttribute(llvm::Attribute::SwiftAsync);
2509
325
      break;
2510
1.53M
    }
2511
2512
1.53M
    if (FI.getExtParameterInfo(ArgNo).isNoEscape())
2513
62
      Attrs.addAttribute(llvm::Attribute::NoCapture);
2514
2515
1.53M
    if (Attrs.hasAttributes()) {
2516
179k
      unsigned FirstIRArg, NumIRArgs;
2517
179k
      std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
2518
359k
      for (unsigned i = 0; i < NumIRArgs; 
i++179k
)
2519
179k
        ArgAttrs[FirstIRArg + i] =
2520
179k
            llvm::AttributeSet::get(getLLVMContext(), Attrs);
2521
179k
    }
2522
1.53M
  }
2523
801k
  assert(ArgNo == FI.arg_size());
2524
2525
0
  AttrList = llvm::AttributeList::get(
2526
801k
      getLLVMContext(), llvm::AttributeSet::get(getLLVMContext(), FuncAttrs),
2527
801k
      llvm::AttributeSet::get(getLLVMContext(), RetAttrs), ArgAttrs);
2528
801k
}
2529
2530
/// An argument came in as a promoted argument; demote it back to its
2531
/// declared type.
2532
static llvm::Value *emitArgumentDemotion(CodeGenFunction &CGF,
2533
                                         const VarDecl *var,
2534
14
                                         llvm::Value *value) {
2535
14
  llvm::Type *varType = CGF.ConvertType(var->getType());
2536
2537
  // This can happen with promotions that actually don't change the
2538
  // underlying type, like the enum promotions.
2539
14
  if (value->getType() == varType) 
return value0
;
2540
2541
14
  assert((varType->isIntegerTy() || varType->isFloatingPointTy())
2542
14
         && "unexpected promotion type");
2543
2544
14
  if (isa<llvm::IntegerType>(varType))
2545
8
    return CGF.Builder.CreateTrunc(value, varType, "arg.unpromote");
2546
2547
6
  return CGF.Builder.CreateFPCast(value, varType, "arg.unpromote");
2548
14
}
2549
2550
/// Returns the attribute (either parameter attribute, or function
2551
/// attribute), which declares argument ArgNo to be non-null.
2552
static const NonNullAttr *getNonNullAttr(const Decl *FD, const ParmVarDecl *PVD,
2553
185k
                                         QualType ArgType, unsigned ArgNo) {
2554
  // FIXME: __attribute__((nonnull)) can also be applied to:
2555
  //   - references to pointers, where the pointee is known to be
2556
  //     nonnull (apparently a Clang extension)
2557
  //   - transparent unions containing pointers
2558
  // In the former case, LLVM IR cannot represent the constraint. In
2559
  // the latter case, we have no guarantee that the transparent union
2560
  // is in fact passed as a pointer.
2561
185k
  if (!ArgType->isAnyPointerType() && 
!ArgType->isBlockPointerType()49.0k
)
2562
48.9k
    return nullptr;
2563
  // First, check attribute on parameter itself.
2564
136k
  if (PVD) {
2565
136k
    if (auto ParmNNAttr = PVD->getAttr<NonNullAttr>())
2566
10
      return ParmNNAttr;
2567
136k
  }
2568
  // Check function attributes.
2569
136k
  if (!FD)
2570
5
    return nullptr;
2571
136k
  for (const auto *NNAttr : FD->specific_attrs<NonNullAttr>()) {
2572
32
    if (NNAttr->isNonNull(ArgNo))
2573
28
      return NNAttr;
2574
32
  }
2575
136k
  return nullptr;
2576
136k
}
2577
2578
namespace {
2579
  struct CopyBackSwiftError final : EHScopeStack::Cleanup {
2580
    Address Temp;
2581
    Address Arg;
2582
16
    CopyBackSwiftError(Address temp, Address arg) : Temp(temp), Arg(arg) {}
2583
16
    void Emit(CodeGenFunction &CGF, Flags flags) override {
2584
16
      llvm::Value *errorValue = CGF.Builder.CreateLoad(Temp);
2585
16
      CGF.Builder.CreateStore(errorValue, Arg);
2586
16
    }
2587
  };
2588
}
2589
2590
void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
2591
                                         llvm::Function *Fn,
2592
377k
                                         const FunctionArgList &Args) {
2593
377k
  if (CurCodeDecl && 
CurCodeDecl->hasAttr<NakedAttr>()368k
)
2594
    // Naked functions don't have prologues.
2595
9
    return;
2596
2597
  // If this is an implicit-return-zero function, go ahead and
2598
  // initialize the return value.  TODO: it might be nice to have
2599
  // a more general mechanism for this that didn't require synthesized
2600
  // return statements.
2601
377k
  if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(CurCodeDecl)) {
2602
333k
    if (FD->hasImplicitReturnZero()) {
2603
5.53k
      QualType RetTy = FD->getReturnType().getUnqualifiedType();
2604
5.53k
      llvm::Type* LLVMTy = CGM.getTypes().ConvertType(RetTy);
2605
5.53k
      llvm::Constant* Zero = llvm::Constant::getNullValue(LLVMTy);
2606
5.53k
      Builder.CreateStore(Zero, ReturnValue);
2607
5.53k
    }
2608
333k
  }
2609
2610
  // FIXME: We no longer need the types from FunctionArgList; lift up and
2611
  // simplify.
2612
2613
377k
  ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), FI);
2614
377k
  assert(Fn->arg_size() == IRFunctionArgs.totalIRArgs());
2615
2616
  // If we're using inalloca, all the memory arguments are GEPs off of the last
2617
  // parameter, which is a pointer to the complete memory area.
2618
0
  Address ArgStruct = Address::invalid();
2619
377k
  if (IRFunctionArgs.hasInallocaArg()) {
2620
41
    ArgStruct = Address(Fn->getArg(IRFunctionArgs.getInallocaArgNo()),
2621
41
                        FI.getArgStructAlignment());
2622
2623
41
    assert(ArgStruct.getType() == FI.getArgStruct()->getPointerTo());
2624
41
  }
2625
2626
  // Name the struct return parameter.
2627
377k
  if (IRFunctionArgs.hasSRetArg()) {
2628
2.91k
    auto AI = Fn->getArg(IRFunctionArgs.getSRetArgNo());
2629
2.91k
    AI->setName("agg.result");
2630
2.91k
    AI->addAttr(llvm::Attribute::NoAlias);
2631
2.91k
  }
2632
2633
  // Track if we received the parameter as a pointer (indirect, byval, or
2634
  // inalloca).  If already have a pointer, EmitParmDecl doesn't need to copy it
2635
  // into a local alloca for us.
2636
377k
  SmallVector<ParamValue, 16> ArgVals;
2637
377k
  ArgVals.reserve(Args.size());
2638
2639
  // Create a pointer value for every parameter declaration.  This usually
2640
  // entails copying one or more LLVM IR arguments into an alloca.  Don't push
2641
  // any cleanups or do anything that might unwind.  We do that separately, so
2642
  // we can push the cleanups in the correct order for the ABI.
2643
377k
  assert(FI.arg_size() == Args.size() &&
2644
377k
         "Mismatch between function signature & arguments.");
2645
0
  unsigned ArgNo = 0;
2646
377k
  CGFunctionInfo::const_arg_iterator info_it = FI.arg_begin();
2647
377k
  for (FunctionArgList::const_iterator i = Args.begin(), e = Args.end();
2648
1.20M
       i != e; 
++i, ++info_it, ++ArgNo829k
) {
2649
829k
    const VarDecl *Arg = *i;
2650
829k
    const ABIArgInfo &ArgI = info_it->info;
2651
2652
829k
    bool isPromoted =
2653
829k
      isa<ParmVarDecl>(Arg) && 
cast<ParmVarDecl>(Arg)->isKNRPromoted()635k
;
2654
    // We are converting from ABIArgInfo type to VarDecl type directly, unless
2655
    // the parameter is promoted. In this case we convert to
2656
    // CGFunctionInfo::ArgInfo type with subsequent argument demotion.
2657
829k
    QualType Ty = isPromoted ? 
info_it->type14
:
Arg->getType()829k
;
2658
829k
    assert(hasScalarEvaluationKind(Ty) ==
2659
829k
           hasScalarEvaluationKind(Arg->getType()));
2660
2661
0
    unsigned FirstIRArg, NumIRArgs;
2662
829k
    std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
2663
2664
829k
    switch (ArgI.getKind()) {
2665
73
    case ABIArgInfo::InAlloca: {
2666
73
      assert(NumIRArgs == 0);
2667
0
      auto FieldIndex = ArgI.getInAllocaFieldIndex();
2668
73
      Address V =
2669
73
          Builder.CreateStructGEP(ArgStruct, FieldIndex, Arg->getName());
2670
73
      if (ArgI.getInAllocaIndirect())
2671
6
        V = Address(Builder.CreateLoad(V),
2672
6
                    getContext().getTypeAlignInChars(Ty));
2673
73
      ArgVals.push_back(ParamValue::forIndirect(V));
2674
73
      break;
2675
0
    }
2676
2677
2.50k
    case ABIArgInfo::Indirect:
2678
2.51k
    case ABIArgInfo::IndirectAliased: {
2679
2.51k
      assert(NumIRArgs == 1);
2680
0
      Address ParamAddr =
2681
2.51k
          Address(Fn->getArg(FirstIRArg), ArgI.getIndirectAlign());
2682
2683
2.51k
      if (!hasScalarEvaluationKind(Ty)) {
2684
        // Aggregates and complex variables are accessed by reference. All we
2685
        // need to do is realign the value, if requested. Also, if the address
2686
        // may be aliased, copy it to ensure that the parameter variable is
2687
        // mutable and has a unique adress, as C requires.
2688
2.13k
        Address V = ParamAddr;
2689
2.13k
        if (ArgI.getIndirectRealign() || 
ArgI.isIndirectAliased()2.10k
) {
2690
45
          Address AlignedTemp = CreateMemTemp(Ty, "coerce");
2691
2692
          // Copy from the incoming argument pointer to the temporary with the
2693
          // appropriate alignment.
2694
          //
2695
          // FIXME: We should have a common utility for generating an aggregate
2696
          // copy.
2697
45
          CharUnits Size = getContext().getTypeSizeInChars(Ty);
2698
45
          Builder.CreateMemCpy(
2699
45
              AlignedTemp.getPointer(), AlignedTemp.getAlignment().getAsAlign(),
2700
45
              ParamAddr.getPointer(), ParamAddr.getAlignment().getAsAlign(),
2701
45
              llvm::ConstantInt::get(IntPtrTy, Size.getQuantity()));
2702
45
          V = AlignedTemp;
2703
45
        }
2704
2.13k
        ArgVals.push_back(ParamValue::forIndirect(V));
2705
2.13k
      } else {
2706
        // Load scalar value from indirect argument.
2707
388
        llvm::Value *V =
2708
388
            EmitLoadOfScalar(ParamAddr, false, Ty, Arg->getBeginLoc());
2709
2710
388
        if (isPromoted)
2711
0
          V = emitArgumentDemotion(*this, Arg, V);
2712
388
        ArgVals.push_back(ParamValue::forDirect(V));
2713
388
      }
2714
2.51k
      break;
2715
2.50k
    }
2716
2717
24.6k
    case ABIArgInfo::Extend:
2718
818k
    case ABIArgInfo::Direct: {
2719
818k
      auto AI = Fn->getArg(FirstIRArg);
2720
818k
      llvm::Type *LTy = ConvertType(Arg->getType());
2721
2722
      // Prepare parameter attributes. So far, only attributes for pointer
2723
      // parameters are prepared. See
2724
      // http://llvm.org/docs/LangRef.html#paramattrs.
2725
818k
      if (ArgI.getDirectOffset() == 0 && 
LTy->isPointerTy()818k
&&
2726
818k
          
ArgI.getCoerceToType()->isPointerTy()352k
) {
2727
352k
        assert(NumIRArgs == 1);
2728
2729
352k
        if (const ParmVarDecl *PVD = dyn_cast<ParmVarDecl>(Arg)) {
2730
          // Set `nonnull` attribute if any.
2731
185k
          if (getNonNullAttr(CurCodeDecl, PVD, PVD->getType(),
2732
185k
                             PVD->getFunctionScopeIndex()) &&
2733
185k
              
!CGM.getCodeGenOpts().NullPointerIsValid22
)
2734
13
            AI->addAttr(llvm::Attribute::NonNull);
2735
2736
185k
          QualType OTy = PVD->getOriginalType();
2737
185k
          if (const auto *ArrTy =
2738
185k
              getContext().getAsConstantArrayType(OTy)) {
2739
            // A C99 array parameter declaration with the static keyword also
2740
            // indicates dereferenceability, and if the size is constant we can
2741
            // use the dereferenceable attribute (which requires the size in
2742
            // bytes).
2743
553
            if (ArrTy->getSizeModifier() == ArrayType::Static) {
2744
6
              QualType ETy = ArrTy->getElementType();
2745
6
              llvm::Align Alignment =
2746
6
                  CGM.getNaturalTypeAlignment(ETy).getAsAlign();
2747
6
              AI->addAttrs(llvm::AttrBuilder().addAlignmentAttr(Alignment));
2748
6
              uint64_t ArrSize = ArrTy->getSize().getZExtValue();
2749
6
              if (!ETy->isIncompleteType() && ETy->isConstantSizeType() &&
2750
6
                  ArrSize) {
2751
4
                llvm::AttrBuilder Attrs;
2752
4
                Attrs.addDereferenceableAttr(
2753
4
                    getContext().getTypeSizeInChars(ETy).getQuantity() *
2754
4
                    ArrSize);
2755
4
                AI->addAttrs(Attrs);
2756
4
              } else 
if (2
getContext().getTargetInfo().getNullPointerValue(
2757
2
                             ETy.getAddressSpace()) == 0 &&
2758
2
                         !CGM.getCodeGenOpts().NullPointerIsValid) {
2759
1
                AI->addAttr(llvm::Attribute::NonNull);
2760
1
              }
2761
6
            }
2762
185k
          } else if (const auto *ArrTy =
2763
185k
                     getContext().getAsVariableArrayType(OTy)) {
2764
            // For C99 VLAs with the static keyword, we don't know the size so
2765
            // we can't use the dereferenceable attribute, but in addrspace(0)
2766
            // we know that it must be nonnull.
2767
102
            if (ArrTy->getSizeModifier() == VariableArrayType::Static) {
2768
2
              QualType ETy = ArrTy->getElementType();
2769
2
              llvm::Align Alignment =
2770
2
                  CGM.getNaturalTypeAlignment(ETy).getAsAlign();
2771
2
              AI->addAttrs(llvm::AttrBuilder().addAlignmentAttr(Alignment));
2772
2
              if (!getContext().getTargetAddressSpace(ETy) &&
2773
2
                  !CGM.getCodeGenOpts().NullPointerIsValid)
2774
1
                AI->addAttr(llvm::Attribute::NonNull);
2775
2
            }
2776
102
          }
2777
2778
          // Set `align` attribute if any.
2779
185k
          const auto *AVAttr = PVD->getAttr<AlignValueAttr>();
2780
185k
          if (!AVAttr)
2781
185k
            if (const auto *TOTy = dyn_cast<TypedefType>(OTy))
2782
6.93k
              AVAttr = TOTy->getDecl()->getAttr<AlignValueAttr>();
2783
185k
          if (AVAttr && 
!SanOpts.has(SanitizerKind::Alignment)13
) {
2784
            // If alignment-assumption sanitizer is enabled, we do *not* add
2785
            // alignment attribute here, but emit normal alignment assumption,
2786
            // so the UBSAN check could function.
2787
10
            llvm::ConstantInt *AlignmentCI =
2788
10
                cast<llvm::ConstantInt>(EmitScalarExpr(AVAttr->getAlignment()));
2789
10
            unsigned AlignmentInt =
2790
10
                AlignmentCI->getLimitedValue(llvm::Value::MaximumAlignment);
2791
10
            if (AI->getParamAlign().valueOrOne() < AlignmentInt) {
2792
10
              AI->removeAttr(llvm::Attribute::AttrKind::Alignment);
2793
10
              AI->addAttrs(llvm::AttrBuilder().addAlignmentAttr(
2794
10
                  llvm::Align(AlignmentInt)));
2795
10
            }
2796
10
          }
2797
185k
        }
2798
2799
        // Set 'noalias' if an argument type has the `restrict` qualifier.
2800
352k
        if (Arg->getType().isRestrictQualified())
2801
31.7k
          AI->addAttr(llvm::Attribute::NoAlias);
2802
352k
      }
2803
2804
      // Prepare the argument value. If we have the trivial case, handle it
2805
      // with no muss and fuss.
2806
818k
      if (!isa<llvm::StructType>(ArgI.getCoerceToType()) &&
2807
818k
          
ArgI.getCoerceToType() == ConvertType(Ty)817k
&&
2808
818k
          
ArgI.getDirectOffset() == 0812k
) {
2809
812k
        assert(NumIRArgs == 1);
2810
2811
        // LLVM expects swifterror parameters to be used in very restricted
2812
        // ways.  Copy the value into a less-restricted temporary.
2813
0
        llvm::Value *V = AI;
2814
812k
        if (FI.getExtParameterInfo(ArgNo).getABI()
2815
812k
              == ParameterABI::SwiftErrorResult) {
2816
16
          QualType pointeeTy = Ty->getPointeeType();
2817
16
          assert(pointeeTy->isPointerType());
2818
0
          Address temp =
2819
16
            CreateMemTemp(pointeeTy, getPointerAlign(), "swifterror.temp");
2820
16
          Address arg = Address(V, getContext().getTypeAlignInChars(pointeeTy));
2821
16
          llvm::Value *incomingErrorValue = Builder.CreateLoad(arg);
2822
16
          Builder.CreateStore(incomingErrorValue, temp);
2823
16
          V = temp.getPointer();
2824
2825
          // Push a cleanup to copy the value back at the end of the function.
2826
          // The convention does not guarantee that the value will be written
2827
          // back if the function exits with an unwind exception.
2828
16
          EHStack.pushCleanup<CopyBackSwiftError>(NormalCleanup, temp, arg);
2829
16
        }
2830
2831
        // Ensure the argument is the correct type.
2832
812k
        if (V->getType() != ArgI.getCoerceToType())
2833
0
          V = Builder.CreateBitCast(V, ArgI.getCoerceToType());
2834
2835
812k
        if (isPromoted)
2836
14
          V = emitArgumentDemotion(*this, Arg, V);
2837
2838
        // Because of merging of function types from multiple decls it is
2839
        // possible for the type of an argument to not match the corresponding
2840
        // type in the function type. Since we are codegening the callee
2841
        // in here, add a cast to the argument type.
2842
812k
        llvm::Type *LTy = ConvertType(Arg->getType());
2843
812k
        if (V->getType() != LTy)
2844
0
          V = Builder.CreateBitCast(V, LTy);
2845
2846
812k
        ArgVals.push_back(ParamValue::forDirect(V));
2847
812k
        break;
2848
812k
      }
2849
2850
      // VLST arguments are coerced to VLATs at the function boundary for
2851
      // ABI consistency. If this is a VLST that was coerced to
2852
      // a VLAT at the function boundary and the types match up, use
2853
      // llvm.experimental.vector.extract to convert back to the original
2854
      // VLST.
2855
6.06k
      if (auto *VecTyTo = dyn_cast<llvm::FixedVectorType>(ConvertType(Ty))) {
2856
1.01k
        llvm::Value *Coerced = Fn->getArg(FirstIRArg);
2857
1.01k
        if (auto *VecTyFrom =
2858
1.01k
                dyn_cast<llvm::ScalableVectorType>(Coerced->getType())) {
2859
          // If we are casting a scalable 16 x i1 predicate vector to a fixed i8
2860
          // vector, bitcast the source and use a vector extract.
2861
36
          auto PredType =
2862
36
              llvm::ScalableVectorType::get(Builder.getInt1Ty(), 16);
2863
36
          if (VecTyFrom == PredType &&
2864
36
              
VecTyTo->getElementType() == Builder.getInt8Ty()4
) {
2865
4
            VecTyFrom = llvm::ScalableVectorType::get(Builder.getInt8Ty(), 2);
2866
4
            Coerced = Builder.CreateBitCast(Coerced, VecTyFrom);
2867
4
          }
2868
36
          if (VecTyFrom->getElementType() == VecTyTo->getElementType()) {
2869
36
            llvm::Value *Zero = llvm::Constant::getNullValue(CGM.Int64Ty);
2870
2871
36
            assert(NumIRArgs == 1);
2872
0
            Coerced->setName(Arg->getName() + ".coerce");
2873
36
            ArgVals.push_back(ParamValue::forDirect(Builder.CreateExtractVector(
2874
36
                VecTyTo, Coerced, Zero, "castFixedSve")));
2875
36
            break;
2876
36
          }
2877
36
        }
2878
1.01k
      }
2879
2880
6.02k
      Address Alloca = CreateMemTemp(Ty, getContext().getDeclAlign(Arg),
2881
6.02k
                                     Arg->getName());
2882
2883
      // Pointer to store into.
2884
6.02k
      Address Ptr = emitAddressAtOffset(*this, Alloca, ArgI);
2885
2886
      // Fast-isel and the optimizer generally like scalar values better than
2887
      // FCAs, so we flatten them if this is safe to do for this argument.
2888
6.02k
      llvm::StructType *STy = dyn_cast<llvm::StructType>(ArgI.getCoerceToType());
2889
6.02k
      if (ArgI.isDirect() && ArgI.getCanBeFlattened() && 
STy5.70k
&&
2890
6.02k
          
STy->getNumElements() > 1999
) {
2891
972
        uint64_t SrcSize = CGM.getDataLayout().getTypeAllocSize(STy);
2892
972
        llvm::Type *DstTy = Ptr.getElementType();
2893
972
        uint64_t DstSize = CGM.getDataLayout().getTypeAllocSize(DstTy);
2894
2895
972
        Address AddrToStoreInto = Address::invalid();
2896
972
        if (SrcSize <= DstSize) {
2897
883
          AddrToStoreInto = Builder.CreateElementBitCast(Ptr, STy);
2898
883
        } else {
2899
89
          AddrToStoreInto =
2900
89
            CreateTempAlloca(STy, Alloca.getAlignment(), "coerce");
2901
89
        }
2902
2903
972
        assert(STy->getNumElements() == NumIRArgs);
2904
3.02k
        for (unsigned i = 0, e = STy->getNumElements(); i != e; 
++i2.05k
) {
2905
2.05k
          auto AI = Fn->getArg(FirstIRArg + i);
2906
2.05k
          AI->setName(Arg->getName() + ".coerce" + Twine(i));
2907
2.05k
          Address EltPtr = Builder.CreateStructGEP(AddrToStoreInto, i);
2908
2.05k
          Builder.CreateStore(AI, EltPtr);
2909
2.05k
        }
2910
2911
972
        if (SrcSize > DstSize) {
2912
89
          Builder.CreateMemCpy(Ptr, AddrToStoreInto, DstSize);
2913
89
        }
2914
2915
5.05k
      } else {
2916
        // Simple case, just do a coerced store of the argument into the alloca.
2917
5.05k
        assert(NumIRArgs == 1);
2918
0
        auto AI = Fn->getArg(FirstIRArg);
2919
5.05k
        AI->setName(Arg->getName() + ".coerce");
2920
5.05k
        CreateCoercedStore(AI, Ptr, /*DstIsVolatile=*/false, *this);
2921
5.05k
      }
2922
2923
      // Match to what EmitParmDecl is expecting for this type.
2924
6.02k
      if (CodeGenFunction::hasScalarEvaluationKind(Ty)) {
2925
1.29k
        llvm::Value *V =
2926
1.29k
            EmitLoadOfScalar(Alloca, false, Ty, Arg->getBeginLoc());
2927
1.29k
        if (isPromoted)
2928
0
          V = emitArgumentDemotion(*this, Arg, V);
2929
1.29k
        ArgVals.push_back(ParamValue::forDirect(V));
2930
4.73k
      } else {
2931
4.73k
        ArgVals.push_back(ParamValue::forIndirect(Alloca));
2932
4.73k
      }
2933
6.02k
      break;
2934
6.06k
    }
2935
2936
571
    case ABIArgInfo::CoerceAndExpand: {
2937
      // Reconstruct into a temporary.
2938
571
      Address alloca = CreateMemTemp(Ty, getContext().getDeclAlign(Arg));
2939
571
      ArgVals.push_back(ParamValue::forIndirect(alloca));
2940
2941
571
      auto coercionType = ArgI.getCoerceAndExpandType();
2942
571
      alloca = Builder.CreateElementBitCast(alloca, coercionType);
2943
2944
571
      unsigned argIndex = FirstIRArg;
2945
1.78k
      for (unsigned i = 0, e = coercionType->getNumElements(); i != e; 
++i1.21k
) {
2946
1.21k
        llvm::Type *eltType = coercionType->getElementType(i);
2947
1.21k
        if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType))
2948
13
          continue;
2949
2950
1.19k
        auto eltAddr = Builder.CreateStructGEP(alloca, i);
2951
1.19k
        auto elt = Fn->getArg(argIndex++);
2952
1.19k
        Builder.CreateStore(elt, eltAddr);
2953
1.19k
      }
2954
571
      assert(argIndex == FirstIRArg + NumIRArgs);
2955
0
      break;
2956
6.06k
    }
2957
2958
101
    case ABIArgInfo::Expand: {
2959
      // If this structure was expanded into multiple arguments then
2960
      // we need to create a temporary and reconstruct it from the
2961
      // arguments.
2962
101
      Address Alloca = CreateMemTemp(Ty, getContext().getDeclAlign(Arg));
2963
101
      LValue LV = MakeAddrLValue(Alloca, Ty);
2964
101
      ArgVals.push_back(ParamValue::forIndirect(Alloca));
2965
2966
101
      auto FnArgIter = Fn->arg_begin() + FirstIRArg;
2967
101
      ExpandTypeFromArgs(Ty, LV, FnArgIter);
2968
101
      assert(FnArgIter == Fn->arg_begin() + FirstIRArg + NumIRArgs);
2969
343
      for (unsigned i = 0, e = NumIRArgs; i != e; 
++i242
) {
2970
242
        auto AI = Fn->getArg(FirstIRArg + i);
2971
242
        AI->setName(Arg->getName() + "." + Twine(i));
2972
242
      }
2973
101
      break;
2974
6.06k
    }
2975
2976
7.09k
    case ABIArgInfo::Ignore:
2977
7.09k
      assert(NumIRArgs == 0);
2978
      // Initialize the local variable appropriately.
2979
7.09k
      if (!hasScalarEvaluationKind(Ty)) {
2980
7.09k
        ArgVals.push_back(ParamValue::forIndirect(CreateMemTemp(Ty)));
2981
7.09k
      } else {
2982
0
        llvm::Value *U = llvm::UndefValue::get(ConvertType(Arg->getType()));
2983
0
        ArgVals.push_back(ParamValue::forDirect(U));
2984
0
      }
2985
7.09k
      break;
2986
829k
    }
2987
829k
  }
2988
2989
377k
  if (getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) {
2990
31.3k
    for (int I = Args.size() - 1; I >= 0; 
--I18.0k
)
2991
18.0k
      EmitParmDecl(*Args[I], ArgVals[I], I + 1);
2992
363k
  } else {
2993
1.17M
    for (unsigned I = 0, E = Args.size(); I != E; 
++I811k
)
2994
811k
      EmitParmDecl(*Args[I], ArgVals[I], I + 1);
2995
363k
  }
2996
377k
}
2997
2998
8
static void eraseUnusedBitCasts(llvm::Instruction *insn) {
2999
12
  while (insn->use_empty()) {
3000
12
    llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(insn);
3001
12
    if (!bitcast) 
return8
;
3002
3003
    // This is "safe" because we would have used a ConstantExpr otherwise.
3004
4
    insn = cast<llvm::Instruction>(bitcast->getOperand(0));
3005
4
    bitcast->eraseFromParent();
3006
4
  }
3007
8
}
3008
3009
/// Try to emit a fused autorelease of a return result.
3010
static llvm::Value *tryEmitFusedAutoreleaseOfResult(CodeGenFunction &CGF,
3011
87
                                                    llvm::Value *result) {
3012
  // We must be immediately followed the cast.
3013
87
  llvm::BasicBlock *BB = CGF.Builder.GetInsertBlock();
3014
87
  if (BB->empty()) 
return nullptr0
;
3015
87
  if (&BB->back() != result) 
return nullptr24
;
3016
3017
63
  llvm::Type *resultType = result->getType();
3018
3019
  // result is in a BasicBlock and is therefore an Instruction.
3020
63
  llvm::Instruction *generator = cast<llvm::Instruction>(result);
3021
3022
63
  SmallVector<llvm::Instruction *, 4> InstsToKill;
3023
3024
  // Look for:
3025
  //  %generator = bitcast %type1* %generator2 to %type2*
3026
95
  while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(generator)) {
3027
    // We would have emitted this as a constant if the operand weren't
3028
    // an Instruction.
3029
32
    generator = cast<llvm::Instruction>(bitcast->getOperand(0));
3030
3031
    // Require the generator to be immediately followed by the cast.
3032
32
    if (generator->getNextNode() != bitcast)
3033
0
      return nullptr;
3034
3035
32
    InstsToKill.push_back(bitcast);
3036
32
  }
3037
3038
  // Look for:
3039
  //   %generator = call i8* @objc_retain(i8* %originalResult)
3040
  // or
3041
  //   %generator = call i8* @objc_retainAutoreleasedReturnValue(i8* %originalResult)
3042
63
  llvm::CallInst *call = dyn_cast<llvm::CallInst>(generator);
3043
63
  if (!call) 
return nullptr13
;
3044
3045
50
  bool doRetainAutorelease;
3046
3047
50
  if (call->getCalledOperand() == CGF.CGM.getObjCEntrypoints().objc_retain) {
3048
8
    doRetainAutorelease = true;
3049
42
  } else if (call->getCalledOperand() ==
3050
42
             CGF.CGM.getObjCEntrypoints().objc_retainAutoreleasedReturnValue) {
3051
19
    doRetainAutorelease = false;
3052
3053
    // If we emitted an assembly marker for this call (and the
3054
    // ARCEntrypoints field should have been set if so), go looking
3055
    // for that call.  If we can't find it, we can't do this
3056
    // optimization.  But it should always be the immediately previous
3057
    // instruction, unless we needed bitcasts around the call.
3058
19
    if (CGF.CGM.getObjCEntrypoints().retainAutoreleasedReturnValueMarker) {
3059
12
      llvm::Instruction *prev = call->getPrevNode();
3060
12
      assert(prev);
3061
12
      if (isa<llvm::BitCastInst>(prev)) {
3062
8
        prev = prev->getPrevNode();
3063
8
        assert(prev);
3064
8
      }
3065
0
      assert(isa<llvm::CallInst>(prev));
3066
0
      assert(cast<llvm::CallInst>(prev)->getCalledOperand() ==
3067
12
             CGF.CGM.getObjCEntrypoints().retainAutoreleasedReturnValueMarker);
3068
0
      InstsToKill.push_back(prev);
3069
12
    }
3070
23
  } else {
3071
23
    return nullptr;
3072
23
  }
3073
3074
27
  result = call->getArgOperand(0);
3075
27
  InstsToKill.push_back(call);
3076
3077
  // Keep killing bitcasts, for sanity.  Note that we no longer care
3078
  // about precise ordering as long as there's exactly one use.
3079
43
  while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(result)) {
3080
16
    if (!bitcast->hasOneUse()) 
break0
;
3081
16
    InstsToKill.push_back(bitcast);
3082
16
    result = bitcast->getOperand(0);
3083
16
  }
3084
3085
  // Delete all the unnecessary instructions, from latest to earliest.
3086
27
  for (auto *I : InstsToKill)
3087
77
    I->eraseFromParent();
3088
3089
  // Do the fused retain/autorelease if we were asked to.
3090
27
  if (doRetainAutorelease)
3091
8
    result = CGF.EmitARCRetainAutoreleaseReturnValue(result);
3092
3093
  // Cast back to the result type.
3094
27
  return CGF.Builder.CreateBitCast(result, resultType);
3095
50
}
3096
3097
/// If this is a +1 of the value of an immutable 'self', remove it.
3098
static llvm::Value *tryRemoveRetainOfSelf(CodeGenFunction &CGF,
3099
122
                                          llvm::Value *result) {
3100
  // This is only applicable to a method with an immutable 'self'.
3101
122
  const ObjCMethodDecl *method =
3102
122
    dyn_cast_or_null<ObjCMethodDecl>(CGF.CurCodeDecl);
3103
122
  if (!method) 
return nullptr103
;
3104
19
  const VarDecl *self = method->getSelfDecl();
3105
19
  if (!self->getType().isConstQualified()) 
return nullptr0
;
3106
3107
  // Look for a retain call.
3108
19
  llvm::CallInst *retainCall =
3109
19
    dyn_cast<llvm::CallInst>(result->stripPointerCasts());
3110
19
  if (!retainCall || retainCall->getCalledOperand() !=
3111
15
                         CGF.CGM.getObjCEntrypoints().objc_retain)
3112
12
    return nullptr;
3113
3114
  // Look for an ordinary load of 'self'.
3115
7
  llvm::Value *retainedValue = retainCall->getArgOperand(0);
3116
7
  llvm::LoadInst *load =
3117
7
    dyn_cast<llvm::LoadInst>(retainedValue->stripPointerCasts());
3118
7
  if (!load || load->isAtomic() || load->isVolatile() ||
3119
7
      load->getPointerOperand() != CGF.GetAddrOfLocalVar(self).getPointer())
3120
3
    return nullptr;
3121
3122
  // Okay!  Burn it all down.  This relies for correctness on the
3123
  // assumption that the retain is emitted as part of the return and
3124
  // that thereafter everything is used "linearly".
3125
4
  llvm::Type *resultType = result->getType();
3126
4
  eraseUnusedBitCasts(cast<llvm::Instruction>(result));
3127
4
  assert(retainCall->use_empty());
3128
0
  retainCall->eraseFromParent();
3129
4
  eraseUnusedBitCasts(cast<llvm::Instruction>(retainedValue));
3130
3131
4
  return CGF.Builder.CreateBitCast(load, resultType);
3132
7
}
3133
3134
/// Emit an ARC autorelease of the result of a function.
3135
///
3136
/// \return the value to actually return from the function
3137
static llvm::Value *emitAutoreleaseOfResult(CodeGenFunction &CGF,
3138
122
                                            llvm::Value *result) {
3139
  // If we're returning 'self', kill the initial retain.  This is a
3140
  // heuristic attempt to "encourage correctness" in the really unfortunate
3141
  // case where we have a return of self during a dealloc and we desperately
3142
  // need to avoid the possible autorelease.
3143
122
  if (llvm::Value *self = tryRemoveRetainOfSelf(CGF, result))
3144
4
    return self;
3145
3146
  // At -O0, try to emit a fused retain/autorelease.
3147
118
  if (CGF.shouldUseFusedARCCalls())
3148
87
    if (llvm::Value *fused = tryEmitFusedAutoreleaseOfResult(CGF, result))
3149
27
      return fused;
3150
3151
91
  return CGF.EmitARCAutoreleaseReturnValue(result);
3152
118
}
3153
3154
/// Heuristically search for a dominating store to the return-value slot.
3155
185k
static llvm::StoreInst *findDominatingStoreToReturnValue(CodeGenFunction &CGF) {
3156
  // Check if a User is a store which pointerOperand is the ReturnValue.
3157
  // We are looking for stores to the ReturnValue, not for stores of the
3158
  // ReturnValue to some other location.
3159
185k
  auto GetStoreIfValid = [&CGF](llvm::User *U) -> llvm::StoreInst * {
3160
183k
    auto *SI = dyn_cast<llvm::StoreInst>(U);
3161
183k
    if (!SI || 
SI->getPointerOperand() != CGF.ReturnValue.getPointer()181k
)
3162
2.29k
      return nullptr;
3163
    // These aren't actually possible for non-coerced returns, and we
3164
    // only care about non-coerced returns on this code path.
3165
181k
    assert(!SI->isAtomic() && !SI->isVolatile());
3166
0
    return SI;
3167
183k
  };
3168
  // If there are multiple uses of the return-value slot, just check
3169
  // for something immediately preceding the IP.  Sometimes this can
3170
  // happen with how we generate implicit-returns; it can also happen
3171
  // with noreturn cleanups.
3172
185k
  if (!CGF.ReturnValue.getPointer()->hasOneUse()) {
3173
7.41k
    llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock();
3174
7.41k
    if (IP->empty()) 
return nullptr1.96k
;
3175
5.45k
    llvm::Instruction *I = &IP->back();
3176
3177
    // Skip lifetime markers
3178
5.45k
    for (llvm::BasicBlock::reverse_iterator II = IP->rbegin(),
3179
5.45k
                                            IE = IP->rend();
3180
5.55k
         II != IE; 
++II108
) {
3181
5.51k
      if (llvm::IntrinsicInst *Intrinsic =
3182
5.51k
              dyn_cast<llvm::IntrinsicInst>(&*II)) {
3183
363
        if (Intrinsic->getIntrinsicID() == llvm::Intrinsic::lifetime_end) {
3184
108
          const llvm::Value *CastAddr = Intrinsic->getArgOperand(1);
3185
108
          ++II;
3186
108
          if (II == IE)
3187
0
            break;
3188
108
          if (isa<llvm::BitCastInst>(&*II) && (CastAddr == &*II))
3189
108
            continue;
3190
108
        }
3191
363
      }
3192
5.40k
      I = &*II;
3193
5.40k
      break;
3194
5.51k
    }
3195
3196
5.45k
    return GetStoreIfValid(I);
3197
7.41k
  }
3198
3199
177k
  llvm::StoreInst *store =
3200
177k
      GetStoreIfValid(CGF.ReturnValue.getPointer()->user_back());
3201
177k
  if (!store) 
return nullptr547
;
3202
3203
  // Now do a first-and-dirty dominance check: just walk up the
3204
  // single-predecessors chain from the current insertion point.
3205
177k
  llvm::BasicBlock *StoreBB = store->getParent();
3206
177k
  llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock();
3207
178k
  while (IP != StoreBB) {
3208
2.01k
    if (!(IP = IP->getSinglePredecessor()))
3209
1.27k
      return nullptr;
3210
2.01k
  }
3211
3212
  // Okay, the store's basic block dominates the insertion point; we
3213
  // can do our thing.
3214
176k
  return store;
3215
177k
}
3216
3217
// Helper functions for EmitCMSEClearRecord
3218
3219
// Set the bits corresponding to a field having width `BitWidth` and located at
3220
// offset `BitOffset` (from the least significant bit) within a storage unit of
3221
// `Bits.size()` bytes. Each element of `Bits` corresponds to one target byte.
3222
// Use little-endian layout, i.e.`Bits[0]` is the LSB.
3223
static void setBitRange(SmallVectorImpl<uint64_t> &Bits, int BitOffset,
3224
125
                        int BitWidth, int CharWidth) {
3225
125
  assert(CharWidth <= 64);
3226
0
  assert(static_cast<unsigned>(BitWidth) <= Bits.size() * CharWidth);
3227
3228
0
  int Pos = 0;
3229
125
  if (BitOffset >= CharWidth) {
3230
39
    Pos += BitOffset / CharWidth;
3231
39
    BitOffset = BitOffset % CharWidth;
3232
39
  }
3233
3234
125
  const uint64_t Used = (uint64_t(1) << CharWidth) - 1;
3235
125
  if (BitOffset + BitWidth >= CharWidth) {
3236
65
    Bits[Pos++] |= (Used << BitOffset) & Used;
3237
65
    BitWidth -= CharWidth - BitOffset;
3238
65
    BitOffset = 0;
3239
65
  }
3240
3241
143
  while (BitWidth >= CharWidth) {
3242
18
    Bits[Pos++] = Used;
3243
18
    BitWidth -= CharWidth;
3244
18
  }
3245
3246
125
  if (BitWidth > 0)
3247
92
    Bits[Pos++] |= (Used >> (CharWidth - BitWidth)) << BitOffset;
3248
125
}
3249
3250
// Set the bits corresponding to a field having width `BitWidth` and located at
3251
// offset `BitOffset` (from the least significant bit) within a storage unit of
3252
// `StorageSize` bytes, located at `StorageOffset` in `Bits`. Each element of
3253
// `Bits` corresponds to one target byte. Use target endian layout.
3254
static void setBitRange(SmallVectorImpl<uint64_t> &Bits, int StorageOffset,
3255
                        int StorageSize, int BitOffset, int BitWidth,
3256
125
                        int CharWidth, bool BigEndian) {
3257
3258
125
  SmallVector<uint64_t, 8> TmpBits(StorageSize);
3259
125
  setBitRange(TmpBits, BitOffset, BitWidth, CharWidth);
3260
3261
125
  if (BigEndian)
3262
50
    std::reverse(TmpBits.begin(), TmpBits.end());
3263
3264
125
  for (uint64_t V : TmpBits)
3265
350
    Bits[StorageOffset++] |= V;
3266
125
}
3267
3268
static void setUsedBits(CodeGenModule &, QualType, int,
3269
                        SmallVectorImpl<uint64_t> &);
3270
3271
// Set the bits in `Bits`, which correspond to the value representations of
3272
// the actual members of the record type `RTy`. Note that this function does
3273
// not handle base classes, virtual tables, etc, since they cannot happen in
3274
// CMSE function arguments or return. The bit mask corresponds to the target
3275
// memory layout, i.e. it's endian dependent.
3276
static void setUsedBits(CodeGenModule &CGM, const RecordType *RTy, int Offset,
3277
153
                        SmallVectorImpl<uint64_t> &Bits) {
3278
153
  ASTContext &Context = CGM.getContext();
3279
153
  int CharWidth = Context.getCharWidth();
3280
153
  const RecordDecl *RD = RTy->getDecl()->getDefinition();
3281
153
  const ASTRecordLayout &ASTLayout = Context.getASTRecordLayout(RD);
3282
153
  const CGRecordLayout &Layout = CGM.getTypes().getCGRecordLayout(RD);
3283
3284
153
  int Idx = 0;
3285
566
  for (auto I = RD->field_begin(), E = RD->field_end(); I != E; 
++I, ++Idx413
) {
3286
413
    const FieldDecl *F = *I;
3287
3288
413
    if (F->isUnnamedBitfield() || 
F->isZeroLengthBitField(Context)283
||
3289
413
        
F->getType()->isIncompleteArrayType()283
)
3290
140
      continue;
3291
3292
273
    if (F->isBitField()) {
3293
125
      const CGBitFieldInfo &BFI = Layout.getBitFieldInfo(F);
3294
125
      setBitRange(Bits, Offset + BFI.StorageOffset.getQuantity(),
3295
125
                  BFI.StorageSize / CharWidth, BFI.Offset,
3296
125
                  BFI.Size, CharWidth,
3297
125
                  CGM.getDataLayout().isBigEndian());
3298
125
      continue;
3299
125
    }
3300
3301
148
    setUsedBits(CGM, F->getType(),
3302
148
                Offset + ASTLayout.getFieldOffset(Idx) / CharWidth, Bits);
3303
148
  }
3304
153
}
3305
3306
// Set the bits in `Bits`, which correspond to the value representations of
3307
// the elements of an array type `ATy`.
3308
static void setUsedBits(CodeGenModule &CGM, const ConstantArrayType *ATy,
3309
34
                        int Offset, SmallVectorImpl<uint64_t> &Bits) {
3310
34
  const ASTContext &Context = CGM.getContext();
3311
3312
34
  QualType ETy = Context.getBaseElementType(ATy);
3313
34
  int Size = Context.getTypeSizeInChars(ETy).getQuantity();
3314
34
  SmallVector<uint64_t, 4> TmpBits(Size);
3315
34
  setUsedBits(CGM, ETy, 0, TmpBits);
3316
3317
142
  for (int I = 0, N = Context.getConstantArrayElementCount(ATy); I < N; 
++I108
) {
3318
108
    auto Src = TmpBits.begin();
3319
108
    auto Dst = Bits.begin() + Offset + I * Size;
3320
270
    for (int J = 0; J < Size; 
++J162
)
3321
162
      *Dst++ |= *Src++;
3322
108
  }
3323
34
}
3324
3325
// Set the bits in `Bits`, which correspond to the value representations of
3326
// the type `QTy`.
3327
static void setUsedBits(CodeGenModule &CGM, QualType QTy, int Offset,
3328
182
                        SmallVectorImpl<uint64_t> &Bits) {
3329
182
  if (const auto *RTy = QTy->getAs<RecordType>())
3330
30
    return setUsedBits(CGM, RTy, Offset, Bits);
3331
3332
152
  ASTContext &Context = CGM.getContext();
3333
152
  if (const auto *ATy = Context.getAsConstantArrayType(QTy))
3334
34
    return setUsedBits(CGM, ATy, Offset, Bits);
3335
3336
118
  int Size = Context.getTypeSizeInChars(QTy).getQuantity();
3337
118
  if (Size <= 0)
3338
0
    return;
3339
3340
118
  std::fill_n(Bits.begin() + Offset, Size,
3341
118
              (uint64_t(1) << Context.getCharWidth()) - 1);
3342
118
}
3343
3344
static uint64_t buildMultiCharMask(const SmallVectorImpl<uint64_t> &Bits,
3345
                                   int Pos, int Size, int CharWidth,
3346
152
                                   bool BigEndian) {
3347
152
  assert(Size > 0);
3348
0
  uint64_t Mask = 0;
3349
152
  if (BigEndian) {
3350
318
    for (auto P = Bits.begin() + Pos, E = Bits.begin() + Pos + Size; P != E;
3351
256
         ++P)
3352
256
      Mask = (Mask << CharWidth) | *P;
3353
90
  } else {
3354
90
    auto P = Bits.begin() + Pos + Size, End = Bits.begin() + Pos;
3355
90
    do
3356
372
      Mask = (Mask << CharWidth) | *--P;
3357
372
    while (P != End);
3358
90
  }
3359
152
  return Mask;
3360
152
}
3361
3362
// Emit code to clear the bits in a record, which aren't a part of any user
3363
// declared member, when the record is a function return.
3364
llvm::Value *CodeGenFunction::EmitCMSEClearRecord(llvm::Value *Src,
3365
                                                  llvm::IntegerType *ITy,
3366
84
                                                  QualType QTy) {
3367
84
  assert(Src->getType() == ITy);
3368
0
  assert(ITy->getScalarSizeInBits() <= 64);
3369
3370
0
  const llvm::DataLayout &DataLayout = CGM.getDataLayout();
3371
84
  int Size = DataLayout.getTypeStoreSize(ITy);
3372
84
  SmallVector<uint64_t, 4> Bits(Size);
3373
84
  setUsedBits(CGM, QTy->castAs<RecordType>(), 0, Bits);
3374
3375
84
  int CharWidth = CGM.getContext().getCharWidth();
3376
84
  uint64_t Mask =
3377
84
      buildMultiCharMask(Bits, 0, Size, CharWidth, DataLayout.isBigEndian());
3378
3379
84
  return Builder.CreateAnd(Src, Mask, "cmse.clear");
3380
84
}
3381
3382
// Emit code to clear the bits in a record, which aren't a part of any user
3383
// declared member, when the record is a function argument.
3384
llvm::Value *CodeGenFunction::EmitCMSEClearRecord(llvm::Value *Src,
3385
                                                  llvm::ArrayType *ATy,
3386
39
                                                  QualType QTy) {
3387
39
  const llvm::DataLayout &DataLayout = CGM.getDataLayout();
3388
39
  int Size = DataLayout.getTypeStoreSize(ATy);
3389
39
  SmallVector<uint64_t, 16> Bits(Size);
3390
39
  setUsedBits(CGM, QTy->castAs<RecordType>(), 0, Bits);
3391
3392
  // Clear each element of the LLVM array.
3393
39
  int CharWidth = CGM.getContext().getCharWidth();
3394
39
  int CharsPerElt =
3395
39
      ATy->getArrayElementType()->getScalarSizeInBits() / CharWidth;
3396
39
  int MaskIndex = 0;
3397
39
  llvm::Value *R = llvm::UndefValue::get(ATy);
3398
107
  for (int I = 0, N = ATy->getArrayNumElements(); I != N; 
++I68
) {
3399
68
    uint64_t Mask = buildMultiCharMask(Bits, MaskIndex, CharsPerElt, CharWidth,
3400
68
                                       DataLayout.isBigEndian());
3401
68
    MaskIndex += CharsPerElt;
3402
68
    llvm::Value *T0 = Builder.CreateExtractValue(Src, I);
3403
68
    llvm::Value *T1 = Builder.CreateAnd(T0, Mask, "cmse.clear");
3404
68
    R = Builder.CreateInsertValue(R, T1, I);
3405
68
  }
3406
3407
39
  return R;
3408
39
}
3409
3410
void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI,
3411
                                         bool EmitRetDbgLoc,
3412
377k
                                         SourceLocation EndLoc) {
3413
377k
  if (FI.isNoReturn()) {
3414
    // Noreturn functions don't return.
3415
27
    EmitUnreachable(EndLoc);
3416
27
    return;
3417
27
  }
3418
3419
377k
  if (CurCodeDecl && 
CurCodeDecl->hasAttr<NakedAttr>()368k
) {
3420
    // Naked functions don't have epilogues.
3421
9
    Builder.CreateUnreachable();
3422
9
    return;
3423
9
  }
3424
3425
  // Functions with no result always return void.
3426
377k
  if (!ReturnValue.isValid()) {
3427
183k
    Builder.CreateRetVoid();
3428
183k
    return;
3429
183k
  }
3430
3431
193k
  llvm::DebugLoc RetDbgLoc;
3432
193k
  llvm::Value *RV = nullptr;
3433
193k
  QualType RetTy = FI.getReturnType();
3434
193k
  const ABIArgInfo &RetAI = FI.getReturnInfo();
3435
3436
193k
  switch (RetAI.getKind()) {
3437
5
  case ABIArgInfo::InAlloca:
3438
    // Aggregrates get evaluated directly into the destination.  Sometimes we
3439
    // need to return the sret value in a register, though.
3440
5
    assert(hasAggregateEvaluationKind(RetTy));
3441
5
    if (RetAI.getInAllocaSRet()) {
3442
5
      llvm::Function::arg_iterator EI = CurFn->arg_end();
3443
5
      --EI;
3444
5
      llvm::Value *ArgStruct = &*EI;
3445
5
      llvm::Value *SRet = Builder.CreateStructGEP(
3446
5
          EI->getType()->getPointerElementType(), ArgStruct,
3447
5
          RetAI.getInAllocaFieldIndex());
3448
5
      llvm::Type *Ty =
3449
5
          cast<llvm::GetElementPtrInst>(SRet)->getResultElementType();
3450
5
      RV = Builder.CreateAlignedLoad(Ty, SRet, getPointerAlign(), "sret");
3451
5
    }
3452
5
    break;
3453
3454
2.91k
  case ABIArgInfo::Indirect: {
3455
2.91k
    auto AI = CurFn->arg_begin();
3456
2.91k
    if (RetAI.isSRetAfterThis())
3457
50
      ++AI;
3458
2.91k
    switch (getEvaluationKind(RetTy)) {
3459
287
    case TEK_Complex: {
3460
287
      ComplexPairTy RT =
3461
287
        EmitLoadOfComplex(MakeAddrLValue(ReturnValue, RetTy), EndLoc);
3462
287
      EmitStoreOfComplex(RT, MakeNaturalAlignAddrLValue(&*AI, RetTy),
3463
287
                         /*isInit*/ true);
3464
287
      break;
3465
0
    }
3466
2.34k
    case TEK_Aggregate:
3467
      // Do nothing; aggregrates get evaluated directly into the destination.
3468
2.34k
      break;
3469
288
    case TEK_Scalar:
3470
288
      EmitStoreOfScalar(Builder.CreateLoad(ReturnValue),
3471
288
                        MakeNaturalAlignAddrLValue(&*AI, RetTy),
3472
288
                        /*isInit*/ true);
3473
288
      break;
3474
2.91k
    }
3475
2.91k
    break;
3476
2.91k
  }
3477
3478
13.6k
  case ABIArgInfo::Extend:
3479
190k
  case ABIArgInfo::Direct:
3480
190k
    if (RetAI.getCoerceToType() == ConvertType(RetTy) &&
3481
190k
        
RetAI.getDirectOffset() == 0185k
) {
3482
      // The internal return value temp always will have pointer-to-return-type
3483
      // type, just do a load.
3484
3485
      // If there is a dominating store to ReturnValue, we can elide
3486
      // the load, zap the store, and usually zap the alloca.
3487
185k
      if (llvm::StoreInst *SI =
3488
185k
              findDominatingStoreToReturnValue(*this)) {
3489
        // Reuse the debug location from the store unless there is
3490
        // cleanup code to be emitted between the store and return
3491
        // instruction.
3492
179k
        if (EmitRetDbgLoc && 
!AutoreleaseResult178k
)
3493
178k
          RetDbgLoc = SI->getDebugLoc();
3494
        // Get the stored value and nuke the now-dead store.
3495
179k
        RV = SI->getValueOperand();
3496
179k
        SI->eraseFromParent();
3497
3498
      // Otherwise, we have to do a simple load.
3499
179k
      } else {
3500
5.53k
        RV = Builder.CreateLoad(ReturnValue);
3501
5.53k
      }
3502
185k
    } else {
3503
      // If the value is offset in memory, apply the offset now.
3504
4.79k
      Address V = emitAddressAtOffset(*this, ReturnValue, RetAI);
3505
3506
4.79k
      RV = CreateCoercedLoad(V, RetAI.getCoerceToType(), *this);
3507
4.79k
    }
3508
3509
    // In ARC, end functions that return a retainable type with a call
3510
    // to objc_autoreleaseReturnValue.
3511
190k
    if (AutoreleaseResult) {
3512
122
#ifndef NDEBUG
3513
      // Type::isObjCRetainabletype has to be called on a QualType that hasn't
3514
      // been stripped of the typedefs, so we cannot use RetTy here. Get the
3515
      // original return type of FunctionDecl, CurCodeDecl, and BlockDecl from
3516
      // CurCodeDecl or BlockInfo.
3517
122
      QualType RT;
3518
3519
122
      if (auto *FD = dyn_cast<FunctionDecl>(CurCodeDecl))
3520
74
        RT = FD->getReturnType();
3521
48
      else if (auto *MD = dyn_cast<ObjCMethodDecl>(CurCodeDecl))
3522
19
        RT = MD->getReturnType();
3523
29
      else if (isa<BlockDecl>(CurCodeDecl))
3524
29
        RT = BlockInfo->BlockExpression->getFunctionType()->getReturnType();
3525
0
      else
3526
0
        llvm_unreachable("Unexpected function/method type");
3527
3528
122
      assert(getLangOpts().ObjCAutoRefCount &&
3529
122
             !FI.isReturnsRetained() &&
3530
122
             RT->isObjCRetainableType());
3531
0
#endif
3532
0
      RV = emitAutoreleaseOfResult(*this, RV);
3533
122
    }
3534
3535
0
    break;
3536
3537
332
  case ABIArgInfo::Ignore:
3538
332
    break;
3539
3540
541
  case ABIArgInfo::CoerceAndExpand: {
3541
541
    auto coercionType = RetAI.getCoerceAndExpandType();
3542
3543
    // Load all of the coerced elements out into results.
3544
541
    llvm::SmallVector<llvm::Value*, 4> results;
3545
541
    Address addr = Builder.CreateElementBitCast(ReturnValue, coercionType);
3546
1.68k
    for (unsigned i = 0, e = coercionType->getNumElements(); i != e; 
++i1.14k
) {
3547
1.14k
      auto coercedEltType = coercionType->getElementType(i);
3548
1.14k
      if (ABIArgInfo::isPaddingForCoerceAndExpand(coercedEltType))
3549
7
        continue;
3550
3551
1.14k
      auto eltAddr = Builder.CreateStructGEP(addr, i);
3552
1.14k
      auto elt = Builder.CreateLoad(eltAddr);
3553
1.14k
      results.push_back(elt);
3554
1.14k
    }
3555
3556
    // If we have one result, it's the single direct result type.
3557
541
    if (results.size() == 1) {
3558
180
      RV = results[0];
3559
3560
    // Otherwise, we need to make a first-class aggregate.
3561
361
    } else {
3562
      // Construct a return type that lacks padding elements.
3563
361
      llvm::Type *returnType = RetAI.getUnpaddedCoerceAndExpandType();
3564
3565
361
      RV = llvm::UndefValue::get(returnType);
3566
1.32k
      for (unsigned i = 0, e = results.size(); i != e; 
++i960
) {
3567
960
        RV = Builder.CreateInsertValue(RV, results[i], i);
3568
960
      }
3569
361
    }
3570
541
    break;
3571
13.6k
  }
3572
0
  case ABIArgInfo::Expand:
3573
0
  case ABIArgInfo::IndirectAliased:
3574
0
    llvm_unreachable("Invalid ABI kind for return argument");
3575
193k
  }
3576
3577
193k
  llvm::Instruction *Ret;
3578
193k
  if (RV) {
3579
190k
    if (CurFuncDecl && 
CurFuncDecl->hasAttr<CmseNSEntryAttr>()189k
) {
3580
      // For certain return types, clear padding bits, as they may reveal
3581
      // sensitive information.
3582
      // Small struct/union types are passed as integers.
3583
90
      auto *ITy = dyn_cast<llvm::IntegerType>(RV->getType());
3584
90
      if (ITy != nullptr && 
isa<RecordType>(RetTy.getCanonicalType())84
)
3585
84
        RV = EmitCMSEClearRecord(RV, ITy, RetTy);
3586
90
    }
3587
190k
    EmitReturnValueCheck(RV);
3588
190k
    Ret = Builder.CreateRet(RV);
3589
190k
  } else {
3590
3.25k
    Ret = Builder.CreateRetVoid();
3591
3.25k
  }
3592
3593
193k
  if (RetDbgLoc)
3594
45.0k
    Ret->setDebugLoc(std::move(RetDbgLoc));
3595
193k
}
3596
3597
190k
void CodeGenFunction::EmitReturnValueCheck(llvm::Value *RV) {
3598
  // A current decl may not be available when emitting vtable thunks.
3599
190k
  if (!CurCodeDecl)
3600
1.22k
    return;
3601
3602
  // If the return block isn't reachable, neither is this check, so don't emit
3603
  // it.
3604
189k
  if (ReturnBlock.isValid() && 
ReturnBlock.getBlock()->use_empty()1.99k
)
3605
222
    return;
3606
3607
189k
  ReturnsNonNullAttr *RetNNAttr = nullptr;
3608
189k
  if (SanOpts.has(SanitizerKind::ReturnsNonnullAttribute))
3609
47
    RetNNAttr = CurCodeDecl->getAttr<ReturnsNonNullAttr>();
3610
3611
189k
  if (!RetNNAttr && 
!requiresReturnValueNullabilityCheck()189k
)
3612
189k
    return;
3613
3614
  // Prefer the returns_nonnull attribute if it's present.
3615
18
  SourceLocation AttrLoc;
3616
18
  SanitizerMask CheckKind;
3617
18
  SanitizerHandler Handler;
3618
18
  if (RetNNAttr) {
3619
5
    assert(!requiresReturnValueNullabilityCheck() &&
3620
5
           "Cannot check nullability and the nonnull attribute");
3621
0
    AttrLoc = RetNNAttr->getLocation();
3622
5
    CheckKind = SanitizerKind::ReturnsNonnullAttribute;
3623
5
    Handler = SanitizerHandler::NonnullReturn;
3624
13
  } else {
3625
13
    if (auto *DD = dyn_cast<DeclaratorDecl>(CurCodeDecl))
3626
7
      if (auto *TSI = DD->getTypeSourceInfo())
3627
7
        if (auto FTL = TSI->getTypeLoc().getAsAdjusted<FunctionTypeLoc>())
3628
7
          AttrLoc = FTL.getReturnLoc().findNullabilityLoc();
3629
13
    CheckKind = SanitizerKind::NullabilityReturn;
3630
13
    Handler = SanitizerHandler::NullabilityReturn;
3631
13
  }
3632
3633
0
  SanitizerScope SanScope(this);
3634
3635
  // Make sure the "return" source location is valid. If we're checking a
3636
  // nullability annotation, make sure the preconditions for the check are met.
3637
18
  llvm::BasicBlock *Check = createBasicBlock("nullcheck");
3638
18
  llvm::BasicBlock *NoCheck = createBasicBlock("no.nullcheck");
3639
18
  llvm::Value *SLocPtr = Builder.CreateLoad(ReturnLocation, "return.sloc.load");
3640
18
  llvm::Value *CanNullCheck = Builder.CreateIsNotNull(SLocPtr);
3641
18
  if (requiresReturnValueNullabilityCheck())
3642
13
    CanNullCheck =
3643
13
        Builder.CreateAnd(CanNullCheck, RetValNullabilityPrecondition);
3644
18
  Builder.CreateCondBr(CanNullCheck, Check, NoCheck);
3645
18
  EmitBlock(Check);
3646
3647
  // Now do the null check.
3648
18
  llvm::Value *Cond = Builder.CreateIsNotNull(RV);
3649
18
  llvm::Constant *StaticData[] = {EmitCheckSourceLocation(AttrLoc)};
3650
18
  llvm::Value *DynamicData[] = {SLocPtr};
3651
18
  EmitCheck(std::make_pair(Cond, CheckKind), Handler, StaticData, DynamicData);
3652
3653
18
  EmitBlock(NoCheck);
3654
3655
18
#ifndef NDEBUG
3656
  // The return location should not be used after the check has been emitted.
3657
18
  ReturnLocation = Address::invalid();
3658
18
#endif
3659
18
}
3660
3661
24.8k
static bool isInAllocaArgument(CGCXXABI &ABI, QualType type) {
3662
24.8k
  const CXXRecordDecl *RD = type->getAsCXXRecordDecl();
3663
24.8k
  return RD && 
ABI.getRecordArgABI(RD) == CGCXXABI::RAA_DirectInMemory2.81k
;
3664
24.8k
}
3665
3666
static AggValueSlot createPlaceholderSlot(CodeGenFunction &CGF,
3667
102
                                          QualType Ty) {
3668
  // FIXME: Generate IR in one pass, rather than going back and fixing up these
3669
  // placeholders.
3670
102
  llvm::Type *IRTy = CGF.ConvertTypeForMem(Ty);
3671
102
  llvm::Type *IRPtrTy = IRTy->getPointerTo();
3672
102
  llvm::Value *Placeholder = llvm::UndefValue::get(IRPtrTy->getPointerTo());
3673
3674
  // FIXME: When we generate this IR in one pass, we shouldn't need
3675
  // this win32-specific alignment hack.
3676
102
  CharUnits Align = CharUnits::fromQuantity(4);
3677
102
  Placeholder = CGF.Builder.CreateAlignedLoad(IRPtrTy, Placeholder, Align);
3678
3679
102
  return AggValueSlot::forAddr(Address(Placeholder, Align),
3680
102
                               Ty.getQualifiers(),
3681
102
                               AggValueSlot::IsNotDestructed,
3682
102
                               AggValueSlot::DoesNotNeedGCBarriers,
3683
102
                               AggValueSlot::IsNotAliased,
3684
102
                               AggValueSlot::DoesNotOverlap);
3685
102
}
3686
3687
void CodeGenFunction::EmitDelegateCallArg(CallArgList &args,
3688
                                          const VarDecl *param,
3689
18.3k
                                          SourceLocation loc) {
3690
  // StartFunction converted the ABI-lowered parameter(s) into a
3691
  // local alloca.  We need to turn that into an r-value suitable
3692
  // for EmitCall.
3693
18.3k
  Address local = GetAddrOfLocalVar(param);
3694
3695
18.3k
  QualType type = param->getType();
3696
3697
18.3k
  if (isInAllocaArgument(CGM.getCXXABI(), type)) {
3698
1
    CGM.ErrorUnsupported(param, "forwarded non-trivially copyable parameter");
3699
1
  }
3700
3701
  // GetAddrOfLocalVar returns a pointer-to-pointer for references,
3702
  // but the argument needs to be the original pointer.
3703
18.3k
  if (type->isReferenceType()) {
3704
9.50k
    args.add(RValue::get(Builder.CreateLoad(local)), type);
3705
3706
  // In ARC, move out of consumed arguments so that the release cleanup
3707
  // entered by StartFunction doesn't cause an over-release.  This isn't
3708
  // optimal -O0 code generation, but it should get cleaned up when
3709
  // optimization is enabled.  This also assumes that delegate calls are
3710
  // performed exactly once for a set of arguments, but that should be safe.
3711
9.50k
  } else 
if (8.85k
getLangOpts().ObjCAutoRefCount8.85k
&&
3712
8.85k
             
param->hasAttr<NSConsumedAttr>()9
&&
3713
8.85k
             
type->isObjCRetainableType()2
) {
3714
2
    llvm::Value *ptr = Builder.CreateLoad(local);
3715
2
    auto null =
3716
2
      llvm::ConstantPointerNull::get(cast<llvm::PointerType>(ptr->getType()));
3717
2
    Builder.CreateStore(null, local);
3718
2
    args.add(RValue::get(ptr), type);
3719
3720
  // For the most part, we just need to load the alloca, except that
3721
  // aggregate r-values are actually pointers to temporaries.
3722
8.85k
  } else {
3723
8.85k
    args.add(convertTempToRValue(local, type, loc), type);
3724
8.85k
  }
3725
3726
  // Deactivate the cleanup for the callee-destructed param that was pushed.
3727
18.3k
  if (type->isRecordType() && 
!CurFuncIsThunk2.51k
&&
3728
18.3k
      
type->castAs<RecordType>()->getDecl()->isParamDestroyedInCallee()2.49k
&&
3729
18.3k
      
param->needsDestruction(getContext())8
) {
3730
6
    EHScopeStack::stable_iterator cleanup =
3731
6
        CalleeDestructedParamCleanups.lookup(cast<ParmVarDecl>(param));
3732
6
    assert(cleanup.isValid() &&
3733
6
           "cleanup for callee-destructed param not recorded");
3734
    // This unreachable is a temporary marker which will be removed later.
3735
0
    llvm::Instruction *isActive = Builder.CreateUnreachable();
3736
6
    args.addArgCleanupDeactivation(cleanup, isActive);
3737
6
  }
3738
18.3k
}
3739
3740
54
static bool isProvablyNull(llvm::Value *addr) {
3741
54
  return isa<llvm::ConstantPointerNull>(addr);
3742
54
}
3743
3744
/// Emit the actual writing-back of a writeback.
3745
static void emitWriteback(CodeGenFunction &CGF,
3746
27
                          const CallArgList::Writeback &writeback) {
3747
27
  const LValue &srcLV = writeback.Source;
3748
27
  Address srcAddr = srcLV.getAddress(CGF);
3749
27
  assert(!isProvablyNull(srcAddr.getPointer()) &&
3750
27
         "shouldn't have writeback for provably null argument");
3751
3752
0
  llvm::BasicBlock *contBB = nullptr;
3753
3754
  // If the argument wasn't provably non-null, we need to null check
3755
  // before doing the store.
3756
27
  bool provablyNonNull = llvm::isKnownNonZero(srcAddr.getPointer(),
3757
27
                                              CGF.CGM.getDataLayout());
3758
27
  if (!provablyNonNull) {
3759
5
    llvm::BasicBlock *writebackBB = CGF.createBasicBlock("icr.writeback");
3760
5
    contBB = CGF.createBasicBlock("icr.done");
3761
3762
5
    llvm::Value *isNull =
3763
5
      CGF.Builder.CreateIsNull(srcAddr.getPointer(), "icr.isnull");
3764
5
    CGF.Builder.CreateCondBr(isNull, contBB, writebackBB);
3765
5
    CGF.EmitBlock(writebackBB);
3766
5
  }
3767
3768
  // Load the value to writeback.
3769
27
  llvm::Value *value = CGF.Builder.CreateLoad(writeback.Temporary);
3770
3771
  // Cast it back, in case we're writing an id to a Foo* or something.
3772
27
  value = CGF.Builder.CreateBitCast(value, srcAddr.getElementType(),
3773
27
                                    "icr.writeback-cast");
3774
3775
  // Perform the writeback.
3776
3777
  // If we have a "to use" value, it's something we need to emit a use
3778
  // of.  This has to be carefully threaded in: if it's done after the
3779
  // release it's potentially undefined behavior (and the optimizer
3780
  // will ignore it), and if it happens before the retain then the
3781
  // optimizer could move the release there.
3782
27
  if (writeback.ToUse) {
3783
8
    assert(srcLV.getObjCLifetime() == Qualifiers::OCL_Strong);
3784
3785
    // Retain the new value.  No need to block-copy here:  the block's
3786
    // being passed up the stack.
3787
0
    value = CGF.EmitARCRetainNonBlock(value);
3788
3789
    // Emit the intrinsic use here.
3790
8
    CGF.EmitARCIntrinsicUse(writeback.ToUse);
3791
3792
    // Load the old value (primitively).
3793
8
    llvm::Value *oldValue = CGF.EmitLoadOfScalar(srcLV, SourceLocation());
3794
3795
    // Put the new value in place (primitively).
3796
8
    CGF.EmitStoreOfScalar(value, srcLV, /*init*/ false);
3797
3798
    // Release the old value.
3799
8
    CGF.EmitARCRelease(oldValue, srcLV.isARCPreciseLifetime());
3800
3801
  // Otherwise, we can just do a normal lvalue store.
3802
19
  } else {
3803
19
    CGF.EmitStoreThroughLValue(RValue::get(value), srcLV);
3804
19
  }
3805
3806
  // Jump to the continuation block.
3807
27
  if (!provablyNonNull)
3808
5
    CGF.EmitBlock(contBB);
3809
27
}
3810
3811
static void emitWritebacks(CodeGenFunction &CGF,
3812
27
                           const CallArgList &args) {
3813
27
  for (const auto &I : args.writebacks())
3814
27
    emitWriteback(CGF, I);
3815
27
}
3816
3817
static void deactivateArgCleanupsBeforeCall(CodeGenFunction &CGF,
3818
147
                                            const CallArgList &CallArgs) {
3819
147
  ArrayRef<CallArgList::CallArgCleanup> Cleanups =
3820
147
    CallArgs.getCleanupsToDeactivate();
3821
  // Iterate in reverse to increase the likelihood of popping the cleanup.
3822
188
  for (const auto &I : llvm::reverse(Cleanups)) {
3823
188
    CGF.DeactivateCleanupBlock(I.Cleanup, I.IsActiveIP);
3824
188
    I.IsActiveIP->eraseFromParent();
3825
188
  }
3826
147
}
3827
3828
27
static const Expr *maybeGetUnaryAddrOfOperand(const Expr *E) {
3829
27
  if (const UnaryOperator *uop = dyn_cast<UnaryOperator>(E->IgnoreParens()))
3830
22
    if (uop->getOpcode() == UO_AddrOf)
3831
22
      return uop->getSubExpr();
3832
5
  return nullptr;
3833
27
}
3834
3835
/// Emit an argument that's being passed call-by-writeback.  That is,
3836
/// we are passing the address of an __autoreleased temporary; it
3837
/// might be copy-initialized with the current value of the given
3838
/// address, but it will definitely be copied out of after the call.
3839
static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args,
3840
27
                             const ObjCIndirectCopyRestoreExpr *CRE) {
3841
27
  LValue srcLV;
3842
3843
  // Make an optimistic effort to emit the address as an l-value.
3844
  // This can fail if the argument expression is more complicated.
3845
27
  if (const Expr *lvExpr = maybeGetUnaryAddrOfOperand(CRE->getSubExpr())) {
3846
22
    srcLV = CGF.EmitLValue(lvExpr);
3847
3848
  // Otherwise, just emit it as a scalar.
3849
22
  } else {
3850
5
    Address srcAddr = CGF.EmitPointerWithAlignment(CRE->getSubExpr());
3851
3852
5
    QualType srcAddrType =
3853
5
      CRE->getSubExpr()->getType()->castAs<PointerType>()->getPointeeType();
3854
5
    srcLV = CGF.MakeAddrLValue(srcAddr, srcAddrType);
3855
5
  }
3856
27
  Address srcAddr = srcLV.getAddress(CGF);
3857
3858
  // The dest and src types don't necessarily match in LLVM terms
3859
  // because of the crazy ObjC compatibility rules.
3860
3861
27
  llvm::PointerType *destType =
3862
27
    cast<llvm::PointerType>(CGF.ConvertType(CRE->getType()));
3863
3864
  // If the address is a constant null, just pass the appropriate null.
3865
27
  if (isProvablyNull(srcAddr.getPointer())) {
3866
0
    args.add(RValue::get(llvm::ConstantPointerNull::get(destType)),
3867
0
             CRE->getType());
3868
0
    return;
3869
0
  }
3870
3871
  // Create the temporary.
3872
27
  Address temp = CGF.CreateTempAlloca(destType->getElementType(),
3873
27
                                      CGF.getPointerAlign(),
3874
27
                                      "icr.temp");
3875
  // Loading an l-value can introduce a cleanup if the l-value is __weak,
3876
  // and that cleanup will be conditional if we can't prove that the l-value
3877
  // isn't null, so we need to register a dominating point so that the cleanups
3878
  // system will make valid IR.
3879
27
  CodeGenFunction::ConditionalEvaluation condEval(CGF);
3880
3881
  // Zero-initialize it if we're not doing a copy-initialization.
3882
27
  bool shouldCopy = CRE->shouldCopy();
3883
27
  if (!shouldCopy) {
3884
4
    llvm::Value *null =
3885
4
      llvm::ConstantPointerNull::get(
3886
4
        cast<llvm::PointerType>(destType->getElementType()));
3887
4
    CGF.Builder.CreateStore(null, temp);
3888
4
  }
3889
3890
27
  llvm::BasicBlock *contBB = nullptr;
3891
27
  llvm::BasicBlock *originBB = nullptr;
3892
3893
  // If the address is *not* known to be non-null, we need to switch.
3894
27
  llvm::Value *finalArgument;
3895
3896
27
  bool provablyNonNull = llvm::isKnownNonZero(srcAddr.getPointer(),
3897
27
                                              CGF.CGM.getDataLayout());
3898
27
  if (provablyNonNull) {
3899
22
    finalArgument = temp.getPointer();
3900
22
  } else {
3901
5
    llvm::Value *isNull =
3902
5
      CGF.Builder.CreateIsNull(srcAddr.getPointer(), "icr.isnull");
3903
3904
5
    finalArgument = CGF.Builder.CreateSelect(isNull,
3905
5
                                   llvm::ConstantPointerNull::get(destType),
3906
5
                                             temp.getPointer(), "icr.argument");
3907
3908
    // If we need to copy, then the load has to be conditional, which
3909
    // means we need control flow.
3910
5
    if (shouldCopy) {
3911
5
      originBB = CGF.Builder.GetInsertBlock();
3912
5
      contBB = CGF.createBasicBlock("icr.cont");
3913
5
      llvm::BasicBlock *copyBB = CGF.createBasicBlock("icr.copy");
3914
5
      CGF.Builder.CreateCondBr(isNull, contBB, copyBB);
3915
5
      CGF.EmitBlock(copyBB);
3916
5
      condEval.begin(CGF);
3917
5
    }
3918
5
  }
3919
3920
27
  llvm::Value *valueToUse = nullptr;
3921
3922
  // Perform a copy if necessary.
3923
27
  if (shouldCopy) {
3924
23
    RValue srcRV = CGF.EmitLoadOfLValue(srcLV, SourceLocation());
3925
23
    assert(srcRV.isScalar());
3926
3927
0
    llvm::Value *src = srcRV.getScalarVal();
3928
23
    src = CGF.Builder.CreateBitCast(src, destType->getElementType(),
3929
23
                                    "icr.cast");
3930
3931
    // Use an ordinary store, not a store-to-lvalue.
3932
23
    CGF.Builder.CreateStore(src, temp);
3933
3934
    // If optimization is enabled, and the value was held in a
3935
    // __strong variable, we need to tell the optimizer that this
3936
    // value has to stay alive until we're doing the store back.
3937
    // This is because the temporary is effectively unretained,
3938
    // and so otherwise we can violate the high-level semantics.
3939
23
    if (CGF.CGM.getCodeGenOpts().OptimizationLevel != 0 &&
3940
23
        
srcLV.getObjCLifetime() == Qualifiers::OCL_Strong10
) {
3941
8
      valueToUse = src;
3942
8
    }
3943
23
  }
3944
3945
  // Finish the control flow if we needed it.
3946
27
  if (shouldCopy && 
!provablyNonNull23
) {
3947
5
    llvm::BasicBlock *copyBB = CGF.Builder.GetInsertBlock();
3948
5
    CGF.EmitBlock(contBB);
3949
3950
    // Make a phi for the value to intrinsically use.
3951
5
    if (valueToUse) {
3952
2
      llvm::PHINode *phiToUse = CGF.Builder.CreatePHI(valueToUse->getType(), 2,
3953
2
                                                      "icr.to-use");
3954
2
      phiToUse->addIncoming(valueToUse, copyBB);
3955
2
      phiToUse->addIncoming(llvm::UndefValue::get(valueToUse->getType()),
3956
2
                            originBB);
3957
2
      valueToUse = phiToUse;
3958
2
    }
3959
3960
5
    condEval.end(CGF);
3961
5
  }
3962
3963
27
  args.addWriteback(srcLV, temp, valueToUse);
3964
27
  args.add(RValue::get(finalArgument), CRE->getType());
3965
27
}
3966
3967
71
void CallArgList::allocateArgumentMemory(CodeGenFunction &CGF) {
3968
71
  assert(!StackBase);
3969
3970
  // Save the stack.
3971
0
  llvm::Function *F = CGF.CGM.getIntrinsic(llvm::Intrinsic::stacksave);
3972
71
  StackBase = CGF.Builder.CreateCall(F, {}, "inalloca.save");
3973
71
}
3974
3975
386k
void CallArgList::freeArgumentMemory(CodeGenFunction &CGF) const {
3976
386k
  if (StackBase) {
3977
    // Restore the stack after the call.
3978
71
    llvm::Function *F = CGF.CGM.getIntrinsic(llvm::Intrinsic::stackrestore);
3979
71
    CGF.Builder.CreateCall(F, StackBase);
3980
71
  }
3981
386k
}
3982
3983
void CodeGenFunction::EmitNonNullArgCheck(RValue RV, QualType ArgType,
3984
                                          SourceLocation ArgLoc,
3985
                                          AbstractCallee AC,
3986
446k
                                          unsigned ParmNum) {
3987
446k
  if (!AC.getDecl() || 
!(426k
SanOpts.has(SanitizerKind::NonnullAttribute)426k
||
3988
426k
                         
SanOpts.has(SanitizerKind::NullabilityArg)426k
))
3989
446k
    return;
3990
3991
  // The param decl may be missing in a variadic function.
3992
77
  auto PVD = ParmNum < AC.getNumParams() ? 
AC.getParamDecl(ParmNum)66
:
nullptr11
;
3993
77
  unsigned ArgNo = PVD ? 
PVD->getFunctionScopeIndex()66
:
ParmNum11
;
3994
3995
  // Prefer the nonnull attribute if it's present.
3996
77
  const NonNullAttr *NNAttr = nullptr;
3997
77
  if (SanOpts.has(SanitizerKind::NonnullAttribute))
3998
25
    NNAttr = getNonNullAttr(AC.getDecl(), PVD, ArgType, ArgNo);
3999
4000
77
  bool CanCheckNullability = false;
4001
77
  if (SanOpts.has(SanitizerKind::NullabilityArg) && 
!NNAttr53
&&
PVD52
) {
4002
44
    auto Nullability = PVD->getType()->getNullability(getContext());
4003
44
    CanCheckNullability = Nullability &&
4004
44
                          
*Nullability == NullabilityKind::NonNull20
&&
4005
44
                          
PVD->getTypeSourceInfo()18
;
4006
44
  }
4007
4008
77
  if (!NNAttr && 
!CanCheckNullability61
)
4009
43
    return;
4010
4011
34
  SourceLocation AttrLoc;
4012
34
  SanitizerMask CheckKind;
4013
34
  SanitizerHandler Handler;
4014
34
  if (NNAttr) {
4015
16
    AttrLoc = NNAttr->getLocation();
4016
16
    CheckKind = SanitizerKind::NonnullAttribute;
4017
16
    Handler = SanitizerHandler::NonnullArg;
4018
18
  } else {
4019
18
    AttrLoc = PVD->getTypeSourceInfo()->getTypeLoc().findNullabilityLoc();
4020
18
    CheckKind = SanitizerKind::NullabilityArg;
4021
18
    Handler = SanitizerHandler::NullabilityArg;
4022
18
  }
4023
4024
34
  SanitizerScope SanScope(this);
4025
34
  llvm::Value *Cond = EmitNonNullRValueCheck(RV, ArgType);
4026
34
  llvm::Constant *StaticData[] = {
4027
34
      EmitCheckSourceLocation(ArgLoc), EmitCheckSourceLocation(AttrLoc),
4028
34
      llvm::ConstantInt::get(Int32Ty, ArgNo + 1),
4029
34
  };
4030
34
  EmitCheck(std::make_pair(Cond, CheckKind), Handler, StaticData, None);
4031
34
}
4032
4033
// Check if the call is going to use the inalloca convention. This needs to
4034
// agree with CGFunctionInfo::usesInAlloca. The CGFunctionInfo is arranged
4035
// later, so we can't check it directly.
4036
static bool hasInAllocaArgs(CodeGenModule &CGM, CallingConv ExplicitCC,
4037
337k
                            ArrayRef<QualType> ArgTypes) {
4038
  // The Swift calling conventions don't go through the target-specific
4039
  // argument classification, they never use inalloca.
4040
  // TODO: Consider limiting inalloca use to only calling conventions supported
4041
  // by MSVC.
4042
337k
  if (ExplicitCC == CC_Swift || 
ExplicitCC == CC_SwiftAsync336k
)
4043
1.28k
    return false;
4044
336k
  if (!CGM.getTarget().getCXXABI().isMicrosoft())
4045
329k
    return false;
4046
7.16k
  return llvm::any_of(ArgTypes, [&](QualType Ty) {
4047
6.51k
    return isInAllocaArgument(CGM.getCXXABI(), Ty);
4048
6.51k
  });
4049
336k
}
4050
4051
#ifndef NDEBUG
4052
// Determine whether the given argument is an Objective-C method
4053
// that may have type parameters in its signature.
4054
11.3k
static bool isObjCMethodWithTypeParams(const ObjCMethodDecl *method) {
4055
11.3k
  const DeclContext *dc = method->getDeclContext();
4056
11.3k
  if (const ObjCInterfaceDecl *classDecl = dyn_cast<ObjCInterfaceDecl>(dc)) {
4057
7.25k
    return classDecl->getTypeParamListAsWritten();
4058
7.25k
  }
4059
4060
4.06k
  if (const ObjCCategoryDecl *catDecl = dyn_cast<ObjCCategoryDecl>(dc)) {
4061
3.37k
    return catDecl->getTypeParamList();
4062
3.37k
  }
4063
4064
697
  return false;
4065
4.06k
}
4066
#endif
4067
4068
/// EmitCallArgs - Emit call arguments for a function.
4069
void CodeGenFunction::EmitCallArgs(
4070
    CallArgList &Args, PrototypeWrapper Prototype,
4071
    llvm::iterator_range<CallExpr::const_arg_iterator> ArgRange,
4072
337k
    AbstractCallee AC, unsigned ParamsToSkip, EvaluationOrder Order) {
4073
337k
  SmallVector<QualType, 16> ArgTypes;
4074
4075
337k
  assert((ParamsToSkip == 0 || Prototype.P) &&
4076
337k
         "Can't skip parameters if type info is not provided");
4077
4078
  // This variable only captures *explicitly* written conventions, not those
4079
  // applied by default via command line flags or target defaults, such as
4080
  // thiscall, aapcs, stdcall via -mrtd, etc. Computing that correctly would
4081
  // require knowing if this is a C++ instance method or being able to see
4082
  // unprototyped FunctionTypes.
4083
0
  CallingConv ExplicitCC = CC_C;
4084
4085
  // First, if a prototype was provided, use those argument types.
4086
337k
  bool IsVariadic = false;
4087
337k
  if (Prototype.P) {
4088
336k
    const auto *MD = Prototype.P.dyn_cast<const ObjCMethodDecl *>();
4089
336k
    if (MD) {
4090
11.3k
      IsVariadic = MD->isVariadic();
4091
11.3k
      ExplicitCC = getCallingConventionForDecl(
4092
11.3k
          MD, CGM.getTarget().getTriple().isOSWindows());
4093
11.3k
      ArgTypes.assign(MD->param_type_begin() + ParamsToSkip,
4094
11.3k
                      MD->param_type_end());
4095
324k
    } else {
4096
324k
      const auto *FPT = Prototype.P.get<const FunctionProtoType *>();
4097
324k
      IsVariadic = FPT->isVariadic();
4098
324k
      ExplicitCC = FPT->getExtInfo().getCC();
4099
324k
      ArgTypes.assign(FPT->param_type_begin() + ParamsToSkip,
4100
324k
                      FPT->param_type_end());
4101
324k
    }
4102
4103
336k
#ifndef NDEBUG
4104
    // Check that the prototyped types match the argument expression types.
4105
336k
    bool isGenericMethod = MD && 
isObjCMethodWithTypeParams(MD)11.3k
;
4106
336k
    CallExpr::const_arg_iterator Arg = ArgRange.begin();
4107
356k
    for (QualType Ty : ArgTypes) {
4108
356k
      assert(Arg != ArgRange.end() && "Running over edge of argument list!");
4109
0
      assert(
4110
356k
          (isGenericMethod || Ty->isVariablyModifiedType() ||
4111
356k
           Ty.getNonReferenceType()->isObjCRetainableType() ||
4112
356k
           getContext()
4113
356k
                   .getCanonicalType(Ty.getNonReferenceType())
4114
356k
                   .getTypePtr() ==
4115
356k
               getContext().getCanonicalType((*Arg)->getType()).getTypePtr()) &&
4116
356k
          "type mismatch in call argument!");
4117
0
      ++Arg;
4118
356k
    }
4119
4120
    // Either we've emitted all the call args, or we have a call to variadic
4121
    // function.
4122
336k
    assert((Arg == ArgRange.end() || IsVariadic) &&
4123
336k
           "Extra arguments in non-variadic function!");
4124
336k
#endif
4125
336k
  }
4126
4127
  // If we still have any arguments, emit them using the type of the argument.
4128
0
  for (auto *A : llvm::make_range(std::next(ArgRange.begin(), ArgTypes.size()),
4129
337k
                                  ArgRange.end()))
4130
90.4k
    ArgTypes.push_back(IsVariadic ? 
getVarArgType(A)89.7k
:
A->getType()730
);
4131
337k
  assert((int)ArgTypes.size() == (ArgRange.end() - ArgRange.begin()));
4132
4133
  // We must evaluate arguments from right to left in the MS C++ ABI,
4134
  // because arguments are destroyed left to right in the callee. As a special
4135
  // case, there are certain language constructs that require left-to-right
4136
  // evaluation, and in those cases we consider the evaluation order requirement
4137
  // to trump the "destruction order is reverse construction order" guarantee.
4138
0
  bool LeftToRight =
4139
337k
      CGM.getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()
4140
337k
          ? 
Order == EvaluationOrder::ForceLeftToRight7.19k
4141
337k
          : 
Order != EvaluationOrder::ForceRightToLeft330k
;
4142
4143
337k
  auto MaybeEmitImplicitObjectSize = [&](unsigned I, const Expr *Arg,
4144
445k
                                         RValue EmittedArg) {
4145
445k
    if (!AC.hasFunctionDecl() || 
I >= AC.getNumParams()415k
)
4146
118k
      return;
4147
327k
    auto *PS = AC.getParamDecl(I)->getAttr<PassObjectSizeAttr>();
4148
327k
    if (PS == nullptr)
4149
326k
      return;
4150
4151
107
    const auto &Context = getContext();
4152
107
    auto SizeTy = Context.getSizeType();
4153
107
    auto T = Builder.getIntNTy(Context.getTypeSize(SizeTy));
4154
107
    assert(EmittedArg.getScalarVal() && "We emitted nothing for the arg?");
4155
0
    llvm::Value *V = evaluateOrEmitBuiltinObjectSize(Arg, PS->getType(), T,
4156
107
                                                     EmittedArg.getScalarVal(),
4157
107
                                                     PS->isDynamic());
4158
107
    Args.add(RValue::get(V), SizeTy);
4159
    // If we're emitting args in reverse, be sure to do so with
4160
    // pass_object_size, as well.
4161
107
    if (!LeftToRight)
4162
9
      std::swap(Args.back(), *(&Args.back() - 1));
4163
107
  };
4164
4165
  // Insert a stack save if we're going to need any inalloca args.
4166
337k
  if (hasInAllocaArgs(CGM, ExplicitCC, ArgTypes)) {
4167
71
    assert(getTarget().getTriple().getArch() == llvm::Triple::x86 &&
4168
71
           "inalloca only supported on x86");
4169
0
    Args.allocateArgumentMemory(*this);
4170
71
  }
4171
4172
  // Evaluate each argument in the appropriate order.
4173
0
  size_t CallArgsStart = Args.size();
4174
784k
  for (unsigned I = 0, E = ArgTypes.size(); I != E; 
++I446k
) {
4175
446k
    unsigned Idx = LeftToRight ? 
I439k
:
E - I - 17.45k
;
4176
446k
    CallExpr::const_arg_iterator Arg = ArgRange.begin() + Idx;
4177
446k
    unsigned InitialArgSize = Args.size();
4178
    // If *Arg is an ObjCIndirectCopyRestoreExpr, check that either the types of
4179
    // the argument and parameter match or the objc method is parameterized.
4180
446k
    assert((!isa<ObjCIndirectCopyRestoreExpr>(*Arg) ||
4181
446k
            getContext().hasSameUnqualifiedType((*Arg)->getType(),
4182
446k
                                                ArgTypes[Idx]) ||
4183
446k
            (isa<ObjCMethodDecl>(AC.getDecl()) &&
4184
446k
             isObjCMethodWithTypeParams(cast<ObjCMethodDecl>(AC.getDecl())))) &&
4185
446k
           "Argument and parameter types don't match");
4186
0
    EmitCallArg(Args, *Arg, ArgTypes[Idx]);
4187
    // In particular, we depend on it being the last arg in Args, and the
4188
    // objectsize bits depend on there only being one arg if !LeftToRight.
4189
446k
    assert(InitialArgSize + 1 == Args.size() &&
4190
446k
           "The code below depends on only adding one arg per EmitCallArg");
4191
0
    (void)InitialArgSize;
4192
    // Since pointer argument are never emitted as LValue, it is safe to emit
4193
    // non-null argument check for r-value only.
4194
446k
    if (!Args.back().hasLValue()) {
4195
445k
      RValue RVArg = Args.back().getKnownRValue();
4196
445k
      EmitNonNullArgCheck(RVArg, ArgTypes[Idx], (*Arg)->getExprLoc(), AC,
4197
445k
                          ParamsToSkip + Idx);
4198
      // @llvm.objectsize should never have side-effects and shouldn't need
4199
      // destruction/cleanups, so we can safely "emit" it after its arg,
4200
      // regardless of right-to-leftness
4201
445k
      MaybeEmitImplicitObjectSize(Idx, *Arg, RVArg);
4202
445k
    }
4203
446k
  }
4204
4205
337k
  if (!LeftToRight) {
4206
    // Un-reverse the arguments we just evaluated so they match up with the LLVM
4207
    // IR function.
4208
8.04k
    std::reverse(Args.begin() + CallArgsStart, Args.end());
4209
8.04k
  }
4210
337k
}
4211
4212
namespace {
4213
4214
struct DestroyUnpassedArg final : EHScopeStack::Cleanup {
4215
  DestroyUnpassedArg(Address Addr, QualType Ty)
4216
178
      : Addr(Addr), Ty(Ty) {}
4217
4218
  Address Addr;
4219
  QualType Ty;
4220
4221
20
  void Emit(CodeGenFunction &CGF, Flags flags) override {
4222
20
    QualType::DestructionKind DtorKind = Ty.isDestructedType();
4223
20
    if (DtorKind == QualType::DK_cxx_destructor) {
4224
18
      const CXXDestructorDecl *Dtor = Ty->getAsCXXRecordDecl()->getDestructor();
4225
18
      assert(!Dtor->isTrivial());
4226
0
      CGF.EmitCXXDestructorCall(Dtor, Dtor_Complete, /*for vbase*/ false,
4227
18
                                /*Delegating=*/false, Addr, Ty);
4228
18
    } else {
4229
2
      CGF.callCStructDestructor(CGF.MakeAddrLValue(Addr, Ty));
4230
2
    }
4231
20
  }
4232
};
4233
4234
struct DisableDebugLocationUpdates {
4235
  CodeGenFunction &CGF;
4236
  bool disabledDebugInfo;
4237
446k
  DisableDebugLocationUpdates(CodeGenFunction &CGF, const Expr *E) : CGF(CGF) {
4238
446k
    if ((disabledDebugInfo = isa<CXXDefaultArgExpr>(E) && 
CGF.getDebugInfo()4.08k
))
4239
2.84k
      CGF.disableDebugInfo();
4240
446k
  }
4241
446k
  ~DisableDebugLocationUpdates() {
4242
446k
    if (disabledDebugInfo)
4243
2.84k
      CGF.enableDebugInfo();
4244
446k
  }
4245
};
4246
4247
} // end anonymous namespace
4248
4249
288
RValue CallArg::getRValue(CodeGenFunction &CGF) const {
4250
288
  if (!HasLV)
4251
288
    return RV;
4252
0
  LValue Copy = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty), Ty);
4253
0
  CGF.EmitAggregateCopy(Copy, LV, Ty, AggValueSlot::DoesNotOverlap,
4254
0
                        LV.isVolatile());
4255
0
  IsUsed = true;
4256
0
  return RValue::getAggregate(Copy.getAddress(CGF));
4257
288
}
4258
4259
1.23k
void CallArg::copyInto(CodeGenFunction &CGF, Address Addr) const {
4260
1.23k
  LValue Dst = CGF.MakeAddrLValue(Addr, Ty);
4261
1.23k
  if (!HasLV && 
RV.isScalar()1.17k
)
4262
800
    CGF.EmitStoreOfScalar(RV.getScalarVal(), Dst, /*isInit=*/true);
4263
435
  else if (!HasLV && 
RV.isComplex()372
)
4264
372
    CGF.EmitStoreOfComplex(RV.getComplexVal(), Dst, /*init=*/true);
4265
63
  else {
4266
63
    auto Addr = HasLV ? LV.getAddress(CGF) : 
RV.getAggregateAddress()0
;
4267
63
    LValue SrcLV = CGF.MakeAddrLValue(Addr, Ty);
4268
    // We assume that call args are never copied into subobjects.
4269
63
    CGF.EmitAggregateCopy(Dst, SrcLV, Ty, AggValueSlot::DoesNotOverlap,
4270
63
                          HasLV ? LV.isVolatileQualified()
4271
63
                                : 
RV.isVolatileQualified()0
);
4272
63
  }
4273
1.23k
  IsUsed = true;
4274
1.23k
}
4275
4276
void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E,
4277
446k
                                  QualType type) {
4278
446k
  DisableDebugLocationUpdates Dis(*this, E);
4279
446k
  if (const ObjCIndirectCopyRestoreExpr *CRE
4280
446k
        = dyn_cast<ObjCIndirectCopyRestoreExpr>(E)) {
4281
27
    assert(getLangOpts().ObjCAutoRefCount);
4282
0
    return emitWritebackArg(*this, args, CRE);
4283
27
  }
4284
4285
446k
  assert(type->isReferenceType() == E->isGLValue() &&
4286
446k
         "reference binding to unmaterialized r-value!");
4287
4288
446k
  if (E->isGLValue()) {
4289
65.4k
    assert(E->getObjectKind() == OK_Ordinary);
4290
0
    return args.add(EmitReferenceBindingToExpr(E), type);
4291
65.4k
  }
4292
4293
381k
  bool HasAggregateEvalKind = hasAggregateEvaluationKind(type);
4294
4295
  // In the Microsoft C++ ABI, aggregate arguments are destructed by the callee.
4296
  // However, we still have to push an EH-only cleanup in case we unwind before
4297
  // we make it to the call.
4298
381k
  if (type->isRecordType() &&
4299
381k
      
type->castAs<RecordType>()->getDecl()->isParamDestroyedInCallee()10.7k
) {
4300
    // If we're using inalloca, use the argument memory.  Otherwise, use a
4301
    // temporary.
4302
376
    AggValueSlot Slot;
4303
376
    if (args.isUsingInAlloca())
4304
102
      Slot = createPlaceholderSlot(*this, type);
4305
274
    else
4306
274
      Slot = CreateAggTemp(type, "agg.tmp");
4307
4308
376
    bool DestroyedInCallee = true, NeedsEHCleanup = true;
4309
376
    if (const auto *RD = type->getAsCXXRecordDecl())
4310
354
      DestroyedInCallee = RD->hasNonTrivialDestructor();
4311
22
    else
4312
22
      NeedsEHCleanup = needsEHCleanup(type.isDestructedType());
4313
4314
376
    if (DestroyedInCallee)
4315
200
      Slot.setExternallyDestructed();
4316
4317
376
    EmitAggExpr(E, Slot);
4318
376
    RValue RV = Slot.asRValue();
4319
376
    args.add(RV, type);
4320
4321
376
    if (DestroyedInCallee && 
NeedsEHCleanup200
) {
4322
      // Create a no-op GEP between the placeholder and the cleanup so we can
4323
      // RAUW it successfully.  It also serves as a marker of the first
4324
      // instruction where the cleanup is active.
4325
182
      pushFullExprCleanup<DestroyUnpassedArg>(EHCleanup, Slot.getAddress(),
4326
182
                                              type);
4327
      // This unreachable is a temporary marker which will be removed later.
4328
182
      llvm::Instruction *IsActive = Builder.CreateUnreachable();
4329
182
      args.addArgCleanupDeactivation(EHStack.getInnermostEHScope(), IsActive);
4330
182
    }
4331
376
    return;
4332
376
  }
4333
4334
380k
  if (HasAggregateEvalKind && 
isa<ImplicitCastExpr>(E)10.4k
&&
4335
380k
      
cast<CastExpr>(E)->getCastKind() == CK_LValueToRValue841
) {
4336
810
    LValue L = EmitLValue(cast<CastExpr>(E)->getSubExpr());
4337
810
    assert(L.isSimple());
4338
0
    args.addUncopiedAggregate(L, type);
4339
810
    return;
4340
810
  }
4341
4342
380k
  args.add(EmitAnyExprToTemp(E), type);
4343
380k
}
4344
4345
89.7k
QualType CodeGenFunction::getVarArgType(const Expr *Arg) {
4346
  // System headers on Windows define NULL to 0 instead of 0LL on Win64. MSVC
4347
  // implicitly widens null pointer constants that are arguments to varargs
4348
  // functions to pointer-sized ints.
4349
89.7k
  if (!getTarget().getTriple().isOSWindows())
4350
89.6k
    return Arg->getType();
4351
4352
94
  if (Arg->getType()->isIntegerType() &&
4353
94
      getContext().getTypeSize(Arg->getType()) <
4354
53
          getContext().getTargetInfo().getPointerWidth(0) &&
4355
94
      Arg->isNullPointerConstant(getContext(),
4356
29
                                 Expr::NPC_ValueDependentIsNotNull)) {
4357
1
    return getContext().getIntPtrType();
4358
1
  }
4359
4360
93
  return Arg->getType();
4361
94
}
4362
4363
// In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
4364
// optimizer it can aggressively ignore unwind edges.
4365
void
4366
1.84k
CodeGenFunction::AddObjCARCExceptionMetadata(llvm::Instruction *Inst) {
4367
1.84k
  if (CGM.getCodeGenOpts().OptimizationLevel != 0 &&
4368
1.84k
      
!CGM.getCodeGenOpts().ObjCAutoRefCountExceptions446
)
4369
356
    Inst->setMetadata("clang.arc.no_objc_arc_exceptions",
4370
356
                      CGM.getNoObjCARCExceptionsMetadata());
4371
1.84k
}
4372
4373
/// Emits a call to the given no-arguments nounwind runtime function.
4374
llvm::CallInst *
4375
CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
4376
508
                                         const llvm::Twine &name) {
4377
508
  return EmitNounwindRuntimeCall(callee, None, name);
4378
508
}
4379
4380
/// Emits a call to the given nounwind runtime function.
4381
llvm::CallInst *
4382
CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
4383
                                         ArrayRef<llvm::Value *> args,
4384
31.8k
                                         const llvm::Twine &name) {
4385
31.8k
  llvm::CallInst *call = EmitRuntimeCall(callee, args, name);
4386
31.8k
  call->setDoesNotThrow();
4387
31.8k
  return call;
4388
31.8k
}
4389
4390
/// Emits a simple call (never an invoke) to the given no-arguments
4391
/// runtime function.
4392
llvm::CallInst *CodeGenFunction::EmitRuntimeCall(llvm::FunctionCallee callee,
4393
5.61k
                                                 const llvm::Twine &name) {
4394
5.61k
  return EmitRuntimeCall(callee, None, name);
4395
5.61k
}
4396
4397
// Calls which may throw must have operand bundles indicating which funclet
4398
// they are nested within.
4399
SmallVector<llvm::OperandBundleDef, 1>
4400
486k
CodeGenFunction::getBundlesForFunclet(llvm::Value *Callee) {
4401
486k
  SmallVector<llvm::OperandBundleDef, 1> BundleList;
4402
  // There is no need for a funclet operand bundle if we aren't inside a
4403
  // funclet.
4404
486k
  if (!CurrentFuncletPad)
4405
485k
    return BundleList;
4406
4407
  // Skip intrinsics which cannot throw.
4408
639
  auto *CalleeFn = dyn_cast<llvm::Function>(Callee->stripPointerCasts());
4409
639
  if (CalleeFn && 
CalleeFn->isIntrinsic()633
&&
CalleeFn->doesNotThrow()38
)
4410
2
    return BundleList;
4411
4412
637
  BundleList.emplace_back("funclet", CurrentFuncletPad);
4413
637
  return BundleList;
4414
639
}
4415
4416
/// Emits a simple call (never an invoke) to the given runtime function.
4417
llvm::CallInst *CodeGenFunction::EmitRuntimeCall(llvm::FunctionCallee callee,
4418
                                                 ArrayRef<llvm::Value *> args,
4419
92.8k
                                                 const llvm::Twine &name) {
4420
92.8k
  llvm::CallInst *call = Builder.CreateCall(
4421
92.8k
      callee, args, getBundlesForFunclet(callee.getCallee()), name);
4422
92.8k
  call->setCallingConv(getRuntimeCC());
4423
92.8k
  return call;
4424
92.8k
}
4425
4426
/// Emits a call or invoke to the given noreturn runtime function.
4427
void CodeGenFunction::EmitNoreturnRuntimeCallOrInvoke(
4428
594
    llvm::FunctionCallee callee, ArrayRef<llvm::Value *> args) {
4429
594
  SmallVector<llvm::OperandBundleDef, 1> BundleList =
4430
594
      getBundlesForFunclet(callee.getCallee());
4431
4432
594
  if (getInvokeDest()) {
4433
154
    llvm::InvokeInst *invoke =
4434
154
      Builder.CreateInvoke(callee,
4435
154
                           getUnreachableBlock(),
4436
154
                           getInvokeDest(),
4437
154
                           args,
4438
154
                           BundleList);
4439
154
    invoke->setDoesNotReturn();
4440
154
    invoke->setCallingConv(getRuntimeCC());
4441
440
  } else {
4442
440
    llvm::CallInst *call = Builder.CreateCall(callee, args, BundleList);
4443
440
    call->setDoesNotReturn();
4444
440
    call->setCallingConv(getRuntimeCC());
4445
440
    Builder.CreateUnreachable();
4446
440
  }
4447
594
}
4448
4449
/// Emits a call or invoke instruction to the given nullary runtime function.
4450
llvm::CallBase *
4451
CodeGenFunction::EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
4452
406
                                         const Twine &name) {
4453
406
  return EmitRuntimeCallOrInvoke(callee, None, name);
4454
406
}
4455
4456
/// Emits a call or invoke instruction to the given runtime function.
4457
llvm::CallBase *
4458
CodeGenFunction::EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
4459
                                         ArrayRef<llvm::Value *> args,
4460
668
                                         const Twine &name) {
4461
668
  llvm::CallBase *call = EmitCallOrInvoke(callee, args, name);
4462
668
  call->setCallingConv(getRuntimeCC());
4463
668
  return call;
4464
668
}
4465
4466
/// Emits a call or invoke instruction to the given function, depending
4467
/// on the current state of the EH stack.
4468
llvm::CallBase *CodeGenFunction::EmitCallOrInvoke(llvm::FunctionCallee Callee,
4469
                                                  ArrayRef<llvm::Value *> Args,
4470
2.22k
                                                  const Twine &Name) {
4471
2.22k
  llvm::BasicBlock *InvokeDest = getInvokeDest();
4472
2.22k
  SmallVector<llvm::OperandBundleDef, 1> BundleList =
4473
2.22k
      getBundlesForFunclet(Callee.getCallee());
4474
4475
2.22k
  llvm::CallBase *Inst;
4476
2.22k
  if (!InvokeDest)
4477
1.94k
    Inst = Builder.CreateCall(Callee, Args, BundleList, Name);
4478
279
  else {
4479
279
    llvm::BasicBlock *ContBB = createBasicBlock("invoke.cont");
4480
279
    Inst = Builder.CreateInvoke(Callee, ContBB, InvokeDest, Args, BundleList,
4481
279
                                Name);
4482
279
    EmitBlock(ContBB);
4483
279
  }
4484
4485
  // In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
4486
  // optimizer it can aggressively ignore unwind edges.
4487
2.22k
  if (CGM.getLangOpts().ObjCAutoRefCount)
4488
4
    AddObjCARCExceptionMetadata(Inst);
4489
4490
2.22k
  return Inst;
4491
2.22k
}
4492
4493
void CodeGenFunction::deferPlaceholderReplacement(llvm::Instruction *Old,
4494
103
                                                  llvm::Value *New) {
4495
103
  DeferredReplacements.push_back(
4496
103
      std::make_pair(llvm::WeakTrackingVH(Old), New));
4497
103
}
4498
4499
namespace {
4500
4501
/// Specify given \p NewAlign as the alignment of return value attribute. If
4502
/// such attribute already exists, re-set it to the maximal one of two options.
4503
LLVM_NODISCARD llvm::AttributeList
4504
maybeRaiseRetAlignmentAttribute(llvm::LLVMContext &Ctx,
4505
                                const llvm::AttributeList &Attrs,
4506
118
                                llvm::Align NewAlign) {
4507
118
  llvm::Align CurAlign = Attrs.getRetAlignment().valueOrOne();
4508
118
  if (CurAlign >= NewAlign)
4509
9
    return Attrs;
4510
109
  llvm::Attribute AlignAttr = llvm::Attribute::getWithAlignment(Ctx, NewAlign);
4511
109
  return Attrs.removeRetAttribute(Ctx, llvm::Attribute::AttrKind::Alignment)
4512
109
      .addRetAttribute(Ctx, AlignAttr);
4513
118
}
4514
4515
template <typename AlignedAttrTy> class AbstractAssumeAlignedAttrEmitter {
4516
protected:
4517
  CodeGenFunction &CGF;
4518
4519
  /// We do nothing if this is, or becomes, nullptr.
4520
  const AlignedAttrTy *AA = nullptr;
4521
4522
  llvm::Value *Alignment = nullptr;      // May or may not be a constant.
4523
  llvm::ConstantInt *OffsetCI = nullptr; // Constant, hopefully zero.
4524
4525
  AbstractAssumeAlignedAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl)
4526
776k
      : CGF(CGF_) {
4527
776k
    if (!FuncDecl)
4528
33.8k
      return;
4529
742k
    AA = FuncDecl->getAttr<AlignedAttrTy>();
4530
742k
  }
CGCall.cpp:(anonymous namespace)::AbstractAssumeAlignedAttrEmitter<clang::AssumeAlignedAttr>::AbstractAssumeAlignedAttrEmitter(clang::CodeGen::CodeGenFunction&, clang::Decl const*)
Line
Count
Source
4526
388k
      : CGF(CGF_) {
4527
388k
    if (!FuncDecl)
4528
16.9k
      return;
4529
371k
    AA = FuncDecl->getAttr<AlignedAttrTy>();
4530
371k
  }
CGCall.cpp:(anonymous namespace)::AbstractAssumeAlignedAttrEmitter<clang::AllocAlignAttr>::AbstractAssumeAlignedAttrEmitter(clang::CodeGen::CodeGenFunction&, clang::Decl const*)
Line
Count
Source
4526
388k
      : CGF(CGF_) {
4527
388k
    if (!FuncDecl)
4528
16.9k
      return;
4529
371k
    AA = FuncDecl->getAttr<AlignedAttrTy>();
4530
371k
  }
4531
4532
public:
4533
  /// If we can, materialize the alignment as an attribute on return value.
4534
  LLVM_NODISCARD llvm::AttributeList
4535
776k
  TryEmitAsCallSiteAttribute(const llvm::AttributeList &Attrs) {
4536
776k
    if (!AA || 
OffsetCI152
||
CGF.SanOpts.has(SanitizerKind::Alignment)147
)
4537
776k
      return Attrs;
4538
138
    const auto *AlignmentCI = dyn_cast<llvm::ConstantInt>(Alignment);
4539
138
    if (!AlignmentCI)
4540
19
      return Attrs;
4541
    // We may legitimately have non-power-of-2 alignment here.
4542
    // If so, this is UB land, emit it via `@llvm.assume` instead.
4543
119
    if (!AlignmentCI->getValue().isPowerOf2())
4544
1
      return Attrs;
4545
118
    llvm::AttributeList NewAttrs = maybeRaiseRetAlignmentAttribute(
4546
118
        CGF.getLLVMContext(), Attrs,
4547
118
        llvm::Align(
4548
118
            AlignmentCI->getLimitedValue(llvm::Value::MaximumAlignment)));
4549
118
    AA = nullptr; // We're done. Disallow doing anything else.
4550
118
    return NewAttrs;
4551
119
  }
CGCall.cpp:(anonymous namespace)::AbstractAssumeAlignedAttrEmitter<clang::AssumeAlignedAttr>::TryEmitAsCallSiteAttribute(llvm::AttributeList const&)
Line
Count
Source
4535
388k
  TryEmitAsCallSiteAttribute(const llvm::AttributeList &Attrs) {
4536
388k
    if (!AA || 
OffsetCI90
||
CGF.SanOpts.has(SanitizerKind::Alignment)85
)
4537
388k
      return Attrs;
4538
82
    const auto *AlignmentCI = dyn_cast<llvm::ConstantInt>(Alignment);
4539
82
    if (!AlignmentCI)
4540
0
      return Attrs;
4541
    // We may legitimately have non-power-of-2 alignment here.
4542
    // If so, this is UB land, emit it via `@llvm.assume` instead.
4543
82
    if (!AlignmentCI->getValue().isPowerOf2())
4544
0
      return Attrs;
4545
82
    llvm::AttributeList NewAttrs = maybeRaiseRetAlignmentAttribute(
4546
82
        CGF.getLLVMContext(), Attrs,
4547
82
        llvm::Align(
4548
82
            AlignmentCI->getLimitedValue(llvm::Value::MaximumAlignment)));
4549
82
    AA = nullptr; // We're done. Disallow doing anything else.
4550
82
    return NewAttrs;
4551
82
  }
CGCall.cpp:(anonymous namespace)::AbstractAssumeAlignedAttrEmitter<clang::AllocAlignAttr>::TryEmitAsCallSiteAttribute(llvm::AttributeList const&)
Line
Count
Source
4535
388k
  TryEmitAsCallSiteAttribute(const llvm::AttributeList &Attrs) {
4536
388k
    if (!AA || 
OffsetCI62
||
CGF.SanOpts.has(SanitizerKind::Alignment)62
)
4537
388k
      return Attrs;
4538
56
    const auto *AlignmentCI = dyn_cast<llvm::ConstantInt>(Alignment);
4539
56
    if (!AlignmentCI)
4540
19
      return Attrs;
4541
    // We may legitimately have non-power-of-2 alignment here.
4542
    // If so, this is UB land, emit it via `@llvm.assume` instead.
4543
37
    if (!AlignmentCI->getValue().isPowerOf2())
4544
1
      return Attrs;
4545
36
    llvm::AttributeList NewAttrs = maybeRaiseRetAlignmentAttribute(
4546
36
        CGF.getLLVMContext(), Attrs,
4547
36
        llvm::Align(
4548
36
            AlignmentCI->getLimitedValue(llvm::Value::MaximumAlignment)));
4549
36
    AA = nullptr; // We're done. Disallow doing anything else.
4550
36
    return NewAttrs;
4551
37
  }
4552
4553
  /// Emit alignment assumption.
4554
  /// This is a general fallback that we take if either there is an offset,
4555
  /// or the alignment is variable or we are sanitizing for alignment.
4556
723k
  void EmitAsAnAssumption(SourceLocation Loc, QualType RetTy, RValue &Ret) {
4557
723k
    if (!AA)
4558
723k
      return;
4559
34
    CGF.emitAlignmentAssumption(Ret.getScalarVal(), RetTy, Loc,
4560
34
                                AA->getLocation(), Alignment, OffsetCI);
4561
34
    AA = nullptr; // We're done. Disallow doing anything else.
4562
34
  }
CGCall.cpp:(anonymous namespace)::AbstractAssumeAlignedAttrEmitter<clang::AssumeAlignedAttr>::EmitAsAnAssumption(clang::SourceLocation, clang::QualType, clang::CodeGen::RValue&)
Line
Count
Source
4556
361k
  void EmitAsAnAssumption(SourceLocation Loc, QualType RetTy, RValue &Ret) {
4557
361k
    if (!AA)
4558
361k
      return;
4559
8
    CGF.emitAlignmentAssumption(Ret.getScalarVal(), RetTy, Loc,
4560
8
                                AA->getLocation(), Alignment, OffsetCI);
4561
8
    AA = nullptr; // We're done. Disallow doing anything else.
4562
8
  }
CGCall.cpp:(anonymous namespace)::AbstractAssumeAlignedAttrEmitter<clang::AllocAlignAttr>::EmitAsAnAssumption(clang::SourceLocation, clang::QualType, clang::CodeGen::RValue&)
Line
Count
Source
4556
361k
  void EmitAsAnAssumption(SourceLocation Loc, QualType RetTy, RValue &Ret) {
4557
361k
    if (!AA)
4558
361k
      return;
4559
26
    CGF.emitAlignmentAssumption(Ret.getScalarVal(), RetTy, Loc,
4560
26
                                AA->getLocation(), Alignment, OffsetCI);
4561
26
    AA = nullptr; // We're done. Disallow doing anything else.
4562
26
  }
4563
};
4564
4565
/// Helper data structure to emit `AssumeAlignedAttr`.
4566
class AssumeAlignedAttrEmitter final
4567
    : public AbstractAssumeAlignedAttrEmitter<AssumeAlignedAttr> {
4568
public:
4569
  AssumeAlignedAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl)
4570
388k
      : AbstractAssumeAlignedAttrEmitter(CGF_, FuncDecl) {
4571
388k
    if (!AA)
4572
388k
      return;
4573
    // It is guaranteed that the alignment/offset are constants.
4574
90
    Alignment = cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AA->getAlignment()));
4575
90
    if (Expr *Offset = AA->getOffset()) {
4576
5
      OffsetCI = cast<llvm::ConstantInt>(CGF.EmitScalarExpr(Offset));
4577
5
      if (OffsetCI->isNullValue()) // Canonicalize zero offset to no offset.
4578
0
        OffsetCI = nullptr;
4579
5
    }
4580
90
  }
4581
};
4582
4583
/// Helper data structure to emit `AllocAlignAttr`.
4584
class AllocAlignAttrEmitter final
4585
    : public AbstractAssumeAlignedAttrEmitter<AllocAlignAttr> {
4586
public:
4587
  AllocAlignAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl,
4588
                        const CallArgList &CallArgs)
4589
388k
      : AbstractAssumeAlignedAttrEmitter(CGF_, FuncDecl) {
4590
388k
    if (!AA)
4591
388k
      return;
4592
    // Alignment may or may not be a constant, and that is okay.
4593
62
    Alignment = CallArgs[AA->getParamIndex().getLLVMIndex()]
4594
62
                    .getRValue(CGF)
4595
62
                    .getScalarVal();
4596
62
  }
4597
};
4598
4599
} // namespace
4600
4601
RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
4602
                                 const CGCallee &Callee,
4603
                                 ReturnValueSlot ReturnValue,
4604
                                 const CallArgList &CallArgs,
4605
                                 llvm::CallBase **callOrInvoke, bool IsMustTail,
4606
388k
                                 SourceLocation Loc) {
4607
  // FIXME: We no longer need the types from CallArgs; lift up and simplify.
4608
4609
388k
  assert(Callee.isOrdinary() || Callee.isVirtual());
4610
4611
  // Handle struct-return functions by passing a pointer to the
4612
  // location that we would like to return into.
4613
0
  QualType RetTy = CallInfo.getReturnType();
4614
388k
  const ABIArgInfo &RetAI = CallInfo.getReturnInfo();
4615
4616
388k
  llvm::FunctionType *IRFuncTy = getTypes().GetFunctionType(CallInfo);
4617
4618
388k
  const Decl *TargetDecl = Callee.getAbstractInfo().getCalleeDecl().getDecl();
4619
388k
  if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(TargetDecl)) {
4620
    // We can only guarantee that a function is called from the correct
4621
    // context/function based on the appropriate target attributes,
4622
    // so only check in the case where we have both always_inline and target
4623
    // since otherwise we could be making a conditional call after a check for
4624
    // the proper cpu features (and it won't cause code generation issues due to
4625
    // function based code generation).
4626
366k
    if (TargetDecl->hasAttr<AlwaysInlineAttr>() &&
4627
366k
        
TargetDecl->hasAttr<TargetAttr>()41.2k
)
4628
13.3k
      checkTargetFeatures(Loc, FD);
4629
4630
    // Some architectures (such as x86-64) have the ABI changed based on
4631
    // attribute-target/features. Give them a chance to diagnose.
4632
366k
    CGM.getTargetCodeGenInfo().checkFunctionCallABI(
4633
366k
        CGM, Loc, dyn_cast_or_null<FunctionDecl>(CurCodeDecl), FD, CallArgs);
4634
366k
  }
4635
4636
388k
#ifndef NDEBUG
4637
388k
  if (!(CallInfo.isVariadic() && 
CallInfo.getArgStruct()78.6k
)) {
4638
    // For an inalloca varargs function, we don't expect CallInfo to match the
4639
    // function pointer's type, because the inalloca struct a will have extra
4640
    // fields in it for the varargs parameters.  Code later in this function
4641
    // bitcasts the function pointer to the type derived from CallInfo.
4642
    //
4643
    // In other cases, we assert that the types match up (until pointers stop
4644
    // having pointee types).
4645
388k
    llvm::Type *TypeFromVal;
4646
388k
    if (Callee.isVirtual())
4647
1.00k
      TypeFromVal = Callee.getVirtualFunctionType();
4648
387k
    else
4649
387k
      TypeFromVal =
4650
387k
          Callee.getFunctionPointer()->getType()->getPointerElementType();
4651
388k
    assert(IRFuncTy == TypeFromVal);
4652
388k
  }
4653
0
#endif
4654
4655
  // 1. Set up the arguments.
4656
4657
  // If we're using inalloca, insert the allocation after the stack save.
4658
  // FIXME: Do this earlier rather than hacking it in here!
4659
0
  Address ArgMemory = Address::invalid();
4660
388k
  if (llvm::StructType *ArgStruct = CallInfo.getArgStruct()) {
4661
72
    const llvm::DataLayout &DL = CGM.getDataLayout();
4662
72
    llvm::Instruction *IP = CallArgs.getStackBase();
4663
72
    llvm::AllocaInst *AI;
4664
72
    if (IP) {
4665
71
      IP = IP->getNextNode();
4666
71
      AI = new llvm::AllocaInst(ArgStruct, DL.getAllocaAddrSpace(),
4667
71
                                "argmem", IP);
4668
71
    } else {
4669
1
      AI = CreateTempAlloca(ArgStruct, "argmem");
4670
1
    }
4671
72
    auto Align = CallInfo.getArgStructAlignment();
4672
72
    AI->setAlignment(Align.getAsAlign());
4673
72
    AI->setUsedWithInAlloca(true);
4674
72
    assert(AI->isUsedWithInAlloca() && !AI->isStaticAlloca());
4675
0
    ArgMemory = Address(AI, Align);
4676
72
  }
4677
4678
0
  ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), CallInfo);
4679
388k
  SmallVector<llvm::Value *, 16> IRCallArgs(IRFunctionArgs.totalIRArgs());
4680
4681
  // If the call returns a temporary with struct return, create a temporary
4682
  // alloca to hold the result, unless one is given to us.
4683
388k
  Address SRetPtr = Address::invalid();
4684
388k
  Address SRetAlloca = Address::invalid();
4685
388k
  llvm::Value *UnusedReturnSizePtr = nullptr;
4686
388k
  if (RetAI.isIndirect() || 
RetAI.isInAlloca()386k
||
RetAI.isCoerceAndExpand()386k
) {
4687
2.09k
    if (!ReturnValue.isNull()) {
4688
1.87k
      SRetPtr = ReturnValue.getValue();
4689
1.87k
    } else {
4690
225
      SRetPtr = CreateMemTemp(RetTy, "tmp", &SRetAlloca);
4691
225
      if (HaveInsertPoint() && ReturnValue.isUnused()) {
4692
133
        llvm::TypeSize size =
4693
133
            CGM.getDataLayout().getTypeAllocSize(ConvertTypeForMem(RetTy));
4694
133
        UnusedReturnSizePtr = EmitLifetimeStart(size, SRetAlloca.getPointer());
4695
133
      }
4696
225
    }
4697
2.09k
    if (IRFunctionArgs.hasSRetArg()) {
4698
1.65k
      IRCallArgs[IRFunctionArgs.getSRetArgNo()] = SRetPtr.getPointer();
4699
1.65k
    } else 
if (446
RetAI.isInAlloca()446
) {
4700
4
      Address Addr =
4701
4
          Builder.CreateStructGEP(ArgMemory, RetAI.getInAllocaFieldIndex());
4702
4
      Builder.CreateStore(SRetPtr.getPointer(), Addr);
4703
4
    }
4704
2.09k
  }
4705
4706
388k
  Address swiftErrorTemp = Address::invalid();
4707
388k
  Address swiftErrorArg = Address::invalid();
4708
4709
  // When passing arguments using temporary allocas, we need to add the
4710
  // appropriate lifetime markers. This vector keeps track of all the lifetime
4711
  // markers that need to be ended right after the call.
4712
388k
  SmallVector<CallLifetimeEnd, 2> CallLifetimeEndAfterCall;
4713
4714
  // Translate all of the arguments as necessary to match the IR lowering.
4715
388k
  assert(CallInfo.arg_size() == CallArgs.size() &&
4716
388k
         "Mismatch between function signature & arguments.");
4717
0
  unsigned ArgNo = 0;
4718
388k
  CGFunctionInfo::const_arg_iterator info_it = CallInfo.arg_begin();
4719
388k
  for (CallArgList::const_iterator I = CallArgs.begin(), E = CallArgs.end();
4720
1.04M
       I != E; 
++I, ++info_it, ++ArgNo661k
) {
4721
661k
    const ABIArgInfo &ArgInfo = info_it->info;
4722
4723
    // Insert a padding argument to ensure proper alignment.
4724
661k
    if (IRFunctionArgs.hasPaddingArg(ArgNo))
4725
17
      IRCallArgs[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
4726
17
          llvm::UndefValue::get(ArgInfo.getPaddingType());
4727
4728
661k
    unsigned FirstIRArg, NumIRArgs;
4729
661k
    std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
4730
4731
661k
    switch (ArgInfo.getKind()) {
4732
156
    case ABIArgInfo::InAlloca: {
4733
156
      assert(NumIRArgs == 0);
4734
0
      assert(getTarget().getTriple().getArch() == llvm::Triple::x86);
4735
156
      if (I->isAggregate()) {
4736
103
        Address Addr = I->hasLValue()
4737
103
                           ? 
I->getKnownLValue().getAddress(*this)0
4738
103
                           : I->getKnownRValue().getAggregateAddress();
4739
103
        llvm::Instruction *Placeholder =
4740
103
            cast<llvm::Instruction>(Addr.getPointer());
4741
4742
103
        if (!ArgInfo.getInAllocaIndirect()) {
4743
          // Replace the placeholder with the appropriate argument slot GEP.
4744
101
          CGBuilderTy::InsertPoint IP = Builder.saveIP();
4745
101
          Builder.SetInsertPoint(Placeholder);
4746
101
          Addr = Builder.CreateStructGEP(ArgMemory,
4747
101
                                         ArgInfo.getInAllocaFieldIndex());
4748
101
          Builder.restoreIP(IP);
4749
101
        } else {
4750
          // For indirect things such as overaligned structs, replace the
4751
          // placeholder with a regular aggregate temporary alloca. Store the
4752
          // address of this alloca into the struct.
4753
2
          Addr = CreateMemTemp(info_it->type, "inalloca.indirect.tmp");
4754
2
          Address ArgSlot = Builder.CreateStructGEP(
4755
2
              ArgMemory, ArgInfo.getInAllocaFieldIndex());
4756
2
          Builder.CreateStore(Addr.getPointer(), ArgSlot);
4757
2
        }
4758
103
        deferPlaceholderReplacement(Placeholder, Addr.getPointer());
4759
103
      } else 
if (53
ArgInfo.getInAllocaIndirect()53
) {
4760
        // Make a temporary alloca and store the address of it into the argument
4761
        // struct.
4762
2
        Address Addr = CreateMemTempWithoutCast(
4763
2
            I->Ty, getContext().getTypeAlignInChars(I->Ty),
4764
2
            "indirect-arg-temp");
4765
2
        I->copyInto(*this, Addr);
4766
2
        Address ArgSlot =
4767
2
            Builder.CreateStructGEP(ArgMemory, ArgInfo.getInAllocaFieldIndex());
4768
2
        Builder.CreateStore(Addr.getPointer(), ArgSlot);
4769
51
      } else {
4770
        // Store the RValue into the argument struct.
4771
51
        Address Addr =
4772
51
            Builder.CreateStructGEP(ArgMemory, ArgInfo.getInAllocaFieldIndex());
4773
51
        unsigned AS = Addr.getType()->getPointerAddressSpace();
4774
51
        llvm::Type *MemType = ConvertTypeForMem(I->Ty)->getPointerTo(AS);
4775
        // There are some cases where a trivial bitcast is not avoidable.  The
4776
        // definition of a type later in a translation unit may change it's type
4777
        // from {}* to (%struct.foo*)*.
4778
51
        if (Addr.getType() != MemType)
4779
1
          Addr = Builder.CreateBitCast(Addr, MemType);
4780
51
        I->copyInto(*this, Addr);
4781
51
      }
4782
156
      break;
4783
0
    }
4784
4785
2.04k
    case ABIArgInfo::Indirect:
4786
2.04k
    case ABIArgInfo::IndirectAliased: {
4787
2.04k
      assert(NumIRArgs == 1);
4788
2.04k
      if (!I->isAggregate()) {
4789
        // Make a temporary alloca to pass the argument.
4790
232
        Address Addr = CreateMemTempWithoutCast(
4791
232
            I->Ty, ArgInfo.getIndirectAlign(), "indirect-arg-temp");
4792
232
        IRCallArgs[FirstIRArg] = Addr.getPointer();
4793
4794
232
        I->copyInto(*this, Addr);
4795
1.81k
      } else {
4796
        // We want to avoid creating an unnecessary temporary+copy here;
4797
        // however, we need one in three cases:
4798
        // 1. If the argument is not byval, and we are required to copy the
4799
        //    source.  (This case doesn't occur on any common architecture.)
4800
        // 2. If the argument is byval, RV is not sufficiently aligned, and
4801
        //    we cannot force it to be sufficiently aligned.
4802
        // 3. If the argument is byval, but RV is not located in default
4803
        //    or alloca address space.
4804
1.81k
        Address Addr = I->hasLValue()
4805
1.81k
                           ? 
I->getKnownLValue().getAddress(*this)319
4806
1.81k
                           : 
I->getKnownRValue().getAggregateAddress()1.49k
;
4807
1.81k
        llvm::Value *V = Addr.getPointer();
4808
1.81k
        CharUnits Align = ArgInfo.getIndirectAlign();
4809
1.81k
        const llvm::DataLayout *TD = &CGM.getDataLayout();
4810
4811
1.81k
        assert((FirstIRArg >= IRFuncTy->getNumParams() ||
4812
1.81k
                IRFuncTy->getParamType(FirstIRArg)->getPointerAddressSpace() ==
4813
1.81k
                    TD->getAllocaAddrSpace()) &&
4814
1.81k
               "indirect argument must be in alloca address space");
4815
4816
0
        bool NeedCopy = false;
4817
4818
1.81k
        if (Addr.getAlignment() < Align &&
4819
1.81k
            llvm::getOrEnforceKnownAlignment(V, Align.getAsAlign(), *TD) <
4820
92
                Align.getAsAlign()) {
4821
2
          NeedCopy = true;
4822
1.80k
        } else if (I->hasLValue()) {
4823
317
          auto LV = I->getKnownLValue();
4824
317
          auto AS = LV.getAddressSpace();
4825
4826
317
          if (!ArgInfo.getIndirectByVal() ||
4827
317
              
(LV.getAlignment() < getContext().getTypeAlignInChars(I->Ty))274
) {
4828
43
            NeedCopy = true;
4829
43
          }
4830
317
          if (!getLangOpts().OpenCL) {
4831
276
            if ((ArgInfo.getIndirectByVal() &&
4832
276
                
(233
AS != LangAS::Default233
&&
4833
233
                 
AS != CGM.getASTAllocaAddressSpace()0
))) {
4834
0
              NeedCopy = true;
4835
0
            }
4836
276
          }
4837
          // For OpenCL even if RV is located in default or alloca address space
4838
          // we don't want to perform address space cast for it.
4839
41
          else if ((ArgInfo.getIndirectByVal() &&
4840
41
                    Addr.getType()->getAddressSpace() != IRFuncTy->
4841
41
                      getParamType(FirstIRArg)->getPointerAddressSpace())) {
4842
18
            NeedCopy = true;
4843
18
          }
4844
317
        }
4845
4846
1.81k
        if (NeedCopy) {
4847
          // Create an aligned temporary, and copy to it.
4848
63
          Address AI = CreateMemTempWithoutCast(
4849
63
              I->Ty, ArgInfo.getIndirectAlign(), "byval-temp");
4850
63
          IRCallArgs[FirstIRArg] = AI.getPointer();
4851
4852
          // Emit lifetime markers for the temporary alloca.
4853
63
          llvm::TypeSize ByvalTempElementSize =
4854
63
              CGM.getDataLayout().getTypeAllocSize(AI.getElementType());
4855
63
          llvm::Value *LifetimeSize =
4856
63
              EmitLifetimeStart(ByvalTempElementSize, AI.getPointer());
4857
4858
          // Add cleanup code to emit the end lifetime marker after the call.
4859
63
          if (LifetimeSize) // In case we disabled lifetime markers.
4860
2
            CallLifetimeEndAfterCall.emplace_back(AI, LifetimeSize);
4861
4862
          // Generate the copy.
4863
63
          I->copyInto(*this, AI);
4864
1.74k
        } else {
4865
          // Skip the extra memcpy call.
4866
1.74k
          auto *T = V->getType()->getPointerElementType()->getPointerTo(
4867
1.74k
              CGM.getDataLayout().getAllocaAddrSpace());
4868
1.74k
          IRCallArgs[FirstIRArg] = getTargetHooks().performAddrSpaceCast(
4869
1.74k
              *this, V, LangAS::Default, CGM.getASTAllocaAddressSpace(), T,
4870
1.74k
              true);
4871
1.74k
        }
4872
1.81k
      }
4873
0
      break;
4874
2.04k
    }
4875
4876
7.24k
    case ABIArgInfo::Ignore:
4877
7.24k
      assert(NumIRArgs == 0);
4878
0
      break;
4879
4880
11.7k
    case ABIArgInfo::Extend:
4881
651k
    case ABIArgInfo::Direct: {
4882
651k
      if (!isa<llvm::StructType>(ArgInfo.getCoerceToType()) &&
4883
651k
          
ArgInfo.getCoerceToType() == ConvertType(info_it->type)650k
&&
4884
651k
          
ArgInfo.getDirectOffset() == 0646k
) {
4885
646k
        assert(NumIRArgs == 1);
4886
0
        llvm::Value *V;
4887
646k
        if (!I->isAggregate())
4888
646k
          V = I->getKnownRValue().getScalarVal();
4889
3
        else
4890
3
          V = Builder.CreateLoad(
4891
3
              I->hasLValue() ? 
I->getKnownLValue().getAddress(*this)0
4892
3
                             : I->getKnownRValue().getAggregateAddress());
4893
4894
        // Implement swifterror by copying into a new swifterror argument.
4895
        // We'll write back in the normal path out of the call.
4896
646k
        if (CallInfo.getExtParameterInfo(ArgNo).getABI()
4897
646k
              == ParameterABI::SwiftErrorResult) {
4898
8
          assert(!swiftErrorTemp.isValid() && "multiple swifterror args");
4899
4900
0
          QualType pointeeTy = I->Ty->getPointeeType();
4901
8
          swiftErrorArg =
4902
8
            Address(V, getContext().getTypeAlignInChars(pointeeTy));
4903
4904
8
          swiftErrorTemp =
4905
8
            CreateMemTemp(pointeeTy, getPointerAlign(), "swifterror.temp");
4906
8
          V = swiftErrorTemp.getPointer();
4907
8
          cast<llvm::AllocaInst>(V)->setSwiftError(true);
4908
4909
8
          llvm::Value *errorValue = Builder.CreateLoad(swiftErrorArg);
4910
8
          Builder.CreateStore(errorValue, swiftErrorTemp);
4911
8
        }
4912
4913
        // We might have to widen integers, but we should never truncate.
4914
646k
        if (ArgInfo.getCoerceToType() != V->getType() &&
4915
646k
            
V->getType()->isIntegerTy()690
)
4916
3
          V = Builder.CreateZExt(V, ArgInfo.getCoerceToType());
4917
4918
        // If the argument doesn't match, perform a bitcast to coerce it.  This
4919
        // can happen due to trivial type mismatches.
4920
646k
        if (FirstIRArg < IRFuncTy->getNumParams() &&
4921
646k
            
V->getType() != IRFuncTy->getParamType(FirstIRArg)557k
)
4922
687
          V = Builder.CreateBitCast(V, IRFuncTy->getParamType(FirstIRArg));
4923
4924
646k
        IRCallArgs[FirstIRArg] = V;
4925
646k
        break;
4926
646k
      }
4927
4928
      // FIXME: Avoid the conversion through memory if possible.
4929
4.64k
      Address Src = Address::invalid();
4930
4.64k
      if (!I->isAggregate()) {
4931
887
        Src = CreateMemTemp(I->Ty, "coerce");
4932
887
        I->copyInto(*this, Src);
4933
3.75k
      } else {
4934
3.75k
        Src = I->hasLValue() ? 
I->getKnownLValue().getAddress(*this)472
4935
3.75k
                             : 
I->getKnownRValue().getAggregateAddress()3.28k
;
4936
3.75k
      }
4937
4938
      // If the value is offset in memory, apply the offset now.
4939
4.64k
      Src = emitAddressAtOffset(*this, Src, ArgInfo);
4940
4941
      // Fast-isel and the optimizer generally like scalar values better than
4942
      // FCAs, so we flatten them if this is safe to do for this argument.
4943
4.64k
      llvm::StructType *STy =
4944
4.64k
            dyn_cast<llvm::StructType>(ArgInfo.getCoerceToType());
4945
4.64k
      if (STy && 
ArgInfo.isDirect()1.26k
&&
ArgInfo.getCanBeFlattened()1.26k
) {
4946
1.23k
        llvm::Type *SrcTy = Src.getElementType();
4947
1.23k
        uint64_t SrcSize = CGM.getDataLayout().getTypeAllocSize(SrcTy);
4948
1.23k
        uint64_t DstSize = CGM.getDataLayout().getTypeAllocSize(STy);
4949
4950
        // If the source type is smaller than the destination type of the
4951
        // coerce-to logic, copy the source value into a temp alloca the size
4952
        // of the destination type to allow loading all of it. The bits past
4953
        // the source value are left undef.
4954
1.23k
        if (SrcSize < DstSize) {
4955
154
          Address TempAlloca
4956
154
            = CreateTempAlloca(STy, Src.getAlignment(),
4957
154
                               Src.getName() + ".coerce");
4958
154
          Builder.CreateMemCpy(TempAlloca, Src, SrcSize);
4959
154
          Src = TempAlloca;
4960
1.08k
        } else {
4961
1.08k
          Src = Builder.CreateBitCast(Src,
4962
1.08k
                                      STy->getPointerTo(Src.getAddressSpace()));
4963
1.08k
        }
4964
4965
1.23k
        assert(NumIRArgs == STy->getNumElements());
4966
3.72k
        for (unsigned i = 0, e = STy->getNumElements(); i != e; 
++i2.48k
) {
4967
2.48k
          Address EltPtr = Builder.CreateStructGEP(Src, i);
4968
2.48k
          llvm::Value *LI = Builder.CreateLoad(EltPtr);
4969
2.48k
          IRCallArgs[FirstIRArg + i] = LI;
4970
2.48k
        }
4971
3.40k
      } else {
4972
        // In the simple case, just pass the coerced loaded value.
4973
3.40k
        assert(NumIRArgs == 1);
4974
0
        llvm::Value *Load =
4975
3.40k
            CreateCoercedLoad(Src, ArgInfo.getCoerceToType(), *this);
4976
4977
3.40k
        if (CallInfo.isCmseNSCall()) {
4978
          // For certain parameter types, clear padding bits, as they may reveal
4979
          // sensitive information.
4980
          // Small struct/union types are passed as integer arrays.
4981
40
          auto *ATy = dyn_cast<llvm::ArrayType>(Load->getType());
4982
40
          if (ATy != nullptr && 
isa<RecordType>(I->Ty.getCanonicalType())39
)
4983
39
            Load = EmitCMSEClearRecord(Load, ATy, I->Ty);
4984
40
        }
4985
3.40k
        IRCallArgs[FirstIRArg] = Load;
4986
3.40k
      }
4987
4988
0
      break;
4989
651k
    }
4990
4991
442
    case ABIArgInfo::CoerceAndExpand: {
4992
442
      auto coercionType = ArgInfo.getCoerceAndExpandType();
4993
442
      auto layout = CGM.getDataLayout().getStructLayout(coercionType);
4994
4995
442
      llvm::Value *tempSize = nullptr;
4996
442
      Address addr = Address::invalid();
4997
442
      Address AllocaAddr = Address::invalid();
4998
442
      if (I->isAggregate()) {
4999
414
        addr = I->hasLValue() ? 
I->getKnownLValue().getAddress(*this)0
5000
414
                              : I->getKnownRValue().getAggregateAddress();
5001
5002
414
      } else {
5003
28
        RValue RV = I->getKnownRValue();
5004
28
        assert(RV.isScalar()); // complex should always just be direct
5005
5006
0
        llvm::Type *scalarType = RV.getScalarVal()->getType();
5007
28
        auto scalarSize = CGM.getDataLayout().getTypeAllocSize(scalarType);
5008
28
        auto scalarAlign = CGM.getDataLayout().getPrefTypeAlignment(scalarType);
5009
5010
        // Materialize to a temporary.
5011
28
        addr = CreateTempAlloca(
5012
28
            RV.getScalarVal()->getType(),
5013
28
            CharUnits::fromQuantity(std::max(
5014
28
                (unsigned)layout->getAlignment().value(), scalarAlign)),
5015
28
            "tmp",
5016
28
            /*ArraySize=*/nullptr, &AllocaAddr);
5017
28
        tempSize = EmitLifetimeStart(scalarSize, AllocaAddr.getPointer());
5018
5019
28
        Builder.CreateStore(RV.getScalarVal(), addr);
5020
28
      }
5021
5022
0
      addr = Builder.CreateElementBitCast(addr, coercionType);
5023
5024
442
      unsigned IRArgPos = FirstIRArg;
5025
1.41k
      for (unsigned i = 0, e = coercionType->getNumElements(); i != e; 
++i973
) {
5026
973
        llvm::Type *eltType = coercionType->getElementType(i);
5027
973
        if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType)) 
continue7
;
5028
966
        Address eltAddr = Builder.CreateStructGEP(addr, i);
5029
966
        llvm::Value *elt = Builder.CreateLoad(eltAddr);
5030
966
        IRCallArgs[IRArgPos++] = elt;
5031
966
      }
5032
442
      assert(IRArgPos == FirstIRArg + NumIRArgs);
5033
5034
442
      if (tempSize) {
5035
0
        EmitLifetimeEnd(tempSize, AllocaAddr.getPointer());
5036
0
      }
5037
5038
442
      break;
5039
651k
    }
5040
5041
31
    case ABIArgInfo::Expand: {
5042
31
      unsigned IRArgPos = FirstIRArg;
5043
31
      ExpandTypeToArgs(I->Ty, *I, IRFuncTy, IRCallArgs, IRArgPos);
5044
31
      assert(IRArgPos == FirstIRArg + NumIRArgs);
5045
0
      break;
5046
651k
    }
5047
661k
    }
5048
661k
  }
5049
5050
388k
  const CGCallee &ConcreteCallee = Callee.prepareConcreteCallee(*this);
5051
388k
  llvm::Value *CalleePtr = ConcreteCallee.getFunctionPointer();
5052
5053
  // If we're using inalloca, set up that argument.
5054
388k
  if (ArgMemory.isValid()) {
5055
72
    llvm::Value *Arg = ArgMemory.getPointer();
5056
72
    if (CallInfo.isVariadic()) {
5057
      // When passing non-POD arguments by value to variadic functions, we will
5058
      // end up with a variadic prototype and an inalloca call site.  In such
5059
      // cases, we can't do any parameter mismatch checks.  Give up and bitcast
5060
      // the callee.
5061
10
      unsigned CalleeAS = CalleePtr->getType()->getPointerAddressSpace();
5062
10
      CalleePtr =
5063
10
          Builder.CreateBitCast(CalleePtr, IRFuncTy->getPointerTo(CalleeAS));
5064
62
    } else {
5065
62
      llvm::Type *LastParamTy =
5066
62
          IRFuncTy->getParamType(IRFuncTy->getNumParams() - 1);
5067
62
      if (Arg->getType() != LastParamTy) {
5068
0
#ifndef NDEBUG
5069
        // Assert that these structs have equivalent element types.
5070
0
        llvm::StructType *FullTy = CallInfo.getArgStruct();
5071
0
        llvm::StructType *DeclaredTy = cast<llvm::StructType>(
5072
0
            cast<llvm::PointerType>(LastParamTy)->getElementType());
5073
0
        assert(DeclaredTy->getNumElements() == FullTy->getNumElements());
5074
0
        for (llvm::StructType::element_iterator DI = DeclaredTy->element_begin(),
5075
0
                                                DE = DeclaredTy->element_end(),
5076