Coverage Report

Created: 2020-11-28 09:00

/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/CodeGen/CGCall.cpp
Line
Count
Source (jump to first uncovered line)
1
//===--- CGCall.cpp - Encapsulate calling convention details --------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// These classes wrap the information about a call or function
10
// definition used to handle ABI compliancy.
11
//
12
//===----------------------------------------------------------------------===//
13
14
#include "CGCall.h"
15
#include "ABIInfo.h"
16
#include "CGBlocks.h"
17
#include "CGCXXABI.h"
18
#include "CGCleanup.h"
19
#include "CGRecordLayout.h"
20
#include "CodeGenFunction.h"
21
#include "CodeGenModule.h"
22
#include "TargetInfo.h"
23
#include "clang/AST/Attr.h"
24
#include "clang/AST/Decl.h"
25
#include "clang/AST/DeclCXX.h"
26
#include "clang/AST/DeclObjC.h"
27
#include "clang/Basic/CodeGenOptions.h"
28
#include "clang/Basic/TargetBuiltins.h"
29
#include "clang/Basic/TargetInfo.h"
30
#include "clang/CodeGen/CGFunctionInfo.h"
31
#include "clang/CodeGen/SwiftCallingConv.h"
32
#include "llvm/ADT/StringExtras.h"
33
#include "llvm/Analysis/ValueTracking.h"
34
#include "llvm/IR/Attributes.h"
35
#include "llvm/IR/CallingConv.h"
36
#include "llvm/IR/DataLayout.h"
37
#include "llvm/IR/InlineAsm.h"
38
#include "llvm/IR/IntrinsicInst.h"
39
#include "llvm/IR/Intrinsics.h"
40
#include "llvm/Transforms/Utils/Local.h"
41
using namespace clang;
42
using namespace CodeGen;
43
44
/***/
45
46
208k
unsigned CodeGenTypes::ClangCallConvToLLVMCallConv(CallingConv CC) {
47
208k
  switch (CC) {
48
203k
  default: return llvm::CallingConv::C;
49
52
  case CC_X86StdCall: return llvm::CallingConv::X86_StdCall;
50
71
  case CC_X86FastCall: return llvm::CallingConv::X86_FastCall;
51
99
  case CC_X86RegCall: return llvm::CallingConv::X86_RegCall;
52
2.65k
  case CC_X86ThisCall: return llvm::CallingConv::X86_ThisCall;
53
14
  case CC_Win64: return llvm::CallingConv::Win64;
54
14
  case CC_X86_64SysV: return llvm::CallingConv::X86_64_SysV;
55
5
  case CC_AAPCS: return llvm::CallingConv::ARM_AAPCS;
56
11
  case CC_AAPCS_VFP: return llvm::CallingConv::ARM_AAPCS_VFP;
57
3
  case CC_IntelOclBicc: return llvm::CallingConv::Intel_OCL_BI;
58
  // TODO: Add support for __pascal to LLVM.
59
3
  case CC_X86Pascal: return llvm::CallingConv::C;
60
  // TODO: Add support for __vectorcall to LLVM.
61
105
  case CC_X86VectorCall: return llvm::CallingConv::X86_VectorCall;
62
2
  case CC_AArch64VectorCall: return llvm::CallingConv::AArch64_VectorCall;
63
269
  case CC_SpirFunction: return llvm::CallingConv::SPIR_FUNC;
64
335
  case CC_OpenCLKernel: return CGM.getTargetCodeGenInfo().getOpenCLKernelCallingConv();
65
10
  case CC_PreserveMost: return llvm::CallingConv::PreserveMost;
66
6
  case CC_PreserveAll: return llvm::CallingConv::PreserveAll;
67
1.13k
  case CC_Swift: return llvm::CallingConv::Swift;
68
208k
  }
69
208k
}
70
71
/// Derives the 'this' type for codegen purposes, i.e. ignoring method CVR
72
/// qualification. Either or both of RD and MD may be null. A null RD indicates
73
/// that there is no meaningful 'this' type, and a null MD can occur when
74
/// calling a method pointer.
75
CanQualType CodeGenTypes::DeriveThisType(const CXXRecordDecl *RD,
76
575k
                                         const CXXMethodDecl *MD) {
77
575k
  QualType RecTy;
78
575k
  if (RD)
79
574k
    RecTy = Context.getTagDeclType(RD)->getCanonicalTypeInternal();
80
688
  else
81
688
    RecTy = Context.VoidTy;
82
83
575k
  if (MD)
84
575k
    RecTy = Context.getAddrSpaceQualType(RecTy, MD->getMethodQualifiers().getAddressSpace());
85
575k
  return Context.getPointerType(CanQualType::CreateUnsafe(RecTy));
86
575k
}
87
88
/// Returns the canonical formal type of the given C++ method.
89
538k
static CanQual<FunctionProtoType> GetFormalType(const CXXMethodDecl *MD) {
90
538k
  return MD->getType()->getCanonicalTypeUnqualified()
91
538k
           .getAs<FunctionProtoType>();
92
538k
}
93
94
/// Returns the "extra-canonicalized" return type, which discards
95
/// qualifiers on the return type.  Codegen doesn't care about them,
96
/// and it makes ABI code a little easier to be able to assume that
97
/// all parameter and return types are top-level unqualified.
98
280k
static CanQualType GetReturnType(QualType RetTy) {
99
280k
  return RetTy->getCanonicalTypeUnqualified().getUnqualifiedType();
100
280k
}
101
102
/// Arrange the argument and result information for a value of the given
103
/// unprototyped freestanding function type.
104
const CGFunctionInfo &
105
580
CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionNoProtoType> FTNP) {
106
  // When translating an unprototyped function type, always use a
107
  // variadic type.
108
580
  return arrangeLLVMFunctionInfo(FTNP->getReturnType().getUnqualifiedType(),
109
580
                                 /*instanceMethod=*/false,
110
580
                                 /*chainCall=*/false, None,
111
580
                                 FTNP->getExtInfo(), {}, RequiredArgs(0));
112
580
}
113
114
static void addExtParameterInfosForCall(
115
         llvm::SmallVectorImpl<FunctionProtoType::ExtParameterInfo> &paramInfos,
116
                                        const FunctionProtoType *proto,
117
                                        unsigned prefixArgs,
118
701
                                        unsigned totalArgs) {
119
701
  assert(proto->hasExtParameterInfos());
120
701
  assert(paramInfos.size() <= prefixArgs);
121
701
  assert(proto->getNumParams() + prefixArgs <= totalArgs);
122
123
701
  paramInfos.reserve(totalArgs);
124
125
  // Add default infos for any prefix args that don't already have infos.
126
701
  paramInfos.resize(prefixArgs);
127
128
  // Add infos for the prototype.
129
1.33k
  for (const auto &ParamInfo : proto->getExtParameterInfos()) {
130
1.33k
    paramInfos.push_back(ParamInfo);
131
    // pass_object_size params have no parameter info.
132
1.33k
    if (ParamInfo.hasPassObjectSize())
133
447
      paramInfos.emplace_back();
134
1.33k
  }
135
136
701
  assert(paramInfos.size() <= totalArgs &&
137
701
         "Did we forget to insert pass_object_size args?");
138
  // Add default infos for the variadic and/or suffix arguments.
139
701
  paramInfos.resize(totalArgs);
140
701
}
141
142
/// Adds the formal parameters in FPT to the given prefix. If any parameter in
143
/// FPT has pass_object_size attrs, then we'll add parameters for those, too.
144
static void appendParameterTypes(const CodeGenTypes &CGT,
145
                                 SmallVectorImpl<CanQualType> &prefix,
146
              SmallVectorImpl<FunctionProtoType::ExtParameterInfo> &paramInfos,
147
913k
                                 CanQual<FunctionProtoType> FPT) {
148
  // Fast path: don't touch param info if we don't need to.
149
913k
  if (!FPT->hasExtParameterInfos()) {
150
912k
    assert(paramInfos.empty() &&
151
912k
           "We have paramInfos, but the prototype doesn't?");
152
912k
    prefix.append(FPT->param_type_begin(), FPT->param_type_end());
153
912k
    return;
154
912k
  }
155
156
497
  unsigned PrefixSize = prefix.size();
157
  // In the vast majority of cases, we'll have precisely FPT->getNumParams()
158
  // parameters; the only thing that can change this is the presence of
159
  // pass_object_size. So, we preallocate for the common case.
160
497
  prefix.reserve(prefix.size() + FPT->getNumParams());
161
162
497
  auto ExtInfos = FPT->getExtParameterInfos();
163
497
  assert(ExtInfos.size() == FPT->getNumParams());
164
1.47k
  for (unsigned I = 0, E = FPT->getNumParams(); I != E; 
++I974
) {
165
974
    prefix.push_back(FPT->getParamType(I));
166
974
    if (ExtInfos[I].hasPassObjectSize())
167
299
      prefix.push_back(CGT.getContext().getSizeType());
168
974
  }
169
170
497
  addExtParameterInfosForCall(paramInfos, FPT.getTypePtr(), PrefixSize,
171
497
                              prefix.size());
172
497
}
173
174
/// Arrange the LLVM function layout for a value of the given function
175
/// type, on top of any implicit parameters already stored.
176
static const CGFunctionInfo &
177
arrangeLLVMFunctionInfo(CodeGenTypes &CGT, bool instanceMethod,
178
                        SmallVectorImpl<CanQualType> &prefix,
179
632k
                        CanQual<FunctionProtoType> FTP) {
180
632k
  SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos;
181
632k
  RequiredArgs Required = RequiredArgs::forPrototypePlus(FTP, prefix.size());
182
  // FIXME: Kill copy.
183
632k
  appendParameterTypes(CGT, prefix, paramInfos, FTP);
184
632k
  CanQualType resultType = FTP->getReturnType().getUnqualifiedType();
185
186
632k
  return CGT.arrangeLLVMFunctionInfo(resultType, instanceMethod,
187
632k
                                     /*chainCall=*/false, prefix,
188
632k
                                     FTP->getExtInfo(), paramInfos,
189
632k
                                     Required);
190
632k
}
191
192
/// Arrange the argument and result information for a value of the
193
/// given freestanding function type.
194
const CGFunctionInfo &
195
431k
CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionProtoType> FTP) {
196
431k
  SmallVector<CanQualType, 16> argTypes;
197
431k
  return ::arrangeLLVMFunctionInfo(*this, /*instanceMethod=*/false, argTypes,
198
431k
                                   FTP);
199
431k
}
200
201
16.6k
static CallingConv getCallingConventionForDecl(const Decl *D, bool IsWindows) {
202
  // Set the appropriate calling convention for the Function.
203
16.6k
  if (D->hasAttr<StdCallAttr>())
204
2
    return CC_X86StdCall;
205
206
16.6k
  if (D->hasAttr<FastCallAttr>())
207
0
    return CC_X86FastCall;
208
209
16.6k
  if (D->hasAttr<RegCallAttr>())
210
0
    return CC_X86RegCall;
211
212
16.6k
  if (D->hasAttr<ThisCallAttr>())
213
0
    return CC_X86ThisCall;
214
215
16.6k
  if (D->hasAttr<VectorCallAttr>())
216
0
    return CC_X86VectorCall;
217
218
16.6k
  if (D->hasAttr<PascalAttr>())
219
0
    return CC_X86Pascal;
220
221
16.6k
  if (PcsAttr *PCS = D->getAttr<PcsAttr>())
222
0
    return (PCS->getPCS() == PcsAttr::AAPCS ? CC_AAPCS : CC_AAPCS_VFP);
223
224
16.6k
  if (D->hasAttr<AArch64VectorPcsAttr>())
225
0
    return CC_AArch64VectorCall;
226
227
16.6k
  if (D->hasAttr<IntelOclBiccAttr>())
228
0
    return CC_IntelOclBicc;
229
230
16.6k
  if (D->hasAttr<MSABIAttr>())
231
2
    return IsWindows ? 
CC_C0
: CC_Win64;
232
233
16.6k
  if (D->hasAttr<SysVABIAttr>())
234
0
    return IsWindows ? CC_X86_64SysV : CC_C;
235
236
16.6k
  if (D->hasAttr<PreserveMostAttr>())
237
0
    return CC_PreserveMost;
238
239
16.6k
  if (D->hasAttr<PreserveAllAttr>())
240
0
    return CC_PreserveAll;
241
242
16.6k
  return CC_C;
243
16.6k
}
244
245
/// Arrange the argument and result information for a call to an
246
/// unknown C++ non-static member function of the given abstract type.
247
/// (A null RD means we don't have any meaningful "this" argument type,
248
///  so fall back to a generic pointer type).
249
/// The member function must be an ordinary function, i.e. not a
250
/// constructor or destructor.
251
const CGFunctionInfo &
252
CodeGenTypes::arrangeCXXMethodType(const CXXRecordDecl *RD,
253
                                   const FunctionProtoType *FTP,
254
200k
                                   const CXXMethodDecl *MD) {
255
200k
  SmallVector<CanQualType, 16> argTypes;
256
257
  // Add the 'this' pointer.
258
200k
  argTypes.push_back(DeriveThisType(RD, MD));
259
260
200k
  return ::arrangeLLVMFunctionInfo(
261
200k
      *this, true, argTypes,
262
200k
      FTP->getCanonicalTypeUnqualified().getAs<FunctionProtoType>());
263
200k
}
264
265
/// Set calling convention for CUDA/HIP kernel.
266
static void setCUDAKernelCallingConvention(CanQualType &FTy, CodeGenModule &CGM,
267
593k
                                           const FunctionDecl *FD) {
268
593k
  if (FD->hasAttr<CUDAGlobalAttr>()) {
269
284
    const FunctionType *FT = FTy->getAs<FunctionType>();
270
284
    CGM.getTargetCodeGenInfo().setCUDAKernelCallingConvention(FT);
271
284
    FTy = FT->getCanonicalTypeUnqualified();
272
284
  }
273
593k
}
274
275
/// Arrange the argument and result information for a declaration or
276
/// definition of the given C++ non-static member function.  The
277
/// member function must be an ordinary function, i.e. not a
278
/// constructor or destructor.
279
const CGFunctionInfo &
280
206k
CodeGenTypes::arrangeCXXMethodDeclaration(const CXXMethodDecl *MD) {
281
206k
  assert(!isa<CXXConstructorDecl>(MD) && "wrong method for constructors!");
282
206k
  assert(!isa<CXXDestructorDecl>(MD) && "wrong method for destructors!");
283
284
206k
  CanQualType FT = GetFormalType(MD).getAs<Type>();
285
206k
  setCUDAKernelCallingConvention(FT, CGM, MD);
286
206k
  auto prototype = FT.getAs<FunctionProtoType>();
287
288
206k
  if (MD->isInstance()) {
289
    // The abstract case is perfectly fine.
290
200k
    const CXXRecordDecl *ThisType = TheCXXABI.getThisArgumentTypeForMethod(MD);
291
200k
    return arrangeCXXMethodType(ThisType, prototype.getTypePtr(), MD);
292
200k
  }
293
294
6.41k
  return arrangeFreeFunctionType(prototype);
295
6.41k
}
296
297
bool CodeGenTypes::inheritingCtorHasParams(
298
1.18k
    const InheritedConstructor &Inherited, CXXCtorType Type) {
299
  // Parameters are unnecessary if we're constructing a base class subobject
300
  // and the inherited constructor lives in a virtual base.
301
1.18k
  return Type == Ctor_Complete ||
302
798
         !Inherited.getShadowDecl()->constructsVirtualBase() ||
303
74
         !Target.getCXXABI().hasConstructorVariants();
304
1.18k
}
305
306
const CGFunctionInfo &
307
281k
CodeGenTypes::arrangeCXXStructorDeclaration(GlobalDecl GD) {
308
281k
  auto *MD = cast<CXXMethodDecl>(GD.getDecl());
309
310
281k
  SmallVector<CanQualType, 16> argTypes;
311
281k
  SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos;
312
281k
  argTypes.push_back(DeriveThisType(MD->getParent(), MD));
313
314
281k
  bool PassParams = true;
315
316
281k
  if (auto *CD = dyn_cast<CXXConstructorDecl>(MD)) {
317
    // A base class inheriting constructor doesn't get forwarded arguments
318
    // needed to construct a virtual base (or base class thereof).
319
166k
    if (auto Inherited = CD->getInheritedConstructor())
320
722
      PassParams = inheritingCtorHasParams(Inherited, GD.getCtorType());
321
166k
  }
322
323
281k
  CanQual<FunctionProtoType> FTP = GetFormalType(MD);
324
325
  // Add the formal parameters.
326
281k
  if (PassParams)
327
281k
    appendParameterTypes(*this, argTypes, paramInfos, FTP);
328
329
281k
  CGCXXABI::AddedStructorArgCounts AddedArgs =
330
281k
      TheCXXABI.buildStructorSignature(GD, argTypes);
331
281k
  if (!paramInfos.empty()) {
332
    // Note: prefix implies after the first param.
333
175
    if (AddedArgs.Prefix)
334
2
      paramInfos.insert(paramInfos.begin() + 1, AddedArgs.Prefix,
335
2
                        FunctionProtoType::ExtParameterInfo{});
336
175
    if (AddedArgs.Suffix)
337
20
      paramInfos.append(AddedArgs.Suffix,
338
20
                        FunctionProtoType::ExtParameterInfo{});
339
175
  }
340
341
281k
  RequiredArgs required =
342
281k
      (PassParams && 
MD->isVariadic()281k
?
RequiredArgs(argTypes.size())91
343
281k
                                      : RequiredArgs::All);
344
345
281k
  FunctionType::ExtInfo extInfo = FTP->getExtInfo();
346
281k
  CanQualType resultType = TheCXXABI.HasThisReturn(GD)
347
7.84k
                               ? argTypes.front()
348
273k
                               : TheCXXABI.hasMostDerivedReturn(GD)
349
1.42k
                                     ? CGM.getContext().VoidPtrTy
350
271k
                                     : Context.VoidTy;
351
281k
  return arrangeLLVMFunctionInfo(resultType, /*instanceMethod=*/true,
352
281k
                                 /*chainCall=*/false, argTypes, extInfo,
353
281k
                                 paramInfos, required);
354
281k
}
355
356
static SmallVector<CanQualType, 16>
357
68.7k
getArgTypesForCall(ASTContext &ctx, const CallArgList &args) {
358
68.7k
  SmallVector<CanQualType, 16> argTypes;
359
68.7k
  for (auto &arg : args)
360
97.1k
    argTypes.push_back(ctx.getCanonicalParamType(arg.Ty));
361
68.7k
  return argTypes;
362
68.7k
}
363
364
static SmallVector<CanQualType, 16>
365
29.8k
getArgTypesForDeclaration(ASTContext &ctx, const FunctionArgList &args) {
366
29.8k
  SmallVector<CanQualType, 16> argTypes;
367
29.8k
  for (auto &arg : args)
368
81.8k
    argTypes.push_back(ctx.getCanonicalParamType(arg->getType()));
369
29.8k
  return argTypes;
370
29.8k
}
371
372
static llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16>
373
getExtParameterInfosForCall(const FunctionProtoType *proto,
374
68.1k
                            unsigned prefixArgs, unsigned totalArgs) {
375
68.1k
  llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16> result;
376
68.1k
  if (proto->hasExtParameterInfos()) {
377
11
    addExtParameterInfosForCall(result, proto, prefixArgs, totalArgs);
378
11
  }
379
68.1k
  return result;
380
68.1k
}
381
382
/// Arrange a call to a C++ method, passing the given arguments.
383
///
384
/// ExtraPrefixArgs is the number of ABI-specific args passed after the `this`
385
/// parameter.
386
/// ExtraSuffixArgs is the number of ABI-specific args passed at the end of
387
/// args.
388
/// PassProtoArgs indicates whether `args` has args for the parameters in the
389
/// given CXXConstructorDecl.
390
const CGFunctionInfo &
391
CodeGenTypes::arrangeCXXConstructorCall(const CallArgList &args,
392
                                        const CXXConstructorDecl *D,
393
                                        CXXCtorType CtorKind,
394
                                        unsigned ExtraPrefixArgs,
395
                                        unsigned ExtraSuffixArgs,
396
50.9k
                                        bool PassProtoArgs) {
397
  // FIXME: Kill copy.
398
50.9k
  SmallVector<CanQualType, 16> ArgTypes;
399
50.9k
  for (const auto &Arg : args)
400
100k
    ArgTypes.push_back(Context.getCanonicalParamType(Arg.Ty));
401
402
  // +1 for implicit this, which should always be args[0].
403
50.9k
  unsigned TotalPrefixArgs = 1 + ExtraPrefixArgs;
404
405
50.9k
  CanQual<FunctionProtoType> FPT = GetFormalType(D);
406
50.9k
  RequiredArgs Required = PassProtoArgs
407
50.9k
                              ? RequiredArgs::forPrototypePlus(
408
50.9k
                                    FPT, TotalPrefixArgs + ExtraSuffixArgs)
409
10
                              : RequiredArgs::All;
410
411
50.9k
  GlobalDecl GD(D, CtorKind);
412
50.9k
  CanQualType ResultType = TheCXXABI.HasThisReturn(GD)
413
2.29k
                               ? ArgTypes.front()
414
48.6k
                               : TheCXXABI.hasMostDerivedReturn(GD)
415
0
                                     ? CGM.getContext().VoidPtrTy
416
48.6k
                                     : Context.VoidTy;
417
418
50.9k
  FunctionType::ExtInfo Info = FPT->getExtInfo();
419
50.9k
  llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16> ParamInfos;
420
  // If the prototype args are elided, we should only have ABI-specific args,
421
  // which never have param info.
422
50.9k
  if (PassProtoArgs && 
FPT->hasExtParameterInfos()50.9k
) {
423
    // ABI-specific suffix arguments are treated the same as variadic arguments.
424
70
    addExtParameterInfosForCall(ParamInfos, FPT.getTypePtr(), TotalPrefixArgs,
425
70
                                ArgTypes.size());
426
70
  }
427
50.9k
  return arrangeLLVMFunctionInfo(ResultType, /*instanceMethod=*/true,
428
50.9k
                                 /*chainCall=*/false, ArgTypes, Info,
429
50.9k
                                 ParamInfos, Required);
430
50.9k
}
431
432
/// Arrange the argument and result information for the declaration or
433
/// definition of the given function.
434
const CGFunctionInfo &
435
478k
CodeGenTypes::arrangeFunctionDeclaration(const FunctionDecl *FD) {
436
478k
  if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(FD))
437
105k
    if (MD->isInstance())
438
91.7k
      return arrangeCXXMethodDeclaration(MD);
439
440
387k
  CanQualType FTy = FD->getType()->getCanonicalTypeUnqualified();
441
442
387k
  assert(isa<FunctionType>(FTy));
443
387k
  setCUDAKernelCallingConvention(FTy, CGM, FD);
444
445
  // When declaring a function without a prototype, always use a
446
  // non-variadic type.
447
387k
  if (CanQual<FunctionNoProtoType> noProto = FTy.getAs<FunctionNoProtoType>()) {
448
14.5k
    return arrangeLLVMFunctionInfo(
449
14.5k
        noProto->getReturnType(), /*instanceMethod=*/false,
450
14.5k
        /*chainCall=*/false, None, noProto->getExtInfo(), {},RequiredArgs::All);
451
14.5k
  }
452
453
372k
  return arrangeFreeFunctionType(FTy.castAs<FunctionProtoType>());
454
372k
}
455
456
/// Arrange the argument and result information for the declaration or
457
/// definition of an Objective-C method.
458
const CGFunctionInfo &
459
5.16k
CodeGenTypes::arrangeObjCMethodDeclaration(const ObjCMethodDecl *MD) {
460
  // It happens that this is the same as a call with no optional
461
  // arguments, except also using the formal 'self' type.
462
5.16k
  return arrangeObjCMessageSendSignature(MD, MD->getSelfDecl()->getType());
463
5.16k
}
464
465
/// Arrange the argument and result information for the function type
466
/// through which to perform a send to the given Objective-C method,
467
/// using the given receiver type.  The receiver type is not always
468
/// the 'self' type of the method or even an Objective-C pointer type.
469
/// This is *not* the right method for actually performing such a
470
/// message send, due to the possibility of optional arguments.
471
const CGFunctionInfo &
472
CodeGenTypes::arrangeObjCMessageSendSignature(const ObjCMethodDecl *MD,
473
16.6k
                                              QualType receiverType) {
474
16.6k
  SmallVector<CanQualType, 16> argTys;
475
16.6k
  SmallVector<FunctionProtoType::ExtParameterInfo, 4> extParamInfos(2);
476
16.6k
  argTys.push_back(Context.getCanonicalParamType(receiverType));
477
16.6k
  argTys.push_back(Context.getCanonicalParamType(Context.getObjCSelType()));
478
  // FIXME: Kill copy?
479
14.1k
  for (const auto *I : MD->parameters()) {
480
14.1k
    argTys.push_back(Context.getCanonicalParamType(I->getType()));
481
14.1k
    auto extParamInfo = FunctionProtoType::ExtParameterInfo().withIsNoEscape(
482
14.1k
        I->hasAttr<NoEscapeAttr>());
483
14.1k
    extParamInfos.push_back(extParamInfo);
484
14.1k
  }
485
486
16.6k
  FunctionType::ExtInfo einfo;
487
16.6k
  bool IsWindows = getContext().getTargetInfo().getTriple().isOSWindows();
488
16.6k
  einfo = einfo.withCallingConv(getCallingConventionForDecl(MD, IsWindows));
489
490
16.6k
  if (getContext().getLangOpts().ObjCAutoRefCount &&
491
818
      MD->hasAttr<NSReturnsRetainedAttr>())
492
114
    einfo = einfo.withProducesResult(true);
493
494
16.6k
  RequiredArgs required =
495
15.7k
    (MD->isVariadic() ? 
RequiredArgs(argTys.size())929
: RequiredArgs::All);
496
497
16.6k
  return arrangeLLVMFunctionInfo(
498
16.6k
      GetReturnType(MD->getReturnType()), /*instanceMethod=*/false,
499
16.6k
      /*chainCall=*/false, argTys, einfo, extParamInfos, required);
500
16.6k
}
501
502
const CGFunctionInfo &
503
CodeGenTypes::arrangeUnprototypedObjCMessageSend(QualType returnType,
504
792
                                                 const CallArgList &args) {
505
792
  auto argTypes = getArgTypesForCall(Context, args);
506
792
  FunctionType::ExtInfo einfo;
507
508
792
  return arrangeLLVMFunctionInfo(
509
792
      GetReturnType(returnType), /*instanceMethod=*/false,
510
792
      /*chainCall=*/false, argTypes, einfo, {}, RequiredArgs::All);
511
792
}
512
513
const CGFunctionInfo &
514
540k
CodeGenTypes::arrangeGlobalDeclaration(GlobalDecl GD) {
515
  // FIXME: Do we need to handle ObjCMethodDecl?
516
540k
  const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
517
518
540k
  if (isa<CXXConstructorDecl>(GD.getDecl()) ||
519
500k
      isa<CXXDestructorDecl>(GD.getDecl()))
520
61.9k
    return arrangeCXXStructorDeclaration(GD);
521
522
478k
  return arrangeFunctionDeclaration(FD);
523
478k
}
524
525
/// Arrange a thunk that takes 'this' as the first parameter followed by
526
/// varargs.  Return a void pointer, regardless of the actual return type.
527
/// The body of the thunk will end in a musttail call to a function of the
528
/// correct type, and the caller will bitcast the function to the correct
529
/// prototype.
530
const CGFunctionInfo &
531
55
CodeGenTypes::arrangeUnprototypedMustTailThunk(const CXXMethodDecl *MD) {
532
55
  assert(MD->isVirtual() && "only methods have thunks");
533
55
  CanQual<FunctionProtoType> FTP = GetFormalType(MD);
534
55
  CanQualType ArgTys[] = {DeriveThisType(MD->getParent(), MD)};
535
55
  return arrangeLLVMFunctionInfo(Context.VoidTy, /*instanceMethod=*/false,
536
55
                                 /*chainCall=*/false, ArgTys,
537
55
                                 FTP->getExtInfo(), {}, RequiredArgs(1));
538
55
}
539
540
const CGFunctionInfo &
541
CodeGenTypes::arrangeMSCtorClosure(const CXXConstructorDecl *CD,
542
35
                                   CXXCtorType CT) {
543
35
  assert(CT == Ctor_CopyingClosure || CT == Ctor_DefaultClosure);
544
545
35
  CanQual<FunctionProtoType> FTP = GetFormalType(CD);
546
35
  SmallVector<CanQualType, 2> ArgTys;
547
35
  const CXXRecordDecl *RD = CD->getParent();
548
35
  ArgTys.push_back(DeriveThisType(RD, CD));
549
35
  if (CT == Ctor_CopyingClosure)
550
6
    ArgTys.push_back(*FTP->param_type_begin());
551
35
  if (RD->getNumVBases() > 0)
552
0
    ArgTys.push_back(Context.IntTy);
553
35
  CallingConv CC = Context.getDefaultCallingConvention(
554
35
      /*IsVariadic=*/false, /*IsCXXMethod=*/true);
555
35
  return arrangeLLVMFunctionInfo(Context.VoidTy, /*instanceMethod=*/true,
556
35
                                 /*chainCall=*/false, ArgTys,
557
35
                                 FunctionType::ExtInfo(CC), {},
558
35
                                 RequiredArgs::All);
559
35
}
560
561
/// Arrange a call as unto a free function, except possibly with an
562
/// additional number of formal parameters considered required.
563
static const CGFunctionInfo &
564
arrangeFreeFunctionLikeCall(CodeGenTypes &CGT,
565
                            CodeGenModule &CGM,
566
                            const CallArgList &args,
567
                            const FunctionType *fnType,
568
                            unsigned numExtraRequiredArgs,
569
165k
                            bool chainCall) {
570
165k
  assert(args.size() >= numExtraRequiredArgs);
571
572
165k
  llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos;
573
574
  // In most cases, there are no optional arguments.
575
165k
  RequiredArgs required = RequiredArgs::All;
576
577
  // If we have a variadic prototype, the required arguments are the
578
  // extra prefix plus the arguments in the prototype.
579
165k
  if (const FunctionProtoType *proto = dyn_cast<FunctionProtoType>(fnType)) {
580
164k
    if (proto->isVariadic())
581
44.1k
      required = RequiredArgs::forPrototypePlus(proto, numExtraRequiredArgs);
582
583
164k
    if (proto->hasExtParameterInfos())
584
123
      addExtParameterInfosForCall(paramInfos, proto, numExtraRequiredArgs,
585
123
                                  args.size());
586
587
  // If we don't have a prototype at all, but we're supposed to
588
  // explicitly use the variadic convention for unprototyped calls,
589
  // treat all of the arguments as required but preserve the nominal
590
  // possibility of variadics.
591
864
  } else if (CGM.getTargetCodeGenInfo()
592
864
                .isNoProtoCallVariadic(args,
593
678
                                       cast<FunctionNoProtoType>(fnType))) {
594
678
    required = RequiredArgs(args.size());
595
678
  }
596
597
  // FIXME: Kill copy.
598
165k
  SmallVector<CanQualType, 16> argTypes;
599
165k
  for (const auto &arg : args)
600
282k
    argTypes.push_back(CGT.getContext().getCanonicalParamType(arg.Ty));
601
165k
  return CGT.arrangeLLVMFunctionInfo(GetReturnType(fnType->getReturnType()),
602
165k
                                     /*instanceMethod=*/false, chainCall,
603
165k
                                     argTypes, fnType->getExtInfo(), paramInfos,
604
165k
                                     required);
605
165k
}
606
607
/// Figure out the rules for calling a function with the given formal
608
/// type using the given arguments.  The arguments are necessary
609
/// because the function might be unprototyped, in which case it's
610
/// target-dependent in crazy ways.
611
const CGFunctionInfo &
612
CodeGenTypes::arrangeFreeFunctionCall(const CallArgList &args,
613
                                      const FunctionType *fnType,
614
164k
                                      bool chainCall) {
615
164k
  return arrangeFreeFunctionLikeCall(*this, CGM, args, fnType,
616
164k
                                     chainCall ? 
18
: 0, chainCall);
617
164k
}
618
619
/// A block function is essentially a free function with an
620
/// extra implicit argument.
621
const CGFunctionInfo &
622
CodeGenTypes::arrangeBlockFunctionCall(const CallArgList &args,
623
570
                                       const FunctionType *fnType) {
624
570
  return arrangeFreeFunctionLikeCall(*this, CGM, args, fnType, 1,
625
570
                                     /*chainCall=*/false);
626
570
}
627
628
const CGFunctionInfo &
629
CodeGenTypes::arrangeBlockFunctionDeclaration(const FunctionProtoType *proto,
630
1.04k
                                              const FunctionArgList &params) {
631
1.04k
  auto paramInfos = getExtParameterInfosForCall(proto, 1, params.size());
632
1.04k
  auto argTypes = getArgTypesForDeclaration(Context, params);
633
634
1.04k
  return arrangeLLVMFunctionInfo(GetReturnType(proto->getReturnType()),
635
1.04k
                                 /*instanceMethod*/ false, /*chainCall*/ false,
636
1.04k
                                 argTypes, proto->getExtInfo(), paramInfos,
637
1.04k
                                 RequiredArgs::forPrototypePlus(proto, 1));
638
1.04k
}
639
640
const CGFunctionInfo &
641
CodeGenTypes::arrangeBuiltinFunctionCall(QualType resultType,
642
951
                                         const CallArgList &args) {
643
  // FIXME: Kill copy.
644
951
  SmallVector<CanQualType, 16> argTypes;
645
951
  for (const auto &Arg : args)
646
3.44k
    argTypes.push_back(Context.getCanonicalParamType(Arg.Ty));
647
951
  return arrangeLLVMFunctionInfo(
648
951
      GetReturnType(resultType), /*instanceMethod=*/false,
649
951
      /*chainCall=*/false, argTypes, FunctionType::ExtInfo(),
650
951
      /*paramInfos=*/ {}, RequiredArgs::All);
651
951
}
652
653
const CGFunctionInfo &
654
CodeGenTypes::arrangeBuiltinFunctionDeclaration(QualType resultType,
655
28.7k
                                                const FunctionArgList &args) {
656
28.7k
  auto argTypes = getArgTypesForDeclaration(Context, args);
657
658
28.7k
  return arrangeLLVMFunctionInfo(
659
28.7k
      GetReturnType(resultType), /*instanceMethod=*/false, /*chainCall=*/false,
660
28.7k
      argTypes, FunctionType::ExtInfo(), {}, RequiredArgs::All);
661
28.7k
}
662
663
const CGFunctionInfo &
664
CodeGenTypes::arrangeBuiltinFunctionDeclaration(CanQualType resultType,
665
384
                                              ArrayRef<CanQualType> argTypes) {
666
384
  return arrangeLLVMFunctionInfo(
667
384
      resultType, /*instanceMethod=*/false, /*chainCall=*/false,
668
384
      argTypes, FunctionType::ExtInfo(), {}, RequiredArgs::All);
669
384
}
670
671
/// Arrange a call to a C++ method, passing the given arguments.
672
///
673
/// numPrefixArgs is the number of ABI-specific prefix arguments we have. It
674
/// does not count `this`.
675
const CGFunctionInfo &
676
CodeGenTypes::arrangeCXXMethodCall(const CallArgList &args,
677
                                   const FunctionProtoType *proto,
678
                                   RequiredArgs required,
679
67.0k
                                   unsigned numPrefixArgs) {
680
67.0k
  assert(numPrefixArgs + 1 <= args.size() &&
681
67.0k
         "Emitting a call with less args than the required prefix?");
682
  // Add one to account for `this`. It's a bit awkward here, but we don't count
683
  // `this` in similar places elsewhere.
684
67.0k
  auto paramInfos =
685
67.0k
    getExtParameterInfosForCall(proto, numPrefixArgs + 1, args.size());
686
687
  // FIXME: Kill copy.
688
67.0k
  auto argTypes = getArgTypesForCall(Context, args);
689
690
67.0k
  FunctionType::ExtInfo info = proto->getExtInfo();
691
67.0k
  return arrangeLLVMFunctionInfo(
692
67.0k
      GetReturnType(proto->getReturnType()), /*instanceMethod=*/true,
693
67.0k
      /*chainCall=*/false, argTypes, info, paramInfos, required);
694
67.0k
}
695
696
17.3k
const CGFunctionInfo &CodeGenTypes::arrangeNullaryFunction() {
697
17.3k
  return arrangeLLVMFunctionInfo(
698
17.3k
      getContext().VoidTy, /*instanceMethod=*/false, /*chainCall=*/false,
699
17.3k
      None, FunctionType::ExtInfo(), {}, RequiredArgs::All);
700
17.3k
}
701
702
const CGFunctionInfo &
703
CodeGenTypes::arrangeCall(const CGFunctionInfo &signature,
704
11.4k
                          const CallArgList &args) {
705
11.4k
  assert(signature.arg_size() <= args.size());
706
11.4k
  if (signature.arg_size() == args.size())
707
10.5k
    return signature;
708
709
911
  SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos;
710
911
  auto sigParamInfos = signature.getExtParameterInfos();
711
911
  if (!sigParamInfos.empty()) {
712
911
    paramInfos.append(sigParamInfos.begin(), sigParamInfos.end());
713
911
    paramInfos.resize(args.size());
714
911
  }
715
716
911
  auto argTypes = getArgTypesForCall(Context, args);
717
718
911
  assert(signature.getRequiredArgs().allowsOptionalArgs());
719
911
  return arrangeLLVMFunctionInfo(signature.getReturnType(),
720
911
                                 signature.isInstanceMethod(),
721
911
                                 signature.isChainCall(),
722
911
                                 argTypes,
723
911
                                 signature.getExtInfo(),
724
911
                                 paramInfos,
725
911
                                 signature.getRequiredArgs());
726
911
}
727
728
namespace clang {
729
namespace CodeGen {
730
void computeSPIRKernelABIInfo(CodeGenModule &CGM, CGFunctionInfo &FI);
731
}
732
}
733
734
/// Arrange the argument and result information for an abstract value
735
/// of a given function type.  This is the method which all of the
736
/// above functions ultimately defer to.
737
const CGFunctionInfo &
738
CodeGenTypes::arrangeLLVMFunctionInfo(CanQualType resultType,
739
                                      bool instanceMethod,
740
                                      bool chainCall,
741
                                      ArrayRef<CanQualType> argTypes,
742
                                      FunctionType::ExtInfo info,
743
                     ArrayRef<FunctionProtoType::ExtParameterInfo> paramInfos,
744
1.27M
                                      RequiredArgs required) {
745
1.27M
  assert(llvm::all_of(argTypes,
746
1.27M
                      [](CanQualType T) { return T.isCanonicalAsParam(); }));
747
748
  // Lookup or create unique function info.
749
1.27M
  llvm::FoldingSetNodeID ID;
750
1.27M
  CGFunctionInfo::Profile(ID, instanceMethod, chainCall, info, paramInfos,
751
1.27M
                          required, resultType, argTypes);
752
753
1.27M
  void *insertPos = nullptr;
754
1.27M
  CGFunctionInfo *FI = FunctionInfos.FindNodeOrInsertPos(ID, insertPos);
755
1.27M
  if (FI)
756
1.07M
    return *FI;
757
758
208k
  unsigned CC = ClangCallConvToLLVMCallConv(info.getCC());
759
760
  // Construct the function info.  We co-allocate the ArgInfos.
761
208k
  FI = CGFunctionInfo::create(CC, instanceMethod, chainCall, info,
762
208k
                              paramInfos, resultType, argTypes, required);
763
208k
  FunctionInfos.InsertNode(FI, insertPos);
764
765
208k
  bool inserted = FunctionsBeingProcessed.insert(FI).second;
766
208k
  (void)inserted;
767
208k
  assert(inserted && "Recursively being processed?");
768
769
  // Compute ABI information.
770
208k
  if (CC == llvm::CallingConv::SPIR_KERNEL) {
771
    // Force target independent argument handling for the host visible
772
    // kernel functions.
773
129
    computeSPIRKernelABIInfo(CGM, *FI);
774
208k
  } else if (info.getCC() == CC_Swift) {
775
1.13k
    swiftcall::computeABIInfo(CGM, *FI);
776
207k
  } else {
777
207k
    getABIInfo().computeInfo(*FI);
778
207k
  }
779
780
  // Loop over all of the computed argument and return value info.  If any of
781
  // them are direct or extend without a specified coerce type, specify the
782
  // default now.
783
208k
  ABIArgInfo &retInfo = FI->getReturnInfo();
784
208k
  if (retInfo.canHaveCoerceToType() && 
retInfo.getCoerceToType() == nullptr115k
)
785
36.7k
    retInfo.setCoerceToType(ConvertType(FI->getReturnType()));
786
787
208k
  for (auto &I : FI->arguments())
788
373k
    if (I.info.canHaveCoerceToType() && 
I.info.getCoerceToType() == nullptr364k
)
789
126k
      I.info.setCoerceToType(ConvertType(I.type));
790
791
208k
  bool erased = FunctionsBeingProcessed.erase(FI); (void)erased;
792
208k
  assert(erased && "Not in set?");
793
794
208k
  return *FI;
795
208k
}
796
797
CGFunctionInfo *CGFunctionInfo::create(unsigned llvmCC,
798
                                       bool instanceMethod,
799
                                       bool chainCall,
800
                                       const FunctionType::ExtInfo &info,
801
                                       ArrayRef<ExtParameterInfo> paramInfos,
802
                                       CanQualType resultType,
803
                                       ArrayRef<CanQualType> argTypes,
804
208k
                                       RequiredArgs required) {
805
208k
  assert(paramInfos.empty() || paramInfos.size() == argTypes.size());
806
208k
  assert(!required.allowsOptionalArgs() ||
807
208k
         required.getNumRequiredArgs() <= argTypes.size());
808
809
208k
  void *buffer =
810
208k
    operator new(totalSizeToAlloc<ArgInfo,             ExtParameterInfo>(
811
208k
                                  argTypes.size() + 1, paramInfos.size()));
812
813
208k
  CGFunctionInfo *FI = new(buffer) CGFunctionInfo();
814
208k
  FI->CallingConvention = llvmCC;
815
208k
  FI->EffectiveCallingConvention = llvmCC;
816
208k
  FI->ASTCallingConvention = info.getCC();
817
208k
  FI->InstanceMethod = instanceMethod;
818
208k
  FI->ChainCall = chainCall;
819
208k
  FI->CmseNSCall = info.getCmseNSCall();
820
208k
  FI->NoReturn = info.getNoReturn();
821
208k
  FI->ReturnsRetained = info.getProducesResult();
822
208k
  FI->NoCallerSavedRegs = info.getNoCallerSavedRegs();
823
208k
  FI->NoCfCheck = info.getNoCfCheck();
824
208k
  FI->Required = required;
825
208k
  FI->HasRegParm = info.getHasRegParm();
826
208k
  FI->RegParm = info.getRegParm();
827
208k
  FI->ArgStruct = nullptr;
828
208k
  FI->ArgStructAlign = 0;
829
208k
  FI->NumArgs = argTypes.size();
830
208k
  FI->HasExtParameterInfos = !paramInfos.empty();
831
208k
  FI->getArgsBuffer()[0].type = resultType;
832
582k
  for (unsigned i = 0, e = argTypes.size(); i != e; 
++i373k
)
833
373k
    FI->getArgsBuffer()[i + 1].type = argTypes[i];
834
228k
  for (unsigned i = 0, e = paramInfos.size(); i != e; 
++i20.4k
)
835
20.4k
    FI->getExtParameterInfosBuffer()[i] = paramInfos[i];
836
208k
  return FI;
837
208k
}
838
839
/***/
840
841
namespace {
842
// ABIArgInfo::Expand implementation.
843
844
// Specifies the way QualType passed as ABIArgInfo::Expand is expanded.
845
struct TypeExpansion {
846
  enum TypeExpansionKind {
847
    // Elements of constant arrays are expanded recursively.
848
    TEK_ConstantArray,
849
    // Record fields are expanded recursively (but if record is a union, only
850
    // the field with the largest size is expanded).
851
    TEK_Record,
852
    // For complex types, real and imaginary parts are expanded recursively.
853
    TEK_Complex,
854
    // All other types are not expandable.
855
    TEK_None
856
  };
857
858
  const TypeExpansionKind Kind;
859
860
2.27k
  TypeExpansion(TypeExpansionKind K) : Kind(K) {}
861
2.27k
  virtual ~TypeExpansion() {}
862
};
863
864
struct ConstantArrayExpansion : TypeExpansion {
865
  QualType EltTy;
866
  uint64_t NumElts;
867
868
  ConstantArrayExpansion(QualType EltTy, uint64_t NumElts)
869
40
      : TypeExpansion(TEK_ConstantArray), EltTy(EltTy), NumElts(NumElts) {}
870
2.31k
  static bool classof(const TypeExpansion *TE) {
871
2.31k
    return TE->Kind == TEK_ConstantArray;
872
2.31k
  }
873
};
874
875
struct RecordExpansion : TypeExpansion {
876
  SmallVector<const CXXBaseSpecifier *, 1> Bases;
877
878
  SmallVector<const FieldDecl *, 1> Fields;
879
880
  RecordExpansion(SmallVector<const CXXBaseSpecifier *, 1> &&Bases,
881
                  SmallVector<const FieldDecl *, 1> &&Fields)
882
      : TypeExpansion(TEK_Record), Bases(std::move(Bases)),
883
721
        Fields(std::move(Fields)) {}
884
2.95k
  static bool classof(const TypeExpansion *TE) {
885
2.95k
    return TE->Kind == TEK_Record;
886
2.95k
  }
887
};
888
889
struct ComplexExpansion : TypeExpansion {
890
  QualType EltTy;
891
892
46
  ComplexExpansion(QualType EltTy) : TypeExpansion(TEK_Complex), EltTy(EltTy) {}
893
1.52k
  static bool classof(const TypeExpansion *TE) {
894
1.52k
    return TE->Kind == TEK_Complex;
895
1.52k
  }
896
};
897
898
struct NoExpansion : TypeExpansion {
899
1.46k
  NoExpansion() : TypeExpansion(TEK_None) {}
900
1.46k
  static bool classof(const TypeExpansion *TE) {
901
1.46k
    return TE->Kind == TEK_None;
902
1.46k
  }
903
};
904
}  // namespace
905
906
static std::unique_ptr<TypeExpansion>
907
2.27k
getTypeExpansion(QualType Ty, const ASTContext &Context) {
908
2.27k
  if (const ConstantArrayType *AT = Context.getAsConstantArrayType(Ty)) {
909
40
    return std::make_unique<ConstantArrayExpansion>(
910
40
        AT->getElementType(), AT->getSize().getZExtValue());
911
40
  }
912
2.23k
  if (const RecordType *RT = Ty->getAs<RecordType>()) {
913
721
    SmallVector<const CXXBaseSpecifier *, 1> Bases;
914
721
    SmallVector<const FieldDecl *, 1> Fields;
915
721
    const RecordDecl *RD = RT->getDecl();
916
721
    assert(!RD->hasFlexibleArrayMember() &&
917
721
           "Cannot expand structure with flexible array.");
918
721
    if (RD->isUnion()) {
919
      // Unions can be here only in degenerative cases - all the fields are same
920
      // after flattening. Thus we have to use the "largest" field.
921
5
      const FieldDecl *LargestFD = nullptr;
922
5
      CharUnits UnionSize = CharUnits::Zero();
923
924
10
      for (const auto *FD : RD->fields()) {
925
10
        if (FD->isZeroLengthBitField(Context))
926
0
          continue;
927
10
        assert(!FD->isBitField() &&
928
10
               "Cannot expand structure with bit-field members.");
929
10
        CharUnits FieldSize = Context.getTypeSizeInChars(FD->getType());
930
10
        if (UnionSize < FieldSize) {
931
5
          UnionSize = FieldSize;
932
5
          LargestFD = FD;
933
5
        }
934
10
      }
935
5
      if (LargestFD)
936
5
        Fields.push_back(LargestFD);
937
716
    } else {
938
716
      if (const auto *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
939
201
        assert(!CXXRD->isDynamicClass() &&
940
201
               "cannot expand vtable pointers in dynamic classes");
941
201
        for (const CXXBaseSpecifier &BS : CXXRD->bases())
942
5
          Bases.push_back(&BS);
943
201
      }
944
945
1.42k
      for (const auto *FD : RD->fields()) {
946
1.42k
        if (FD->isZeroLengthBitField(Context))
947
0
          continue;
948
1.42k
        assert(!FD->isBitField() &&
949
1.42k
               "Cannot expand structure with bit-field members.");
950
1.42k
        Fields.push_back(FD);
951
1.42k
      }
952
716
    }
953
721
    return std::make_unique<RecordExpansion>(std::move(Bases),
954
721
                                              std::move(Fields));
955
721
  }
956
1.50k
  if (const ComplexType *CT = Ty->getAs<ComplexType>()) {
957
46
    return std::make_unique<ComplexExpansion>(CT->getElementType());
958
46
  }
959
1.46k
  return std::make_unique<NoExpansion>();
960
1.46k
}
961
962
1.33k
static int getExpansionSize(QualType Ty, const ASTContext &Context) {
963
1.33k
  auto Exp = getTypeExpansion(Ty, Context);
964
1.33k
  if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
965
24
    return CAExp->NumElts * getExpansionSize(CAExp->EltTy, Context);
966
24
  }
967
1.31k
  if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
968
433
    int Res = 0;
969
433
    for (auto BS : RExp->Bases)
970
3
      Res += getExpansionSize(BS->getType(), Context);
971
433
    for (auto FD : RExp->Fields)
972
856
      Res += getExpansionSize(FD->getType(), Context);
973
433
    return Res;
974
433
  }
975
877
  if (isa<ComplexExpansion>(Exp.get()))
976
28
    return 2;
977
849
  assert(isa<NoExpansion>(Exp.get()));
978
849
  return 1;
979
849
}
980
981
void
982
CodeGenTypes::getExpandedTypes(QualType Ty,
983
518
                               SmallVectorImpl<llvm::Type *>::iterator &TI) {
984
518
  auto Exp = getTypeExpansion(Ty, Context);
985
518
  if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
986
40
    for (int i = 0, n = CAExp->NumElts; i < n; 
i++32
) {
987
32
      getExpandedTypes(CAExp->EltTy, TI);
988
32
    }
989
510
  } else if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
990
162
    for (auto BS : RExp->Bases)
991
1
      getExpandedTypes(BS->getType(), TI);
992
162
    for (auto FD : RExp->Fields)
993
316
      getExpandedTypes(FD->getType(), TI);
994
348
  } else if (auto CExp = dyn_cast<ComplexExpansion>(Exp.get())) {
995
11
    llvm::Type *EltTy = ConvertType(CExp->EltTy);
996
11
    *TI++ = EltTy;
997
11
    *TI++ = EltTy;
998
337
  } else {
999
337
    assert(isa<NoExpansion>(Exp.get()));
1000
337
    *TI++ = ConvertType(Ty);
1001
337
  }
1002
518
}
1003
1004
static void forConstantArrayExpansion(CodeGenFunction &CGF,
1005
                                      ConstantArrayExpansion *CAE,
1006
                                      Address BaseAddr,
1007
8
                                      llvm::function_ref<void(Address)> Fn) {
1008
8
  CharUnits EltSize = CGF.getContext().getTypeSizeInChars(CAE->EltTy);
1009
8
  CharUnits EltAlign =
1010
8
    BaseAddr.getAlignment().alignmentOfArrayElement(EltSize);
1011
1012
40
  for (int i = 0, n = CAE->NumElts; i < n; 
i++32
) {
1013
32
    llvm::Value *EltAddr =
1014
32
      CGF.Builder.CreateConstGEP2_32(nullptr, BaseAddr.getPointer(), 0, i);
1015
32
    Fn(Address(EltAddr, EltAlign));
1016
32
  }
1017
8
}
1018
1019
void CodeGenFunction::ExpandTypeFromArgs(QualType Ty, LValue LV,
1020
344
                                         llvm::Function::arg_iterator &AI) {
1021
344
  assert(LV.isSimple() &&
1022
344
         "Unexpected non-simple lvalue during struct expansion.");
1023
1024
344
  auto Exp = getTypeExpansion(Ty, getContext());
1025
344
  if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
1026
8
    forConstantArrayExpansion(
1027
32
        *this, CAExp, LV.getAddress(*this), [&](Address EltAddr) {
1028
32
          LValue LV = MakeAddrLValue(EltAddr, CAExp->EltTy);
1029
32
          ExpandTypeFromArgs(CAExp->EltTy, LV, AI);
1030
32
        });
1031
336
  } else if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
1032
99
    Address This = LV.getAddress(*this);
1033
1
    for (const CXXBaseSpecifier *BS : RExp->Bases) {
1034
      // Perform a single step derived-to-base conversion.
1035
1
      Address Base =
1036
1
          GetAddressOfBaseClass(This, Ty->getAsCXXRecordDecl(), &BS, &BS + 1,
1037
1
                                /*NullCheckValue=*/false, SourceLocation());
1038
1
      LValue SubLV = MakeAddrLValue(Base, BS->getType());
1039
1040
      // Recurse onto bases.
1041
1
      ExpandTypeFromArgs(BS->getType(), SubLV, AI);
1042
1
    }
1043
211
    for (auto FD : RExp->Fields) {
1044
      // FIXME: What are the right qualifiers here?
1045
211
      LValue SubLV = EmitLValueForFieldInitialization(LV, FD);
1046
211
      ExpandTypeFromArgs(FD->getType(), SubLV, AI);
1047
211
    }
1048
237
  } else if (isa<ComplexExpansion>(Exp.get())) {
1049
3
    auto realValue = &*AI++;
1050
3
    auto imagValue = &*AI++;
1051
3
    EmitStoreOfComplex(ComplexPairTy(realValue, imagValue), LV, /*init*/ true);
1052
234
  } else {
1053
    // Call EmitStoreOfScalar except when the lvalue is a bitfield to emit a
1054
    // primitive store.
1055
234
    assert(isa<NoExpansion>(Exp.get()));
1056
234
    if (LV.isBitField())
1057
0
      EmitStoreThroughLValue(RValue::get(&*AI++), LV);
1058
234
    else
1059
234
      EmitStoreOfScalar(&*AI++, LV);
1060
234
  }
1061
344
}
1062
1063
void CodeGenFunction::ExpandTypeToArgs(
1064
    QualType Ty, CallArg Arg, llvm::FunctionType *IRFuncTy,
1065
74
    SmallVectorImpl<llvm::Value *> &IRCallArgs, unsigned &IRCallArgPos) {
1066
74
  auto Exp = getTypeExpansion(Ty, getContext());
1067
74
  if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
1068
0
    Address Addr = Arg.hasLValue() ? Arg.getKnownLValue().getAddress(*this)
1069
0
                                   : Arg.getKnownRValue().getAggregateAddress();
1070
0
    forConstantArrayExpansion(
1071
0
        *this, CAExp, Addr, [&](Address EltAddr) {
1072
0
          CallArg EltArg = CallArg(
1073
0
              convertTempToRValue(EltAddr, CAExp->EltTy, SourceLocation()),
1074
0
              CAExp->EltTy);
1075
0
          ExpandTypeToArgs(CAExp->EltTy, EltArg, IRFuncTy, IRCallArgs,
1076
0
                           IRCallArgPos);
1077
0
        });
1078
74
  } else if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
1079
12
    Address This = Arg.hasLValue() ? Arg.getKnownLValue().getAddress(*this)
1080
15
                                   : Arg.getKnownRValue().getAggregateAddress();
1081
0
    for (const CXXBaseSpecifier *BS : RExp->Bases) {
1082
      // Perform a single step derived-to-base conversion.
1083
0
      Address Base =
1084
0
          GetAddressOfBaseClass(This, Ty->getAsCXXRecordDecl(), &BS, &BS + 1,
1085
0
                                /*NullCheckValue=*/false, SourceLocation());
1086
0
      CallArg BaseArg = CallArg(RValue::getAggregate(Base), BS->getType());
1087
1088
      // Recurse onto bases.
1089
0
      ExpandTypeToArgs(BS->getType(), BaseArg, IRFuncTy, IRCallArgs,
1090
0
                       IRCallArgPos);
1091
0
    }
1092
1093
27
    LValue LV = MakeAddrLValue(This, Ty);
1094
44
    for (auto FD : RExp->Fields) {
1095
44
      CallArg FldArg =
1096
44
          CallArg(EmitRValueForField(LV, FD, SourceLocation()), FD->getType());
1097
44
      ExpandTypeToArgs(FD->getType(), FldArg, IRFuncTy, IRCallArgs,
1098
44
                       IRCallArgPos);
1099
44
    }
1100
47
  } else if (isa<ComplexExpansion>(Exp.get())) {
1101
4
    ComplexPairTy CV = Arg.getKnownRValue().getComplexVal();
1102
4
    IRCallArgs[IRCallArgPos++] = CV.first;
1103
4
    IRCallArgs[IRCallArgPos++] = CV.second;
1104
43
  } else {
1105
43
    assert(isa<NoExpansion>(Exp.get()));
1106
43
    auto RV = Arg.getKnownRValue();
1107
43
    assert(RV.isScalar() &&
1108
43
           "Unexpected non-scalar rvalue during struct expansion.");
1109
1110
    // Insert a bitcast as needed.
1111
43
    llvm::Value *V = RV.getScalarVal();
1112
43
    if (IRCallArgPos < IRFuncTy->getNumParams() &&
1113
43
        V->getType() != IRFuncTy->getParamType(IRCallArgPos))
1114
0
      V = Builder.CreateBitCast(V, IRFuncTy->getParamType(IRCallArgPos));
1115
1116
43
    IRCallArgs[IRCallArgPos++] = V;
1117
43
  }
1118
74
}
1119
1120
/// Create a temporary allocation for the purposes of coercion.
1121
static Address CreateTempAllocaForCoercion(CodeGenFunction &CGF, llvm::Type *Ty,
1122
                                           CharUnits MinAlign,
1123
279
                                           const Twine &Name = "tmp") {
1124
  // Don't use an alignment that's worse than what LLVM would prefer.
1125
279
  auto PrefAlign = CGF.CGM.getDataLayout().getPrefTypeAlignment(Ty);
1126
279
  CharUnits Align = std::max(MinAlign, CharUnits::fromQuantity(PrefAlign));
1127
1128
279
  return CGF.CreateTempAlloca(Ty, Align, Name + ".coerce");
1129
279
}
1130
1131
/// EnterStructPointerForCoercedAccess - Given a struct pointer that we are
1132
/// accessing some number of bytes out of it, try to gep into the struct to get
1133
/// at its inner goodness.  Dive as deep as possible without entering an element
1134
/// with an in-memory size smaller than DstSize.
1135
static Address
1136
EnterStructPointerForCoercedAccess(Address SrcPtr,
1137
                                   llvm::StructType *SrcSTy,
1138
18.7k
                                   uint64_t DstSize, CodeGenFunction &CGF) {
1139
  // We can't dive into a zero-element struct.
1140
18.7k
  if (SrcSTy->getNumElements() == 0) 
return SrcPtr0
;
1141
1142
18.7k
  llvm::Type *FirstElt = SrcSTy->getElementType(0);
1143
1144
  // If the first elt is at least as large as what we're looking for, or if the
1145
  // first element is the same size as the whole struct, we can enter it. The
1146
  // comparison must be made on the store size and not the alloca size. Using
1147
  // the alloca size may overstate the size of the load.
1148
18.7k
  uint64_t FirstEltSize =
1149
18.7k
    CGF.CGM.getDataLayout().getTypeStoreSize(FirstElt);
1150
18.7k
  if (FirstEltSize < DstSize &&
1151
4.76k
      FirstEltSize < CGF.CGM.getDataLayout().getTypeStoreSize(SrcSTy))
1152
4.60k
    return SrcPtr;
1153
1154
  // GEP into the first element.
1155
14.1k
  SrcPtr = CGF.Builder.CreateStructGEP(SrcPtr, 0, "coerce.dive");
1156
1157
  // If the first element is a struct, recurse.
1158
14.1k
  llvm::Type *SrcTy = SrcPtr.getElementType();
1159
14.1k
  if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy))
1160
4.01k
    return EnterStructPointerForCoercedAccess(SrcPtr, SrcSTy, DstSize, CGF);
1161
1162
10.0k
  return SrcPtr;
1163
10.0k
}
1164
1165
/// CoerceIntOrPtrToIntOrPtr - Convert a value Val to the specific Ty where both
1166
/// are either integers or pointers.  This does a truncation of the value if it
1167
/// is too large or a zero extension if it is too small.
1168
///
1169
/// This behaves as if the value were coerced through memory, so on big-endian
1170
/// targets the high bits are preserved in a truncation, while little-endian
1171
/// targets preserve the low bits.
1172
static llvm::Value *CoerceIntOrPtrToIntOrPtr(llvm::Value *Val,
1173
                                             llvm::Type *Ty,
1174
8.93k
                                             CodeGenFunction &CGF) {
1175
8.93k
  if (Val->getType() == Ty)
1176
8.49k
    return Val;
1177
1178
444
  if (isa<llvm::PointerType>(Val->getType())) {
1179
    // If this is Pointer->Pointer avoid conversion to and from int.
1180
342
    if (isa<llvm::PointerType>(Ty))
1181
144
      return CGF.Builder.CreateBitCast(Val, Ty, "coerce.val");
1182
1183
    // Convert the pointer to an integer so we can play with its width.
1184
198
    Val = CGF.Builder.CreatePtrToInt(Val, CGF.IntPtrTy, "coerce.val.pi");
1185
198
  }
1186
1187
300
  llvm::Type *DestIntTy = Ty;
1188
300
  if (isa<llvm::PointerType>(DestIntTy))
1189
25
    DestIntTy = CGF.IntPtrTy;
1190
1191
300
  if (Val->getType() != DestIntTy) {
1192
77
    const llvm::DataLayout &DL = CGF.CGM.getDataLayout();
1193
77
    if (DL.isBigEndian()) {
1194
      // Preserve the high bits on big-endian targets.
1195
      // That is what memory coercion does.
1196
19
      uint64_t SrcSize = DL.getTypeSizeInBits(Val->getType());
1197
19
      uint64_t DstSize = DL.getTypeSizeInBits(DestIntTy);
1198
1199
19
      if (SrcSize > DstSize) {
1200
10
        Val = CGF.Builder.CreateLShr(Val, SrcSize - DstSize, "coerce.highbits");
1201
10
        Val = CGF.Builder.CreateTrunc(Val, DestIntTy, "coerce.val.ii");
1202
9
      } else {
1203
9
        Val = CGF.Builder.CreateZExt(Val, DestIntTy, "coerce.val.ii");
1204
9
        Val = CGF.Builder.CreateShl(Val, DstSize - SrcSize, "coerce.highbits");
1205
9
      }
1206
58
    } else {
1207
      // Little-endian targets preserve the low bits. No shifts required.
1208
58
      Val = CGF.Builder.CreateIntCast(Val, DestIntTy, false, "coerce.val.ii");
1209
58
    }
1210
77
  }
1211
1212
300
  if (isa<llvm::PointerType>(Ty))
1213
25
    Val = CGF.Builder.CreateIntToPtr(Val, Ty, "coerce.val.ip");
1214
300
  return Val;
1215
444
}
1216
1217
1218
1219
/// CreateCoercedLoad - Create a load from \arg SrcPtr interpreted as
1220
/// a pointer to an object of type \arg Ty, known to be aligned to
1221
/// \arg SrcAlign bytes.
1222
///
1223
/// This safely handles the case when the src type is smaller than the
1224
/// destination type; in this situation the values of bits which not
1225
/// present in the src are undefined.
1226
static llvm::Value *CreateCoercedLoad(Address Src, llvm::Type *Ty,
1227
7.86k
                                      CodeGenFunction &CGF) {
1228
7.86k
  llvm::Type *SrcTy = Src.getElementType();
1229
1230
  // If SrcTy and Ty are the same, just do a load.
1231
7.86k
  if (SrcTy == Ty)
1232
25
    return CGF.Builder.CreateLoad(Src);
1233
1234
7.84k
  llvm::TypeSize DstSize = CGF.CGM.getDataLayout().getTypeAllocSize(Ty);
1235
1236
7.84k
  if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy)) {
1237
6.47k
    Src = EnterStructPointerForCoercedAccess(Src, SrcSTy,
1238
6.47k
                                             DstSize.getFixedSize(), CGF);
1239
6.47k
    SrcTy = Src.getElementType();
1240
6.47k
  }
1241
1242
7.84k
  llvm::TypeSize SrcSize = CGF.CGM.getDataLayout().getTypeAllocSize(SrcTy);
1243
1244
  // If the source and destination are integer or pointer types, just do an
1245
  // extension or truncation to the desired type.
1246
7.84k
  if ((isa<llvm::IntegerType>(Ty) || 
isa<llvm::PointerType>(Ty)4.99k
) &&
1247
5.82k
      (isa<llvm::IntegerType>(SrcTy) || 
isa<llvm::PointerType>(SrcTy)4.65k
)) {
1248
4.34k
    llvm::Value *Load = CGF.Builder.CreateLoad(Src);
1249
4.34k
    return CoerceIntOrPtrToIntOrPtr(Load, Ty, CGF);
1250
4.34k
  }
1251
1252
  // If load is legal, just bitcast the src pointer.
1253
3.49k
  if (!SrcSize.isScalable() && !DstSize.isScalable() &&
1254
3.46k
      SrcSize.getFixedSize() >= DstSize.getFixedSize()) {
1255
    // Generally SrcSize is never greater than DstSize, since this means we are
1256
    // losing bits. However, this can happen in cases where the structure has
1257
    // additional padding, for example due to a user specified alignment.
1258
    //
1259
    // FIXME: Assert that we aren't truncating non-padding bits when have access
1260
    // to that information.
1261
3.31k
    Src = CGF.Builder.CreateBitCast(Src,
1262
3.31k
                                    Ty->getPointerTo(Src.getAddressSpace()));
1263
3.31k
    return CGF.Builder.CreateLoad(Src);
1264
3.31k
  }
1265
1266
  // Otherwise do coercion through memory. This is stupid, but simple.
1267
180
  Address Tmp =
1268
180
      CreateTempAllocaForCoercion(CGF, Ty, Src.getAlignment(), Src.getName());
1269
180
  CGF.Builder.CreateMemCpy(
1270
180
      Tmp.getPointer(), Tmp.getAlignment().getAsAlign(), Src.getPointer(),
1271
180
      Src.getAlignment().getAsAlign(),
1272
180
      llvm::ConstantInt::get(CGF.IntPtrTy, SrcSize.getKnownMinSize()));
1273
180
  return CGF.Builder.CreateLoad(Tmp);
1274
180
}
1275
1276
// Function to store a first-class aggregate into memory.  We prefer to
1277
// store the elements rather than the aggregate to be more friendly to
1278
// fast-isel.
1279
// FIXME: Do we need to recurse here?
1280
void CodeGenFunction::EmitAggregateStore(llvm::Value *Val, Address Dest,
1281
5.13k
                                         bool DestIsVolatile) {
1282
  // Prefer scalar stores to first-class aggregate stores.
1283
5.13k
  if (llvm::StructType *STy = dyn_cast<llvm::StructType>(Val->getType())) {
1284
4.20k
    for (unsigned i = 0, e = STy->getNumElements(); i != e; 
++i2.79k
) {
1285
2.79k
      Address EltPtr = Builder.CreateStructGEP(Dest, i);
1286
2.79k
      llvm::Value *Elt = Builder.CreateExtractValue(Val, i);
1287
2.79k
      Builder.CreateStore(Elt, EltPtr, DestIsVolatile);
1288
2.79k
    }
1289
3.72k
  } else {
1290
3.72k
    Builder.CreateStore(Val, Dest, DestIsVolatile);
1291
3.72k
  }
1292
5.13k
}
1293
1294
/// CreateCoercedStore - Create a store to \arg DstPtr from \arg Src,
1295
/// where the source and destination may have different types.  The
1296
/// destination is known to be aligned to \arg DstAlign bytes.
1297
///
1298
/// This safely handles the case when the src type is larger than the
1299
/// destination type; the upper bits of the src will be lost.
1300
static void CreateCoercedStore(llvm::Value *Src,
1301
                               Address Dst,
1302
                               bool DstIsVolatile,
1303
9.87k
                               CodeGenFunction &CGF) {
1304
9.87k
  llvm::Type *SrcTy = Src->getType();
1305
9.87k
  llvm::Type *DstTy = Dst.getElementType();
1306
9.87k
  if (SrcTy == DstTy) {
1307
158
    CGF.Builder.CreateStore(Src, Dst, DstIsVolatile);
1308
158
    return;
1309
158
  }
1310
1311
9.71k
  llvm::TypeSize SrcSize = CGF.CGM.getDataLayout().getTypeAllocSize(SrcTy);
1312
1313
9.71k
  if (llvm::StructType *DstSTy = dyn_cast<llvm::StructType>(DstTy)) {
1314
8.22k
    Dst = EnterStructPointerForCoercedAccess(Dst, DstSTy,
1315
8.22k
                                             SrcSize.getFixedSize(), CGF);
1316
8.22k
    DstTy = Dst.getElementType();
1317
8.22k
  }
1318
1319
9.71k
  llvm::PointerType *SrcPtrTy = llvm::dyn_cast<llvm::PointerType>(SrcTy);
1320
9.71k
  llvm::PointerType *DstPtrTy = llvm::dyn_cast<llvm::PointerType>(DstTy);
1321
9.71k
  if (SrcPtrTy && 
DstPtrTy3.17k
&&
1322
3.17k
      SrcPtrTy->getAddressSpace() != DstPtrTy->getAddressSpace()) {
1323
16
    Src = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Src, DstTy);
1324
16
    CGF.Builder.CreateStore(Src, Dst, DstIsVolatile);
1325
16
    return;
1326
16
  }
1327
1328
  // If the source and destination are integer or pointer types, just do an
1329
  // extension or truncation to the desired type.
1330
9.70k
  if ((isa<llvm::IntegerType>(SrcTy) || 
isa<llvm::PointerType>(SrcTy)6.73k
) &&
1331
6.12k
      (isa<llvm::IntegerType>(DstTy) || 
isa<llvm::PointerType>(DstTy)4.71k
)) {
1332
4.59k
    Src = CoerceIntOrPtrToIntOrPtr(Src, DstTy, CGF);
1333
4.59k
    CGF.Builder.CreateStore(Src, Dst, DstIsVolatile);
1334
4.59k
    return;
1335
4.59k
  }
1336
1337
5.11k
  llvm::TypeSize DstSize = CGF.CGM.getDataLayout().getTypeAllocSize(DstTy);
1338
1339
  // If store is legal, just bitcast the src pointer.
1340
5.11k
  if (isa<llvm::ScalableVectorType>(SrcTy) ||
1341
5.07k
      isa<llvm::ScalableVectorType>(DstTy) ||
1342
5.07k
      SrcSize.getFixedSize() <= DstSize.getFixedSize()) {
1343
5.01k
    Dst = CGF.Builder.CreateElementBitCast(Dst, SrcTy);
1344
5.01k
    CGF.EmitAggregateStore(Src, Dst, DstIsVolatile);
1345
99
  } else {
1346
    // Otherwise do coercion through memory. This is stupid, but
1347
    // simple.
1348
1349
    // Generally SrcSize is never greater than DstSize, since this means we are
1350
    // losing bits. However, this can happen in cases where the structure has
1351
    // additional padding, for example due to a user specified alignment.
1352
    //
1353
    // FIXME: Assert that we aren't truncating non-padding bits when have access
1354
    // to that information.
1355
99
    Address Tmp = CreateTempAllocaForCoercion(CGF, SrcTy, Dst.getAlignment());
1356
99
    CGF.Builder.CreateStore(Src, Tmp);
1357
99
    CGF.Builder.CreateMemCpy(
1358
99
        Dst.getPointer(), Dst.getAlignment().getAsAlign(), Tmp.getPointer(),
1359
99
        Tmp.getAlignment().getAsAlign(),
1360
99
        llvm::ConstantInt::get(CGF.IntPtrTy, DstSize.getFixedSize()));
1361
99
  }
1362
5.11k
}
1363
1364
static Address emitAddressAtOffset(CodeGenFunction &CGF, Address addr,
1365
19.8k
                                   const ABIArgInfo &info) {
1366
19.8k
  if (unsigned offset = info.getDirectOffset()) {
1367
3
    addr = CGF.Builder.CreateElementBitCast(addr, CGF.Int8Ty);
1368
3
    addr = CGF.Builder.CreateConstInBoundsByteGEP(addr,
1369
3
                                             CharUnits::fromQuantity(offset));
1370
3
    addr = CGF.Builder.CreateElementBitCast(addr, info.getCoerceToType());
1371
3
  }
1372
19.8k
  return addr;
1373
19.8k
}
1374
1375
namespace {
1376
1377
/// Encapsulates information about the way function arguments from
1378
/// CGFunctionInfo should be passed to actual LLVM IR function.
1379
class ClangToLLVMArgMapping {
1380
  static const unsigned InvalidIndex = ~0U;
1381
  unsigned InallocaArgNo;
1382
  unsigned SRetArgNo;
1383
  unsigned TotalIRArgs;
1384
1385
  /// Arguments of LLVM IR function corresponding to single Clang argument.
1386
  struct IRArgs {
1387
    unsigned PaddingArgIndex;
1388
    // Argument is expanded to IR arguments at positions
1389
    // [FirstArgIndex, FirstArgIndex + NumberOfArgs).
1390
    unsigned FirstArgIndex;
1391
    unsigned NumberOfArgs;
1392
1393
    IRArgs()
1394
        : PaddingArgIndex(InvalidIndex), FirstArgIndex(InvalidIndex),
1395
2.25M
          NumberOfArgs(0) {}
1396
  };
1397
1398
  SmallVector<IRArgs, 8> ArgInfo;
1399
1400
public:
1401
  ClangToLLVMArgMapping(const ASTContext &Context, const CGFunctionInfo &FI,
1402
                        bool OnlyRequiredArgs = false)
1403
      : InallocaArgNo(InvalidIndex), SRetArgNo(InvalidIndex), TotalIRArgs(0),
1404
2.25M
        ArgInfo(OnlyRequiredArgs ? FI.getNumRequiredArgs() : FI.arg_size()) {
1405
2.25M
    construct(Context, FI, OnlyRequiredArgs);
1406
2.25M
  }
1407
1408
2.72M
  bool hasInallocaArg() const { return InallocaArgNo != InvalidIndex; }
1409
558
  unsigned getInallocaArgNo() const {
1410
558
    assert(hasInallocaArg());
1411
558
    return InallocaArgNo;
1412
558
  }
1413
1414
1.95M
  bool hasSRetArg() const { return SRetArgNo != InvalidIndex; }
1415
14.0k
  unsigned getSRetArgNo() const {
1416
14.0k
    assert(hasSRetArg());
1417
14.0k
    return SRetArgNo;
1418
14.0k
  }
1419
1420
2.25M
  unsigned totalIRArgs() const { return TotalIRArgs; }
1421
1422
3.18M
  bool hasPaddingArg(unsigned ArgNo) const {
1423
3.18M
    assert(ArgNo < ArgInfo.size());
1424
3.18M
    return ArgInfo[ArgNo].PaddingArgIndex != InvalidIndex;
1425
3.18M
  }
1426
69
  unsigned getPaddingArgNo(unsigned ArgNo) const {
1427
69
    assert(hasPaddingArg(ArgNo));
1428
69
    return ArgInfo[ArgNo].PaddingArgIndex;
1429
69
  }
1430
1431
  /// Returns index of first IR argument corresponding to ArgNo, and their
1432
  /// quantity.
1433
3.02M
  std::pair<unsigned, unsigned> getIRArgs(unsigned ArgNo) const {
1434
3.02M
    assert(ArgNo < ArgInfo.size());
1435
3.02M
    return std::make_pair(ArgInfo[ArgNo].FirstArgIndex,
1436
3.02M
                          ArgInfo[ArgNo].NumberOfArgs);
1437
3.02M
  }
1438
1439
private:
1440
  void construct(const ASTContext &Context, const CGFunctionInfo &FI,
1441
                 bool OnlyRequiredArgs);
1442
};
1443
1444
void ClangToLLVMArgMapping::construct(const ASTContext &Context,
1445
                                      const CGFunctionInfo &FI,
1446
2.25M
                                      bool OnlyRequiredArgs) {
1447
2.25M
  unsigned IRArgNo = 0;
1448
2.25M
  bool SwapThisWithSRet = false;
1449
2.25M
  const ABIArgInfo &RetAI = FI.getReturnInfo();
1450
1451
2.25M
  if (RetAI.getKind() == ABIArgInfo::Indirect) {
1452
14.0k
    SwapThisWithSRet = RetAI.isSRetAfterThis();
1453
13.6k
    SRetArgNo = SwapThisWithSRet ? 
1392
: IRArgNo++;
1454
14.0k
  }
1455
1456
2.25M
  unsigned ArgNo = 0;
1457
1.26M
  unsigned NumArgs = OnlyRequiredArgs ? 
FI.getNumRequiredArgs()995k
: FI.arg_size();
1458
5.92M
  for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(); ArgNo < NumArgs;
1459
3.66M
       ++I, ++ArgNo) {
1460
3.66M
    assert(I != FI.arg_end());
1461
3.66M
    QualType ArgType = I->type;
1462
3.66M
    const ABIArgInfo &AI = I->info;
1463
    // Collect data about IR arguments corresponding to Clang argument ArgNo.
1464
3.66M
    auto &IRArgs = ArgInfo[ArgNo];
1465
1466
3.66M
    if (AI.getPaddingType())
1467
124
      IRArgs.PaddingArgIndex = IRArgNo++;
1468
1469
3.66M
    switch (AI.getKind()) {
1470
3.58M
    case ABIArgInfo::Extend:
1471
3.58M
    case ABIArgInfo::Direct: {
1472
      // FIXME: handle sseregparm someday...
1473
3.58M
      llvm::StructType *STy = dyn_cast<llvm::StructType>(AI.getCoerceToType());
1474
3.58M
      if (AI.isDirect() && 
AI.getCanBeFlattened()3.48M
&&
STy3.47M
) {
1475
8.06k
        IRArgs.NumberOfArgs = STy->getNumElements();
1476
3.57M
      } else {
1477
3.57M
        IRArgs.NumberOfArgs = 1;
1478
3.57M
      }
1479
3.58M
      break;
1480
3.58M
    }
1481
15.0k
    case ABIArgInfo::Indirect:
1482
15.0k
    case ABIArgInfo::IndirectAliased:
1483
15.0k
      IRArgs.NumberOfArgs = 1;
1484
15.0k
      break;
1485
64.9k
    case ABIArgInfo::Ignore:
1486
64.9k
    case ABIArgInfo::InAlloca:
1487
      // ignore and inalloca doesn't have matching LLVM parameters.
1488
64.9k
      IRArgs.NumberOfArgs = 0;
1489
64.9k
      break;
1490
3.45k
    case ABIArgInfo::CoerceAndExpand:
1491
3.45k
      IRArgs.NumberOfArgs = AI.getCoerceAndExpandTypeSequence().size();
1492
3.45k
      break;
1493
451
    case ABIArgInfo::Expand:
1494
451
      IRArgs.NumberOfArgs = getExpansionSize(ArgType, Context);
1495
451
      break;
1496
3.66M
    }
1497
1498
3.66M
    if (IRArgs.NumberOfArgs > 0) {
1499
3.60M
      IRArgs.FirstArgIndex = IRArgNo;
1500
3.60M
      IRArgNo += IRArgs.NumberOfArgs;
1501
3.60M
    }
1502
1503
    // Skip over the sret parameter when it comes second.  We already handled it
1504
    // above.
1505
3.66M
    if (IRArgNo == 1 && 
SwapThisWithSRet2.09M
)
1506
392
      IRArgNo++;
1507
3.66M
  }
1508
2.25M
  assert(ArgNo == ArgInfo.size());
1509
1510
2.25M
  if (FI.usesInAlloca())
1511
459
    InallocaArgNo = IRArgNo++;
1512
1513
2.25M
  TotalIRArgs = IRArgNo;
1514
2.25M
}
1515
}  // namespace
1516
1517
/***/
1518
1519
20.7k
bool CodeGenModule::ReturnTypeUsesSRet(const CGFunctionInfo &FI) {
1520
20.7k
  const auto &RI = FI.getReturnInfo();
1521
20.7k
  return RI.isIndirect() || 
(20.6k
RI.isInAlloca()20.6k
&&
RI.getInAllocaSRet()1
);
1522
20.7k
}
1523
1524
13.2k
bool CodeGenModule::ReturnSlotInterferesWithArgs(const CGFunctionInfo &FI) {
1525
13.2k
  return ReturnTypeUsesSRet(FI) &&
1526
40
         getTargetCodeGenInfo().doesReturnSlotInterfereWithArgs();
1527
13.2k
}
1528
1529
12.1k
bool CodeGenModule::ReturnTypeUsesFPRet(QualType ResultType) {
1530
12.1k
  if (const BuiltinType *BT = ResultType->getAs<BuiltinType>()) {
1531
5.16k
    switch (BT->getKind()) {
1532
5.09k
    default:
1533
5.09k
      return false;
1534
18
    case BuiltinType::Float:
1535
18
      return getTarget().useObjCFPRetForRealType(TargetInfo::Float);
1536
46
    case BuiltinType::Double:
1537
46
      return getTarget().useObjCFPRetForRealType(TargetInfo::Double);
1538
4
    case BuiltinType::LongDouble:
1539
4
      return getTarget().useObjCFPRetForRealType(TargetInfo::LongDouble);
1540
6.98k
    }
1541
6.98k
  }
1542
1543
6.98k
  return false;
1544
6.98k
}
1545
1546
12.1k
bool CodeGenModule::ReturnTypeUsesFP2Ret(QualType ResultType) {
1547
12.1k
  if (const ComplexType *CT = ResultType->getAs<ComplexType>()) {
1548
14
    if (const BuiltinType *BT = CT->getElementType()->getAs<BuiltinType>()) {
1549
14
      if (BT->getKind() == BuiltinType::LongDouble)
1550
2
        return getTarget().useObjCFP2RetForComplexLongDouble();
1551
12.1k
    }
1552
14
  }
1553
1554
12.1k
  return false;
1555
12.1k
}
1556
1557
5.68k
llvm::FunctionType *CodeGenTypes::GetFunctionType(GlobalDecl GD) {
1558
5.68k
  const CGFunctionInfo &FI = arrangeGlobalDeclaration(GD);
1559
5.68k
  return GetFunctionType(FI);
1560
5.68k
}
1561
1562
llvm::FunctionType *
1563
995k
CodeGenTypes::GetFunctionType(const CGFunctionInfo &FI) {
1564
1565
995k
  bool Inserted = FunctionsBeingProcessed.insert(&FI).second;
1566
995k
  (void)Inserted;
1567
995k
  assert(Inserted && "Recursively being processed?");
1568
1569
995k
  llvm::Type *resultType = nullptr;
1570
995k
  const ABIArgInfo &retAI = FI.getReturnInfo();
1571
995k
  switch (retAI.getKind()) {
1572
0
  case ABIArgInfo::Expand:
1573
0
  case ABIArgInfo::IndirectAliased:
1574
0
    llvm_unreachable("Invalid ABI kind for return argument");
1575
1576
514k
  case ABIArgInfo::Extend:
1577
514k
  case ABIArgInfo::Direct:
1578
514k
    resultType = retAI.getCoerceToType();
1579
514k
    break;
1580
1581
18
  case ABIArgInfo::InAlloca:
1582
18
    if (retAI.getInAllocaSRet()) {
1583
      // sret things on win32 aren't void, they return the sret pointer.
1584
18
      QualType ret = FI.getReturnType();
1585
18
      llvm::Type *ty = ConvertType(ret);
1586
18
      unsigned addressSpace = Context.getTargetAddressSpace(ret);
1587
18
      resultType = llvm::PointerType::get(ty, addressSpace);
1588
0
    } else {
1589
0
      resultType = llvm::Type::getVoidTy(getLLVMContext());
1590
0
    }
1591
18
    break;
1592
1593
479k
  case ABIArgInfo::Indirect:
1594
479k
  case ABIArgInfo::Ignore:
1595
479k
    resultType = llvm::Type::getVoidTy(getLLVMContext());
1596
479k
    break;
1597
1598
1.42k
  case ABIArgInfo::CoerceAndExpand:
1599
1.42k
    resultType = retAI.getUnpaddedCoerceAndExpandType();
1600
1.42k
    break;
1601
995k
  }
1602
1603
995k
  ClangToLLVMArgMapping IRFunctionArgs(getContext(), FI, true);
1604
995k
  SmallVector<llvm::Type*, 8> ArgTypes(IRFunctionArgs.totalIRArgs());
1605
1606
  // Add type for sret argument.
1607
995k
  if (IRFunctionArgs.hasSRetArg()) {
1608
5.72k
    QualType Ret = FI.getReturnType();
1609
5.72k
    llvm::Type *Ty = ConvertType(Ret);
1610
5.72k
    unsigned AddressSpace = Context.getTargetAddressSpace(Ret);
1611
5.72k
    ArgTypes[IRFunctionArgs.getSRetArgNo()] =
1612
5.72k
        llvm::PointerType::get(Ty, AddressSpace);
1613
5.72k
  }
1614
1615
  // Add type for inalloca argument.
1616
995k
  if (IRFunctionArgs.hasInallocaArg()) {
1617
191
    auto ArgStruct = FI.getArgStruct();
1618
191
    assert(ArgStruct);
1619
191
    ArgTypes[IRFunctionArgs.getInallocaArgNo()] = ArgStruct->getPointerTo();
1620
191
  }
1621
1622
  // Add in all of the required arguments.
1623
995k
  unsigned ArgNo = 0;
1624
995k
  CGFunctionInfo::const_arg_iterator it = FI.arg_begin(),
1625
995k
                                     ie = it + FI.getNumRequiredArgs();
1626
2.57M
  for (; it != ie; 
++it, ++ArgNo1.57M
) {
1627
1.57M
    const ABIArgInfo &ArgInfo = it->info;
1628
1629
    // Insert a padding type to ensure proper alignment.
1630
1.57M
    if (IRFunctionArgs.hasPaddingArg(ArgNo))
1631
47
      ArgTypes[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
1632
47
          ArgInfo.getPaddingType();
1633
1634
1.57M
    unsigned FirstIRArg, NumIRArgs;
1635
1.57M
    std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
1636
1637
1.57M
    switch (ArgInfo.getKind()) {
1638
32.0k
    case ABIArgInfo::Ignore:
1639
32.0k
    case ABIArgInfo::InAlloca:
1640
32.0k
      assert(NumIRArgs == 0);
1641
32.0k
      break;
1642
1643
6.23k
    case ABIArgInfo::Indirect: {
1644
6.23k
      assert(NumIRArgs == 1);
1645
      // indirect arguments are always on the stack, which is alloca addr space.
1646
6.23k
      llvm::Type *LTy = ConvertTypeForMem(it->type);
1647
6.23k
      ArgTypes[FirstIRArg] = LTy->getPointerTo(
1648
6.23k
          CGM.getDataLayout().getAllocaAddrSpace());
1649
6.23k
      break;
1650
32.0k
    }
1651
19
    case ABIArgInfo::IndirectAliased: {
1652
19
      assert(NumIRArgs == 1);
1653
19
      llvm::Type *LTy = ConvertTypeForMem(it->type);
1654
19
      ArgTypes[FirstIRArg] = LTy->getPointerTo(ArgInfo.getIndirectAddrSpace());
1655
19
      break;
1656
32.0k
    }
1657
1.53M
    case ABIArgInfo::Extend:
1658
1.53M
    case ABIArgInfo::Direct: {
1659
      // Fast-isel and the optimizer generally like scalar values better than
1660
      // FCAs, so we flatten them if this is safe to do for this argument.
1661
1.53M
      llvm::Type *argType = ArgInfo.getCoerceToType();
1662
1.53M
      llvm::StructType *st = dyn_cast<llvm::StructType>(argType);
1663
1.53M
      if (st && 
ArgInfo.isDirect()3.66k
&&
ArgInfo.getCanBeFlattened()3.66k
) {
1664
3.46k
        assert(NumIRArgs == st->getNumElements());
1665
10.5k
        for (unsigned i = 0, e = st->getNumElements(); i != e; 
++i7.04k
)
1666
7.04k
          ArgTypes[FirstIRArg + i] = st->getElementType(i);
1667
1.53M
      } else {
1668
1.53M
        assert(NumIRArgs == 1);
1669
1.53M
        ArgTypes[FirstIRArg] = argType;
1670
1.53M
      }
1671
1.53M
      break;
1672
1.53M
    }
1673
1674
1.44k
    case ABIArgInfo::CoerceAndExpand: {
1675
1.44k
      auto ArgTypesIter = ArgTypes.begin() + FirstIRArg;
1676
3.11k
      for (auto EltTy : ArgInfo.getCoerceAndExpandTypeSequence()) {
1677
3.11k
        *ArgTypesIter++ = EltTy;
1678
3.11k
      }
1679
1.44k
      assert(ArgTypesIter == ArgTypes.begin() + FirstIRArg + NumIRArgs);
1680
1.44k
      break;
1681
1.53M
    }
1682
1683
169
    case ABIArgInfo::Expand:
1684
169
      auto ArgTypesIter = ArgTypes.begin() + FirstIRArg;
1685
169
      getExpandedTypes(it->type, ArgTypesIter);
1686
169
      assert(ArgTypesIter == ArgTypes.begin() + FirstIRArg + NumIRArgs);
1687
169
      break;
1688
1.57M
    }
1689
1.57M
  }
1690
1691
995k
  bool Erased = FunctionsBeingProcessed.erase(&FI); (void)Erased;
1692
995k
  assert(Erased && "Not in set?");
1693
1694
995k
  return llvm::FunctionType::get(resultType, ArgTypes, FI.isVariadic());
1695
995k
}
1696
1697
5.66k
llvm::Type *CodeGenTypes::GetFunctionTypeForVTable(GlobalDecl GD) {
1698
5.66k
  const CXXMethodDecl *MD = cast<CXXMethodDecl>(GD.getDecl());
1699
5.66k
  const FunctionProtoType *FPT = MD->getType()->getAs<FunctionProtoType>();
1700
1701
5.66k
  if (!isFuncTypeConvertible(FPT))
1702
39
    return llvm::StructType::get(getLLVMContext());
1703
1704
5.63k
  return GetFunctionType(GD);
1705
5.63k
}
1706
1707
static void AddAttributesFromFunctionProtoType(ASTContext &Ctx,
1708
                                               llvm::AttrBuilder &FuncAttrs,
1709
1.23M
                                               const FunctionProtoType *FPT) {
1710
1.23M
  if (!FPT)
1711
650k
    return;
1712
1713
582k
  if (!isUnresolvedExceptionSpec(FPT->getExceptionSpecType()) &&
1714
582k
      FPT->isNothrow())
1715
175k
    FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
1716
582k
}
1717
1718
void CodeGenModule::getDefaultFunctionAttributes(StringRef Name,
1719
                                                 bool HasOptnone,
1720
                                                 bool AttrOnCallSite,
1721
647k
                                               llvm::AttrBuilder &FuncAttrs) {
1722
  // OptimizeNoneAttr takes precedence over -Os or -Oz. No warning needed.
1723
647k
  if (!HasOptnone) {
1724
646k
    if (CodeGenOpts.OptimizeSize)
1725
1.13k
      FuncAttrs.addAttribute(llvm::Attribute::OptimizeForSize);
1726
646k
    if (CodeGenOpts.OptimizeSize == 2)
1727
717
      FuncAttrs.addAttribute(llvm::Attribute::MinSize);
1728
646k
  }
1729
1730
647k
  if (CodeGenOpts.DisableRedZone)
1731
25
    FuncAttrs.addAttribute(llvm::Attribute::NoRedZone);
1732
647k
  if (CodeGenOpts.IndirectTlsSegRefs)
1733
1
    FuncAttrs.addAttribute("indirect-tls-seg-refs");
1734
647k
  if (CodeGenOpts.NoImplicitFloat)
1735
16
    FuncAttrs.addAttribute(llvm::Attribute::NoImplicitFloat);
1736
1737
647k
  if (AttrOnCallSite) {
1738
    // Attributes that should go on the call site only.
1739
324k
    if (!CodeGenOpts.SimplifyLibCalls ||
1740
147k
        CodeGenOpts.isNoBuiltinFunc(Name.data()))
1741
177k
      FuncAttrs.addAttribute(llvm::Attribute::NoBuiltin);
1742
324k
    if (!CodeGenOpts.TrapFuncName.empty())
1743
0
      FuncAttrs.addAttribute("trap-func-name", CodeGenOpts.TrapFuncName);
1744
322k
  } else {
1745
322k
    StringRef FpKind;
1746
322k
    switch (CodeGenOpts.getFramePointer()) {
1747
200k
    case CodeGenOptions::FramePointerKind::None:
1748
200k
      FpKind = "none";
1749
200k
      break;
1750
184
    case CodeGenOptions::FramePointerKind::NonLeaf:
1751
184
      FpKind = "non-leaf";
1752
184
      break;
1753
122k
    case CodeGenOptions::FramePointerKind::All:
1754
122k
      FpKind = "all";
1755
122k
      break;
1756
322k
    }
1757
322k
    FuncAttrs.addAttribute("frame-pointer", FpKind);
1758
1759
322k
    FuncAttrs.addAttribute("less-precise-fpmad",
1760
322k
                           llvm::toStringRef(CodeGenOpts.LessPreciseFPMAD));
1761
1762
322k
    if (CodeGenOpts.NullPointerIsValid)
1763
238
      FuncAttrs.addAttribute(llvm::Attribute::NullPointerIsValid);
1764
1765
322k
    if (CodeGenOpts.FPDenormalMode != llvm::DenormalMode::getIEEE())
1766
2
      FuncAttrs.addAttribute("denormal-fp-math",
1767
2
                             CodeGenOpts.FPDenormalMode.str());
1768
322k
    if (CodeGenOpts.FP32DenormalMode != CodeGenOpts.FPDenormalMode) {
1769
16
      FuncAttrs.addAttribute(
1770
16
          "denormal-fp-math-f32",
1771
16
          CodeGenOpts.FP32DenormalMode.str());
1772
16
    }
1773
1774
322k
    FuncAttrs.addAttribute("no-trapping-math",
1775
322k
                           llvm::toStringRef(LangOpts.getFPExceptionMode() ==
1776
322k
                                             LangOptions::FPE_Ignore));
1777
1778
    // Strict (compliant) code is the default, so only add this attribute to
1779
    // indicate that we are trying to workaround a problem case.
1780
322k
    if (!CodeGenOpts.StrictFloatCastOverflow)
1781
1
      FuncAttrs.addAttribute("strict-float-cast-overflow", "false");
1782
1783
    // TODO: Are these all needed?
1784
    // unsafe/inf/nan/nsz are handled by instruction-level FastMathFlags.
1785
322k
    FuncAttrs.addAttribute("no-infs-fp-math",
1786
322k
                           llvm::toStringRef(LangOpts.NoHonorInfs));
1787
322k
    FuncAttrs.addAttribute("no-nans-fp-math",
1788
322k
                           llvm::toStringRef(LangOpts.NoHonorNaNs));
1789
322k
    FuncAttrs.addAttribute("unsafe-fp-math",
1790
322k
                           llvm::toStringRef(LangOpts.UnsafeFPMath));
1791
322k
    FuncAttrs.addAttribute("use-soft-float",
1792
322k
                           llvm::toStringRef(CodeGenOpts.SoftFloat));
1793
322k
    FuncAttrs.addAttribute("stack-protector-buffer-size",
1794
322k
                           llvm::utostr(CodeGenOpts.SSPBufferSize));
1795
322k
    FuncAttrs.addAttribute("no-signed-zeros-fp-math",
1796
322k
                           llvm::toStringRef(LangOpts.NoSignedZero));
1797
1798
    // TODO: Reciprocal estimate codegen options should apply to instructions?
1799
322k
    const std::vector<std::string> &Recips = CodeGenOpts.Reciprocals;
1800
322k
    if (!Recips.empty())
1801
1
      FuncAttrs.addAttribute("reciprocal-estimates",
1802
1
                             llvm::join(Recips, ","));
1803
1804
322k
    if (!CodeGenOpts.PreferVectorWidth.empty() &&
1805
3
        CodeGenOpts.PreferVectorWidth != "none")
1806
2
      FuncAttrs.addAttribute("prefer-vector-width",
1807
2
                             CodeGenOpts.PreferVectorWidth);
1808
1809
322k
    if (CodeGenOpts.StackRealignment)
1810
1
      FuncAttrs.addAttribute("stackrealign");
1811
322k
    if (CodeGenOpts.Backchain)
1812
2
      FuncAttrs.addAttribute("backchain");
1813
322k
    if (CodeGenOpts.EnableSegmentedStacks)
1814
6
      FuncAttrs.addAttribute("split-stack");
1815
1816
322k
    if (CodeGenOpts.SpeculativeLoadHardening)
1817
9
      FuncAttrs.addAttribute(llvm::Attribute::SpeculativeLoadHardening);
1818
322k
  }
1819
1820
647k
  if (getLangOpts().assumeFunctionsAreConvergent()) {
1821
    // Conservatively, mark all functions and calls in CUDA and OpenCL as
1822
    // convergent (meaning, they may call an intrinsically convergent op, such
1823
    // as __syncthreads() / barrier(), and so can't have certain optimizations
1824
    // applied around them).  LLVM will remove this attribute where it safely
1825
    // can.
1826
74.7k
    FuncAttrs.addAttribute(llvm::Attribute::Convergent);
1827
74.7k
  }
1828
1829
647k
  if (getLangOpts().CUDA && 
getLangOpts().CUDAIsDevice1.31k
) {
1830
    // Exceptions aren't supported in CUDA device code.
1831
814
    FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
1832
814
  }
1833
1834
24
  for (StringRef Attr : CodeGenOpts.DefaultFunctionAttrs) {
1835
24
    StringRef Var, Value;
1836
24
    std::tie(Var, Value) = Attr.split('=');
1837
24
    FuncAttrs.addAttribute(Var, Value);
1838
24
  }
1839
647k
}
1840
1841
22
void CodeGenModule::addDefaultFunctionDefinitionAttributes(llvm::Function &F) {
1842
22
  llvm::AttrBuilder FuncAttrs;
1843
22
  getDefaultFunctionAttributes(F.getName(), F.hasOptNone(),
1844
22
                               /* AttrOnCallSite = */ false, FuncAttrs);
1845
  // TODO: call GetCPUAndFeaturesAttributes?
1846
22
  F.addAttributes(llvm::AttributeList::FunctionIndex, FuncAttrs);
1847
22
}
1848
1849
void CodeGenModule::addDefaultFunctionDefinitionAttributes(
1850
0
                                                   llvm::AttrBuilder &attrs) {
1851
0
  getDefaultFunctionAttributes(/*function name*/ "", /*optnone*/ false,
1852
0
                               /*for call*/ false, attrs);
1853
0
  GetCPUAndFeaturesAttributes(GlobalDecl(), attrs);
1854
0
}
1855
1856
static void addNoBuiltinAttributes(llvm::AttrBuilder &FuncAttrs,
1857
                                   const LangOptions &LangOpts,
1858
647k
                                   const NoBuiltinAttr *NBA = nullptr) {
1859
2.40k
  auto AddNoBuiltinAttr = [&FuncAttrs](StringRef BuiltinName) {
1860
2.40k
    SmallString<32> AttributeName;
1861
2.40k
    AttributeName += "no-builtin-";
1862
2.40k
    AttributeName += BuiltinName;
1863
2.40k
    FuncAttrs.addAttribute(AttributeName);
1864
2.40k
  };
1865
1866
  // First, handle the language options passed through -fno-builtin.
1867
647k
  if (LangOpts.NoBuiltin) {
1868
    // -fno-builtin disables them all.
1869
366k
    FuncAttrs.addAttribute("no-builtins");
1870
366k
    return;
1871
366k
  }
1872
1873
  // Then, add attributes for builtins specified through -fno-builtin-<name>.
1874
280k
  llvm::for_each(LangOpts.NoBuiltinFuncs, AddNoBuiltinAttr);
1875
1876
  // Now, let's check the __attribute__((no_builtin("...")) attribute added to
1877
  // the source.
1878
280k
  if (!NBA)
1879
280k
    return;
1880
1881
  // If there is a wildcard in the builtin names specified through the
1882
  // attribute, disable them all.
1883
9
  if (llvm::is_contained(NBA->builtinNames(), "*")) {
1884
1
    FuncAttrs.addAttribute("no-builtins");
1885
1
    return;
1886
1
  }
1887
1888
  // And last, add the rest of the builtin names.
1889
8
  llvm::for_each(NBA->builtinNames(), AddNoBuiltinAttr);
1890
8
}
1891
1892
/// Construct the IR attribute list of a function or call.
1893
///
1894
/// When adding an attribute, please consider where it should be handled:
1895
///
1896
///   - getDefaultFunctionAttributes is for attributes that are essentially
1897
///     part of the global target configuration (but perhaps can be
1898
///     overridden on a per-function basis).  Adding attributes there
1899
///     will cause them to also be set in frontends that build on Clang's
1900
///     target-configuration logic, as well as for code defined in library
1901
///     modules such as CUDA's libdevice.
1902
///
1903
///   - ConstructAttributeList builds on top of getDefaultFunctionAttributes
1904
///     and adds declaration-specific, convention-specific, and
1905
///     frontend-specific logic.  The last is of particular importance:
1906
///     attributes that restrict how the frontend generates code must be
1907
///     added here rather than getDefaultFunctionAttributes.
1908
///
1909
void CodeGenModule::ConstructAttributeList(
1910
    StringRef Name, const CGFunctionInfo &FI, CGCalleeInfo CalleeInfo,
1911
647k
    llvm::AttributeList &AttrList, unsigned &CallingConv, bool AttrOnCallSite) {
1912
647k
  llvm::AttrBuilder FuncAttrs;
1913
647k
  llvm::AttrBuilder RetAttrs;
1914
1915
  // Collect function IR attributes from the CC lowering.
1916
  // We'll collect the paramete and result attributes later.
1917
647k
  CallingConv = FI.getEffectiveCallingConvention();
1918
647k
  if (FI.isNoReturn())
1919
1.07k
    FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
1920
647k
  if (FI.isCmseNSCall())
1921
62
    FuncAttrs.addAttribute("cmse_nonsecure_call");
1922
1923
  // Collect function IR attributes from the callee prototype if we have one.
1924
647k
  AddAttributesFromFunctionProtoType(getContext(), FuncAttrs,
1925
647k
                                     CalleeInfo.getCalleeFunctionProtoType());
1926
1927
647k
  const Decl *TargetDecl = CalleeInfo.getCalleeDecl().getDecl();
1928
1929
647k
  bool HasOptnone = false;
1930
  // The NoBuiltinAttr attached to the target FunctionDecl.
1931
647k
  const NoBuiltinAttr *NBA = nullptr;
1932
1933
  // Collect function IR attributes based on declaration-specific
1934
  // information.
1935
  // FIXME: handle sseregparm someday...
1936
647k
  if (TargetDecl) {
1937
616k
    if (TargetDecl->hasAttr<ReturnsTwiceAttr>())
1938
41
      FuncAttrs.addAttribute(llvm::Attribute::ReturnsTwice);
1939
616k
    if (TargetDecl->hasAttr<NoThrowAttr>())
1940
5.14k
      FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
1941
616k
    if (TargetDecl->hasAttr<NoReturnAttr>())
1942
0
      FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
1943
616k
    if (TargetDecl->hasAttr<ColdAttr>())
1944
286
      FuncAttrs.addAttribute(llvm::Attribute::Cold);
1945
616k
    if (TargetDecl->hasAttr<NoDuplicateAttr>())
1946
6
      FuncAttrs.addAttribute(llvm::Attribute::NoDuplicate);
1947
616k
    if (TargetDecl->hasAttr<ConvergentAttr>())
1948
6
      FuncAttrs.addAttribute(llvm::Attribute::Convergent);
1949
1950
616k
    if (const FunctionDecl *Fn = dyn_cast<FunctionDecl>(TargetDecl)) {
1951
585k
      AddAttributesFromFunctionProtoType(
1952
585k
          getContext(), FuncAttrs, Fn->getType()->getAs<FunctionProtoType>());
1953
585k
      if (AttrOnCallSite && 
Fn->isReplaceableGlobalAllocationFunction()305k
) {
1954
        // A sane operator new returns a non-aliasing pointer.
1955
3.90k
        auto Kind = Fn->getDeclName().getCXXOverloadedOperator();
1956
3.90k
        if (getCodeGenOpts().AssumeSaneOperatorNew &&
1957
3.89k
            (Kind == OO_New || 
Kind == OO_Array_New2.69k
))
1958
1.48k
          RetAttrs.addAttribute(llvm::Attribute::NoAlias);
1959
3.90k
      }
1960
585k
      const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Fn);
1961
585k
      const bool IsVirtualCall = MD && 
MD->isVirtual()267k
;
1962
      // Don't use [[noreturn]], _Noreturn or [[no_builtin]] for a call to a
1963
      // virtual function. These attributes are not inherited by overloads.
1964
585k
      if (!(AttrOnCallSite && 
IsVirtualCall305k
)) {
1965
580k
        if (Fn->isNoReturn())
1966
2.74k
          FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
1967
580k
        NBA = Fn->getAttr<NoBuiltinAttr>();
1968
580k
      }
1969
585k
    }
1970
1971
    // 'const', 'pure' and 'noalias' attributed functions are also nounwind.
1972
616k
    if (TargetDecl->hasAttr<ConstAttr>()) {
1973
1.43k
      FuncAttrs.addAttribute(llvm::Attribute::ReadNone);
1974
1.43k
      FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
1975
614k
    } else if (TargetDecl->hasAttr<PureAttr>()) {
1976
193
      FuncAttrs.addAttribute(llvm::Attribute::ReadOnly);
1977
193
      FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
1978
614k
    } else if (TargetDecl->hasAttr<NoAliasAttr>()) {
1979
2
      FuncAttrs.addAttribute(llvm::Attribute::ArgMemOnly);
1980
2
      FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
1981
2
    }
1982
616k
    if (TargetDecl->hasAttr<RestrictAttr>())
1983
54
      RetAttrs.addAttribute(llvm::Attribute::NoAlias);
1984
616k
    if (TargetDecl->hasAttr<ReturnsNonNullAttr>() &&
1985
2.30k
        !CodeGenOpts.NullPointerIsValid)
1986
2.30k
      RetAttrs.addAttribute(llvm::Attribute::NonNull);
1987
616k
    if (TargetDecl->hasAttr<AnyX86NoCallerSavedRegistersAttr>())
1988
14
      FuncAttrs.addAttribute("no_caller_saved_registers");
1989
616k
    if (TargetDecl->hasAttr<AnyX86NoCfCheckAttr>())
1990
2
      FuncAttrs.addAttribute(llvm::Attribute::NoCfCheck);
1991
1992
616k
    HasOptnone = TargetDecl->hasAttr<OptimizeNoneAttr>();
1993
616k
    if (auto *AllocSize = TargetDecl->getAttr<AllocSizeAttr>()) {
1994
2.57k
      Optional<unsigned> NumElemsParam;
1995
2.57k
      if (AllocSize->getNumElemsParam().isValid())
1996
133
        NumElemsParam = AllocSize->getNumElemsParam().getLLVMIndex();
1997
2.57k
      FuncAttrs.addAllocSizeAttr(AllocSize->getElemSizeParam().getLLVMIndex(),
1998
2.57k
                                 NumElemsParam);
1999
2.57k
    }
2000
2001
616k
    if (TargetDecl->hasAttr<OpenCLKernelAttr>()) {
2002
447
      if (getLangOpts().OpenCLVersion <= 120) {
2003
        // OpenCL v1.2 Work groups are always uniform
2004
312
        FuncAttrs.addAttribute("uniform-work-group-size", "true");
2005
135
      } else {
2006
        // OpenCL v2.0 Work groups may be whether uniform or not.
2007
        // '-cl-uniform-work-group-size' compile option gets a hint
2008
        // to the compiler that the global work-size be a multiple of
2009
        // the work-group size specified to clEnqueueNDRangeKernel
2010
        // (i.e. work groups are uniform).
2011
135
        FuncAttrs.addAttribute("uniform-work-group-size",
2012
135
                               llvm::toStringRef(CodeGenOpts.UniformWGSize));
2013
135
      }
2014
447
    }
2015
616k
  }
2016
2017
  // Attach "no-builtins" attributes to:
2018
  // * call sites: both `nobuiltin` and "no-builtins" or "no-builtin-<name>".
2019
  // * definitions: "no-builtins" or "no-builtin-<name>" only.
2020
  // The attributes can come from:
2021
  // * LangOpts: -ffreestanding, -fno-builtin, -fno-builtin-<name>
2022
  // * FunctionDecl attributes: __attribute__((no_builtin(...)))
2023
647k
  addNoBuiltinAttributes(FuncAttrs, getLangOpts(), NBA);
2024
2025
  // Collect function IR attributes based on global settiings.
2026
647k
  getDefaultFunctionAttributes(Name, HasOptnone, AttrOnCallSite, FuncAttrs);
2027
2028
  // Override some default IR attributes based on declaration-specific
2029
  // information.
2030
647k
  if (TargetDecl) {
2031
616k
    if (TargetDecl->hasAttr<NoSpeculativeLoadHardeningAttr>())
2032
13
      FuncAttrs.removeAttribute(llvm::Attribute::SpeculativeLoadHardening);
2033
616k
    if (TargetDecl->hasAttr<SpeculativeLoadHardeningAttr>())
2034
13
      FuncAttrs.addAttribute(llvm::Attribute::SpeculativeLoadHardening);
2035
616k
    if (TargetDecl->hasAttr<NoSplitStackAttr>())
2036
8
      FuncAttrs.removeAttribute("split-stack");
2037
2038
    // Add NonLazyBind attribute to function declarations when -fno-plt
2039
    // is used.
2040
    // FIXME: what if we just haven't processed the function definition
2041
    // yet, or if it's an external definition like C99 inline?
2042
616k
    if (CodeGenOpts.NoPLT) {
2043
45
      if (auto *Fn = dyn_cast<FunctionDecl>(TargetDecl)) {
2044
45
        if (!Fn->isDefined() && 
!AttrOnCallSite22
) {
2045
12
          FuncAttrs.addAttribute(llvm::Attribute::NonLazyBind);
2046
12
        }
2047
45
      }
2048
45
    }
2049
616k
  }
2050
2051
  // Collect non-call-site function IR attributes from declaration-specific
2052
  // information.
2053
647k
  if (!AttrOnCallSite) {
2054
322k
    if (TargetDecl && 
TargetDecl->hasAttr<CmseNSEntryAttr>()307k
)
2055
94
      FuncAttrs.addAttribute("cmse_nonsecure_entry");
2056
2057
    // Whether tail calls are enabled.
2058
322k
    auto shouldDisableTailCalls = [&] {
2059
      // Should this be honored in getDefaultFunctionAttributes?
2060
322k
      if (CodeGenOpts.DisableTailCalls)
2061
3
        return true;
2062
2063
322k
      if (!TargetDecl)
2064
14.7k
        return false;
2065
2066
307k
      if (TargetDecl->hasAttr<DisableTailCallsAttr>() ||
2067
307k
          TargetDecl->hasAttr<AnyX86InterruptAttr>())
2068
144
        return true;
2069
2070
307k
      if (CodeGenOpts.NoEscapingBlockTailCalls) {
2071
17
        if (const auto *BD = dyn_cast<BlockDecl>(TargetDecl))
2072
8
          if (!BD->doesNotEscape())
2073
4
            return true;
2074
307k
      }
2075
2076
307k
      return false;
2077
307k
    };
2078
322k
    FuncAttrs.addAttribute("disable-tail-calls",
2079
322k
                           llvm::toStringRef(shouldDisableTailCalls()));
2080
2081
    // CPU/feature overrides.  addDefaultFunctionDefinitionAttributes
2082
    // handles these separately to set them based on the global defaults.
2083
322k
    GetCPUAndFeaturesAttributes(CalleeInfo.getCalleeDecl(), FuncAttrs);
2084
322k
  }
2085
2086
  // Collect attributes from arguments and return values.
2087
647k
  ClangToLLVMArgMapping IRFunctionArgs(getContext(), FI);
2088
2089
647k
  QualType RetTy = FI.getReturnType();
2090
647k
  const ABIArgInfo &RetAI = FI.getReturnInfo();
2091
647k
  switch (RetAI.getKind()) {
2092
22.9k
  case ABIArgInfo::Extend:
2093
22.9k
    if (RetAI.isSignExt())
2094
12.0k
      RetAttrs.addAttribute(llvm::Attribute::SExt);
2095
10.9k
    else
2096
10.9k
      RetAttrs.addAttribute(llvm::Attribute::ZExt);
2097
22.9k
    LLVM_FALLTHROUGH;
2098
351k
  case ABIArgInfo::Direct:
2099
351k
    if (RetAI.getInReg())
2100
30
      RetAttrs.addAttribute(llvm::Attribute::InReg);
2101
351k
    break;
2102
290k
  case ABIArgInfo::Ignore:
2103
290k
    break;
2104
2105
4.31k
  case ABIArgInfo::InAlloca:
2106
4.31k
  case ABIArgInfo::Indirect: {
2107
    // inalloca and sret disable readnone and readonly
2108
4.31k
    FuncAttrs.removeAttribute(llvm::Attribute::ReadOnly)
2109
4.31k
      .removeAttribute(llvm::Attribute::ReadNone);
2110
4.31k
    break;
2111
4.31k
  }
2112
2113
983
  case ABIArgInfo::CoerceAndExpand:
2114
983
    break;
2115
2116
0
  case ABIArgInfo::Expand:
2117
0
  case ABIArgInfo::IndirectAliased:
2118
0
    llvm_unreachable("Invalid ABI kind for return argument");
2119
647k
  }
2120
2121
647k
  if (const auto *RefTy = RetTy->getAs<ReferenceType>()) {
2122
64.4k
    QualType PTy = RefTy->getPointeeType();
2123
64.4k
    if (!PTy->isIncompleteType() && 
PTy->isConstantSizeType()64.4k
)
2124
64.4k
      RetAttrs.addDereferenceableAttr(
2125
64.4k
          getMinimumObjectSize(PTy).getQuantity());
2126
64.4k
    if (getContext().getTargetAddressSpace(PTy) == 0 &&
2127
64.4k
        !CodeGenOpts.NullPointerIsValid)
2128
64.4k
      RetAttrs.addAttribute(llvm::Attribute::NonNull);
2129
64.4k
    if (PTy->isObjectType()) {
2130
63.8k
      llvm::Align Alignment =
2131
63.8k
          getNaturalPointeeTypeAlignment(RetTy).getAsAlign();
2132
63.8k
      RetAttrs.addAlignmentAttr(Alignment);
2133
63.8k
    }
2134
64.4k
  }
2135
2136
647k
  bool hasUsedSRet = false;
2137
647k
  SmallVector<llvm::AttributeSet, 4> ArgAttrs(IRFunctionArgs.totalIRArgs());
2138
2139
  // Attach attributes to sret.
2140
647k
  if (IRFunctionArgs.hasSRetArg()) {
2141
4.30k
    llvm::AttrBuilder SRETAttrs;
2142
4.30k
    SRETAttrs.addStructRetAttr(getTypes().ConvertTypeForMem(RetTy));
2143
4.30k
    hasUsedSRet = true;
2144
4.30k
    if (RetAI.getInReg())
2145
55
      SRETAttrs.addAttribute(llvm::Attribute::InReg);
2146
4.30k
    SRETAttrs.addAlignmentAttr(RetAI.getIndirectAlign().getQuantity());
2147
4.30k
    ArgAttrs[IRFunctionArgs.getSRetArgNo()] =
2148
4.30k
        llvm::AttributeSet::get(getLLVMContext(), SRETAttrs);
2149
4.30k
  }
2150
2151
  // Attach attributes to inalloca argument.
2152
647k
  if (IRFunctionArgs.hasInallocaArg()) {
2153
157
    llvm::AttrBuilder Attrs;
2154
157
    Attrs.addAttribute(llvm::Attribute::InAlloca);
2155
157
    ArgAttrs[IRFunctionArgs.getInallocaArgNo()] =
2156
157
        llvm::AttributeSet::get(getLLVMContext(), Attrs);
2157
157
  }
2158
2159
  // Apply `nonnull` and `dereferencable(N)` to the `this` argument.
2160
647k
  if (FI.isInstanceMethod() && 
!IRFunctionArgs.hasInallocaArg()250k
&&
2161
250k
      !FI.arg_begin()->type->isVoidPointerType()) {
2162
250k
    auto IRArgs = IRFunctionArgs.getIRArgs(0);
2163
2164
250k
    assert(IRArgs.second == 1 && "Expected only a single `this` pointer.");
2165
2166
250k
    llvm::AttrBuilder Attrs;
2167
2168
250k
    if (!CodeGenOpts.NullPointerIsValid &&
2169
250k
        getContext().getTargetAddressSpace(FI.arg_begin()->type) == 0) {
2170
250k
      Attrs.addAttribute(llvm::Attribute::NonNull);
2171
250k
    }
2172
2173
250k
    Attrs.addDereferenceableAttr(
2174
250k
        getMinimumObjectSize(
2175
250k
            FI.arg_begin()->type.castAs<PointerType>()->getPointeeType())
2176
250k
            .getQuantity());
2177
2178
250k
    ArgAttrs[IRArgs.first] = llvm::AttributeSet::get(getLLVMContext(), Attrs);
2179
250k
  }
2180
2181
647k
  unsigned ArgNo = 0;
2182
647k
  for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(),
2183
647k
                                          E = FI.arg_end();
2184
1.70M
       I != E; 
++I, ++ArgNo1.06M
) {
2185
1.06M
    QualType ParamType = I->type;
2186
1.06M
    const ABIArgInfo &AI = I->info;
2187
1.06M
    llvm::AttrBuilder Attrs;
2188
2189
    // Add attribute for padding argument, if necessary.
2190
1.06M
    if (IRFunctionArgs.hasPaddingArg(ArgNo)) {
2191
47
      if (AI.getPaddingInReg()) {
2192
5
        ArgAttrs[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
2193
5
            llvm::AttributeSet::get(
2194
5
                getLLVMContext(),
2195
5
                llvm::AttrBuilder().addAttribute(llvm::Attribute::InReg));
2196
5
      }
2197
47
    }
2198
2199
    // 'restrict' -> 'noalias' is done in EmitFunctionProlog when we
2200
    // have the corresponding parameter variable.  It doesn't make
2201
    // sense to do it here because parameters are so messed up.
2202
1.06M
    switch (AI.getKind()) {
2203
29.4k
    case ABIArgInfo::Extend:
2204
29.4k
      if (AI.isSignExt())
2205
14.3k
        Attrs.addAttribute(llvm::Attribute::SExt);
2206
15.1k
      else
2207
15.1k
        Attrs.addAttribute(llvm::Attribute::ZExt);
2208
29.4k
      LLVM_FALLTHROUGH;
2209
1.03M
    case ABIArgInfo::Direct:
2210
1.03M
      if (ArgNo == 0 && 
FI.isChainCall()575k
)
2211
8
        Attrs.addAttribute(llvm::Attribute::Nest);
2212
1.03M
      else if (AI.getInReg())
2213
586
        Attrs.addAttribute(llvm::Attribute::InReg);
2214
1.03M
      break;
2215
2216
4.62k
    case ABIArgInfo::Indirect: {
2217
4.62k
      if (AI.getInReg())
2218
24
        Attrs.addAttribute(llvm::Attribute::InReg);
2219
2220
4.62k
      if (AI.getIndirectByVal())
2221
1.82k
        Attrs.addByValAttr(getTypes().ConvertTypeForMem(ParamType));
2222
2223
4.62k
      auto *Decl = ParamType->getAsRecordDecl();
2224
4.62k
      if (CodeGenOpts.PassByValueIsNoAlias && 
Decl7
&&
2225
7
          Decl->getArgPassingRestrictions() == RecordDecl::APK_CanPassInRegs)
2226
        // When calling the function, the pointer passed in will be the only
2227
        // reference to the underlying object. Mark it accordingly.
2228
5
        Attrs.addAttribute(llvm::Attribute::NoAlias);
2229
2230
      // TODO: We could add the byref attribute if not byval, but it would
2231
      // require updating many testcases.
2232
2233
4.62k
      CharUnits Align = AI.getIndirectAlign();
2234
2235
      // In a byval argument, it is important that the required
2236
      // alignment of the type is honored, as LLVM might be creating a
2237
      // *new* stack object, and needs to know what alignment to give
2238
      // it. (Sometimes it can deduce a sensible alignment on its own,
2239
      // but not if clang decides it must emit a packed struct, or the
2240
      // user specifies increased alignment requirements.)
2241
      //
2242
      // This is different from indirect *not* byval, where the object
2243
      // exists already, and the align attribute is purely
2244
      // informative.
2245
4.62k
      assert(!Align.isZero());
2246
2247
      // For now, only add this when we have a byval argument.
2248
      // TODO: be less lazy about updating test cases.
2249
4.62k
      if (AI.getIndirectByVal())
2250
1.82k
        Attrs.addAlignmentAttr(Align.getQuantity());
2251
2252
      // byval disables readnone and readonly.
2253
4.62k
      FuncAttrs.removeAttribute(llvm::Attribute::ReadOnly)
2254
4.62k
        .removeAttribute(llvm::Attribute::ReadNone);
2255
2256
4.62k
      break;
2257
29.4k
    }
2258
12
    case ABIArgInfo::IndirectAliased: {
2259
12
      CharUnits Align = AI.getIndirectAlign();
2260
12
      Attrs.addByRefAttr(getTypes().ConvertTypeForMem(ParamType));
2261
12
      Attrs.addAlignmentAttr(Align.getQuantity());
2262
12
      break;
2263
29.4k
    }
2264
17.3k
    case ABIArgInfo::Ignore:
2265
17.3k
    case ABIArgInfo::Expand:
2266
17.3k
    case ABIArgInfo::CoerceAndExpand:
2267
17.3k
      break;
2268
2269
311
    case ABIArgInfo::InAlloca:
2270
      // inalloca disables readnone and readonly.
2271
311
      FuncAttrs.removeAttribute(llvm::Attribute::ReadOnly)
2272
311
          .removeAttribute(llvm::Attribute::ReadNone);
2273
311
      continue;
2274
1.06M
    }
2275
2276
1.06M
    if (const auto *RefTy = ParamType->getAs<ReferenceType>()) {
2277
133k
      QualType PTy = RefTy->getPointeeType();
2278
133k
      if (!PTy->isIncompleteType() && 
PTy->isConstantSizeType()133k
)
2279
133k
        Attrs.addDereferenceableAttr(
2280
133k
            getMinimumObjectSize(PTy).getQuantity());
2281
133k
      if (getContext().getTargetAddressSpace(PTy) == 0 &&
2282
133k
          !CodeGenOpts.NullPointerIsValid)
2283
133k
        Attrs.addAttribute(llvm::Attribute::NonNull);
2284
133k
      if (PTy->isObjectType()) {
2285
131k
        llvm::Align Alignment =
2286
131k
            getNaturalPointeeTypeAlignment(ParamType).getAsAlign();
2287
131k
        Attrs.addAlignmentAttr(Alignment);
2288
131k
      }
2289
133k
    }
2290
2291
1.06M
    switch (FI.getExtParameterInfo(ArgNo).getABI()) {
2292
1.06M
    case ParameterABI::Ordinary:
2293
1.06M
      break;
2294
2295
48
    case ParameterABI::SwiftIndirectResult: {
2296
      // Add 'sret' if we haven't already used it for something, but
2297
      // only if the result is void.
2298
48
      if (!hasUsedSRet && 
RetTy->isVoidType()24
) {
2299
8
        Attrs.addStructRetAttr(getTypes().ConvertTypeForMem(ParamType));
2300
8
        hasUsedSRet = true;
2301
8
      }
2302
2303
      // Add 'noalias' in either case.
2304
48
      Attrs.addAttribute(llvm::Attribute::NoAlias);
2305
2306
      // Add 'dereferenceable' and 'alignment'.
2307
48
      auto PTy = ParamType->getPointeeType();
2308
48
      if (!PTy->isIncompleteType() && PTy->isConstantSizeType()) {
2309
48
        auto info = getContext().getTypeInfoInChars(PTy);
2310
48
        Attrs.addDereferenceableAttr(info.Width.getQuantity());
2311
48
        Attrs.addAlignmentAttr(info.Align.getAsAlign());
2312
48
      }
2313
48
      break;
2314
0
    }
2315
2316
24
    case ParameterABI::SwiftErrorResult:
2317
24
      Attrs.addAttribute(llvm::Attribute::SwiftError);
2318
24
      break;
2319
2320
40
    case ParameterABI::SwiftContext:
2321
40
      Attrs.addAttribute(llvm::Attribute::SwiftSelf);
2322
40
      break;
2323
1.06M
    }
2324
2325
1.06M
    if (FI.getExtParameterInfo(ArgNo).isNoEscape())
2326
58
      Attrs.addAttribute(llvm::Attribute::NoCapture);
2327
2328
1.06M
    if (Attrs.hasAttributes()) {
2329
165k
      unsigned FirstIRArg, NumIRArgs;
2330
165k
      std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
2331
330k
      for (unsigned i = 0; i < NumIRArgs; 
i++165k
)
2332
165k
        ArgAttrs[FirstIRArg + i] =
2333
165k
            llvm::AttributeSet::get(getLLVMContext(), Attrs);
2334
165k
    }
2335
1.06M
  }
2336
647k
  assert(ArgNo == FI.arg_size());
2337
2338
647k
  AttrList = llvm::AttributeList::get(
2339
647k
      getLLVMContext(), llvm::AttributeSet::get(getLLVMContext(), FuncAttrs),
2340
647k
      llvm::AttributeSet::get(getLLVMContext(), RetAttrs), ArgAttrs);
2341
647k
}
2342
2343
/// An argument came in as a promoted argument; demote it back to its
2344
/// declared type.
2345
static llvm::Value *emitArgumentDemotion(CodeGenFunction &CGF,
2346
                                         const VarDecl *var,
2347
14
                                         llvm::Value *value) {
2348
14
  llvm::Type *varType = CGF.ConvertType(var->getType());
2349
2350
  // This can happen with promotions that actually don't change the
2351
  // underlying type, like the enum promotions.
2352
14
  if (value->getType() == varType) 
return value0
;
2353
2354
14
  assert((varType->isIntegerTy() || varType->isFloatingPointTy())
2355
14
         && "unexpected promotion type");
2356
2357
14
  if (isa<llvm::IntegerType>(varType))
2358
8
    return CGF.Builder.CreateTrunc(value, varType, "arg.unpromote");
2359
2360
6
  return CGF.Builder.CreateFPCast(value, varType, "arg.unpromote");
2361
6
}
2362
2363
/// Returns the attribute (either parameter attribute, or function
2364
/// attribute), which declares argument ArgNo to be non-null.
2365
static const NonNullAttr *getNonNullAttr(const Decl *FD, const ParmVarDecl *PVD,
2366
103k
                                         QualType ArgType, unsigned ArgNo) {
2367
  // FIXME: __attribute__((nonnull)) can also be applied to:
2368
  //   - references to pointers, where the pointee is known to be
2369
  //     nonnull (apparently a Clang extension)
2370
  //   - transparent unions containing pointers
2371
  // In the former case, LLVM IR cannot represent the constraint. In
2372
  // the latter case, we have no guarantee that the transparent union
2373
  // is in fact passed as a pointer.
2374
103k
  if (!ArgType->isAnyPointerType() && 
!ArgType->isBlockPointerType()47.3k
)
2375
47.2k
    return nullptr;
2376
  // First, check attribute on parameter itself.
2377
56.0k
  if (PVD) {
2378
56.0k
    if (auto ParmNNAttr = PVD->getAttr<NonNullAttr>())
2379
10
      return ParmNNAttr;
2380
56.0k
  }
2381
  // Check function attributes.
2382
56.0k
  if (!FD)
2383
5
    return nullptr;
2384
56.0k
  for (const auto *NNAttr : FD->specific_attrs<NonNullAttr>()) {
2385
32
    if (NNAttr->isNonNull(ArgNo))
2386
28
      return NNAttr;
2387
32
  }
2388
56.0k
  return nullptr;
2389
56.0k
}
2390
2391
namespace {
2392
  struct CopyBackSwiftError final : EHScopeStack::Cleanup {
2393
    Address Temp;
2394
    Address Arg;
2395
16
    CopyBackSwiftError(Address temp, Address arg) : Temp(temp), Arg(arg) {}
2396
16
    void Emit(CodeGenFunction &CGF, Flags flags) override {
2397
16
      llvm::Value *errorValue = CGF.Builder.CreateLoad(Temp);
2398
16
      CGF.Builder.CreateStore(errorValue, Arg);
2399
16
    }
2400
  };
2401
}
2402
2403
void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
2404
                                         llvm::Function *Fn,
2405
293k
                                         const FunctionArgList &Args) {
2406
293k
  if (CurCodeDecl && 
CurCodeDecl->hasAttr<NakedAttr>()285k
)
2407
    // Naked functions don't have prologues.
2408
7
    return;
2409
2410
  // If this is an implicit-return-zero function, go ahead and
2411
  // initialize the return value.  TODO: it might be nice to have
2412
  // a more general mechanism for this that didn't require synthesized
2413
  // return statements.
2414
293k
  if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(CurCodeDecl)) {
2415
251k
    if (FD->hasImplicitReturnZero()) {
2416
5.30k
      QualType RetTy = FD->getReturnType().getUnqualifiedType();
2417
5.30k
      llvm::Type* LLVMTy = CGM.getTypes().ConvertType(RetTy);
2418
5.30k
      llvm::Constant* Zero = llvm::Constant::getNullValue(LLVMTy);
2419
5.30k
      Builder.CreateStore(Zero, ReturnValue);
2420
5.30k
    }
2421
251k
  }
2422
2423
  // FIXME: We no longer need the types from FunctionArgList; lift up and
2424
  // simplify.
2425
2426
293k
  ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), FI);
2427
293k
  assert(Fn->arg_size() == IRFunctionArgs.totalIRArgs());
2428
2429
  // If we're using inalloca, all the memory arguments are GEPs off of the last
2430
  // parameter, which is a pointer to the complete memory area.
2431
293k
  Address ArgStruct = Address::invalid();
2432
293k
  if (IRFunctionArgs.hasInallocaArg()) {
2433
41
    ArgStruct = Address(Fn->getArg(IRFunctionArgs.getInallocaArgNo()),
2434
41
                        FI.getArgStructAlignment());
2435
2436
41
    assert(ArgStruct.getType() == FI.getArgStruct()->getPointerTo());
2437
41
  }
2438
2439
  // Name the struct return parameter.
2440
293k
  if (IRFunctionArgs.hasSRetArg()) {
2441
2.54k
    auto AI = Fn->getArg(IRFunctionArgs.getSRetArgNo());
2442
2.54k
    AI->setName("agg.result");
2443
2.54k
    AI->addAttr(llvm::Attribute::NoAlias);
2444
2.54k
  }
2445
2446
  // Track if we received the parameter as a pointer (indirect, byval, or
2447
  // inalloca).  If already have a pointer, EmitParmDecl doesn't need to copy it
2448
  // into a local alloca for us.
2449
293k
  SmallVector<ParamValue, 16> ArgVals;
2450
293k
  ArgVals.reserve(Args.size());
2451
2452
  // Create a pointer value for every parameter declaration.  This usually
2453
  // entails copying one or more LLVM IR arguments into an alloca.  Don't push
2454
  // any cleanups or do anything that might unwind.  We do that separately, so
2455
  // we can push the cleanups in the correct order for the ABI.
2456
293k
  assert(FI.arg_size() == Args.size() &&
2457
293k
         "Mismatch between function signature & arguments.");
2458
293k
  unsigned ArgNo = 0;
2459
293k
  CGFunctionInfo::const_arg_iterator info_it = FI.arg_begin();
2460
293k
  for (FunctionArgList::const_iterator i = Args.begin(), e = Args.end();
2461
777k
       i != e; 
++i, ++info_it, ++ArgNo483k
) {
2462
483k
    const VarDecl *Arg = *i;
2463
483k
    const ABIArgInfo &ArgI = info_it->info;
2464
2465
483k
    bool isPromoted =
2466
483k
      isa<ParmVarDecl>(Arg) && 
cast<ParmVarDecl>(Arg)->isKNRPromoted()298k
;
2467
    // We are converting from ABIArgInfo type to VarDecl type directly, unless
2468
    // the parameter is promoted. In this case we convert to
2469
    // CGFunctionInfo::ArgInfo type with subsequent argument demotion.
2470
483k
    QualType Ty = isPromoted ? 
info_it->type14
: Arg->getType();
2471
483k
    assert(hasScalarEvaluationKind(Ty) ==
2472
483k
           hasScalarEvaluationKind(Arg->getType()));
2473
2474
483k
    unsigned FirstIRArg, NumIRArgs;
2475
483k
    std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
2476
2477
483k
    switch (ArgI.getKind()) {
2478
73
    case ABIArgInfo::InAlloca: {
2479
73
      assert(NumIRArgs == 0);
2480
73
      auto FieldIndex = ArgI.getInAllocaFieldIndex();
2481
73
      Address V =
2482
73
          Builder.CreateStructGEP(ArgStruct, FieldIndex, Arg->getName());
2483
73
      if (ArgI.getInAllocaIndirect())
2484
6
        V = Address(Builder.CreateLoad(V),
2485
6
                    getContext().getTypeAlignInChars(Ty));
2486
73
      ArgVals.push_back(ParamValue::forIndirect(V));
2487
73
      break;
2488
0
    }
2489
2490
2.24k
    case ABIArgInfo::Indirect:
2491
2.24k
    case ABIArgInfo::IndirectAliased: {
2492
2.24k
      assert(NumIRArgs == 1);
2493
2.24k
      Address ParamAddr =
2494
2.24k
          Address(Fn->getArg(FirstIRArg), ArgI.getIndirectAlign());
2495
2496
2.24k
      if (!hasScalarEvaluationKind(Ty)) {
2497
        // Aggregates and complex variables are accessed by reference. All we
2498
        // need to do is realign the value, if requested. Also, if the address
2499
        // may be aliased, copy it to ensure that the parameter variable is
2500
        // mutable and has a unique adress, as C requires.
2501
1.87k
        Address V = ParamAddr;
2502
1.87k
        if (ArgI.getIndirectRealign() || 
ArgI.isIndirectAliased()1.84k
) {
2503
41
          Address AlignedTemp = CreateMemTemp(Ty, "coerce");
2504
2505
          // Copy from the incoming argument pointer to the temporary with the
2506
          // appropriate alignment.
2507
          //
2508
          // FIXME: We should have a common utility for generating an aggregate
2509
          // copy.
2510
41
          CharUnits Size = getContext().getTypeSizeInChars(Ty);
2511
41
          Builder.CreateMemCpy(
2512
41
              AlignedTemp.getPointer(), AlignedTemp.getAlignment().getAsAlign(),
2513
41
              ParamAddr.getPointer(), ParamAddr.getAlignment().getAsAlign(),
2514
41
              llvm::ConstantInt::get(IntPtrTy, Size.getQuantity()));
2515
41
          V = AlignedTemp;
2516
41
        }
2517
1.87k
        ArgVals.push_back(ParamValue::forIndirect(V));
2518
370
      } else {
2519
        // Load scalar value from indirect argument.
2520
370
        llvm::Value *V =
2521
370
            EmitLoadOfScalar(ParamAddr, false, Ty, Arg->getBeginLoc());
2522
2523
370
        if (isPromoted)
2524
0
          V = emitArgumentDemotion(*this, Arg, V);
2525
370
        ArgVals.push_back(ParamValue::forDirect(V));
2526
370
      }
2527
2.24k
      break;
2528
2.24k
    }
2529
2530
472k
    case ABIArgInfo::Extend:
2531
472k
    case ABIArgInfo::Direct: {
2532
472k
      auto AI = Fn->getArg(FirstIRArg);
2533
472k
      llvm::Type *LTy = ConvertType(Arg->getType());
2534
2535
      // Prepare parameter attributes. So far, only attributes for pointer
2536
      // parameters are prepared. See
2537
      // http://llvm.org/docs/LangRef.html#paramattrs.
2538
472k
      if (ArgI.getDirectOffset() == 0 && 
LTy->isPointerTy()472k
&&
2539
260k
          ArgI.getCoerceToType()->isPointerTy()) {
2540
260k
        assert(NumIRArgs == 1);
2541
2542
260k
        if (const ParmVarDecl *PVD = dyn_cast<ParmVarDecl>(Arg)) {
2543
          // Set `nonnull` attribute if any.
2544
103k
          if (getNonNullAttr(CurCodeDecl, PVD, PVD->getType(),
2545
103k
                             PVD->getFunctionScopeIndex()) &&
2546
22
              !CGM.getCodeGenOpts().NullPointerIsValid)
2547
13
            AI->addAttr(llvm::Attribute::NonNull);
2548
2549
103k
          QualType OTy = PVD->getOriginalType();
2550
103k
          if (const auto *ArrTy =
2551
534
              getContext().getAsConstantArrayType(OTy)) {
2552
            // A C99 array parameter declaration with the static keyword also
2553
            // indicates dereferenceability, and if the size is constant we can
2554
            // use the dereferenceable attribute (which requires the size in
2555
            // bytes).
2556
534
            if (ArrTy->getSizeModifier() == ArrayType::Static) {
2557
6
              QualType ETy = ArrTy->getElementType();
2558
6
              llvm::Align Alignment =
2559
6
                  CGM.getNaturalTypeAlignment(ETy).getAsAlign();
2560
6
              AI->addAttrs(llvm::AttrBuilder().addAlignmentAttr(Alignment));
2561
6
              uint64_t ArrSize = ArrTy->getSize().getZExtValue();
2562
6
              if (!ETy->isIncompleteType() && ETy->isConstantSizeType() &&
2563
6
                  ArrSize) {
2564
4
                llvm::AttrBuilder Attrs;
2565
4
                Attrs.addDereferenceableAttr(
2566
4
                    getContext().getTypeSizeInChars(ETy).getQuantity() *
2567
4
                    ArrSize);
2568
4
                AI->addAttrs(Attrs);
2569
2
              } else if (getContext().getTargetInfo().getNullPointerValue(
2570
2
                             ETy.getAddressSpace()) == 0 &&
2571
2
                         !CGM.getCodeGenOpts().NullPointerIsValid) {
2572
1
                AI->addAttr(llvm::Attribute::NonNull);
2573
1
              }
2574
6
            }
2575
102k
          } else if (const auto *ArrTy =
2576
102
                     getContext().getAsVariableArrayType(OTy)) {
2577
            // For C99 VLAs with the static keyword, we don't know the size so
2578
            // we can't use the dereferenceable attribute, but in addrspace(0)
2579
            // we know that it must be nonnull.
2580
102
            if (ArrTy->getSizeModifier() == VariableArrayType::Static) {
2581
2
              QualType ETy = ArrTy->getElementType();
2582
2
              llvm::Align Alignment =
2583
2
                  CGM.getNaturalTypeAlignment(ETy).getAsAlign();
2584
2
              AI->addAttrs(llvm::AttrBuilder().addAlignmentAttr(Alignment));
2585
2
              if (!getContext().getTargetAddressSpace(ETy) &&
2586
2
                  !CGM.getCodeGenOpts().NullPointerIsValid)
2587
1
                AI->addAttr(llvm::Attribute::NonNull);
2588
2
            }
2589
102
          }
2590
2591
          // Set `align` attribute if any.
2592
103k
          const auto *AVAttr = PVD->getAttr<AlignValueAttr>();
2593
103k
          if (!AVAttr)
2594
103k
            if (const auto *TOTy = dyn_cast<TypedefType>(OTy))
2595
6.54k
              AVAttr = TOTy->getDecl()->getAttr<AlignValueAttr>();
2596
103k
          if (AVAttr && 
!SanOpts.has(SanitizerKind::Alignment)13
) {
2597
            // If alignment-assumption sanitizer is enabled, we do *not* add
2598
            // alignment attribute here, but emit normal alignment assumption,
2599
            // so the UBSAN check could function.
2600
10
            llvm::ConstantInt *AlignmentCI =
2601
10
                cast<llvm::ConstantInt>(EmitScalarExpr(AVAttr->getAlignment()));
2602
10
            unsigned AlignmentInt =
2603
10
                AlignmentCI->getLimitedValue(llvm::Value::MaximumAlignment);
2604
10
            if (AI->getParamAlign().valueOrOne() < AlignmentInt) {
2605
10
              AI->removeAttr(llvm::Attribute::AttrKind::Alignment);
2606
10
              AI->addAttrs(llvm::AttrBuilder().addAlignmentAttr(
2607
10
                  llvm::Align(AlignmentInt)));
2608
10
            }
2609
10
          }
2610
103k
        }
2611
2612
        // Set 'noalias' if an argument type has the `restrict` qualifier.
2613
260k
        if (Arg->getType().isRestrictQualified())
2614
30.9k
          AI->addAttr(llvm::Attribute::NoAlias);
2615
260k
      }
2616
2617
      // Prepare the argument value. If we have the trivial case, handle it
2618
      // with no muss and fuss.
2619
472k
      if (!isa<llvm::StructType>(ArgI.getCoerceToType()) &&
2620
471k
          ArgI.getCoerceToType() == ConvertType(Ty) &&
2621
466k
          ArgI.getDirectOffset() == 0) {
2622
466k
        assert(NumIRArgs == 1);
2623
2624
        // LLVM expects swifterror parameters to be used in very restricted
2625
        // ways.  Copy the value into a less-restricted temporary.
2626
466k
        llvm::Value *V = AI;
2627
466k
        if (FI.getExtParameterInfo(ArgNo).getABI()
2628
16
              == ParameterABI::SwiftErrorResult) {
2629
16
          QualType pointeeTy = Ty->getPointeeType();
2630
16
          assert(pointeeTy->isPointerType());
2631
16
          Address temp =
2632
16
            CreateMemTemp(pointeeTy, getPointerAlign(), "swifterror.temp");
2633
16
          Address arg = Address(V, getContext().getTypeAlignInChars(pointeeTy));
2634
16
          llvm::Value *incomingErrorValue = Builder.CreateLoad(arg);
2635
16
          Builder.CreateStore(incomingErrorValue, temp);
2636
16
          V = temp.getPointer();
2637
2638
          // Push a cleanup to copy the value back at the end of the function.
2639
          // The convention does not guarantee that the value will be written
2640
          // back if the function exits with an unwind exception.
2641
16
          EHStack.pushCleanup<CopyBackSwiftError>(NormalCleanup, temp, arg);
2642
16
        }
2643
2644
        // Ensure the argument is the correct type.
2645
466k
        if (V->getType() != ArgI.getCoerceToType())
2646
0
          V = Builder.CreateBitCast(V, ArgI.getCoerceToType());
2647
2648
466k
        if (isPromoted)
2649
14
          V = emitArgumentDemotion(*this, Arg, V);
2650
2651
        // Because of merging of function types from multiple decls it is
2652
        // possible for the type of an argument to not match the corresponding
2653
        // type in the function type. Since we are codegening the callee
2654
        // in here, add a cast to the argument type.
2655
466k
        llvm::Type *LTy = ConvertType(Arg->getType());
2656
466k
        if (V->getType() != LTy)
2657
0
          V = Builder.CreateBitCast(V, LTy);
2658
2659
466k
        ArgVals.push_back(ParamValue::forDirect(V));
2660
466k
        break;
2661
466k
      }
2662
2663
5.84k
      Address Alloca = CreateMemTemp(Ty, getContext().getDeclAlign(Arg),
2664
5.84k
                                     Arg->getName());
2665
2666
      // Pointer to store into.
2667
5.84k
      Address Ptr = emitAddressAtOffset(*this, Alloca, ArgI);
2668
2669
      // Fast-isel and the optimizer generally like scalar values better than
2670
      // FCAs, so we flatten them if this is safe to do for this argument.
2671
5.84k
      llvm::StructType *STy = dyn_cast<llvm::StructType>(ArgI.getCoerceToType());
2672
5.84k
      if (ArgI.isDirect() && ArgI.getCanBeFlattened() && 
STy5.57k
&&
2673
954
          STy->getNumElements() > 1) {
2674
929
        uint64_t SrcSize = CGM.getDataLayout().getTypeAllocSize(STy);
2675
929
        llvm::Type *DstTy = Ptr.getElementType();
2676
929
        uint64_t DstSize = CGM.getDataLayout().getTypeAllocSize(DstTy);
2677
2678
929
        Address AddrToStoreInto = Address::invalid();
2679
929
        if (SrcSize <= DstSize) {
2680
842
          AddrToStoreInto = Builder.CreateElementBitCast(Ptr, STy);
2681
87
        } else {
2682
87
          AddrToStoreInto =
2683
87
            CreateTempAlloca(STy, Alloca.getAlignment(), "coerce");
2684
87
        }
2685
2686
929
        assert(STy->getNumElements() == NumIRArgs);
2687
2.89k
        for (unsigned i = 0, e = STy->getNumElements(); i != e; 
++i1.96k
) {
2688
1.96k
          auto AI = Fn->getArg(FirstIRArg + i);
2689
1.96k
          AI->setName(Arg->getName() + ".coerce" + Twine(i));
2690
1.96k
          Address EltPtr = Builder.CreateStructGEP(AddrToStoreInto, i);
2691
1.96k
          Builder.CreateStore(AI, EltPtr);
2692
1.96k
        }
2693
2694
929
        if (SrcSize > DstSize) {
2695
87
          Builder.CreateMemCpy(Ptr, AddrToStoreInto, DstSize);
2696
87
        }
2697
2698
4.91k
      } else {
2699
        // Simple case, just do a coerced store of the argument into the alloca.
2700
4.91k
        assert(NumIRArgs == 1);
2701
4.91k
        auto AI = Fn->getArg(FirstIRArg);
2702
4.91k
        AI->setName(Arg->getName() + ".coerce");
2703
4.91k
        CreateCoercedStore(AI, Ptr, /*DstIsVolatile=*/false, *this);
2704
4.91k
      }
2705
2706
      // Match to what EmitParmDecl is expecting for this type.
2707
5.84k
      if (CodeGenFunction::hasScalarEvaluationKind(Ty)) {
2708
1.28k
        llvm::Value *V =
2709
1.28k
            EmitLoadOfScalar(Alloca, false, Ty, Arg->getBeginLoc());
2710
1.28k
        if (isPromoted)
2711
0
          V = emitArgumentDemotion(*this, Arg, V);
2712
1.28k
        ArgVals.push_back(ParamValue::forDirect(V));
2713
4.56k
      } else {
2714
4.56k
        ArgVals.push_back(ParamValue::forIndirect(Alloca));
2715
4.56k
      }
2716
5.84k
      break;
2717
5.84k
    }
2718
2719
565
    case ABIArgInfo::CoerceAndExpand: {
2720
      // Reconstruct into a temporary.
2721
565
      Address alloca = CreateMemTemp(Ty, getContext().getDeclAlign(Arg));
2722
565
      ArgVals.push_back(ParamValue::forIndirect(alloca));
2723
2724
565
      auto coercionType = ArgI.getCoerceAndExpandType();
2725
565
      alloca = Builder.CreateElementBitCast(alloca, coercionType);
2726
2727
565
      unsigned argIndex = FirstIRArg;
2728
1.75k
      for (unsigned i = 0, e = coercionType->getNumElements(); i != e; 
++i1.19k
) {
2729
1.19k
        llvm::Type *eltType = coercionType->getElementType(i);
2730
1.19k
        if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType))
2731
7
          continue;
2732
2733
1.18k
        auto eltAddr = Builder.CreateStructGEP(alloca, i);
2734
1.18k
        auto elt = Fn->getArg(argIndex++);
2735
1.18k
        Builder.CreateStore(elt, eltAddr);
2736
1.18k
      }
2737
565
      assert(argIndex == FirstIRArg + NumIRArgs);
2738
565
      break;
2739
5.84k
    }
2740
2741
100
    case ABIArgInfo::Expand: {
2742
      // If this structure was expanded into multiple arguments then
2743
      // we need to create a temporary and reconstruct it from the
2744
      // arguments.
2745
100
      Address Alloca = CreateMemTemp(Ty, getContext().getDeclAlign(Arg));
2746
100
      LValue LV = MakeAddrLValue(Alloca, Ty);
2747
100
      ArgVals.push_back(ParamValue::forIndirect(Alloca));
2748
2749
100
      auto FnArgIter = Fn->arg_begin() + FirstIRArg;
2750
100
      ExpandTypeFromArgs(Ty, LV, FnArgIter);
2751
100
      assert(FnArgIter == Fn->arg_begin() + FirstIRArg + NumIRArgs);
2752
340
      for (unsigned i = 0, e = NumIRArgs; i != e; 
++i240
) {
2753
240
        auto AI = Fn->getArg(FirstIRArg + i);
2754
240
        AI->setName(Arg->getName() + "." + Twine(i));
2755
240
      }
2756
100
      break;
2757
5.84k
    }
2758
2759
7.91k
    case ABIArgInfo::Ignore:
2760
7.91k
      assert(NumIRArgs == 0);
2761
      // Initialize the local variable appropriately.
2762
7.91k
      if (!hasScalarEvaluationKind(Ty)) {
2763
7.91k
        ArgVals.push_back(ParamValue::forIndirect(CreateMemTemp(Ty)));
2764
0
      } else {
2765
0
        llvm::Value *U = llvm::UndefValue::get(ConvertType(Arg->getType()));
2766
0
        ArgVals.push_back(ParamValue::forDirect(U));
2767
0
      }
2768
7.91k
      break;
2769
483k
    }
2770
483k
  }
2771
2772
293k
  if (getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) {
2773
30.9k
    for (int I = Args.size() - 1; I >= 0; 
--I17.8k
)
2774
17.8k
      EmitParmDecl(*Args[I], ArgVals[I], I + 1);
2775
280k
  } else {
2776
746k
    for (unsigned I = 0, E = Args.size(); I != E; 
++I465k
)
2777
465k
      EmitParmDecl(*Args[I], ArgVals[I], I + 1);
2778
280k
  }
2779
293k
}
2780
2781
8
static void eraseUnusedBitCasts(llvm::Instruction *insn) {
2782
12
  while (insn->use_empty()) {
2783
12
    llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(insn);
2784
12
    if (!bitcast) 
return8
;
2785
2786
    // This is "safe" because we would have used a ConstantExpr otherwise.
2787
4
    insn = cast<llvm::Instruction>(bitcast->getOperand(0));
2788
4
    bitcast->eraseFromParent();
2789
4
  }
2790
8
}
2791
2792
/// Try to emit a fused autorelease of a return result.
2793
static llvm::Value *tryEmitFusedAutoreleaseOfResult(CodeGenFunction &CGF,
2794
87
                                                    llvm::Value *result) {
2795
  // We must be immediately followed the cast.
2796
87
  llvm::BasicBlock *BB = CGF.Builder.GetInsertBlock();
2797
87
  if (BB->empty()) 
return nullptr0
;
2798
87
  if (&BB->back() != result) 
return nullptr24
;
2799
2800
63
  llvm::Type *resultType = result->getType();
2801
2802
  // result is in a BasicBlock and is therefore an Instruction.
2803
63
  llvm::Instruction *generator = cast<llvm::Instruction>(result);
2804
2805
63
  SmallVector<llvm::Instruction *, 4> InstsToKill;
2806
2807
  // Look for:
2808
  //  %generator = bitcast %type1* %generator2 to %type2*
2809
95
  while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(generator)) {
2810
    // We would have emitted this as a constant if the operand weren't
2811
    // an Instruction.
2812
32
    generator = cast<llvm::Instruction>(bitcast->getOperand(0));
2813
2814
    // Require the generator to be immediately followed by the cast.
2815
32
    if (generator->getNextNode() != bitcast)
2816
0
      return nullptr;
2817
2818
32
    InstsToKill.push_back(bitcast);
2819
32
  }
2820
2821
  // Look for:
2822
  //   %generator = call i8* @objc_retain(i8* %originalResult)
2823
  // or
2824
  //   %generator = call i8* @objc_retainAutoreleasedReturnValue(i8* %originalResult)
2825
63
  llvm::CallInst *call = dyn_cast<llvm::CallInst>(generator);
2826
63
  if (!call) 
return nullptr13
;
2827
2828
50
  bool doRetainAutorelease;
2829
2830
50
  if (call->getCalledOperand() == CGF.CGM.getObjCEntrypoints().objc_retain) {
2831
8
    doRetainAutorelease = true;
2832
42
  } else if (call->getCalledOperand() ==
2833
19
             CGF.CGM.getObjCEntrypoints().objc_retainAutoreleasedReturnValue) {
2834
19
    doRetainAutorelease = false;
2835
2836
    // If we emitted an assembly marker for this call (and the
2837
    // ARCEntrypoints field should have been set if so), go looking
2838
    // for that call.  If we can't find it, we can't do this
2839
    // optimization.  But it should always be the immediately previous
2840
    // instruction, unless we needed bitcasts around the call.
2841
19
    if (CGF.CGM.getObjCEntrypoints().retainAutoreleasedReturnValueMarker) {
2842
12
      llvm::Instruction *prev = call->getPrevNode();
2843
12
      assert(prev);
2844
12
      if (isa<llvm::BitCastInst>(prev)) {
2845
8
        prev = prev->getPrevNode();
2846
8
        assert(prev);
2847
8
      }
2848
12
      assert(isa<llvm::CallInst>(prev));
2849
12
      assert(cast<llvm::CallInst>(prev)->getCalledOperand() ==
2850
12
             CGF.CGM.getObjCEntrypoints().retainAutoreleasedReturnValueMarker);
2851
12
      InstsToKill.push_back(prev);
2852
12
    }
2853
23
  } else {
2854
23
    return nullptr;
2855
23
  }
2856
2857
27
  result = call->getArgOperand(0);
2858
27
  InstsToKill.push_back(call);
2859
2860
  // Keep killing bitcasts, for sanity.  Note that we no longer care
2861
  // about precise ordering as long as there's exactly one use.
2862
43
  while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(result)) {
2863
16
    if (!bitcast->hasOneUse()) 
break0
;
2864
16
    InstsToKill.push_back(bitcast);
2865
16
    result = bitcast->getOperand(0);
2866
16
  }
2867
2868
  // Delete all the unnecessary instructions, from latest to earliest.
2869
27
  for (auto *I : InstsToKill)
2870
77
    I->eraseFromParent();
2871
2872
  // Do the fused retain/autorelease if we were asked to.
2873
27
  if (doRetainAutorelease)
2874
8
    result = CGF.EmitARCRetainAutoreleaseReturnValue(result);
2875
2876
  // Cast back to the result type.
2877
27
  return CGF.Builder.CreateBitCast(result, resultType);
2878
27
}
2879
2880
/// If this is a +1 of the value of an immutable 'self', remove it.
2881
static llvm::Value *tryRemoveRetainOfSelf(CodeGenFunction &CGF,
2882
113
                                          llvm::Value *result) {
2883
  // This is only applicable to a method with an immutable 'self'.
2884
113
  const ObjCMethodDecl *method =
2885
113
    dyn_cast_or_null<ObjCMethodDecl>(CGF.CurCodeDecl);
2886
113
  if (!method) 
return nullptr94
;
2887
19
  const VarDecl *self = method->getSelfDecl();
2888
19
  if (!self->getType().isConstQualified()) 
return nullptr0
;
2889
2890
  // Look for a retain call.
2891
19
  llvm::CallInst *retainCall =
2892
19
    dyn_cast<llvm::CallInst>(result->stripPointerCasts());
2893
19
  if (!retainCall || retainCall->getCalledOperand() !=
2894
15
                         CGF.CGM.getObjCEntrypoints().objc_retain)
2895
12
    return nullptr;
2896
2897
  // Look for an ordinary load of 'self'.
2898
7
  llvm::Value *retainedValue = retainCall->getArgOperand(0);
2899
7
  llvm::LoadInst *load =
2900
7
    dyn_cast<llvm::LoadInst>(retainedValue->stripPointerCasts());
2901
7
  if (!load || load->isAtomic() || load->isVolatile() ||
2902
7
      load->getPointerOperand() != CGF.GetAddrOfLocalVar(self).getPointer())
2903
3
    return nullptr;
2904
2905
  // Okay!  Burn it all down.  This relies for correctness on the
2906
  // assumption that the retain is emitted as part of the return and
2907
  // that thereafter everything is used "linearly".
2908
4
  llvm::Type *resultType = result->getType();
2909
4
  eraseUnusedBitCasts(cast<llvm::Instruction>(result));
2910
4
  assert(retainCall->use_empty());
2911
4
  retainCall->eraseFromParent();
2912
4
  eraseUnusedBitCasts(cast<llvm::Instruction>(retainedValue));
2913
2914
4
  return CGF.Builder.CreateBitCast(load, resultType);
2915
4
}
2916
2917
/// Emit an ARC autorelease of the result of a function.
2918
///
2919
/// \return the value to actually return from the function
2920
static llvm::Value *emitAutoreleaseOfResult(CodeGenFunction &CGF,
2921
113
                                            llvm::Value *result) {
2922
  // If we're returning 'self', kill the initial retain.  This is a
2923
  // heuristic attempt to "encourage correctness" in the really unfortunate
2924
  // case where we have a return of self during a dealloc and we desperately
2925
  // need to avoid the possible autorelease.
2926
113
  if (llvm::Value *self = tryRemoveRetainOfSelf(CGF, result))
2927
4
    return self;
2928
2929
  // At -O0, try to emit a fused retain/autorelease.
2930
109
  if (CGF.shouldUseFusedARCCalls())
2931
87
    if (llvm::Value *fused = tryEmitFusedAutoreleaseOfResult(CGF, result))
2932
27
      return fused;
2933
2934
82
  return CGF.EmitARCAutoreleaseReturnValue(result);
2935
82
}
2936
2937
/// Heuristically search for a dominating store to the return-value slot.
2938
134k
static llvm::StoreInst *findDominatingStoreToReturnValue(CodeGenFunction &CGF) {
2939
  // Check if a User is a store which pointerOperand is the ReturnValue.
2940
  // We are looking for stores to the ReturnValue, not for stores of the
2941
  // ReturnValue to some other location.
2942
132k
  auto GetStoreIfValid = [&CGF](llvm::User *U) -> llvm::StoreInst * {
2943
132k
    auto *SI = dyn_cast<llvm::StoreInst>(U);
2944
132k
    if (!SI || 
SI->getPointerOperand() != CGF.ReturnValue.getPointer()130k
)
2945
2.20k
      return nullptr;
2946
    // These aren't actually possible for non-coerced returns, and we
2947
    // only care about non-coerced returns on this code path.
2948
130k
    assert(!SI->isAtomic() && !SI->isVolatile());
2949
130k
    return SI;
2950
130k
  };
2951
  // If there are multiple uses of the return-value slot, just check
2952
  // for something immediately preceding the IP.  Sometimes this can
2953
  // happen with how we generate implicit-returns; it can also happen
2954
  // with noreturn cleanups.
2955
134k
  if (!CGF.ReturnValue.getPointer()->hasOneUse()) {
2956
6.91k
    llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock();
2957
6.91k
    if (IP->empty()) 
return nullptr1.72k
;
2958
5.19k
    llvm::Instruction *I = &IP->back();
2959
2960
    // Skip lifetime markers
2961
5.19k
    for (llvm::BasicBlock::reverse_iterator II = IP->rbegin(),
2962
5.19k
                                            IE = IP->rend();
2963
5.28k
         II != IE; 
++II88
) {
2964
5.23k
      if (llvm::IntrinsicInst *Intrinsic =
2965
341
              dyn_cast<llvm::IntrinsicInst>(&*II)) {
2966
341
        if (Intrinsic->getIntrinsicID() == llvm::Intrinsic::lifetime_end) {
2967
88
          const llvm::Value *CastAddr = Intrinsic->getArgOperand(1);
2968
88
          ++II;
2969
88
          if (II == IE)
2970
0
            break;
2971
88
          if (isa<llvm::BitCastInst>(&*II) && (CastAddr == &*II))
2972
88
            continue;
2973
5.15k
        }
2974
341
      }
2975
5.15k
      I = &*II;
2976
5.15k
      break;
2977
5.15k
    }
2978
2979
5.19k
    return GetStoreIfValid(I);
2980
5.19k
  }
2981
2982
127k
  llvm::StoreInst *store =
2983
127k
      GetStoreIfValid(CGF.ReturnValue.getPointer()->user_back());
2984
127k
  if (!store) 
return nullptr509
;
2985
2986
  // Now do a first-and-dirty dominance check: just walk up the
2987
  // single-predecessors chain from the current insertion point.
2988
127k
  llvm::BasicBlock *StoreBB = store->getParent();
2989
127k
  llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock();
2990
127k
  while (IP != StoreBB) {
2991
2.00k
    if (!(IP = IP->getSinglePredecessor()))
2992
1.27k
      return nullptr;
2993
2.00k
  }
2994
2995
  // Okay, the store's basic block dominates the insertion point; we
2996
  // can do our thing.
2997
125k
  return store;
2998
127k
}
2999
3000
// Helper functions for EmitCMSEClearRecord
3001
3002
// Set the bits corresponding to a field having width `BitWidth` and located at
3003
// offset `BitOffset` (from the least significant bit) within a storage unit of
3004
// `Bits.size()` bytes. Each element of `Bits` corresponds to one target byte.
3005
// Use little-endian layout, i.e.`Bits[0]` is the LSB.
3006
static void setBitRange(SmallVectorImpl<uint64_t> &Bits, int BitOffset,
3007
125
                        int BitWidth, int CharWidth) {
3008
125
  assert(CharWidth <= 64);
3009
125
  assert(static_cast<unsigned>(BitWidth) <= Bits.size() * CharWidth);
3010
3011
125
  int Pos = 0;
3012
125
  if (BitOffset >= CharWidth) {
3013
39
    Pos += BitOffset / CharWidth;
3014
39
    BitOffset = BitOffset % CharWidth;
3015
39
  }
3016
3017
125
  const uint64_t Used = (uint64_t(1) << CharWidth) - 1;
3018
125
  if (BitOffset + BitWidth >= CharWidth) {
3019
65
    Bits[Pos++] |= (Used << BitOffset) & Used;
3020
65
    BitWidth -= CharWidth - BitOffset;
3021
65
    BitOffset = 0;
3022
65
  }
3023
3024
143
  while (BitWidth >= CharWidth) {
3025
18
    Bits[Pos++] = Used;
3026
18
    BitWidth -= CharWidth;
3027
18
  }
3028
3029
125
  if (BitWidth > 0)
3030
92
    Bits[Pos++] |= (Used >> (CharWidth - BitWidth)) << BitOffset;
3031
125
}
3032
3033
// Set the bits corresponding to a field having width `BitWidth` and located at
3034
// offset `BitOffset` (from the least significant bit) within a storage unit of
3035
// `StorageSize` bytes, located at `StorageOffset` in `Bits`. Each element of
3036
// `Bits` corresponds to one target byte. Use target endian layout.
3037
static void setBitRange(SmallVectorImpl<uint64_t> &Bits, int StorageOffset,
3038
                        int StorageSize, int BitOffset, int BitWidth,
3039
125
                        int CharWidth, bool BigEndian) {
3040
3041
125
  SmallVector<uint64_t, 8> TmpBits(StorageSize);
3042
125
  setBitRange(TmpBits, BitOffset, BitWidth, CharWidth);
3043
3044
125
  if (BigEndian)
3045
50
    std::reverse(TmpBits.begin(), TmpBits.end());
3046
3047
125
  for (uint64_t V : TmpBits)
3048
350
    Bits[StorageOffset++] |= V;
3049
125
}
3050
3051
static void setUsedBits(CodeGenModule &, QualType, int,
3052
                        SmallVectorImpl<uint64_t> &);
3053
3054
// Set the bits in `Bits`, which correspond to the value representations of
3055
// the actual members of the record type `RTy`. Note that this function does
3056
// not handle base classes, virtual tables, etc, since they cannot happen in
3057
// CMSE function arguments or return. The bit mask corresponds to the target
3058
// memory layout, i.e. it's endian dependent.
3059
static void setUsedBits(CodeGenModule &CGM, const RecordType *RTy, int Offset,
3060
153
                        SmallVectorImpl<uint64_t> &Bits) {
3061
153
  ASTContext &Context = CGM.getContext();
3062
153
  int CharWidth = Context.getCharWidth();
3063
153
  const RecordDecl *RD = RTy->getDecl()->getDefinition();
3064
153
  const ASTRecordLayout &ASTLayout = Context.getASTRecordLayout(RD);
3065
153
  const CGRecordLayout &Layout = CGM.getTypes().getCGRecordLayout(RD);
3066
3067
153
  int Idx = 0;
3068
566
  for (auto I = RD->field_begin(), E = RD->field_end(); I != E; 
++I, ++Idx413
) {
3069
413
    const FieldDecl *F = *I;
3070
3071
413
    if (F->isUnnamedBitfield() || 
F->isZeroLengthBitField(Context)283
||
3072
283
        F->getType()->isIncompleteArrayType())
3073
140
      continue;
3074
3075
273
    if (F->isBitField()) {
3076
125
      const CGBitFieldInfo &BFI = Layout.getBitFieldInfo(F);
3077
125
      setBitRange(Bits, Offset + BFI.StorageOffset.getQuantity(),
3078
125
                  BFI.StorageSize / CharWidth, BFI.Offset,
3079
125
                  BFI.Size, CharWidth,
3080
125
                  CGM.getDataLayout().isBigEndian());
3081
125
      continue;
3082
125
    }
3083
3084
148
    setUsedBits(CGM, F->getType(),
3085
148
                Offset + ASTLayout.getFieldOffset(Idx) / CharWidth, Bits);
3086
148
  }
3087
153
}
3088
3089
// Set the bits in `Bits`, which correspond to the value representations of
3090
// the elements of an array type `ATy`.
3091
static void setUsedBits(CodeGenModule &CGM, const ConstantArrayType *ATy,
3092
34
                        int Offset, SmallVectorImpl<uint64_t> &Bits) {
3093
34
  const ASTContext &Context = CGM.getContext();
3094
3095
34
  QualType ETy = Context.getBaseElementType(ATy);
3096
34
  int Size = Context.getTypeSizeInChars(ETy).getQuantity();
3097
34
  SmallVector<uint64_t, 4> TmpBits(Size);
3098
34
  setUsedBits(CGM, ETy, 0, TmpBits);
3099
3100
142
  for (int I = 0, N = Context.getConstantArrayElementCount(ATy); I < N; 
++I108
) {
3101
108
    auto Src = TmpBits.begin();
3102
108
    auto Dst = Bits.begin() + Offset + I * Size;
3103
270
    for (int J = 0; J < Size; 
++J162
)
3104
162
      *Dst++ |= *Src++;
3105
108
  }
3106
34
}
3107
3108
// Set the bits in `Bits`, which correspond to the value representations of
3109
// the type `QTy`.
3110
static void setUsedBits(CodeGenModule &CGM, QualType QTy, int Offset,
3111
182
                        SmallVectorImpl<uint64_t> &Bits) {
3112
182
  if (const auto *RTy = QTy->getAs<RecordType>())
3113
30
    return setUsedBits(CGM, RTy, Offset, Bits);
3114
3115
152
  ASTContext &Context = CGM.getContext();
3116
152
  if (const auto *ATy = Context.getAsConstantArrayType(QTy))
3117
34
    return setUsedBits(CGM, ATy, Offset, Bits);
3118
3119
118
  int Size = Context.getTypeSizeInChars(QTy).getQuantity();
3120
118
  if (Size <= 0)
3121
0
    return;
3122
3123
118
  std::fill_n(Bits.begin() + Offset, Size,
3124
118
              (uint64_t(1) << Context.getCharWidth()) - 1);
3125
118
}
3126
3127
static uint64_t buildMultiCharMask(const SmallVectorImpl<uint64_t> &Bits,
3128
                                   int Pos, int Size, int CharWidth,
3129
152
                                   bool BigEndian) {
3130
152
  assert(Size > 0);
3131
152
  uint64_t Mask = 0;
3132
152
  if (BigEndian) {
3133
318
    for (auto P = Bits.begin() + Pos, E = Bits.begin() + Pos + Size; P != E;
3134
256
         ++P)
3135
256
      Mask = (Mask << CharWidth) | *P;
3136
90
  } else {
3137
90
    auto P = Bits.begin() + Pos + Size, End = Bits.begin() + Pos;
3138
90
    do
3139
372
      Mask = (Mask << CharWidth) | *--P;
3140
372
    while (P != End);
3141
90
  }
3142
152
  return Mask;
3143
152
}
3144
3145
// Emit code to clear the bits in a record, which aren't a part of any user
3146
// declared member, when the record is a function return.
3147
llvm::Value *CodeGenFunction::EmitCMSEClearRecord(llvm::Value *Src,
3148
                                                  llvm::IntegerType *ITy,
3149
84
                                                  QualType QTy) {
3150
84
  assert(Src->getType() == ITy);
3151
84
  assert(ITy->getScalarSizeInBits() <= 64);
3152
3153
84
  const llvm::DataLayout &DataLayout = CGM.getDataLayout();
3154
84
  int Size = DataLayout.getTypeStoreSize(ITy);
3155
84
  SmallVector<uint64_t, 4> Bits(Size);
3156
84
  setUsedBits(CGM, QTy->castAs<RecordType>(), 0, Bits);
3157
3158
84
  int CharWidth = CGM.getContext().getCharWidth();
3159
84
  uint64_t Mask =
3160
84
      buildMultiCharMask(Bits, 0, Size, CharWidth, DataLayout.isBigEndian());
3161
3162
84
  return Builder.CreateAnd(Src, Mask, "cmse.clear");
3163
84
}
3164
3165
// Emit code to clear the bits in a record, which aren't a part of any user
3166
// declared member, when the record is a function argument.
3167
llvm::Value *CodeGenFunction::EmitCMSEClearRecord(llvm::Value *Src,
3168
                                                  llvm::ArrayType *ATy,
3169
39
                                                  QualType QTy) {
3170
39
  const llvm::DataLayout &DataLayout = CGM.getDataLayout();
3171
39
  int Size = DataLayout.getTypeStoreSize(ATy);
3172
39
  SmallVector<uint64_t, 16> Bits(Size);
3173
39
  setUsedBits(CGM, QTy->castAs<RecordType>(), 0, Bits);
3174
3175
  // Clear each element of the LLVM array.
3176
39
  int CharWidth = CGM.getContext().getCharWidth();
3177
39
  int CharsPerElt =
3178
39
      ATy->getArrayElementType()->getScalarSizeInBits() / CharWidth;
3179
39
  int MaskIndex = 0;
3180
39
  llvm::Value *R = llvm::UndefValue::get(ATy);
3181
107
  for (int I = 0, N = ATy->getArrayNumElements(); I != N; 
++I68
) {
3182
68
    uint64_t Mask = buildMultiCharMask(Bits, MaskIndex, CharsPerElt, CharWidth,
3183
68
                                       DataLayout.isBigEndian());
3184
68
    MaskIndex += CharsPerElt;
3185
68
    llvm::Value *T0 = Builder.CreateExtractValue(Src, I);
3186
68
    llvm::Value *T1 = Builder.CreateAnd(T0, Mask, "cmse.clear");
3187
68
    R = Builder.CreateInsertValue(R, T1, I);
3188
68
  }
3189
3190
39
  return R;
3191
39
}
3192
3193
void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI,
3194
                                         bool EmitRetDbgLoc,
3195
293k
                                         SourceLocation EndLoc) {
3196
293k
  if (FI.isNoReturn()) {
3197
    // Noreturn functions don't return.
3198
13
    EmitUnreachable(EndLoc);
3199
13
    return;
3200
13
  }
3201
3202
293k
  if (CurCodeDecl && 
CurCodeDecl->hasAttr<NakedAttr>()285k
) {
3203
    // Naked functions don't have epilogues.
3204
7
    Builder.CreateUnreachable();
3205
7
    return;
3206
7
  }
3207
3208
  // Functions with no result always return void.
3209
293k
  if (!ReturnValue.isValid()) {
3210
150k
    Builder.CreateRetVoid();
3211
150k
    return;
3212
150k
  }
3213
3214
142k
  llvm::DebugLoc RetDbgLoc;
3215
142k
  llvm::Value *RV = nullptr;
3216
142k
  QualType RetTy = FI.getReturnType();
3217
142k
  const ABIArgInfo &RetAI = FI.getReturnInfo();
3218
3219
142k
  switch (RetAI.getKind()) {
3220
5
  case ABIArgInfo::InAlloca:
3221
    // Aggregrates get evaluated directly into the destination.  Sometimes we
3222
    // need to return the sret value in a register, though.
3223
5
    assert(hasAggregateEvaluationKind(RetTy));
3224
5
    if (RetAI.getInAllocaSRet()) {
3225
5
      llvm::Function::arg_iterator EI = CurFn->arg_end();
3226
5
      --EI;
3227
5
      llvm::Value *ArgStruct = &*EI;
3228
5
      llvm::Value *SRet = Builder.CreateStructGEP(
3229
5
          nullptr, ArgStruct, RetAI.getInAllocaFieldIndex());
3230
5
      RV = Builder.CreateAlignedLoad(SRet, getPointerAlign(), "sret");
3231
5
    }
3232
5
    break;
3233
3234
2.54k
  case ABIArgInfo::Indirect: {
3235
2.54k
    auto AI = CurFn->arg_begin();
3236
2.54k
    if (RetAI.isSRetAfterThis())
3237
50
      ++AI;
3238
2.54k
    switch (getEvaluationKind(RetTy)) {
3239
216
    case TEK_Complex: {
3240
216
      ComplexPairTy RT =
3241
216
        EmitLoadOfComplex(MakeAddrLValue(ReturnValue, RetTy), EndLoc);
3242
216
      EmitStoreOfComplex(RT, MakeNaturalAlignAddrLValue(&*AI, RetTy),
3243
216
                         /*isInit*/ true);
3244
216
      break;
3245
0
    }
3246
2.04k
    case TEK_Aggregate:
3247
      // Do nothing; aggregrates get evaluated directly into the destination.
3248
2.04k
      break;
3249
283
    case TEK_Scalar:
3250
283
      EmitStoreOfScalar(Builder.CreateLoad(ReturnValue),
3251
283
                        MakeNaturalAlignAddrLValue(&*AI, RetTy),
3252
283
                        /*isInit*/ true);
3253
283
      break;
3254
2.54k
    }
3255
2.54k
    break;
3256
2.54k
  }
3257
3258
139k
  case ABIArgInfo::Extend:
3259
139k
  case ABIArgInfo::Direct:
3260
139k
    if (RetAI.getCoerceToType() == ConvertType(RetTy) &&
3261
134k
        RetAI.getDirectOffset() == 0) {
3262
      // The internal return value temp always will have pointer-to-return-type
3263
      // type, just do a load.
3264
3265
      // If there is a dominating store to ReturnValue, we can elide
3266
      // the load, zap the store, and usually zap the alloca.
3267
134k
      if (llvm::StoreInst *SI =
3268
129k
              findDominatingStoreToReturnValue(*this)) {
3269
        // Reuse the debug location from the store unless there is
3270
        // cleanup code to be emitted between the store and return
3271
        // instruction.
3272
129k
        if (EmitRetDbgLoc && 
!AutoreleaseResult128k
)
3273
127k
          RetDbgLoc = SI->getDebugLoc();
3274
        // Get the stored value and nuke the now-dead store.
3275
129k
        RV = SI->getValueOperand();
3276
129k
        SI->eraseFromParent();
3277
3278
      // Otherwise, we have to do a simple load.
3279
5.20k
      } else {
3280
5.20k
        RV = Builder.CreateLoad(ReturnValue);
3281
5.20k
      }
3282
4.53k
    } else {
3283
      // If the value is offset in memory, apply the offset now.
3284
4.53k
      Address V = emitAddressAtOffset(*this, ReturnValue, RetAI);
3285
3286
4.53k
      RV = CreateCoercedLoad(V, RetAI.getCoerceToType(), *this);
3287
4.53k
    }
3288
3289
    // In ARC, end functions that return a retainable type with a call
3290
    // to objc_autoreleaseReturnValue.
3291
139k
    if (AutoreleaseResult) {
3292
113
#ifndef NDEBUG
3293
      // Type::isObjCRetainabletype has to be called on a QualType that hasn't
3294
      // been stripped of the typedefs, so we cannot use RetTy here. Get the
3295
      // original return type of FunctionDecl, CurCodeDecl, and BlockDecl from
3296
      // CurCodeDecl or BlockInfo.
3297
113
      QualType RT;
3298
3299
113
      if (auto *FD = dyn_cast<FunctionDecl>(CurCodeDecl))
3300
70
        RT = FD->getReturnType();
3301
43
      else if (auto *MD = dyn_cast<ObjCMethodDecl>(CurCodeDecl))
3302
19
        RT = MD->getReturnType();
3303
24
      else if (isa<BlockDecl>(CurCodeDecl))
3304
24
        RT = BlockInfo->BlockExpression->getFunctionType()->getReturnType();
3305
24
      else
3306
0
        llvm_unreachable("Unexpected function/method type");
3307
3308
113
      assert(getLangOpts().ObjCAutoRefCount &&
3309
113
             !FI.isReturnsRetained() &&
3310
113
             RT->isObjCRetainableType());
3311
113
#endif
3312
113
      RV = emitAutoreleaseOfResult(*this, RV);
3313
113
    }
3314
3315
139k
    break;
3316
3317
322
  case ABIArgInfo::Ignore:
3318
322
    break;
3319
3320
541
  case ABIArgInfo::CoerceAndExpand: {
3321
541
    auto coercionType = RetAI.getCoerceAndExpandType();
3322
3323
    // Load all of the coerced elements out into results.
3324
541
    llvm::SmallVector<llvm::Value*, 4> results;
3325
541
    Address addr = Builder.CreateElementBitCast(ReturnValue, coercionType);
3326
1.68k
    for (unsigned i = 0, e = coercionType->getNumElements(); i != e; 
++i1.14k
) {
3327
1.14k
      auto coercedEltType = coercionType->getElementType(i);
3328
1.14k
      if (ABIArgInfo::isPaddingForCoerceAndExpand(coercedEltType))
3329
7
        continue;
3330
3331
1.14k
      auto eltAddr = Builder.CreateStructGEP(addr, i);
3332
1.14k
      auto elt = Builder.CreateLoad(eltAddr);
3333
1.14k
      results.push_back(elt);
3334
1.14k
    }
3335
3336
    // If we have one result, it's the single direct result type.
3337
541
    if (results.size() == 1) {
3338
180
      RV = results[0];
3339
3340
    // Otherwise, we need to make a first-class aggregate.
3341
361
    } else {
3342
      // Construct a return type that lacks padding elements.
3343
361
      llvm::Type *returnType = RetAI.getUnpaddedCoerceAndExpandType();
3344
3345
361
      RV = llvm::UndefValue::get(returnType);
3346
1.32k
      for (unsigned i = 0, e = results.size(); i != e; 
++i960
) {
3347
960
        RV = Builder.CreateInsertValue(RV, results[i], i);
3348
960
      }
3349
361
    }
3350
541
    break;
3351
139k
  }
3352
0
  case ABIArgInfo::Expand:
3353
0
  case ABIArgInfo::IndirectAliased:
3354
0
    llvm_unreachable("Invalid ABI kind for return argument");
3355
142k
  }
3356
3357
142k
  llvm::Instruction *Ret;
3358
142k
  if (RV) {
3359
139k
    if (CurFuncDecl && 
CurFuncDecl->hasAttr<CmseNSEntryAttr>()138k
) {
3360
      // For certain return types, clear padding bits, as they may reveal
3361
      // sensitive information.
3362
      // Small struct/union types are passed as integers.
3363
90
      auto *ITy = dyn_cast<llvm::IntegerType>(RV->getType());
3364
90
      if (ITy != nullptr && 
isa<RecordType>(RetTy.getCanonicalType())84
)
3365
84
        RV = EmitCMSEClearRecord(RV, ITy, RetTy);
3366
90
    }
3367
139k
    EmitReturnValueCheck(RV);
3368
139k
    Ret = Builder.CreateRet(RV);
3369
2.86k
  } else {
3370
2.86k
    Ret = Builder.CreateRetVoid();
3371
2.86k
  }
3372
3373
142k
  if (RetDbgLoc)
3374
40.0k
    Ret->setDebugLoc(std::move(RetDbgLoc));
3375
142k
}
3376
3377
139k
void CodeGenFunction::EmitReturnValueCheck(llvm::Value *RV) {
3378
  // A current decl may not be available when emitting vtable thunks.
3379
139k
  if (!CurCodeDecl)
3380
1.20k
    return;
3381
3382
  // If the return block isn't reachable, neither is this check, so don't emit
3383
  // it.
3384
138k
  if (ReturnBlock.isValid() && 
ReturnBlock.getBlock()->use_empty()1.75k
)
3385
214
    return;
3386
3387
138k
  ReturnsNonNullAttr *RetNNAttr = nullptr;
3388
138k
  if (SanOpts.has(SanitizerKind::ReturnsNonnullAttribute))
3389
47
    RetNNAttr = CurCodeDecl->getAttr<ReturnsNonNullAttr>();
3390
3391
138k
  if (!RetNNAttr && 
!requiresReturnValueNullabilityCheck()138k
)
3392
138k
    return;
3393
3394
  // Prefer the returns_nonnull attribute if it's present.
3395
18
  SourceLocation AttrLoc;
3396
18
  SanitizerMask CheckKind;
3397
18
  SanitizerHandler Handler;
3398
18
  if (RetNNAttr) {
3399
5
    assert(!requiresReturnValueNullabilityCheck() &&
3400
5
           "Cannot check nullability and the nonnull attribute");
3401
5
    AttrLoc = RetNNAttr->getLocation();
3402
5
    CheckKind = SanitizerKind::ReturnsNonnullAttribute;
3403
5
    Handler = SanitizerHandler::NonnullReturn;
3404
13
  } else {
3405
13
    if (auto *DD = dyn_cast<DeclaratorDecl>(CurCodeDecl))
3406
7
      if (auto *TSI = DD->getTypeSourceInfo())
3407
7
        if (auto FTL = TSI->getTypeLoc().getAsAdjusted<FunctionTypeLoc>())
3408
7
          AttrLoc = FTL.getReturnLoc().findNullabilityLoc();
3409
13
    CheckKind = SanitizerKind::NullabilityReturn;
3410
13
    Handler = SanitizerHandler::NullabilityReturn;
3411
13
  }
3412
3413
18
  SanitizerScope SanScope(this);
3414
3415
  // Make sure the "return" source location is valid. If we're checking a
3416
  // nullability annotation, make sure the preconditions for the check are met.
3417
18
  llvm::BasicBlock *Check = createBasicBlock("nullcheck");
3418
18
  llvm::BasicBlock *NoCheck = createBasicBlock("no.nullcheck");
3419
18
  llvm::Value *SLocPtr = Builder.CreateLoad(ReturnLocation, "return.sloc.load");
3420
18
  llvm::Value *CanNullCheck = Builder.CreateIsNotNull(SLocPtr);
3421
18
  if (requiresReturnValueNullabilityCheck())
3422
13
    CanNullCheck =
3423
13
        Builder.CreateAnd(CanNullCheck, RetValNullabilityPrecondition);
3424
18
  Builder.CreateCondBr(CanNullCheck, Check, NoCheck);
3425
18
  EmitBlock(Check);
3426
3427
  // Now do the null check.
3428
18
  llvm::Value *Cond = Builder.CreateIsNotNull(RV);
3429
18
  llvm::Constant *StaticData[] = {EmitCheckSourceLocation(AttrLoc)};
3430
18
  llvm::Value *DynamicData[] = {SLocPtr};
3431
18
  EmitCheck(std::make_pair(Cond, CheckKind), Handler, StaticData, DynamicData);
3432
3433
18
  EmitBlock(NoCheck);
3434
3435
18
#ifndef NDEBUG
3436
  // The return location should not be used after the check has been emitted.
3437
18
  ReturnLocation = Address::invalid();
3438
18
#endif
3439
18
}
3440
3441
23.9k
static bool isInAllocaArgument(CGCXXABI &ABI, QualType type) {
3442
23.9k
  const CXXRecordDecl *RD = type->getAsCXXRecordDecl();
3443
23.9k
  return RD && 
ABI.getRecordArgABI(RD) == CGCXXABI::RAA_DirectInMemory2.79k
;
3444
23.9k
}
3445
3446
static AggValueSlot createPlaceholderSlot(CodeGenFunction &CGF,
3447
98
                                          QualType Ty) {
3448
  // FIXME: Generate IR in one pass, rather than going back and fixing up these
3449
  // placeholders.
3450
98
  llvm::Type *IRTy = CGF.ConvertTypeForMem(Ty);
3451
98
  llvm::Type *IRPtrTy = IRTy->getPointerTo();
3452
98
  llvm::Value *Placeholder = llvm::UndefValue::get(IRPtrTy->getPointerTo());
3453
3454
  // FIXME: When we generate this IR in one pass, we shouldn't need
3455
  // this win32-specific alignment hack.
3456
98
  CharUnits Align = CharUnits::fromQuantity(4);
3457
98
  Placeholder = CGF.Builder.CreateAlignedLoad(IRPtrTy, Placeholder, Align);
3458
3459
98
  return AggValueSlot::forAddr(Address(Placeholder, Align),
3460
98
                               Ty.getQualifiers(),
3461
98
                               AggValueSlot::IsNotDestructed,
3462
98
                               AggValueSlot::DoesNotNeedGCBarriers,
3463
98
                               AggValueSlot::IsNotAliased,
3464
98
                               AggValueSlot::DoesNotOverlap);
3465
98
}
3466
3467
void CodeGenFunction::EmitDelegateCallArg(CallArgList &args,
3468
                                          const VarDecl *param,
3469
17.5k
                                          SourceLocation loc) {
3470
  // StartFunction converted the ABI-lowered parameter(s) into a
3471
  // local alloca.  We need to turn that into an r-value suitable
3472
  // for EmitCall.
3473
17.5k
  Address local = GetAddrOfLocalVar(param);
3474
3475
17.5k
  QualType type = param->getType();
3476
3477
17.5k
  if (isInAllocaArgument(CGM.getCXXABI(), type)) {
3478
1
    CGM.ErrorUnsupported(param, "forwarded non-trivially copyable parameter");
3479
1
  }
3480
3481
  // GetAddrOfLocalVar returns a pointer-to-pointer for references,
3482
  // but the argument needs to be the original pointer.
3483
17.5k
  if (type->isReferenceType()) {
3484
9.04k
    args.add(RValue::get(Builder.CreateLoad(local)), type);
3485
3486
  // In ARC, move out of consumed arguments so that the release cleanup
3487
  // entered by StartFunction doesn't cause an over-release.  This isn't
3488
  // optimal -O0 code generation, but it should get cleaned up when
3489
  // optimization is enabled.  This also assumes that delegate calls are
3490
  // performed exactly once for a set of arguments, but that should be safe.
3491
8.53k
  } else if (getLangOpts().ObjCAutoRefCount &&
3492
9
             param->hasAttr<NSConsumedAttr>() &&
3493
2
             type->isObjCRetainableType()) {
3494
2
    llvm::Value *ptr = Builder.CreateLoad(local);
3495
2
    auto null =
3496
2
      llvm::ConstantPointerNull::get(cast<llvm::PointerType>(ptr->getType()));
3497
2
    Builder.CreateStore(null, local);
3498
2
    args.add(RValue::get(ptr), type);
3499
3500
  // For the most part, we just need to load the alloca, except that
3501
  // aggregate r-values are actually pointers to temporaries.
3502
8.53k
  } else {
3503
8.53k
    args.add(convertTempToRValue(local, type, loc), type);
3504
8.53k
  }
3505
3506
  // Deactivate the cleanup for the callee-destructed param that was pushed.
3507
17.5k
  if (hasAggregateEvaluationKind(type) && 
!CurFuncIsThunk2.51k
&&
3508
2.49k
      type->castAs<RecordType>()->getDecl()->isParamDestroyedInCallee() &&
3509
8
      param->needsDestruction(getContext())) {
3510
6
    EHScopeStack::stable_iterator cleanup =
3511
6
        CalleeDestructedParamCleanups.lookup(cast<ParmVarDecl>(param));
3512
6
    assert(cleanup.isValid() &&
3513
6
           "cleanup for callee-destructed param not recorded");
3514
    // This unreachable is a temporary marker which will be removed later.
3515
6
    llvm::Instruction *isActive = Builder.CreateUnreachable();
3516
6
    args.addArgCleanupDeactivation(cleanup, isActive);
3517
6
  }
3518
17.5k
}
3519
3520
52
static bool isProvablyNull(llvm::Value *addr) {
3521
52
  return isa<llvm::ConstantPointerNull>(addr);
3522
52
}
3523
3524
/// Emit the actual writing-back of a writeback.
3525
static void emitWriteback(CodeGenFunction &CGF,
3526
26
                          const CallArgList::Writeback &writeback) {
3527
26
  const LValue &srcLV = writeback.Source;
3528
26
  Address srcAddr = srcLV.getAddress(CGF);
3529
26
  assert(!isProvablyNull(srcAddr.getPointer()) &&
3530
26
         "shouldn't have writeback for provably null argument");
3531
3532
26
  llvm::BasicBlock *contBB = nullptr;
3533
3534
  // If the argument wasn't provably non-null, we need to null check
3535
  // before doing the store.
3536
26
  bool provablyNonNull = llvm::isKnownNonZero(srcAddr.getPointer(),
3537
26
                                              CGF.CGM.getDataLayout());
3538
26
  if (!provablyNonNull) {
3539
5
    llvm::BasicBlock *writebackBB = CGF.createBasicBlock("icr.writeback");
3540
5
    contBB = CGF.createBasicBlock("icr.done");
3541
3542
5
    llvm::Value *isNull =
3543
5
      CGF.Builder.CreateIsNull(srcAddr.getPointer(), "icr.isnull");
3544
5
    CGF.Builder.CreateCondBr(isNull, contBB, writebackBB);
3545
5
    CGF.EmitBlock(writebackBB);
3546
5
  }
3547
3548
  // Load the value to writeback.
3549
26
  llvm::Value *value = CGF.Builder.CreateLoad(writeback.Temporary);
3550
3551
  // Cast it back, in case we're writing an id to a Foo* or something.
3552
26
  value = CGF.Builder.CreateBitCast(value, srcAddr.getElementType(),
3553
26
                                    "icr.writeback-cast");
3554
3555
  // Perform the writeback.
3556
3557
  // If we have a "to use" value, it's something we need to emit a use
3558
  // of.  This has to be carefully threaded in: if it's done after the
3559
  // release it's potentially undefined behavior (and the optimizer
3560
  // will ignore it), and if it happens before the retain then the
3561
  // optimizer could move the release there.
3562
26
  if (writeback.ToUse) {
3563
7
    assert(srcLV.getObjCLifetime() == Qualifiers::OCL_Strong);
3564
3565
    // Retain the new value.  No need to block-copy here:  the block's
3566
    // being passed up the stack.
3567
7
    value = CGF.EmitARCRetainNonBlock(value);
3568
3569
    // Emit the intrinsic use here.
3570
7
    CGF.EmitARCIntrinsicUse(writeback.ToUse);
3571
3572
    // Load the old value (primitively).
3573
7
    llvm::Value *oldValue = CGF.EmitLoadOfScalar(srcLV, SourceLocation());
3574
3575
    // Put the new value in place (primitively).
3576
7
    CGF.EmitStoreOfScalar(value, srcLV, /*init*/ false);
3577
3578
    // Release the old value.
3579
7
    CGF.EmitARCRelease(oldValue, srcLV.isARCPreciseLifetime());
3580
3581
  // Otherwise, we can just do a normal lvalue store.
3582
19
  } else {
3583
19
    CGF.EmitStoreThroughLValue(RValue::get(value), srcLV);
3584
19
  }
3585
3586
  // Jump to the continuation block.
3587
26
  if (!provablyNonNull)
3588
5
    CGF.EmitBlock(contBB);
3589
26
}
3590
3591
static void emitWritebacks(CodeGenFunction &CGF,
3592
26
                           const CallArgList &args) {
3593
26
  for (const auto &I : args.writebacks())
3594
26
    emitWriteback(CGF, I);
3595
26
}
3596
3597
static void deactivateArgCleanupsBeforeCall(CodeGenFunction &CGF,
3598
136
                                            const CallArgList &CallArgs) {
3599
136
  ArrayRef<CallArgList::CallArgCleanup> Cleanups =
3600
136
    CallArgs.getCleanupsToDeactivate();
3601
  // Iterate in reverse to increase the likelihood of popping the cleanup.
3602
174
  for (const auto &I : llvm::reverse(Cleanups)) {
3603
174
    CGF.DeactivateCleanupBlock(I.Cleanup, I.IsActiveIP);
3604
174
    I.IsActiveIP->eraseFromParent();
3605
174
  }
3606
136
}
3607
3608
26
static const Expr *maybeGetUnaryAddrOfOperand(const Expr *E) {
3609
26
  if (const UnaryOperator *uop = dyn_cast<UnaryOperator>(E->IgnoreParens()))
3610
21
    if (uop->getOpcode() == UO_AddrOf)
3611
21
      return uop->getSubExpr();
3612
5
  return nullptr;
3613
5
}
3614
3615
/// Emit an argument that's being passed call-by-writeback.  That is,
3616
/// we are passing the address of an __autoreleased temporary; it
3617
/// might be copy-initialized with the current value of the given
3618
/// address, but it will definitely be copied out of after the call.
3619
static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args,
3620
26
                             const ObjCIndirectCopyRestoreExpr *CRE) {
3621
26
  LValue srcLV;
3622
3623
  // Make an optimistic effort to emit the address as an l-value.
3624
  // This can fail if the argument expression is more complicated.
3625
26
  if (const Expr *lvExpr = maybeGetUnaryAddrOfOperand(CRE->getSubExpr())) {
3626
21
    srcLV = CGF.EmitLValue(lvExpr);
3627
3628
  // Otherwise, just emit it as a scalar.
3629
5
  } else {
3630
5
    Address srcAddr = CGF.EmitPointerWithAlignment(CRE->getSubExpr());
3631
3632
5
    QualType srcAddrType =
3633
5
      CRE->getSubExpr()->getType()->castAs<PointerType>()->getPointeeType();
3634
5
    srcLV = CGF.MakeAddrLValue(srcAddr, srcAddrType);
3635
5
  }
3636
26
  Address srcAddr = srcLV.getAddress(CGF);
3637
3638
  // The dest and src types don't necessarily match in LLVM terms
3639
  // because of the crazy ObjC compatibility rules.
3640
3641
26
  llvm::PointerType *destType =
3642
26
    cast<llvm::PointerType>(CGF.ConvertType(CRE->getType()));
3643
3644
  // If the address is a constant null, just pass the appropriate null.
3645
26
  if (isProvablyNull(srcAddr.getPointer())) {
3646
0
    args.add(RValue::get(llvm::ConstantPointerNull::get(destType)),
3647
0
             CRE->getType());
3648
0
    return;
3649
0
  }
3650
3651
  // Create the temporary.
3652
26
  Address temp = CGF.CreateTempAlloca(destType->getElementType(),
3653
26
                                      CGF.getPointerAlign(),
3654
26
                                      "icr.temp");
3655
  // Loading an l-value can introduce a cleanup if the l-value is __weak,
3656
  // and that cleanup will be conditional if we can't prove that the l-value
3657
  // isn't null, so we need to register a dominating point so that the cleanups
3658
  // system will make valid IR.
3659
26
  CodeGenFunction::ConditionalEvaluation condEval(CGF);
3660
3661
  // Zero-initialize it if we're not doing a copy-initialization.
3662
26
  bool shouldCopy = CRE->shouldCopy();
3663
26
  if (!shouldCopy) {
3664
4
    llvm::Value *null =
3665
4
      llvm::ConstantPointerNull::get(
3666
4
        cast<llvm::PointerType>(destType->getElementType()));
3667
4
    CGF.Builder.CreateStore(null, temp);
3668
4
  }
3669
3670
26
  llvm::BasicBlock *contBB = nullptr;
3671
26
  llvm::BasicBlock *originBB = nullptr;
3672
3673
  // If the address is *not* known to be non-null, we need to switch.
3674
26
  llvm::Value *finalArgument;
3675
3676
26
  bool provablyNonNull = llvm::isKnownNonZero(srcAddr.getPointer(),
3677
26
                                              CGF.CGM.getDataLayout());
3678
26
  if (provablyNonNull) {
3679
21
    finalArgument = temp.getPointer();
3680
5
  } else {
3681
5
    llvm::Value *isNull =
3682
5
      CGF.Builder.CreateIsNull(srcAddr.getPointer(), "icr.isnull");
3683
3684
5
    finalArgument = CGF.Builder.CreateSelect(isNull,
3685
5
                                   llvm::ConstantPointerNull::get(destType),
3686
5
                                             temp.getPointer(), "icr.argument");
3687
3688
    // If we need to copy, then the load has to be conditional, which
3689
    // means we need control flow.
3690
5
    if (shouldCopy) {
3691
5
      originBB = CGF.Builder.GetInsertBlock();
3692
5
      contBB = CGF.createBasicBlock("icr.cont");
3693
5
      llvm::BasicBlock *copyBB = CGF.createBasicBlock("icr.copy");
3694
5
      CGF.Builder.CreateCondBr(isNull, contBB, copyBB);
3695
5
      CGF.EmitBlock(copyBB);
3696
5
      condEval.begin(CGF);
3697
5
    }
3698
5
  }
3699
3700
26
  llvm::Value *valueToUse = nullptr;
3701
3702
  // Perform a copy if necessary.
3703
26
  if (shouldCopy) {
3704
22
    RValue srcRV = CGF.EmitLoadOfLValue(srcLV, SourceLocation());
3705
22
    assert(srcRV.isScalar());
3706
3707
22
    llvm::Value *src = srcRV.getScalarVal();
3708
22
    src = CGF.Builder.CreateBitCast(src, destType->getElementType(),
3709
22
                                    "icr.cast");
3710
3711
    // Use an ordinary store, not a store-to-lvalue.
3712
22
    CGF.Builder.CreateStore(src, temp);
3713
3714
    // If optimization is enabled, and the value was held in a
3715
    // __strong variable, we need to tell the optimizer that this
3716
    // value has to stay alive until we're doing the store back.
3717
    // This is because the temporary is effectively unretained,
3718
    // and so otherwise we can violate the high-level semantics.
3719
22
    if (CGF.CGM.getCodeGenOpts().OptimizationLevel != 0 &&
3720
9
        srcLV.getObjCLifetime() == Qualifiers::OCL_Strong) {
3721
7
      valueToUse = src;
3722
7
    }
3723
22
  }
3724
3725
  // Finish the control flow if we needed it.
3726
26
  if (shouldCopy && 
!provablyNonNull22
) {
3727
5
    llvm::BasicBlock *copyBB = CGF.Builder.GetInsertBlock();
3728
5
    CGF.EmitBlock(contBB);
3729
3730
    // Make a phi for the value to intrinsically use.
3731
5
    if (valueToUse) {
3732
2
      llvm::PHINode *phiToUse = CGF.Builder.CreatePHI(valueToUse->getType(), 2,
3733
2
                                                      "icr.to-use");
3734
2
      phiToUse->addIncoming(valueToUse, copyBB);
3735
2
      phiToUse->addIncoming(llvm::UndefValue::get(valueToUse->getType()),
3736
2
                            originBB);
3737
2
      valueToUse = phiToUse;
3738
2
    }
3739
3740
5
    condEval.end(CGF);
3741
5
  }
3742
3743
26
  args.addWriteback(srcLV, temp, valueToUse);
3744
26
  args.add(RValue::get(finalArgument), CRE->getType());
3745
26
}
3746
3747
69
void CallArgList::allocateArgumentMemory(CodeGenFunction &CGF) {
3748
69
  assert(!StackBase);
3749
3750
  // Save the stack.
3751
69
  llvm::Function *F = CGF.CGM.getIntrinsic(llvm::Intrinsic::stacksave);
3752
69
  StackBase = CGF.Builder.CreateCall(F, {}, "inalloca.save");
3753
69
}
3754
3755
322k
void CallArgList::freeArgumentMemory(CodeGenFunction &CGF) const {
3756
322k
  if (StackBase) {
3757
    // Restore the stack after the call.
3758
69
    llvm::Function *F = CGF.CGM.getIntrinsic(llvm::Intrinsic::stackrestore);
3759
69
    CGF.Builder.CreateCall(F, StackBase);
3760
69
  }
3761
322k
}
3762
3763
void CodeGenFunction::EmitNonNullArgCheck(RValue RV, QualType ArgType,
3764
                                          SourceLocation ArgLoc,
3765
                                          AbstractCallee AC,
3766
344k
                                          unsigned ParmNum) {
3767
344k
  if (!AC.getDecl() || 
!(333k
SanOpts.has(SanitizerKind::NonnullAttribute)333k
||
3768
333k
                         SanOpts.has(SanitizerKind::NullabilityArg)))
3769
343k
    return;
3770
3771
  // The param decl may be missing in a variadic function.
3772
75
  auto PVD = ParmNum < AC.getNumParams() ? 
AC.getParamDecl(ParmNum)64
:
nullptr11
;
3773
64
  unsigned ArgNo = PVD ? PVD->getFunctionScopeIndex() : 
ParmNum11
;
3774
3775
  // Prefer the nonnull attribute if it's present.
3776
75
  const NonNullAttr *NNAttr = nullptr;
3777
75
  if (SanOpts.has(SanitizerKind::NonnullAttribute))
3778
23
    NNAttr = getNonNullAttr(AC.getDecl(), PVD, ArgType, ArgNo);
3779
3780
75
  bool CanCheckNullability = false;
3781
75
  if (SanOpts.has(SanitizerKind::NullabilityArg) && 
!NNAttr53
&&
PVD52
) {
3782
44
    auto Nullability = PVD->getType()->getNullability(getContext());
3783
44
    CanCheckNullability = Nullability &&
3784
20
                          *Nullability == NullabilityKind::NonNull &&
3785
18
                          PVD->getTypeSourceInfo();
3786
44
  }
3787
3788
75
  if (!NNAttr && 
!CanCheckNullability59
)
3789
41
    return;
3790
3791
34
  SourceLocation AttrLoc;
3792
34
  SanitizerMask CheckKind;
3793
34
  SanitizerHandler Handler;
3794
34
  if (NNAttr) {
3795
16
    AttrLoc = NNAttr->getLocation();
3796
16
    CheckKind = SanitizerKind::NonnullAttribute;
3797
16
    Handler = SanitizerHandler::NonnullArg;
3798
18
  } else {
3799
18
    AttrLoc = PVD->getTypeSourceInfo()->getTypeLoc().findNullabilityLoc();
3800
18
    CheckKind = SanitizerKind::NullabilityArg;
3801
18
    Handler = SanitizerHandler::NullabilityArg;
3802
18
  }
3803
3804
34
  SanitizerScope SanScope(this);
3805
34
  llvm::Value *Cond = EmitNonNullRValueCheck(RV, ArgType);
3806
34
  llvm::Constant *StaticData[] = {
3807
34
      EmitCheckSourceLocation(ArgLoc), EmitCheckSourceLocation(AttrLoc),
3808
34
      llvm::ConstantInt::get(Int32Ty, ArgNo + 1),
3809
34
  };
3810
34
  EmitCheck(std::make_pair(Cond, CheckKind), Handler, StaticData, None);
3811
34
}
3812
3813
void CodeGenFunction::EmitCallArgs(
3814
    CallArgList &Args, ArrayRef<QualType> ArgTypes,
3815
    llvm::iterator_range<CallExpr::const_arg_iterator> ArgRange,
3816
277k
    AbstractCallee AC, unsigned ParamsToSkip, EvaluationOrder Order) {
3817
277k
  assert((int)ArgTypes.size() == (ArgRange.end() - ArgRange.begin()));
3818
3819
  // We *have* to evaluate arguments from right to left in the MS C++ ABI,
3820
  // because arguments are destroyed left to right in the callee. As a special
3821
  // case, there are certain language constructs that require left-to-right
3822
  // evaluation, and in those cases we consider the evaluation order requirement
3823
  // to trump the "destruction order is reverse construction order" guarantee.
3824
277k
  bool LeftToRight =
3825
277k
      CGM.getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()
3826
7.03k
          ? Order == EvaluationOrder::ForceLeftToRight
3827
270k
          : Order != EvaluationOrder::ForceRightToLeft;
3828
3829
277k
  auto MaybeEmitImplicitObjectSize = [&](unsigned I, const Expr *Arg,
3830
343k
                                         RValue EmittedArg) {
3831
343k
    if (!AC.hasFunctionDecl() || 
I >= AC.getNumParams()321k
)
3832
73.0k
      return;
3833
270k
    auto *PS = AC.getParamDecl(I)->getAttr<PassObjectSizeAttr>();
3834
270k
    if (PS == nullptr)
3835
270k
      return;
3836
3837
107
    const auto &Context = getContext();
3838
107
    auto SizeTy = Context.getSizeType();
3839
107
    auto T = Builder.getIntNTy(Context.getTypeSize(SizeTy));
3840
107
    assert(EmittedArg.getScalarVal() && "We emitted nothing for the arg?");
3841
107
    llvm::Value *V = evaluateOrEmitBuiltinObjectSize(Arg, PS->getType(), T,
3842
107
                                                     EmittedArg.getScalarVal(),
3843
107
                                                     PS->isDynamic());
3844
107
    Args.add(RValue::get(V), SizeTy);
3845
    // If we're emitting args in reverse, be sure to do so with
3846
    // pass_object_size, as well.
3847
107
    if (!LeftToRight)
3848
9
      std::swap(Args.back(), *(&Args.back() - 1));
3849
107
  };
3850
3851
  // Insert a stack save if we're going to need any inalloca args.
3852
277k
  bool HasInAllocaArgs = false;
3853
277k
  if (CGM.getTarget().getCXXABI().isMicrosoft()) {
3854
7.03k
    for (ArrayRef<QualType>::iterator I = ArgTypes.begin(), E = ArgTypes.end();
3855
13.4k
         I != E && 
!HasInAllocaArgs6.44k
;
++I6.41k
)
3856
6.41k
      HasInAllocaArgs = isInAllocaArgument(CGM.getCXXABI(), *I);
3857
7.03k
    if (HasInAllocaArgs) {
3858
69
      assert(getTarget().getTriple().getArch() == llvm::Triple::x86);
3859
69
      Args.allocateArgumentMemory(*this);
3860
69
    }
3861
7.03k
  }
3862
3863
  // Evaluate each argument in the appropriate order.
3864
277k
  size_t CallArgsStart = Args.size();
3865
622k
  for (unsigned I = 0, E = ArgTypes.size(); I != E; 
++I344k
) {
3866
337k
    unsigned Idx = LeftToRight ? I : 
E - I - 17.15k
;
3867
344k
    CallExpr::const_arg_iterator Arg = ArgRange.begin() + Idx;
3868
344k
    unsigned InitialArgSize = Args.size();
3869
    // If *Arg is an ObjCIndirectCopyRestoreExpr, check that either the types of
3870
    // the argument and parameter match or the objc method is parameterized.
3871
344k
    assert((!isa<ObjCIndirectCopyRestoreExpr>(*Arg) ||
3872
344k
            getContext().hasSameUnqualifiedType((*Arg)->getType(),
3873
344k
                                                ArgTypes[Idx]) ||
3874
344k
            (isa<ObjCMethodDecl>(AC.getDecl()) &&
3875
344k
             isObjCMethodWithTypeParams(cast<ObjCMethodDecl>(AC.getDecl())))) &&
3876
344k
           "Argument and parameter types don't match");
3877
344k
    EmitCallArg(Args, *Arg, ArgTypes[Idx]);
3878
    // In particular, we depend on it being the last arg in Args, and the
3879
    // objectsize bits depend on there only being one arg if !LeftToRight.
3880
344k
    assert(InitialArgSize + 1 == Args.size() &&
3881
344k
           "The code below depends on only adding one arg per EmitCallArg");
3882
344k
    (void)InitialArgSize;
3883
    // Since pointer argument are never emitted as LValue, it is safe to emit
3884
    // non-null argument check for r-value only.
3885
344k
    if (!Args.back().hasLValue()) {
3886
343k
      RValue RVArg = Args.back().getKnownRValue();
3887
343k
      EmitNonNullArgCheck(RVArg, ArgTypes[Idx], (*Arg)->getExprLoc(), AC,
3888
343k
                          ParamsToSkip + Idx);
3889
      // @llvm.objectsize should never have side-effects and shouldn't need
3890
      // destruction/cleanups, so we can safely "emit" it after its arg,
3891
      // regardless of right-to-leftness
3892
343k
      MaybeEmitImplicitObjectSize(Idx, *Arg, RVArg);
3893
343k
    }
3894
344k
  }
3895
3896
277k
  if (!LeftToRight) {
3897
    // Un-reverse the arguments we just evaluated so they match up with the LLVM
3898
    // IR function.
3899
7.70k
    std::reverse(Args.begin() + CallArgsStart, Args.end());
3900
7.70k
  }
3901
277k
}
3902
3903
namespace {
3904
3905
struct DestroyUnpassedArg final : EHScopeStack::Cleanup {
3906
  DestroyUnpassedArg(Address Addr, QualType Ty)
3907
164
      : Addr(Addr), Ty(Ty) {}
3908
3909
  Address Addr;
3910
  QualType Ty;
3911
3912
18
  void Emit(CodeGenFunction &CGF, Flags flags) override {
3913
18
    QualType::DestructionKind DtorKind = Ty.isDestructedType();
3914
18
    if (DtorKind == QualType::DK_cxx_destructor) {
3915
16
      const CXXDestructorDecl *Dtor = Ty->getAsCXXRecordDecl()->getDestructor();
3916
16
      assert(!Dtor->isTrivial());
3917
16
      CGF.EmitCXXDestructorCall(Dtor, Dtor_Complete, /*for vbase*/ false,
3918
16
                                /*Delegating=*/false, Addr, Ty);
3919
2
    } else {
3920
2
      CGF.callCStructDestructor(CGF.MakeAddrLValue(Addr, Ty));
3921
2
    }
3922
18
  }
3923
};
3924
3925
struct DisableDebugLocationUpdates {
3926
  CodeGenFunction &CGF;
3927
  bool disabledDebugInfo;
3928
344k
  DisableDebugLocationUpdates(CodeGenFunction &CGF, const Expr *E) : CGF(CGF) {
3929
344k
    if ((disabledDebugInfo = isa<CXXDefaultArgExpr>(E) && 
CGF.getDebugInfo()3.75k
))
3930
2.62k
      CGF.disableDebugInfo();
3931
344k
  }
3932
344k
  ~DisableDebugLocationUpdates() {
3933
344k
    if (disabledDebugInfo)
3934
2.62k
      CGF.enableDebugInfo();
3935
344k
  }
3936
};
3937
3938
} // end anonymous namespace
3939
3940
249
RValue CallArg::getRValue(CodeGenFunction &CGF) const {
3941
249
  if (!HasLV)
3942
249
    return RV;
3943
0
  LValue Copy = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty), Ty);
3944
0
  CGF.EmitAggregateCopy(Copy, LV, Ty, AggValueSlot::DoesNotOverlap,
3945
0
                        LV.isVolatile());
3946
0
  IsUsed = true;
3947
0
  return RValue::getAggregate(Copy.getAddress(CGF));
3948
0
}
3949
3950
1.19k
void CallArg::copyInto(CodeGenFunction &CGF, Address Addr) const {
3951
1.19k
  LValue Dst = CGF.MakeAddrLValue(Addr, Ty);
3952
1.19k
  if (!HasLV && 
RV.isScalar()1.14k
)
3953
780
    CGF.EmitStoreOfScalar(RV.getScalarVal(), Dst, /*isInit=*/true);
3954
411
  else if (!HasLV && 
RV.isComplex()369
)
3955
369
    CGF.EmitStoreOfComplex(RV.getComplexVal(), Dst, /*init=*/true);
3956
42
  else {
3957
42
    auto Addr = HasLV ? LV.getAddress(CGF) : 
RV.getAggregateAddress()0
;
3958
42
    LValue SrcLV = CGF.MakeAddrLValue(Addr, Ty);
3959
    // We assume that call args are never copied into subobjects.
3960
42
    CGF.EmitAggregateCopy(Dst, SrcLV, Ty, AggValueSlot::DoesNotOverlap,
3961
42
                          HasLV ? LV.isVolatileQualified()
3962
0
                                : RV.isVolatileQualified());
3963
42
  }
3964
1.19k
  IsUsed = true;
3965
1.19k
}
3966
3967
void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E,
3968
344k
                                  QualType type) {
3969
344k
  DisableDebugLocationUpdates Dis(*this, E);
3970
344k
  if (const ObjCIndirectCopyRestoreExpr *CRE
3971
26
        = dyn_cast<ObjCIndirectCopyRestoreExpr>(E)) {
3972
26
    assert(getLangOpts().ObjCAutoRefCount);
3973
26
    return emitWritebackArg(*this, args, CRE);
3974
26
  }
3975
3976
344k
  assert(type->isReferenceType() == E->isGLValue() &&
3977
344k
         "reference binding to unmaterialized r-value!");
3978
3979
344k
  if (E->isGLValue()) {
3980
61.3k
    assert(E->getObjectKind() == OK_Ordinary);
3981
61.3k
    return args.add(EmitReferenceBindingToExpr(E), type);
3982
61.3k
  }
3983
3984
283k
  bool HasAggregateEvalKind = hasAggregateEvaluationKind(type);
3985
3986
  // In the Microsoft C++ ABI, aggregate arguments are destructed by the callee.
3987
  // However, we still have to push an EH-only cleanup in case we unwind before
3988
  // we make it to the call.
3989
283k
  if (HasAggregateEvalKind &&
3990
11.4k
      type->castAs<RecordType>()->getDecl()->isParamDestroyedInCallee()) {
3991
    // If we're using inalloca, use the argument memory.  Otherwise, use a
3992
    // temporary.
3993
341
    AggValueSlot Slot;
3994
341
    if (args.isUsingInAlloca())
3995
98
      Slot = createPlaceholderSlot(*this, type);
3996
243
    else
3997
243
      Slot = CreateAggTemp(type, "agg.tmp");
3998
3999
341
    bool DestroyedInCallee = true, NeedsEHCleanup = true;
4000
341
    if (const auto *RD = type->getAsCXXRecordDecl())
4001
327
      DestroyedInCallee = RD->hasNonTrivialDestructor();
4002
14
    else
4003
14
      NeedsEHCleanup = needsEHCleanup(type.isDestructedType());
4004
4005
341
    if (DestroyedInCallee)
4006
178
      Slot.setExternallyDestructed();
4007
4008
341
    EmitAggExpr(E, Slot);
4009
341
    RValue RV = Slot.asRValue();
4010
341
    args.add(RV, type);
4011
4012
341
    if (DestroyedInCallee && 
NeedsEHCleanup178
) {
4013
      // Create a no-op GEP between the placeholder and the cleanup so we can
4014
      // RAUW it successfully.  It also serves as a marker of the first
4015
      // instruction where the cleanup is active.
4016
168
      pushFullExprCleanup<DestroyUnpassedArg>(EHCleanup, Slot.getAddress(),
4017
168
                                              type);
4018
      // This unreachable is a temporary marker which will be removed later.
4019
168
      llvm::Instruction *IsActive = Builder.CreateUnreachable();
4020
168
      args.addArgCleanupDeactivation(EHStack.getInnermostEHScope(), IsActive);
4021
168
    }
4022
341
    return;
4023
341
  }
4024
4025
282k
  if (HasAggregateEvalKind && 
isa<ImplicitCastExpr>(E)11.0k
&&
4026
737
      cast<CastExpr>(E)->getCastKind() == CK_LValueToRValue) {
4027
714
    LValue L = EmitLValue(cast<CastExpr>(E)->getSubExpr());
4028
714
    assert(L.isSimple());
4029
714
    args.addUncopiedAggregate(L, type);
4030
714
    return;
4031
714
  }
4032
4033
282k
  args.add(EmitAnyExprToTemp(E), type);
4034
282k
}
4035
4036
52.5k
QualType CodeGenFunction::getVarArgType(const Expr *Arg) {
4037
  // System headers on Windows define NULL to 0 instead of 0LL on Win64. MSVC
4038
  // implicitly widens null pointer constants that are arguments to varargs
4039
  // functions to pointer-sized ints.
4040
52.5k
  if (!getTarget().getTriple().isOSWindows())
4041
52.4k
    return Arg->getType();
4042
4043
67
  if (Arg->getType()->isIntegerType() &&
4044
44
      getContext().getTypeSize(Arg->getType()) <
4045
44
          getContext().getTargetInfo().getPointerWidth(0) &&
4046
20
      Arg->isNullPointerConstant(getContext(),
4047
1
                                 Expr::NPC_ValueDependentIsNotNull)) {
4048
1
    return getContext().getIntPtrType();
4049
1
  }
4050
4051
66
  return Arg->getType();
4052
66
}
4053
4054
// In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
4055
// optimizer it can aggressively ignore unwind edges.
4056
void
4057
1.68k
CodeGenFunction::AddObjCARCExceptionMetadata(llvm::Instruction *Inst) {
4058
1.68k
  if (CGM.getCodeGenOpts().OptimizationLevel != 0 &&
4059
405
      !CGM.getCodeGenOpts().ObjCAutoRefCountExceptions)
4060
317
    Inst->setMetadata("clang.arc.no_objc_arc_exceptions",
4061
317
                      CGM.getNoObjCARCExceptionsMetadata());
4062
1.68k
}
4063
4064
/// Emits a call to the given no-arguments nounwind runtime function.
4065
llvm::CallInst *
4066
CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
4067
501
                                         const llvm::Twine &name) {
4068
501
  return EmitNounwindRuntimeCall(callee, None, name);
4069
501
}
4070
4071
/// Emits a call to the given nounwind runtime function.
4072
llvm::CallInst *
4073
CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee,
4074
                                         ArrayRef<llvm::Value *> args,
4075
31.3k
                                         const llvm::Twine &name) {
4076
31.3k
  llvm::CallInst *call = EmitRuntimeCall(callee, args, name);
4077
31.3k
  call->setDoesNotThrow();
4078
31.3k
  return call;
4079
31.3k
}
4080
4081
/// Emits a simple call (never an invoke) to the given no-arguments
4082
/// runtime function.
4083
llvm::CallInst *CodeGenFunction::EmitRuntimeCall(llvm::FunctionCallee callee,
4084
6.68k
                                                 const llvm::Twine &name) {
4085
6.68k
  return EmitRuntimeCall(callee, None, name);
4086
6.68k
}
4087
4088
// Calls which may throw must have operand bundles indicating which funclet
4089
// they are nested within.
4090
SmallVector<llvm::OperandBundleDef, 1>
4091
422k
CodeGenFunction::getBundlesForFunclet(llvm::Value *Callee) {
4092
422k
  SmallVector<llvm::OperandBundleDef, 1> BundleList;
4093
  // There is no need for a funclet operand bundle if we aren't inside a
4094
  // funclet.
4095
422k
  if (!CurrentFuncletPad)
4096
421k
    return BundleList;
4097
4098
  // Skip intrinsics which cannot throw.
4099
604
  auto *CalleeFn = dyn_cast<llvm::Function>(Callee->stripPointerCasts());
4100
604
  if (CalleeFn && 
CalleeFn->isIntrinsic()598
&&
CalleeFn->doesNotThrow()38
)
4101
2
    return BundleList;
4102
4103
602
  BundleList.emplace_back("funclet", CurrentFuncletPad);
4104
602
  return BundleList;
4105
602
}
4106
4107
/// Emits a simple call (never an invoke) to the given runtime function.
4108
llvm::CallInst *CodeGenFunction::EmitRuntimeCall(llvm::FunctionCallee callee,
4109
                                                 ArrayRef<llvm::Value *> args,
4110
93.5k
                                                 const llvm::Twine &name) {
4111
93.5k
  llvm::CallInst *call = Builder.CreateCall(
4112
93.5k
      callee, args, getBundlesForFunclet(callee.getCallee()), name);
4113
93.5k
  call->setCallingConv(getRuntimeCC());
4114
93.5k
  return call;
4115
93.5k
}
4116
4117
/// Emits a call or invoke to the given noreturn runtime function.
4118
void CodeGenFunction::EmitNoreturnRuntimeCallOrInvoke(
4119
569
    llvm::FunctionCallee callee, ArrayRef<llvm::Value *> args) {
4120
569
  SmallVector<llvm::OperandBundleDef, 1> BundleList =
4121
569
      getBundlesForFunclet(callee.getCallee());
4122
4123
569
  if (getInvokeDest()) {
4124
154
    llvm::InvokeInst *invoke =
4125
154
      Builder.CreateInvoke(callee,
4126
154
                           getUnreachableBlock(),
4127
154
                           getInvokeDest(),
4128
154
                           args,
4129
154
                           BundleList);
4130
154
    invoke->setDoesNotReturn();
4131
154
    invoke->setCallingConv(getRuntimeCC());
4132
415
  } else {
4133
415
    llvm::CallInst *call = Builder.CreateCall(callee, args, BundleList);
4134
415
    call->setDoesNotReturn();
4135
415
    call->setCallingConv(getRuntimeCC());
4136
415
    Builder.CreateUnreachable();
4137
415
  }
4138
569
}
4139
4140
/// Emits a call or invoke instruction to the given nullary runtime function.
4141
llvm::CallBase *
4142
CodeGenFunction::EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
4143
389
                                         const Twine &name) {
4144
389
  return EmitRuntimeCallOrInvoke(callee, None, name);
4145
389
}
4146
4147
/// Emits a call or invoke instruction to the given runtime function.
4148
llvm::CallBase *
4149
CodeGenFunction::EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee,
4150
                                         ArrayRef<llvm::Value *> args,
4151
606
                                         const Twine &name) {
4152
606
  llvm::CallBase *call = EmitCallOrInvoke(callee, args, name);
4153
606
  call->setCallingConv(getRuntimeCC());
4154
606
  return call;
4155
606
}
4156
4157
/// Emits a call or invoke instruction to the given function, depending
4158
/// on the current state of the EH stack.
4159
llvm::CallBase *CodeGenFunction::EmitCallOrInvoke(llvm::FunctionCallee Callee,
4160
                                                  ArrayRef<llvm::Value *> Args,
4161
2.24k
                                                  const Twine &Name) {
4162
2.24k
  llvm::BasicBlock *InvokeDest = getInvokeDest();
4163
2.24k
  SmallVector<llvm::OperandBundleDef, 1> BundleList =
4164
2.24k
      getBundlesForFunclet(Callee.getCallee());
4165
4166
2.24k
  llvm::CallBase *Inst;
4167
2.24k
  if (!InvokeDest)
4168
1.97k
    Inst = Builder.CreateCall(Callee, Args, BundleList, Name);
4169
266
  else {
4170
266
    llvm::BasicBlock *ContBB = createBasicBlock("invoke.cont");
4171
266
    Inst = Builder.CreateInvoke(Callee, ContBB, InvokeDest, Args, BundleList,
4172
266
                                Name);
4173
266
    EmitBlock(ContBB);
4174
266
  }
4175
4176
  // In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
4177
  // optimizer it can aggressively ignore unwind edges.
4178
2.24k
  if (CGM.getLangOpts().ObjCAutoRefCount)
4179
3
    AddObjCARCExceptionMetadata(Inst);
4180
4181
2.24k
  return Inst;
4182
2.24k
}
4183
4184
void CodeGenFunction::deferPlaceholderReplacement(llvm::Instruction *Old,
4185
99
                                                  llvm::Value *New) {
4186
99
  DeferredReplacements.push_back(std::make_pair(Old, New));
4187
99
}
4188
4189
namespace {
4190
4191
/// Specify given \p NewAlign as the alignment of return value attribute. If
4192
/// such attribute already exists, re-set it to the maximal one of two options.
4193
LLVM_NODISCARD llvm::AttributeList
4194
maybeRaiseRetAlignmentAttribute(llvm::LLVMContext &Ctx,
4195
                                const llvm::AttributeList &Attrs,
4196
28
                                llvm::Align NewAlign) {
4197
28
  llvm::Align CurAlign = Attrs.getRetAlignment().valueOrOne();
4198
28
  if (CurAlign >= NewAlign)
4199
2
    return Attrs;
4200
26
  llvm::Attribute AlignAttr = llvm::Attribute::getWithAlignment(Ctx, NewAlign);
4201
26
  return Attrs
4202
26
      .removeAttribute(Ctx, llvm::AttributeList::ReturnIndex,
4203
26
                       llvm::Attribute::AttrKind::Alignment)
4204
26
      .addAttribute(Ctx, llvm::AttributeList::ReturnIndex, AlignAttr);
4205
26
}
4206
4207
template <typename AlignedAttrTy> class AbstractAssumeAlignedAttrEmitter {
4208
protected:
4209
  CodeGenFunction &CGF;
4210
4211
  /// We do nothing if this is, or becomes, nullptr.
4212
  const AlignedAttrTy *AA = nullptr;
4213
4214
  llvm::Value *Alignment = nullptr;      // May or may not be a constant.
4215
  llvm::ConstantInt *OffsetCI = nullptr; // Constant, hopefully zero.
4216
4217
  AbstractAssumeAlignedAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl)
4218
648k
      : CGF(CGF_) {
4219
648k
    if (!FuncDecl)
4220
32.2k
      return;
4221
616k
    AA = FuncDecl->getAttr<AlignedAttrTy>();
4222
616k
  }
CGCall.cpp:(anonymous namespace)::AbstractAssumeAlignedAttrEmitter<clang::AssumeAlignedAttr>::AbstractAssumeAlignedAttrEmitter(clang::CodeGen::CodeGenFunction&, clang::Decl const*)
Line
Count
Source
4218
324k
      : CGF(CGF_) {
4219
324k
    if (!FuncDecl)
4220
16.1k
      return;
4221
308k
    AA = FuncDecl->getAttr<AlignedAttrTy>();
4222
308k
  }
CGCall.cpp:(anonymous namespace)::AbstractAssumeAlignedAttrEmitter<clang::AllocAlignAttr>::AbstractAssumeAlignedAttrEmitter(clang::CodeGen::CodeGenFunction&, clang::Decl const*)
Line
Count
Source
4218
324k
      : CGF(CGF_) {
4219
324k
    if (!FuncDecl)
4220
16.1k
      return;
4221
308k
    AA = FuncDecl->getAttr<AlignedAttrTy>();
4222
308k
  }
4223
4224
public:
4225
  /// If we can, materialize the alignment as an attribute on return value.
4226
  LLVM_NODISCARD llvm::AttributeList
4227
648k
  TryEmitAsCallSiteAttribute(const llvm::AttributeList &Attrs) {
4228
648k
    if (!AA || 
OffsetCI55
||
CGF.SanOpts.has(SanitizerKind::Alignment)50
)
4229
648k
      return Attrs;
4230
41
    const auto *AlignmentCI = dyn_cast<llvm::ConstantInt>(Alignment);
4231
41
    if (!AlignmentCI)
4232
12
      return Attrs;
4233
    // We may legitimately have non-power-of-2 alignment here.
4234
    // If so, this is UB land, emit it via `@llvm.assume` instead.
4235
29
    if (!AlignmentCI->getValue().isPowerOf2())
4236
1
      return Attrs;
4237
28
    llvm::AttributeList NewAttrs = maybeRaiseRetAlignmentAttribute(
4238
28
        CGF.getLLVMContext(), Attrs,
4239
28
        llvm::Align(
4240
28
            AlignmentCI->getLimitedValue(llvm::Value::MaximumAlignment)));
4241
28
    AA = nullptr; // We're done. Disallow doing anything else.
4242
28
    return NewAttrs;
4243
28
  }
CGCall.cpp:(anonymous namespace)::AbstractAssumeAlignedAttrEmitter<clang::AssumeAlignedAttr>::TryEmitAsCallSiteAttribute(llvm::AttributeList const&)
Line
Count
Source
4227
324k
  TryEmitAsCallSiteAttribute(const llvm::AttributeList &Attrs) {
4228
324k
    if (!AA || 
OffsetCI14
||
CGF.SanOpts.has(SanitizerKind::Alignment)9
)
4229
324k
      return Attrs;
4230
6
    const auto *AlignmentCI = dyn_cast<llvm::ConstantInt>(Alignment);
4231
6
    if (!AlignmentCI)
4232
0
      return Attrs;
4233
    // We may legitimately have non-power-of-2 alignment here.
4234
    // If so, this is UB land, emit it via `@llvm.assume` instead.
4235
6
    if (!AlignmentCI->getValue().isPowerOf2())
4236
0
      return Attrs;
4237
6
    llvm::AttributeList NewAttrs = maybeRaiseRetAlignmentAttribute(
4238
6
        CGF.getLLVMContext(), Attrs,
4239
6
        llvm::Align(
4240
6
            AlignmentCI->getLimitedValue(llvm::Value::MaximumAlignment)));
4241
6
    AA = nullptr; // We're done. Disallow doing anything else.
4242
6
    return NewAttrs;
4243
6
  }
CGCall.cpp:(anonymous namespace)::AbstractAssumeAlignedAttrEmitter<clang::AllocAlignAttr>::TryEmitAsCallSiteAttribute(llvm::AttributeList const&)
Line
Count
Source
4227
324k
  TryEmitAsCallSiteAttribute(const llvm::AttributeList &Attrs) {
4228
324k
    if (!AA || 
OffsetCI41
||
CGF.SanOpts.has(SanitizerKind::Alignment)41
)
4229
324k
      return Attrs;
4230
35
    const auto *AlignmentCI = dyn_cast<llvm::ConstantInt>(Alignment);
4231
35
    if (!AlignmentCI)
4232
12
      return Attrs;
4233
    // We may legitimately have non-power-of-2 alignment here.
4234
    // If so, this is UB land, emit it via `@llvm.assume` instead.
4235
23
    if (!AlignmentCI->getValue().isPowerOf2())
4236
1
      return Attrs;
4237
22
    llvm::AttributeList NewAttrs = maybeRaiseRetAlignmentAttribute(
4238
22
        CGF.getLLVMContext(), Attrs,
4239
22
        llvm::Align(
4240
22
            AlignmentCI->getLimitedValue(llvm::Value::MaximumAlignment)));
4241
22
    AA = nullptr; // We're done. Disallow doing anything else.
4242
22
    return NewAttrs;
4243
22
  }
4244
4245
  /// Emit alignment assumption.
4246
  /// This is a general fallback that we take if either there is an offset,
4247
  /// or the alignment is variable or we are sanitizing for alignment.
4248
599k
  void EmitAsAnAssumption(SourceLocation Loc, QualType RetTy, RValue &Ret) {
4249
599k
    if (!AA)
4250
598k
      return;
4251
27
    CGF.emitAlignmentAssumption(Ret.getScalarVal(), RetTy, Loc,
4252
27
                                AA->getLocation(), Alignment, OffsetCI);
4253
27
    AA = nullptr; // We're done. Disallow doing anything else.
4254
27
  }
CGCall.cpp:(anonymous namespace)::AbstractAssumeAlignedAttrEmitter<clang::AssumeAlignedAttr>::EmitAsAnAssumption(clang::SourceLocation, clang::QualType, clang::CodeGen::RValue&)
Line
Count
Source
4248
299k
  void EmitAsAnAssumption(SourceLocation Loc, QualType RetTy, RValue &Ret) {
4249
299k
    if (!AA)
4250
299k
      return;
4251
8
    CGF.emitAlignmentAssumption(Ret.getScalarVal(), RetTy, Loc,
4252
8
                                AA->getLocation(), Alignment, OffsetCI);
4253
8
    AA = nullptr; // We're done. Disallow doing anything else.
4254
8
  }
CGCall.cpp:(anonymous namespace)::AbstractAssumeAlignedAttrEmitter<clang::AllocAlignAttr>::EmitAsAnAssumption(clang::SourceLocation, clang::QualType, clang::CodeGen::RValue&)
Line
Count
Source
4248
299k
  void EmitAsAnAssumption(SourceLocation Loc, QualType RetTy, RValue &Ret) {
4249
299k
    if (!AA)
4250
299k
      return;
4251
19
    CGF.emitAlignmentAssumption(Ret.getScalarVal(), RetTy, Loc,
4252
19
                                AA->getLocation(), Alignment, OffsetCI);
4253
19
    AA = nullptr; // We're done. Disallow doing anything else.
4254
19
  }
4255
};
4256
4257
/// Helper data structure to emit `AssumeAlignedAttr`.
4258
class AssumeAlignedAttrEmitter final
4259
    : public AbstractAssumeAlignedAttrEmitter<AssumeAlignedAttr> {
4260
public:
4261
  AssumeAlignedAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl)
4262
324k
      : AbstractAssumeAlignedAttrEmitter(CGF_, FuncDecl) {
4263
324k
    if (!AA)
4264
324k
      return;
4265
    // It is guaranteed that the alignment/offset are constants.
4266
14
    Alignment = cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AA->getAlignment()));
4267
14
    if (Expr *Offset = AA->getOffset()) {
4268
5
      OffsetCI = cast<llvm::ConstantInt>(CGF.EmitScalarExpr(Offset));
4269
5
      if (OffsetCI->isNullValue()) // Canonicalize zero offset to no offset.
4270
0
        OffsetCI = nullptr;
4271
5
    }
4272
14
  }
4273
};
4274
4275
/// Helper data structure to emit `AllocAlignAttr`.
4276
class AllocAlignAttrEmitter final
4277
    : public AbstractAssumeAlignedAttrEmitter<AllocAlignAttr> {
4278
public:
4279
  AllocAlignAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl,
4280
                        const CallArgList &CallArgs)
4281
324k
      : AbstractAssumeAlignedAttrEmitter(CGF_, FuncDecl) {
4282
324k
    if (!AA)
4283
324k
      return;
4284
    // Alignment may or may not be a constant, and that is okay.
4285
41
    Alignment = CallArgs[AA->getParamIndex().getLLVMIndex()]
4286
41
                    .getRValue(CGF)
4287
41
                    .getScalarVal();
4288
41
  }
4289
};
4290
4291
} // namespace
4292
4293
RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
4294
                                 const CGCallee &Callee,
4295
                                 ReturnValueSlot ReturnValue,
4296
                                 const CallArgList &CallArgs,
4297
                                 llvm::CallBase **callOrInvoke,
4298
324k
                                 SourceLocation Loc) {
4299
  // FIXME: We no longer need the types from CallArgs; lift up and simplify.
4300
4301
324k
  assert(Callee.isOrdinary() || Callee.isVirtual());
4302
4303
  // Handle struct-return functions by passing a pointer to the
4304
  // location that we would like to return into.
4305
324k
  QualType RetTy = CallInfo.getReturnType();
4306
324k
  const ABIArgInfo &RetAI = CallInfo.getReturnInfo();
4307
4308
324k
  llvm::FunctionType *IRFuncTy = getTypes().GetFunctionType(CallInfo);
4309
4310
324k
  const Decl *TargetDecl = Callee.getAbstractInfo().getCalleeDecl().getDecl();
4311
324k
  if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(TargetDecl)) {
4312
    // We can only guarantee that a function is called from the correct
4313
    // context/function based on the appropriate target attributes,
4314
    // so only check in the case where we have both always_inline and target
4315
    // since otherwise we could be making a conditional call after a check for
4316
    // the proper cpu features (and it won't cause code generation issues due to
4317
    // function based code generation).
4318
305k
    if (TargetDecl->hasAttr<AlwaysInlineAttr>() &&
4319
35.9k
        TargetDecl->hasAttr<TargetAttr>())
4320
11.9k
      checkTargetFeatures(Loc, FD);
4321
4322
    // Some architectures (such as x86-64) have the ABI changed based on
4323
    // attribute-target/features. Give them a chance to diagnose.
4324
305k
    CGM.getTargetCodeGenInfo().checkFunctionCallABI(
4325
305k
        CGM, Loc, dyn_cast_or_null<FunctionDecl>(CurCodeDecl), FD, CallArgs);
4326
305k
  }
4327
4328
324k
#ifndef NDEBUG
4329
324k
  if (!(CallInfo.isVariadic() && 
CallInfo.getArgStruct()45.7k
)) {
4330
    // For an inalloca varargs function, we don't expect CallInfo to match the
4331
    // function pointer's type, because the inalloca struct a will have extra
4332
    // fields in it for the varargs parameters.  Code later in this function
4333
    // bitcasts the function pointer to the type derived from CallInfo.
4334
    //
4335
    // In other cases, we assert that the types match up (until pointers stop
4336
    // having pointee types).
4337
324k
    llvm::Type *TypeFromVal;
4338
324k
    if (Callee.isVirtual())
4339
977
      TypeFromVal = Callee.getVirtualFunctionType();
4340
323k
    else
4341
323k
      TypeFromVal =
4342
323k
          Callee.getFunctionPointer()->getType()->getPointerElementType();
4343
324k
    assert(IRFuncTy == TypeFromVal);
4344
324k
  }
4345
324k
#endif
4346
4347
  // 1. Set up the arguments.
4348
4349
  // If we're using inalloca, insert the allocation after the stack save.
4350
  // FIXME: Do this earlier rather than hacking it in here!
4351
324k
  Address ArgMemory = Address::invalid();
4352
324k
  if (llvm::StructType *ArgStruct = CallInfo.getArgStruct()) {
4353
70
    const llvm::DataLayout &DL = CGM.getDataLayout();
4354
70
    llvm::Instruction *IP = CallArgs.getStackBase();
4355
70
    llvm::AllocaInst *AI;
4356
70
    if (IP) {
4357
69
      IP = IP->getNextNode();
4358
69
      AI = new llvm::AllocaInst(ArgStruct, DL.getAllocaAddrSpace(),
4359
69
                                "argmem", IP);
4360
1
    } else {
4361
1
      AI = CreateTempAlloca(ArgStruct, "argmem");
4362
1
    }
4363
70
    auto Align = CallInfo.getArgStructAlignment();
4364
70
    AI->setAlignment(Align.getAsAlign());
4365
70
    AI->setUsedWithInAlloca(true);
4366
70
    assert(AI->isUsedWithInAlloca() && !AI->isStaticAlloca());
4367
70
    ArgMemory = Address(AI, Align);
4368
70
  }
4369
4370
324k
  ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), CallInfo);
4371
324k
  SmallVector<llvm::Value *, 16> IRCallArgs(IRFunctionArgs.totalIRArgs());
4372
4373
  // If the call returns a temporary with struct return, create a temporary
4374
  // alloca to hold the result, unless one is given to us.
4375
324k
  Address SRetPtr = Address::invalid();
4376
324k
  Address SRetAlloca = Address::invalid();
4377
324k
  llvm::Value *UnusedReturnSizePtr = nullptr;
4378
324k
  if (RetAI.isIndirect() || 
RetAI.isInAlloca()322k
||
RetAI.isCoerceAndExpand()322k
) {
4379
1.87k
    if (!ReturnValue.isNull()) {
4380
1.66k
      SRetPtr = ReturnValue.getValue();
4381
206
    } else {
4382
206
      SRetPtr = CreateMemTemp(RetTy, "tmp", &SRetAlloca);
4383
206
      if (HaveInsertPoint() && ReturnValue.isUnused()) {
4384
128
        uint64_t size =
4385
128
            CGM.getDataLayout().getTypeAllocSize(ConvertTypeForMem(RetTy));
4386
128
        UnusedReturnSizePtr = EmitLifetimeStart(size, SRetAlloca.getPointer());
4387
128
      }
4388
206
    }
4389
1.87k
    if (IRFunctionArgs.hasSRetArg()) {
4390
1.42k
      IRCallArgs[IRFunctionArgs.getSRetArgNo()] = SRetPtr.getPointer();
4391
446
    } else if (RetAI.isInAlloca()) {
4392
4
      Address Addr =
4393
4
          Builder.CreateStructGEP(ArgMemory, RetAI.getInAllocaFieldIndex());
4394
4
      Builder.CreateStore(SRetPtr.getPointer(), Addr);
4395
4
    }
4396
1.87k
  }
4397
4398
324k
  Address swiftErrorTemp = Address::invalid();
4399
324k
  Address swiftErrorArg = Address::invalid();
4400
4401
  // When passing arguments using temporary allocas, we need to add the
4402
  // appropriate lifetime markers. This vector keeps track of all the lifetime
4403
  // markers that need to be ended right after the call.
4404
324k
  SmallVector<CallLifetimeEnd, 2> CallLifetimeEndAfterCall;
4405
4406
  // Translate all of the arguments as necessary to match the IR lowering.
4407
324k
  assert(CallInfo.arg_size() == CallArgs.size() &&
4408
324k
         "Mismatch between function signature & arguments.");
4409
324k
  unsigned ArgNo = 0;
4410
324k
  CGFunctionInfo::const_arg_iterator info_it = CallInfo.arg_begin();
4411
324k
  for (CallArgList::const_iterator I = CallArgs.begin(), E = CallArgs.end();
4412
867k
       I != E; 
++I, ++info_it, ++ArgNo543k
) {
4413
543k
    const ABIArgInfo &ArgInfo = info_it->info;
4414
4415
    // Insert a padding argument to ensure proper alignment.
4416
543k
    if (IRFunctionArgs.hasPaddingArg(ArgNo))
4417
17
      IRCallArgs[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
4418
17
          llvm::UndefValue::get(ArgInfo.getPaddingType());
4419
4420
543k
    unsigned FirstIRArg, NumIRArgs;
4421
543k
    std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
4422
4423
543k
    switch (ArgInfo.getKind()) {
4424
152
    case ABIArgInfo::InAlloca: {
4425
152
      assert(NumIRArgs == 0);
4426
152
      assert(getTarget().getTriple().getArch() == llvm::Triple::x86);
4427
152
      if (I->isAggregate()) {
4428
99
        Address Addr = I->hasLValue()
4429
0
                           ? I->getKnownLValue().getAddress(*this)
4430
99
                           : I->getKnownRValue().getAggregateAddress();
4431
99
        llvm::Instruction *Placeholder =
4432
99
            cast<llvm::Instruction>(Addr.getPointer());
4433
4434
99
        if (!ArgInfo.getInAllocaIndirect()) {
4435
          // Replace the placeholder with the appropriate argument slot GEP.
4436
97
          CGBuilderTy::InsertPoint IP = Builder.saveIP();
4437
97
          Builder.SetInsertPoint(Placeholder);
4438
97
          Addr = Builder.CreateStructGEP(ArgMemory,
4439
97
                                         ArgInfo.getInAllocaFieldIndex());
4440
97
          Builder.restoreIP(IP);
4441
2
        } else {
4442
          // For indirect things such as overaligned structs, replace the
4443
          // placeholder with a regular aggregate temporary alloca. Store the
4444
          // address of this alloca into the struct.
4445
2
          Addr = CreateMemTemp(info_it->type, "inalloca.indirect.tmp");
4446
2
          Address ArgSlot = Builder.CreateStructGEP(
4447
2
              ArgMemory, ArgInfo.getInAllocaFieldIndex());
4448
2
          Builder.CreateStore(Addr.getPointer(), ArgSlot);
4449
2
        }
4450
99
        deferPlaceholderReplacement(Placeholder, Addr.getPointer());
4451
53
      } else if (ArgInfo.getInAllocaIndirect()) {
4452
        // Make a temporary alloca and store the address of it into the argument
4453
        // struct.
4454
2
        Address Addr = CreateMemTempWithoutCast(
4455
2
            I->Ty, getContext().getTypeAlignInChars(I->Ty),
4456
2
            "indirect-arg-temp");
4457
2
        I->copyInto(*this, Addr);
4458
2
        Address ArgSlot =
4459
2
            Builder.CreateStructGEP(ArgMemory, ArgInfo.getInAllocaFieldIndex());
4460
2
        Builder.CreateStore(Addr.getPointer(), ArgSlot);
4461
51
      } else {
4462
        // Store the RValue into the argument struct.
4463
51
        Address Addr =
4464
51
            Builder.CreateStructGEP(ArgMemory, ArgInfo.getInAllocaFieldIndex());
4465
51
        unsigned AS = Addr.getType()->getPointerAddressSpace();
4466
51
        llvm::Type *MemType = ConvertTypeForMem(I->Ty)->getPointerTo(AS);
4467
        // There are some cases where a trivial bitcast is not avoidable.  The
4468
        // definition of a type later in a translation unit may change it's type
4469
        // from {}* to (%struct.foo*)*.
4470
51
        if (Addr.getType() != MemType)
4471
1
          Addr = Builder.CreateBitCast(Addr, MemType);
4472
51
        I->copyInto(*this, Addr);
4473
51
      }
4474
152
      break;
4475
0
    }
4476
4477
1.87k
    case ABIArgInfo::Indirect:
4478
1.87k
    case ABIArgInfo::IndirectAliased: {
4479
1.87k
      assert(NumIRArgs == 1);
4480
1.87k
      if (!I->isAggregate()) {
4481
        // Make a temporary alloca to pass the argument.
4482
214
        Address Addr = CreateMemTempWithoutCast(
4483
214
            I->Ty, ArgInfo.getIndirectAlign(), "indirect-arg-temp");
4484
214
        IRCallArgs[FirstIRArg] = Addr.getPointer();
4485
4486
214
        I->copyInto(*this, Addr);
4487
1.66k
      } else {
4488
        // We want to avoid creating an unnecessary temporary+copy here;
4489
        // however, we need one in three cases:
4490
        // 1. If the argument is not byval, and we are required to copy the
4491
        //    source.  (This case doesn't occur on any common architecture.)
4492
        // 2. If the argument is byval, RV is not sufficiently aligned, and
4493
        //    we cannot force it to be sufficiently aligned.
4494
        // 3. If the argument is byval, but RV is not located in default
4495
        //    or alloca address space.
4496
1.66k
        Address Addr = I->hasLValue()
4497
290
                           ? I->getKnownLValue().getAddress(*this)
4498
1.37k
                           : I->getKnownRValue().getAggregateAddress();
4499
1.66k
        llvm::Value *V = Addr.getPointer();
4500
1.66k
        CharUnits Align = ArgInfo.getIndirectAlign();
4501
1.66k
        const llvm::DataLayout *TD = &CGM.getDataLayout();
4502
4503
1.66k
        assert((FirstIRArg >= IRFuncTy->getNumParams() ||
4504
1.66k
                IRFuncTy->getParamType(FirstIRArg)->getPointerAddressSpace() ==
4505
1.66k
                    TD->getAllocaAddrSpace()) &&
4506
1.66k
               "indirect argument must be in alloca address space");
4507
4508
1.66k
        bool NeedCopy = false;
4509
4510
1.66k
        if (Addr.getAlignment() < Align &&
4511
89
            llvm::getOrEnforceKnownAlignment(V, Align.getAsAlign(), *TD) <
4512
2
                Align.getAsAlign()) {
4513
2
          NeedCopy = true;
4514
1.65k
        } else if (I->hasLValue()) {
4515
288
          auto LV = I->getKnownLValue();
4516
288
          auto AS = LV.getAddressSpace();
4517
4518
288
          if (!ArgInfo.getIndirectByVal() ||
4519
261
              (LV.getAlignment() < getContext().getTypeAlignInChars(I->Ty))) {
4520
27
            NeedCopy = true;
4521
27
          }
4522
288
          if (!getLangOpts().OpenCL) {
4523
258
            if ((ArgInfo.getIndirectByVal() &&
4524
231
                (AS != LangAS::Default &&
4525
0
                 AS != CGM.getASTAllocaAddressSpace()))) {
4526
0
              NeedCopy = true;
4527
0
            }
4528
258
          }
4529
          // For OpenCL even if RV is located in default or alloca address space
4530
          // we don't want to perform address space cast for it.
4531
30
          else if ((ArgInfo.getIndirectByVal() &&
4532
30
                    Addr.getType()->getAddressSpace() != IRFuncTy->
4533
13
                      getParamType(FirstIRArg)->getPointerAddressSpace())) {
4534
13
            NeedCopy = true;
4535
13
          }
4536
288
        }
4537
4538
1.66k
        if (NeedCopy) {
4539
          // Create an aligned temporary, and copy to it.
4540
42
          Address AI = CreateMemTempWithoutCast(
4541
42
              I->Ty, ArgInfo.getIndirectAlign(), "byval-temp");
4542
42
          IRCallArgs[FirstIRArg] = AI.getPointer();
4543
4544
          // Emit lifetime markers for the temporary alloca.
4545
42
          uint64_t ByvalTempElementSize =
4546
42
              CGM.getDataLayout().getTypeAllocSize(AI.getElementType());
4547
42
          llvm::Value *LifetimeSize =
4548
42
              EmitLifetimeStart(ByvalTempElementSize, AI.getPointer());
4549
4550
          // Add cleanup code to emit the end lifetime marker after the call.
4551
42
          if (LifetimeSize) // In case we disabled lifetime markers.
4552
2
            CallLifetimeEndAfterCall.emplace_back(AI, LifetimeSize);
4553
4554
          // Generate the copy.
4555
42
          I->copyInto(*this, AI);
4556
1.61k
        } else {
4557
          // Skip the extra memcpy call.
4558
1.61k
          auto *T = V->getType()->getPointerElementType()->getPointerTo(
4559
1.61k
              CGM.getDataLayout().getAllocaAddrSpace());
4560
1.61k
          IRCallArgs[FirstIRArg] = getTargetHooks().performAddrSpaceCast(
4561
1.61k
              *this, V, LangAS::Default, CGM.getASTAllocaAddressSpace(), T,
4562
1.61k
              true);
4563
1.61k
        }
4564
1.66k
      }
4565
1.87k
      break;
4566
1.87k
    }
4567
4568
8.16k
    case ABIArgInfo::Ignore:
4569
8.16k
      assert(NumIRArgs == 0);
4570
8.16k
      break;
4571
4572
532k
    case ABIArgInfo::Extend:
4573
532k
    case ABIArgInfo::Direct: {
4574
532k
      if (!isa<llvm::StructType>(ArgInfo.getCoerceToType()) &&
4575
531k
          ArgInfo.getCoerceToType() == ConvertType(info_it->type) &&
4576
528k
          ArgInfo.getDirectOffset() == 0) {
4577
528k
        assert(NumIRArgs == 1);
4578
528k
        llvm::Value *V;
4579
528k
        if (!I->isAggregate())
4580
528k
          V = I->getKnownRValue().getScalarVal();
4581
3
        else
4582
3
          V = Builder.CreateLoad(
4583
0
              I->hasLValue() ? I->getKnownLValue().getAddress(*this)
4584
3
                             : I->getKnownRValue().getAggregateAddress());
4585
4586
        // Implement swifterror by copying into a new swifterror argument.
4587
        // We'll write back in the normal path out of the call.
4588
528k
        if (CallInfo.getExtParameterInfo(ArgNo).getABI()
4589
8
              == ParameterABI::SwiftErrorResult) {
4590
8
          assert(!swiftErrorTemp.isValid() && "multiple swifterror args");
4591
4592
8
          QualType pointeeTy = I->Ty->getPointeeType();
4593
8
          swiftErrorArg =
4594
8
            Address(V, getContext().getTypeAlignInChars(pointeeTy));
4595
4596
8
          swiftErrorTemp =
4597
8
            CreateMemTemp(pointeeTy, getPointerAlign(), "swifterror.temp");
4598
8
          V = swiftErrorTemp.getPointer();
4599
8
          cast<llvm::AllocaInst>(V)->setSwiftError(true);
4600
4601
8
          llvm::Value *errorValue = Builder.CreateLoad(swiftErrorArg);
4602
8
          Builder.CreateStore(errorValue, swiftErrorTemp);
4603
8
        }
4604
4605
        // We might have to widen integers, but we should never truncate.
4606
528k
        if (ArgInfo.getCoerceToType() != V->getType() &&
4607
696
            V->getType()->isIntegerTy())
4608
2
          V = Builder.CreateZExt(V, ArgInfo.getCoerceToType());
4609
4610
        // If the argument doesn't match, perform a bitcast to coerce it.  This
4611
        // can happen due to trivial type mismatches.
4612
528k
        if (FirstIRArg < IRFuncTy->getNumParams() &&
4613
475k
            V->getType() != IRFuncTy->getParamType(FirstIRArg))
4614
694
          V = Builder.CreateBitCast(V, IRFuncTy->getParamType(FirstIRArg));
4615
4616
528k
        IRCallArgs[FirstIRArg] = V;
4617
528k
        break;
4618
528k
      }
4619
4620
      // FIXME: Avoid the conversion through memory if possible.
4621
4.52k
      Address Src = Address::invalid();
4622
4.52k
      if (!I->isAggregate()) {
4623
882
        Src = CreateMemTemp(I->Ty, "coerce");
4624
882
        I->copyInto(*this, Src);
4625
3.63k
      } else {
4626
405
        Src = I->hasLValue() ? I->getKnownLValue().getAddress(*this)
4627
3.23k
                             : I->getKnownRValue().getAggregateAddress();
4628
3.63k
      }
4629
4630
      // If the value is offset in memory, apply the offset now.
4631
4.52k
      Src = emitAddressAtOffset(*this, Src, ArgInfo);
4632
4633
      // Fast-isel and the optimizer generally like scalar values better than
4634
      // FCAs, so we flatten them if this is safe to do for this argument.
4635
4.52k
      llvm::StructType *STy =
4636
4.52k
            dyn_cast<llvm::StructType>(ArgInfo.getCoerceToType());
4637
4.52k
      if (STy && 
ArgInfo.isDirect()1.21k
&&
ArgInfo.getCanBeFlattened()1.21k
) {
4638
1.18k
        llvm::Type *SrcTy = Src.getElementType();
4639
1.18k
        uint64_t SrcSize = CGM.getDataLayout().getTypeAllocSize(SrcTy);
4640
1.18k
        uint64_t DstSize = CGM.getDataLayout().getTypeAllocSize(STy);
4641
4642
        // If the source type is smaller than the destination type of the
4643
        // coerce-to logic, copy the source value into a temp alloca the size
4644
        // of the destination type to allow loading all of it. The bits past
4645
        // the source value are left undef.
4646
1.18k
        if (SrcSize < DstSize) {
4647
142
          Address TempAlloca
4648
142
            = CreateTempAlloca(STy, Src.getAlignment(),
4649
142
                               Src.getName() + ".coerce");
4650
142
          Builder.CreateMemCpy(TempAlloca, Src, SrcSize);
4651
142
          Src = TempAlloca;
4652
1.04k
        } else {
4653
1.04k
          Src = Builder.CreateBitCast(Src,
4654
1.04k
                                      STy->getPointerTo(Src.getAddressSpace()));
4655
1.04k
        }
4656
4657
1.18k
        assert(NumIRArgs == STy->getNumElements());
4658
3.56k
        for (unsigned i = 0, e = STy->getNumElements(); i != e; 
++i2.38k
) {
4659
2.38k
          Address EltPtr = Builder.CreateStructGEP(Src, i);
4660
2.38k
          llvm::Value *LI = Builder.CreateLoad(EltPtr);
4661
2.38k
          IRCallArgs[FirstIRArg + i] = LI;
4662
2.38k
        }
4663
3.33k
      } else {
4664
        // In the simple case, just pass the coerced loaded value.
4665
3.33k
        assert(NumIRArgs == 1);
4666
3.33k
        llvm::Value *Load =
4667
3.33k
            CreateCoercedLoad(Src, ArgInfo.getCoerceToType(), *this);
4668
4669
3.33k
        if (CallInfo.isCmseNSCall()) {
4670
          // For certain parameter types, clear padding bits, as they may reveal
4671
          // sensitive information.
4672
          // Small struct/union types are passed as integer arrays.
4673
40
          auto *ATy = dyn_cast<llvm::ArrayType>(Load->getType());
4674
40
          if (ATy != nullptr && 
isa<RecordType>(I->Ty.getCanonicalType())39
)
4675
39
            Load = EmitCMSEClearRecord(Load, ATy, I->Ty);
4676
40
        }
4677
3.33k
        IRCallArgs[FirstIRArg] = Load;
4678
3.33k
      }
4679
4680
4.52k
      break;
4681
4.52k
    }
4682
4683
442
    case ABIArgInfo::CoerceAndExpand: {
4684
442
      auto coercionType = ArgInfo.getCoerceAndExpandType();
4685
442
      auto layout = CGM.getDataLayout().getStructLayout(coercionType);
4686
4687
442
      llvm::Value *tempSize = nullptr;
4688
442
      Address addr = Address::invalid();
4689
442
      Address AllocaAddr = Address::invalid();
4690
442
      if (I->isAggregate()) {
4691
0
        addr = I->hasLValue() ? I->getKnownLValue().getAddress(*this)
4692
414
                              : I->getKnownRValue().getAggregateAddress();
4693
4694
28
      } else {
4695
28
        RValue RV = I->getKnownRValue();
4696
28
        assert(RV.isScalar()); // complex should always just be direct
4697
4698
28
        llvm::Type *scalarType = RV.getScalarVal()->getType();
4699
28
        auto scalarSize = CGM.getDataLayout().getTypeAllocSize(scalarType);
4700
28
        auto scalarAlign = CGM.getDataLayout().getPrefTypeAlignment(scalarType);
4701
4702
        // Materialize to a temporary.
4703
28
        addr = CreateTempAlloca(
4704
28
            RV.getScalarVal()->getType(),
4705
28
            CharUnits::fromQuantity(std::max(
4706
28
                (unsigned)layout->getAlignment().value(), scalarAlign)),
4707
28
            "tmp",
4708
28
            /*ArraySize=*/nullptr, &AllocaAddr);
4709
28
        tempSize = EmitLifetimeStart(scalarSize, AllocaAddr.getPointer());
4710
4711
28
        Builder.CreateStore(RV.getScalarVal(), addr);
4712
28
      }
4713
4714
442
      addr = Builder.CreateElementBitCast(addr, coercionType);
4715
4716
442
      unsigned IRArgPos = FirstIRArg;
4717
1.41k
      for (unsigned i = 0, e = coercionType->getNumElements(); i != e; 
++i973
) {
4718
973
        llvm::Type *eltType = coercionType->getElementType(i);
4719
973
        if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType)) 
continue7
;
4720
966
        Address eltAddr = Builder.CreateStructGEP(addr, i);
4721
966
        llvm::Value *elt = Builder.CreateLoad(eltAddr);
4722
966
        IRCallArgs[IRArgPos++] = elt;
4723
966
      }
4724
442
      assert(IRArgPos == FirstIRArg + NumIRArgs);
4725
4726
442
      if (tempSize) {
4727
0
        EmitLifetimeEnd(tempSize, AllocaAddr.getPointer());
4728
0
      }
4729
4730
442
      break;
4731
4.52k
    }
4732
4733
30
    case ABIArgInfo::Expand: {
4734
30
      unsigned IRArgPos = FirstIRArg;
4735
30
      ExpandTypeToArgs(I->Ty, *I, IRFuncTy, IRCallArgs, IRArgPos);
4736
30
      assert(IRArgPos == FirstIRArg + NumIRArgs);
4737
30
      break;
4738
4.52k
    }
4739
543k
    }
4740
543k
  }
4741
4742
324k
  const CGCallee &ConcreteCallee = Callee.prepareConcreteCallee(*this);
4743
324k
  llvm::Value *CalleePtr = ConcreteCallee.getFunctionPointer();
4744
4745
  // If we're using inalloca, set up that argument.
4746
324k
  if (ArgMemory.isValid()) {
4747
70
    llvm::Value *Arg = ArgMemory.getPointer();
4748
70
    if (CallInfo.isVariadic()) {
4749
      // When passing non-POD arguments by value to variadic functions, we will
4750
      // end up with a variadic prototype and an inalloca call site.  In such
4751
      // cases, we can't do any parameter mismatch checks.  Give up and bitcast
4752
      // the callee.
4753
10
      unsigned CalleeAS = CalleePtr->getType()->getPointerAddressSpace();
4754
10
      CalleePtr =
4755
10
          Builder.CreateBitCast(CalleePtr, IRFuncTy->getPointerTo(CalleeAS));
4756
60
    } else {
4757
60
      llvm::Type *LastParamTy =
4758
60
          IRFuncTy->getParamType(IRFuncTy->getNumParams() - 1);
4759
60
      if (Arg->getType() != LastParamTy) {
4760
0
#ifndef NDEBUG
4761
        // Assert that these structs have equivalent element types.
4762
0
        llvm::StructType *FullTy = CallInfo.getArgStruct();
4763
0
        llvm::StructType *DeclaredTy = cast<llvm::StructType>(
4764
0
            cast<llvm::PointerType>(LastParamTy)->getElementType());
4765
0
        assert(DeclaredTy->getNumElements() == FullTy->getNumElements());
4766
0
        for (llvm::StructType::element_iterator DI = DeclaredTy->element_begin(),
4767
0
                                                DE = DeclaredTy->element_end(),
4768
0
                                                FI = FullTy->element_begin();
4769
0
             DI != DE; ++DI, ++FI)
4770
0
          assert(*DI == *FI);
4771
0
#endif
4772
0
        Arg = Builder.CreateBitCast(Arg, LastParamTy);
4773
0
      }
4774
60
    }
4775
70
    assert(IRFunctionArgs.hasInallocaArg());
4776
70
    IRCallArgs[IRFunctionArgs.getInallocaArgNo()] = Arg;
4777
70
  }
4778
4779
  // 2. Prepare the function pointer.
4780
4781
  // If the callee is a bitcast of a non-variadic function to have a
4782
  // variadic function pointer type, check to see if we can remove the
4783
  // bitcast.  This comes up with unprototyped functions.
4784
  //
4785
  // This makes the IR nicer, but more importantly it ensures that we
4786
  // can inline the function at -O0 if it is marked always_inline.
4787
324k
  auto simplifyVariadicCallee = [](llvm::FunctionType *CalleeFT,
4788
324k
                                   llvm::Value *Ptr) -> llvm::Function * {
4789
324k
    if (!CalleeFT->isVarArg())
4790
278k
      return nullptr;
4791
4792
    // Get underlying value if it's a bitcast
4793
45.7k
    if (llvm::ConstantExpr *CE = dyn_cast<llvm::ConstantExpr>(Ptr)) {
4794
1.32k
      if (CE->getOpcode() == llvm::Instruction::BitCast)
4795
1.32k
        Ptr = CE->getOperand(0);
4796
1.32k
    }
4797
4798
45.7k
    llvm::Function *OrigFn = dyn_cast<llvm::Function>(Ptr);
4799
45.7k
    if (!OrigFn)
4800
48
      return nullptr;
4801
4802
45.7k
    llvm::FunctionType *OrigFT = OrigFn->getFunctionType();
4803
4804
    // If the original type is variadic, or if any of the component types
4805
    // disagree, we cannot remove the cast.
4806
45.7k
    if (OrigFT->isVarArg() ||
4807
357
        OrigFT->getNumParams() != CalleeFT->getNumParams() ||
4808
356
        OrigFT->getReturnType() != CalleeFT->getReturnType())
4809
45.3k
      return nullptr;
4810
4811
359
    
for (unsigned i = 0, e = OrigFT->getNumParams(); 356
i != e;
++i3
)
4812
3
      if (OrigFT->getParamType(i) != CalleeFT->getParamType(i))
4813
0
        return nullptr;
4814
4815
356
    return OrigFn;
4816
356
  };
4817
4818
324k
  if (llvm::Function *OrigFn = simplifyVariadicCallee(IRFuncTy, CalleePtr)) {
4819
356
    CalleePtr = OrigFn;
4820
356
    IRFuncTy = OrigFn->getFunctionType();
4821
356
  }
4822
4823
  // 3. Perform the actual call.
4824
4825
  // Deactivate any cleanups that we're supposed to do immediately before
4826
  // the call.
4827
324k
  if (!CallArgs.getCleanupsToDeactivate().empty())
4828
136
    deactivateArgCleanupsBeforeCall(*this, CallArgs);
4829
4830
  // Assert that the arguments we computed match up.  The IR verifier
4831
  // will catch this, but this is a common enough source of problems
4832
  // during IRGen changes that it's way better for debugging to catch
4833
  // it ourselves here.
4834
324k
#ifndef NDEBUG
4835
324k
  assert(IRCallArgs.size() == IRFuncTy->getNumParams() || IRFuncTy->isVarArg());
4836
862k
  for (unsigned i = 0; i < IRCallArgs.size(); 
++i538k
) {
4837
    // Inalloca argument can have different type.
4838
538k
    if (IRFunctionArgs.hasInallocaArg() &&
4839
99
        i == IRFunctionArgs.getInallocaArgNo())
4840
70
      continue;
4841
538k
    if (i < IRFuncTy->getNumParams())
4842
538k
      assert(IRCallArgs[i]->getType() == IRFuncTy->getParamType(i));
4843
538k
  }
4844
324k
#endif
4845
4846
  // Update the largest vector width if any arguments have vector types.
4847
862k
  for (unsigned i = 0; i < IRCallArgs.size(); 
++i538k
) {
4848
538k
    if (auto *VT = dyn_cast<llvm::VectorType>(IRCallArgs[i]->getType()))
4849
51.4k
      LargestVectorWidth =
4850
51.4k
          std::max((uint64_t)LargestVectorWidth,
4851
51.4k
                   VT->getPrimitiveSizeInBits().getKnownMinSize());
4852
538k
  }
4853
4854
  // Compute the calling convention and attributes.
4855
324k
  unsigned CallingConv;
4856
324k
  llvm::AttributeList Attrs;
4857
324k
  CGM.ConstructAttributeList(CalleePtr->getName(), CallInfo,
4858
324k
                             Callee.getAbstractInfo(), Attrs, CallingConv,
4859
324k
                             /*AttrOnCallSite=*/true);
4860
4861
324k
  if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(CurFuncDecl))
4862
315k
    if (FD->hasAttr<StrictFPAttr>())
4863
      // All calls within a strictfp function are marked strictfp
4864
791
      Attrs =
4865
791
        Attrs.addAttribute(getLLVMContext(), llvm::AttributeList::FunctionIndex,
4866
791
                           llvm::Attribute::StrictFP);
4867
4868
  // Add call-site nomerge attribute if exists.
4869
324k
  if (InNoMergeAttributedStmt)
4870
10
    Attrs =
4871
10
      Attrs.addAttribute(getLLVMContext(), llvm::AttributeList::FunctionIndex,
4872
10
                         llvm::Attribute::NoMerge);
4873
4874
  // Apply some call-site-specific attributes.
4875
  // TODO: work this into building the attribute set.
4876
4877
  // Apply always_inline to all calls within flatten functions.
4878
  // FIXME: should this really take priority over __try, below?
4879
324k
  if (CurCodeDecl && 
CurCodeDecl->hasAttr<FlattenAttr>()323k
&&
4880
3
      !(TargetDecl && TargetDecl->hasAttr<NoInlineAttr>())) {
4881
2
    Attrs =
4882
2
        Attrs.addAttribute(getLLVMContext(), llvm::AttributeList::FunctionIndex,
4883
2
                           llvm::Attribute::AlwaysInline);
4884
2
  }
4885
4886
  // Disable inlining inside SEH __try blocks.
4887
324k
  if (isSEHTryScope()) {
4888
101
    Attrs =
4889
101
        Attrs.addAttribute(getLLVMContext(), llvm::AttributeList::FunctionIndex,
4890
101
                           llvm::Attribute::NoInline);
4891
101
  }
4892
4893
  // Decide whether to use a call or an invoke.
4894
324k
  bool CannotThrow;
4895
324k
  if (currentFunctionUsesSEHTry()) {
4896
    // SEH cares about asynchronous exceptions, so everything can "throw."
4897
247
    CannotThrow = false;
4898
324k
  } else if (isCleanupPadScope() &&
4899
107
             EHPersonality::get(*this).isMSVCXXPersonality()) {
4900
    // The MSVC++ personality will implicitly terminate the program if an
4901
    // exception is thrown during a cleanup outside of a try/catch.
4902
    // We don't need to model anything in IR to get this behavior.
4903
65
    CannotThrow = true;
4904
324k
  } else {
4905
    // Otherwise, nounwind call sites will never throw.
4906
324k
    CannotThrow = Attrs.hasFnAttribute(llvm::Attribute::NoUnwind);
4907
4908
324k
    if (auto *FPtr = dyn_cast<llvm::Function>(CalleePtr))
4909
305k
      if (FPtr->hasFnAttribute(llvm::Attribute::NoUnwind))
4910
129k
        CannotThrow = true;
4911
324k
  }
4912
4913
  // If we made a temporary, be sure to clean up after ourselves. Note that we
4914
  // can't depend on being inside of an ExprWithCleanups, so we need to manually
4915
  // pop this cleanup later on. Being eager about this is OK, since this
4916
  // temporary is 'invisible' outside of the callee.
4917
324k
  if (UnusedReturnSizePtr)
4918
25
    pushFullExprCleanup<CallLifetimeEnd>(NormalEHLifetimeMarker, SRetAlloca,
4919
25
                                         UnusedReturnSizePtr);
4920
4921
194k
  llvm::BasicBlock *InvokeDest = CannotThrow ? 
nullptr130k
: getInvokeDest();
4922
4923
324k
  SmallVector<llvm::OperandBundleDef, 1> BundleList =
4924
324k
      getBundlesForFunclet(CalleePtr);
4925
4926
324k
  if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(CurFuncDecl))
4927
315k
    if (FD->hasAttr<StrictFPAttr>())
4928
      // All calls within a strictfp function are marked strictfp
4929
791
      Attrs =
4930
791
        Attrs.addAttribute(getLLVMContext(), llvm::AttributeList::FunctionIndex,
4931
791
                           llvm::Attribute::StrictFP);
4932
4933
324k
  AssumeAlignedAttrEmitter AssumeAlignedAttrEmitter(*this, TargetDecl);
4934
324k
  Attrs = AssumeAlignedAttrEmitter.TryEmitAsCallSiteAttribute(Attrs);
4935
4936
324k
  AllocAlignAttrEmitter AllocAlignAttrEmitter(*this, TargetDecl, CallArgs);
4937
324k
  Attrs = AllocAlignAttrEmitter.TryEmitAsCallSiteAttribute(Attrs);
4938
4939
  // Emit the actual call/invoke instruction.
4940
324k
  llvm::CallBase *CI;
4941
324k
  if (!InvokeDest) {
4942
296k
    CI = Builder.CreateCall(IRFuncTy, CalleePtr, IRCallArgs, BundleList);
4943
27.9k
  } else {
4944
27.9k
    llvm::BasicBlock *Cont = createBasicBlock("invoke.cont");
4945
27.9k
    CI = Builder.CreateInvoke(IRFuncTy, CalleePtr, Cont, InvokeDest, IRCallArgs,
4946
27.9k
                              BundleList);
4947
27.9k
    EmitBlock(Cont);
4948
27.9k
  }
4949
324k
  if (callOrInvoke)
4950
177k
    *callOrInvoke = CI;
4951
4952
  // If this is within a function that has the guard(nocf) attribute and is an
4953
  // indirect call, add the "guard_nocf" attribute to this call to indicate that
4954
  // Control Flow Guard checks should not be added, even if the call is inlined.
4955
324k
  if (const auto *FD = dyn_cast_or_null<FunctionDecl>(CurFuncDecl)) {
4956
315k
    if (const auto *A = FD->getAttr<CFGuardAttr>()) {
4957
9
      if (A->getGuard() == CFGuardAttr::GuardArg::nocf && !CI->getCalledFunction())
4958
7
        Attrs = Attrs.addAttribute(
4959
7
            getLLVMContext(), llvm::AttributeList::FunctionIndex, "guard_nocf");
4960
9
    }
4961
315k
  }
4962
4963
  // Apply the attributes and calling convention.
4964
324k
  CI->setAttributes(Attrs);
4965
324k
  CI->setCallingConv(static_cast<llvm::CallingConv::ID>(CallingConv));
4966
4967
  // Apply various metadata.
4968
4969
324k
  if (!CI->getType()->isVoidTy())
4970
196k
    CI->setName("call");
4971
4972
  // Update largest vector width from the return type.
4973
324k
  if (auto *VT = dyn_cast<llvm::VectorType>(CI->getType()))
4974
28.5k
    LargestVectorWidth =
4975
28.5k
        std::max((uint64_t)LargestVectorWidth,
4976
28.5k
                 VT->getPrimitiveSizeInBits().getKnownMinSize());
4977
4978
  // Insert instrumentation or attach profile metadata at indirect call sites.
4979
  // For more details, see the comment before the definition of
4980
  // IPVK_IndirectCallTarget in InstrProfData.inc.
4981
324k
  if (!CI->getCalledFunction())
4982
18.5k
    PGO.valueProfile(Builder, llvm::IPVK_IndirectCallTarget,
4983
18.5k
                     CI, CalleePtr);
4984
4985
  // In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC
4986
  // optimizer it can aggressively ignore unwind edges.
4987
324k
  if (CGM.getLangOpts().ObjCAutoRefCount)
4988
1.68k
    AddObjCARCExceptionMetadata(CI);
4989
4990
  // Suppress tail calls if requested.
4991
324k
  if (llvm::CallInst *Call = dyn_cast<llvm::CallInst>(CI)) {
4992
296k
    if (TargetDecl && 
TargetDecl->hasAttr<NotTailCalledAttr>()280k
)
4993
293
      Call->setTailCallKind(llvm::CallInst::TCK_NoTail);
4994
296k
  }
4995
4996
  // Add metadata for calls to MSAllocator functions
4997
324k
  if (getDebugInfo() && 
TargetDecl203k
&&
4998
191k
      TargetDecl->hasAttr<MSAllocatorAttr>())
4999
4
    getDebugInfo()->addHeapAllocSiteMetadata(CI, RetTy->getPointeeType(), Loc);
5000
5001
  // 4. Finish the call.
5002
5003
  // If the call doesn't return, finish the basic block and clear the
5004
  // insertion point; this allows the rest of IRGen to discard
5005
  // unreachable code.
5006
324k
  if (CI->doesNotReturn()) {
5007
1.79k
    if (UnusedReturnSizePtr)
5008
0
      PopCleanupBlock();
5009
5010
    // Strip away the noreturn attribute to better diagnose unreachable UB.
5011
1.79k
    if (SanOpts.has(SanitizerKind::Unreachable)) {
5012
      // Also remove from function since CallBase::hasFnAttr additionally checks
5013
      // attributes of the called function.
5014
8
      if (auto *F = CI->getCalledFunction())
5015
7
        F->removeFnAttr(llvm::Attribute::NoReturn);
5016
8
      CI->removeAttribute(llvm::AttributeList::FunctionIndex,
5017
8
                          llvm::Attribute::NoReturn);
5018
5019
      // Avoid incompatibility with ASan which relies on the `noreturn`
5020
      // attribute to insert handler calls.
5021
8
      if (SanOpts.hasOneOf(SanitizerKind::Address |
5022
2
                           SanitizerKind::KernelAddress)) {
5023
2
        SanitizerScope SanScope(this);
5024
2
        llvm::IRBuilder<>::InsertPointGuard IPGuard(Builder);
5025
2
        Builder.SetInsertPoint(CI);
5026
2
        auto *FnType = llvm::FunctionType::get(CGM.VoidTy, /*isVarArg=*/false);
5027
2
        llvm::FunctionCallee Fn =
5028
2
            CGM.CreateRuntimeFunction(FnType, "__asan_handle_no_return");
5029
2
        EmitNounwindRuntimeCall(Fn);
5030
2
      }
5031
8
    }
5032
5033
1.79k
    EmitUnreachable(Loc);
5034
1.79k
    Builder.ClearInsertionPoint();
5035
5036
    // FIXME: For now, emit a dummy basic block because expr emitters in
5037
    // generally are not ready to handle emitting expressions at unreachable
5038
    // points.
5039
1.79k
    EnsureInsertPoint();
5040
5041
    // Return a reasonable RValue.
5042
1.79k
    return GetUndefRValue(RetTy);
5043
1.79k
  }
5044
5045
  // Perform the swifterror writeback.
5046
322k
  if (swiftErrorTemp.isValid()) {
5047
8
    llvm::Value *errorResult = Builder.CreateLoad(swiftErrorTemp);
5048
8
    Builder.CreateStore(errorResult, swiftErrorArg);
5049
8
  }
5050
5051
  // Emit any call-associated writebacks immediately.  Arguably this
5052
  // should happen after any return-value munging.
5053
322k
  if (CallArgs.hasWritebacks())
5054
26
    emitWritebacks(*this, CallArgs);
5055
5056
  // The stack cleanup for inalloca arguments has to run out of the normal
5057
  // lexical order, so deactivate it and run it manually here.
5058
322k
  CallArgs.freeArgumentMemory(*this);
5059
5060
  // Extract the return value.
5061
322k
  RValue Ret = [&] {
5062
322k
    switch (RetAI.getKind()) {
5063
442
    case ABIArgInfo::CoerceAndExpand: {
5064
442
      auto coercionType = RetAI.getCoerceAndExpandType();
5065
5066
442
      Address addr = SRetPtr;
5067
442
      addr = Builder.CreateElementBitCast(addr, coercionType);
5068
5069
442
      assert(CI->getType() == RetAI.getUnpaddedCoerceAndExpandType());
5070
442
      bool requiresExtract = isa<llvm::StructType>(CI->getType());
5071
5072
442
      unsigned unpaddedIndex = 0;
5073
1.41k
      for (unsigned i = 0, e = coercionType->getNumElements(); i != e; 
++i973
) {
5074
973
        llvm::Type *eltType = coercionType->getElementType(i);
5075
973
        if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType)) 
continue7
;
5076
966
        Address eltAddr = Builder.CreateStructGEP(addr, i);
5077
966
        llvm::Value *elt = CI;
5078
966
        if (requiresExtract)
5079
810
          elt = Builder.CreateExtractValue(elt, unpaddedIndex++);
5080
966
        else
5081
966
          assert(unpaddedIndex == 0);
5082
966
        Builder.CreateStore(elt, eltAddr);
5083
966
      }
5084
      // FALLTHROUGH
5085
442
      LLVM_FALLTHROUGH;
5086
442
    }
5087
5088
1.87k
    case ABIArgInfo::InAlloca:
5089
1.87k
    case ABIArgInfo::Indirect: {
5090
1.87k
      RValue ret = convertTempToRValue(SRetPtr, RetTy, SourceLocation());
5091
1.87k
      if (UnusedReturnSizePtr)
5092
25
        PopCleanupBlock();
5093
1.87k
      return ret;
5094
1.87k
    }
5095
5096
124k
    case ABIArgInfo::Ignore:
5097
      // If we are ignoring an argument that had a result, make sure to
5098
      // construct the appropriate return value for our caller.
5099
124k
      return GetUndefRValue(RetTy);
5100
5101
196k
    case ABIArgInfo::Extend:
5102
196k
    case ABIArgInfo::Direct: {
5103
196k
      llvm::Type *RetIRTy = ConvertType(RetTy);
5104
196k
      if (RetAI.getCoerceToType() == RetIRTy && 
RetAI.getDirectOffset() == 0191k
) {
5105
191k
        switch (getEvaluationKind(RetTy)) {
5106
261
        case TEK_Complex: {
5107
261
          llvm::Value *Real = Builder.CreateExtractValue(CI, 0);
5108
261
          llvm::Value *Imag = Builder.CreateExtractValue(CI, 1);
5109
261
          return RValue::getComplex(std::make_pair(Real, Imag));
5110
0
        }
5111
115
        case TEK_Aggregate: {
5112
115
          Address DestPtr = ReturnValue.getValue();
5113
115
          bool DestIsVolatile = ReturnValue.isVolatile();
5114
5115
115
          if (!DestPtr.isValid()) {
5116
5
            DestPtr = CreateMemTemp(RetTy, "agg.tmp");
5117
5
            DestIsVolatile = false;
5118
5
          }
51