/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/CodeGen/CGCall.cpp
Line | Count | Source (jump to first uncovered line) |
1 | | //===--- CGCall.cpp - Encapsulate calling convention details --------------===// |
2 | | // |
3 | | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | | // See https://llvm.org/LICENSE.txt for license information. |
5 | | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | | // |
7 | | //===----------------------------------------------------------------------===// |
8 | | // |
9 | | // These classes wrap the information about a call or function |
10 | | // definition used to handle ABI compliancy. |
11 | | // |
12 | | //===----------------------------------------------------------------------===// |
13 | | |
14 | | #include "CGCall.h" |
15 | | #include "ABIInfo.h" |
16 | | #include "CGBlocks.h" |
17 | | #include "CGCXXABI.h" |
18 | | #include "CGCleanup.h" |
19 | | #include "CGRecordLayout.h" |
20 | | #include "CodeGenFunction.h" |
21 | | #include "CodeGenModule.h" |
22 | | #include "TargetInfo.h" |
23 | | #include "clang/AST/Attr.h" |
24 | | #include "clang/AST/Decl.h" |
25 | | #include "clang/AST/DeclCXX.h" |
26 | | #include "clang/AST/DeclObjC.h" |
27 | | #include "clang/Basic/CodeGenOptions.h" |
28 | | #include "clang/Basic/TargetBuiltins.h" |
29 | | #include "clang/Basic/TargetInfo.h" |
30 | | #include "clang/CodeGen/CGFunctionInfo.h" |
31 | | #include "clang/CodeGen/SwiftCallingConv.h" |
32 | | #include "llvm/ADT/StringExtras.h" |
33 | | #include "llvm/Analysis/ValueTracking.h" |
34 | | #include "llvm/IR/Assumptions.h" |
35 | | #include "llvm/IR/Attributes.h" |
36 | | #include "llvm/IR/CallingConv.h" |
37 | | #include "llvm/IR/DataLayout.h" |
38 | | #include "llvm/IR/InlineAsm.h" |
39 | | #include "llvm/IR/IntrinsicInst.h" |
40 | | #include "llvm/IR/Intrinsics.h" |
41 | | #include "llvm/IR/Type.h" |
42 | | #include "llvm/Transforms/Utils/Local.h" |
43 | | using namespace clang; |
44 | | using namespace CodeGen; |
45 | | |
46 | | /***/ |
47 | | |
48 | 238k | unsigned CodeGenTypes::ClangCallConvToLLVMCallConv(CallingConv CC) { |
49 | 238k | switch (CC) { |
50 | 233k | default: return llvm::CallingConv::C; |
51 | 56 | case CC_X86StdCall: return llvm::CallingConv::X86_StdCall; |
52 | 74 | case CC_X86FastCall: return llvm::CallingConv::X86_FastCall; |
53 | 108 | case CC_X86RegCall: return llvm::CallingConv::X86_RegCall; |
54 | 2.67k | case CC_X86ThisCall: return llvm::CallingConv::X86_ThisCall; |
55 | 19 | case CC_Win64: return llvm::CallingConv::Win64; |
56 | 14 | case CC_X86_64SysV: return llvm::CallingConv::X86_64_SysV; |
57 | 5 | case CC_AAPCS: return llvm::CallingConv::ARM_AAPCS; |
58 | 11 | case CC_AAPCS_VFP: return llvm::CallingConv::ARM_AAPCS_VFP; |
59 | 3 | case CC_IntelOclBicc: return llvm::CallingConv::Intel_OCL_BI; |
60 | | // TODO: Add support for __pascal to LLVM. |
61 | 3 | case CC_X86Pascal: return llvm::CallingConv::C; |
62 | | // TODO: Add support for __vectorcall to LLVM. |
63 | 112 | case CC_X86VectorCall: return llvm::CallingConv::X86_VectorCall; |
64 | 2 | case CC_AArch64VectorCall: return llvm::CallingConv::AArch64_VectorCall; |
65 | 2 | case CC_AArch64SVEPCS: return llvm::CallingConv::AArch64_SVE_VectorCall; |
66 | 431 | case CC_SpirFunction: return llvm::CallingConv::SPIR_FUNC; |
67 | 381 | case CC_OpenCLKernel: return CGM.getTargetCodeGenInfo().getOpenCLKernelCallingConv(); |
68 | 10 | case CC_PreserveMost: return llvm::CallingConv::PreserveMost; |
69 | 6 | case CC_PreserveAll: return llvm::CallingConv::PreserveAll; |
70 | 1.14k | case CC_Swift: return llvm::CallingConv::Swift; |
71 | 73 | case CC_SwiftAsync: return llvm::CallingConv::SwiftTail; |
72 | 238k | } |
73 | 238k | } |
74 | | |
75 | | /// Derives the 'this' type for codegen purposes, i.e. ignoring method CVR |
76 | | /// qualification. Either or both of RD and MD may be null. A null RD indicates |
77 | | /// that there is no meaningful 'this' type, and a null MD can occur when |
78 | | /// calling a method pointer. |
79 | | CanQualType CodeGenTypes::DeriveThisType(const CXXRecordDecl *RD, |
80 | 622k | const CXXMethodDecl *MD) { |
81 | 622k | QualType RecTy; |
82 | 622k | if (RD) |
83 | 621k | RecTy = Context.getTagDeclType(RD)->getCanonicalTypeInternal(); |
84 | 688 | else |
85 | 688 | RecTy = Context.VoidTy; |
86 | | |
87 | 622k | if (MD) |
88 | 621k | RecTy = Context.getAddrSpaceQualType(RecTy, MD->getMethodQualifiers().getAddressSpace()); |
89 | 622k | return Context.getPointerType(CanQualType::CreateUnsafe(RecTy)); |
90 | 622k | } |
91 | | |
92 | | /// Returns the canonical formal type of the given C++ method. |
93 | 577k | static CanQual<FunctionProtoType> GetFormalType(const CXXMethodDecl *MD) { |
94 | 577k | return MD->getType()->getCanonicalTypeUnqualified() |
95 | 577k | .getAs<FunctionProtoType>(); |
96 | 577k | } |
97 | | |
98 | | /// Returns the "extra-canonicalized" return type, which discards |
99 | | /// qualifiers on the return type. Codegen doesn't care about them, |
100 | | /// and it makes ABI code a little easier to be able to assume that |
101 | | /// all parameter and return types are top-level unqualified. |
102 | 318k | static CanQualType GetReturnType(QualType RetTy) { |
103 | 318k | return RetTy->getCanonicalTypeUnqualified().getUnqualifiedType(); |
104 | 318k | } |
105 | | |
106 | | /// Arrange the argument and result information for a value of the given |
107 | | /// unprototyped freestanding function type. |
108 | | const CGFunctionInfo & |
109 | 389 | CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionNoProtoType> FTNP) { |
110 | | // When translating an unprototyped function type, always use a |
111 | | // variadic type. |
112 | 389 | return arrangeLLVMFunctionInfo(FTNP->getReturnType().getUnqualifiedType(), |
113 | 389 | /*instanceMethod=*/false, |
114 | 389 | /*chainCall=*/false, None, |
115 | 389 | FTNP->getExtInfo(), {}, RequiredArgs(0)); |
116 | 389 | } |
117 | | |
118 | | static void addExtParameterInfosForCall( |
119 | | llvm::SmallVectorImpl<FunctionProtoType::ExtParameterInfo> ¶mInfos, |
120 | | const FunctionProtoType *proto, |
121 | | unsigned prefixArgs, |
122 | 1.24k | unsigned totalArgs) { |
123 | 1.24k | assert(proto->hasExtParameterInfos()); |
124 | 0 | assert(paramInfos.size() <= prefixArgs); |
125 | 0 | assert(proto->getNumParams() + prefixArgs <= totalArgs); |
126 | | |
127 | 0 | paramInfos.reserve(totalArgs); |
128 | | |
129 | | // Add default infos for any prefix args that don't already have infos. |
130 | 1.24k | paramInfos.resize(prefixArgs); |
131 | | |
132 | | // Add infos for the prototype. |
133 | 2.06k | for (const auto &ParamInfo : proto->getExtParameterInfos()) { |
134 | 2.06k | paramInfos.push_back(ParamInfo); |
135 | | // pass_object_size params have no parameter info. |
136 | 2.06k | if (ParamInfo.hasPassObjectSize()) |
137 | 447 | paramInfos.emplace_back(); |
138 | 2.06k | } |
139 | | |
140 | 1.24k | assert(paramInfos.size() <= totalArgs && |
141 | 1.24k | "Did we forget to insert pass_object_size args?"); |
142 | | // Add default infos for the variadic and/or suffix arguments. |
143 | 0 | paramInfos.resize(totalArgs); |
144 | 1.24k | } |
145 | | |
146 | | /// Adds the formal parameters in FPT to the given prefix. If any parameter in |
147 | | /// FPT has pass_object_size attrs, then we'll add parameters for those, too. |
148 | | static void appendParameterTypes(const CodeGenTypes &CGT, |
149 | | SmallVectorImpl<CanQualType> &prefix, |
150 | | SmallVectorImpl<FunctionProtoType::ExtParameterInfo> ¶mInfos, |
151 | 1.00M | CanQual<FunctionProtoType> FPT) { |
152 | | // Fast path: don't touch param info if we don't need to. |
153 | 1.00M | if (!FPT->hasExtParameterInfos()) { |
154 | 1.00M | assert(paramInfos.empty() && |
155 | 1.00M | "We have paramInfos, but the prototype doesn't?"); |
156 | 0 | prefix.append(FPT->param_type_begin(), FPT->param_type_end()); |
157 | 1.00M | return; |
158 | 1.00M | } |
159 | | |
160 | 825 | unsigned PrefixSize = prefix.size(); |
161 | | // In the vast majority of cases, we'll have precisely FPT->getNumParams() |
162 | | // parameters; the only thing that can change this is the presence of |
163 | | // pass_object_size. So, we preallocate for the common case. |
164 | 825 | prefix.reserve(prefix.size() + FPT->getNumParams()); |
165 | | |
166 | 825 | auto ExtInfos = FPT->getExtParameterInfos(); |
167 | 825 | assert(ExtInfos.size() == FPT->getNumParams()); |
168 | 2.24k | for (unsigned I = 0, E = FPT->getNumParams(); I != E; ++I1.42k ) { |
169 | 1.42k | prefix.push_back(FPT->getParamType(I)); |
170 | 1.42k | if (ExtInfos[I].hasPassObjectSize()) |
171 | 299 | prefix.push_back(CGT.getContext().getSizeType()); |
172 | 1.42k | } |
173 | | |
174 | 825 | addExtParameterInfosForCall(paramInfos, FPT.getTypePtr(), PrefixSize, |
175 | 825 | prefix.size()); |
176 | 825 | } |
177 | | |
178 | | /// Arrange the LLVM function layout for a value of the given function |
179 | | /// type, on top of any implicit parameters already stored. |
180 | | static const CGFunctionInfo & |
181 | | arrangeLLVMFunctionInfo(CodeGenTypes &CGT, bool instanceMethod, |
182 | | SmallVectorImpl<CanQualType> &prefix, |
183 | 707k | CanQual<FunctionProtoType> FTP) { |
184 | 707k | SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos; |
185 | 707k | RequiredArgs Required = RequiredArgs::forPrototypePlus(FTP, prefix.size()); |
186 | | // FIXME: Kill copy. |
187 | 707k | appendParameterTypes(CGT, prefix, paramInfos, FTP); |
188 | 707k | CanQualType resultType = FTP->getReturnType().getUnqualifiedType(); |
189 | | |
190 | 707k | return CGT.arrangeLLVMFunctionInfo(resultType, instanceMethod, |
191 | 707k | /*chainCall=*/false, prefix, |
192 | 707k | FTP->getExtInfo(), paramInfos, |
193 | 707k | Required); |
194 | 707k | } |
195 | | |
196 | | /// Arrange the argument and result information for a value of the |
197 | | /// given freestanding function type. |
198 | | const CGFunctionInfo & |
199 | 485k | CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionProtoType> FTP) { |
200 | 485k | SmallVector<CanQualType, 16> argTypes; |
201 | 485k | return ::arrangeLLVMFunctionInfo(*this, /*instanceMethod=*/false, argTypes, |
202 | 485k | FTP); |
203 | 485k | } |
204 | | |
205 | | static CallingConv getCallingConventionForDecl(const ObjCMethodDecl *D, |
206 | 27.6k | bool IsWindows) { |
207 | | // Set the appropriate calling convention for the Function. |
208 | 27.6k | if (D->hasAttr<StdCallAttr>()) |
209 | 2 | return CC_X86StdCall; |
210 | | |
211 | 27.6k | if (D->hasAttr<FastCallAttr>()) |
212 | 0 | return CC_X86FastCall; |
213 | | |
214 | 27.6k | if (D->hasAttr<RegCallAttr>()) |
215 | 0 | return CC_X86RegCall; |
216 | | |
217 | 27.6k | if (D->hasAttr<ThisCallAttr>()) |
218 | 0 | return CC_X86ThisCall; |
219 | | |
220 | 27.6k | if (D->hasAttr<VectorCallAttr>()) |
221 | 0 | return CC_X86VectorCall; |
222 | | |
223 | 27.6k | if (D->hasAttr<PascalAttr>()) |
224 | 0 | return CC_X86Pascal; |
225 | | |
226 | 27.6k | if (PcsAttr *PCS = D->getAttr<PcsAttr>()) |
227 | 0 | return (PCS->getPCS() == PcsAttr::AAPCS ? CC_AAPCS : CC_AAPCS_VFP); |
228 | | |
229 | 27.6k | if (D->hasAttr<AArch64VectorPcsAttr>()) |
230 | 0 | return CC_AArch64VectorCall; |
231 | | |
232 | 27.6k | if (D->hasAttr<AArch64SVEPcsAttr>()) |
233 | 0 | return CC_AArch64SVEPCS; |
234 | | |
235 | 27.6k | if (D->hasAttr<IntelOclBiccAttr>()) |
236 | 0 | return CC_IntelOclBicc; |
237 | | |
238 | 27.6k | if (D->hasAttr<MSABIAttr>()) |
239 | 2 | return IsWindows ? CC_C0 : CC_Win64; |
240 | | |
241 | 27.6k | if (D->hasAttr<SysVABIAttr>()) |
242 | 0 | return IsWindows ? CC_X86_64SysV : CC_C; |
243 | | |
244 | 27.6k | if (D->hasAttr<PreserveMostAttr>()) |
245 | 0 | return CC_PreserveMost; |
246 | | |
247 | 27.6k | if (D->hasAttr<PreserveAllAttr>()) |
248 | 0 | return CC_PreserveAll; |
249 | | |
250 | 27.6k | return CC_C; |
251 | 27.6k | } |
252 | | |
253 | | /// Arrange the argument and result information for a call to an |
254 | | /// unknown C++ non-static member function of the given abstract type. |
255 | | /// (A null RD means we don't have any meaningful "this" argument type, |
256 | | /// so fall back to a generic pointer type). |
257 | | /// The member function must be an ordinary function, i.e. not a |
258 | | /// constructor or destructor. |
259 | | const CGFunctionInfo & |
260 | | CodeGenTypes::arrangeCXXMethodType(const CXXRecordDecl *RD, |
261 | | const FunctionProtoType *FTP, |
262 | 221k | const CXXMethodDecl *MD) { |
263 | 221k | SmallVector<CanQualType, 16> argTypes; |
264 | | |
265 | | // Add the 'this' pointer. |
266 | 221k | argTypes.push_back(DeriveThisType(RD, MD)); |
267 | | |
268 | 221k | return ::arrangeLLVMFunctionInfo( |
269 | 221k | *this, true, argTypes, |
270 | 221k | FTP->getCanonicalTypeUnqualified().getAs<FunctionProtoType>()); |
271 | 221k | } |
272 | | |
273 | | /// Set calling convention for CUDA/HIP kernel. |
274 | | static void setCUDAKernelCallingConvention(CanQualType &FTy, CodeGenModule &CGM, |
275 | 644k | const FunctionDecl *FD) { |
276 | 644k | if (FD->hasAttr<CUDAGlobalAttr>()) { |
277 | 404 | const FunctionType *FT = FTy->getAs<FunctionType>(); |
278 | 404 | CGM.getTargetCodeGenInfo().setCUDAKernelCallingConvention(FT); |
279 | 404 | FTy = FT->getCanonicalTypeUnqualified(); |
280 | 404 | } |
281 | 644k | } |
282 | | |
283 | | /// Arrange the argument and result information for a declaration or |
284 | | /// definition of the given C++ non-static member function. The |
285 | | /// member function must be an ordinary function, i.e. not a |
286 | | /// constructor or destructor. |
287 | | const CGFunctionInfo & |
288 | 227k | CodeGenTypes::arrangeCXXMethodDeclaration(const CXXMethodDecl *MD) { |
289 | 227k | assert(!isa<CXXConstructorDecl>(MD) && "wrong method for constructors!"); |
290 | 0 | assert(!isa<CXXDestructorDecl>(MD) && "wrong method for destructors!"); |
291 | | |
292 | 0 | CanQualType FT = GetFormalType(MD).getAs<Type>(); |
293 | 227k | setCUDAKernelCallingConvention(FT, CGM, MD); |
294 | 227k | auto prototype = FT.getAs<FunctionProtoType>(); |
295 | | |
296 | 227k | if (MD->isInstance()) { |
297 | | // The abstract case is perfectly fine. |
298 | 221k | const CXXRecordDecl *ThisType = TheCXXABI.getThisArgumentTypeForMethod(MD); |
299 | 221k | return arrangeCXXMethodType(ThisType, prototype.getTypePtr(), MD); |
300 | 221k | } |
301 | | |
302 | 5.93k | return arrangeFreeFunctionType(prototype); |
303 | 227k | } |
304 | | |
305 | | bool CodeGenTypes::inheritingCtorHasParams( |
306 | 1.18k | const InheritedConstructor &Inherited, CXXCtorType Type) { |
307 | | // Parameters are unnecessary if we're constructing a base class subobject |
308 | | // and the inherited constructor lives in a virtual base. |
309 | 1.18k | return Type == Ctor_Complete || |
310 | 1.18k | !Inherited.getShadowDecl()->constructsVirtualBase()804 || |
311 | 1.18k | !Target.getCXXABI().hasConstructorVariants()74 ; |
312 | 1.18k | } |
313 | | |
314 | | const CGFunctionInfo & |
315 | 295k | CodeGenTypes::arrangeCXXStructorDeclaration(GlobalDecl GD) { |
316 | 295k | auto *MD = cast<CXXMethodDecl>(GD.getDecl()); |
317 | | |
318 | 295k | SmallVector<CanQualType, 16> argTypes; |
319 | 295k | SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos; |
320 | 295k | argTypes.push_back(DeriveThisType(MD->getParent(), MD)); |
321 | | |
322 | 295k | bool PassParams = true; |
323 | | |
324 | 295k | if (auto *CD = dyn_cast<CXXConstructorDecl>(MD)) { |
325 | | // A base class inheriting constructor doesn't get forwarded arguments |
326 | | // needed to construct a virtual base (or base class thereof). |
327 | 175k | if (auto Inherited = CD->getInheritedConstructor()) |
328 | 726 | PassParams = inheritingCtorHasParams(Inherited, GD.getCtorType()); |
329 | 175k | } |
330 | | |
331 | 295k | CanQual<FunctionProtoType> FTP = GetFormalType(MD); |
332 | | |
333 | | // Add the formal parameters. |
334 | 295k | if (PassParams) |
335 | 295k | appendParameterTypes(*this, argTypes, paramInfos, FTP); |
336 | | |
337 | 295k | CGCXXABI::AddedStructorArgCounts AddedArgs = |
338 | 295k | TheCXXABI.buildStructorSignature(GD, argTypes); |
339 | 295k | if (!paramInfos.empty()) { |
340 | | // Note: prefix implies after the first param. |
341 | 175 | if (AddedArgs.Prefix) |
342 | 2 | paramInfos.insert(paramInfos.begin() + 1, AddedArgs.Prefix, |
343 | 2 | FunctionProtoType::ExtParameterInfo{}); |
344 | 175 | if (AddedArgs.Suffix) |
345 | 20 | paramInfos.append(AddedArgs.Suffix, |
346 | 20 | FunctionProtoType::ExtParameterInfo{}); |
347 | 175 | } |
348 | | |
349 | 295k | RequiredArgs required = |
350 | 295k | (PassParams && MD->isVariadic()295k ? RequiredArgs(argTypes.size())88 |
351 | 295k | : RequiredArgs::All294k ); |
352 | | |
353 | 295k | FunctionType::ExtInfo extInfo = FTP->getExtInfo(); |
354 | 295k | CanQualType resultType = TheCXXABI.HasThisReturn(GD) |
355 | 295k | ? argTypes.front()7.75k |
356 | 295k | : TheCXXABI.hasMostDerivedReturn(GD)287k |
357 | 287k | ? CGM.getContext().VoidPtrTy1.43k |
358 | 287k | : Context.VoidTy285k ; |
359 | 295k | return arrangeLLVMFunctionInfo(resultType, /*instanceMethod=*/true, |
360 | 295k | /*chainCall=*/false, argTypes, extInfo, |
361 | 295k | paramInfos, required); |
362 | 295k | } |
363 | | |
364 | | static SmallVector<CanQualType, 16> |
365 | 77.1k | getArgTypesForCall(ASTContext &ctx, const CallArgList &args) { |
366 | 77.1k | SmallVector<CanQualType, 16> argTypes; |
367 | 77.1k | for (auto &arg : args) |
368 | 108k | argTypes.push_back(ctx.getCanonicalParamType(arg.Ty)); |
369 | 77.1k | return argTypes; |
370 | 77.1k | } |
371 | | |
372 | | static SmallVector<CanQualType, 16> |
373 | 30.8k | getArgTypesForDeclaration(ASTContext &ctx, const FunctionArgList &args) { |
374 | 30.8k | SmallVector<CanQualType, 16> argTypes; |
375 | 30.8k | for (auto &arg : args) |
376 | 82.7k | argTypes.push_back(ctx.getCanonicalParamType(arg->getType())); |
377 | 30.8k | return argTypes; |
378 | 30.8k | } |
379 | | |
380 | | static llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16> |
381 | | getExtParameterInfosForCall(const FunctionProtoType *proto, |
382 | 76.6k | unsigned prefixArgs, unsigned totalArgs) { |
383 | 76.6k | llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16> result; |
384 | 76.6k | if (proto->hasExtParameterInfos()) { |
385 | 41 | addExtParameterInfosForCall(result, proto, prefixArgs, totalArgs); |
386 | 41 | } |
387 | 76.6k | return result; |
388 | 76.6k | } |
389 | | |
390 | | /// Arrange a call to a C++ method, passing the given arguments. |
391 | | /// |
392 | | /// ExtraPrefixArgs is the number of ABI-specific args passed after the `this` |
393 | | /// parameter. |
394 | | /// ExtraSuffixArgs is the number of ABI-specific args passed at the end of |
395 | | /// args. |
396 | | /// PassProtoArgs indicates whether `args` has args for the parameters in the |
397 | | /// given CXXConstructorDecl. |
398 | | const CGFunctionInfo & |
399 | | CodeGenTypes::arrangeCXXConstructorCall(const CallArgList &args, |
400 | | const CXXConstructorDecl *D, |
401 | | CXXCtorType CtorKind, |
402 | | unsigned ExtraPrefixArgs, |
403 | | unsigned ExtraSuffixArgs, |
404 | 54.3k | bool PassProtoArgs) { |
405 | | // FIXME: Kill copy. |
406 | 54.3k | SmallVector<CanQualType, 16> ArgTypes; |
407 | 54.3k | for (const auto &Arg : args) |
408 | 106k | ArgTypes.push_back(Context.getCanonicalParamType(Arg.Ty)); |
409 | | |
410 | | // +1 for implicit this, which should always be args[0]. |
411 | 54.3k | unsigned TotalPrefixArgs = 1 + ExtraPrefixArgs; |
412 | | |
413 | 54.3k | CanQual<FunctionProtoType> FPT = GetFormalType(D); |
414 | 54.3k | RequiredArgs Required = PassProtoArgs |
415 | 54.3k | ? RequiredArgs::forPrototypePlus( |
416 | 54.3k | FPT, TotalPrefixArgs + ExtraSuffixArgs) |
417 | 54.3k | : RequiredArgs::All10 ; |
418 | | |
419 | 54.3k | GlobalDecl GD(D, CtorKind); |
420 | 54.3k | CanQualType ResultType = TheCXXABI.HasThisReturn(GD) |
421 | 54.3k | ? ArgTypes.front()2.37k |
422 | 54.3k | : TheCXXABI.hasMostDerivedReturn(GD)51.9k |
423 | 51.9k | ? CGM.getContext().VoidPtrTy0 |
424 | 51.9k | : Context.VoidTy; |
425 | | |
426 | 54.3k | FunctionType::ExtInfo Info = FPT->getExtInfo(); |
427 | 54.3k | llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16> ParamInfos; |
428 | | // If the prototype args are elided, we should only have ABI-specific args, |
429 | | // which never have param info. |
430 | 54.3k | if (PassProtoArgs && FPT->hasExtParameterInfos()54.3k ) { |
431 | | // ABI-specific suffix arguments are treated the same as variadic arguments. |
432 | 70 | addExtParameterInfosForCall(ParamInfos, FPT.getTypePtr(), TotalPrefixArgs, |
433 | 70 | ArgTypes.size()); |
434 | 70 | } |
435 | 54.3k | return arrangeLLVMFunctionInfo(ResultType, /*instanceMethod=*/true, |
436 | 54.3k | /*chainCall=*/false, ArgTypes, Info, |
437 | 54.3k | ParamInfos, Required); |
438 | 54.3k | } |
439 | | |
440 | | /// Arrange the argument and result information for the declaration or |
441 | | /// definition of the given function. |
442 | | const CGFunctionInfo & |
443 | 517k | CodeGenTypes::arrangeFunctionDeclaration(const FunctionDecl *FD) { |
444 | 517k | if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(FD)) |
445 | 113k | if (MD->isInstance()) |
446 | 100k | return arrangeCXXMethodDeclaration(MD); |
447 | | |
448 | 416k | CanQualType FTy = FD->getType()->getCanonicalTypeUnqualified(); |
449 | | |
450 | 416k | assert(isa<FunctionType>(FTy)); |
451 | 0 | setCUDAKernelCallingConvention(FTy, CGM, FD); |
452 | | |
453 | | // When declaring a function without a prototype, always use a |
454 | | // non-variadic type. |
455 | 416k | if (CanQual<FunctionNoProtoType> noProto = FTy.getAs<FunctionNoProtoType>()) { |
456 | 4.28k | return arrangeLLVMFunctionInfo( |
457 | 4.28k | noProto->getReturnType(), /*instanceMethod=*/false, |
458 | 4.28k | /*chainCall=*/false, None, noProto->getExtInfo(), {},RequiredArgs::All); |
459 | 4.28k | } |
460 | | |
461 | 412k | return arrangeFreeFunctionType(FTy.castAs<FunctionProtoType>()); |
462 | 416k | } |
463 | | |
464 | | /// Arrange the argument and result information for the declaration or |
465 | | /// definition of an Objective-C method. |
466 | | const CGFunctionInfo & |
467 | 5.21k | CodeGenTypes::arrangeObjCMethodDeclaration(const ObjCMethodDecl *MD) { |
468 | | // It happens that this is the same as a call with no optional |
469 | | // arguments, except also using the formal 'self' type. |
470 | 5.21k | return arrangeObjCMessageSendSignature(MD, MD->getSelfDecl()->getType()); |
471 | 5.21k | } |
472 | | |
473 | | /// Arrange the argument and result information for the function type |
474 | | /// through which to perform a send to the given Objective-C method, |
475 | | /// using the given receiver type. The receiver type is not always |
476 | | /// the 'self' type of the method or even an Objective-C pointer type. |
477 | | /// This is *not* the right method for actually performing such a |
478 | | /// message send, due to the possibility of optional arguments. |
479 | | const CGFunctionInfo & |
480 | | CodeGenTypes::arrangeObjCMessageSendSignature(const ObjCMethodDecl *MD, |
481 | 16.3k | QualType receiverType) { |
482 | 16.3k | SmallVector<CanQualType, 16> argTys; |
483 | 16.3k | SmallVector<FunctionProtoType::ExtParameterInfo, 4> extParamInfos(2); |
484 | 16.3k | argTys.push_back(Context.getCanonicalParamType(receiverType)); |
485 | 16.3k | argTys.push_back(Context.getCanonicalParamType(Context.getObjCSelType())); |
486 | | // FIXME: Kill copy? |
487 | 16.3k | for (const auto *I : MD->parameters()) { |
488 | 13.5k | argTys.push_back(Context.getCanonicalParamType(I->getType())); |
489 | 13.5k | auto extParamInfo = FunctionProtoType::ExtParameterInfo().withIsNoEscape( |
490 | 13.5k | I->hasAttr<NoEscapeAttr>()); |
491 | 13.5k | extParamInfos.push_back(extParamInfo); |
492 | 13.5k | } |
493 | | |
494 | 16.3k | FunctionType::ExtInfo einfo; |
495 | 16.3k | bool IsWindows = getContext().getTargetInfo().getTriple().isOSWindows(); |
496 | 16.3k | einfo = einfo.withCallingConv(getCallingConventionForDecl(MD, IsWindows)); |
497 | | |
498 | 16.3k | if (getContext().getLangOpts().ObjCAutoRefCount && |
499 | 16.3k | MD->hasAttr<NSReturnsRetainedAttr>()880 ) |
500 | 114 | einfo = einfo.withProducesResult(true); |
501 | | |
502 | 16.3k | RequiredArgs required = |
503 | 16.3k | (MD->isVariadic() ? RequiredArgs(argTys.size())874 : RequiredArgs::All15.4k ); |
504 | | |
505 | 16.3k | return arrangeLLVMFunctionInfo( |
506 | 16.3k | GetReturnType(MD->getReturnType()), /*instanceMethod=*/false, |
507 | 16.3k | /*chainCall=*/false, argTys, einfo, extParamInfos, required); |
508 | 16.3k | } |
509 | | |
510 | | const CGFunctionInfo & |
511 | | CodeGenTypes::arrangeUnprototypedObjCMessageSend(QualType returnType, |
512 | 800 | const CallArgList &args) { |
513 | 800 | auto argTypes = getArgTypesForCall(Context, args); |
514 | 800 | FunctionType::ExtInfo einfo; |
515 | | |
516 | 800 | return arrangeLLVMFunctionInfo( |
517 | 800 | GetReturnType(returnType), /*instanceMethod=*/false, |
518 | 800 | /*chainCall=*/false, argTypes, einfo, {}, RequiredArgs::All); |
519 | 800 | } |
520 | | |
521 | | const CGFunctionInfo & |
522 | 581k | CodeGenTypes::arrangeGlobalDeclaration(GlobalDecl GD) { |
523 | | // FIXME: Do we need to handle ObjCMethodDecl? |
524 | 581k | const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl()); |
525 | | |
526 | 581k | if (isa<CXXConstructorDecl>(GD.getDecl()) || |
527 | 581k | isa<CXXDestructorDecl>(GD.getDecl())539k ) |
528 | 64.1k | return arrangeCXXStructorDeclaration(GD); |
529 | | |
530 | 517k | return arrangeFunctionDeclaration(FD); |
531 | 581k | } |
532 | | |
533 | | /// Arrange a thunk that takes 'this' as the first parameter followed by |
534 | | /// varargs. Return a void pointer, regardless of the actual return type. |
535 | | /// The body of the thunk will end in a musttail call to a function of the |
536 | | /// correct type, and the caller will bitcast the function to the correct |
537 | | /// prototype. |
538 | | const CGFunctionInfo & |
539 | 55 | CodeGenTypes::arrangeUnprototypedMustTailThunk(const CXXMethodDecl *MD) { |
540 | 55 | assert(MD->isVirtual() && "only methods have thunks"); |
541 | 0 | CanQual<FunctionProtoType> FTP = GetFormalType(MD); |
542 | 55 | CanQualType ArgTys[] = {DeriveThisType(MD->getParent(), MD)}; |
543 | 55 | return arrangeLLVMFunctionInfo(Context.VoidTy, /*instanceMethod=*/false, |
544 | 55 | /*chainCall=*/false, ArgTys, |
545 | 55 | FTP->getExtInfo(), {}, RequiredArgs(1)); |
546 | 55 | } |
547 | | |
548 | | const CGFunctionInfo & |
549 | | CodeGenTypes::arrangeMSCtorClosure(const CXXConstructorDecl *CD, |
550 | 34 | CXXCtorType CT) { |
551 | 34 | assert(CT == Ctor_CopyingClosure || CT == Ctor_DefaultClosure); |
552 | | |
553 | 0 | CanQual<FunctionProtoType> FTP = GetFormalType(CD); |
554 | 34 | SmallVector<CanQualType, 2> ArgTys; |
555 | 34 | const CXXRecordDecl *RD = CD->getParent(); |
556 | 34 | ArgTys.push_back(DeriveThisType(RD, CD)); |
557 | 34 | if (CT == Ctor_CopyingClosure) |
558 | 3 | ArgTys.push_back(*FTP->param_type_begin()); |
559 | 34 | if (RD->getNumVBases() > 0) |
560 | 0 | ArgTys.push_back(Context.IntTy); |
561 | 34 | CallingConv CC = Context.getDefaultCallingConvention( |
562 | 34 | /*IsVariadic=*/false, /*IsCXXMethod=*/true); |
563 | 34 | return arrangeLLVMFunctionInfo(Context.VoidTy, /*instanceMethod=*/true, |
564 | 34 | /*chainCall=*/false, ArgTys, |
565 | 34 | FunctionType::ExtInfo(CC), {}, |
566 | 34 | RequiredArgs::All); |
567 | 34 | } |
568 | | |
569 | | /// Arrange a call as unto a free function, except possibly with an |
570 | | /// additional number of formal parameters considered required. |
571 | | static const CGFunctionInfo & |
572 | | arrangeFreeFunctionLikeCall(CodeGenTypes &CGT, |
573 | | CodeGenModule &CGM, |
574 | | const CallArgList &args, |
575 | | const FunctionType *fnType, |
576 | | unsigned numExtraRequiredArgs, |
577 | 193k | bool chainCall) { |
578 | 193k | assert(args.size() >= numExtraRequiredArgs); |
579 | | |
580 | 0 | llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos; |
581 | | |
582 | | // In most cases, there are no optional arguments. |
583 | 193k | RequiredArgs required = RequiredArgs::All; |
584 | | |
585 | | // If we have a variadic prototype, the required arguments are the |
586 | | // extra prefix plus the arguments in the prototype. |
587 | 193k | if (const FunctionProtoType *proto = dyn_cast<FunctionProtoType>(fnType)) { |
588 | 193k | if (proto->isVariadic()) |
589 | 76.2k | required = RequiredArgs::forPrototypePlus(proto, numExtraRequiredArgs); |
590 | | |
591 | 193k | if (proto->hasExtParameterInfos()) |
592 | 305 | addExtParameterInfosForCall(paramInfos, proto, numExtraRequiredArgs, |
593 | 305 | args.size()); |
594 | | |
595 | | // If we don't have a prototype at all, but we're supposed to |
596 | | // explicitly use the variadic convention for unprototyped calls, |
597 | | // treat all of the arguments as required but preserve the nominal |
598 | | // possibility of variadics. |
599 | 193k | } else if (526 CGM.getTargetCodeGenInfo() |
600 | 526 | .isNoProtoCallVariadic(args, |
601 | 526 | cast<FunctionNoProtoType>(fnType))) { |
602 | 423 | required = RequiredArgs(args.size()); |
603 | 423 | } |
604 | | |
605 | | // FIXME: Kill copy. |
606 | 193k | SmallVector<CanQualType, 16> argTypes; |
607 | 193k | for (const auto &arg : args) |
608 | 336k | argTypes.push_back(CGT.getContext().getCanonicalParamType(arg.Ty)); |
609 | 193k | return CGT.arrangeLLVMFunctionInfo(GetReturnType(fnType->getReturnType()), |
610 | 193k | /*instanceMethod=*/false, chainCall, |
611 | 193k | argTypes, fnType->getExtInfo(), paramInfos, |
612 | 193k | required); |
613 | 193k | } |
614 | | |
615 | | /// Figure out the rules for calling a function with the given formal |
616 | | /// type using the given arguments. The arguments are necessary |
617 | | /// because the function might be unprototyped, in which case it's |
618 | | /// target-dependent in crazy ways. |
619 | | const CGFunctionInfo & |
620 | | CodeGenTypes::arrangeFreeFunctionCall(const CallArgList &args, |
621 | | const FunctionType *fnType, |
622 | 193k | bool chainCall) { |
623 | 193k | return arrangeFreeFunctionLikeCall(*this, CGM, args, fnType, |
624 | 193k | chainCall ? 18 : 0193k , chainCall); |
625 | 193k | } |
626 | | |
627 | | /// A block function is essentially a free function with an |
628 | | /// extra implicit argument. |
629 | | const CGFunctionInfo & |
630 | | CodeGenTypes::arrangeBlockFunctionCall(const CallArgList &args, |
631 | 620 | const FunctionType *fnType) { |
632 | 620 | return arrangeFreeFunctionLikeCall(*this, CGM, args, fnType, 1, |
633 | 620 | /*chainCall=*/false); |
634 | 620 | } |
635 | | |
636 | | const CGFunctionInfo & |
637 | | CodeGenTypes::arrangeBlockFunctionDeclaration(const FunctionProtoType *proto, |
638 | 1.17k | const FunctionArgList ¶ms) { |
639 | 1.17k | auto paramInfos = getExtParameterInfosForCall(proto, 1, params.size()); |
640 | 1.17k | auto argTypes = getArgTypesForDeclaration(Context, params); |
641 | | |
642 | 1.17k | return arrangeLLVMFunctionInfo(GetReturnType(proto->getReturnType()), |
643 | 1.17k | /*instanceMethod*/ false, /*chainCall*/ false, |
644 | 1.17k | argTypes, proto->getExtInfo(), paramInfos, |
645 | 1.17k | RequiredArgs::forPrototypePlus(proto, 1)); |
646 | 1.17k | } |
647 | | |
648 | | const CGFunctionInfo & |
649 | | CodeGenTypes::arrangeBuiltinFunctionCall(QualType resultType, |
650 | 1.00k | const CallArgList &args) { |
651 | | // FIXME: Kill copy. |
652 | 1.00k | SmallVector<CanQualType, 16> argTypes; |
653 | 1.00k | for (const auto &Arg : args) |
654 | 3.68k | argTypes.push_back(Context.getCanonicalParamType(Arg.Ty)); |
655 | 1.00k | return arrangeLLVMFunctionInfo( |
656 | 1.00k | GetReturnType(resultType), /*instanceMethod=*/false, |
657 | 1.00k | /*chainCall=*/false, argTypes, FunctionType::ExtInfo(), |
658 | 1.00k | /*paramInfos=*/ {}, RequiredArgs::All); |
659 | 1.00k | } |
660 | | |
661 | | const CGFunctionInfo & |
662 | | CodeGenTypes::arrangeBuiltinFunctionDeclaration(QualType resultType, |
663 | 29.7k | const FunctionArgList &args) { |
664 | 29.7k | auto argTypes = getArgTypesForDeclaration(Context, args); |
665 | | |
666 | 29.7k | return arrangeLLVMFunctionInfo( |
667 | 29.7k | GetReturnType(resultType), /*instanceMethod=*/false, /*chainCall=*/false, |
668 | 29.7k | argTypes, FunctionType::ExtInfo(), {}, RequiredArgs::All); |
669 | 29.7k | } |
670 | | |
671 | | const CGFunctionInfo & |
672 | | CodeGenTypes::arrangeBuiltinFunctionDeclaration(CanQualType resultType, |
673 | 374 | ArrayRef<CanQualType> argTypes) { |
674 | 374 | return arrangeLLVMFunctionInfo( |
675 | 374 | resultType, /*instanceMethod=*/false, /*chainCall=*/false, |
676 | 374 | argTypes, FunctionType::ExtInfo(), {}, RequiredArgs::All); |
677 | 374 | } |
678 | | |
679 | | /// Arrange a call to a C++ method, passing the given arguments. |
680 | | /// |
681 | | /// numPrefixArgs is the number of ABI-specific prefix arguments we have. It |
682 | | /// does not count `this`. |
683 | | const CGFunctionInfo & |
684 | | CodeGenTypes::arrangeCXXMethodCall(const CallArgList &args, |
685 | | const FunctionProtoType *proto, |
686 | | RequiredArgs required, |
687 | 75.4k | unsigned numPrefixArgs) { |
688 | 75.4k | assert(numPrefixArgs + 1 <= args.size() && |
689 | 75.4k | "Emitting a call with less args than the required prefix?"); |
690 | | // Add one to account for `this`. It's a bit awkward here, but we don't count |
691 | | // `this` in similar places elsewhere. |
692 | 0 | auto paramInfos = |
693 | 75.4k | getExtParameterInfosForCall(proto, numPrefixArgs + 1, args.size()); |
694 | | |
695 | | // FIXME: Kill copy. |
696 | 75.4k | auto argTypes = getArgTypesForCall(Context, args); |
697 | | |
698 | 75.4k | FunctionType::ExtInfo info = proto->getExtInfo(); |
699 | 75.4k | return arrangeLLVMFunctionInfo( |
700 | 75.4k | GetReturnType(proto->getReturnType()), /*instanceMethod=*/true, |
701 | 75.4k | /*chainCall=*/false, argTypes, info, paramInfos, required); |
702 | 75.4k | } |
703 | | |
704 | 17.6k | const CGFunctionInfo &CodeGenTypes::arrangeNullaryFunction() { |
705 | 17.6k | return arrangeLLVMFunctionInfo( |
706 | 17.6k | getContext().VoidTy, /*instanceMethod=*/false, /*chainCall=*/false, |
707 | 17.6k | None, FunctionType::ExtInfo(), {}, RequiredArgs::All); |
708 | 17.6k | } |
709 | | |
710 | | const CGFunctionInfo & |
711 | | CodeGenTypes::arrangeCall(const CGFunctionInfo &signature, |
712 | 11.1k | const CallArgList &args) { |
713 | 11.1k | assert(signature.arg_size() <= args.size()); |
714 | 11.1k | if (signature.arg_size() == args.size()) |
715 | 10.2k | return signature; |
716 | | |
717 | 856 | SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos; |
718 | 856 | auto sigParamInfos = signature.getExtParameterInfos(); |
719 | 856 | if (!sigParamInfos.empty()) { |
720 | 856 | paramInfos.append(sigParamInfos.begin(), sigParamInfos.end()); |
721 | 856 | paramInfos.resize(args.size()); |
722 | 856 | } |
723 | | |
724 | 856 | auto argTypes = getArgTypesForCall(Context, args); |
725 | | |
726 | 856 | assert(signature.getRequiredArgs().allowsOptionalArgs()); |
727 | 0 | return arrangeLLVMFunctionInfo(signature.getReturnType(), |
728 | 856 | signature.isInstanceMethod(), |
729 | 856 | signature.isChainCall(), |
730 | 856 | argTypes, |
731 | 856 | signature.getExtInfo(), |
732 | 856 | paramInfos, |
733 | 856 | signature.getRequiredArgs()); |
734 | 11.1k | } |
735 | | |
736 | | namespace clang { |
737 | | namespace CodeGen { |
738 | | void computeSPIRKernelABIInfo(CodeGenModule &CGM, CGFunctionInfo &FI); |
739 | | } |
740 | | } |
741 | | |
742 | | /// Arrange the argument and result information for an abstract value |
743 | | /// of a given function type. This is the method which all of the |
744 | | /// above functions ultimately defer to. |
745 | | const CGFunctionInfo & |
746 | | CodeGenTypes::arrangeLLVMFunctionInfo(CanQualType resultType, |
747 | | bool instanceMethod, |
748 | | bool chainCall, |
749 | | ArrayRef<CanQualType> argTypes, |
750 | | FunctionType::ExtInfo info, |
751 | | ArrayRef<FunctionProtoType::ExtParameterInfo> paramInfos, |
752 | 1.39M | RequiredArgs required) { |
753 | 1.39M | assert(llvm::all_of(argTypes, |
754 | 1.39M | [](CanQualType T) { return T.isCanonicalAsParam(); })); |
755 | | |
756 | | // Lookup or create unique function info. |
757 | 0 | llvm::FoldingSetNodeID ID; |
758 | 1.39M | CGFunctionInfo::Profile(ID, instanceMethod, chainCall, info, paramInfos, |
759 | 1.39M | required, resultType, argTypes); |
760 | | |
761 | 1.39M | void *insertPos = nullptr; |
762 | 1.39M | CGFunctionInfo *FI = FunctionInfos.FindNodeOrInsertPos(ID, insertPos); |
763 | 1.39M | if (FI) |
764 | 1.16M | return *FI; |
765 | | |
766 | 238k | unsigned CC = ClangCallConvToLLVMCallConv(info.getCC()); |
767 | | |
768 | | // Construct the function info. We co-allocate the ArgInfos. |
769 | 238k | FI = CGFunctionInfo::create(CC, instanceMethod, chainCall, info, |
770 | 238k | paramInfos, resultType, argTypes, required); |
771 | 238k | FunctionInfos.InsertNode(FI, insertPos); |
772 | | |
773 | 238k | bool inserted = FunctionsBeingProcessed.insert(FI).second; |
774 | 238k | (void)inserted; |
775 | 238k | assert(inserted && "Recursively being processed?"); |
776 | | |
777 | | // Compute ABI information. |
778 | 238k | if (CC == llvm::CallingConv::SPIR_KERNEL) { |
779 | | // Force target independent argument handling for the host visible |
780 | | // kernel functions. |
781 | 179 | computeSPIRKernelABIInfo(CGM, *FI); |
782 | 237k | } else if (info.getCC() == CC_Swift || info.getCC() == CC_SwiftAsync236k ) { |
783 | 1.21k | swiftcall::computeABIInfo(CGM, *FI); |
784 | 236k | } else { |
785 | 236k | getABIInfo().computeInfo(*FI); |
786 | 236k | } |
787 | | |
788 | | // Loop over all of the computed argument and return value info. If any of |
789 | | // them are direct or extend without a specified coerce type, specify the |
790 | | // default now. |
791 | 238k | ABIArgInfo &retInfo = FI->getReturnInfo(); |
792 | 238k | if (retInfo.canHaveCoerceToType() && retInfo.getCoerceToType() == nullptr136k ) |
793 | 41.9k | retInfo.setCoerceToType(ConvertType(FI->getReturnType())); |
794 | | |
795 | 238k | for (auto &I : FI->arguments()) |
796 | 434k | if (I.info.canHaveCoerceToType() && I.info.getCoerceToType() == nullptr426k ) |
797 | 141k | I.info.setCoerceToType(ConvertType(I.type)); |
798 | | |
799 | 238k | bool erased = FunctionsBeingProcessed.erase(FI); (void)erased; |
800 | 238k | assert(erased && "Not in set?"); |
801 | | |
802 | 0 | return *FI; |
803 | 1.39M | } |
804 | | |
805 | | CGFunctionInfo *CGFunctionInfo::create(unsigned llvmCC, |
806 | | bool instanceMethod, |
807 | | bool chainCall, |
808 | | const FunctionType::ExtInfo &info, |
809 | | ArrayRef<ExtParameterInfo> paramInfos, |
810 | | CanQualType resultType, |
811 | | ArrayRef<CanQualType> argTypes, |
812 | 238k | RequiredArgs required) { |
813 | 238k | assert(paramInfos.empty() || paramInfos.size() == argTypes.size()); |
814 | 0 | assert(!required.allowsOptionalArgs() || |
815 | 238k | required.getNumRequiredArgs() <= argTypes.size()); |
816 | | |
817 | 0 | void *buffer = |
818 | 238k | operator new(totalSizeToAlloc<ArgInfo, ExtParameterInfo>( |
819 | 238k | argTypes.size() + 1, paramInfos.size())); |
820 | | |
821 | 238k | CGFunctionInfo *FI = new(buffer) CGFunctionInfo(); |
822 | 238k | FI->CallingConvention = llvmCC; |
823 | 238k | FI->EffectiveCallingConvention = llvmCC; |
824 | 238k | FI->ASTCallingConvention = info.getCC(); |
825 | 238k | FI->InstanceMethod = instanceMethod; |
826 | 238k | FI->ChainCall = chainCall; |
827 | 238k | FI->CmseNSCall = info.getCmseNSCall(); |
828 | 238k | FI->NoReturn = info.getNoReturn(); |
829 | 238k | FI->ReturnsRetained = info.getProducesResult(); |
830 | 238k | FI->NoCallerSavedRegs = info.getNoCallerSavedRegs(); |
831 | 238k | FI->NoCfCheck = info.getNoCfCheck(); |
832 | 238k | FI->Required = required; |
833 | 238k | FI->HasRegParm = info.getHasRegParm(); |
834 | 238k | FI->RegParm = info.getRegParm(); |
835 | 238k | FI->ArgStruct = nullptr; |
836 | 238k | FI->ArgStructAlign = 0; |
837 | 238k | FI->NumArgs = argTypes.size(); |
838 | 238k | FI->HasExtParameterInfos = !paramInfos.empty(); |
839 | 238k | FI->getArgsBuffer()[0].type = resultType; |
840 | 238k | FI->MaxVectorWidth = 0; |
841 | 672k | for (unsigned i = 0, e = argTypes.size(); i != e; ++i434k ) |
842 | 434k | FI->getArgsBuffer()[i + 1].type = argTypes[i]; |
843 | 258k | for (unsigned i = 0, e = paramInfos.size(); i != e; ++i20.1k ) |
844 | 20.1k | FI->getExtParameterInfosBuffer()[i] = paramInfos[i]; |
845 | 238k | return FI; |
846 | 238k | } |
847 | | |
848 | | /***/ |
849 | | |
850 | | namespace { |
851 | | // ABIArgInfo::Expand implementation. |
852 | | |
853 | | // Specifies the way QualType passed as ABIArgInfo::Expand is expanded. |
854 | | struct TypeExpansion { |
855 | | enum TypeExpansionKind { |
856 | | // Elements of constant arrays are expanded recursively. |
857 | | TEK_ConstantArray, |
858 | | // Record fields are expanded recursively (but if record is a union, only |
859 | | // the field with the largest size is expanded). |
860 | | TEK_Record, |
861 | | // For complex types, real and imaginary parts are expanded recursively. |
862 | | TEK_Complex, |
863 | | // All other types are not expandable. |
864 | | TEK_None |
865 | | }; |
866 | | |
867 | | const TypeExpansionKind Kind; |
868 | | |
869 | 2.36k | TypeExpansion(TypeExpansionKind K) : Kind(K) {} |
870 | 2.36k | virtual ~TypeExpansion() {} |
871 | | }; |
872 | | |
873 | | struct ConstantArrayExpansion : TypeExpansion { |
874 | | QualType EltTy; |
875 | | uint64_t NumElts; |
876 | | |
877 | | ConstantArrayExpansion(QualType EltTy, uint64_t NumElts) |
878 | 40 | : TypeExpansion(TEK_ConstantArray), EltTy(EltTy), NumElts(NumElts) {} |
879 | 2.36k | static bool classof(const TypeExpansion *TE) { |
880 | 2.36k | return TE->Kind == TEK_ConstantArray; |
881 | 2.36k | } |
882 | | }; |
883 | | |
884 | | struct RecordExpansion : TypeExpansion { |
885 | | SmallVector<const CXXBaseSpecifier *, 1> Bases; |
886 | | |
887 | | SmallVector<const FieldDecl *, 1> Fields; |
888 | | |
889 | | RecordExpansion(SmallVector<const CXXBaseSpecifier *, 1> &&Bases, |
890 | | SmallVector<const FieldDecl *, 1> &&Fields) |
891 | | : TypeExpansion(TEK_Record), Bases(std::move(Bases)), |
892 | 761 | Fields(std::move(Fields)) {} |
893 | 2.32k | static bool classof(const TypeExpansion *TE) { |
894 | 2.32k | return TE->Kind == TEK_Record; |
895 | 2.32k | } |
896 | | }; |
897 | | |
898 | | struct ComplexExpansion : TypeExpansion { |
899 | | QualType EltTy; |
900 | | |
901 | 46 | ComplexExpansion(QualType EltTy) : TypeExpansion(TEK_Complex), EltTy(EltTy) {} |
902 | 1.56k | static bool classof(const TypeExpansion *TE) { |
903 | 1.56k | return TE->Kind == TEK_Complex; |
904 | 1.56k | } |
905 | | }; |
906 | | |
907 | | struct NoExpansion : TypeExpansion { |
908 | 1.51k | NoExpansion() : TypeExpansion(TEK_None) {} |
909 | 1.51k | static bool classof(const TypeExpansion *TE) { |
910 | 1.51k | return TE->Kind == TEK_None; |
911 | 1.51k | } |
912 | | }; |
913 | | } // namespace |
914 | | |
915 | | static std::unique_ptr<TypeExpansion> |
916 | 2.36k | getTypeExpansion(QualType Ty, const ASTContext &Context) { |
917 | 2.36k | if (const ConstantArrayType *AT = Context.getAsConstantArrayType(Ty)) { |
918 | 40 | return std::make_unique<ConstantArrayExpansion>( |
919 | 40 | AT->getElementType(), AT->getSize().getZExtValue()); |
920 | 40 | } |
921 | 2.32k | if (const RecordType *RT = Ty->getAs<RecordType>()) { |
922 | 761 | SmallVector<const CXXBaseSpecifier *, 1> Bases; |
923 | 761 | SmallVector<const FieldDecl *, 1> Fields; |
924 | 761 | const RecordDecl *RD = RT->getDecl(); |
925 | 761 | assert(!RD->hasFlexibleArrayMember() && |
926 | 761 | "Cannot expand structure with flexible array."); |
927 | 761 | if (RD->isUnion()) { |
928 | | // Unions can be here only in degenerative cases - all the fields are same |
929 | | // after flattening. Thus we have to use the "largest" field. |
930 | 5 | const FieldDecl *LargestFD = nullptr; |
931 | 5 | CharUnits UnionSize = CharUnits::Zero(); |
932 | | |
933 | 10 | for (const auto *FD : RD->fields()) { |
934 | 10 | if (FD->isZeroLengthBitField(Context)) |
935 | 0 | continue; |
936 | 10 | assert(!FD->isBitField() && |
937 | 10 | "Cannot expand structure with bit-field members."); |
938 | 0 | CharUnits FieldSize = Context.getTypeSizeInChars(FD->getType()); |
939 | 10 | if (UnionSize < FieldSize) { |
940 | 5 | UnionSize = FieldSize; |
941 | 5 | LargestFD = FD; |
942 | 5 | } |
943 | 10 | } |
944 | 5 | if (LargestFD) |
945 | 5 | Fields.push_back(LargestFD); |
946 | 756 | } else { |
947 | 756 | if (const auto *CXXRD = dyn_cast<CXXRecordDecl>(RD)) { |
948 | 241 | assert(!CXXRD->isDynamicClass() && |
949 | 241 | "cannot expand vtable pointers in dynamic classes"); |
950 | 0 | llvm::append_range(Bases, llvm::make_pointer_range(CXXRD->bases())); |
951 | 241 | } |
952 | | |
953 | 1.47k | for (const auto *FD : RD->fields()) { |
954 | 1.47k | if (FD->isZeroLengthBitField(Context)) |
955 | 0 | continue; |
956 | 1.47k | assert(!FD->isBitField() && |
957 | 1.47k | "Cannot expand structure with bit-field members."); |
958 | 0 | Fields.push_back(FD); |
959 | 1.47k | } |
960 | 756 | } |
961 | 0 | return std::make_unique<RecordExpansion>(std::move(Bases), |
962 | 761 | std::move(Fields)); |
963 | 761 | } |
964 | 1.56k | if (const ComplexType *CT = Ty->getAs<ComplexType>()) { |
965 | 46 | return std::make_unique<ComplexExpansion>(CT->getElementType()); |
966 | 46 | } |
967 | 1.51k | return std::make_unique<NoExpansion>(); |
968 | 1.56k | } |
969 | | |
970 | 1.38k | static int getExpansionSize(QualType Ty, const ASTContext &Context) { |
971 | 1.38k | auto Exp = getTypeExpansion(Ty, Context); |
972 | 1.38k | if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) { |
973 | 24 | return CAExp->NumElts * getExpansionSize(CAExp->EltTy, Context); |
974 | 24 | } |
975 | 1.36k | if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) { |
976 | 456 | int Res = 0; |
977 | 456 | for (auto BS : RExp->Bases) |
978 | 3 | Res += getExpansionSize(BS->getType(), Context); |
979 | 456 | for (auto FD : RExp->Fields) |
980 | 887 | Res += getExpansionSize(FD->getType(), Context); |
981 | 456 | return Res; |
982 | 456 | } |
983 | 908 | if (isa<ComplexExpansion>(Exp.get())) |
984 | 28 | return 2; |
985 | 880 | assert(isa<NoExpansion>(Exp.get())); |
986 | 0 | return 1; |
987 | 908 | } |
988 | | |
989 | | void |
990 | | CodeGenTypes::getExpandedTypes(QualType Ty, |
991 | 546 | SmallVectorImpl<llvm::Type *>::iterator &TI) { |
992 | 546 | auto Exp = getTypeExpansion(Ty, Context); |
993 | 546 | if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) { |
994 | 40 | for (int i = 0, n = CAExp->NumElts; i < n; i++32 ) { |
995 | 32 | getExpandedTypes(CAExp->EltTy, TI); |
996 | 32 | } |
997 | 538 | } else if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) { |
998 | 174 | for (auto BS : RExp->Bases) |
999 | 1 | getExpandedTypes(BS->getType(), TI); |
1000 | 174 | for (auto FD : RExp->Fields) |
1001 | 332 | getExpandedTypes(FD->getType(), TI); |
1002 | 364 | } else if (auto CExp = dyn_cast<ComplexExpansion>(Exp.get())) { |
1003 | 11 | llvm::Type *EltTy = ConvertType(CExp->EltTy); |
1004 | 11 | *TI++ = EltTy; |
1005 | 11 | *TI++ = EltTy; |
1006 | 353 | } else { |
1007 | 353 | assert(isa<NoExpansion>(Exp.get())); |
1008 | 0 | *TI++ = ConvertType(Ty); |
1009 | 353 | } |
1010 | 546 | } |
1011 | | |
1012 | | static void forConstantArrayExpansion(CodeGenFunction &CGF, |
1013 | | ConstantArrayExpansion *CAE, |
1014 | | Address BaseAddr, |
1015 | 8 | llvm::function_ref<void(Address)> Fn) { |
1016 | 8 | CharUnits EltSize = CGF.getContext().getTypeSizeInChars(CAE->EltTy); |
1017 | 8 | CharUnits EltAlign = |
1018 | 8 | BaseAddr.getAlignment().alignmentOfArrayElement(EltSize); |
1019 | 8 | llvm::Type *EltTy = CGF.ConvertTypeForMem(CAE->EltTy); |
1020 | | |
1021 | 40 | for (int i = 0, n = CAE->NumElts; i < n; i++32 ) { |
1022 | 32 | llvm::Value *EltAddr = CGF.Builder.CreateConstGEP2_32( |
1023 | 32 | BaseAddr.getElementType(), BaseAddr.getPointer(), 0, i); |
1024 | 32 | Fn(Address(EltAddr, EltTy, EltAlign)); |
1025 | 32 | } |
1026 | 8 | } |
1027 | | |
1028 | | void CodeGenFunction::ExpandTypeFromArgs(QualType Ty, LValue LV, |
1029 | 349 | llvm::Function::arg_iterator &AI) { |
1030 | 349 | assert(LV.isSimple() && |
1031 | 349 | "Unexpected non-simple lvalue during struct expansion."); |
1032 | | |
1033 | 0 | auto Exp = getTypeExpansion(Ty, getContext()); |
1034 | 349 | if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) { |
1035 | 8 | forConstantArrayExpansion( |
1036 | 32 | *this, CAExp, LV.getAddress(*this), [&](Address EltAddr) { |
1037 | 32 | LValue LV = MakeAddrLValue(EltAddr, CAExp->EltTy); |
1038 | 32 | ExpandTypeFromArgs(CAExp->EltTy, LV, AI); |
1039 | 32 | }); |
1040 | 341 | } else if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) { |
1041 | 101 | Address This = LV.getAddress(*this); |
1042 | 101 | for (const CXXBaseSpecifier *BS : RExp->Bases) { |
1043 | | // Perform a single step derived-to-base conversion. |
1044 | 1 | Address Base = |
1045 | 1 | GetAddressOfBaseClass(This, Ty->getAsCXXRecordDecl(), &BS, &BS + 1, |
1046 | 1 | /*NullCheckValue=*/false, SourceLocation()); |
1047 | 1 | LValue SubLV = MakeAddrLValue(Base, BS->getType()); |
1048 | | |
1049 | | // Recurse onto bases. |
1050 | 1 | ExpandTypeFromArgs(BS->getType(), SubLV, AI); |
1051 | 1 | } |
1052 | 214 | for (auto FD : RExp->Fields) { |
1053 | | // FIXME: What are the right qualifiers here? |
1054 | 214 | LValue SubLV = EmitLValueForFieldInitialization(LV, FD); |
1055 | 214 | ExpandTypeFromArgs(FD->getType(), SubLV, AI); |
1056 | 214 | } |
1057 | 240 | } else if (isa<ComplexExpansion>(Exp.get())) { |
1058 | 3 | auto realValue = &*AI++; |
1059 | 3 | auto imagValue = &*AI++; |
1060 | 3 | EmitStoreOfComplex(ComplexPairTy(realValue, imagValue), LV, /*init*/ true); |
1061 | 237 | } else { |
1062 | | // Call EmitStoreOfScalar except when the lvalue is a bitfield to emit a |
1063 | | // primitive store. |
1064 | 237 | assert(isa<NoExpansion>(Exp.get())); |
1065 | 0 | llvm::Value *Arg = &*AI++; |
1066 | 237 | if (LV.isBitField()) { |
1067 | 0 | EmitStoreThroughLValue(RValue::get(Arg), LV); |
1068 | 237 | } else { |
1069 | | // TODO: currently there are some places are inconsistent in what LLVM |
1070 | | // pointer type they use (see D118744). Once clang uses opaque pointers |
1071 | | // all LLVM pointer types will be the same and we can remove this check. |
1072 | 237 | if (Arg->getType()->isPointerTy()) { |
1073 | 6 | Address Addr = LV.getAddress(*this); |
1074 | 6 | Arg = Builder.CreateBitCast(Arg, Addr.getElementType()); |
1075 | 6 | } |
1076 | 237 | EmitStoreOfScalar(Arg, LV); |
1077 | 237 | } |
1078 | 237 | } |
1079 | 349 | } |
1080 | | |
1081 | | void CodeGenFunction::ExpandTypeToArgs( |
1082 | | QualType Ty, CallArg Arg, llvm::FunctionType *IRFuncTy, |
1083 | 81 | SmallVectorImpl<llvm::Value *> &IRCallArgs, unsigned &IRCallArgPos) { |
1084 | 81 | auto Exp = getTypeExpansion(Ty, getContext()); |
1085 | 81 | if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) { |
1086 | 0 | Address Addr = Arg.hasLValue() ? Arg.getKnownLValue().getAddress(*this) |
1087 | 0 | : Arg.getKnownRValue().getAggregateAddress(); |
1088 | 0 | forConstantArrayExpansion( |
1089 | 0 | *this, CAExp, Addr, [&](Address EltAddr) { |
1090 | 0 | CallArg EltArg = CallArg( |
1091 | 0 | convertTempToRValue(EltAddr, CAExp->EltTy, SourceLocation()), |
1092 | 0 | CAExp->EltTy); |
1093 | 0 | ExpandTypeToArgs(CAExp->EltTy, EltArg, IRFuncTy, IRCallArgs, |
1094 | 0 | IRCallArgPos); |
1095 | 0 | }); |
1096 | 81 | } else if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) { |
1097 | 30 | Address This = Arg.hasLValue() ? Arg.getKnownLValue().getAddress(*this)12 |
1098 | 30 | : Arg.getKnownRValue().getAggregateAddress()18 ; |
1099 | 30 | for (const CXXBaseSpecifier *BS : RExp->Bases) { |
1100 | | // Perform a single step derived-to-base conversion. |
1101 | 0 | Address Base = |
1102 | 0 | GetAddressOfBaseClass(This, Ty->getAsCXXRecordDecl(), &BS, &BS + 1, |
1103 | 0 | /*NullCheckValue=*/false, SourceLocation()); |
1104 | 0 | CallArg BaseArg = CallArg(RValue::getAggregate(Base), BS->getType()); |
1105 | | |
1106 | | // Recurse onto bases. |
1107 | 0 | ExpandTypeToArgs(BS->getType(), BaseArg, IRFuncTy, IRCallArgs, |
1108 | 0 | IRCallArgPos); |
1109 | 0 | } |
1110 | | |
1111 | 30 | LValue LV = MakeAddrLValue(This, Ty); |
1112 | 48 | for (auto FD : RExp->Fields) { |
1113 | 48 | CallArg FldArg = |
1114 | 48 | CallArg(EmitRValueForField(LV, FD, SourceLocation()), FD->getType()); |
1115 | 48 | ExpandTypeToArgs(FD->getType(), FldArg, IRFuncTy, IRCallArgs, |
1116 | 48 | IRCallArgPos); |
1117 | 48 | } |
1118 | 51 | } else if (isa<ComplexExpansion>(Exp.get())) { |
1119 | 4 | ComplexPairTy CV = Arg.getKnownRValue().getComplexVal(); |
1120 | 4 | IRCallArgs[IRCallArgPos++] = CV.first; |
1121 | 4 | IRCallArgs[IRCallArgPos++] = CV.second; |
1122 | 47 | } else { |
1123 | 47 | assert(isa<NoExpansion>(Exp.get())); |
1124 | 0 | auto RV = Arg.getKnownRValue(); |
1125 | 47 | assert(RV.isScalar() && |
1126 | 47 | "Unexpected non-scalar rvalue during struct expansion."); |
1127 | | |
1128 | | // Insert a bitcast as needed. |
1129 | 0 | llvm::Value *V = RV.getScalarVal(); |
1130 | 47 | if (IRCallArgPos < IRFuncTy->getNumParams() && |
1131 | 47 | V->getType() != IRFuncTy->getParamType(IRCallArgPos)) |
1132 | 1 | V = Builder.CreateBitCast(V, IRFuncTy->getParamType(IRCallArgPos)); |
1133 | | |
1134 | 47 | IRCallArgs[IRCallArgPos++] = V; |
1135 | 47 | } |
1136 | 81 | } |
1137 | | |
1138 | | /// Create a temporary allocation for the purposes of coercion. |
1139 | | static Address CreateTempAllocaForCoercion(CodeGenFunction &CGF, llvm::Type *Ty, |
1140 | | CharUnits MinAlign, |
1141 | 247 | const Twine &Name = "tmp") { |
1142 | | // Don't use an alignment that's worse than what LLVM would prefer. |
1143 | 247 | auto PrefAlign = CGF.CGM.getDataLayout().getPrefTypeAlignment(Ty); |
1144 | 247 | CharUnits Align = std::max(MinAlign, CharUnits::fromQuantity(PrefAlign)); |
1145 | | |
1146 | 247 | return CGF.CreateTempAlloca(Ty, Align, Name + ".coerce"); |
1147 | 247 | } |
1148 | | |
1149 | | /// EnterStructPointerForCoercedAccess - Given a struct pointer that we are |
1150 | | /// accessing some number of bytes out of it, try to gep into the struct to get |
1151 | | /// at its inner goodness. Dive as deep as possible without entering an element |
1152 | | /// with an in-memory size smaller than DstSize. |
1153 | | static Address |
1154 | | EnterStructPointerForCoercedAccess(Address SrcPtr, |
1155 | | llvm::StructType *SrcSTy, |
1156 | 18.9k | uint64_t DstSize, CodeGenFunction &CGF) { |
1157 | | // We can't dive into a zero-element struct. |
1158 | 18.9k | if (SrcSTy->getNumElements() == 0) return SrcPtr0 ; |
1159 | | |
1160 | 18.9k | llvm::Type *FirstElt = SrcSTy->getElementType(0); |
1161 | | |
1162 | | // If the first elt is at least as large as what we're looking for, or if the |
1163 | | // first element is the same size as the whole struct, we can enter it. The |
1164 | | // comparison must be made on the store size and not the alloca size. Using |
1165 | | // the alloca size may overstate the size of the load. |
1166 | 18.9k | uint64_t FirstEltSize = |
1167 | 18.9k | CGF.CGM.getDataLayout().getTypeStoreSize(FirstElt); |
1168 | 18.9k | if (FirstEltSize < DstSize && |
1169 | 18.9k | FirstEltSize < CGF.CGM.getDataLayout().getTypeStoreSize(SrcSTy)5.68k ) |
1170 | 5.54k | return SrcPtr; |
1171 | | |
1172 | | // GEP into the first element. |
1173 | 13.4k | SrcPtr = CGF.Builder.CreateStructGEP(SrcPtr, 0, "coerce.dive"); |
1174 | | |
1175 | | // If the first element is a struct, recurse. |
1176 | 13.4k | llvm::Type *SrcTy = SrcPtr.getElementType(); |
1177 | 13.4k | if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy)) |
1178 | 3.16k | return EnterStructPointerForCoercedAccess(SrcPtr, SrcSTy, DstSize, CGF); |
1179 | | |
1180 | 10.2k | return SrcPtr; |
1181 | 13.4k | } |
1182 | | |
1183 | | /// CoerceIntOrPtrToIntOrPtr - Convert a value Val to the specific Ty where both |
1184 | | /// are either integers or pointers. This does a truncation of the value if it |
1185 | | /// is too large or a zero extension if it is too small. |
1186 | | /// |
1187 | | /// This behaves as if the value were coerced through memory, so on big-endian |
1188 | | /// targets the high bits are preserved in a truncation, while little-endian |
1189 | | /// targets preserve the low bits. |
1190 | | static llvm::Value *CoerceIntOrPtrToIntOrPtr(llvm::Value *Val, |
1191 | | llvm::Type *Ty, |
1192 | 8.98k | CodeGenFunction &CGF) { |
1193 | 8.98k | if (Val->getType() == Ty) |
1194 | 8.57k | return Val; |
1195 | | |
1196 | 418 | if (isa<llvm::PointerType>(Val->getType())) { |
1197 | | // If this is Pointer->Pointer avoid conversion to and from int. |
1198 | 302 | if (isa<llvm::PointerType>(Ty)) |
1199 | 91 | return CGF.Builder.CreateBitCast(Val, Ty, "coerce.val"); |
1200 | | |
1201 | | // Convert the pointer to an integer so we can play with its width. |
1202 | 211 | Val = CGF.Builder.CreatePtrToInt(Val, CGF.IntPtrTy, "coerce.val.pi"); |
1203 | 211 | } |
1204 | | |
1205 | 327 | llvm::Type *DestIntTy = Ty; |
1206 | 327 | if (isa<llvm::PointerType>(DestIntTy)) |
1207 | 31 | DestIntTy = CGF.IntPtrTy; |
1208 | | |
1209 | 327 | if (Val->getType() != DestIntTy) { |
1210 | 85 | const llvm::DataLayout &DL = CGF.CGM.getDataLayout(); |
1211 | 85 | if (DL.isBigEndian()) { |
1212 | | // Preserve the high bits on big-endian targets. |
1213 | | // That is what memory coercion does. |
1214 | 30 | uint64_t SrcSize = DL.getTypeSizeInBits(Val->getType()); |
1215 | 30 | uint64_t DstSize = DL.getTypeSizeInBits(DestIntTy); |
1216 | | |
1217 | 30 | if (SrcSize > DstSize) { |
1218 | 11 | Val = CGF.Builder.CreateLShr(Val, SrcSize - DstSize, "coerce.highbits"); |
1219 | 11 | Val = CGF.Builder.CreateTrunc(Val, DestIntTy, "coerce.val.ii"); |
1220 | 19 | } else { |
1221 | 19 | Val = CGF.Builder.CreateZExt(Val, DestIntTy, "coerce.val.ii"); |
1222 | 19 | Val = CGF.Builder.CreateShl(Val, DstSize - SrcSize, "coerce.highbits"); |
1223 | 19 | } |
1224 | 55 | } else { |
1225 | | // Little-endian targets preserve the low bits. No shifts required. |
1226 | 55 | Val = CGF.Builder.CreateIntCast(Val, DestIntTy, false, "coerce.val.ii"); |
1227 | 55 | } |
1228 | 85 | } |
1229 | | |
1230 | 327 | if (isa<llvm::PointerType>(Ty)) |
1231 | 31 | Val = CGF.Builder.CreateIntToPtr(Val, Ty, "coerce.val.ip"); |
1232 | 327 | return Val; |
1233 | 418 | } |
1234 | | |
1235 | | |
1236 | | |
1237 | | /// CreateCoercedLoad - Create a load from \arg SrcPtr interpreted as |
1238 | | /// a pointer to an object of type \arg Ty, known to be aligned to |
1239 | | /// \arg SrcAlign bytes. |
1240 | | /// |
1241 | | /// This safely handles the case when the src type is smaller than the |
1242 | | /// destination type; in this situation the values of bits which not |
1243 | | /// present in the src are undefined. |
1244 | | static llvm::Value *CreateCoercedLoad(Address Src, llvm::Type *Ty, |
1245 | 9.18k | CodeGenFunction &CGF) { |
1246 | 9.18k | llvm::Type *SrcTy = Src.getElementType(); |
1247 | | |
1248 | | // If SrcTy and Ty are the same, just do a load. |
1249 | 9.18k | if (SrcTy == Ty) |
1250 | 31 | return CGF.Builder.CreateLoad(Src); |
1251 | | |
1252 | 9.15k | llvm::TypeSize DstSize = CGF.CGM.getDataLayout().getTypeAllocSize(Ty); |
1253 | | |
1254 | 9.15k | if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy)) { |
1255 | 7.01k | Src = EnterStructPointerForCoercedAccess(Src, SrcSTy, |
1256 | 7.01k | DstSize.getFixedSize(), CGF); |
1257 | 7.01k | SrcTy = Src.getElementType(); |
1258 | 7.01k | } |
1259 | | |
1260 | 9.15k | llvm::TypeSize SrcSize = CGF.CGM.getDataLayout().getTypeAllocSize(SrcTy); |
1261 | | |
1262 | | // If the source and destination are integer or pointer types, just do an |
1263 | | // extension or truncation to the desired type. |
1264 | 9.15k | if ((isa<llvm::IntegerType>(Ty) || isa<llvm::PointerType>(Ty)6.21k ) && |
1265 | 9.15k | (5.89k isa<llvm::IntegerType>(SrcTy)5.89k || isa<llvm::PointerType>(SrcTy)4.67k )) { |
1266 | 4.38k | llvm::Value *Load = CGF.Builder.CreateLoad(Src); |
1267 | 4.38k | return CoerceIntOrPtrToIntOrPtr(Load, Ty, CGF); |
1268 | 4.38k | } |
1269 | | |
1270 | | // If load is legal, just bitcast the src pointer. |
1271 | 4.76k | if (!SrcSize.isScalable() && !DstSize.isScalable() && |
1272 | 4.76k | SrcSize.getFixedSize() >= DstSize.getFixedSize()4.74k ) { |
1273 | | // Generally SrcSize is never greater than DstSize, since this means we are |
1274 | | // losing bits. However, this can happen in cases where the structure has |
1275 | | // additional padding, for example due to a user specified alignment. |
1276 | | // |
1277 | | // FIXME: Assert that we aren't truncating non-padding bits when have access |
1278 | | // to that information. |
1279 | 4.59k | Src = CGF.Builder.CreateElementBitCast(Src, Ty); |
1280 | 4.59k | return CGF.Builder.CreateLoad(Src); |
1281 | 4.59k | } |
1282 | | |
1283 | | // If coercing a fixed vector to a scalable vector for ABI compatibility, and |
1284 | | // the types match, use the llvm.experimental.vector.insert intrinsic to |
1285 | | // perform the conversion. |
1286 | 172 | if (auto *ScalableDst = dyn_cast<llvm::ScalableVectorType>(Ty)) { |
1287 | 26 | if (auto *FixedSrc = dyn_cast<llvm::FixedVectorType>(SrcTy)) { |
1288 | | // If we are casting a fixed i8 vector to a scalable 16 x i1 predicate |
1289 | | // vector, use a vector insert and bitcast the result. |
1290 | 26 | bool NeedsBitcast = false; |
1291 | 26 | auto PredType = |
1292 | 26 | llvm::ScalableVectorType::get(CGF.Builder.getInt1Ty(), 16); |
1293 | 26 | llvm::Type *OrigType = Ty; |
1294 | 26 | if (ScalableDst == PredType && |
1295 | 26 | FixedSrc->getElementType() == CGF.Builder.getInt8Ty()5 ) { |
1296 | 5 | ScalableDst = llvm::ScalableVectorType::get(CGF.Builder.getInt8Ty(), 2); |
1297 | 5 | NeedsBitcast = true; |
1298 | 5 | } |
1299 | 26 | if (ScalableDst->getElementType() == FixedSrc->getElementType()) { |
1300 | 26 | auto *Load = CGF.Builder.CreateLoad(Src); |
1301 | 26 | auto *UndefVec = llvm::UndefValue::get(ScalableDst); |
1302 | 26 | auto *Zero = llvm::Constant::getNullValue(CGF.CGM.Int64Ty); |
1303 | 26 | llvm::Value *Result = CGF.Builder.CreateInsertVector( |
1304 | 26 | ScalableDst, UndefVec, Load, Zero, "castScalableSve"); |
1305 | 26 | if (NeedsBitcast) |
1306 | 5 | Result = CGF.Builder.CreateBitCast(Result, OrigType); |
1307 | 26 | return Result; |
1308 | 26 | } |
1309 | 26 | } |
1310 | 26 | } |
1311 | | |
1312 | | // Otherwise do coercion through memory. This is stupid, but simple. |
1313 | 146 | Address Tmp = |
1314 | 146 | CreateTempAllocaForCoercion(CGF, Ty, Src.getAlignment(), Src.getName()); |
1315 | 146 | CGF.Builder.CreateMemCpy( |
1316 | 146 | Tmp.getPointer(), Tmp.getAlignment().getAsAlign(), Src.getPointer(), |
1317 | 146 | Src.getAlignment().getAsAlign(), |
1318 | 146 | llvm::ConstantInt::get(CGF.IntPtrTy, SrcSize.getKnownMinSize())); |
1319 | 146 | return CGF.Builder.CreateLoad(Tmp); |
1320 | 172 | } |
1321 | | |
1322 | | // Function to store a first-class aggregate into memory. We prefer to |
1323 | | // store the elements rather than the aggregate to be more friendly to |
1324 | | // fast-isel. |
1325 | | // FIXME: Do we need to recurse here? |
1326 | | void CodeGenFunction::EmitAggregateStore(llvm::Value *Val, Address Dest, |
1327 | 5.95k | bool DestIsVolatile) { |
1328 | | // Prefer scalar stores to first-class aggregate stores. |
1329 | 5.95k | if (llvm::StructType *STy = dyn_cast<llvm::StructType>(Val->getType())) { |
1330 | 5.08k | for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i3.37k ) { |
1331 | 3.37k | Address EltPtr = Builder.CreateStructGEP(Dest, i); |
1332 | 3.37k | llvm::Value *Elt = Builder.CreateExtractValue(Val, i); |
1333 | 3.37k | Builder.CreateStore(Elt, EltPtr, DestIsVolatile); |
1334 | 3.37k | } |
1335 | 4.23k | } else { |
1336 | 4.23k | Builder.CreateStore(Val, Dest, DestIsVolatile); |
1337 | 4.23k | } |
1338 | 5.95k | } |
1339 | | |
1340 | | /// CreateCoercedStore - Create a store to \arg DstPtr from \arg Src, |
1341 | | /// where the source and destination may have different types. The |
1342 | | /// destination is known to be aligned to \arg DstAlign bytes. |
1343 | | /// |
1344 | | /// This safely handles the case when the src type is larger than the |
1345 | | /// destination type; the upper bits of the src will be lost. |
1346 | | static void CreateCoercedStore(llvm::Value *Src, |
1347 | | Address Dst, |
1348 | | bool DstIsVolatile, |
1349 | 10.6k | CodeGenFunction &CGF) { |
1350 | 10.6k | llvm::Type *SrcTy = Src->getType(); |
1351 | 10.6k | llvm::Type *DstTy = Dst.getElementType(); |
1352 | 10.6k | if (SrcTy == DstTy) { |
1353 | 144 | CGF.Builder.CreateStore(Src, Dst, DstIsVolatile); |
1354 | 144 | return; |
1355 | 144 | } |
1356 | | |
1357 | 10.5k | llvm::TypeSize SrcSize = CGF.CGM.getDataLayout().getTypeAllocSize(SrcTy); |
1358 | | |
1359 | 10.5k | if (llvm::StructType *DstSTy = dyn_cast<llvm::StructType>(DstTy)) { |
1360 | 8.81k | Dst = EnterStructPointerForCoercedAccess(Dst, DstSTy, |
1361 | 8.81k | SrcSize.getFixedSize(), CGF); |
1362 | 8.81k | DstTy = Dst.getElementType(); |
1363 | 8.81k | } |
1364 | | |
1365 | 10.5k | llvm::PointerType *SrcPtrTy = llvm::dyn_cast<llvm::PointerType>(SrcTy); |
1366 | 10.5k | llvm::PointerType *DstPtrTy = llvm::dyn_cast<llvm::PointerType>(DstTy); |
1367 | 10.5k | if (SrcPtrTy && DstPtrTy3.12k && |
1368 | 10.5k | SrcPtrTy->getAddressSpace() != DstPtrTy->getAddressSpace()3.12k ) { |
1369 | 22 | Src = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Src, DstTy); |
1370 | 22 | CGF.Builder.CreateStore(Src, Dst, DstIsVolatile); |
1371 | 22 | return; |
1372 | 22 | } |
1373 | | |
1374 | | // If the source and destination are integer or pointer types, just do an |
1375 | | // extension or truncation to the desired type. |
1376 | 10.5k | if ((isa<llvm::IntegerType>(SrcTy) || isa<llvm::PointerType>(SrcTy)7.37k ) && |
1377 | 10.5k | (6.25k isa<llvm::IntegerType>(DstTy)6.25k || isa<llvm::PointerType>(DstTy)4.78k )) { |
1378 | 4.60k | Src = CoerceIntOrPtrToIntOrPtr(Src, DstTy, CGF); |
1379 | 4.60k | CGF.Builder.CreateStore(Src, Dst, DstIsVolatile); |
1380 | 4.60k | return; |
1381 | 4.60k | } |
1382 | | |
1383 | 5.92k | llvm::TypeSize DstSize = CGF.CGM.getDataLayout().getTypeAllocSize(DstTy); |
1384 | | |
1385 | | // If store is legal, just bitcast the src pointer. |
1386 | 5.92k | if (isa<llvm::ScalableVectorType>(SrcTy) || |
1387 | 5.92k | isa<llvm::ScalableVectorType>(DstTy)5.92k || |
1388 | 5.92k | SrcSize.getFixedSize() <= DstSize.getFixedSize()5.92k ) { |
1389 | 5.82k | Dst = CGF.Builder.CreateElementBitCast(Dst, SrcTy); |
1390 | 5.82k | CGF.EmitAggregateStore(Src, Dst, DstIsVolatile); |
1391 | 5.82k | } else { |
1392 | | // Otherwise do coercion through memory. This is stupid, but |
1393 | | // simple. |
1394 | | |
1395 | | // Generally SrcSize is never greater than DstSize, since this means we are |
1396 | | // losing bits. However, this can happen in cases where the structure has |
1397 | | // additional padding, for example due to a user specified alignment. |
1398 | | // |
1399 | | // FIXME: Assert that we aren't truncating non-padding bits when have access |
1400 | | // to that information. |
1401 | 101 | Address Tmp = CreateTempAllocaForCoercion(CGF, SrcTy, Dst.getAlignment()); |
1402 | 101 | CGF.Builder.CreateStore(Src, Tmp); |
1403 | 101 | CGF.Builder.CreateMemCpy( |
1404 | 101 | Dst.getPointer(), Dst.getAlignment().getAsAlign(), Tmp.getPointer(), |
1405 | 101 | Tmp.getAlignment().getAsAlign(), |
1406 | 101 | llvm::ConstantInt::get(CGF.IntPtrTy, DstSize.getFixedSize())); |
1407 | 101 | } |
1408 | 5.92k | } |
1409 | | |
1410 | | static Address emitAddressAtOffset(CodeGenFunction &CGF, Address addr, |
1411 | 22.1k | const ABIArgInfo &info) { |
1412 | 22.1k | if (unsigned offset = info.getDirectOffset()) { |
1413 | 3 | addr = CGF.Builder.CreateElementBitCast(addr, CGF.Int8Ty); |
1414 | 3 | addr = CGF.Builder.CreateConstInBoundsByteGEP(addr, |
1415 | 3 | CharUnits::fromQuantity(offset)); |
1416 | 3 | addr = CGF.Builder.CreateElementBitCast(addr, info.getCoerceToType()); |
1417 | 3 | } |
1418 | 22.1k | return addr; |
1419 | 22.1k | } |
1420 | | |
1421 | | namespace { |
1422 | | |
1423 | | /// Encapsulates information about the way function arguments from |
1424 | | /// CGFunctionInfo should be passed to actual LLVM IR function. |
1425 | | class ClangToLLVMArgMapping { |
1426 | | static const unsigned InvalidIndex = ~0U; |
1427 | | unsigned InallocaArgNo; |
1428 | | unsigned SRetArgNo; |
1429 | | unsigned TotalIRArgs; |
1430 | | |
1431 | | /// Arguments of LLVM IR function corresponding to single Clang argument. |
1432 | | struct IRArgs { |
1433 | | unsigned PaddingArgIndex; |
1434 | | // Argument is expanded to IR arguments at positions |
1435 | | // [FirstArgIndex, FirstArgIndex + NumberOfArgs). |
1436 | | unsigned FirstArgIndex; |
1437 | | unsigned NumberOfArgs; |
1438 | | |
1439 | | IRArgs() |
1440 | | : PaddingArgIndex(InvalidIndex), FirstArgIndex(InvalidIndex), |
1441 | 2.48M | NumberOfArgs(0) {} |
1442 | | }; |
1443 | | |
1444 | | SmallVector<IRArgs, 8> ArgInfo; |
1445 | | |
1446 | | public: |
1447 | | ClangToLLVMArgMapping(const ASTContext &Context, const CGFunctionInfo &FI, |
1448 | | bool OnlyRequiredArgs = false) |
1449 | | : InallocaArgNo(InvalidIndex), SRetArgNo(InvalidIndex), TotalIRArgs(0), |
1450 | 2.48M | ArgInfo(OnlyRequiredArgs ? FI.getNumRequiredArgs() : FI.arg_size()) { |
1451 | 2.48M | construct(Context, FI, OnlyRequiredArgs); |
1452 | 2.48M | } |
1453 | | |
1454 | 3.00M | bool hasInallocaArg() const { return InallocaArgNo != InvalidIndex; } |
1455 | 580 | unsigned getInallocaArgNo() const { |
1456 | 580 | assert(hasInallocaArg()); |
1457 | 0 | return InallocaArgNo; |
1458 | 580 | } |
1459 | | |
1460 | 2.14M | bool hasSRetArg() const { return SRetArgNo != InvalidIndex; } |
1461 | 22.2k | unsigned getSRetArgNo() const { |
1462 | 22.2k | assert(hasSRetArg()); |
1463 | 0 | return SRetArgNo; |
1464 | 22.2k | } |
1465 | | |
1466 | 2.48M | unsigned totalIRArgs() const { return TotalIRArgs; } |
1467 | | |
1468 | 3.49M | bool hasPaddingArg(unsigned ArgNo) const { |
1469 | 3.49M | assert(ArgNo < ArgInfo.size()); |
1470 | 0 | return ArgInfo[ArgNo].PaddingArgIndex != InvalidIndex; |
1471 | 3.49M | } |
1472 | 69 | unsigned getPaddingArgNo(unsigned ArgNo) const { |
1473 | 69 | assert(hasPaddingArg(ArgNo)); |
1474 | 0 | return ArgInfo[ArgNo].PaddingArgIndex; |
1475 | 69 | } |
1476 | | |
1477 | | /// Returns index of first IR argument corresponding to ArgNo, and their |
1478 | | /// quantity. |
1479 | 3.97M | std::pair<unsigned, unsigned> getIRArgs(unsigned ArgNo) const { |
1480 | 3.97M | assert(ArgNo < ArgInfo.size()); |
1481 | 0 | return std::make_pair(ArgInfo[ArgNo].FirstArgIndex, |
1482 | 3.97M | ArgInfo[ArgNo].NumberOfArgs); |
1483 | 3.97M | } |
1484 | | |
1485 | | private: |
1486 | | void construct(const ASTContext &Context, const CGFunctionInfo &FI, |
1487 | | bool OnlyRequiredArgs); |
1488 | | }; |
1489 | | |
1490 | | void ClangToLLVMArgMapping::construct(const ASTContext &Context, |
1491 | | const CGFunctionInfo &FI, |
1492 | 2.48M | bool OnlyRequiredArgs) { |
1493 | 2.48M | unsigned IRArgNo = 0; |
1494 | 2.48M | bool SwapThisWithSRet = false; |
1495 | 2.48M | const ABIArgInfo &RetAI = FI.getReturnInfo(); |
1496 | | |
1497 | 2.48M | if (RetAI.getKind() == ABIArgInfo::Indirect) { |
1498 | 22.2k | SwapThisWithSRet = RetAI.isSRetAfterThis(); |
1499 | 22.2k | SRetArgNo = SwapThisWithSRet ? 1455 : IRArgNo++21.8k ; |
1500 | 22.2k | } |
1501 | | |
1502 | 2.48M | unsigned ArgNo = 0; |
1503 | 2.48M | unsigned NumArgs = OnlyRequiredArgs ? FI.getNumRequiredArgs()1.08M : FI.arg_size()1.39M ; |
1504 | 6.50M | for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(); ArgNo < NumArgs; |
1505 | 4.02M | ++I, ++ArgNo) { |
1506 | 4.02M | assert(I != FI.arg_end()); |
1507 | 0 | QualType ArgType = I->type; |
1508 | 4.02M | const ABIArgInfo &AI = I->info; |
1509 | | // Collect data about IR arguments corresponding to Clang argument ArgNo. |
1510 | 4.02M | auto &IRArgs = ArgInfo[ArgNo]; |
1511 | | |
1512 | 4.02M | if (AI.getPaddingType()) |
1513 | 124 | IRArgs.PaddingArgIndex = IRArgNo++; |
1514 | | |
1515 | 4.02M | switch (AI.getKind()) { |
1516 | 94.4k | case ABIArgInfo::Extend: |
1517 | 3.94M | case ABIArgInfo::Direct: { |
1518 | | // FIXME: handle sseregparm someday... |
1519 | 3.94M | llvm::StructType *STy = dyn_cast<llvm::StructType>(AI.getCoerceToType()); |
1520 | 3.94M | if (AI.isDirect() && AI.getCanBeFlattened()3.85M && STy3.84M ) { |
1521 | 8.64k | IRArgs.NumberOfArgs = STy->getNumElements(); |
1522 | 3.93M | } else { |
1523 | 3.93M | IRArgs.NumberOfArgs = 1; |
1524 | 3.93M | } |
1525 | 3.94M | break; |
1526 | 94.4k | } |
1527 | 17.5k | case ABIArgInfo::Indirect: |
1528 | 17.6k | case ABIArgInfo::IndirectAliased: |
1529 | 17.6k | IRArgs.NumberOfArgs = 1; |
1530 | 17.6k | break; |
1531 | 54.7k | case ABIArgInfo::Ignore: |
1532 | 55.6k | case ABIArgInfo::InAlloca: |
1533 | | // ignore and inalloca doesn't have matching LLVM parameters. |
1534 | 55.6k | IRArgs.NumberOfArgs = 0; |
1535 | 55.6k | break; |
1536 | 3.47k | case ABIArgInfo::CoerceAndExpand: |
1537 | 3.47k | IRArgs.NumberOfArgs = AI.getCoerceAndExpandTypeSequence().size(); |
1538 | 3.47k | break; |
1539 | 474 | case ABIArgInfo::Expand: |
1540 | 474 | IRArgs.NumberOfArgs = getExpansionSize(ArgType, Context); |
1541 | 474 | break; |
1542 | 4.02M | } |
1543 | | |
1544 | 4.02M | if (IRArgs.NumberOfArgs > 0) { |
1545 | 3.96M | IRArgs.FirstArgIndex = IRArgNo; |
1546 | 3.96M | IRArgNo += IRArgs.NumberOfArgs; |
1547 | 3.96M | } |
1548 | | |
1549 | | // Skip over the sret parameter when it comes second. We already handled it |
1550 | | // above. |
1551 | 4.02M | if (IRArgNo == 1 && SwapThisWithSRet2.29M ) |
1552 | 455 | IRArgNo++; |
1553 | 4.02M | } |
1554 | 2.48M | assert(ArgNo == ArgInfo.size()); |
1555 | | |
1556 | 2.48M | if (FI.usesInAlloca()) |
1557 | 477 | InallocaArgNo = IRArgNo++; |
1558 | | |
1559 | 2.48M | TotalIRArgs = IRArgNo; |
1560 | 2.48M | } |
1561 | | } // namespace |
1562 | | |
1563 | | /***/ |
1564 | | |
1565 | 20.1k | bool CodeGenModule::ReturnTypeUsesSRet(const CGFunctionInfo &FI) { |
1566 | 20.1k | const auto &RI = FI.getReturnInfo(); |
1567 | 20.1k | return RI.isIndirect() || (20.0k RI.isInAlloca()20.0k && RI.getInAllocaSRet()1 ); |
1568 | 20.1k | } |
1569 | | |
1570 | 12.9k | bool CodeGenModule::ReturnSlotInterferesWithArgs(const CGFunctionInfo &FI) { |
1571 | 12.9k | return ReturnTypeUsesSRet(FI) && |
1572 | 12.9k | getTargetCodeGenInfo().doesReturnSlotInterfereWithArgs()44 ; |
1573 | 12.9k | } |
1574 | | |
1575 | 11.7k | bool CodeGenModule::ReturnTypeUsesFPRet(QualType ResultType) { |
1576 | 11.7k | if (const BuiltinType *BT = ResultType->getAs<BuiltinType>()) { |
1577 | 4.92k | switch (BT->getKind()) { |
1578 | 4.86k | default: |
1579 | 4.86k | return false; |
1580 | 18 | case BuiltinType::Float: |
1581 | 18 | return getTarget().useObjCFPRetForRealType(FloatModeKind::Float); |
1582 | 43 | case BuiltinType::Double: |
1583 | 43 | return getTarget().useObjCFPRetForRealType(FloatModeKind::Double); |
1584 | 4 | case BuiltinType::LongDouble: |
1585 | 4 | return getTarget().useObjCFPRetForRealType(FloatModeKind::LongDouble); |
1586 | 4.92k | } |
1587 | 4.92k | } |
1588 | | |
1589 | 6.85k | return false; |
1590 | 11.7k | } |
1591 | | |
1592 | 11.7k | bool CodeGenModule::ReturnTypeUsesFP2Ret(QualType ResultType) { |
1593 | 11.7k | if (const ComplexType *CT = ResultType->getAs<ComplexType>()) { |
1594 | 14 | if (const BuiltinType *BT = CT->getElementType()->getAs<BuiltinType>()) { |
1595 | 14 | if (BT->getKind() == BuiltinType::LongDouble) |
1596 | 2 | return getTarget().useObjCFP2RetForComplexLongDouble(); |
1597 | 14 | } |
1598 | 14 | } |
1599 | | |
1600 | 11.7k | return false; |
1601 | 11.7k | } |
1602 | | |
1603 | 5.50k | llvm::FunctionType *CodeGenTypes::GetFunctionType(GlobalDecl GD) { |
1604 | 5.50k | const CGFunctionInfo &FI = arrangeGlobalDeclaration(GD); |
1605 | 5.50k | return GetFunctionType(FI); |
1606 | 5.50k | } |
1607 | | |
1608 | | llvm::FunctionType * |
1609 | 1.08M | CodeGenTypes::GetFunctionType(const CGFunctionInfo &FI) { |
1610 | | |
1611 | 1.08M | bool Inserted = FunctionsBeingProcessed.insert(&FI).second; |
1612 | 1.08M | (void)Inserted; |
1613 | 1.08M | assert(Inserted && "Recursively being processed?"); |
1614 | | |
1615 | 0 | llvm::Type *resultType = nullptr; |
1616 | 1.08M | const ABIArgInfo &retAI = FI.getReturnInfo(); |
1617 | 1.08M | switch (retAI.getKind()) { |
1618 | 0 | case ABIArgInfo::Expand: |
1619 | 0 | case ABIArgInfo::IndirectAliased: |
1620 | 0 | llvm_unreachable("Invalid ABI kind for return argument"); |
1621 | |
|
1622 | 35.0k | case ABIArgInfo::Extend: |
1623 | 567k | case ABIArgInfo::Direct: |
1624 | 567k | resultType = retAI.getCoerceToType(); |
1625 | 567k | break; |
1626 | | |
1627 | 18 | case ABIArgInfo::InAlloca: |
1628 | 18 | if (retAI.getInAllocaSRet()) { |
1629 | | // sret things on win32 aren't void, they return the sret pointer. |
1630 | 18 | QualType ret = FI.getReturnType(); |
1631 | 18 | llvm::Type *ty = ConvertType(ret); |
1632 | 18 | unsigned addressSpace = Context.getTargetAddressSpace(ret); |
1633 | 18 | resultType = llvm::PointerType::get(ty, addressSpace); |
1634 | 18 | } else { |
1635 | 0 | resultType = llvm::Type::getVoidTy(getLLVMContext()); |
1636 | 0 | } |
1637 | 18 | break; |
1638 | | |
1639 | 9.13k | case ABIArgInfo::Indirect: |
1640 | 517k | case ABIArgInfo::Ignore: |
1641 | 517k | resultType = llvm::Type::getVoidTy(getLLVMContext()); |
1642 | 517k | break; |
1643 | | |
1644 | 1.42k | case ABIArgInfo::CoerceAndExpand: |
1645 | 1.42k | resultType = retAI.getUnpaddedCoerceAndExpandType(); |
1646 | 1.42k | break; |
1647 | 1.08M | } |
1648 | | |
1649 | 1.08M | ClangToLLVMArgMapping IRFunctionArgs(getContext(), FI, true); |
1650 | 1.08M | SmallVector<llvm::Type*, 8> ArgTypes(IRFunctionArgs.totalIRArgs()); |
1651 | | |
1652 | | // Add type for sret argument. |
1653 | 1.08M | if (IRFunctionArgs.hasSRetArg()) { |
1654 | 9.13k | QualType Ret = FI.getReturnType(); |
1655 | 9.13k | llvm::Type *Ty = ConvertType(Ret); |
1656 | 9.13k | unsigned AddressSpace = Context.getTargetAddressSpace(Ret); |
1657 | 9.13k | ArgTypes[IRFunctionArgs.getSRetArgNo()] = |
1658 | 9.13k | llvm::PointerType::get(Ty, AddressSpace); |
1659 | 9.13k | } |
1660 | | |
1661 | | // Add type for inalloca argument. |
1662 | 1.08M | if (IRFunctionArgs.hasInallocaArg()) { |
1663 | 200 | auto ArgStruct = FI.getArgStruct(); |
1664 | 200 | assert(ArgStruct); |
1665 | 0 | ArgTypes[IRFunctionArgs.getInallocaArgNo()] = ArgStruct->getPointerTo(); |
1666 | 200 | } |
1667 | | |
1668 | | // Add in all of the required arguments. |
1669 | 0 | unsigned ArgNo = 0; |
1670 | 1.08M | CGFunctionInfo::const_arg_iterator it = FI.arg_begin(), |
1671 | 1.08M | ie = it + FI.getNumRequiredArgs(); |
1672 | 2.78M | for (; it != ie; ++it, ++ArgNo1.69M ) { |
1673 | 1.69M | const ABIArgInfo &ArgInfo = it->info; |
1674 | | |
1675 | | // Insert a padding type to ensure proper alignment. |
1676 | 1.69M | if (IRFunctionArgs.hasPaddingArg(ArgNo)) |
1677 | 47 | ArgTypes[IRFunctionArgs.getPaddingArgNo(ArgNo)] = |
1678 | 47 | ArgInfo.getPaddingType(); |
1679 | | |
1680 | 1.69M | unsigned FirstIRArg, NumIRArgs; |
1681 | 1.69M | std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo); |
1682 | | |
1683 | 1.69M | switch (ArgInfo.getKind()) { |
1684 | 27.0k | case ABIArgInfo::Ignore: |
1685 | 27.3k | case ABIArgInfo::InAlloca: |
1686 | 27.3k | assert(NumIRArgs == 0); |
1687 | 0 | break; |
1688 | | |
1689 | 7.91k | case ABIArgInfo::Indirect: { |
1690 | 7.91k | assert(NumIRArgs == 1); |
1691 | | // indirect arguments are always on the stack, which is alloca addr space. |
1692 | 0 | llvm::Type *LTy = ConvertTypeForMem(it->type); |
1693 | 7.91k | ArgTypes[FirstIRArg] = LTy->getPointerTo( |
1694 | 7.91k | CGM.getDataLayout().getAllocaAddrSpace()); |
1695 | 7.91k | break; |
1696 | 27.0k | } |
1697 | 29 | case ABIArgInfo::IndirectAliased: { |
1698 | 29 | assert(NumIRArgs == 1); |
1699 | 0 | llvm::Type *LTy = ConvertTypeForMem(it->type); |
1700 | 29 | ArgTypes[FirstIRArg] = LTy->getPointerTo(ArgInfo.getIndirectAddrSpace()); |
1701 | 29 | break; |
1702 | 27.0k | } |
1703 | 39.7k | case ABIArgInfo::Extend: |
1704 | 1.65M | case ABIArgInfo::Direct: { |
1705 | | // Fast-isel and the optimizer generally like scalar values better than |
1706 | | // FCAs, so we flatten them if this is safe to do for this argument. |
1707 | 1.65M | llvm::Type *argType = ArgInfo.getCoerceToType(); |
1708 | 1.65M | llvm::StructType *st = dyn_cast<llvm::StructType>(argType); |
1709 | 1.65M | if (st && ArgInfo.isDirect()3.96k && ArgInfo.getCanBeFlattened()3.96k ) { |
1710 | 3.76k | assert(NumIRArgs == st->getNumElements()); |
1711 | 11.3k | for (unsigned i = 0, e = st->getNumElements(); i != e; ++i7.59k ) |
1712 | 7.59k | ArgTypes[FirstIRArg + i] = st->getElementType(i); |
1713 | 1.65M | } else { |
1714 | 1.65M | assert(NumIRArgs == 1); |
1715 | 0 | ArgTypes[FirstIRArg] = argType; |
1716 | 1.65M | } |
1717 | 0 | break; |
1718 | 39.7k | } |
1719 | | |
1720 | 1.45k | case ABIArgInfo::CoerceAndExpand: { |
1721 | 1.45k | auto ArgTypesIter = ArgTypes.begin() + FirstIRArg; |
1722 | 3.12k | for (auto EltTy : ArgInfo.getCoerceAndExpandTypeSequence()) { |
1723 | 3.12k | *ArgTypesIter++ = EltTy; |
1724 | 3.12k | } |
1725 | 1.45k | assert(ArgTypesIter == ArgTypes.begin() + FirstIRArg + NumIRArgs); |
1726 | 0 | break; |
1727 | 39.7k | } |
1728 | | |
1729 | 181 | case ABIArgInfo::Expand: |
1730 | 181 | auto ArgTypesIter = ArgTypes.begin() + FirstIRArg; |
1731 | 181 | getExpandedTypes(it->type, ArgTypesIter); |
1732 | 181 | assert(ArgTypesIter == ArgTypes.begin() + FirstIRArg + NumIRArgs); |
1733 | 0 | break; |
1734 | 1.69M | } |
1735 | 1.69M | } |
1736 | | |
1737 | 1.08M | bool Erased = FunctionsBeingProcessed.erase(&FI); (void)Erased; |
1738 | 1.08M | assert(Erased && "Not in set?"); |
1739 | | |
1740 | 0 | return llvm::FunctionType::get(resultType, ArgTypes, FI.isVariadic()); |
1741 | 1.08M | } |
1742 | | |
1743 | 5.49k | llvm::Type *CodeGenTypes::GetFunctionTypeForVTable(GlobalDecl GD) { |
1744 | 5.49k | const CXXMethodDecl *MD = cast<CXXMethodDecl>(GD.getDecl()); |
1745 | 5.49k | const FunctionProtoType *FPT = MD->getType()->getAs<FunctionProtoType>(); |
1746 | | |
1747 | 5.49k | if (!isFuncTypeConvertible(FPT)) |
1748 | 39 | return llvm::StructType::get(getLLVMContext()); |
1749 | | |
1750 | 5.45k | return GetFunctionType(GD); |
1751 | 5.49k | } |
1752 | | |
1753 | | static void AddAttributesFromFunctionProtoType(ASTContext &Ctx, |
1754 | | llvm::AttrBuilder &FuncAttrs, |
1755 | 1.36M | const FunctionProtoType *FPT) { |
1756 | 1.36M | if (!FPT) |
1757 | 711k | return; |
1758 | | |
1759 | 657k | if (!isUnresolvedExceptionSpec(FPT->getExceptionSpecType()) && |
1760 | 657k | FPT->isNothrow()) |
1761 | 192k | FuncAttrs.addAttribute(llvm::Attribute::NoUnwind); |
1762 | 657k | } |
1763 | | |
1764 | | static void AddAttributesFromAssumes(llvm::AttrBuilder &FuncAttrs, |
1765 | 717k | const Decl *Callee) { |
1766 | 717k | if (!Callee) |
1767 | 31.8k | return; |
1768 | | |
1769 | 685k | SmallVector<StringRef, 4> Attrs; |
1770 | | |
1771 | 685k | for (const AssumptionAttr *AA : Callee->specific_attrs<AssumptionAttr>()) |
1772 | 481 | AA->getAssumption().split(Attrs, ","); |
1773 | | |
1774 | 685k | if (!Attrs.empty()) |
1775 | 198 | FuncAttrs.addAttribute(llvm::AssumptionAttrKey, |
1776 | 198 | llvm::join(Attrs.begin(), Attrs.end(), ",")); |
1777 | 685k | } |
1778 | | |
1779 | | bool CodeGenModule::MayDropFunctionReturn(const ASTContext &Context, |
1780 | 48 | QualType ReturnType) { |
1781 | | // We can't just discard the return value for a record type with a |
1782 | | // complex destructor or a non-trivially copyable type. |
1783 | 48 | if (const RecordType *RT = |
1784 | 48 | ReturnType.getCanonicalType()->getAs<RecordType>()) { |
1785 | 24 | if (const auto *ClassDecl = dyn_cast<CXXRecordDecl>(RT->getDecl())) |
1786 | 24 | return ClassDecl->hasTrivialDestructor(); |
1787 | 24 | } |
1788 | 24 | return ReturnType.isTriviallyCopyableType(Context); |
1789 | 48 | } |
1790 | | |
1791 | | void CodeGenModule::getDefaultFunctionAttributes(StringRef Name, |
1792 | | bool HasOptnone, |
1793 | | bool AttrOnCallSite, |
1794 | 717k | llvm::AttrBuilder &FuncAttrs) { |
1795 | | // OptimizeNoneAttr takes precedence over -Os or -Oz. No warning needed. |
1796 | 717k | if (!HasOptnone) { |
1797 | 717k | if (CodeGenOpts.OptimizeSize) |
1798 | 1.07k | FuncAttrs.addAttribute(llvm::Attribute::OptimizeForSize); |
1799 | 717k | if (CodeGenOpts.OptimizeSize == 2) |
1800 | 717 | FuncAttrs.addAttribute(llvm::Attribute::MinSize); |
1801 | 717k | } |
1802 | | |
1803 | 717k | if (CodeGenOpts.DisableRedZone) |
1804 | 25 | FuncAttrs.addAttribute(llvm::Attribute::NoRedZone); |
1805 | 717k | if (CodeGenOpts.IndirectTlsSegRefs) |
1806 | 1 | FuncAttrs.addAttribute("indirect-tls-seg-refs"); |
1807 | 717k | if (CodeGenOpts.NoImplicitFloat) |
1808 | 16 | FuncAttrs.addAttribute(llvm::Attribute::NoImplicitFloat); |
1809 | | |
1810 | 717k | if (AttrOnCallSite) { |
1811 | | // Attributes that should go on the call site only. |
1812 | | // FIXME: Look for 'BuiltinAttr' on the function rather than re-checking |
1813 | | // the -fno-builtin-foo list. |
1814 | 366k | if (!CodeGenOpts.SimplifyLibCalls || LangOpts.isNoBuiltinFunc(Name)174k ) |
1815 | 192k | FuncAttrs.addAttribute(llvm::Attribute::NoBuiltin); |
1816 | 366k | if (!CodeGenOpts.TrapFuncName.empty()) |
1817 | 0 | FuncAttrs.addAttribute("trap-func-name", CodeGenOpts.TrapFuncName); |
1818 | 366k | } else { |
1819 | 350k | StringRef FpKind; |
1820 | 350k | switch (CodeGenOpts.getFramePointer()) { |
1821 | 214k | case CodeGenOptions::FramePointerKind::None: |
1822 | 214k | FpKind = "none"; |
1823 | 214k | break; |
1824 | 165 | case CodeGenOptions::FramePointerKind::NonLeaf: |
1825 | 165 | FpKind = "non-leaf"; |
1826 | 165 | break; |
1827 | 136k | case CodeGenOptions::FramePointerKind::All: |
1828 | 136k | FpKind = "all"; |
1829 | 136k | break; |
1830 | 350k | } |
1831 | 350k | FuncAttrs.addAttribute("frame-pointer", FpKind); |
1832 | | |
1833 | 350k | if (CodeGenOpts.LessPreciseFPMAD) |
1834 | 6 | FuncAttrs.addAttribute("less-precise-fpmad", "true"); |
1835 | | |
1836 | 350k | if (CodeGenOpts.NullPointerIsValid) |
1837 | 238 | FuncAttrs.addAttribute(llvm::Attribute::NullPointerIsValid); |
1838 | | |
1839 | 350k | if (CodeGenOpts.FPDenormalMode != llvm::DenormalMode::getIEEE()) |
1840 | 10 | FuncAttrs.addAttribute("denormal-fp-math", |
1841 | 10 | CodeGenOpts.FPDenormalMode.str()); |
1842 | 350k | if (CodeGenOpts.FP32DenormalMode != CodeGenOpts.FPDenormalMode) { |
1843 | 16 | FuncAttrs.addAttribute( |
1844 | 16 | "denormal-fp-math-f32", |
1845 | 16 | CodeGenOpts.FP32DenormalMode.str()); |
1846 | 16 | } |
1847 | | |
1848 | 350k | if (LangOpts.getFPExceptionMode() == LangOptions::FPE_Ignore) |
1849 | 349k | FuncAttrs.addAttribute("no-trapping-math", "true"); |
1850 | | |
1851 | | // TODO: Are these all needed? |
1852 | | // unsafe/inf/nan/nsz are handled by instruction-level FastMathFlags. |
1853 | 350k | if (LangOpts.NoHonorInfs) |
1854 | 265 | FuncAttrs.addAttribute("no-infs-fp-math", "true"); |
1855 | 350k | if (LangOpts.NoHonorNaNs) |
1856 | 265 | FuncAttrs.addAttribute("no-nans-fp-math", "true"); |
1857 | 350k | if (LangOpts.ApproxFunc) |
1858 | 253 | FuncAttrs.addAttribute("approx-func-fp-math", "true"); |
1859 | 350k | if (LangOpts.UnsafeFPMath) |
1860 | 252 | FuncAttrs.addAttribute("unsafe-fp-math", "true"); |
1861 | 350k | if (CodeGenOpts.SoftFloat) |
1862 | 61 | FuncAttrs.addAttribute("use-soft-float", "true"); |
1863 | 350k | FuncAttrs.addAttribute("stack-protector-buffer-size", |
1864 | 350k | llvm::utostr(CodeGenOpts.SSPBufferSize)); |
1865 | 350k | if (LangOpts.NoSignedZero) |
1866 | 258 | FuncAttrs.addAttribute("no-signed-zeros-fp-math", "true"); |
1867 | | |
1868 | | // TODO: Reciprocal estimate codegen options should apply to instructions? |
1869 | 350k | const std::vector<std::string> &Recips = CodeGenOpts.Reciprocals; |
1870 | 350k | if (!Recips.empty()) |
1871 | 1 | FuncAttrs.addAttribute("reciprocal-estimates", |
1872 | 1 | llvm::join(Recips, ",")); |
1873 | | |
1874 | 350k | if (!CodeGenOpts.PreferVectorWidth.empty() && |
1875 | 350k | CodeGenOpts.PreferVectorWidth != "none"3 ) |
1876 | 2 | FuncAttrs.addAttribute("prefer-vector-width", |
1877 | 2 | CodeGenOpts.PreferVectorWidth); |
1878 | | |
1879 | 350k | if (CodeGenOpts.StackRealignment) |
1880 | 1 | FuncAttrs.addAttribute("stackrealign"); |
1881 | 350k | if (CodeGenOpts.Backchain) |
1882 | 2 | FuncAttrs.addAttribute("backchain"); |
1883 | 350k | if (CodeGenOpts.EnableSegmentedStacks) |
1884 | 6 | FuncAttrs.addAttribute("split-stack"); |
1885 | | |
1886 | 350k | if (CodeGenOpts.SpeculativeLoadHardening) |
1887 | 9 | FuncAttrs.addAttribute(llvm::Attribute::SpeculativeLoadHardening); |
1888 | | |
1889 | | // Add zero-call-used-regs attribute. |
1890 | 350k | switch (CodeGenOpts.getZeroCallUsedRegs()) { |
1891 | 350k | case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::Skip: |
1892 | 350k | FuncAttrs.removeAttribute("zero-call-used-regs"); |
1893 | 350k | break; |
1894 | 10 | case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedGPRArg: |
1895 | 10 | FuncAttrs.addAttribute("zero-call-used-regs", "used-gpr-arg"); |
1896 | 10 | break; |
1897 | 10 | case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedGPR: |
1898 | 10 | FuncAttrs.addAttribute("zero-call-used-regs", "used-gpr"); |
1899 | 10 | break; |
1900 | 10 | case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::UsedArg: |
1901 | 10 | FuncAttrs.addAttribute("zero-call-used-regs", "used-arg"); |
1902 | 10 | break; |
1903 | 10 | case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::Used: |
1904 | 10 | FuncAttrs.addAttribute("zero-call-used-regs", "used"); |
1905 | 10 | break; |
1906 | 10 | case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllGPRArg: |
1907 | 10 | FuncAttrs.addAttribute("zero-call-used-regs", "all-gpr-arg"); |
1908 | 10 | break; |
1909 | 10 | case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllGPR: |
1910 | 10 | FuncAttrs.addAttribute("zero-call-used-regs", "all-gpr"); |
1911 | 10 | break; |
1912 | 10 | case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::AllArg: |
1913 | 10 | FuncAttrs.addAttribute("zero-call-used-regs", "all-arg"); |
1914 | 10 | break; |
1915 | 10 | case llvm::ZeroCallUsedRegs::ZeroCallUsedRegsKind::All: |
1916 | 10 | FuncAttrs.addAttribute("zero-call-used-regs", "all"); |
1917 | 10 | break; |
1918 | 350k | } |
1919 | 350k | } |
1920 | | |
1921 | 717k | if (getLangOpts().assumeFunctionsAreConvergent()) { |
1922 | | // Conservatively, mark all functions and calls in CUDA and OpenCL as |
1923 | | // convergent (meaning, they may call an intrinsically convergent op, such |
1924 | | // as __syncthreads() / barrier(), and so can't have certain optimizations |
1925 | | // applied around them). LLVM will remove this attribute where it safely |
1926 | | // can. |
1927 | 132k | FuncAttrs.addAttribute(llvm::Attribute::Convergent); |
1928 | 132k | } |
1929 | | |
1930 | 717k | if (getLangOpts().CUDA && getLangOpts().CUDAIsDevice1.28k ) { |
1931 | | // Exceptions aren't supported in CUDA device code. |
1932 | 465 | FuncAttrs.addAttribute(llvm::Attribute::NoUnwind); |
1933 | 465 | } |
1934 | | |
1935 | 717k | for (StringRef Attr : CodeGenOpts.DefaultFunctionAttrs) { |
1936 | 32 | StringRef Var, Value; |
1937 | 32 | std::tie(Var, Value) = Attr.split('='); |
1938 | 32 | FuncAttrs.addAttribute(Var, Value); |
1939 | 32 | } |
1940 | 717k | } |
1941 | | |
1942 | 3 | void CodeGenModule::addDefaultFunctionDefinitionAttributes(llvm::Function &F) { |
1943 | 3 | llvm::AttrBuilder FuncAttrs(F.getContext()); |
1944 | 3 | getDefaultFunctionAttributes(F.getName(), F.hasOptNone(), |
1945 | 3 | /* AttrOnCallSite = */ false, FuncAttrs); |
1946 | | // TODO: call GetCPUAndFeaturesAttributes? |
1947 | 3 | F.addFnAttrs(FuncAttrs); |
1948 | 3 | } |
1949 | | |
1950 | | void CodeGenModule::addDefaultFunctionDefinitionAttributes( |
1951 | 0 | llvm::AttrBuilder &attrs) { |
1952 | 0 | getDefaultFunctionAttributes(/*function name*/ "", /*optnone*/ false, |
1953 | 0 | /*for call*/ false, attrs); |
1954 | 0 | GetCPUAndFeaturesAttributes(GlobalDecl(), attrs); |
1955 | 0 | } |
1956 | | |
1957 | | static void addNoBuiltinAttributes(llvm::AttrBuilder &FuncAttrs, |
1958 | | const LangOptions &LangOpts, |
1959 | 717k | const NoBuiltinAttr *NBA = nullptr) { |
1960 | 717k | auto AddNoBuiltinAttr = [&FuncAttrs](StringRef BuiltinName) { |
1961 | 2.40k | SmallString<32> AttributeName; |
1962 | 2.40k | AttributeName += "no-builtin-"; |
1963 | 2.40k | AttributeName += BuiltinName; |
1964 | 2.40k | FuncAttrs.addAttribute(AttributeName); |
1965 | 2.40k | }; |
1966 | | |
1967 | | // First, handle the language options passed through -fno-builtin. |
1968 | 717k | if (LangOpts.NoBuiltin) { |
1969 | | // -fno-builtin disables them all. |
1970 | 444k | FuncAttrs.addAttribute("no-builtins"); |
1971 | 444k | return; |
1972 | 444k | } |
1973 | | |
1974 | | // Then, add attributes for builtins specified through -fno-builtin-<name>. |
1975 | 273k | llvm::for_each(LangOpts.NoBuiltinFuncs, AddNoBuiltinAttr); |
1976 | | |
1977 | | // Now, let's check the __attribute__((no_builtin("...")) attribute added to |
1978 | | // the source. |
1979 | 273k | if (!NBA) |
1980 | 273k | return; |
1981 | | |
1982 | | // If there is a wildcard in the builtin names specified through the |
1983 | | // attribute, disable them all. |
1984 | 11 | if (llvm::is_contained(NBA->builtinNames(), "*")) { |
1985 | 1 | FuncAttrs.addAttribute("no-builtins"); |
1986 | 1 | return; |
1987 | 1 | } |
1988 | | |
1989 | | // And last, add the rest of the builtin names. |
1990 | 10 | llvm::for_each(NBA->builtinNames(), AddNoBuiltinAttr); |
1991 | 10 | } |
1992 | | |
1993 | | static bool DetermineNoUndef(QualType QTy, CodeGenTypes &Types, |
1994 | | const llvm::DataLayout &DL, const ABIArgInfo &AI, |
1995 | 1.20M | bool CheckCoerce = true) { |
1996 | 1.20M | llvm::Type *Ty = Types.ConvertTypeForMem(QTy); |
1997 | 1.20M | if (AI.getKind() == ABIArgInfo::Indirect) |
1998 | 4.18k | return true; |
1999 | 1.20M | if (AI.getKind() == ABIArgInfo::Extend) |
2000 | 43.0k | return true; |
2001 | 1.16M | if (!DL.typeSizeEqualsStoreSize(Ty)) |
2002 | | // TODO: This will result in a modest amount of values not marked noundef |
2003 | | // when they could be. We care about values that *invisibly* contain undef |
2004 | | // bits from the perspective of LLVM IR. |
2005 | 43 | return false; |
2006 | 1.16M | if (CheckCoerce && AI.canHaveCoerceToType()1.06M ) { |
2007 | 1.04M | llvm::Type *CoerceTy = AI.getCoerceToType(); |
2008 | 1.04M | if (llvm::TypeSize::isKnownGT(DL.getTypeSizeInBits(CoerceTy), |
2009 | 1.04M | DL.getTypeSizeInBits(Ty))) |
2010 | | // If we're coercing to a type with a greater size than the canonical one, |
2011 | | // we're introducing new undef bits. |
2012 | | // Coercing to a type of smaller or equal size is ok, as we know that |
2013 | | // there's no internal padding (typeSizeEqualsStoreSize). |
2014 | 645 | return false; |
2015 | 1.04M | } |
2016 | 1.16M | if (QTy->isBitIntType()) |
2017 | 3 | return true; |
2018 | 1.16M | if (QTy->isReferenceType()) |
2019 | 184k | return true; |
2020 | 976k | if (QTy->isNullPtrType()) |
2021 | 198 | return false; |
2022 | 976k | if (QTy->isMemberPointerType()) |
2023 | | // TODO: Some member pointers are `noundef`, but it depends on the ABI. For |
2024 | | // now, never mark them. |
2025 | 279 | return false; |
2026 | 975k | if (QTy->isScalarType()) { |
2027 | 776k | if (const ComplexType *Complex = dyn_cast<ComplexType>(QTy)) |
2028 | 1.09k | return DetermineNoUndef(Complex->getElementType(), Types, DL, AI, false); |
2029 | 775k | return true; |
2030 | 776k | } |
2031 | 199k | if (const VectorType *Vector = dyn_cast<VectorType>(QTy)) |
2032 | 95.3k | return DetermineNoUndef(Vector->getElementType(), Types, DL, AI, false); |
2033 | 104k | if (const MatrixType *Matrix = dyn_cast<MatrixType>(QTy)) |
2034 | 340 | return DetermineNoUndef(Matrix->getElementType(), Types, DL, AI, false); |
2035 | 103k | if (const ArrayType *Array = dyn_cast<ArrayType>(QTy)) |
2036 | 0 | return DetermineNoUndef(Array->getElementType(), Types, DL, AI, false); |
2037 | | |
2038 | | // TODO: Some structs may be `noundef`, in specific situations. |
2039 | 103k | return false; |
2040 | 103k | } |
2041 | | |
2042 | | /// Construct the IR attribute list of a function or call. |
2043 | | /// |
2044 | | /// When adding an attribute, please consider where it should be handled: |
2045 | | /// |
2046 | | /// - getDefaultFunctionAttributes is for attributes that are essentially |
2047 | | /// part of the global target configuration (but perhaps can be |
2048 | | /// overridden on a per-function basis). Adding attributes there |
2049 | | /// will cause them to also be set in frontends that build on Clang's |
2050 | | /// target-configuration logic, as well as for code defined in library |
2051 | | /// modules such as CUDA's libdevice. |
2052 | | /// |
2053 | | /// - ConstructAttributeList builds on top of getDefaultFunctionAttributes |
2054 | | /// and adds declaration-specific, convention-specific, and |
2055 | | /// frontend-specific logic. The last is of particular importance: |
2056 | | /// attributes that restrict how the frontend generates code must be |
2057 | | /// added here rather than getDefaultFunctionAttributes. |
2058 | | /// |
2059 | | void CodeGenModule::ConstructAttributeList(StringRef Name, |
2060 | | const CGFunctionInfo &FI, |
2061 | | CGCalleeInfo CalleeInfo, |
2062 | | llvm::AttributeList &AttrList, |
2063 | | unsigned &CallingConv, |
2064 | 717k | bool AttrOnCallSite, bool IsThunk) { |
2065 | 717k | llvm::AttrBuilder FuncAttrs(getLLVMContext()); |
2066 | 717k | llvm::AttrBuilder RetAttrs(getLLVMContext()); |
2067 | | |
2068 | | // Collect function IR attributes from the CC lowering. |
2069 | | // We'll collect the paramete and result attributes later. |
2070 | 717k | CallingConv = FI.getEffectiveCallingConvention(); |
2071 | 717k | if (FI.isNoReturn()) |
2072 | 1.28k | FuncAttrs.addAttribute(llvm::Attribute::NoReturn); |
2073 | 717k | if (FI.isCmseNSCall()) |
2074 | 62 | FuncAttrs.addAttribute("cmse_nonsecure_call"); |
2075 | | |
2076 | | // Collect function IR attributes from the callee prototype if we have one. |
2077 | 717k | AddAttributesFromFunctionProtoType(getContext(), FuncAttrs, |
2078 | 717k | CalleeInfo.getCalleeFunctionProtoType()); |
2079 | | |
2080 | 717k | const Decl *TargetDecl = CalleeInfo.getCalleeDecl().getDecl(); |
2081 | | |
2082 | | // Attach assumption attributes to the declaration. If this is a call |
2083 | | // site, attach assumptions from the caller to the call as well. |
2084 | 717k | AddAttributesFromAssumes(FuncAttrs, TargetDecl); |
2085 | | |
2086 | 717k | bool HasOptnone = false; |
2087 | | // The NoBuiltinAttr attached to the target FunctionDecl. |
2088 | 717k | const NoBuiltinAttr *NBA = nullptr; |
2089 | | |
2090 | | // Collect function IR attributes based on declaration-specific |
2091 | | // information. |
2092 | | // FIXME: handle sseregparm someday... |
2093 | 717k | if (TargetDecl) { |
2094 | 685k | if (TargetDecl->hasAttr<ReturnsTwiceAttr>()) |
2095 | 43 | FuncAttrs.addAttribute(llvm::Attribute::ReturnsTwice); |
2096 | 685k | if (TargetDecl->hasAttr<NoThrowAttr>()) |
2097 | 5.55k | FuncAttrs.addAttribute(llvm::Attribute::NoUnwind); |
2098 | 685k | if (TargetDecl->hasAttr<NoReturnAttr>()) |
2099 | 0 | FuncAttrs.addAttribute(llvm::Attribute::NoReturn); |
2100 | 685k | if (TargetDecl->hasAttr<ColdAttr>()) |
2101 | 387 | FuncAttrs.addAttribute(llvm::Attribute::Cold); |
2102 | 685k | if (TargetDecl->hasAttr<HotAttr>()) |
2103 | 3 | FuncAttrs.addAttribute(llvm::Attribute::Hot); |
2104 | 685k | if (TargetDecl->hasAttr<NoDuplicateAttr>()) |
2105 | 4 | FuncAttrs.addAttribute(llvm::Attribute::NoDuplicate); |
2106 | 685k | if (TargetDecl->hasAttr<ConvergentAttr>()) |
2107 | 3 | FuncAttrs.addAttribute(llvm::Attribute::Convergent); |
2108 | | |
2109 | 685k | if (const FunctionDecl *Fn = dyn_cast<FunctionDecl>(TargetDecl)) { |
2110 | 651k | AddAttributesFromFunctionProtoType( |
2111 | 651k | getContext(), FuncAttrs, Fn->getType()->getAs<FunctionProtoType>()); |
2112 | 651k | if (AttrOnCallSite && Fn->isReplaceableGlobalAllocationFunction()344k ) { |
2113 | | // A sane operator new returns a non-aliasing pointer. |
2114 | 3.94k | auto Kind = Fn->getDeclName().getCXXOverloadedOperator(); |
2115 | 3.94k | if (getCodeGenOpts().AssumeSaneOperatorNew && |
2116 | 3.94k | (3.93k Kind == OO_New3.93k || Kind == OO_Array_New2.62k )) |
2117 | 1.60k | RetAttrs.addAttribute(llvm::Attribute::NoAlias); |
2118 | 3.94k | } |
2119 | 651k | const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Fn); |
2120 | 651k | const bool IsVirtualCall = MD && MD->isVirtual()288k ; |
2121 | | // Don't use [[noreturn]], _Noreturn or [[no_builtin]] for a call to a |
2122 | | // virtual function. These attributes are not inherited by overloads. |
2123 | 651k | if (!(AttrOnCallSite && IsVirtualCall344k )) { |
2124 | 646k | if (Fn->isNoReturn()) |
2125 | 3.61k | FuncAttrs.addAttribute(llvm::Attribute::NoReturn); |
2126 | 646k | NBA = Fn->getAttr<NoBuiltinAttr>(); |
2127 | 646k | } |
2128 | | // Only place nomerge attribute on call sites, never functions. This |
2129 | | // allows it to work on indirect virtual function calls. |
2130 | 651k | if (AttrOnCallSite && TargetDecl->hasAttr<NoMergeAttr>()344k ) |
2131 | 12 | FuncAttrs.addAttribute(llvm::Attribute::NoMerge); |
2132 | 651k | } |
2133 | | |
2134 | | // 'const', 'pure' and 'noalias' attributed functions are also nounwind. |
2135 | 685k | if (TargetDecl->hasAttr<ConstAttr>()) { |
2136 | 1.79k | FuncAttrs.addAttribute(llvm::Attribute::ReadNone); |
2137 | 1.79k | FuncAttrs.addAttribute(llvm::Attribute::NoUnwind); |
2138 | | // gcc specifies that 'const' functions have greater restrictions than |
2139 | | // 'pure' functions, so they also cannot have infinite loops. |
2140 | 1.79k | FuncAttrs.addAttribute(llvm::Attribute::WillReturn); |
2141 | 683k | } else if (TargetDecl->hasAttr<PureAttr>()) { |
2142 | 211 | FuncAttrs.addAttribute(llvm::Attribute::ReadOnly); |
2143 | 211 | FuncAttrs.addAttribute(llvm::Attribute::NoUnwind); |
2144 | | // gcc specifies that 'pure' functions cannot have infinite loops. |
2145 | 211 | FuncAttrs.addAttribute(llvm::Attribute::WillReturn); |
2146 | 683k | } else if (TargetDecl->hasAttr<NoAliasAttr>()) { |
2147 | 2 | FuncAttrs.addAttribute(llvm::Attribute::ArgMemOnly); |
2148 | 2 | FuncAttrs.addAttribute(llvm::Attribute::NoUnwind); |
2149 | 2 | } |
2150 | 685k | if (TargetDecl->hasAttr<RestrictAttr>()) |
2151 | 56 | RetAttrs.addAttribute(llvm::Attribute::NoAlias); |
2152 | 685k | if (TargetDecl->hasAttr<ReturnsNonNullAttr>() && |
2153 | 685k | !CodeGenOpts.NullPointerIsValid2.49k ) |
2154 | 2.49k | RetAttrs.addAttribute(llvm::Attribute::NonNull); |
2155 | 685k | if (TargetDecl->hasAttr<AnyX86NoCallerSavedRegistersAttr>()) |
2156 | 14 | FuncAttrs.addAttribute("no_caller_saved_registers"); |
2157 | 685k | if (TargetDecl->hasAttr<AnyX86NoCfCheckAttr>()) |
2158 | 2 | FuncAttrs.addAttribute(llvm::Attribute::NoCfCheck); |
2159 | 685k | if (TargetDecl->hasAttr<LeafAttr>()) |
2160 | 1 | FuncAttrs.addAttribute(llvm::Attribute::NoCallback); |
2161 | | |
2162 | 685k | HasOptnone = TargetDecl->hasAttr<OptimizeNoneAttr>(); |
2163 | 685k | if (auto *AllocSize = TargetDecl->getAttr<AllocSizeAttr>()) { |
2164 | 2.85k | Optional<unsigned> NumElemsParam; |
2165 | 2.85k | if (AllocSize->getNumElemsParam().isValid()) |
2166 | 144 | NumElemsParam = AllocSize->getNumElemsParam().getLLVMIndex(); |
2167 | 2.85k | FuncAttrs.addAllocSizeAttr(AllocSize->getElemSizeParam().getLLVMIndex(), |
2168 | 2.85k | NumElemsParam); |
2169 | 2.85k | } |
2170 | | |
2171 | 685k | if (TargetDecl->hasAttr<OpenCLKernelAttr>()) { |
2172 | 383 | if (getLangOpts().OpenCLVersion <= 120) { |
2173 | | // OpenCL v1.2 Work groups are always uniform |
2174 | 277 | FuncAttrs.addAttribute("uniform-work-group-size", "true"); |
2175 | 277 | } else { |
2176 | | // OpenCL v2.0 Work groups may be whether uniform or not. |
2177 | | // '-cl-uniform-work-group-size' compile option gets a hint |
2178 | | // to the compiler that the global work-size be a multiple of |
2179 | | // the work-group size specified to clEnqueueNDRangeKernel |
2180 | | // (i.e. work groups are uniform). |
2181 | 106 | FuncAttrs.addAttribute("uniform-work-group-size", |
2182 | 106 | llvm::toStringRef(CodeGenOpts.UniformWGSize)); |
2183 | 106 | } |
2184 | 383 | } |
2185 | 685k | } |
2186 | | |
2187 | | // Attach "no-builtins" attributes to: |
2188 | | // * call sites: both `nobuiltin` and "no-builtins" or "no-builtin-<name>". |
2189 | | // * definitions: "no-builtins" or "no-builtin-<name>" only. |
2190 | | // The attributes can come from: |
2191 | | // * LangOpts: -ffreestanding, -fno-builtin, -fno-builtin-<name> |
2192 | | // * FunctionDecl attributes: __attribute__((no_builtin(...))) |
2193 | 717k | addNoBuiltinAttributes(FuncAttrs, getLangOpts(), NBA); |
2194 | | |
2195 | | // Collect function IR attributes based on global settiings. |
2196 | 717k | getDefaultFunctionAttributes(Name, HasOptnone, AttrOnCallSite, FuncAttrs); |
2197 | | |
2198 | | // Override some default IR attributes based on declaration-specific |
2199 | | // information. |
2200 | 717k | if (TargetDecl) { |
2201 | 685k | if (TargetDecl->hasAttr<NoSpeculativeLoadHardeningAttr>()) |
2202 | 13 | FuncAttrs.removeAttribute(llvm::Attribute::SpeculativeLoadHardening); |
2203 | 685k | if (TargetDecl->hasAttr<SpeculativeLoadHardeningAttr>()) |
2204 | 13 | FuncAttrs.addAttribute(llvm::Attribute::SpeculativeLoadHardening); |
2205 | 685k | if (TargetDecl->hasAttr<NoSplitStackAttr>()) |
2206 | 8 | FuncAttrs.removeAttribute("split-stack"); |
2207 | 685k | if (TargetDecl->hasAttr<ZeroCallUsedRegsAttr>()) { |
2208 | | // A function "__attribute__((...))" overrides the command-line flag. |
2209 | 81 | auto Kind = |
2210 | 81 | TargetDecl->getAttr<ZeroCallUsedRegsAttr>()->getZeroCallUsedRegs(); |
2211 | 81 | FuncAttrs.removeAttribute("zero-call-used-regs"); |
2212 | 81 | FuncAttrs.addAttribute( |
2213 | 81 | "zero-call-used-regs", |
2214 | 81 | ZeroCallUsedRegsAttr::ConvertZeroCallUsedRegsKindToStr(Kind)); |
2215 | 81 | } |
2216 | | |
2217 | | // Add NonLazyBind attribute to function declarations when -fno-plt |
2218 | | // is used. |
2219 | | // FIXME: what if we just haven't processed the function definition |
2220 | | // yet, or if it's an external definition like C99 inline? |
2221 | 685k | if (CodeGenOpts.NoPLT) { |
2222 | 45 | if (auto *Fn = dyn_cast<FunctionDecl>(TargetDecl)) { |
2223 | 45 | if (!Fn->isDefined() && !AttrOnCallSite22 ) { |
2224 | 12 | FuncAttrs.addAttribute(llvm::Attribute::NonLazyBind); |
2225 | 12 | } |
2226 | 45 | } |
2227 | 45 | } |
2228 | 685k | } |
2229 | | |
2230 | | // Add "sample-profile-suffix-elision-policy" attribute for internal linkage |
2231 | | // functions with -funique-internal-linkage-names. |
2232 | 717k | if (TargetDecl && CodeGenOpts.UniqueInternalLinkageNames685k ) { |
2233 | 47 | if (isa<FunctionDecl>(TargetDecl)) { |
2234 | 47 | if (this->getFunctionLinkage(CalleeInfo.getCalleeDecl()) == |
2235 | 47 | llvm::GlobalValue::InternalLinkage) |
2236 | 34 | FuncAttrs.addAttribute("sample-profile-suffix-elision-policy", |
2237 | 34 | "selected"); |
2238 | 47 | } |
2239 | 47 | } |
2240 | | |
2241 | | // Collect non-call-site function IR attributes from declaration-specific |
2242 | | // information. |
2243 | 717k | if (!AttrOnCallSite) { |
2244 | 350k | if (TargetDecl && TargetDecl->hasAttr<CmseNSEntryAttr>()335k ) |
2245 | 94 | FuncAttrs.addAttribute("cmse_nonsecure_entry"); |
2246 | | |
2247 | | // Whether tail calls are enabled. |
2248 | 350k | auto shouldDisableTailCalls = [&] { |
2249 | | // Should this be honored in getDefaultFunctionAttributes? |
2250 | 350k | if (CodeGenOpts.DisableTailCalls) |
2251 | 3 | return true; |
2252 | | |
2253 | 350k | if (!TargetDecl) |
2254 | 14.9k | return false; |
2255 | | |
2256 | 335k | if (TargetDecl->hasAttr<DisableTailCallsAttr>() || |
2257 | 335k | TargetDecl->hasAttr<AnyX86InterruptAttr>()335k ) |
2258 | 147 | return true; |
2259 | | |
2260 | 335k | if (CodeGenOpts.NoEscapingBlockTailCalls) { |
2261 | 17 | if (const auto *BD = dyn_cast<BlockDecl>(TargetDecl)) |
2262 | 8 | if (!BD->doesNotEscape()) |
2263 | 4 | return true; |
2264 | 17 | } |
2265 | | |
2266 | 335k | return false; |
2267 | 335k | }; |
2268 | 350k | if (shouldDisableTailCalls()) |
2269 | 154 | FuncAttrs.addAttribute("disable-tail-calls", "true"); |
2270 | | |
2271 | | // CPU/feature overrides. addDefaultFunctionDefinitionAttributes |
2272 | | // handles these separately to set them based on the global defaults. |
2273 | 350k | GetCPUAndFeaturesAttributes(CalleeInfo.getCalleeDecl(), FuncAttrs); |
2274 | 350k | } |
2275 | | |
2276 | | // Collect attributes from arguments and return values. |
2277 | 717k | ClangToLLVMArgMapping IRFunctionArgs(getContext(), FI); |
2278 | | |
2279 | 717k | QualType RetTy = FI.getReturnType(); |
2280 | 717k | const ABIArgInfo &RetAI = FI.getReturnInfo(); |
2281 | 717k | const llvm::DataLayout &DL = getDataLayout(); |
2282 | | |
2283 | | // C++ explicitly makes returning undefined values UB. C's rule only applies |
2284 | | // to used values, so we never mark them noundef for now. |
2285 | 717k | bool HasStrictReturn = getLangOpts().CPlusPlus; |
2286 | 717k | if (TargetDecl && HasStrictReturn685k ) { |
2287 | 554k | if (const FunctionDecl *FDecl = dyn_cast<FunctionDecl>(TargetDecl)) |
2288 | 523k | HasStrictReturn &= !FDecl->isExternC(); |
2289 | 30.6k | else if (const VarDecl *VDecl = dyn_cast<VarDecl>(TargetDecl)) |
2290 | | // Function pointer |
2291 | 4.98k | HasStrictReturn &= !VDecl->isExternC(); |
2292 | 554k | } |
2293 | | |
2294 | | // We don't want to be too aggressive with the return checking, unless |
2295 | | // it's explicit in the code opts or we're using an appropriate sanitizer. |
2296 | | // Try to respect what the programmer intended. |
2297 | 717k | HasStrictReturn &= getCodeGenOpts().StrictReturn || |
2298 | 717k | !MayDropFunctionReturn(getContext(), RetTy)27 || |
2299 | 717k | getLangOpts().Sanitize.has(SanitizerKind::Memory)21 || |
2300 | 717k | getLangOpts().Sanitize.has(SanitizerKind::Return)21 ; |
2301 | | |
2302 | | // Determine if the return type could be partially undef |
2303 | 717k | if (CodeGenOpts.EnableNoundefAttrs && HasStrictReturn576k ) { |
2304 | 415k | if (!RetTy->isVoidType() && RetAI.getKind() != ABIArgInfo::Indirect171k && |
2305 | 415k | DetermineNoUndef(RetTy, getTypes(), DL, RetAI)169k ) |
2306 | 149k | RetAttrs.addAttribute(llvm::Attribute::NoUndef); |
2307 | 415k | } |
2308 | | |
2309 | 717k | switch (RetAI.getKind()) { |
2310 | 21.6k | case ABIArgInfo::Extend: |
2311 | 21.6k | if (RetAI.isSignExt()) |
2312 | 8.18k | RetAttrs.addAttribute(llvm::Attribute::SExt); |
2313 | 13.5k | else |
2314 | 13.5k | RetAttrs.addAttribute(llvm::Attribute::ZExt); |
2315 | 21.6k | LLVM_FALLTHROUGH; |
2316 | 398k | case ABIArgInfo::Direct: |
2317 | 398k | if (RetAI.getInReg()) |
2318 | 16 | RetAttrs.addAttribute(llvm::Attribute::InReg); |
2319 | 398k | break; |
2320 | 311k | case ABIArgInfo::Ignore: |
2321 | 311k | break; |
2322 | | |
2323 | 13 | case ABIArgInfo::InAlloca: |
2324 | 6.79k | case ABIArgInfo::Indirect: { |
2325 | | // inalloca and sret disable readnone and readonly |
2326 | 6.79k | FuncAttrs.removeAttribute(llvm::Attribute::ReadOnly) |
2327 | 6.79k | .removeAttribute(llvm::Attribute::ReadNone); |
2328 | 6.79k | break; |
2329 | 13 | } |
2330 | | |
2331 | 983 | case ABIArgInfo::CoerceAndExpand: |
2332 | 983 | break; |
2333 | | |
2334 | 0 | case ABIArgInfo::Expand: |
2335 | 0 | case ABIArgInfo::IndirectAliased: |
2336 | 0 | llvm_unreachable("Invalid ABI kind for return argument"); |
2337 | 717k | } |
2338 | | |
2339 | 717k | if (!IsThunk) { |
2340 | | // FIXME: fix this properly, https://reviews.llvm.org/D100388 |
2341 | 716k | if (const auto *RefTy = RetTy->getAs<ReferenceType>()) { |
2342 | 67.0k | QualType PTy = RefTy->getPointeeType(); |
2343 | 67.0k | if (!PTy->isIncompleteType() && PTy->isConstantSizeType()67.0k ) |
2344 | 67.0k | RetAttrs.addDereferenceableAttr( |
2345 | 67.0k | getMinimumObjectSize(PTy).getQuantity()); |
2346 | 67.0k | if (getContext().getTargetAddressSpace(PTy) == 0 && |
2347 | 67.0k | !CodeGenOpts.NullPointerIsValid67.0k ) |
2348 | 67.0k | RetAttrs.addAttribute(llvm::Attribute::NonNull); |
2349 | 67.0k | if (PTy->isObjectType()) { |
2350 | 66.0k | llvm::Align Alignment = |
2351 | 66.0k | getNaturalPointeeTypeAlignment(RetTy).getAsAlign(); |
2352 | 66.0k | RetAttrs.addAlignmentAttr(Alignment); |
2353 | 66.0k | } |
2354 | 67.0k | } |
2355 | 716k | } |
2356 | | |
2357 | 717k | bool hasUsedSRet = false; |
2358 | 717k | SmallVector<llvm::AttributeSet, 4> ArgAttrs(IRFunctionArgs.totalIRArgs()); |
2359 | | |
2360 | | // Attach attributes to sret. |
2361 | 717k | if (IRFunctionArgs.hasSRetArg()) { |
2362 | 6.77k | llvm::AttrBuilder SRETAttrs(getLLVMContext()); |
2363 | 6.77k | SRETAttrs.addStructRetAttr(getTypes().ConvertTypeForMem(RetTy)); |
2364 | 6.77k | hasUsedSRet = true; |
2365 | 6.77k | if (RetAI.getInReg()) |
2366 | 64 | SRETAttrs.addAttribute(llvm::Attribute::InReg); |
2367 | 6.77k | SRETAttrs.addAlignmentAttr(RetAI.getIndirectAlign().getQuantity()); |
2368 | 6.77k | ArgAttrs[IRFunctionArgs.getSRetArgNo()] = |
2369 | 6.77k | llvm::AttributeSet::get(getLLVMContext(), SRETAttrs); |
2370 | 6.77k | } |
2371 | | |
2372 | | // Attach attributes to inalloca argument. |
2373 | 717k | if (IRFunctionArgs.hasInallocaArg()) { |
2374 | 162 | llvm::AttrBuilder Attrs(getLLVMContext()); |
2375 | 162 | Attrs.addInAllocaAttr(FI.getArgStruct()); |
2376 | 162 | ArgAttrs[IRFunctionArgs.getInallocaArgNo()] = |
2377 | 162 | llvm::AttributeSet::get(getLLVMContext(), Attrs); |
2378 | 162 | } |
2379 | | |
2380 | | // Apply `nonnull`, `dereferencable(N)` and `align N` to the `this` argument, |
2381 | | // unless this is a thunk function. |
2382 | | // FIXME: fix this properly, https://reviews.llvm.org/D100388 |
2383 | 717k | if (FI.isInstanceMethod() && !IRFunctionArgs.hasInallocaArg()272k && |
2384 | 717k | !FI.arg_begin()->type->isVoidPointerType()272k && !IsThunk271k ) { |
2385 | 271k | auto IRArgs = IRFunctionArgs.getIRArgs(0); |
2386 | | |
2387 | 271k | assert(IRArgs.second == 1 && "Expected only a single `this` pointer."); |
2388 | | |
2389 | 0 | llvm::AttrBuilder Attrs(getLLVMContext()); |
2390 | | |
2391 | 271k | QualType ThisTy = |
2392 | 271k | FI.arg_begin()->type.castAs<PointerType>()->getPointeeType(); |
2393 | | |
2394 | 271k | if (!CodeGenOpts.NullPointerIsValid && |
2395 | 271k | getContext().getTargetAddressSpace(FI.arg_begin()->type) == 0270k ) { |
2396 | 270k | Attrs.addAttribute(llvm::Attribute::NonNull); |
2397 | 270k | Attrs.addDereferenceableAttr(getMinimumObjectSize(ThisTy).getQuantity()); |
2398 | 270k | } else { |
2399 | | // FIXME dereferenceable should be correct here, regardless of |
2400 | | // NullPointerIsValid. However, dereferenceable currently does not always |
2401 | | // respect NullPointerIsValid and may imply nonnull and break the program. |
2402 | | // See https://reviews.llvm.org/D66618 for discussions. |
2403 | 322 | Attrs.addDereferenceableOrNullAttr( |
2404 | 322 | getMinimumObjectSize( |
2405 | 322 | FI.arg_begin()->type.castAs<PointerType>()->getPointeeType()) |
2406 | 322 | .getQuantity()); |
2407 | 322 | } |
2408 | | |
2409 | 271k | llvm::Align Alignment = |
2410 | 271k | getNaturalTypeAlignment(ThisTy, /*BaseInfo=*/nullptr, |
2411 | 271k | /*TBAAInfo=*/nullptr, /*forPointeeType=*/true) |
2412 | 271k | .getAsAlign(); |
2413 | 271k | Attrs.addAlignmentAttr(Alignment); |
2414 | | |
2415 | 271k | ArgAttrs[IRArgs.first] = llvm::AttributeSet::get(getLLVMContext(), Attrs); |
2416 | 271k | } |
2417 | | |
2418 | 0 | unsigned ArgNo = 0; |
2419 | 717k | for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(), |
2420 | 717k | E = FI.arg_end(); |
2421 | 1.90M | I != E; ++I, ++ArgNo1.18M ) { |
2422 | 1.18M | QualType ParamType = I->type; |
2423 | 1.18M | const ABIArgInfo &AI = I->info; |
2424 | 1.18M | llvm::AttrBuilder Attrs(getLLVMContext()); |
2425 | | |
2426 | | // Add attribute for padding argument, if necessary. |
2427 | 1.18M | if (IRFunctionArgs.hasPaddingArg(ArgNo)) { |
2428 | 47 | if (AI.getPaddingInReg()) { |
2429 | 5 | ArgAttrs[IRFunctionArgs.getPaddingArgNo(ArgNo)] = |
2430 | 5 | llvm::AttributeSet::get( |
2431 | 5 | getLLVMContext(), |
2432 | 5 | llvm::AttrBuilder(getLLVMContext()).addAttribute(llvm::Attribute::InReg)); |
2433 | 5 | } |
2434 | 47 | } |
2435 | | |
2436 | | // Decide whether the argument we're handling could be partially undef |
2437 | 1.18M | if (CodeGenOpts.EnableNoundefAttrs && |
2438 | 1.18M | DetermineNoUndef(ParamType, getTypes(), DL, AI)942k ) { |
2439 | 857k | Attrs.addAttribute(llvm::Attribute::NoUndef); |
2440 | 857k | } |
2441 | | |
2442 | | // 'restrict' -> 'noalias' is done in EmitFunctionProlog when we |
2443 | | // have the corresponding parameter variable. It doesn't make |
2444 | | // sense to do it here because parameters are so messed up. |
2445 | 1.18M | switch (AI.getKind()) { |
2446 | 27.5k | case ABIArgInfo::Extend: |
2447 | 27.5k | if (AI.isSignExt()) |
2448 | 11.5k | Attrs.addAttribute(llvm::Attribute::SExt); |
2449 | 16.0k | else |
2450 | 16.0k | Attrs.addAttribute(llvm::Attribute::ZExt); |
2451 | 27.5k | LLVM_FALLTHROUGH; |
2452 | 1.16M | case ABIArgInfo::Direct: |
2453 | 1.16M | if (ArgNo == 0 && FI.isChainCall()644k ) |
2454 | 8 | Attrs.addAttribute(llvm::Attribute::Nest); |
2455 | 1.16M | else if (AI.getInReg()) |
2456 | 580 | Attrs.addAttribute(llvm::Attribute::InReg); |
2457 | 1.16M | Attrs.addStackAlignmentAttr(llvm::MaybeAlign(AI.getDirectAlign())); |
2458 | 1.16M | break; |
2459 | | |
2460 | 5.08k | case ABIArgInfo::Indirect: { |
2461 | 5.08k | if (AI.getInReg()) |
2462 | 24 | Attrs.addAttribute(llvm::Attribute::InReg); |
2463 | | |
2464 | 5.08k | if (AI.getIndirectByVal()) |
2465 | 1.89k | Attrs.addByValAttr(getTypes().ConvertTypeForMem(ParamType)); |
2466 | | |
2467 | 5.08k | auto *Decl = ParamType->getAsRecordDecl(); |
2468 | 5.08k | if (CodeGenOpts.PassByValueIsNoAlias && Decl7 && |
2469 | 5.08k | Decl->getArgPassingRestrictions() == RecordDecl::APK_CanPassInRegs7 ) |
2470 | | // When calling the function, the pointer passed in will be the only |
2471 | | // reference to the underlying object. Mark it accordingly. |
2472 | 5 | Attrs.addAttribute(llvm::Attribute::NoAlias); |
2473 | | |
2474 | | // TODO: We could add the byref attribute if not byval, but it would |
2475 | | // require updating many testcases. |
2476 | | |
2477 | 5.08k | CharUnits Align = AI.getIndirectAlign(); |
2478 | | |
2479 | | // In a byval argument, it is important that the required |
2480 | | // alignment of the type is honored, as LLVM might be creating a |
2481 | | // *new* stack object, and needs to know what alignment to give |
2482 | | // it. (Sometimes it can deduce a sensible alignment on its own, |
2483 | | // but not if clang decides it must emit a packed struct, or the |
2484 | | // user specifies increased alignment requirements.) |
2485 | | // |
2486 | | // This is different from indirect *not* byval, where the object |
2487 | | // exists already, and the align attribute is purely |
2488 | | // informative. |
2489 | 5.08k | assert(!Align.isZero()); |
2490 | | |
2491 | | // For now, only add this when we have a byval argument. |
2492 | | // TODO: be less lazy about updating test cases. |
2493 | 5.08k | if (AI.getIndirectByVal()) |
2494 | 1.89k | Attrs.addAlignmentAttr(Align.getQuantity()); |
2495 | | |
2496 | | // byval disables readnone and readonly. |
2497 | 5.08k | FuncAttrs.removeAttribute(llvm::Attribute::ReadOnly) |
2498 | 5.08k | .removeAttribute(llvm::Attribute::ReadNone); |
2499 | | |
2500 | 5.08k | break; |
2501 | 27.5k | } |
2502 | 14 | case ABIArgInfo::IndirectAliased: { |
2503 | 14 | CharUnits Align = AI.getIndirectAlign(); |
2504 | 14 | Attrs.addByRefAttr(getTypes().ConvertTypeForMem(ParamType)); |
2505 | 14 | Attrs.addAlignmentAttr(Align.getQuantity()); |
2506 | 14 | break; |
2507 | 27.5k | } |
2508 | 13.9k | case ABIArgInfo::Ignore: |
2509 | 14.1k | case ABIArgInfo::Expand: |
2510 | 15.1k | case ABIArgInfo::CoerceAndExpand: |
2511 | 15.1k | break; |
2512 | | |
2513 | 321 | case ABIArgInfo::InAlloca: |
2514 | | // inalloca disables readnone and readonly. |
2515 | 321 | FuncAttrs.removeAttribute(llvm::Attribute::ReadOnly) |
2516 | 321 | .removeAttribute(llvm::Attribute::ReadNone); |
2517 | 321 | continue; |
2518 | 1.18M | } |
2519 | | |
2520 | 1.18M | if (const auto *RefTy = ParamType->getAs<ReferenceType>()) { |
2521 | 136k | QualType PTy = RefTy->getPointeeType(); |
2522 | 136k | if (!PTy->isIncompleteType() && PTy->isConstantSizeType()135k ) |
2523 | 135k | Attrs.addDereferenceableAttr( |
2524 | 135k | getMinimumObjectSize(PTy).getQuantity()); |
2525 | 136k | if (getContext().getTargetAddressSpace(PTy) == 0 && |
2526 | 136k | !CodeGenOpts.NullPointerIsValid135k ) |
2527 | 135k | Attrs.addAttribute(llvm::Attribute::NonNull); |
2528 | 136k | if (PTy->isObjectType()) { |
2529 | 132k | llvm::Align Alignment = |
2530 | 132k | getNaturalPointeeTypeAlignment(ParamType).getAsAlign(); |
2531 | 132k | Attrs.addAlignmentAttr(Alignment); |
2532 | 132k | } |
2533 | 136k | } |
2534 | | |
2535 | | // From OpenCL spec v3.0.10 section 6.3.5 Alignment of Types: |
2536 | | // > For arguments to a __kernel function declared to be a pointer to a |
2537 | | // > data type, the OpenCL compiler can assume that the pointee is always |
2538 | | // > appropriately aligned as required by the data type. |
2539 | 1.18M | if (TargetDecl && TargetDecl->hasAttr<OpenCLKernelAttr>()1.11M && |
2540 | 1.18M | ParamType->isPointerType()312 ) { |
2541 | 166 | QualType PTy = ParamType->getPointeeType(); |
2542 | 166 | if (!PTy->isIncompleteType() && PTy->isConstantSizeType()165 ) { |
2543 | 165 | llvm::Align Alignment = |
2544 | 165 | getNaturalPointeeTypeAlignment(ParamType).getAsAlign(); |
2545 | 165 | Attrs.addAlignmentAttr(Alignment); |
2546 | 165 | } |
2547 | 166 | } |
2548 | | |
2549 | 1.18M | switch (FI.getExtParameterInfo(ArgNo).getABI()) { |
2550 | 1.18M | case ParameterABI::Ordinary: |
2551 | 1.18M | break; |
2552 | | |
2553 | 48 | case ParameterABI::SwiftIndirectResult: { |
2554 | | // Add 'sret' if we haven't already used it for something, but |
2555 | | // only if the result is void. |
2556 | 48 | if (!hasUsedSRet && RetTy->isVoidType()24 ) { |
2557 | 8 | Attrs.addStructRetAttr(getTypes().ConvertTypeForMem(ParamType)); |
2558 | 8 | hasUsedSRet = true; |
2559 | 8 | } |
2560 | | |
2561 | | // Add 'noalias' in either case. |
2562 | 48 | Attrs.addAttribute(llvm::Attribute::NoAlias); |
2563 | | |
2564 | | // Add 'dereferenceable' and 'alignment'. |
2565 | 48 | auto PTy = ParamType->getPointeeType(); |
2566 | 48 | if (!PTy->isIncompleteType() && PTy->isConstantSizeType()) { |
2567 | 48 | auto info = getContext().getTypeInfoInChars(PTy); |
2568 | 48 | Attrs.addDereferenceableAttr(info.Width.getQuantity()); |
2569 | 48 | Attrs.addAlignmentAttr(info.Align.getAsAlign()); |
2570 | 48 | } |
2571 | 48 | break; |
2572 | 0 | } |
2573 | | |
2574 | 24 | case ParameterABI::SwiftErrorResult: |
2575 | 24 | Attrs.addAttribute(llvm::Attribute::SwiftError); |
2576 | 24 | break; |
2577 | | |
2578 | 40 | case ParameterABI::SwiftContext: |
2579 | 40 | Attrs.addAttribute(llvm::Attribute::SwiftSelf); |
2580 | 40 | break; |
2581 | | |
2582 | 325 | case ParameterABI::SwiftAsyncContext: |
2583 | 325 | Attrs.addAttribute(llvm::Attribute::SwiftAsync); |
2584 | 325 | break; |
2585 | 1.18M | } |
2586 | | |
2587 | 1.18M | if (FI.getExtParameterInfo(ArgNo).isNoEscape()) |
2588 | 62 | Attrs.addAttribute(llvm::Attribute::NoCapture); |
2589 | | |
2590 | 1.18M | if (Attrs.hasAttributes()) { |
2591 | 869k | unsigned FirstIRArg, NumIRArgs; |
2592 | 869k | std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo); |
2593 | 1.74M | for (unsigned i = 0; i < NumIRArgs; i++870k ) |
2594 | 870k | ArgAttrs[FirstIRArg + i] = ArgAttrs[FirstIRArg + i].addAttributes( |
2595 | 870k | getLLVMContext(), llvm::AttributeSet::get(getLLVMContext(), Attrs)); |
2596 | 869k | } |
2597 | 1.18M | } |
2598 | 717k | assert(ArgNo == FI.arg_size()); |
2599 | | |
2600 | 0 | AttrList = llvm::AttributeList::get( |
2601 | 717k | getLLVMContext(), llvm::AttributeSet::get(getLLVMContext(), FuncAttrs), |
2602 | 717k | llvm::AttributeSet::get(getLLVMContext(), RetAttrs), ArgAttrs); |
2603 | 717k | } |
2604 | | |
2605 | | /// An argument came in as a promoted argument; demote it back to its |
2606 | | /// declared type. |
2607 | | static llvm::Value *emitArgumentDemotion(CodeGenFunction &CGF, |
2608 | | const VarDecl *var, |
2609 | 14 | llvm::Value *value) { |
2610 | 14 | llvm::Type *varType = CGF.ConvertType(var->getType()); |
2611 | | |
2612 | | // This can happen with promotions that actually don't change the |
2613 | | // underlying type, like the enum promotions. |
2614 | 14 | if (value->getType() == varType) return value0 ; |
2615 | | |
2616 | 14 | assert((varType->isIntegerTy() || varType->isFloatingPointTy()) |
2617 | 14 | && "unexpected promotion type"); |
2618 | | |
2619 | 14 | if (isa<llvm::IntegerType>(varType)) |
2620 | 8 | return CGF.Builder.CreateTrunc(value, varType, "arg.unpromote"); |
2621 | | |
2622 | 6 | return CGF.Builder.CreateFPCast(value, varType, "arg.unpromote"); |
2623 | 14 | } |
2624 | | |
2625 | | /// Returns the attribute (either parameter attribute, or function |
2626 | | /// attribute), which declares argument ArgNo to be non-null. |
2627 | | static const NonNullAttr *getNonNullAttr(const Decl *FD, const ParmVarDecl *PVD, |
2628 | 114k | QualType ArgType, unsigned ArgNo) { |
2629 | | // FIXME: __attribute__((nonnull)) can also be applied to: |
2630 | | // - references to pointers, where the pointee is known to be |
2631 | | // nonnull (apparently a Clang extension) |
2632 | | // - transparent unions containing pointers |
2633 | | // In the former case, LLVM IR cannot represent the constraint. In |
2634 | | // the latter case, we have no guarantee that the transparent union |
2635 | | // is in fact passed as a pointer. |
2636 | 114k | if (!ArgType->isAnyPointerType() && !ArgType->isBlockPointerType()47.7k ) |
2637 | 47.6k | return nullptr; |
2638 | | // First, check attribute on parameter itself. |
2639 | 66.4k | if (PVD) { |
2640 | 66.4k | if (auto ParmNNAttr = PVD->getAttr<NonNullAttr>()) |
2641 | 10 | return ParmNNAttr; |
2642 | 66.4k | } |
2643 | | // Check function attributes. |
2644 | 66.4k | if (!FD) |
2645 | 5 | return nullptr; |
2646 | 66.4k | for (const auto *NNAttr : FD->specific_attrs<NonNullAttr>()) { |
2647 | 32 | if (NNAttr->isNonNull(ArgNo)) |
2648 | 28 | return NNAttr; |
2649 | 32 | } |
2650 | 66.3k | return nullptr; |
2651 | 66.4k | } |
2652 | | |
2653 | | namespace { |
2654 | | struct CopyBackSwiftError final : EHScopeStack::Cleanup { |
2655 | | Address Temp; |
2656 | | Address Arg; |
2657 | 16 | CopyBackSwiftError(Address temp, Address arg) : Temp(temp), Arg(arg) {} |
2658 | 16 | void Emit(CodeGenFunction &CGF, Flags flags) override { |
2659 | 16 | llvm::Value *errorValue = CGF.Builder.CreateLoad(Temp); |
2660 | 16 | CGF.Builder.CreateStore(errorValue, Arg); |
2661 | 16 | } |
2662 | | }; |
2663 | | } |
2664 | | |
2665 | | void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI, |
2666 | | llvm::Function *Fn, |
2667 | 314k | const FunctionArgList &Args) { |
2668 | 314k | if (CurCodeDecl && CurCodeDecl->hasAttr<NakedAttr>()305k ) |
2669 | | // Naked functions don't have prologues. |
2670 | 10 | return; |
2671 | | |
2672 | | // If this is an implicit-return-zero function, go ahead and |
2673 | | // initialize the return value. TODO: it might be nice to have |
2674 | | // a more general mechanism for this that didn't require synthesized |
2675 | | // return statements. |
2676 | 314k | if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(CurCodeDecl)) { |
2677 | 270k | if (FD->hasImplicitReturnZero()) { |
2678 | 5.66k | QualType RetTy = FD->getReturnType().getUnqualifiedType(); |
2679 | 5.66k | llvm::Type* LLVMTy = CGM.getTypes().ConvertType(RetTy); |
2680 | 5.66k | llvm::Constant* Zero = llvm::Constant::getNullValue(LLVMTy); |
2681 | 5.66k | Builder.CreateStore(Zero, ReturnValue); |
2682 | 5.66k | } |
2683 | 270k | } |
2684 | | |
2685 | | // FIXME: We no longer need the types from FunctionArgList; lift up and |
2686 | | // simplify. |
2687 | | |
2688 | 314k | ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), FI); |
2689 | 314k | assert(Fn->arg_size() == IRFunctionArgs.totalIRArgs()); |
2690 | | |
2691 | | // If we're using inalloca, all the memory arguments are GEPs off of the last |
2692 | | // parameter, which is a pointer to the complete memory area. |
2693 | 0 | Address ArgStruct = Address::invalid(); |
2694 | 314k | if (IRFunctionArgs.hasInallocaArg()) { |
2695 | 41 | ArgStruct = Address(Fn->getArg(IRFunctionArgs.getInallocaArgNo()), |
2696 | 41 | FI.getArgStruct(), FI.getArgStructAlignment()); |
2697 | | |
2698 | 41 | assert(ArgStruct.getType() == FI.getArgStruct()->getPointerTo()); |
2699 | 41 | } |
2700 | | |
2701 | | // Name the struct return parameter. |
2702 | 314k | if (IRFunctionArgs.hasSRetArg()) { |
2703 | 4.14k | auto AI = Fn->getArg(IRFunctionArgs.getSRetArgNo()); |
2704 | 4.14k | AI->setName("agg.result"); |
2705 | 4.14k | AI->addAttr(llvm::Attribute::NoAlias); |
2706 | 4.14k | } |
2707 | | |
2708 | | // Track if we received the parameter as a pointer (indirect, byval, or |
2709 | | // inalloca). If already have a pointer, EmitParmDecl doesn't need to copy it |
2710 | | // into a local alloca for us. |
2711 | 314k | SmallVector<ParamValue, 16> ArgVals; |
2712 | 314k | ArgVals.reserve(Args.size()); |
2713 | | |
2714 | | // Create a pointer value for every parameter declaration. This usually |
2715 | | // entails copying one or more LLVM IR arguments into an alloca. Don't push |
2716 | | // any cleanups or do anything that might unwind. We do that separately, so |
2717 | | // we can push the cleanups in the correct order for the ABI. |
2718 | 314k | assert(FI.arg_size() == Args.size() && |
2719 | 314k | "Mismatch between function signature & arguments."); |
2720 | 0 | unsigned ArgNo = 0; |
2721 | 314k | CGFunctionInfo::const_arg_iterator info_it = FI.arg_begin(); |
2722 | 314k | for (FunctionArgList::const_iterator i = Args.begin(), e = Args.end(); |
2723 | 841k | i != e; ++i, ++info_it, ++ArgNo526k ) { |
2724 | 526k | const VarDecl *Arg = *i; |
2725 | 526k | const ABIArgInfo &ArgI = info_it->info; |
2726 | | |
2727 | 526k | bool isPromoted = |
2728 | 526k | isa<ParmVarDecl>(Arg) && cast<ParmVarDecl>(Arg)->isKNRPromoted()335k ; |
2729 | | // We are converting from ABIArgInfo type to VarDecl type directly, unless |
2730 | | // the parameter is promoted. In this case we convert to |
2731 | | // CGFunctionInfo::ArgInfo type with subsequent argument demotion. |
2732 | 526k | QualType Ty = isPromoted ? info_it->type14 : Arg->getType()526k ; |
2733 | 526k | assert(hasScalarEvaluationKind(Ty) == |
2734 | 526k | hasScalarEvaluationKind(Arg->getType())); |
2735 | | |
2736 | 0 | unsigned FirstIRArg, NumIRArgs; |
2737 | 526k | std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo); |
2738 | | |
2739 | 526k | switch (ArgI.getKind()) { |
2740 | 73 | case ABIArgInfo::InAlloca: { |
2741 | 73 | assert(NumIRArgs == 0); |
2742 | 0 | auto FieldIndex = ArgI.getInAllocaFieldIndex(); |
2743 | 73 | Address V = |
2744 | 73 | Builder.CreateStructGEP(ArgStruct, FieldIndex, Arg->getName()); |
2745 | 73 | if (ArgI.getInAllocaIndirect()) |
2746 | 6 | V = Address(Builder.CreateLoad(V), ConvertTypeForMem(Ty), |
2747 | 6 | getContext().getTypeAlignInChars(Ty)); |
2748 | 73 | ArgVals.push_back(ParamValue::forIndirect(V)); |
2749 | 73 | break; |
2750 | 0 | } |
2751 | | |
2752 | 2.48k | case ABIArgInfo::Indirect: |
2753 | 2.50k | case ABIArgInfo::IndirectAliased: { |
2754 | 2.50k | assert(NumIRArgs == 1); |
2755 | 0 | Address ParamAddr = Address(Fn->getArg(FirstIRArg), ConvertTypeForMem(Ty), |
2756 | 2.50k | ArgI.getIndirectAlign()); |
2757 | | |
2758 | 2.50k | if (!hasScalarEvaluationKind(Ty)) { |
2759 | | // Aggregates and complex variables are accessed by reference. All we |
2760 | | // need to do is realign the value, if requested. Also, if the address |
2761 | | // may be aliased, copy it to ensure that the parameter variable is |
2762 | | // mutable and has a unique adress, as C requires. |
2763 | 2.16k | Address V = ParamAddr; |
2764 | 2.16k | if (ArgI.getIndirectRealign() || ArgI.isIndirectAliased()2.13k ) { |
2765 | 42 | Address AlignedTemp = CreateMemTemp(Ty, "coerce"); |
2766 | | |
2767 | | // Copy from the incoming argument pointer to the temporary with the |
2768 | | // appropriate alignment. |
2769 | | // |
2770 | | // FIXME: We should have a common utility for generating an aggregate |
2771 | | // copy. |
2772 | 42 | CharUnits Size = getContext().getTypeSizeInChars(Ty); |
2773 | 42 | Builder.CreateMemCpy( |
2774 | 42 | AlignedTemp.getPointer(), AlignedTemp.getAlignment().getAsAlign(), |
2775 | 42 | ParamAddr.getPointer(), ParamAddr.getAlignment().getAsAlign(), |
2776 | 42 | llvm::ConstantInt::get(IntPtrTy, Size.getQuantity())); |
2777 | 42 | V = AlignedTemp; |
2778 | 42 | } |
2779 | 2.16k | ArgVals.push_back(ParamValue::forIndirect(V)); |
2780 | 2.16k | } else { |
2781 | | // Load scalar value from indirect argument. |
2782 | 338 | llvm::Value *V = |
2783 | 338 | EmitLoadOfScalar(ParamAddr, false, Ty, Arg->getBeginLoc()); |
2784 | | |
2785 | 338 | if (isPromoted) |
2786 | 0 | V = emitArgumentDemotion(*this, Arg, V); |
2787 | 338 | ArgVals.push_back(ParamValue::forDirect(V)); |
2788 | 338 | } |
2789 | 2.50k | break; |
2790 | 2.48k | } |
2791 | | |
2792 | 17.7k | case ABIArgInfo::Extend: |
2793 | 516k | case ABIArgInfo::Direct: { |
2794 | 516k | auto AI = Fn->getArg(FirstIRArg); |
2795 | 516k | llvm::Type *LTy = ConvertType(Arg->getType()); |
2796 | | |
2797 | | // Prepare parameter attributes. So far, only attributes for pointer |
2798 | | // parameters are prepared. See |
2799 | | // http://llvm.org/docs/LangRef.html#paramattrs. |
2800 | 516k | if (ArgI.getDirectOffset() == 0 && LTy->isPointerTy()516k && |
2801 | 516k | ArgI.getCoerceToType()->isPointerTy()277k ) { |
2802 | 277k | assert(NumIRArgs == 1); |
2803 | | |
2804 | 277k | if (const ParmVarDecl *PVD = dyn_cast<ParmVarDecl>(Arg)) { |
2805 | | // Set `nonnull` attribute if any. |
2806 | 114k | if (getNonNullAttr(CurCodeDecl, PVD, PVD->getType(), |
2807 | 114k | PVD->getFunctionScopeIndex()) && |
2808 | 114k | !CGM.getCodeGenOpts().NullPointerIsValid22 ) |
2809 | 13 | AI->addAttr(llvm::Attribute::NonNull); |
2810 | | |
2811 | 114k | QualType OTy = PVD->getOriginalType(); |
2812 | 114k | if (const auto *ArrTy = |
2813 | 114k | getContext().getAsConstantArrayType(OTy)) { |
2814 | | // A C99 array parameter declaration with the static keyword also |
2815 | | // indicates dereferenceability, and if the size is constant we can |
2816 | | // use the dereferenceable attribute (which requires the size in |
2817 | | // bytes). |
2818 | 578 | if (ArrTy->getSizeModifier() == ArrayType::Static) { |
2819 | 5 | QualType ETy = ArrTy->getElementType(); |
2820 | 5 | llvm::Align Alignment = |
2821 | 5 | CGM.getNaturalTypeAlignment(ETy).getAsAlign(); |
2822 | 5 | AI->addAttrs(llvm::AttrBuilder(getLLVMContext()).addAlignmentAttr(Alignment)); |
2823 | 5 | uint64_t ArrSize = ArrTy->getSize().getZExtValue(); |
2824 | 5 | if (!ETy->isIncompleteType() && ETy->isConstantSizeType() && |
2825 | 5 | ArrSize) { |
2826 | 3 | llvm::AttrBuilder Attrs(getLLVMContext()); |
2827 | 3 | Attrs.addDereferenceableAttr( |
2828 | 3 | getContext().getTypeSizeInChars(ETy).getQuantity() * |
2829 | 3 | ArrSize); |
2830 | 3 | AI->addAttrs(Attrs); |
2831 | 3 | } else if (2 getContext().getTargetInfo().getNullPointerValue( |
2832 | 2 | ETy.getAddressSpace()) == 0 && |
2833 | 2 | !CGM.getCodeGenOpts().NullPointerIsValid) { |
2834 | 1 | AI->addAttr(llvm::Attribute::NonNull); |
2835 | 1 | } |
2836 | 5 | } |
2837 | 113k | } else if (const auto *ArrTy = |
2838 | 113k | getContext().getAsVariableArrayType(OTy)) { |
2839 | | // For C99 VLAs with the static keyword, we don't know the size so |
2840 | | // we can't use the dereferenceable attribute, but in addrspace(0) |
2841 | | // we know that it must be nonnull. |
2842 | 102 | if (ArrTy->getSizeModifier() == VariableArrayType::Static) { |
2843 | 2 | QualType ETy = ArrTy->getElementType(); |
2844 | 2 | llvm::Align Alignment = |
2845 | 2 | CGM.getNaturalTypeAlignment(ETy).getAsAlign(); |
2846 | 2 | AI->addAttrs(llvm::AttrBuilder(getLLVMContext()).addAlignmentAttr(Alignment)); |
2847 | 2 | if (!getContext().getTargetAddressSpace(ETy) && |
2848 | 2 | !CGM.getCodeGenOpts().NullPointerIsValid) |
2849 | 1 | AI->addAttr(llvm::Attribute::NonNull); |
2850 | 2 | } |
2851 | 102 | } |
2852 | | |
2853 | | // Set `align` attribute if any. |
2854 | 114k | const auto *AVAttr = PVD->getAttr<AlignValueAttr>(); |
2855 | 114k | if (!AVAttr) |
2856 | 114k | if (const auto *TOTy = dyn_cast<TypedefType>(OTy)) |
2857 | 6.99k | AVAttr = TOTy->getDecl()->getAttr<AlignValueAttr>(); |
2858 | 114k | if (AVAttr && !SanOpts.has(SanitizerKind::Alignment)11 ) { |
2859 | | // If alignment-assumption sanitizer is enabled, we do *not* add |
2860 | | // alignment attribute here, but emit normal alignment assumption, |
2861 | | // so the UBSAN check could function. |
2862 | 8 | llvm::ConstantInt *AlignmentCI = |
2863 | 8 | cast<llvm::ConstantInt>(EmitScalarExpr(AVAttr->getAlignment())); |
2864 | 8 | uint64_t AlignmentInt = |
2865 | 8 | AlignmentCI->getLimitedValue(llvm::Value::MaximumAlignment); |
2866 | 8 | if (AI->getParamAlign().valueOrOne() < AlignmentInt) { |
2867 | 8 | AI->removeAttr(llvm::Attribute::AttrKind::Alignment); |
2868 | 8 | AI->addAttrs(llvm::AttrBuilder(getLLVMContext()).addAlignmentAttr( |
2869 | 8 | llvm::Align(AlignmentInt))); |
2870 | 8 | } |
2871 | 8 | } |
2872 | 114k | } |
2873 | | |
2874 | | // Set 'noalias' if an argument type has the `restrict` qualifier. |
2875 | 277k | if (Arg->getType().isRestrictQualified()) |
2876 | 32.0k | AI->addAttr(llvm::Attribute::NoAlias); |
2877 | 277k | } |
2878 | | |
2879 | | // Prepare the argument value. If we have the trivial case, handle it |
2880 | | // with no muss and fuss. |
2881 | 516k | if (!isa<llvm::StructType>(ArgI.getCoerceToType()) && |
2882 | 516k | ArgI.getCoerceToType() == ConvertType(Ty)515k && |
2883 | 516k | ArgI.getDirectOffset() == 0510k ) { |
2884 | 510k | assert(NumIRArgs == 1); |
2885 | | |
2886 | | // LLVM expects swifterror parameters to be used in very restricted |
2887 | | // ways. Copy the value into a less-restricted temporary. |
2888 | 0 | llvm::Value *V = AI; |
2889 | 510k | if (FI.getExtParameterInfo(ArgNo).getABI() |
2890 | 510k | == ParameterABI::SwiftErrorResult) { |
2891 | 16 | QualType pointeeTy = Ty->getPointeeType(); |
2892 | 16 | assert(pointeeTy->isPointerType()); |
2893 | 0 | Address temp = |
2894 | 16 | CreateMemTemp(pointeeTy, getPointerAlign(), "swifterror.temp"); |
2895 | 16 | Address arg(V, ConvertTypeForMem(pointeeTy), |
2896 | 16 | getContext().getTypeAlignInChars(pointeeTy)); |
2897 | 16 | llvm::Value *incomingErrorValue = Builder.CreateLoad(arg); |
2898 | 16 | Builder.CreateStore(incomingErrorValue, temp); |
2899 | 16 | V = temp.getPointer(); |
2900 | | |
2901 | | // Push a cleanup to copy the value back at the end of the function. |
2902 | | // The convention does not guarantee that the value will be written |
2903 | | // back if the function exits with an unwind exception. |
2904 | 16 | EHStack.pushCleanup<CopyBackSwiftError>(NormalCleanup, temp, arg); |
2905 | 16 | } |
2906 | | |
2907 | | // Ensure the argument is the correct type. |
2908 | 510k | if (V->getType() != ArgI.getCoerceToType()) |
2909 | 0 | V = Builder.CreateBitCast(V, ArgI.getCoerceToType()); |
2910 | | |
2911 | 510k | if (isPromoted) |
2912 | 14 | V = emitArgumentDemotion(*this, Arg, V); |
2913 | | |
2914 | | // Because of merging of function types from multiple decls it is |
2915 | | // possible for the type of an argument to not match the corresponding |
2916 | | // type in the function type. Since we are codegening the callee |
2917 | | // in here, add a cast to the argument type. |
2918 | 510k | llvm::Type *LTy = ConvertType(Arg->getType()); |
2919 | 510k | if (V->getType() != LTy) |
2920 | 0 | V = Builder.CreateBitCast(V, LTy); |
2921 | | |
2922 | 510k | ArgVals.push_back(ParamValue::forDirect(V)); |
2923 | 510k | break; |
2924 | 510k | } |
2925 | | |
2926 | | // VLST arguments are coerced to VLATs at the function boundary for |
2927 | | // ABI consistency. If this is a VLST that was coerced to |
2928 | | // a VLAT at the function boundary and the types match up, use |
2929 | | // llvm.experimental.vector.extract to convert back to the original |
2930 | | // VLST. |
2931 | 5.89k | if (auto *VecTyTo = dyn_cast<llvm::FixedVectorType>(ConvertType(Ty))) { |
2932 | 1.04k | llvm::Value *Coerced = Fn->getArg(FirstIRArg); |
2933 | 1.04k | if (auto *VecTyFrom = |
2934 | 1.04k | dyn_cast<llvm::ScalableVectorType>(Coerced->getType())) { |
2935 | | // If we are casting a scalable 16 x i1 predicate vector to a fixed i8 |
2936 | | // vector, bitcast the source and use a vector extract. |
2937 | 36 | auto PredType = |
2938 | 36 | llvm::ScalableVectorType::get(Builder.getInt1Ty(), 16); |
2939 | 36 | if (VecTyFrom == PredType && |
2940 | 36 | VecTyTo->getElementType() == Builder.getInt8Ty()4 ) { |
2941 | 4 | VecTyFrom = llvm::ScalableVectorType::get(Builder.getInt8Ty(), 2); |
2942 | 4 | Coerced = Builder.CreateBitCast(Coerced, VecTyFrom); |
2943 | 4 | } |
2944 | 36 | if (VecTyFrom->getElementType() == VecTyTo->getElementType()) { |
2945 | 36 | llvm::Value *Zero = llvm::Constant::getNullValue(CGM.Int64Ty); |
2946 | | |
2947 | 36 | assert(NumIRArgs == 1); |
2948 | 0 | Coerced->setName(Arg->getName() + ".coerce"); |
2949 | 36 | ArgVals.push_back(ParamValue::forDirect(Builder.CreateExtractVector( |
2950 | 36 | VecTyTo, Coerced, Zero, "castFixedSve"))); |
2951 | 36 | break; |
2952 | 36 | } |
2953 | 36 | } |
2954 | 1.04k | } |
2955 | | |
2956 | 5.86k | Address Alloca = CreateMemTemp(Ty, getContext().getDeclAlign(Arg), |
2957 | 5.86k | Arg->getName()); |
2958 | | |
2959 | | // Pointer to store into. |
2960 | 5.86k | Address Ptr = emitAddressAtOffset(*this, Alloca, ArgI); |
2961 | | |
2962 | | // Fast-isel and the optimizer generally like scalar values better than |
2963 | | // FCAs, so we flatten them if this is safe to do for this argument. |
2964 | 5.86k | llvm::StructType *STy = dyn_cast<llvm::StructType>(ArgI.getCoerceToType()); |
2965 | 5.86k | if (ArgI.isDirect() && ArgI.getCanBeFlattened() && STy5.59k && |
2966 | 5.86k | STy->getNumElements() > 11.00k ) { |
2967 | 977 | uint64_t SrcSize = CGM.getDataLayout().getTypeAllocSize(STy); |
2968 | 977 | llvm::Type *DstTy = Ptr.getElementType(); |
2969 | 977 | uint64_t DstSize = CGM.getDataLayout().getTypeAllocSize(DstTy); |
2970 | | |
2971 | 977 | Address AddrToStoreInto = Address::invalid(); |
2972 | 977 | if (SrcSize <= DstSize) { |
2973 | 888 | AddrToStoreInto = Builder.CreateElementBitCast(Ptr, STy); |
2974 | 888 | } else { |
2975 | 89 | AddrToStoreInto = |
2976 | 89 | CreateTempAlloca(STy, Alloca.getAlignment(), "coerce"); |
2977 | 89 | } |
2978 | | |
2979 | 977 | assert(STy->getNumElements() == NumIRArgs); |
2980 | 2.99k | for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i2.01k ) { |
2981 | 2.01k | auto AI = Fn->getArg(FirstIRArg + i); |
2982 | 2.01k | AI->setName(Arg->getName() + ".coerce" + Twine(i)); |
2983 | 2.01k | Address EltPtr = Builder.CreateStructGEP(AddrToStoreInto, i); |
2984 | 2.01k | Builder.CreateStore(AI, EltPtr); |
2985 | 2.01k | } |
2986 | | |
2987 | 977 | if (SrcSize > DstSize) { |
2988 | 89 | Builder.CreateMemCpy(Ptr, AddrToStoreInto, DstSize); |
2989 | 89 | } |
2990 | | |
2991 | 4.88k | } else { |
2992 | | // Simple case, just do a coerced store of the argument into the alloca. |
2993 | 4.88k | assert(NumIRArgs == 1); |
2994 | 0 | auto AI = Fn->getArg(FirstIRArg); |
2995 | 4.88k | AI->setName(Arg->getName() + ".coerce"); |
2996 | 4.88k | CreateCoercedStore(AI, Ptr, /*DstIsVolatile=*/false, *this); |
2997 | 4.88k | } |
2998 | | |
2999 | | // Match to what EmitParmDecl is expecting for this type. |
3000 | 5.86k | if (CodeGenFunction::hasScalarEvaluationKind(Ty)) { |
3001 | 1.24k | llvm::Value *V = |
3002 | 1.24k | EmitLoadOfScalar(Alloca, false, Ty, Arg->getBeginLoc()); |
3003 | 1.24k | if (isPromoted) |
3004 | 0 | V = emitArgumentDemotion(*this, Arg, V); |
3005 | 1.24k | ArgVals.push_back(ParamValue::forDirect(V)); |
3006 | 4.61k | } else { |
3007 | 4.61k | ArgVals.push_back(ParamValue::forIndirect(Alloca)); |
3008 | 4.61k | } |
3009 | 5.86k | break; |
3010 | 5.89k | } |
3011 | | |
3012 | 571 | case ABIArgInfo::CoerceAndExpand: { |
3013 | | // Reconstruct into a temporary. |
3014 | 571 | Address alloca = CreateMemTemp(Ty, getContext().getDeclAlign(Arg)); |
3015 | 571 | ArgVals.push_back(ParamValue::forIndirect(alloca)); |
3016 | | |
3017 | 571 | auto coercionType = ArgI.getCoerceAndExpandType(); |
3018 | 571 | alloca = Builder.CreateElementBitCast(alloca, coercionType); |
3019 | | |
3020 | 571 | unsigned argIndex = FirstIRArg; |
3021 | 1.78k | for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i1.21k ) { |
3022 | 1.21k | llvm::Type *eltType = coercionType->getElementType(i); |
3023 | 1.21k | if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType)) |
3024 | 13 | continue; |
3025 | | |
3026 | 1.19k | auto eltAddr = Builder.CreateStructGEP(alloca, i); |
3027 | 1.19k | auto elt = Fn->getArg(argIndex++); |
3028 | 1.19k | Builder.CreateStore(elt, eltAddr); |
3029 | 1.19k | } |
3030 | 571 | assert(argIndex == FirstIRArg + NumIRArgs); |
3031 | 0 | break; |
3032 | 5.89k | } |
3033 | | |
3034 | 102 | case ABIArgInfo::Expand: { |
3035 | | // If this structure was expanded into multiple arguments then |
3036 | | // we need to create a temporary and reconstruct it from the |
3037 | | // arguments. |
3038 | 102 | Address Alloca = CreateMemTemp(Ty, getContext().getDeclAlign(Arg)); |
3039 | 102 | LValue LV = MakeAddrLValue(Alloca, Ty); |
3040 | 102 | ArgVals.push_back(ParamValue::forIndirect(Alloca)); |
3041 | | |
3042 | 102 | auto FnArgIter = Fn->arg_begin() + FirstIRArg; |
3043 | 102 | ExpandTypeFromArgs(Ty, LV, FnArgIter); |
3044 | 102 | assert(FnArgIter == Fn->arg_begin() + FirstIRArg + NumIRArgs); |
3045 | 345 | for (unsigned i = 0, e = NumIRArgs; i != e; ++i243 ) { |
3046 | 243 | auto AI = Fn->getArg(FirstIRArg + i); |
3047 | 243 | AI->setName(Arg->getName() + "." + Twine(i)); |
3048 | 243 | } |
3049 | 102 | break; |
3050 | 5.89k | } |
3051 | | |
3052 | 6.79k | case ABIArgInfo::Ignore: |
3053 | 6.79k | assert(NumIRArgs == 0); |
3054 | | // Initialize the local variable appropriately. |
3055 | 6.79k | if (!hasScalarEvaluationKind(Ty)) { |
3056 | 6.79k | ArgVals.push_back(ParamValue::forIndirect(CreateMemTemp(Ty))); |
3057 | 6.79k | } else { |
3058 | 0 | llvm::Value *U = llvm::UndefValue::get(ConvertType(Arg->getType())); |
3059 | 0 | ArgVals.push_back(ParamValue::forDirect(U)); |
3060 | 0 | } |
3061 | 6.79k | break; |
3062 | 526k | } |
3063 | 526k | } |
3064 | | |
3065 | 314k | if (getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) { |
3066 | 31.7k | for (int I = Args.size() - 1; I >= 0; --I18.2k ) |
3067 | 18.2k | EmitParmDecl(*Args[I], ArgVals[I], I + 1); |
3068 | 301k | } else { |
3069 | 809k | for (unsigned I = 0, E = Args.size(); I != E; ++I508k ) |
3070 | 508k | EmitParmDecl(*Args[I], ArgVals[I], I + 1); |
3071 | 301k | } |
3072 | 314k | } |
3073 | | |
3074 | 8 | static void eraseUnusedBitCasts(llvm::Instruction *insn) { |
3075 | 12 | while (insn->use_empty()) { |
3076 | 12 | llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(insn); |
3077 | 12 | if (!bitcast) return8 ; |
3078 | | |
3079 | | // This is "safe" because we would have used a ConstantExpr otherwise. |
3080 | 4 | insn = cast<llvm::Instruction>(bitcast->getOperand(0)); |
3081 | 4 | bitcast->eraseFromParent(); |
3082 | 4 | } |
3083 | 8 | } |
3084 | | |
3085 | | /// Try to emit a fused autorelease of a return result. |
3086 | | static llvm::Value *tryEmitFusedAutoreleaseOfResult(CodeGenFunction &CGF, |
3087 | 87 | llvm::Value *result) { |
3088 | | // We must be immediately followed the cast. |
3089 | 87 | llvm::BasicBlock *BB = CGF.Builder.GetInsertBlock(); |
3090 | 87 | if (BB->empty()) return nullptr0 ; |
3091 | 87 | if (&BB->back() != result) return nullptr24 ; |
3092 | | |
3093 | 63 | llvm::Type *resultType = result->getType(); |
3094 | | |
3095 | | // result is in a BasicBlock and is therefore an Instruction. |
3096 | 63 | llvm::Instruction *generator = cast<llvm::Instruction>(result); |
3097 | | |
3098 | 63 | SmallVector<llvm::Instruction *, 4> InstsToKill; |
3099 | | |
3100 | | // Look for: |
3101 | | // %generator = bitcast %type1* %generator2 to %type2* |
3102 | 94 | while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(generator)) { |
3103 | | // We would have emitted this as a constant if the operand weren't |
3104 | | // an Instruction. |
3105 | 31 | generator = cast<llvm::Instruction>(bitcast->getOperand(0)); |
3106 | | |
3107 | | // Require the generator to be immediately followed by the cast. |
3108 | 31 | if (generator->getNextNode() != bitcast) |
3109 | 0 | return nullptr; |
3110 | | |
3111 | 31 | InstsToKill.push_back(bitcast); |
3112 | 31 | } |
3113 | | |
3114 | | // Look for: |
3115 | | // %generator = call i8* @objc_retain(i8* %originalResult) |
3116 | | // or |
3117 | | // %generator = call i8* @objc_retainAutoreleasedReturnValue(i8* %originalResult) |
3118 | 63 | llvm::CallInst *call = dyn_cast<llvm::CallInst>(generator); |
3119 | 63 | if (!call) return nullptr13 ; |
3120 | | |
3121 | 50 | bool doRetainAutorelease; |
3122 | | |
3123 | 50 | if (call->getCalledOperand() == CGF.CGM.getObjCEntrypoints().objc_retain) { |
3124 | 8 | doRetainAutorelease = true; |
3125 | 42 | } else if (call->getCalledOperand() == |
3126 | 42 | CGF.CGM.getObjCEntrypoints().objc_retainAutoreleasedReturnValue) { |
3127 | 19 | doRetainAutorelease = false; |
3128 | | |
3129 | | // If we emitted an assembly marker for this call (and the |
3130 | | // ARCEntrypoints field should have been set if so), go looking |
3131 | | // for that call. If we can't find it, we can't do this |
3132 | | // optimization. But it should always be the immediately previous |
3133 | | // instruction, unless we needed bitcasts around the call. |
3134 | 19 | if (CGF.CGM.getObjCEntrypoints().retainAutoreleasedReturnValueMarker) { |
3135 | 12 | llvm::Instruction *prev = call->getPrevNode(); |
3136 | 12 | assert(prev); |
3137 | 12 | if (isa<llvm::BitCastInst>(prev)) { |
3138 | 8 | prev = prev->getPrevNode(); |
3139 | 8 | assert(prev); |
3140 | 8 | } |
3141 | 0 | assert(isa<llvm::CallInst>(prev)); |
3142 | 0 | assert(cast<llvm::CallInst>(prev)->getCalledOperand() == |
3143 | 12 | CGF.CGM.getObjCEntrypoints().retainAutoreleasedReturnValueMarker); |
3144 | 0 | InstsToKill.push_back(prev); |
3145 | 12 | } |
3146 | 23 | } else { |
3147 | 23 | return nullptr; |
3148 | 23 | } |
3149 | | |
3150 | 27 | result = call->getArgOperand(0); |
3151 | 27 | InstsToKill.push_back(call); |
3152 | | |
3153 | | // Keep killing bitcasts, for sanity. Note that we no longer care |
3154 | | // about precise ordering as long as there's exactly one use. |
3155 | 42 | while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(result)) { |
3156 | 15 | if (!bitcast->hasOneUse()) break0 ; |
3157 | 15 | InstsToKill.push_back(bitcast); |
3158 | 15 | result = bitcast->getOperand(0); |
3159 | 15 | } |
3160 | | |
3161 | | // Delete all the unnecessary instructions, from latest to earliest. |
3162 | 27 | for (auto *I : InstsToKill) |
3163 | 75 | I->eraseFromParent(); |
3164 | | |
3165 | | // Do the fused retain/autorelease if we were asked to. |
3166 | 27 | if (doRetainAutorelease) |
3167 | 8 | result = CGF.EmitARCRetainAutoreleaseReturnValue(result); |
3168 | | |
3169 | | // Cast back to the result type. |
3170 | 27 | return CGF.Builder.CreateBitCast(result, resultType); |
3171 | 50 | } |
3172 | | |
3173 | | /// If this is a +1 of the value of an immutable 'self', remove it. |
3174 | | static llvm::Value *tryRemoveRetainOfSelf(CodeGenFunction &CGF, |
3175 | 122 | llvm::Value *result) { |
3176 | | // This is only applicable to a method with an immutable 'self'. |
3177 | 122 | const ObjCMethodDecl *method = |
3178 | 122 | dyn_cast_or_null<ObjCMethodDecl>(CGF.CurCodeDecl); |
3179 | 122 | if (!method) return nullptr103 ; |
3180 | 19 | const VarDecl *self = method->getSelfDecl(); |
3181 | 19 | if (!self->getType().isConstQualified()) return nullptr0 ; |
3182 | | |
3183 | | // Look for a retain call. |
3184 | 19 | llvm::CallInst *retainCall = |
3185 | 19 | dyn_cast<llvm::CallInst>(result->stripPointerCasts()); |
3186 | 19 | if (!retainCall || retainCall->getCalledOperand() != |
3187 | 15 | CGF.CGM.getObjCEntrypoints().objc_retain) |
3188 | 12 | return nullptr; |
3189 | | |
3190 | | // Look for an ordinary load of 'self'. |
3191 | 7 | llvm::Value *retainedValue = retainCall->getArgOperand(0); |
3192 | 7 | llvm::LoadInst *load = |
3193 | 7 | dyn_cast<llvm::LoadInst>(retainedValue->stripPointerCasts()); |
3194 | 7 | if (!load || load->isAtomic() || load->isVolatile() || |
3195 | 7 | load->getPointerOperand() != CGF.GetAddrOfLocalVar(self).getPointer()) |
3196 | 3 | return nullptr; |
3197 | | |
3198 | | // Okay! Burn it all down. This relies for correctness on the |
3199 | | // assumption that the retain is emitted as part of the return and |
3200 | | // that thereafter everything is used "linearly". |
3201 | 4 | llvm::Type *resultType = result->getType(); |
3202 | 4 | eraseUnusedBitCasts(cast<llvm::Instruction>(result)); |
3203 | 4 | assert(retainCall->use_empty()); |
3204 | 0 | retainCall->eraseFromParent(); |
3205 | 4 | eraseUnusedBitCasts(cast<llvm::Instruction>(retainedValue)); |
3206 | | |
3207 | 4 | return CGF.Builder.CreateBitCast(load, resultType); |
3208 | 7 | } |
3209 | | |
3210 | | /// Emit an ARC autorelease of the result of a function. |
3211 | | /// |
3212 | | /// \return the value to actually return from the function |
3213 | | static llvm::Value *emitAutoreleaseOfResult(CodeGenFunction &CGF, |
3214 | 122 | llvm::Value *result) { |
3215 | | // If we're returning 'self', kill the initial retain. This is a |
3216 | | // heuristic attempt to "encourage correctness" in the really unfortunate |
3217 | | // case where we have a return of self during a dealloc and we desperately |
3218 | | // need to avoid the possible autorelease. |
3219 | 122 | if (llvm::Value *self = tryRemoveRetainOfSelf(CGF, result)) |
3220 | 4 | return self; |
3221 | | |
3222 | | // At -O0, try to emit a fused retain/autorelease. |
3223 | 118 | if (CGF.shouldUseFusedARCCalls()) |
3224 | 87 | if (llvm::Value *fused = tryEmitFusedAutoreleaseOfResult(CGF, result)) |
3225 | 27 | return fused; |
3226 | | |
3227 | 91 | return CGF.EmitARCAutoreleaseReturnValue(result); |
3228 | 118 | } |
3229 | | |
3230 | | /// Heuristically search for a dominating store to the return-value slot. |
3231 | 144k | static llvm::StoreInst *findDominatingStoreToReturnValue(CodeGenFunction &CGF) { |
3232 | | // Check if a User is a store which pointerOperand is the ReturnValue. |
3233 | | // We are looking for stores to the ReturnValue, not for stores of the |
3234 | | // ReturnValue to some other location. |
3235 | 144k | auto GetStoreIfValid = [&CGF](llvm::User *U) -> llvm::StoreInst * { |
3236 | 142k | auto *SI = dyn_cast<llvm::StoreInst>(U); |
3237 | 142k | if (!SI || SI->getPointerOperand() != CGF.ReturnValue.getPointer()140k || |
3238 | 142k | SI->getValueOperand()->getType() != CGF.ReturnValue.getElementType()139k ) |
3239 | 2.18k | return nullptr; |
3240 | | // These aren't actually possible for non-coerced returns, and we |
3241 | | // only care about non-coerced returns on this code path. |
3242 | 139k | assert(!SI->isAtomic() && !SI->isVolatile()); |
3243 | 0 | return SI; |
3244 | 142k | }; |
3245 | | // If there are multiple uses of the return-value slot, just check |
3246 | | // for something immediately preceding the IP. Sometimes this can |
3247 | | // happen with how we generate implicit-returns; it can also happen |
3248 | | // with noreturn cleanups. |
3249 | 144k | if (!CGF.ReturnValue.getPointer()->hasOneUse()) { |
3250 | 7.38k | llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock(); |
3251 | 7.38k | if (IP->empty()) return nullptr1.88k ; |
3252 | | |
3253 | | // Look at directly preceding instruction, skipping bitcasts and lifetime |
3254 | | // markers. |
3255 | 5.70k | for (llvm::Instruction &I : make_range(IP->rbegin(), IP->rend()))5.49k { |
3256 | 5.70k | if (isa<llvm::BitCastInst>(&I)) |
3257 | 152 | continue; |
3258 | 5.55k | if (auto *II = dyn_cast<llvm::IntrinsicInst>(&I)) |
3259 | 350 | if (II->getIntrinsicID() == llvm::Intrinsic::lifetime_end) |
3260 | 95 | continue; |
3261 | | |
3262 | 5.45k | return GetStoreIfValid(&I); |
3263 | 5.55k | } |
3264 | 41 | return nullptr; |
3265 | 5.49k | } |
3266 | | |
3267 | 136k | llvm::StoreInst *store = |
3268 | 136k | GetStoreIfValid(CGF.ReturnValue.getPointer()->user_back()); |
3269 | 136k | if (!store) return nullptr512 ; |
3270 | | |
3271 | | // Now do a first-and-dirty dominance check: just walk up the |
3272 | | // single-predecessors chain from the current insertion point. |
3273 | 136k | llvm::BasicBlock *StoreBB = store->getParent(); |
3274 | 136k | llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock(); |
3275 | 136k | while (IP != StoreBB) { |
3276 | 2.00k | if (!(IP = IP->getSinglePredecessor())) |
3277 | 1.26k | return nullptr; |
3278 | 2.00k | } |
3279 | | |
3280 | | // Okay, the store's basic block dominates the insertion point; we |
3281 | | // can do our thing. |
3282 | 134k | return store; |
3283 | 136k | } |
3284 | | |
3285 | | // Helper functions for EmitCMSEClearRecord |
3286 | | |
3287 | | // Set the bits corresponding to a field having width `BitWidth` and located at |
3288 | | // offset `BitOffset` (from the least significant bit) within a storage unit of |
3289 | | // `Bits.size()` bytes. Each element of `Bits` corresponds to one target byte. |
3290 | | // Use little-endian layout, i.e.`Bits[0]` is the LSB. |
3291 | | static void setBitRange(SmallVectorImpl<uint64_t> &Bits, int BitOffset, |
3292 | 125 | int BitWidth, int CharWidth) { |
3293 | 125 | assert(CharWidth <= 64); |
3294 | 0 | assert(static_cast<unsigned>(BitWidth) <= Bits.size() * CharWidth); |
3295 | | |
3296 | 0 | int Pos = 0; |
3297 | 125 | if (BitOffset >= CharWidth) { |
3298 | 39 | Pos += BitOffset / CharWidth; |
3299 | 39 | BitOffset = BitOffset % CharWidth; |
3300 | 39 | } |
3301 | | |
3302 | 125 | const uint64_t Used = (uint64_t(1) << CharWidth) - 1; |
3303 | 125 | if (BitOffset + BitWidth >= CharWidth) { |
3304 | 65 | Bits[Pos++] |= (Used << BitOffset) & Used; |
3305 | 65 | BitWidth -= CharWidth - BitOffset; |
3306 | 65 | BitOffset = 0; |
3307 | 65 | } |
3308 | | |
3309 | 143 | while (BitWidth >= CharWidth) { |
3310 | 18 | Bits[Pos++] = Used; |
3311 | 18 | BitWidth -= CharWidth; |
3312 | 18 | } |
3313 | | |
3314 | 125 | if (BitWidth > 0) |
3315 | 92 | Bits[Pos++] |= (Used >> (CharWidth - BitWidth)) << BitOffset; |
3316 | 125 | } |
3317 | | |
3318 | | // Set the bits corresponding to a field having width `BitWidth` and located at |
3319 | | // offset `BitOffset` (from the least significant bit) within a storage unit of |
3320 | | // `StorageSize` bytes, located at `StorageOffset` in `Bits`. Each element of |
3321 | | // `Bits` corresponds to one target byte. Use target endian layout. |
3322 | | static void setBitRange(SmallVectorImpl<uint64_t> &Bits, int StorageOffset, |
3323 | | int StorageSize, int BitOffset, int BitWidth, |
3324 | 125 | int CharWidth, bool BigEndian) { |
3325 | | |
3326 | 125 | SmallVector<uint64_t, 8> TmpBits(StorageSize); |
3327 | 125 | setBitRange(TmpBits, BitOffset, BitWidth, CharWidth); |
3328 | | |
3329 | 125 | if (BigEndian) |
3330 | 50 | std::reverse(TmpBits.begin(), TmpBits.end()); |
3331 | | |
3332 | 125 | for (uint64_t V : TmpBits) |
3333 | 350 | Bits[StorageOffset++] |= V; |
3334 | 125 | } |
3335 | | |
3336 | | static void setUsedBits(CodeGenModule &, QualType, int, |
3337 | | SmallVectorImpl<uint64_t> &); |
3338 | | |
3339 | | // Set the bits in `Bits`, which correspond to the value representations of |
3340 | | // the actual members of the record type `RTy`. Note that this function does |
3341 | | // not handle base classes, virtual tables, etc, since they cannot happen in |
3342 | | // CMSE function arguments or return. The bit mask corresponds to the target |
3343 | | // memory layout, i.e. it's endian dependent. |
3344 | | static void setUsedBits(CodeGenModule &CGM, const RecordType *RTy, int Offset, |
3345 | 153 | SmallVectorImpl<uint64_t> &Bits) { |
3346 | 153 | ASTContext &Context = CGM.getContext(); |
3347 | 153 | int CharWidth = Context.getCharWidth(); |
3348 | 153 | const RecordDecl *RD = RTy->getDecl()->getDefinition(); |
3349 | 153 | const ASTRecordLayout &ASTLayout = Context.getASTRecordLayout(RD); |
3350 | 153 | const CGRecordLayout &Layout = CGM.getTypes().getCGRecordLayout(RD); |
3351 | | |
3352 | 153 | int Idx = 0; |
3353 | 566 | for (auto I = RD->field_begin(), E = RD->field_end(); I != E; ++I, ++Idx413 ) { |
3354 | 413 | const FieldDecl *F = *I; |
3355 | | |
3356 | 413 | if (F->isUnnamedBitfield() || F->isZeroLengthBitField(Context)283 || |
3357 | 413 | F->getType()->isIncompleteArrayType()283 ) |
3358 | 140 | continue; |
3359 | | |
3360 | 273 | if (F->isBitField()) { |
3361 | 125 | const CGBitFieldInfo &BFI = Layout.getBitFieldInfo(F); |
3362 | 125 | setBitRange(Bits, Offset + BFI.StorageOffset.getQuantity(), |
3363 | 125 | BFI.StorageSize / CharWidth, BFI.Offset, |
3364 | 125 | BFI.Size, CharWidth, |
3365 | 125 | CGM.getDataLayout().isBigEndian()); |
3366 | 125 | continue; |
3367 | 125 | } |
3368 | | |
3369 | 148 | setUsedBits(CGM, F->getType(), |
3370 | 148 | Offset + ASTLayout.getFieldOffset(Idx) / CharWidth, Bits); |
3371 | 148 | } |
3372 | 153 | } |
3373 | | |
3374 | | // Set the bits in `Bits`, which correspond to the value representations of |
3375 | | // the elements of an array type `ATy`. |
3376 | | static void setUsedBits(CodeGenModule &CGM, const ConstantArrayType *ATy, |
3377 | 34 | int Offset, SmallVectorImpl<uint64_t> &Bits) { |
3378 | 34 | const ASTContext &Context = CGM.getContext(); |
3379 | | |
3380 | 34 | QualType ETy = Context.getBaseElementType(ATy); |
3381 | 34 | int Size = Context.getTypeSizeInChars(ETy).getQuantity(); |
3382 | 34 | SmallVector<uint64_t, 4> TmpBits(Size); |
3383 | 34 | setUsedBits(CGM, ETy, 0, TmpBits); |
3384 | | |
3385 | 142 | for (int I = 0, N = Context.getConstantArrayElementCount(ATy); I < N; ++I108 ) { |
3386 | 108 | auto Src = TmpBits.begin(); |
3387 | 108 | auto Dst = Bits.begin() + Offset + I * Size; |
3388 | 270 | for (int J = 0; J < Size; ++J162 ) |
3389 | 162 | *Dst++ |= *Src++; |
3390 | 108 | } |
3391 | 34 | } |
3392 | | |
3393 | | // Set the bits in `Bits`, which correspond to the value representations of |
3394 | | // the type `QTy`. |
3395 | | static void setUsedBits(CodeGenModule &CGM, QualType QTy, int Offset, |
3396 | 182 | SmallVectorImpl<uint64_t> &Bits) { |
3397 | 182 | if (const auto *RTy = QTy->getAs<RecordType>()) |
3398 | 30 | return setUsedBits(CGM, RTy, Offset, Bits); |
3399 | | |
3400 | 152 | ASTContext &Context = CGM.getContext(); |
3401 | 152 | if (const auto *ATy = Context.getAsConstantArrayType(QTy)) |
3402 | 34 | return setUsedBits(CGM, ATy, Offset, Bits); |
3403 | | |
3404 | 118 | int Size = Context.getTypeSizeInChars(QTy).getQuantity(); |
3405 | 118 | if (Size <= 0) |
3406 | 0 | return; |
3407 | | |
3408 | 118 | std::fill_n(Bits.begin() + Offset, Size, |
3409 | 118 | (uint64_t(1) << Context.getCharWidth()) - 1); |
3410 | 118 | } |
3411 | | |
3412 | | static uint64_t buildMultiCharMask(const SmallVectorImpl<uint64_t> &Bits, |
3413 | | int Pos, int Size, int CharWidth, |
3414 | 152 | bool BigEndian) { |
3415 | 152 | assert(Size > 0); |
3416 | 0 | uint64_t Mask = 0; |
3417 | 152 | if (BigEndian) { |
3418 | 318 | for (auto P = Bits.begin() + Pos, E = Bits.begin() + Pos + Size; P != E; |
3419 | 256 | ++P) |
3420 | 256 | Mask = (Mask << CharWidth) | *P; |
3421 | 90 | } else { |
3422 | 90 | auto P = Bits.begin() + Pos + Size, End = Bits.begin() + Pos; |
3423 | 90 | do |
3424 | 372 | Mask = (Mask << CharWidth) | *--P; |
3425 | 372 | while (P != End); |
3426 | 90 | } |
3427 | 152 | return Mask; |
3428 | 152 | } |
3429 | | |
3430 | | // Emit code to clear the bits in a record, which aren't a part of any user |
3431 | | // declared member, when the record is a function return. |
3432 | | llvm::Value *CodeGenFunction::EmitCMSEClearRecord(llvm::Value *Src, |
3433 | | llvm::IntegerType *ITy, |
3434 | 84 | QualType QTy) { |
3435 | 84 | assert(Src->getType() == ITy); |
3436 | 0 | assert(ITy->getScalarSizeInBits() <= 64); |
3437 | | |
3438 | 0 | const llvm::DataLayout &DataLayout = CGM.getDataLayout(); |
3439 | 84 | int Size = DataLayout.getTypeStoreSize(ITy); |
3440 | 84 | SmallVector<uint64_t, 4> Bits(Size); |
3441 | 84 | setUsedBits(CGM, QTy->castAs<RecordType>(), 0, Bits); |
3442 | | |
3443 | 84 | int CharWidth = CGM.getContext().getCharWidth(); |
3444 | 84 | uint64_t Mask = |
3445 | 84 | buildMultiCharMask(Bits, 0, Size, CharWidth, DataLayout.isBigEndian()); |
3446 | | |
3447 | 84 | return Builder.CreateAnd(Src, Mask, "cmse.clear"); |
3448 | 84 | } |
3449 | | |
3450 | | // Emit code to clear the bits in a record, which aren't a part of any user |
3451 | | // declared member, when the record is a function argument. |
3452 | | llvm::Value *CodeGenFunction::EmitCMSEClearRecord(llvm::Value *Src, |
3453 | | llvm::ArrayType *ATy, |
3454 | 39 | QualType QTy) { |
3455 | 39 | const llvm::DataLayout &DataLayout = CGM.getDataLayout(); |
3456 | 39 | int Size = DataLayout.getTypeStoreSize(ATy); |
3457 | 39 | SmallVector<uint64_t, 16> Bits(Size); |
3458 | 39 | setUsedBits(CGM, QTy->castAs<RecordType>(), 0, Bits); |
3459 | | |
3460 | | // Clear each element of the LLVM array. |
3461 | 39 | int CharWidth = CGM.getContext().getCharWidth(); |
3462 | 39 | int CharsPerElt = |
3463 | 39 | ATy->getArrayElementType()->getScalarSizeInBits() / CharWidth; |
3464 | 39 | int MaskIndex = 0; |
3465 | 39 | llvm::Value *R = llvm::UndefValue::get(ATy); |
3466 | 107 | for (int I = 0, N = ATy->getArrayNumElements(); I != N; ++I68 ) { |
3467 | 68 | uint64_t Mask = buildMultiCharMask(Bits, MaskIndex, CharsPerElt, CharWidth, |
3468 | 68 | DataLayout.isBigEndian()); |
3469 | 68 | MaskIndex += CharsPerElt; |
3470 | 68 | llvm::Value *T0 = Builder.CreateExtractValue(Src, I); |
3471 | 68 | llvm::Value *T1 = Builder.CreateAnd(T0, Mask, "cmse.clear"); |
3472 | 68 | R = Builder.CreateInsertValue(R, T1, I); |
3473 | 68 | } |
3474 | | |
3475 | 39 | return R; |
3476 | 39 | } |
3477 | | |
3478 | | void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI, |
3479 | | bool EmitRetDbgLoc, |
3480 | 314k | SourceLocation EndLoc) { |
3481 | 314k | if (FI.isNoReturn()) { |
3482 | | // Noreturn functions don't return. |
3483 | 12 | EmitUnreachable(EndLoc); |
3484 | 12 | return; |
3485 | 12 | } |
3486 | | |
3487 | 314k | if (CurCodeDecl && CurCodeDecl->hasAttr<NakedAttr>()305k ) { |
3488 | | // Naked functions don't have epilogues. |
3489 | 10 | Builder.CreateUnreachable(); |
3490 | 10 | return; |
3491 | 10 | } |
3492 | | |
3493 | | // Functions with no result always return void. |
3494 | 314k | if (!ReturnValue.isValid()) { |
3495 | 159k | Builder.CreateRetVoid(); |
3496 | 159k | return; |
3497 | 159k | } |
3498 | | |
3499 | 154k | llvm::DebugLoc RetDbgLoc; |
3500 | 154k | llvm::Value *RV = nullptr; |
3501 | 154k | QualType RetTy = FI.getReturnType(); |
3502 | 154k | const ABIArgInfo &RetAI = FI.getReturnInfo(); |
3503 | | |
3504 | 154k | switch (RetAI.getKind()) { |
3505 | 5 | case ABIArgInfo::InAlloca: |
3506 | | // Aggregrates get evaluated directly into the destination. Sometimes we |
3507 | | // need to return the sret value in a register, though. |
3508 | 5 | assert(hasAggregateEvaluationKind(RetTy)); |
3509 | 5 | if (RetAI.getInAllocaSRet()) { |
3510 | 5 | llvm::Function::arg_iterator EI = CurFn->arg_end(); |
3511 | 5 | --EI; |
3512 | 5 | llvm::Value *ArgStruct = &*EI; |
3513 | 5 | llvm::Value *SRet = Builder.CreateStructGEP( |
3514 | 5 | FI.getArgStruct(), ArgStruct, RetAI.getInAllocaFieldIndex()); |
3515 | 5 | llvm::Type *Ty = |
3516 | 5 | cast<llvm::GetElementPtrInst>(SRet)->getResultElementType(); |
3517 | 5 | RV = Builder.CreateAlignedLoad(Ty, SRet, getPointerAlign(), "sret"); |
3518 | 5 | } |
3519 | 5 | break; |
3520 | | |
3521 | 4.14k | case ABIArgInfo::Indirect: { |
3522 | 4.14k | auto AI = CurFn->arg_begin(); |
3523 | 4.14k | if (RetAI.isSRetAfterThis()) |
3524 | 54 | ++AI; |
3525 | 4.14k | switch (getEvaluationKind(RetTy)) { |
3526 | 292 | case TEK_Complex: { |
3527 | 292 | ComplexPairTy RT = |
3528 | 292 | EmitLoadOfComplex(MakeAddrLValue(ReturnValue, RetTy), EndLoc); |
3529 | 292 | EmitStoreOfComplex(RT, MakeNaturalAlignAddrLValue(&*AI, RetTy), |
3530 | 292 | /*isInit*/ true); |
3531 | 292 | break; |
3532 | 0 | } |
3533 | 2.33k | case TEK_Aggregate: |
3534 | | // Do nothing; aggregrates get evaluated directly into the destination. |
3535 | 2.33k | break; |
3536 | 1.51k | case TEK_Scalar: { |
3537 | 1.51k | LValueBaseInfo BaseInfo; |
3538 | 1.51k | TBAAAccessInfo TBAAInfo; |
3539 | 1.51k | CharUnits Alignment = |
3540 | 1.51k | CGM.getNaturalTypeAlignment(RetTy, &BaseInfo, &TBAAInfo); |
3541 | 1.51k | Address ArgAddr(&*AI, ConvertType(RetTy), Alignment); |
3542 | 1.51k | LValue ArgVal = |
3543 | 1.51k | LValue::MakeAddr(ArgAddr, RetTy, getContext(), BaseInfo, TBAAInfo); |
3544 | 1.51k | EmitStoreOfScalar( |
3545 | 1.51k | Builder.CreateLoad(ReturnValue), ArgVal, /*isInit*/ true); |
3546 | 1.51k | break; |
3547 | 0 | } |
3548 | 4.14k | } |
3549 | 4.14k | break; |
3550 | 4.14k | } |
3551 | | |
3552 | 11.2k | case ABIArgInfo::Extend: |
3553 | 149k | case ABIArgInfo::Direct: |
3554 | 149k | if (RetAI.getCoerceToType() == ConvertType(RetTy) && |
3555 | 149k | RetAI.getDirectOffset() == 0144k ) { |
3556 | | // The internal return value temp always will have pointer-to-return-type |
3557 | | // type, just do a load. |
3558 | | |
3559 | | // If there is a dominating store to ReturnValue, we can elide |
3560 | | // the load, zap the store, and usually zap the alloca. |
3561 | 144k | if (llvm::StoreInst *SI = |
3562 | 144k | findDominatingStoreToReturnValue(*this)) { |
3563 | | // Reuse the debug location from the store unless there is |
3564 | | // cleanup code to be emitted between the store and return |
3565 | | // instruction. |
3566 | 138k | if (EmitRetDbgLoc && !AutoreleaseResult137k ) |
3567 | 137k | RetDbgLoc = SI->getDebugLoc(); |
3568 | | // Get the stored value and nuke the now-dead store. |
3569 | 138k | RV = SI->getValueOperand(); |
3570 | 138k | SI->eraseFromParent(); |
3571 | | |
3572 | | // Otherwise, we have to do a simple load. |
3573 | 138k | } else { |
3574 | 5.38k | RV = Builder.CreateLoad(ReturnValue); |
3575 | 5.38k | } |
3576 | 144k | } else { |
3577 | | // If the value is offset in memory, apply the offset now. |
3578 | 5.79k | Address V = emitAddressAtOffset(*this, ReturnValue, RetAI); |
3579 | | |
3580 | 5.79k | RV = CreateCoercedLoad(V, RetAI.getCoerceToType(), *this); |
3581 | 5.79k | } |
3582 | | |
3583 | | // In ARC, end functions that return a retainable type with a call |
3584 | | // to objc_autoreleaseReturnValue. |
3585 | 149k | if (AutoreleaseResult) { |
3586 | 122 | #ifndef NDEBUG |
3587 | | // Type::isObjCRetainabletype has to be called on a QualType that hasn't |
3588 | | // been stripped of the typedefs, so we cannot use RetTy here. Get the |
3589 | | // original return type of FunctionDecl, CurCodeDecl, and BlockDecl from |
3590 | | // CurCodeDecl or BlockInfo. |
3591 | 122 | QualType RT; |
3592 | | |
3593 | 122 | if (auto *FD = dyn_cast<FunctionDecl>(CurCodeDecl)) |
3594 | 74 | RT = FD->getReturnType(); |
3595 | 48 | else if (auto *MD = dyn_cast<ObjCMethodDecl>(CurCodeDecl)) |
3596 | 19 | RT = MD->getReturnType(); |
3597 | 29 | else if (isa<BlockDecl>(CurCodeDecl)) |
3598 | 29 | RT = BlockInfo->BlockExpression->getFunctionType()->getReturnType(); |
3599 | 0 | else |
3600 | 0 | llvm_unreachable("Unexpected function/method type"); |
3601 | | |
3602 | 122 | assert(getLangOpts().ObjCAutoRefCount && |
3603 | 122 | !FI.isReturnsRetained() && |
3604 | 122 | RT->isObjCRetainableType()); |
3605 | 0 | #endif |
3606 | 0 | RV = emitAutoreleaseOfResult(*this, RV); |
3607 | 122 | } |
3608 | | |
3609 | 0 | break; |
3610 | | |
3611 | 409 | case ABIArgInfo::Ignore: |
3612 | 409 | break; |
3613 | | |
3614 | 541 | case ABIArgInfo::CoerceAndExpand: { |
3615 | 541 | auto coercionType = RetAI.getCoerceAndExpandType(); |
3616 | | |
3617 | | // Load all of the coerced elements out into results. |
3618 | 541 | llvm::SmallVector<llvm::Value*, 4> results; |
3619 | 541 | Address addr = Builder.CreateElementBitCast(ReturnValue, coercionType); |
3620 | 1.68k | for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i1.14k ) { |
3621 | 1.14k | auto coercedEltType = coercionType->getElementType(i); |
3622 | 1.14k | if (ABIArgInfo::isPaddingForCoerceAndExpand(coercedEltType)) |
3623 | 7 | continue; |
3624 | | |
3625 | 1.14k | auto eltAddr = Builder.CreateStructGEP(addr, i); |
3626 | 1.14k | auto elt = Builder.CreateLoad(eltAddr); |
3627 | 1.14k | results.push_back(elt); |
3628 | 1.14k | } |
3629 | | |
3630 | | // If we have one result, it's the single direct result type. |
3631 | 541 | if (results.size() == 1) { |
3632 | 180 | RV = results[0]; |
3633 | | |
3634 | | // Otherwise, we need to make a first-class aggregate. |
3635 | 361 | } else { |
3636 | | // Construct a return type that lacks padding elements. |
3637 | 361 | llvm::Type *returnType = RetAI.getUnpaddedCoerceAndExpandType(); |
3638 | | |
3639 | 361 | RV = llvm::UndefValue::get(returnType); |
3640 | 1.32k | for (unsigned i = 0, e = results.size(); i != e; ++i960 ) { |
3641 | 960 | RV = Builder.CreateInsertValue(RV, results[i], i); |
3642 | 960 | } |
3643 | 361 | } |
3644 | 541 | break; |
3645 | 11.2k | } |
3646 | 0 | case ABIArgInfo::Expand: |
3647 | 0 | case ABIArgInfo::IndirectAliased: |
3648 | 0 | llvm_unreachable("Invalid ABI kind for return argument"); |
3649 | 154k | } |
3650 | | |
3651 | 154k | llvm::Instruction *Ret; |
3652 | 154k | if (RV) { |
3653 | 150k | if (CurFuncDecl && CurFuncDecl->hasAttr<CmseNSEntryAttr>()149k ) { |
3654 | | // For certain return types, clear padding bits, as they may reveal |
3655 | | // sensitive information. |
3656 | | // Small struct/union types are passed as integers. |
3657 | 90 | auto *ITy = dyn_cast<llvm::IntegerType>(RV->getType()); |
3658 | 90 | if (ITy != nullptr && isa<RecordType>(RetTy.getCanonicalType())84 ) |
3659 | 84 | RV = EmitCMSEClearRecord(RV, ITy, RetTy); |
3660 | 90 | } |
3661 | 150k | EmitReturnValueCheck(RV); |
3662 | 150k | Ret = Builder.CreateRet(RV); |
3663 | 150k | } else { |
3664 | 4.55k | Ret = Builder.CreateRetVoid(); |
3665 | 4.55k | } |
3666 | | |
3667 | 154k | if (RetDbgLoc) |
3668 | 44.1k | Ret->setDebugLoc(std::move(RetDbgLoc)); |
3669 | 154k | } |
3670 | | |
3671 | 150k | void CodeGenFunction::EmitReturnValueCheck(llvm::Value *RV) { |
3672 | | // A current decl may not be available when emitting vtable thunks. |
3673 | 150k | if (!CurCodeDecl) |
3674 | 1.27k | return; |
3675 | | |
3676 | | // If the return block isn't reachable, neither is this check, so don't emit |
3677 | | // it. |
3678 | 149k | if (ReturnBlock.isValid() && ReturnBlock.getBlock()->use_empty()1.93k ) |
3679 | 233 | return; |
3680 | | |
3681 | 148k | ReturnsNonNullAttr *RetNNAttr = nullptr; |
3682 | 148k | if (SanOpts.has(SanitizerKind::ReturnsNonnullAttribute)) |
3683 | 45 | RetNNAttr = CurCodeDecl->getAttr<ReturnsNonNullAttr>(); |
3684 | | |
3685 | 148k | if (!RetNNAttr && !requiresReturnValueNullabilityCheck()148k ) |
3686 | 148k | return; |
3687 | | |
3688 | | // Prefer the returns_nonnull attribute if it's present. |
3689 | 18 | SourceLocation AttrLoc; |
3690 | 18 | SanitizerMask CheckKind; |
3691 | 18 | SanitizerHandler Handler; |
3692 | 18 | if (RetNNAttr) { |
3693 | 5 | assert(!requiresReturnValueNullabilityCheck() && |
3694 | 5 | "Cannot check nullability and the nonnull attribute"); |
3695 | 0 | AttrLoc = RetNNAttr->getLocation(); |
3696 | 5 | CheckKind = SanitizerKind::ReturnsNonnullAttribute; |
3697 | 5 | Handler = SanitizerHandler::NonnullReturn; |
3698 | 13 | } else { |
3699 | 13 | if (auto *DD = dyn_cast<DeclaratorDecl>(CurCodeDecl)) |
3700 | 7 | if (auto *TSI = DD->getTypeSourceInfo()) |
3701 | 7 | if (auto FTL = TSI->getTypeLoc().getAsAdjusted<FunctionTypeLoc>()) |
3702 | 7 | AttrLoc = FTL.getReturnLoc().findNullabilityLoc(); |
3703 | 13 | CheckKind = SanitizerKind::NullabilityReturn; |
3704 | 13 | Handler = SanitizerHandler::NullabilityReturn; |
3705 | 13 | } |
3706 | | |
3707 | 0 | SanitizerScope SanScope(this); |
3708 | | |
3709 | | // Make sure the "return" source location is valid. If we're checking a |
3710 | | // nullability annotation, make sure the preconditions for the check are met. |
3711 | 18 | llvm::BasicBlock *Check = createBasicBlock("nullcheck"); |
3712 | 18 | llvm::BasicBlock *NoCheck = createBasicBlock("no.nullcheck"); |
3713 | 18 | llvm::Value *SLocPtr = Builder.CreateLoad(ReturnLocation, "return.sloc.load"); |
3714 | 18 | llvm::Value *CanNullCheck = Builder.CreateIsNotNull(SLocPtr); |
3715 | 18 | if (requiresReturnValueNullabilityCheck()) |
3716 | 13 | CanNullCheck = |
3717 | 13 | Builder.CreateAnd(CanNullCheck, RetValNullabilityPrecondition); |
3718 | 18 | Builder.CreateCondBr(CanNullCheck, Check, NoCheck); |
3719 | 18 | EmitBlock(Check); |
3720 | | |
3721 | | // Now do the null check. |
3722 | 18 | llvm::Value *Cond = Builder.CreateIsNotNull(RV); |
3723 | 18 | llvm::Constant *StaticData[] = {EmitCheckSourceLocation(AttrLoc)}; |
3724 | 18 | llvm::Value *DynamicData[] = {SLocPtr}; |
3725 | 18 | EmitCheck(std::make_pair(Cond, CheckKind), Handler, StaticData, DynamicData); |
3726 | | |
3727 | 18 | EmitBlock(NoCheck); |
3728 | | |
3729 | 18 | #ifndef NDEBUG |
3730 | | // The return location should not be used after the check has been emitted. |
3731 | 18 | ReturnLocation = Address::invalid(); |
3732 | 18 | #endif |
3733 | 18 | } |
3734 | | |
3735 | 24.6k | static bool isInAllocaArgument(CGCXXABI &ABI, QualType type) { |
3736 | 24.6k | const CXXRecordDecl *RD = type->getAsCXXRecordDecl(); |
3737 | 24.6k | return RD && ABI.getRecordArgABI(RD) == CGCXXABI::RAA_DirectInMemory2.62k ; |
3738 | 24.6k | } |
3739 | | |
3740 | | static AggValueSlot createPlaceholderSlot(CodeGenFunction &CGF, |
3741 | 106 | QualType Ty) { |
3742 | | // FIXME: Generate IR in one pass, rather than going back and fixing up these |
3743 | | // placeholders. |
3744 | 106 | llvm::Type *IRTy = CGF.ConvertTypeForMem(Ty); |
3745 | 106 | llvm::Type *IRPtrTy = IRTy->getPointerTo(); |
3746 | 106 | llvm::Value *Placeholder = llvm::UndefValue::get(IRPtrTy->getPointerTo()); |
3747 | | |
3748 | | // FIXME: When we generate this IR in one pass, we shouldn't need |
3749 | | // this win32-specific alignment hack. |
3750 | 106 | CharUnits Align = CharUnits::fromQuantity(4); |
3751 | 106 | Placeholder = CGF.Builder.CreateAlignedLoad(IRPtrTy, Placeholder, Align); |
3752 | | |
3753 | 106 | return AggValueSlot::forAddr(Address(Placeholder, IRTy, Align), |
3754 | 106 | Ty.getQualifiers(), |
3755 | 106 | AggValueSlot::IsNotDestructed, |
3756 | 106 | AggValueSlot::DoesNotNeedGCBarriers, |
3757 | 106 | AggValueSlot::IsNotAliased, |
3758 | 106 | AggValueSlot::DoesNotOverlap); |
3759 | 106 | } |
3760 | | |
3761 | | void CodeGenFunction::EmitDelegateCallArg(CallArgList &args, |
3762 | | const VarDecl *param, |
3763 | 17.9k | SourceLocation loc) { |
3764 | | // StartFunction converted the ABI-lowered parameter(s) into a |
3765 | | // local alloca. We need to turn that into an r-value suitable |
3766 | | // for EmitCall. |
3767 | 17.9k | Address local = GetAddrOfLocalVar(param); |
3768 | | |
3769 | 17.9k | QualType type = param->getType(); |
3770 | | |
3771 | 17.9k | if (isInAllocaArgument(CGM.getCXXABI(), type)) { |
3772 | 1 | CGM.ErrorUnsupported(param, "forwarded non-trivially copyable parameter"); |
3773 | 1 | } |
3774 | | |
3775 | | // GetAddrOfLocalVar returns a pointer-to-pointer for references, |
3776 | | // but the argument needs to be the original pointer. |
3777 | 17.9k | if (type->isReferenceType()) { |
3778 | 9.27k | args.add(RValue::get(Builder.CreateLoad(local)), type); |
3779 | | |
3780 | | // In ARC, move out of consumed arguments so that the release cleanup |
3781 | | // entered by StartFunction doesn't cause an over-release. This isn't |
3782 | | // optimal -O0 code generation, but it should get cleaned up when |
3783 | | // optimization is enabled. This also assumes that delegate calls are |
3784 | | // performed exactly once for a set of arguments, but that should be safe. |
3785 | 9.27k | } else if (8.67k getLangOpts().ObjCAutoRefCount8.67k && |
3786 | 8.67k | param->hasAttr<NSConsumedAttr>()9 && |
3787 | 8.67k | type->isObjCRetainableType()2 ) { |
3788 | 2 | llvm::Value *ptr = Builder.CreateLoad(local); |
3789 | 2 | auto null = |
3790 | 2 | llvm::ConstantPointerNull::get(cast<llvm::PointerType>(ptr->getType())); |
3791 | 2 | Builder.CreateStore(null, local); |
3792 | 2 | args.add(RValue::get(ptr), type); |
3793 | | |
3794 | | // For the most part, we just need to load the alloca, except that |
3795 | | // aggregate r-values are actually pointers to temporaries. |
3796 | 8.66k | } else { |
3797 | 8.66k | args.add(convertTempToRValue(local, type, loc), type); |
3798 | 8.66k | } |
3799 | | |
3800 | | // Deactivate the cleanup for the callee-destructed param that was pushed. |
3801 | 17.9k | if (type->isRecordType() && !CurFuncIsThunk2.29k && |
3802 | 17.9k | type->castAs<RecordType>()->getDecl()->isParamDestroyedInCallee()2.26k && |
3803 | 17.9k | param->needsDestruction(getContext())8 ) { |
3804 | 6 | EHScopeStack::stable_iterator cleanup = |
3805 | 6 | CalleeDestructedParamCleanups.lookup(cast<ParmVarDecl>(param)); |
3806 | 6 | assert(cleanup.isValid() && |
3807 | 6 | "cleanup for callee-destructed param not recorded"); |
3808 | | // This unreachable is a temporary marker which will be removed later. |
3809 | 0 | llvm::Instruction *isActive = Builder.CreateUnreachable(); |
3810 | 6 | args.addArgCleanupDeactivation(cleanup, isActive); |
3811 | 6 | } |
3812 | 17.9k | } |
3813 | | |
3814 | 54 | static bool isProvablyNull(llvm::Value *addr) { |
3815 | 54 | return isa<llvm::ConstantPointerNull>(addr); |
3816 | 54 | } |
3817 | | |
3818 | | /// Emit the actual writing-back of a writeback. |
3819 | | static void emitWriteback(CodeGenFunction &CGF, |
3820 | 27 | const CallArgList::Writeback &writeback) { |
3821 | 27 | const LValue &srcLV = writeback.Source; |
3822 | 27 | Address srcAddr = srcLV.getAddress(CGF); |
3823 | 27 | assert(!isProvablyNull(srcAddr.getPointer()) && |
3824 | 27 | "shouldn't have writeback for provably null argument"); |
3825 | | |
3826 | 0 | llvm::BasicBlock *contBB = nullptr; |
3827 | | |
3828 | | // If the argument wasn't provably non-null, we need to null check |
3829 | | // before doing the store. |
3830 | 27 | bool provablyNonNull = llvm::isKnownNonZero(srcAddr.getPointer(), |
3831 | 27 | CGF.CGM.getDataLayout()); |
3832 | 27 | if (!provablyNonNull) { |
3833 | 5 | llvm::BasicBlock *writebackBB = CGF.createBasicBlock("icr.writeback"); |
3834 | 5 | contBB = CGF.createBasicBlock("icr.done"); |
3835 | | |
3836 | 5 | llvm::Value *isNull = |
3837 | 5 | CGF.Builder.CreateIsNull(srcAddr.getPointer(), "icr.isnull"); |
3838 | 5 | CGF.Builder.CreateCondBr(isNull, contBB, writebackBB); |
3839 | 5 | CGF.EmitBlock(writebackBB); |
3840 | 5 | } |
3841 | | |
3842 | | // Load the value to writeback. |
3843 | 27 | llvm::Value *value = CGF.Builder.CreateLoad(writeback.Temporary); |
3844 | | |
3845 | | // Cast it back, in case we're writing an id to a Foo* or something. |
3846 | 27 | value = CGF.Builder.CreateBitCast(value, srcAddr.getElementType(), |
3847 | 27 | "icr.writeback-cast"); |
3848 | | |
3849 | | // Perform the writeback. |
3850 | | |
3851 | | // If we have a "to use" value, it's something we need to emit a use |
3852 | | // of. This has to be carefully threaded in: if it's done after the |
3853 | | // release it's potentially undefined behavior (and the optimizer |
3854 | | // will ignore it), and if it happens before the retain then the |
3855 | | // optimizer could move the release there. |
3856 | 27 | if (writeback.ToUse) { |
3857 | 8 | assert(srcLV.getObjCLifetime() == Qualifiers::OCL_Strong); |
3858 | | |
3859 | | // Retain the new value. No need to block-copy here: the block's |
3860 | | // being passed up the stack. |
3861 | 0 | value = CGF.EmitARCRetainNonBlock(value); |
3862 | | |
3863 | | // Emit the intrinsic use here. |
3864 | 8 | CGF.EmitARCIntrinsicUse(writeback.ToUse); |
3865 | | |
3866 | | // Load the old value (primitively). |
3867 | 8 | llvm::Value *oldValue = CGF.EmitLoadOfScalar(srcLV, SourceLocation()); |
3868 | | |
3869 | | // Put the new value in place (primitively). |
3870 | 8 | CGF.EmitStoreOfScalar(value, srcLV, /*init*/ false); |
3871 | | |
3872 | | // Release the old value. |
3873 | 8 | CGF.EmitARCRelease(oldValue, srcLV.isARCPreciseLifetime()); |
3874 | | |
3875 | | // Otherwise, we can just do a normal lvalue store. |
3876 | 19 | } else { |
3877 | 19 | CGF.EmitStoreThroughLValue(RValue::get(value), srcLV); |
3878 | 19 | } |
3879 | | |
3880 | | // Jump to the continuation block. |
3881 | 27 | if (!provablyNonNull) |
3882 | 5 | CGF.EmitBlock(contBB); |
3883 | 27 | } |
3884 | | |
3885 | | static void emitWritebacks(CodeGenFunction &CGF, |
3886 | 27 | const CallArgList &args) { |
3887 | 27 | for (const auto &I : args.writebacks()) |
3888 | 27 | emitWriteback(CGF, I); |
3889 | 27 | } |
3890 | | |
3891 | | static void deactivateArgCleanupsBeforeCall(CodeGenFunction &CGF, |
3892 | 149 | const CallArgList &CallArgs) { |
3893 | 149 | ArrayRef<CallArgList::CallArgCleanup> Cleanups = |
3894 | 149 | CallArgs.getCleanupsToDeactivate(); |
3895 | | // Iterate in reverse to increase the likelihood of popping the cleanup. |
3896 | 192 | for (const auto &I : llvm::reverse(Cleanups)) { |
3897 | 192 | CGF.DeactivateCleanupBlock(I.Cleanup, I.IsActiveIP); |
3898 | 192 | I.IsActiveIP->eraseFromParent(); |
3899 | 192 | } |
3900 | 149 | } |
3901 | | |
3902 | 27 | static const Expr *maybeGetUnaryAddrOfOperand(const Expr *E) { |
3903 | 27 | if (const UnaryOperator *uop = dyn_cast<UnaryOperator>(E->IgnoreParens())) |
3904 | 22 | if (uop->getOpcode() == UO_AddrOf) |
3905 | 22 | return uop->getSubExpr(); |
3906 | 5 | return nullptr; |
3907 | 27 | } |
3908 | | |
3909 | | /// Emit an argument that's being passed call-by-writeback. That is, |
3910 | | /// we are passing the address of an __autoreleased temporary; it |
3911 | | /// might be copy-initialized with the current value of the given |
3912 | | /// address, but it will definitely be copied out of after the call. |
3913 | | static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args, |
3914 | 27 | const ObjCIndirectCopyRestoreExpr *CRE) { |
3915 | 27 | LValue srcLV; |
3916 | | |
3917 | | // Make an optimistic effort to emit the address as an l-value. |
3918 | | // This can fail if the argument expression is more complicated. |
3919 | 27 | if (const Expr *lvExpr = maybeGetUnaryAddrOfOperand(CRE->getSubExpr())) { |
3920 | 22 | srcLV = CGF.EmitLValue(lvExpr); |
3921 | | |
3922 | | // Otherwise, just emit it as a scalar. |
3923 | 22 | } else { |
3924 | 5 | Address srcAddr = CGF.EmitPointerWithAlignment(CRE->getSubExpr()); |
3925 | | |
3926 | 5 | QualType srcAddrType = |
3927 | 5 | CRE->getSubExpr()->getType()->castAs<PointerType>()->getPointeeType(); |
3928 | 5 | srcLV = CGF.MakeAddrLValue(srcAddr, srcAddrType); |
3929 | 5 | } |
3930 | 27 | Address srcAddr = srcLV.getAddress(CGF); |
3931 | | |
3932 | | // The dest and src types don't necessarily match in LLVM terms |
3933 | | // because of the crazy ObjC compatibility rules. |
3934 | | |
3935 | 27 | llvm::PointerType *destType = |
3936 | 27 | cast<llvm::PointerType>(CGF.ConvertType(CRE->getType())); |
3937 | 27 | llvm::Type *destElemType = |
3938 | 27 | CGF.ConvertTypeForMem(CRE->getType()->getPointeeType()); |
3939 | | |
3940 | | // If the address is a constant null, just pass the appropriate null. |
3941 | 27 | if (isProvablyNull(srcAddr.getPointer())) { |
3942 | 0 | args.add(RValue::get(llvm::ConstantPointerNull::get(destType)), |
3943 | 0 | CRE->getType()); |
3944 | 0 | return; |
3945 | 0 | } |
3946 | | |
3947 | | // Create the temporary. |
3948 | 27 | Address temp = |
3949 | 27 | CGF.CreateTempAlloca(destElemType, CGF.getPointerAlign(), "icr.temp"); |
3950 | | // Loading an l-value can introduce a cleanup if the l-value is __weak, |
3951 | | // and that cleanup will be conditional if we can't prove that the l-value |
3952 | | // isn't null, so we need to register a dominating point so that the cleanups |
3953 | | // system will make valid IR. |
3954 | 27 | CodeGenFunction::ConditionalEvaluation condEval(CGF); |
3955 | | |
3956 | | // Zero-initialize it if we're not doing a copy-initialization. |
3957 | 27 | bool shouldCopy = CRE->shouldCopy(); |
3958 | 27 | if (!shouldCopy) { |
3959 | 4 | llvm::Value *null = |
3960 | 4 | llvm::ConstantPointerNull::get(cast<llvm::PointerType>(destElemType)); |
3961 | 4 | CGF.Builder.CreateStore(null, temp); |
3962 | 4 | } |
3963 | | |
3964 | 27 | llvm::BasicBlock *contBB = nullptr; |
3965 | 27 | llvm::BasicBlock *originBB = nullptr; |
3966 | | |
3967 | | // If the address is *not* known to be non-null, we need to switch. |
3968 | 27 | llvm::Value *finalArgument; |
3969 | | |
3970 | 27 | bool provablyNonNull = llvm::isKnownNonZero(srcAddr.getPointer(), |
3971 | 27 | CGF.CGM.getDataLayout()); |
3972 | 27 | if (provablyNonNull) { |
3973 | 22 | finalArgument = temp.getPointer(); |
3974 | 22 | } else { |
3975 | 5 | llvm::Value *isNull = |
3976 | 5 | CGF.Builder.CreateIsNull(srcAddr.getPointer(), "icr.isnull"); |
3977 | | |
3978 | 5 | finalArgument = CGF.Builder.CreateSelect(isNull, |
3979 | 5 | llvm::ConstantPointerNull::get(destType), |
3980 | 5 | temp.getPointer(), "icr.argument"); |
3981 | | |
3982 | | // If we need to copy, then the load has to be conditional, which |
3983 | | // means we need control flow. |
3984 | 5 | if (shouldCopy) { |
3985 | 5 | originBB = CGF.Builder.GetInsertBlock(); |
3986 | 5 | contBB = CGF.createBasicBlock("icr.cont"); |
3987 | 5 | llvm::BasicBlock *copyBB = CGF.createBasicBlock("icr.copy"); |
3988 | 5 | CGF.Builder.CreateCondBr(isNull, contBB, copyBB); |
3989 | 5 | CGF.EmitBlock(copyBB); |
3990 | 5 | condEval.begin(CGF); |
3991 | 5 | } |
3992 | 5 | } |
3993 | | |
3994 | 27 | llvm::Value *valueToUse = nullptr; |
3995 | | |
3996 | | // Perform a copy if necessary. |
3997 | 27 | if (shouldCopy) { |
3998 | 23 | RValue srcRV = CGF.EmitLoadOfLValue(srcLV, SourceLocation()); |
3999 | 23 | assert(srcRV.isScalar()); |
4000 | | |
4001 | 0 | llvm::Value *src = srcRV.getScalarVal(); |
4002 | 23 | src = CGF.Builder.CreateBitCast(src, destElemType, "icr.cast"); |
4003 | | |
4004 | | // Use an ordinary store, not a store-to-lvalue. |
4005 | 23 | CGF.Builder.CreateStore(src, temp); |
4006 | | |
4007 | | // If optimization is enabled, and the value was held in a |
4008 | | // __strong variable, we need to tell the optimizer that this |
4009 | | // value has to stay alive until we're doing the store back. |
4010 | | // This is because the temporary is effectively unretained, |
4011 | | // and so otherwise we can violate the high-level semantics. |
4012 | 23 | if (CGF.CGM.getCodeGenOpts().OptimizationLevel != 0 && |
4013 | 23 | srcLV.getObjCLifetime() == Qualifiers::OCL_Strong10 ) { |
4014 | 8 | valueToUse = src; |
4015 | 8 | } |
4016 | 23 | } |
4017 | | |
4018 | | // Finish the control flow if we needed it. |
4019 | 27 | if (shouldCopy && !provablyNonNull23 ) { |
4020 | 5 | llvm::BasicBlock *copyBB = CGF.Builder.GetInsertBlock(); |
4021 | 5 | CGF.EmitBlock(contBB); |
4022 | | |
4023 | | // Make a phi for the value to intrinsically use. |
4024 | 5 | if (valueToUse) { |
4025 | 2 | llvm::PHINode *phiToUse = CGF.Builder.CreatePHI(valueToUse->getType(), 2, |
4026 | 2 | "icr.to-use"); |
4027 | 2 | phiToUse->addIncoming(valueToUse, copyBB); |
4028 | 2 | phiToUse->addIncoming(llvm::UndefValue::get(valueToUse->getType()), |
4029 | 2 | originBB); |
4030 | 2 | valueToUse = phiToUse; |
4031 | 2 | } |
4032 | | |
4033 | 5 | condEval.end(CGF); |
4034 | 5 | } |
4035 | | |
4036 | 27 | args.addWriteback(srcLV, temp, valueToUse); |
4037 | 27 | args.add(RValue::get(finalArgument), CRE->getType()); |
4038 | 27 | } |
4039 | | |
4040 | 73 | void CallArgList::allocateArgumentMemory(CodeGenFunction &CGF) { |
4041 | 73 | assert(!StackBase); |
4042 | | |
4043 | | // Save the stack. |
4044 | 0 | llvm::Function *F = CGF.CGM.getIntrinsic(llvm::Intrinsic::stacksave); |
4045 | 73 | StackBase = CGF.Builder.CreateCall(F, {}, "inalloca.save"); |
4046 | 73 | } |
4047 | | |
4048 | 364k | void CallArgList::freeArgumentMemory(CodeGenFunction &CGF) const { |
4049 | 364k | if (StackBase) { |
4050 | | // Restore the stack after the call. |
4051 | 73 | llvm::Function *F = CGF.CGM.getIntrinsic(llvm::Intrinsic::stackrestore); |
4052 | 73 | CGF.Builder.CreateCall(F, StackBase); |
4053 | 73 | } |
4054 | 364k | } |
4055 | | |
4056 | | void CodeGenFunction::EmitNonNullArgCheck(RValue RV, QualType ArgType, |
4057 | | SourceLocation ArgLoc, |
4058 | | AbstractCallee AC, |
4059 | 402k | unsigned ParmNum) { |
4060 | 402k | if (!AC.getDecl() || !(383k SanOpts.has(SanitizerKind::NonnullAttribute)383k || |
4061 | 383k | SanOpts.has(SanitizerKind::NullabilityArg)383k )) |
4062 | 402k | return; |
4063 | | |
4064 | | // The param decl may be missing in a variadic function. |
4065 | 77 | auto PVD = ParmNum < AC.getNumParams() ? AC.getParamDecl(ParmNum)66 : nullptr11 ; |
4066 | 77 | unsigned ArgNo = PVD ? PVD->getFunctionScopeIndex()66 : ParmNum11 ; |
4067 | | |
4068 | | // Prefer the nonnull attribute if it's present. |
4069 | 77 | const NonNullAttr *NNAttr = nullptr; |
4070 | 77 | if (SanOpts.has(SanitizerKind::NonnullAttribute)) |
4071 | 25 | NNAttr = getNonNullAttr(AC.getDecl(), PVD, ArgType, ArgNo); |
4072 | | |
4073 | 77 | bool CanCheckNullability = false; |
4074 | 77 | if (SanOpts.has(SanitizerKind::NullabilityArg) && !NNAttr53 && PVD52 ) { |
4075 | 44 | auto Nullability = PVD->getType()->getNullability(getContext()); |
4076 | 44 | CanCheckNullability = Nullability && |
4077 | 44 | *Nullability == NullabilityKind::NonNull20 && |
4078 | 44 | PVD->getTypeSourceInfo()18 ; |
4079 | 44 | } |
4080 | | |
4081 | 77 | if (!NNAttr && !CanCheckNullability61 ) |
4082 | 43 | return; |
4083 | | |
4084 | 34 | SourceLocation AttrLoc; |
4085 | 34 | SanitizerMask CheckKind; |
4086 | 34 | SanitizerHandler Handler; |
4087 | 34 | if (NNAttr) { |
4088 | 16 | AttrLoc = NNAttr->getLocation(); |
4089 | 16 | CheckKind = SanitizerKind::NonnullAttribute; |
4090 | 16 | Handler = SanitizerHandler::NonnullArg; |
4091 | 18 | } else { |
4092 | 18 | AttrLoc = PVD->getTypeSourceInfo()->getTypeLoc().findNullabilityLoc(); |
4093 | 18 | CheckKind = SanitizerKind::NullabilityArg; |
4094 | 18 | Handler = SanitizerHandler::NullabilityArg; |
4095 | 18 | } |
4096 | | |
4097 | 34 | SanitizerScope SanScope(this); |
4098 | 34 | llvm::Value *Cond = EmitNonNullRValueCheck(RV, ArgType); |
4099 | 34 | llvm::Constant *StaticData[] = { |
4100 | 34 | EmitCheckSourceLocation(ArgLoc), EmitCheckSourceLocation(AttrLoc), |
4101 | 34 | llvm::ConstantInt::get(Int32Ty, ArgNo + 1), |
4102 | 34 | }; |
4103 | 34 | EmitCheck(std::make_pair(Cond, CheckKind), Handler, StaticData, None); |
4104 | 34 | } |
4105 | | |
4106 | | // Check if the call is going to use the inalloca convention. This needs to |
4107 | | // agree with CGFunctionInfo::usesInAlloca. The CGFunctionInfo is arranged |
4108 | | // later, so we can't check it directly. |
4109 | | static bool hasInAllocaArgs(CodeGenModule &CGM, CallingConv ExplicitCC, |
4110 | 316k | ArrayRef<QualType> ArgTypes) { |
4111 | | // The Swift calling conventions don't go through the target-specific |
4112 | | // argument classification, they never use inalloca. |
4113 | | // TODO: Consider limiting inalloca use to only calling conventions supported |
4114 | | // by MSVC. |
4115 | 316k | if (ExplicitCC == CC_Swift || ExplicitCC == CC_SwiftAsync315k ) |
4116 | 1.28k | return false; |
4117 | 315k | if (!CGM.getTarget().getCXXABI().isMicrosoft()) |
4118 | 308k | return false; |
4119 | 7.34k | return llvm::any_of(ArgTypes, [&](QualType Ty) { |
4120 | 6.72k | return isInAllocaArgument(CGM.getCXXABI(), Ty); |
4121 | 6.72k | }); |
4122 | 315k | } |
4123 | | |
4124 | | #ifndef NDEBUG |
4125 | | // Determine whether the given argument is an Objective-C method |
4126 | | // that may have type parameters in its signature. |
4127 | 11.3k | static bool isObjCMethodWithTypeParams(const ObjCMethodDecl *method) { |
4128 | 11.3k | const DeclContext *dc = method->getDeclContext(); |
4129 | 11.3k | if (const ObjCInterfaceDecl *classDecl = dyn_cast<ObjCInterfaceDecl>(dc)) { |
4130 | 7.25k | return classDecl->getTypeParamListAsWritten(); |
4131 | 7.25k | } |
4132 | | |
4133 | 4.07k | if (const ObjCCategoryDecl *catDecl = dyn_cast<ObjCCategoryDecl>(dc)) { |
4134 | 3.37k | return catDecl->getTypeParamList(); |
4135 | 3.37k | } |
4136 | | |
4137 | 700 | return false; |
4138 | 4.07k | } |
4139 | | #endif |
4140 | | |
4141 | | /// EmitCallArgs - Emit call arguments for a function. |
4142 | | void CodeGenFunction::EmitCallArgs( |
4143 | | CallArgList &Args, PrototypeWrapper Prototype, |
4144 | | llvm::iterator_range<CallExpr::const_arg_iterator> ArgRange, |
4145 | 316k | AbstractCallee AC, unsigned ParamsToSkip, EvaluationOrder Order) { |
4146 | 316k | SmallVector<QualType, 16> ArgTypes; |
4147 | | |
4148 | 316k | assert((ParamsToSkip == 0 || Prototype.P) && |
4149 | 316k | "Can't skip parameters if type info is not provided"); |
4150 | | |
4151 | | // This variable only captures *explicitly* written conventions, not those |
4152 | | // applied by default via command line flags or target defaults, such as |
4153 | | // thiscall, aapcs, stdcall via -mrtd, etc. Computing that correctly would |
4154 | | // require knowing if this is a C++ instance method or being able to see |
4155 | | // unprototyped FunctionTypes. |
4156 | 0 | CallingConv ExplicitCC = CC_C; |
4157 | | |
4158 | | // First, if a prototype was provided, use those argument types. |
4159 | 316k | bool IsVariadic = false; |
4160 | 316k | if (Prototype.P) { |
4161 | 315k | const auto *MD = Prototype.P.dyn_cast<const ObjCMethodDecl *>(); |
4162 | 315k | if (MD) { |
4163 | 11.3k | IsVariadic = MD->isVariadic(); |
4164 | 11.3k | ExplicitCC = getCallingConventionForDecl( |
4165 | 11.3k | MD, CGM.getTarget().getTriple().isOSWindows()); |
4166 | 11.3k | ArgTypes.assign(MD->param_type_begin() + ParamsToSkip, |
4167 | 11.3k | MD->param_type_end()); |
4168 | 304k | } else { |
4169 | 304k | const auto *FPT = Prototype.P.get<const FunctionProtoType *>(); |
4170 | 304k | IsVariadic = FPT->isVariadic(); |
4171 | 304k | ExplicitCC = FPT->getExtInfo().getCC(); |
4172 | 304k | ArgTypes.assign(FPT->param_type_begin() + ParamsToSkip, |
4173 | 304k | FPT->param_type_end()); |
4174 | 304k | } |
4175 | | |
4176 | 315k | #ifndef NDEBUG |
4177 | | // Check that the prototyped types match the argument expression types. |
4178 | 315k | bool isGenericMethod = MD && isObjCMethodWithTypeParams(MD)11.3k ; |
4179 | 315k | CallExpr::const_arg_iterator Arg = ArgRange.begin(); |
4180 | 315k | for (QualType Ty : ArgTypes) { |
4181 | 313k | assert(Arg != ArgRange.end() && "Running over edge of argument list!"); |
4182 | 0 | assert( |
4183 | 313k | (isGenericMethod || Ty->isVariablyModifiedType() || |
4184 | 313k | Ty.getNonReferenceType()->isObjCRetainableType() || |
4185 | 313k | getContext() |
4186 | 313k | .getCanonicalType(Ty.getNonReferenceType()) |
4187 | 313k | .getTypePtr() == |
4188 | 313k | getContext().getCanonicalType((*Arg)->getType()).getTypePtr()) && |
4189 | 313k | "type mismatch in call argument!"); |
4190 | 0 | ++Arg; |
4191 | 313k | } |
4192 | | |
4193 | | // Either we've emitted all the call args, or we have a call to variadic |
4194 | | // function. |
4195 | 315k | assert((Arg == ArgRange.end() || IsVariadic) && |
4196 | 315k | "Extra arguments in non-variadic function!"); |
4197 | 315k | #endif |
4198 | 315k | } |
4199 | | |
4200 | | // If we still have any arguments, emit them using the type of the argument. |
4201 | 0 | for (auto *A : llvm::drop_begin(ArgRange, ArgTypes.size())) |
4202 | 89.7k | ArgTypes.push_back(IsVariadic ? getVarArgType(A)88.9k : A->getType()803 ); |
4203 | 316k | assert((int)ArgTypes.size() == (ArgRange.end() - ArgRange.begin())); |
4204 | | |
4205 | | // We must evaluate arguments from right to left in the MS C++ ABI, |
4206 | | // because arguments are destroyed left to right in the callee. As a special |
4207 | | // case, there are certain language constructs that require left-to-right |
4208 | | // evaluation, and in those cases we consider the evaluation order requirement |
4209 | | // to trump the "destruction order is reverse construction order" guarantee. |
4210 | 0 | bool LeftToRight = |
4211 | 316k | CGM.getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee() |
4212 | 316k | ? Order == EvaluationOrder::ForceLeftToRight7.36k |
4213 | 316k | : Order != EvaluationOrder::ForceRightToLeft309k ; |
4214 | | |
4215 | 316k | auto MaybeEmitImplicitObjectSize = [&](unsigned I, const Expr *Arg, |
4216 | 402k | RValue EmittedArg) { |
4217 | 402k | if (!AC.hasFunctionDecl() || I >= AC.getNumParams()371k ) |
4218 | 117k | return; |
4219 | 284k | auto *PS = AC.getParamDecl(I)->getAttr<PassObjectSizeAttr>(); |
4220 | 284k | if (PS == nullptr) |
4221 | 284k | return; |
4222 | | |
4223 | 107 | const auto &Context = getContext(); |
4224 | 107 | auto SizeTy = Context.getSizeType(); |
4225 | 107 | auto T = Builder.getIntNTy(Context.getTypeSize(SizeTy)); |
4226 | 107 | assert(EmittedArg.getScalarVal() && "We emitted nothing for the arg?"); |
4227 | 0 | llvm::Value *V = evaluateOrEmitBuiltinObjectSize(Arg, PS->getType(), T, |
4228 | 107 | EmittedArg.getScalarVal(), |
4229 | 107 | PS->isDynamic()); |
4230 | 107 | Args.add(RValue::get(V), SizeTy); |
4231 | | // If we're emitting args in reverse, be sure to do so with |
4232 | | // pass_object_size, as well. |
4233 | 107 | if (!LeftToRight) |
4234 | 9 | std::swap(Args.back(), *(&Args.back() - 1)); |
4235 | 107 | }; |
4236 | | |
4237 | | // Insert a stack save if we're going to need any inalloca args. |
4238 | 316k | if (hasInAllocaArgs(CGM, ExplicitCC, ArgTypes)) { |
4239 | 73 | assert(getTarget().getTriple().getArch() == llvm::Triple::x86 && |
4240 | 73 | "inalloca only supported on x86"); |
4241 | 0 | Args.allocateArgumentMemory(*this); |
4242 | 73 | } |
4243 | | |
4244 | | // Evaluate each argument in the appropriate order. |
4245 | 0 | size_t CallArgsStart = Args.size(); |
4246 | 719k | for (unsigned I = 0, E = ArgTypes.size(); I != E; ++I403k ) { |
4247 | 403k | unsigned Idx = LeftToRight ? I395k : E - I - 17.67k ; |
4248 | 403k | CallExpr::const_arg_iterator Arg = ArgRange.begin() + Idx; |
4249 | 403k | unsigned InitialArgSize = Args.size(); |
4250 | | // If *Arg is an ObjCIndirectCopyRestoreExpr, check that either the types of |
4251 | | // the argument and parameter match or the objc method is parameterized. |
4252 | 403k | assert((!isa<ObjCIndirectCopyRestoreExpr>(*Arg) || |
4253 | 403k | getContext().hasSameUnqualifiedType((*Arg)->getType(), |
4254 | 403k | ArgTypes[Idx]) || |
4255 | 403k | (isa<ObjCMethodDecl>(AC.getDecl()) && |
4256 | 403k | isObjCMethodWithTypeParams(cast<ObjCMethodDecl>(AC.getDecl())))) && |
4257 | 403k | "Argument and parameter types don't match"); |
4258 | 0 | EmitCallArg(Args, *Arg, ArgTypes[Idx]); |
4259 | | // In particular, we depend on it being the last arg in Args, and the |
4260 | | // objectsize bits depend on there only being one arg if !LeftToRight. |
4261 | 403k | assert(InitialArgSize + 1 == Args.size() && |
4262 | 403k | "The code below depends on only adding one arg per EmitCallArg"); |
4263 | 0 | (void)InitialArgSize; |
4264 | | // Since pointer argument are never emitted as LValue, it is safe to emit |
4265 | | // non-null argument check for r-value only. |
4266 | 403k | if (!Args.back().hasLValue()) { |
4267 | 402k | RValue RVArg = Args.back().getKnownRValue(); |
4268 | 402k | EmitNonNullArgCheck(RVArg, ArgTypes[Idx], (*Arg)->getExprLoc(), AC, |
4269 | 402k | ParamsToSkip + Idx); |
4270 | | // @llvm.objectsize should never have side-effects and shouldn't need |
4271 | | // destruction/cleanups, so we can safely "emit" it after its arg, |
4272 | | // regardless of right-to-leftness |
4273 | 402k | MaybeEmitImplicitObjectSize(Idx, *Arg, RVArg); |
4274 | 402k | } |
4275 | 403k | } |
4276 | | |
4277 | 316k | if (!LeftToRight) { |
4278 | | // Un-reverse the arguments we just evaluated so they match up with the LLVM |
4279 | | // IR function. |
4280 | 8.23k | std::reverse(Args.begin() + CallArgsStart, Args.end()); |
4281 | 8.23k | } |
4282 | 316k | } |
4283 | | |
4284 | | namespace { |
4285 | | |
4286 | | struct DestroyUnpassedArg final : EHScopeStack::Cleanup { |
4287 | | DestroyUnpassedArg(Address Addr, QualType Ty) |
4288 | 182 | : Addr(Addr), Ty(Ty) {} |
4289 | | |
4290 | | Address Addr; |
4291 | | QualType Ty; |
4292 | | |
4293 | 20 | void Emit(CodeGenFunction &CGF, Flags flags) override { |
4294 | 20 | QualType::DestructionKind DtorKind = Ty.isDestructedType(); |
4295 | 20 | if (DtorKind == QualType::DK_cxx_destructor) { |
4296 | 18 | const CXXDestructorDecl *Dtor = Ty->getAsCXXRecordDecl()->getDestructor(); |
4297 | 18 | assert(!Dtor->isTrivial()); |
4298 | 0 | CGF.EmitCXXDestructorCall(Dtor, Dtor_Complete, /*for vbase*/ false, |
4299 | 18 | /*Delegating=*/false, Addr, Ty); |
4300 | 18 | } else { |
4301 | 2 | CGF.callCStructDestructor(CGF.MakeAddrLValue(Addr, Ty)); |
4302 | 2 | } |
4303 | 20 | } |
4304 | | }; |
4305 | | |
4306 | | struct DisableDebugLocationUpdates { |
4307 | | CodeGenFunction &CGF; |
4308 | | bool disabledDebugInfo; |
4309 | 403k | DisableDebugLocationUpdates(CodeGenFunction &CGF, const Expr *E) : CGF(CGF) { |
4310 | 403k | if ((disabledDebugInfo = isa<CXXDefaultArgExpr>(E) && CGF.getDebugInfo()4.11k )) |
4311 | 2.80k | CGF.disableDebugInfo(); |
4312 | 403k | } |
4313 | 403k | ~DisableDebugLocationUpdates() { |
4314 | 403k | if (disabledDebugInfo) |
4315 | 2.80k | CGF.enableDebugInfo(); |
4316 | 403k | } |
4317 | | }; |
4318 | | |
4319 | | } // end anonymous namespace |
4320 | | |
4321 | 233 | RValue CallArg::getRValue(CodeGenFunction &CGF) const { |
4322 | 233 | if (!HasLV) |
4323 | 233 | return RV; |
4324 | 0 | LValue Copy = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty), Ty); |
4325 | 0 | CGF.EmitAggregateCopy(Copy, LV, Ty, AggValueSlot::DoesNotOverlap, |
4326 | 0 | LV.isVolatile()); |
4327 | 0 | IsUsed = true; |
4328 | 0 | return RValue::getAggregate(Copy.getAddress(CGF)); |
4329 | 233 | } |
4330 | | |
4331 | 1.21k | void CallArg::copyInto(CodeGenFunction &CGF, Address Addr) const { |
4332 | 1.21k | LValue Dst = CGF.MakeAddrLValue(Addr, Ty); |
4333 | 1.21k | if (!HasLV && RV.isScalar()1.15k ) |
4334 | 818 | CGF.EmitStoreOfScalar(RV.getScalarVal(), Dst, /*isInit=*/true); |
4335 | 401 | else if (!HasLV && RV.isComplex()338 ) |
4336 | 338 | CGF.EmitStoreOfComplex(RV.getComplexVal(), Dst, /*init=*/true); |
4337 | 63 | else { |
4338 | 63 | auto Addr = HasLV ? LV.getAddress(CGF) : RV.getAggregateAddress()0 ; |
4339 | 63 | LValue SrcLV = CGF.MakeAddrLValue(Addr, Ty); |
4340 | | // We assume that call args are never copied into subobjects. |
4341 | 63 | CGF.EmitAggregateCopy(Dst, SrcLV, Ty, AggValueSlot::DoesNotOverlap, |
4342 | 63 | HasLV ? LV.isVolatileQualified() |
4343 | 63 | : RV.isVolatileQualified()0 ); |
4344 | 63 | } |
4345 | 1.21k | IsUsed = true; |
4346 | 1.21k | } |
4347 | | |
4348 | | void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E, |
4349 | 403k | QualType type) { |
4350 | 403k | DisableDebugLocationUpdates Dis(*this, E); |
4351 | 403k | if (const ObjCIndirectCopyRestoreExpr *CRE |
4352 | 403k | = dyn_cast<ObjCIndirectCopyRestoreExpr>(E)) { |
4353 | 27 | assert(getLangOpts().ObjCAutoRefCount); |
4354 | 0 | return emitWritebackArg(*this, args, CRE); |
4355 | 27 | } |
4356 | | |
4357 | 403k | assert(type->isReferenceType() == E->isGLValue() && |
4358 | 403k | "reference binding to unmaterialized r-value!"); |
4359 | | |
4360 | 403k | if (E->isGLValue()) { |
4361 | 63.6k | assert(E->getObjectKind() == OK_Ordinary); |
4362 | 0 | return args.add(EmitReferenceBindingToExpr(E), type); |
4363 | 63.6k | } |
4364 | | |
4365 | 339k | bool HasAggregateEvalKind = hasAggregateEvaluationKind(type); |
4366 | | |
4367 | | // In the Microsoft C++ ABI, aggregate arguments are destructed by the callee. |
4368 | | // However, we still have to push an EH-only cleanup in case we unwind before |
4369 | | // we make it to the call. |
4370 | 339k | if (type->isRecordType() && |
4371 | 339k | type->castAs<RecordType>()->getDecl()->isParamDestroyedInCallee()10.7k ) { |
4372 | | // If we're using inalloca, use the argument memory. Otherwise, use a |
4373 | | // temporary. |
4374 | 417 | AggValueSlot Slot = args.isUsingInAlloca() |
4375 | 417 | ? createPlaceholderSlot(*this, type)106 : CreateAggTemp(type, "agg.tmp")311 ; |
4376 | | |
4377 | 417 | bool DestroyedInCallee = true, NeedsEHCleanup = true; |
4378 | 417 | if (const auto *RD = type->getAsCXXRecordDecl()) |
4379 | 395 | DestroyedInCallee = RD->hasNonTrivialDestructor(); |
4380 | 22 | else |
4381 | 22 | NeedsEHCleanup = needsEHCleanup(type.isDestructedType()); |
4382 | | |
4383 | 417 | if (DestroyedInCallee) |
4384 | 204 | Slot.setExternallyDestructed(); |
4385 | | |
4386 | 417 | EmitAggExpr(E, Slot); |
4387 | 417 | RValue RV = Slot.asRValue(); |
4388 | 417 | args.add(RV, type); |
4389 | | |
4390 | 417 | if (DestroyedInCallee && NeedsEHCleanup204 ) { |
4391 | | // Create a no-op GEP between the placeholder and the cleanup so we can |
4392 | | // RAUW it successfully. It also serves as a marker of the first |
4393 | | // instruction where the cleanup is active. |
4394 | 186 | pushFullExprCleanup<DestroyUnpassedArg>(EHCleanup, Slot.getAddress(), |
4395 | 186 | type); |
4396 | | // This unreachable is a temporary marker which will be removed later. |
4397 | 186 | llvm::Instruction *IsActive = Builder.CreateUnreachable(); |
4398 | 186 | args.addArgCleanupDeactivation(EHStack.stable_begin(), IsActive); |
4399 | 186 | } |
4400 | 417 | return; |
4401 | 417 | } |
4402 | | |
4403 | 339k | if (HasAggregateEvalKind && isa<ImplicitCastExpr>(E)10.3k && |
4404 | 339k | cast<CastExpr>(E)->getCastKind() == CK_LValueToRValue865 ) { |
4405 | 800 | LValue L = EmitLValue(cast<CastExpr>(E)->getSubExpr()); |
4406 | 800 | assert(L.isSimple()); |
4407 | 0 | args.addUncopiedAggregate(L, type); |
4408 | 800 | return; |
4409 | 800 | } |
4410 | | |
4411 | 338k | args.add(EmitAnyExprToTemp(E), type); |
4412 | 338k | } |
4413 | | |
4414 | 88.9k | QualType CodeGenFunction::getVarArgType(const Expr *Arg) { |
4415 | | // System headers on Windows define NULL to 0 instead of 0LL on Win64. MSVC |
4416 | | // implicitly widens null pointer constants that are arguments to varargs |
4417 | | // functions to pointer-sized ints. |
4418 | 88.9k | if (!getTarget().getTriple().isOSWindows()) |
4419 | 88.8k | return Arg->getType(); |
4420 | | |
4421 | 94 | if (Arg->getType()->isIntegerType() && |
4422 | 94 | getContext().getTypeSize(Arg->getType()) < |
4423 | 53 | getContext().getTargetInfo().getPointerWidth(0) && |
4424 | 94 | Arg->isNullPointerConstant(getContext(), |
4425 | 29 | Expr::NPC_ValueDependentIsNotNull)) { |
4426 | 1 | return getContext().getIntPtrType(); |
4427 | 1 | } |
4428 | | |
4429 | 93 | return Arg->getType(); |
4430 | 94 | } |
4431 | | |
4432 | | // In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC |
4433 | | // optimizer it can aggressively ignore unwind edges. |
4434 | | void |
4435 | 1.87k | CodeGenFunction::AddObjCARCExceptionMetadata(llvm::Instruction *Inst) { |
4436 | 1.87k | if (CGM.getCodeGenOpts().OptimizationLevel != 0 && |
4437 | 1.87k | !CGM.getCodeGenOpts().ObjCAutoRefCountExceptions468 ) |
4438 | 378 | Inst->setMetadata("clang.arc.no_objc_arc_exceptions", |
4439 | 378 | CGM.getNoObjCARCExceptionsMetadata()); |
4440 | 1.87k | } |
4441 | | |
4442 | | /// Emits a call to the given no-arguments nounwind runtime function. |
4443 | | llvm::CallInst * |
4444 | | CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee, |
4445 | 373 | const llvm::Twine &name) { |
4446 | 373 | return EmitNounwindRuntimeCall(callee, None, name); |
4447 | 373 | } |
4448 | | |
4449 | | /// Emits a call to the given nounwind runtime function. |
4450 | | llvm::CallInst * |
4451 | | CodeGenFunction::EmitNounwindRuntimeCall(llvm::FunctionCallee callee, |
4452 | | ArrayRef<llvm::Value *> args, |
4453 | 32.0k | const llvm::Twine &name) { |
4454 | 32.0k | llvm::CallInst *call = EmitRuntimeCall(callee, args, name); |
4455 | 32.0k | call->setDoesNotThrow(); |
4456 | 32.0k | return call; |
4457 | 32.0k | } |
4458 | | |
4459 | | /// Emits a simple call (never an invoke) to the given no-arguments |
4460 | | /// runtime function. |
4461 | | llvm::CallInst *CodeGenFunction::EmitRuntimeCall(llvm::FunctionCallee callee, |
4462 | 5.60k | const llvm::Twine &name) { |
4463 | 5.60k | return EmitRuntimeCall(callee, None, name); |
4464 | 5.60k | } |
4465 | | |
4466 | | // Calls which may throw must have operand bundles indicating which funclet |
4467 | | // they are nested within. |
4468 | | SmallVector<llvm::OperandBundleDef, 1> |
4469 | 464k | CodeGenFunction::getBundlesForFunclet(llvm::Value *Callee) { |
4470 | 464k | SmallVector<llvm::OperandBundleDef, 1> BundleList; |
4471 | | // There is no need for a funclet operand bundle if we aren't inside a |
4472 | | // funclet. |
4473 | 464k | if (!CurrentFuncletPad) |
4474 | 464k | return BundleList; |
4475 | | |
4476 | | // Skip intrinsics which cannot throw. |
4477 | 243 | auto *CalleeFn = dyn_cast<llvm::Function>(Callee->stripPointerCasts()); |
4478 | 243 | if (CalleeFn && CalleeFn->isIntrinsic()237 && CalleeFn->doesNotThrow()2 ) |
4479 | 2 | return BundleList; |
4480 | | |
4481 | 241 | BundleList.emplace_back("funclet", CurrentFuncletPad); |
4482 | 241 | return BundleList; |
4483 | 243 | } |
4484 | | |
4485 | | /// Emits a simple call (never an invoke) to the given runtime function. |
4486 | | llvm::CallInst *CodeGenFunction::EmitRuntimeCall(llvm::FunctionCallee callee, |
4487 | | ArrayRef<llvm::Value *> args, |
4488 | 93.3k | const llvm::Twine &name) { |
4489 | 93.3k | llvm::CallInst *call = Builder.CreateCall( |
4490 | 93.3k | callee, args, getBundlesForFunclet(callee.getCallee()), name); |
4491 | 93.3k | call->setCallingConv(getRuntimeCC()); |
4492 | 93.3k | return call; |
4493 | 93.3k | } |
4494 | | |
4495 | | /// Emits a call or invoke to the given noreturn runtime function. |
4496 | | void CodeGenFunction::EmitNoreturnRuntimeCallOrInvoke( |
4497 | 734 | llvm::FunctionCallee callee, ArrayRef<llvm::Value *> args) { |
4498 | 734 | SmallVector<llvm::OperandBundleDef, 1> BundleList = |
4499 | 734 | getBundlesForFunclet(callee.getCallee()); |
4500 | | |
4501 | 734 | if (getInvokeDest()) { |
4502 | 134 | llvm::InvokeInst *invoke = |
4503 | 134 | Builder.CreateInvoke(callee, |
4504 | 134 | getUnreachableBlock(), |
4505 | 134 | getInvokeDest(), |
4506 | 134 | args, |
4507 | 134 | BundleList); |
4508 | 134 | invoke->setDoesNotReturn(); |
4509 | 134 | invoke->setCallingConv(getRuntimeCC()); |
4510 | 600 | } else { |
4511 | 600 | llvm::CallInst *call = Builder.CreateCall(callee, args, BundleList); |
4512 | 600 | call->setDoesNotReturn(); |
4513 | 600 | call->setCallingConv(getRuntimeCC()); |
4514 | 600 | Builder.CreateUnreachable(); |
4515 | 600 | } |
4516 | 734 | } |
4517 | | |
4518 | | /// Emits a call or invoke instruction to the given nullary runtime function. |
4519 | | llvm::CallBase * |
4520 | | CodeGenFunction::EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee, |
4521 | 367 | const Twine &name) { |
4522 | 367 | return EmitRuntimeCallOrInvoke(callee, None, name); |
4523 | 367 | } |
4524 | | |
4525 | | /// Emits a call or invoke instruction to the given runtime function. |
4526 | | llvm::CallBase * |
4527 | | CodeGenFunction::EmitRuntimeCallOrInvoke(llvm::FunctionCallee callee, |
4528 | | ArrayRef<llvm::Value *> args, |
4529 | 623 | const Twine &name) { |
4530 | 623 | llvm::CallBase *call = EmitCallOrInvoke(callee, args, name); |
4531 | 623 | call->setCallingConv(getRuntimeCC()); |
4532 | 623 | return call; |
4533 | 623 | } |
4534 | | |
4535 | | /// Emits a call or invoke instruction to the given function, depending |
4536 | | /// on the current state of the EH stack. |
4537 | | llvm::CallBase *CodeGenFunction::EmitCallOrInvoke(llvm::FunctionCallee Callee, |
4538 | | ArrayRef<llvm::Value *> Args, |
4539 | 2.18k | const Twine &Name) { |
4540 | 2.18k | llvm::BasicBlock *InvokeDest = getInvokeDest(); |
4541 | 2.18k | SmallVector<llvm::OperandBundleDef, 1> BundleList = |
4542 | 2.18k | getBundlesForFunclet(Callee.getCallee()); |
4543 | | |
4544 | 2.18k | llvm::CallBase *Inst; |
4545 | 2.18k | if (!InvokeDest) |
4546 | 1.91k | Inst = Builder.CreateCall(Callee, Args, BundleList, Name); |
4547 | 267 | else { |
4548 | 267 | llvm::BasicBlock *ContBB = createBasicBlock("invoke.cont"); |
4549 | 267 | Inst = Builder.CreateInvoke(Callee, ContBB, InvokeDest, Args, BundleList, |
4550 | 267 | Name); |
4551 | 267 | EmitBlock(ContBB); |
4552 | 267 | } |
4553 | | |
4554 | | // In ObjC ARC mode with no ObjC ARC exception safety, tell the ARC |
4555 | | // optimizer it can aggressively ignore unwind edges. |
4556 | 2.18k | if (CGM.getLangOpts().ObjCAutoRefCount) |
4557 | 8 | AddObjCARCExceptionMetadata(Inst); |
4558 | | |
4559 | 2.18k | return Inst; |
4560 | 2.18k | } |
4561 | | |
4562 | | void CodeGenFunction::deferPlaceholderReplacement(llvm::Instruction *Old, |
4563 | 107 | llvm::Value *New) { |
4564 | 107 | DeferredReplacements.push_back( |
4565 | 107 | std::make_pair(llvm::WeakTrackingVH(Old), New)); |
4566 | 107 | } |
4567 | | |
4568 | | namespace { |
4569 | | |
4570 | | /// Specify given \p NewAlign as the alignment of return value attribute. If |
4571 | | /// such attribute already exists, re-set it to the maximal one of two options. |
4572 | | LLVM_NODISCARD llvm::AttributeList |
4573 | | maybeRaiseRetAlignmentAttribute(llvm::LLVMContext &Ctx, |
4574 | | const llvm::AttributeList &Attrs, |
4575 | 32 | llvm::Align NewAlign) { |
4576 | 32 | llvm::Align CurAlign = Attrs.getRetAlignment().valueOrOne(); |
4577 | 32 | if (CurAlign >= NewAlign) |
4578 | 2 | return Attrs; |
4579 | 30 | llvm::Attribute AlignAttr = llvm::Attribute::getWithAlignment(Ctx, NewAlign); |
4580 | 30 | return Attrs.removeRetAttribute(Ctx, llvm::Attribute::AttrKind::Alignment) |
4581 | 30 | .addRetAttribute(Ctx, AlignAttr); |
4582 | 32 | } |
4583 | | |
4584 | | template <typename AlignedAttrTy> class AbstractAssumeAlignedAttrEmitter { |
4585 | | protected: |
4586 | | CodeGenFunction &CGF; |
4587 | | |
4588 | | /// We do nothing if this is, or becomes, nullptr. |
4589 | | const AlignedAttrTy *AA = nullptr; |
4590 | | |
4591 | | llvm::Value *Alignment = nullptr; // May or may not be a constant. |
4592 | | llvm::ConstantInt *OffsetCI = nullptr; // Constant, hopefully zero. |
4593 | | |
4594 | | AbstractAssumeAlignedAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl) |
4595 | 733k | : CGF(CGF_) { |
4596 | 733k | if (!FuncDecl) |
4597 | 33.8k | return; |
4598 | 699k | AA = FuncDecl->getAttr<AlignedAttrTy>(); |
4599 | 699k | } CGCall.cpp:(anonymous namespace)::AbstractAssumeAlignedAttrEmitter<clang::AssumeAlignedAttr>::AbstractAssumeAlignedAttrEmitter(clang::CodeGen::CodeGenFunction&, clang::Decl const*) Line | Count | Source | 4595 | 366k | : CGF(CGF_) { | 4596 | 366k | if (!FuncDecl) | 4597 | 16.9k | return; | 4598 | 349k | AA = FuncDecl->getAttr<AlignedAttrTy>(); | 4599 | 349k | } |
CGCall.cpp:(anonymous namespace)::AbstractAssumeAlignedAttrEmitter<clang::AllocAlignAttr>::AbstractAssumeAlignedAttrEmitter(clang::CodeGen::CodeGenFunction&, clang::Decl const*) Line | Count | Source | 4595 | 366k | : CGF(CGF_) { | 4596 | 366k | if (!FuncDecl) | 4597 | 16.9k | return; | 4598 | 349k | AA = FuncDecl->getAttr<AlignedAttrTy>(); | 4599 | 349k | } |
|
4600 | | |
4601 | | public: |
4602 | | /// If we can, materialize the alignment as an attribute on return value. |
4603 | | LLVM_NODISCARD llvm::AttributeList |
4604 | 733k | TryEmitAsCallSiteAttribute(const llvm::AttributeList &Attrs) { |
4605 | 733k | if (!AA || OffsetCI63 || CGF.SanOpts.has(SanitizerKind::Alignment)58 ) |
4606 | 733k | return Attrs; |
4607 | 49 | const auto *AlignmentCI = dyn_cast<llvm::ConstantInt>(Alignment); |
4608 | 49 | if (!AlignmentCI) |
4609 | 16 | return Attrs; |
4610 | | // We may legitimately have non-power-of-2 alignment here. |
4611 | | // If so, this is UB land, emit it via `@llvm.assume` instead. |
4612 | 33 | if (!AlignmentCI->getValue().isPowerOf2()) |
4613 | 1 | return Attrs; |
4614 | 32 | llvm::AttributeList NewAttrs = maybeRaiseRetAlignmentAttribute( |
4615 | 32 | CGF.getLLVMContext(), Attrs, |
4616 | 32 | llvm::Align( |
4617 | 32 | AlignmentCI->getLimitedValue(llvm::Value::MaximumAlignment))); |
4618 | 32 | AA = nullptr; // We're done. Disallow doing anything else. |
4619 | 32 | return NewAttrs; |
4620 | 33 | } CGCall.cpp:(anonymous namespace)::AbstractAssumeAlignedAttrEmitter<clang::AssumeAlignedAttr>::TryEmitAsCallSiteAttribute(llvm::AttributeList const&) Line | Count | Source | 4604 | 366k | TryEmitAsCallSiteAttribute(const llvm::AttributeList &Attrs) { | 4605 | 366k | if (!AA || OffsetCI14 || CGF.SanOpts.has(SanitizerKind::Alignment)9 ) | 4606 | 366k | return Attrs; | 4607 | 6 | const auto *AlignmentCI = dyn_cast<llvm::ConstantInt>(Alignment); | 4608 | 6 | if (!AlignmentCI) | 4609 | 0 | return Attrs; | 4610 | | // We may legitimately have non-power-of-2 alignment here. | 4611 | | // If so, this is UB land, emit it via `@llvm.assume` instead. | 4612 | 6 | if (!AlignmentCI->getValue().isPowerOf2()) | 4613 | 0 | return Attrs; | 4614 | 6 | llvm::AttributeList NewAttrs = maybeRaiseRetAlignmentAttribute( | 4615 | 6 | CGF.getLLVMContext(), Attrs, | 4616 | 6 | llvm::Align( | 4617 | 6 | AlignmentCI->getLimitedValue(llvm::Value::MaximumAlignment))); | 4618 | 6 | AA = nullptr; // We're done. Disallow doing anything else. | 4619 | 6 | return NewAttrs; | 4620 | 6 | } |
CGCall.cpp:(anonymous namespace)::AbstractAssumeAlignedAttrEmitter<clang::AllocAlignAttr>::TryEmitAsCallSiteAttribute(llvm::AttributeList const&) Line | Count | Source | 4604 | 366k | TryEmitAsCallSiteAttribute(const llvm::AttributeList &Attrs) { | 4605 | 366k | if (!AA || OffsetCI49 || CGF.SanOpts.has(SanitizerKind::Alignment)49 ) | 4606 | 366k | return Attrs; | 4607 | 43 | const auto *AlignmentCI = dyn_cast<llvm::ConstantInt>(Alignment); | 4608 | 43 | if (!AlignmentCI) | 4609 | 16 | return Attrs; | 4610 | | // We may legitimately have non-power-of-2 alignment here. | 4611 | | // If so, this is UB land, emit it via `@llvm.assume` instead. | 4612 | 27 | if (!AlignmentCI->getValue().isPowerOf2()) | 4613 | 1 | return Attrs; | 4614 | 26 | llvm::AttributeList NewAttrs = maybeRaiseRetAlignmentAttribute( | 4615 | 26 | CGF.getLLVMContext(), Attrs, | 4616 | 26 | llvm::Align( | 4617 | 26 | AlignmentCI->getLimitedValue(llvm::Value::MaximumAlignment))); | 4618 | 26 | AA = nullptr; // We're done. Disallow doing anything else. | 4619 | 26 | return NewAttrs; | 4620 | 27 | } |
|
4621 | | |
4622 | | /// Emit alignment assumption. |
4623 | | /// This is a general fallback that we take if either there is an offset, |
4624 | | /// or the alignment is variable or we are sanitizing for alignment. |
4625 | 679k | void EmitAsAnAssumption(SourceLocation Loc, QualType RetTy, RValue &Ret) { |
4626 | 679k | if (!AA) |
4627 | 679k | return; |
4628 | 31 | CGF.emitAlignmentAssumption(Ret.getScalarVal(), RetTy, Loc, |
4629 | 31 | AA->getLocation(), Alignment, OffsetCI); |
4630 | 31 | AA = nullptr; // We're done. Disallow doing anything else. |
4631 | 31 | } CGCall.cpp:(anonymous namespace)::AbstractAssumeAlignedAttrEmitter<clang::AssumeAlignedAttr>::EmitAsAnAssumption(clang::SourceLocation, clang::QualType, clang::CodeGen::RValue&) Line | Count | Source | 4625 | 339k | void EmitAsAnAssumption(SourceLocation Loc, QualType RetTy, RValue &Ret) { | 4626 | 339k | if (!AA) | 4627 | 339k | return; | 4628 | 8 | CGF.emitAlignmentAssumption(Ret.getScalarVal(), RetTy, Loc, | 4629 | 8 | AA->getLocation(), Alignment, OffsetCI); | 4630 | 8 | AA = nullptr; // We're done. Disallow doing anything else. | 4631 | 8 | } |
CGCall.cpp:(anonymous namespace)::AbstractAssumeAlignedAttrEmitter<clang::AllocAlignAttr>::EmitAsAnAssumption(clang::SourceLocation, clang::QualType, clang::CodeGen::RValue&) Line | Count | Source | 4625 | 339k | void EmitAsAnAssumption(SourceLocation Loc, QualType RetTy, RValue &Ret) { | 4626 | 339k | if (!AA) | 4627 | 339k | return; | 4628 | 23 | CGF.emitAlignmentAssumption(Ret.getScalarVal(), RetTy, Loc, | 4629 | 23 | AA->getLocation(), Alignment, OffsetCI); | 4630 | 23 | AA = nullptr; // We're done. Disallow doing anything else. | 4631 | 23 | } |
|
4632 | | }; |
4633 | | |
4634 | | /// Helper data structure to emit `AssumeAlignedAttr`. |
4635 | | class AssumeAlignedAttrEmitter final |
4636 | | : public AbstractAssumeAlignedAttrEmitter<AssumeAlignedAttr> { |
4637 | | public: |
4638 | | AssumeAlignedAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl) |
4639 | 366k | : AbstractAssumeAlignedAttrEmitter(CGF_, FuncDecl) { |
4640 | 366k | if (!AA) |
4641 | 366k | return; |
4642 | | // It is guaranteed that the alignment/offset are constants. |
4643 | 14 | Alignment = cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AA->getAlignment())); |
4644 | 14 | if (Expr *Offset = AA->getOffset()) { |
4645 | 5 | OffsetCI = cast<llvm::ConstantInt>(CGF.EmitScalarExpr(Offset)); |
4646 | 5 | if (OffsetCI->isNullValue()) // Canonicalize zero offset to no offset. |
4647 | 0 | OffsetCI = nullptr; |
4648 | 5 | } |
4649 | 14 | } |
4650 | | }; |
4651 | | |
4652 | | /// Helper data structure to emit `AllocAlignAttr`. |
4653 | | class AllocAlignAttrEmitter final |
4654 | | : public AbstractAssumeAlignedAttrEmitter<AllocAlignAttr> { |
4655 | | public: |
4656 | | AllocAlignAttrEmitter(CodeGenFunction &CGF_, const Decl *FuncDecl, |
4657 | | const CallArgList &CallArgs) |
4658 | 366k | : AbstractAssumeAlignedAttrEmitter(CGF_, FuncDecl) { |
4659 | 366k | if (!AA) |
4660 | 366k | return; |
4661 | | // Alignment may or may not be a constant, and that is okay. |
4662 | 49 | Alignment = CallArgs[AA->getParamIndex().getLLVMIndex()] |
4663 | 49 | .getRValue(CGF) |
4664 | 49 | .getScalarVal(); |
4665 | 49 | } |
4666 | | }; |
4667 | | |
4668 | | } // namespace |
4669 | | |
4670 | 985k | static unsigned getMaxVectorWidth(const llvm::Type *Ty) { |
4671 | 985k | if (auto *VT = dyn_cast<llvm::VectorType>(Ty)) |
4672 | 47.2k | return VT->getPrimitiveSizeInBits().getKnownMinSize(); |
4673 | 938k | if (auto *AT = dyn_cast<llvm::ArrayType>(Ty)) |
4674 | 530 | return getMaxVectorWidth(AT->getElementType()); |
4675 | | |
4676 | 937k | unsigned MaxVectorWidth = 0; |
4677 | 937k | if (auto *ST = dyn_cast<llvm::StructType>(Ty)) |
4678 | 2.37k | for (auto *I : ST->elements()) |
4679 | 4.91k | MaxVectorWidth = std::max(MaxVectorWidth, getMaxVectorWidth(I)); |
4680 | 937k | return MaxVectorWidth; |
4681 | 938k | } |
4682 | | |
4683 | | RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo, |
4684 | | const CGCallee &Callee, |
4685 | | ReturnValueSlot ReturnValue, |
4686 | | const CallArgList &CallArgs, |
4687 | | llvm::CallBase **callOrInvoke, bool IsMustTail, |
4688 | 366k | SourceLocation Loc) { |
4689 | | // FIXME: We no longer need the types from CallArgs; lift up and simplify. |
4690 | | |
4691 | 366k | assert(Callee.isOrdinary() || Callee.isVirtual()); |
4692 | | |
4693 | | // Handle struct-return functions by passing a pointer to the |
4694 | | // location that we would like to return into. |
4695 | 0 | QualType RetTy = CallInfo.getReturnType(); |
4696 | 366k | const ABIArgInfo &RetAI = CallInfo.getReturnInfo(); |
4697 | | |
4698 | 366k | llvm::FunctionType *IRFuncTy = getTypes().GetFunctionType(CallInfo); |
4699 | | |
4700 | 366k | const Decl *TargetDecl = Callee.getAbstractInfo().getCalleeDecl().getDecl(); |
4701 | 366k | if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(TargetDecl)) { |
4702 | | // We can only guarantee that a function is called from the correct |
4703 | | // context/function based on the appropriate target attributes, |
4704 | | // so only check in the case where we have both always_inline and target |
4705 | | // since otherwise we could be making a conditional call after a check for |
4706 | | // the proper cpu features (and it won't cause code generation issues due to |
4707 | | // function based code generation). |
4708 | 344k | if (TargetDecl->hasAttr<AlwaysInlineAttr>() && |
4709 | 344k | TargetDecl->hasAttr<TargetAttr>()21.9k ) |
4710 | 14.8k | checkTargetFeatures(Loc, FD); |
4711 | | |
4712 | | // Some architectures (such as x86-64) have the ABI changed based on |
4713 | | // attribute-target/features. Give them a chance to diagnose. |
4714 | 344k | CGM.getTargetCodeGenInfo().checkFunctionCallABI( |
4715 | 344k | CGM, Loc, dyn_cast_or_null<FunctionDecl>(CurCodeDecl), FD, CallArgs); |
4716 | 344k | } |
4717 | | |
4718 | 366k | #ifndef NDEBUG |
4719 | 366k | if (!(CallInfo.isVariadic() && CallInfo.getArgStruct()77.5k )) { |
4720 | | // For an inalloca varargs function, we don't expect CallInfo to match the |
4721 | | // function pointer's type, because the inalloca struct a will have extra |
4722 | | // fields in it for the varargs parameters. Code later in this function |
4723 | | // bitcasts the function pointer to the type derived from CallInfo. |
4724 | | // |
4725 | | // In other cases, we assert that the types match up (until pointers stop |
4726 | | // having pointee types). |
4727 | 366k | if (Callee.isVirtual()) |
4728 | 957 | assert(IRFuncTy == Callee.getVirtualFunctionType()); |
4729 | 365k | else { |
4730 | 365k | llvm::PointerType *PtrTy = |
4731 | 365k | llvm::cast<llvm::PointerType>(Callee.getFunctionPointer()->getType()); |
4732 | 365k | assert(PtrTy->isOpaqueOrPointeeTypeMatches(IRFuncTy)); |
4733 | 365k | } |
4734 | 366k | } |
4735 | 0 | #endif |
4736 | | |
4737 | | // 1. Set up the arguments. |
4738 | | |
4739 | | // If we're using inalloca, insert the allocation after the stack save. |
4740 | | // FIXME: Do this earlier rather than hacking it in here! |
4741 | 0 | Address ArgMemory = Address::invalid(); |
4742 | 366k | if (llvm::StructType *ArgStruct = CallInfo.getArgStruct()) { |
4743 | 74 | const llvm::DataLayout &DL = CGM.getDataLayout(); |
4744 | 74 | llvm::Instruction *IP = CallArgs.getStackBase(); |
4745 | 74 | llvm::AllocaInst *AI; |
4746 | 74 | if (IP) { |
4747 | 73 | IP = IP->getNextNode(); |
4748 | 73 | AI = new llvm::AllocaInst(ArgStruct, DL.getAllocaAddrSpace(), |
4749 | 73 | "argmem", IP); |
4750 | 73 | } else { |
4751 | 1 | AI = CreateTempAlloca(ArgStruct, "argmem"); |
4752 | 1 | } |
4753 | 74 | auto Align = CallInfo.getArgStructAlignment(); |
4754 | 74 | AI->setAlignment(Align.getAsAlign()); |
4755 | 74 | AI->setUsedWithInAlloca(true); |
4756 | 74 | assert(AI->isUsedWithInAlloca() && !AI->isStaticAlloca()); |
4757 | 0 | ArgMemory = Address(AI, ArgStruct, Align); |
4758 | 74 | } |
4759 | | |
4760 | 0 | ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), CallInfo); |
4761 | 366k | SmallVector<llvm::Value *, 16> IRCallArgs(IRFunctionArgs.totalIRArgs()); |
4762 | | |
4763 | | // If the call returns a temporary with struct return, create a temporary |
4764 | | // alloca to hold the result, unless one is given to us. |
4765 | 366k | Address SRetPtr = Address::invalid(); |
4766 | 366k | Address SRetAlloca = Address::invalid(); |
4767 | 366k | llvm::Value *UnusedReturnSizePtr = nullptr; |
4768 | 366k | if (RetAI.isIndirect() || RetAI.isInAlloca()364k || RetAI.isCoerceAndExpand()364k ) { |
4769 | 2.66k | if (!ReturnValue.isNull()) { |
4770 | 1.87k | SRetPtr = ReturnValue.getValue(); |
4771 | 1.87k | } else { |
4772 | 791 | SRetPtr = CreateMemTemp(RetTy, "tmp", &SRetAlloca); |
4773 | 791 | if (HaveInsertPoint() && ReturnValue.isUnused()) { |
4774 | 130 | llvm::TypeSize size = |
4775 | 130 | CGM.getDataLayout().getTypeAllocSize(ConvertTypeForMem(RetTy)); |
4776 | 130 | UnusedReturnSizePtr = EmitLifetimeStart(size, SRetAlloca.getPointer()); |
4777 | 130 | } |
4778 | 791 | } |
4779 | 2.66k | if (IRFunctionArgs.hasSRetArg()) { |
4780 | 2.21k | IRCallArgs[IRFunctionArgs.getSRetArgNo()] = SRetPtr.getPointer(); |
4781 | 2.21k | } else if (446 RetAI.isInAlloca()446 ) { |
4782 | 4 | Address Addr = |
4783 | 4 | Builder.CreateStructGEP(ArgMemory, RetAI.getInAllocaFieldIndex()); |
4784 | 4 | Builder.CreateStore(SRetPtr.getPointer(), Addr); |
4785 | 4 | } |
4786 | 2.66k | } |
4787 | | |
4788 | 366k | Address swiftErrorTemp = Address::invalid(); |
4789 | 366k | Address swiftErrorArg = Address::invalid(); |
4790 | | |
4791 | | // When passing arguments using temporary allocas, we need to add the |
4792 | | // appropriate lifetime markers. This vector keeps track of all the lifetime |
4793 | | // markers that need to be ended right after the call. |
4794 | 366k | SmallVector<CallLifetimeEnd, 2> CallLifetimeEndAfterCall; |
4795 | | |
4796 | | // Translate all of the arguments as necessary to match the IR lowering. |
4797 | 366k | assert(CallInfo.arg_size() == CallArgs.size() && |
4798 | 366k | "Mismatch between function signature & arguments."); |
4799 | 0 | unsigned ArgNo = 0; |
4800 | 366k | CGFunctionInfo::const_arg_iterator info_it = CallInfo.arg_begin(); |
4801 | 366k | for (CallArgList::const_iterator I = CallArgs.begin(), E = CallArgs.end(); |
4802 | 983k | I != E; ++I, ++info_it, ++ArgNo616k ) { |
4803 | 616k | const ABIArgInfo &ArgInfo = info_it->info; |
4804 | | |
4805 | | // Insert a padding argument to ensure proper alignment. |
4806 | 616k | if (IRFunctionArgs.hasPaddingArg(ArgNo)) |
4807 | 17 | IRCallArgs[IRFunctionArgs.getPaddingArgNo(ArgNo)] = |
4808 | 17 | llvm::UndefValue::get(ArgInfo.getPaddingType()); |
4809 | | |
4810 | 616k | unsigned FirstIRArg, NumIRArgs; |
4811 | 616k | std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo); |
4812 | | |
4813 | 616k | switch (ArgInfo.getKind()) { |
4814 | 160 | case ABIArgInfo::InAlloca: { |
4815 | 160 | assert(NumIRArgs == 0); |
4816 | 0 | assert(getTarget().getTriple().getArch() == llvm::Triple::x86); |
4817 | 160 | if (I->isAggregate()) { |
4818 | 107 | Address Addr = I->hasLValue() |
4819 | 107 | ? I->getKnownLValue().getAddress(*this)0 |
4820 | 107 | : I->getKnownRValue().getAggregateAddress(); |
4821 | 107 | llvm::Instruction *Placeholder = |
4822 | 107 | cast<llvm::Instruction>(Addr.getPointer()); |
4823 | | |
4824 | 107 | if (!ArgInfo.getInAllocaIndirect()) { |
4825 | | // Replace the placeholder with the appropriate argument slot GEP. |
4826 | 105 | CGBuilderTy::InsertPoint IP = Builder.saveIP(); |
4827 | 105 | Builder.SetInsertPoint(Placeholder); |
4828 | 105 | Addr = Builder.CreateStructGEP(ArgMemory, |
4829 | 105 | ArgInfo.getInAllocaFieldIndex()); |
4830 | 105 | Builder.restoreIP(IP); |
4831 | 105 | } else { |
4832 | | // For indirect things such as overaligned structs, replace the |
4833 | | // placeholder with a regular aggregate temporary alloca. Store the |
4834 | | // address of this alloca into the struct. |
4835 | 2 | Addr = CreateMemTemp(info_it->type, "inalloca.indirect.tmp"); |
4836 | 2 | Address ArgSlot = Builder.CreateStructGEP( |
4837 | 2 | ArgMemory, ArgInfo.getInAllocaFieldIndex()); |
4838 | 2 | Builder.CreateStore(Addr.getPointer(), ArgSlot); |
4839 | 2 | } |
4840 | 107 | deferPlaceholderReplacement(Placeholder, Addr.getPointer()); |
4841 | 107 | } else if (53 ArgInfo.getInAllocaIndirect()53 ) { |
4842 | | // Make a temporary alloca and store the address of it into the argument |
4843 | | // struct. |
4844 | 2 | Address Addr = CreateMemTempWithoutCast( |
4845 | 2 | I->Ty, getContext().getTypeAlignInChars(I->Ty), |
4846 | 2 | "indirect-arg-temp"); |
4847 | 2 | I->copyInto(*this, Addr); |
4848 | 2 | Address ArgSlot = |
4849 | 2 | Builder.CreateStructGEP(ArgMemory, ArgInfo.getInAllocaFieldIndex()); |
4850 | 2 | Builder.CreateStore(Addr.getPointer(), ArgSlot); |
4851 | 51 | } else { |
4852 | | // Store the RValue into the argument struct. |
4853 | 51 | Address Addr = |
4854 | 51 | Builder.CreateStructGEP(ArgMemory, ArgInfo.getInAllocaFieldIndex()); |
4855 | | // There are some cases where a trivial bitcast is not avoidable. The |
4856 | | // definition of a type later in a translation unit may change it's type |
4857 | | // from {}* to (%struct.foo*)*. |
4858 | 51 | Addr = Builder.CreateElementBitCast(Addr, ConvertTypeForMem(I->Ty)); |
4859 | 51 | I->copyInto(*this, Addr); |
4860 | 51 | } |
4861 | 160 | break; |
4862 | 0 | } |
4863 | | |
4864 | 2.07k | case ABIArgInfo::Indirect: |
4865 | 2.07k | case ABIArgInfo::IndirectAliased: { |
4866 | 2.07k | assert(NumIRArgs == 1); |
4867 | 2.07k | if (!I->isAggregate()) { |
4868 | | // Make a temporary alloca to pass the argument. |
4869 | 232 | Address Addr = CreateMemTempWithoutCast( |
4870 | 232 | I->Ty, ArgInfo.getIndirectAlign(), "indirect-arg-temp"); |
4871 | 232 | IRCallArgs[FirstIRArg] = Addr.getPointer(); |
4872 | | |
4873 | 232 | I->copyInto(*this, Addr); |
4874 | 1.84k | } else { |
4875 | | // We want to avoid creating an unnecessary temporary+copy here; |
4876 | | // however, we need one in three cases: |
4877 | | // 1. If the argument is not byval, and we are required to copy the |
4878 | | // source. (This case doesn't occur on any common architecture.) |
4879 | | // 2. If the argument is byval, RV is not sufficiently aligned, and |
4880 | | // we cannot force it to be sufficiently aligned. |
4881 | | // 3. If the argument is byval, but RV is not located in default |
4882 | | // or alloca address space. |
4883 | 1.84k | Address Addr = I->hasLValue() |
4884 | 1.84k | ? I->getKnownLValue().getAddress(*this)313 |
4885 | 1.84k | : I->getKnownRValue().getAggregateAddress()1.53k ; |
4886 | 1.84k | llvm::Value *V = Addr.getPointer(); |
4887 | 1.84k | CharUnits Align = ArgInfo.getIndirectAlign(); |
4888 | 1.84k | const llvm::DataLayout *TD = &CGM.getDataLayout(); |
4889 | | |
4890 | 1.84k | assert((FirstIRArg >= IRFuncTy->getNumParams() || |
4891 | 1.84k | IRFuncTy->getParamType(FirstIRArg)->getPointerAddressSpace() == |
4892 | 1.84k | TD->getAllocaAddrSpace()) && |
4893 | 1.84k | "indirect argument must be in alloca address space"); |
4894 | | |
4895 | 0 | bool NeedCopy = false; |
4896 | | |
4897 | 1.84k | if (Addr.getAlignment() < Align && |
4898 | 1.84k | llvm::getOrEnforceKnownAlignment(V, Align.getAsAlign(), *TD) < |
4899 | 88 | Align.getAsAlign()) { |
4900 | 2 | NeedCopy = true; |
4901 | 1.84k | } else if (I->hasLValue()) { |
4902 | 311 | auto LV = I->getKnownLValue(); |
4903 | 311 | auto AS = LV.getAddressSpace(); |
4904 | | |
4905 | 311 | if (!ArgInfo.getIndirectByVal() || |
4906 | 311 | (LV.getAlignment() < getContext().getTypeAlignInChars(I->Ty))268 ) { |
4907 | 43 | NeedCopy = true; |
4908 | 43 | } |
4909 | 311 | if (!getLangOpts().OpenCL) { |
4910 | 270 | if ((ArgInfo.getIndirectByVal() && |
4911 | 270 | (227 AS != LangAS::Default227 && |
4912 | 227 | AS != CGM.getASTAllocaAddressSpace()0 ))) { |
4913 | 0 | NeedCopy = true; |
4914 | 0 | } |
4915 | 270 | } |
4916 | | // For OpenCL even if RV is located in default or alloca address space |
4917 | | // we don't want to perform address space cast for it. |
4918 | 41 | else if ((ArgInfo.getIndirectByVal() && |
4919 | 41 | Addr.getType()->getAddressSpace() != IRFuncTy-> |
4920 | 41 | getParamType(FirstIRArg)->getPointerAddressSpace())) { |
4921 | 18 | NeedCopy = true; |
4922 | 18 | } |
4923 | 311 | } |
4924 | | |
4925 | 1.84k | if (NeedCopy) { |
4926 | | // Create an aligned temporary, and copy to it. |
4927 | 63 | Address AI = CreateMemTempWithoutCast( |
4928 | 63 | I->Ty, ArgInfo.getIndirectAlign(), "byval-temp"); |
4929 | 63 | IRCallArgs[FirstIRArg] = AI.getPointer(); |
4930 | | |
4931 | | // Emit lifetime markers for the temporary alloca. |
4932 | 63 | llvm::TypeSize ByvalTempElementSize = |
4933 | 63 | CGM.getDataLayout().getTypeAllocSize(AI.getElementType()); |
4934 | 63 | llvm::Value *LifetimeSize = |
4935 | 63 | EmitLifetimeStart(ByvalTempElementSize, AI.getPointer()); |
4936 | | |
4937 | | // Add cleanup code to emit the end lifetime marker after the call. |
4938 | 63 | if (LifetimeSize) // In case we disabled lifetime markers. |
4939 | 2 | CallLifetimeEndAfterCall.emplace_back(AI, LifetimeSize); |
4940 | | |
4941 | | // Generate the copy. |
4942 | 63 | I->copyInto(*this, AI); |
4943 | 1.78k | } else { |
4944 | | // Skip the extra memcpy call. |
4945 | 1.78k | auto *T = llvm::PointerType::getWithSamePointeeType( |
4946 | 1.78k | cast<llvm::PointerType>(V->getType()), |
4947 | 1.78k | CGM.getDataLayout().getAllocaAddrSpace()); |
4948 | 1.78k | IRCallArgs[FirstIRArg] = getTargetHooks().performAddrSpaceCast( |
4949 | 1.78k | *this, V, LangAS::Default, CGM.getASTAllocaAddressSpace(), T, |
4950 | 1.78k | true); |
4951 | 1.78k | } |
4952 | 1.84k | } |
4953 | 0 | break; |
4954 | 2.07k | } |
4955 | | |
4956 | 6.99k | case ABIArgInfo::Ignore: |
4957 | 6.99k | assert(NumIRArgs == 0); |
4958 | 0 | break; |
4959 | | |
4960 | 9.45k | case ABIArgInfo::Extend: |
4961 | 606k | case ABIArgInfo::Direct: { |
4962 | 606k | if (!isa<llvm::StructType>(ArgInfo.getCoerceToType()) && |
4963 | 606k | ArgInfo.getCoerceToType() == ConvertType(info_it->type)605k && |
4964 | 606k | ArgInfo.getDirectOffset() == 0602k ) { |
4965 | 602k | assert(NumIRArgs == 1); |
4966 | 0 | llvm::Value *V; |
4967 | 602k | if (!I->isAggregate()) |
4968 | 602k | V = I->getKnownRValue().getScalarVal(); |
4969 | 0 | else |
4970 | 0 | V = Builder.CreateLoad( |
4971 | 0 | I->hasLValue() ? I->getKnownLValue().getAddress(*this) |
4972 | 0 | : I->getKnownRValue().getAggregateAddress()); |
4973 | | |
4974 | | // Implement swifterror by copying into a new swifterror argument. |
4975 | | // We'll write back in the normal path out of the call. |
4976 | 602k | if (CallInfo.getExtParameterInfo(ArgNo).getABI() |
4977 | 602k | == ParameterABI::SwiftErrorResult) { |
4978 | 8 | assert(!swiftErrorTemp.isValid() && "multiple swifterror args"); |
4979 | | |
4980 | 0 | QualType pointeeTy = I->Ty->getPointeeType(); |
4981 | 8 | swiftErrorArg = Address(V, ConvertTypeForMem(pointeeTy), |
4982 | 8 | getContext().getTypeAlignInChars(pointeeTy)); |
4983 | | |
4984 | 8 | swiftErrorTemp = |
4985 | 8 | CreateMemTemp(pointeeTy, getPointerAlign(), "swifterror.temp"); |
4986 | 8 | V = swiftErrorTemp.getPointer(); |
4987 | 8 | cast<llvm::AllocaInst>(V)->setSwiftError(true); |
4988 | | |
4989 | 8 | llvm::Value *errorValue = Builder.CreateLoad(swiftErrorArg); |
4990 | 8 | Builder.CreateStore(errorValue, swiftErrorTemp); |
4991 | 8 | } |
4992 | | |
4993 | | // We might have to widen integers, but we should never truncate. |
4994 | 602k | if (ArgInfo.getCoerceToType() != V->getType() && |
4995 | 602k | V->getType()->isIntegerTy()227 ) |
4996 | 3 | V = Builder.CreateZExt(V, ArgInfo.getCoerceToType()); |
4997 | | |
4998 | | // If the argument doesn't match, perform a bitcast to coerce it. This |
4999 | | // can happen due to trivial type mismatches. |
5000 | 602k | if (FirstIRArg < IRFuncTy->getNumParams() && |
5001 | 602k | V->getType() != IRFuncTy->getParamType(FirstIRArg)513k ) |
5002 | 224 | V = Builder.CreateBitCast(V, IRFuncTy->getParamType(FirstIRArg)); |
5003 | | |
5004 | 602k | IRCallArgs[FirstIRArg] = V; |
5005 | 602k | break; |
5006 | 602k | } |
5007 | | |
5008 | | // FIXME: Avoid the conversion through memory if possible. |
5009 | 4.65k | Address Src = Address::invalid(); |
5010 | 4.65k | if (!I->isAggregate()) { |
5011 | 871 | Src = CreateMemTemp(I->Ty, "coerce"); |
5012 | 871 | I->copyInto(*this, Src); |
5013 | 3.78k | } else { |
5014 | 3.78k | Src = I->hasLValue() ? I->getKnownLValue().getAddress(*this)468 |
5015 | 3.78k | : I->getKnownRValue().getAggregateAddress()3.31k ; |
5016 | 3.78k | } |
5017 | | |
5018 | | // If the value is offset in memory, apply the offset now. |
5019 | 4.65k | Src = emitAddressAtOffset(*this, Src, ArgInfo); |
5020 | | |
5021 | | // Fast-isel and the optimizer generally like scalar values better than |
5022 | | // FCAs, so we flatten them if this is safe to do for this argument. |
5023 | 4.65k | llvm::StructType *STy = |
5024 | 4.65k | dyn_cast<llvm::StructType>(ArgInfo.getCoerceToType()); |
5025 | 4.65k | if (STy && ArgInfo.isDirect()1.29k && ArgInfo.getCanBeFlattened()1.29k ) { |
5026 | 1.26k | llvm::Type *SrcTy = Src.getElementType(); |
5027 | 1.26k | uint64_t SrcSize = CGM.getDataLayout().getTypeAllocSize(SrcTy); |
5028 | 1.26k | uint64_t DstSize = CGM.getDataLayout().getTypeAllocSize(STy); |
5029 | | |
5030 | | // If the source type is smaller than the destination type of the |
5031 | | // coerce-to logic, copy the source value into a temp alloca the size |
5032 | | // of the destination type to allow loading all of it. The bits past |
5033 | | // the source value are left undef. |
5034 | 1.26k | if (SrcSize < DstSize) { |
5035 | 184 | Address TempAlloca |
5036 | 184 | = CreateTempAlloca(STy, Src.getAlignment(), |
5037 | 184 | Src.getName() + ".coerce"); |
5038 | 184 | Builder.CreateMemCpy(TempAlloca, Src, SrcSize); |
5039 | 184 | Src = TempAlloca; |
5040 | 1.08k | } else { |
5041 | 1.08k | Src = Builder.CreateElementBitCast(Src, STy); |
5042 | 1.08k | } |
5043 | | |
5044 | 1.26k | assert(NumIRArgs == STy->getNumElements()); |
5045 | 3.80k | for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i2.53k ) { |
5046 | 2.53k | Address EltPtr = Builder.CreateStructGEP(Src, i); |
5047 | 2.53k | llvm::Value *LI = Builder.CreateLoad(EltPtr); |
5048 | 2.53k | IRCallArgs[FirstIRArg + i] = LI; |
5049 | 2.53k | } |
5050 | 3.38k | } else { |
5051 | | // In the simple case, just pass the coerced loaded value. |
5052 | 3.38k | assert(NumIRArgs == 1); |
5053 | 0 | llvm::Value *Load = |
5054 | 3.38k | CreateCoercedLoad(Src, ArgInfo.getCoerceToType(), *this); |
5055 | | |
5056 | 3.38k | if (CallInfo.isCmseNSCall()) { |
5057 | | // For certain parameter types, clear padding bits, as they may reveal |
5058 | | // sensitive information. |
5059 | | // Small struct/union types are passed as integer arrays. |
5060 | 40 | auto *ATy = dyn_cast<llvm::ArrayType>(Load->getType()); |
5061 | 40 | if (ATy != nullptr && isa<RecordType>(I->Ty.getCanonicalType())39 ) |
5062 | 39 | Load = EmitCMSEClearRecord(Load, ATy, I->Ty); |
5063 | 40 | } |
5064 | 3.38k | IRCallArgs[FirstIRArg] = Load; |
5065 | 3.38k | } |
5066 | | |
5067 | 0 | break; |
5068 | 606k | } |
5069 | | |
5070 | 442 | case ABIArgInfo::CoerceAndExpand: { |
5071 | 442 | auto coercionType = ArgInfo.getCoerceAndExpandType(); |
|