Coverage Report

Created: 2020-09-15 12:33

/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/CodeGen/CGDecl.cpp
Line
Count
Source (jump to first uncovered line)
1
//===--- CGDecl.cpp - Emit LLVM Code for declarations ---------------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This contains code to emit Decl nodes as LLVM code.
10
//
11
//===----------------------------------------------------------------------===//
12
13
#include "CGBlocks.h"
14
#include "CGCXXABI.h"
15
#include "CGCleanup.h"
16
#include "CGDebugInfo.h"
17
#include "CGOpenCLRuntime.h"
18
#include "CGOpenMPRuntime.h"
19
#include "CodeGenFunction.h"
20
#include "CodeGenModule.h"
21
#include "ConstantEmitter.h"
22
#include "PatternInit.h"
23
#include "TargetInfo.h"
24
#include "clang/AST/ASTContext.h"
25
#include "clang/AST/Attr.h"
26
#include "clang/AST/CharUnits.h"
27
#include "clang/AST/Decl.h"
28
#include "clang/AST/DeclObjC.h"
29
#include "clang/AST/DeclOpenMP.h"
30
#include "clang/Basic/CodeGenOptions.h"
31
#include "clang/Basic/SourceManager.h"
32
#include "clang/Basic/TargetInfo.h"
33
#include "clang/CodeGen/CGFunctionInfo.h"
34
#include "clang/Sema/Sema.h"
35
#include "llvm/Analysis/ValueTracking.h"
36
#include "llvm/IR/DataLayout.h"
37
#include "llvm/IR/GlobalVariable.h"
38
#include "llvm/IR/Intrinsics.h"
39
#include "llvm/IR/Type.h"
40
41
using namespace clang;
42
using namespace CodeGen;
43
44
static_assert(clang::Sema::MaximumAlignment <= llvm::Value::MaximumAlignment,
45
              "Clang max alignment greater than what LLVM supports?");
46
47
156k
void CodeGenFunction::EmitDecl(const Decl &D) {
48
156k
  switch (D.getKind()) {
49
0
  case Decl::BuiltinTemplate:
50
0
  case Decl::TranslationUnit:
51
0
  case Decl::ExternCContext:
52
0
  case Decl::Namespace:
53
0
  case Decl::UnresolvedUsingTypename:
54
0
  case Decl::ClassTemplateSpecialization:
55
0
  case Decl::ClassTemplatePartialSpecialization:
56
0
  case Decl::VarTemplateSpecialization:
57
0
  case Decl::VarTemplatePartialSpecialization:
58
0
  case Decl::TemplateTypeParm:
59
0
  case Decl::UnresolvedUsingValue:
60
0
  case Decl::NonTypeTemplateParm:
61
0
  case Decl::CXXDeductionGuide:
62
0
  case Decl::CXXMethod:
63
0
  case Decl::CXXConstructor:
64
0
  case Decl::CXXDestructor:
65
0
  case Decl::CXXConversion:
66
0
  case Decl::Field:
67
0
  case Decl::MSProperty:
68
0
  case Decl::IndirectField:
69
0
  case Decl::ObjCIvar:
70
0
  case Decl::ObjCAtDefsField:
71
0
  case Decl::ParmVar:
72
0
  case Decl::ImplicitParam:
73
0
  case Decl::ClassTemplate:
74
0
  case Decl::VarTemplate:
75
0
  case Decl::FunctionTemplate:
76
0
  case Decl::TypeAliasTemplate:
77
0
  case Decl::TemplateTemplateParm:
78
0
  case Decl::ObjCMethod:
79
0
  case Decl::ObjCCategory:
80
0
  case Decl::ObjCProtocol:
81
0
  case Decl::ObjCInterface:
82
0
  case Decl::ObjCCategoryImpl:
83
0
  case Decl::ObjCImplementation:
84
0
  case Decl::ObjCProperty:
85
0
  case Decl::ObjCCompatibleAlias:
86
0
  case Decl::PragmaComment:
87
0
  case Decl::PragmaDetectMismatch:
88
0
  case Decl::AccessSpec:
89
0
  case Decl::LinkageSpec:
90
0
  case Decl::Export:
91
0
  case Decl::ObjCPropertyImpl:
92
0
  case Decl::FileScopeAsm:
93
0
  case Decl::Friend:
94
0
  case Decl::FriendTemplate:
95
0
  case Decl::Block:
96
0
  case Decl::Captured:
97
0
  case Decl::ClassScopeFunctionSpecialization:
98
0
  case Decl::UsingShadow:
99
0
  case Decl::ConstructorUsingShadow:
100
0
  case Decl::ObjCTypeParam:
101
0
  case Decl::Binding:
102
0
    llvm_unreachable("Declaration should not be in declstmts!");
103
2.37k
  case Decl::Record:    // struct/union/class X;
104
2.37k
  case Decl::CXXRecord: // struct/union/class X; [C++]
105
2.37k
    if (CGDebugInfo *DI = getDebugInfo())
106
1.68k
      if (cast<RecordDecl>(D).getDefinition())
107
1.67k
        DI->EmitAndRetainType(getContext().getRecordType(cast<RecordDecl>(&D)));
108
2.37k
    return;
109
81
  case Decl::Enum:      // enum X;
110
81
    if (CGDebugInfo *DI = getDebugInfo())
111
33
      if (cast<EnumDecl>(D).getDefinition())
112
31
        DI->EmitAndRetainType(getContext().getEnumType(cast<EnumDecl>(&D)));
113
81
    return;
114
3.02k
  case Decl::Function:     // void X();
115
3.02k
  case Decl::EnumConstant: // enum ? { X = ? }
116
3.02k
  case Decl::StaticAssert: // static_assert(X, ""); [C++0x]
117
3.02k
  case Decl::Label:        // __label__ x;
118
3.02k
  case Decl::Import:
119
3.02k
  case Decl::MSGuid:    // __declspec(uuid("..."))
120
3.02k
  case Decl::OMPThreadPrivate:
121
3.02k
  case Decl::OMPAllocate:
122
3.02k
  case Decl::OMPCapturedExpr:
123
3.02k
  case Decl::OMPRequires:
124
3.02k
  case Decl::Empty:
125
3.02k
  case Decl::Concept:
126
3.02k
  case Decl::LifetimeExtendedTemporary:
127
3.02k
  case Decl::RequiresExprBody:
128
    // None of these decls require codegen support.
129
3.02k
    return;
130
3.02k
131
9
  case Decl::NamespaceAlias:
132
9
    if (CGDebugInfo *DI = getDebugInfo())
133
8
        DI->EmitNamespaceAlias(cast<NamespaceAliasDecl>(D));
134
9
    return;
135
7.31k
  case Decl::Using:          // using X; [C++]
136
7.31k
    if (CGDebugInfo *DI = getDebugInfo())
137
84
        DI->EmitUsingDecl(cast<UsingDecl>(D));
138
7.31k
    return;
139
0
  case Decl::UsingPack:
140
0
    for (auto *Using : cast<UsingPackDecl>(D).expansions())
141
0
      EmitDecl(*Using);
142
0
    return;
143
236
  case Decl::UsingDirective: // using namespace X; [C++]
144
236
    if (CGDebugInfo *DI = getDebugInfo())
145
233
      DI->EmitUsingDirective(cast<UsingDirectiveDecl>(D));
146
236
    return;
147
139k
  case Decl::Var:
148
139k
  case Decl::Decomposition: {
149
139k
    const VarDecl &VD = cast<VarDecl>(D);
150
139k
    assert(VD.isLocalVarDecl() &&
151
139k
           "Should not see file-scope variables inside a function!");
152
139k
    EmitVarDecl(VD);
153
139k
    if (auto *DD = dyn_cast<DecompositionDecl>(&VD))
154
7
      for (auto *B : DD->bindings())
155
17
        if (auto *HD = B->getHoldingVar())
156
4
          EmitVarDecl(*HD);
157
139k
    return;
158
139k
  }
159
139k
160
70
  case Decl::OMPDeclareReduction:
161
70
    return CGM.EmitOMPDeclareReduction(cast<OMPDeclareReductionDecl>(&D), this);
162
139k
163
0
  case Decl::OMPDeclareMapper:
164
0
    return CGM.EmitOMPDeclareMapper(cast<OMPDeclareMapperDecl>(&D), this);
165
139k
166
4.35k
  case Decl::Typedef:      // typedef int X;
167
4.35k
  case Decl::TypeAlias: {  // using X = int; [C++0x]
168
4.35k
    QualType Ty = cast<TypedefNameDecl>(D).getUnderlyingType();
169
4.35k
    if (CGDebugInfo *DI = getDebugInfo())
170
3.83k
      DI->EmitAndRetainType(Ty);
171
4.35k
    if (Ty->isVariablyModifiedType())
172
21
      EmitVariablyModifiedType(Ty);
173
4.35k
    return;
174
4.35k
  }
175
156k
  }
176
156k
}
177
178
/// EmitVarDecl - This method handles emission of any variable declaration
179
/// inside a function, including static vars etc.
180
202k
void CodeGenFunction::EmitVarDecl(const VarDecl &D) {
181
202k
  if (D.hasExternalStorage())
182
    // Don't emit it now, allow it to be emitted lazily on its first use.
183
133
    return;
184
202k
185
  // Some function-scope variable does not have static storage but still
186
  // needs to be emitted like a static variable, e.g. a function-scope
187
  // variable in constant address space in OpenCL.
188
202k
  if (D.getStorageDuration() != SD_Automatic) {
189
    // Static sampler variables translated to function calls.
190
10.4k
    if (D.getType()->isSamplerT())
191
3
      return;
192
10.4k
193
10.4k
    llvm::GlobalValue::LinkageTypes Linkage =
194
10.4k
        CGM.getLLVMLinkageVarDefinition(&D, /*IsConstant=*/false);
195
10.4k
196
    // FIXME: We need to force the emission/use of a guard variable for
197
    // some variables even if we can constant-evaluate them because
198
    // we can't guarantee every translation unit will constant-evaluate them.
199
10.4k
200
10.4k
    return EmitStaticVarDecl(D, Linkage);
201
10.4k
  }
202
191k
203
191k
  if (D.getType().getAddressSpace() == LangAS::opencl_local)
204
122
    return CGM.getOpenCLRuntime().EmitWorkGroupLocalVarDecl(*this, D);
205
191k
206
191k
  assert(D.hasLocalStorage());
207
191k
  return EmitAutoVarDecl(D);
208
191k
}
209
210
10.4k
static std::string getStaticDeclName(CodeGenModule &CGM, const VarDecl &D) {
211
10.4k
  if (CGM.getLangOpts().CPlusPlus)
212
9.95k
    return CGM.getMangledName(&D).str();
213
540
214
  // If this isn't C++, we don't need a mangled name, just a pretty one.
215
540
  assert(!D.isExternallyVisible() && "name shouldn't matter");
216
540
  std::string ContextName;
217
540
  const DeclContext *DC = D.getDeclContext();
218
540
  if (auto *CD = dyn_cast<CapturedDecl>(DC))
219
1
    DC = cast<DeclContext>(CD->getNonClosureContext());
220
540
  if (const auto *FD = dyn_cast<FunctionDecl>(DC))
221
532
    ContextName = std::string(CGM.getMangledName(FD));
222
8
  else if (const auto *BD = dyn_cast<BlockDecl>(DC))
223
5
    ContextName = std::string(CGM.getBlockMangledName(GlobalDecl(), BD));
224
3
  else if (const auto *OMD = dyn_cast<ObjCMethodDecl>(DC))
225
3
    ContextName = OMD->getSelector().getAsString();
226
3
  else
227
0
    llvm_unreachable("Unknown context for static var decl");
228
540
229
540
  ContextName += "." + D.getNameAsString();
230
540
  return ContextName;
231
540
}
232
233
llvm::Constant *CodeGenModule::getOrCreateStaticVarDecl(
234
11.1k
    const VarDecl &D, llvm::GlobalValue::LinkageTypes Linkage) {
235
  // In general, we don't always emit static var decls once before we reference
236
  // them. It is possible to reference them before emitting the function that
237
  // contains them, and it is possible to emit the containing function multiple
238
  // times.
239
11.1k
  if (llvm::Constant *ExistingGV = StaticLocalDeclMap[&D])
240
667
    return ExistingGV;
241
10.4k
242
10.4k
  QualType Ty = D.getType();
243
10.4k
  assert(Ty->isConstantSizeType() && "VLAs can't be static");
244
10.4k
245
  // Use the label if the variable is renamed with the asm-label extension.
246
10.4k
  std::string Name;
247
10.4k
  if (D.hasAttr<AsmLabelAttr>())
248
2
    Name = std::string(getMangledName(&D));
249
10.4k
  else
250
10.4k
    Name = getStaticDeclName(*this, D);
251
10.4k
252
10.4k
  llvm::Type *LTy = getTypes().ConvertTypeForMem(Ty);
253
10.4k
  LangAS AS = GetGlobalVarAddressSpace(&D);
254
10.4k
  unsigned TargetAS = getContext().getTargetAddressSpace(AS);
255
10.4k
256
  // OpenCL variables in local address space and CUDA shared
257
  // variables cannot have an initializer.
258
10.4k
  llvm::Constant *Init = nullptr;
259
10.4k
  if (Ty.getAddressSpace() == LangAS::opencl_local ||
260
10.3k
      D.hasAttr<CUDASharedAttr>() || 
D.hasAttr<LoaderUninitializedAttr>()10.3k
)
261
147
    Init = llvm::UndefValue::get(LTy);
262
10.3k
  else
263
10.3k
    Init = EmitNullConstant(Ty);
264
10.4k
265
10.4k
  llvm::GlobalVariable *GV = new llvm::GlobalVariable(
266
10.4k
      getModule(), LTy, Ty.isConstant(getContext()), Linkage, Init, Name,
267
10.4k
      nullptr, llvm::GlobalVariable::NotThreadLocal, TargetAS);
268
10.4k
  GV->setAlignment(getContext().getDeclAlign(&D).getAsAlign());
269
10.4k
270
10.4k
  if (supportsCOMDAT() && 
GV->isWeakForLinker()2.56k
)
271
250
    GV->setComdat(TheModule.getOrInsertComdat(GV->getName()));
272
10.4k
273
10.4k
  if (D.getTLSKind())
274
118
    setTLSMode(GV, D);
275
10.4k
276
10.4k
  setGVProperties(GV, &D);
277
10.4k
278
  // Make sure the result is of the correct type.
279
10.4k
  LangAS ExpectedAS = Ty.getAddressSpace();
280
10.4k
  llvm::Constant *Addr = GV;
281
10.4k
  if (AS != ExpectedAS) {
282
32
    Addr = getTargetCodeGenInfo().performAddrSpaceCast(
283
32
        *this, GV, AS, ExpectedAS,
284
32
        LTy->getPointerTo(getContext().getTargetAddressSpace(ExpectedAS)));
285
32
  }
286
10.4k
287
10.4k
  setStaticLocalDeclAddress(&D, Addr);
288
10.4k
289
  // Ensure that the static local gets initialized by making sure the parent
290
  // function gets emitted eventually.
291
10.4k
  const Decl *DC = cast<Decl>(D.getDeclContext());
292
10.4k
293
  // We can't name blocks or captured statements directly, so try to emit their
294
  // parents.
295
10.4k
  if (isa<BlockDecl>(DC) || 
isa<CapturedDecl>(DC)10.4k
) {
296
296
    DC = DC->getNonClosureContext();
297
    // FIXME: Ensure that global blocks get emitted.
298
296
    if (!DC)
299
21
      return Addr;
300
10.4k
  }
301
10.4k
302
10.4k
  GlobalDecl GD;
303
10.4k
  if (const auto *CD = dyn_cast<CXXConstructorDecl>(DC))
304
11
    GD = GlobalDecl(CD, Ctor_Base);
305
10.4k
  else if (const auto *DD = dyn_cast<CXXDestructorDecl>(DC))
306
4
    GD = GlobalDecl(DD, Dtor_Base);
307
10.4k
  else if (const auto *FD = dyn_cast<FunctionDecl>(DC))
308
10.3k
    GD = GlobalDecl(FD);
309
122
  else {
310
    // Don't do anything for Obj-C method decls or global closures. We should
311
    // never defer them.
312
122
    assert(isa<ObjCMethodDecl>(DC) && "unexpected parent code decl");
313
122
  }
314
10.4k
  if (GD.getDecl()) {
315
    // Disable emission of the parent function for the OpenMP device codegen.
316
10.3k
    CGOpenMPRuntime::DisableAutoDeclareTargetRAII NoDeclTarget(*this);
317
10.3k
    (void)GetAddrOfGlobal(GD);
318
10.3k
  }
319
10.4k
320
10.4k
  return Addr;
321
10.4k
}
322
323
/// AddInitializerToStaticVarDecl - Add the initializer for 'D' to the
324
/// global variable that has already been created for it.  If the initializer
325
/// has a different type than GV does, this may free GV and return a different
326
/// one.  Otherwise it just returns GV.
327
llvm::GlobalVariable *
328
CodeGenFunction::AddInitializerToStaticVarDecl(const VarDecl &D,
329
8.51k
                                               llvm::GlobalVariable *GV) {
330
8.51k
  ConstantEmitter emitter(*this);
331
8.51k
  llvm::Constant *Init = emitter.tryEmitForInitializer(D);
332
8.51k
333
  // If constant emission failed, then this should be a C++ static
334
  // initializer.
335
8.51k
  if (!Init) {
336
7.15k
    if (!getLangOpts().CPlusPlus)
337
0
      CGM.ErrorUnsupported(D.getInit(), "constant l-value expression");
338
7.15k
    else if (HaveInsertPoint()) {
339
      // Since we have a static initializer, this global variable can't
340
      // be constant.
341
7.15k
      GV->setConstant(false);
342
7.15k
343
7.15k
      EmitCXXGuardedInit(D, GV, /*PerformInit*/true);
344
7.15k
    }
345
7.15k
    return GV;
346
7.15k
  }
347
1.35k
348
  // The initializer may differ in type from the global. Rewrite
349
  // the global to match the initializer.  (We have to do this
350
  // because some types, like unions, can't be completely represented
351
  // in the LLVM type system.)
352
1.35k
  if (GV->getValueType() != Init->getType()) {
353
89
    llvm::GlobalVariable *OldGV = GV;
354
89
355
89
    GV = new llvm::GlobalVariable(CGM.getModule(), Init->getType(),
356
89
                                  OldGV->isConstant(),
357
89
                                  OldGV->getLinkage(), Init, "",
358
89
                                  /*InsertBefore*/ OldGV,
359
89
                                  OldGV->getThreadLocalMode(),
360
89
                           CGM.getContext().getTargetAddressSpace(D.getType()));
361
89
    GV->setVisibility(OldGV->getVisibility());
362
89
    GV->setDSOLocal(OldGV->isDSOLocal());
363
89
    GV->setComdat(OldGV->getComdat());
364
89
365
    // Steal the name of the old global
366
89
    GV->takeName(OldGV);
367
89
368
    // Replace all uses of the old global with the new global
369
89
    llvm::Constant *NewPtrForOldDecl =
370
89
    llvm::ConstantExpr::getBitCast(GV, OldGV->getType());
371
89
    OldGV->replaceAllUsesWith(NewPtrForOldDecl);
372
89
373
    // Erase the old global, since it is no longer used.
374
89
    OldGV->eraseFromParent();
375
89
  }
376
1.35k
377
1.35k
  GV->setConstant(CGM.isTypeConstant(D.getType(), true));
378
1.35k
  GV->setInitializer(Init);
379
1.35k
380
1.35k
  emitter.finalize(GV);
381
1.35k
382
1.35k
  if (D.needsDestruction(getContext()) == QualType::DK_cxx_destructor &&
383
49
      HaveInsertPoint()) {
384
    // We have a constant initializer, but a nontrivial destructor. We still
385
    // need to perform a guarded "initialization" in order to register the
386
    // destructor.
387
48
    EmitCXXGuardedInit(D, GV, /*PerformInit*/false);
388
48
  }
389
1.35k
390
1.35k
  return GV;
391
1.35k
}
392
393
void CodeGenFunction::EmitStaticVarDecl(const VarDecl &D,
394
10.5k
                                      llvm::GlobalValue::LinkageTypes Linkage) {
395
  // Check to see if we already have a global variable for this
396
  // declaration.  This can happen when double-emitting function
397
  // bodies, e.g. with complete and base constructors.
398
10.5k
  llvm::Constant *addr = CGM.getOrCreateStaticVarDecl(D, Linkage);
399
10.5k
  CharUnits alignment = getContext().getDeclAlign(&D);
400
10.5k
401
  // Store into LocalDeclMap before generating initializer to handle
402
  // circular references.
403
10.5k
  setAddrOfLocalVar(&D, Address(addr, alignment));
404
10.5k
405
  // We can't have a VLA here, but we can have a pointer to a VLA,
406
  // even though that doesn't really make any sense.
407
  // Make sure to evaluate VLA bounds now so that we have them for later.
408
10.5k
  if (D.getType()->isVariablyModifiedType())
409
1
    EmitVariablyModifiedType(D.getType());
410
10.5k
411
  // Save the type in case adding the initializer forces a type change.
412
10.5k
  llvm::Type *expectedType = addr->getType();
413
10.5k
414
10.5k
  llvm::GlobalVariable *var =
415
10.5k
    cast<llvm::GlobalVariable>(addr->stripPointerCasts());
416
10.5k
417
  // CUDA's local and local static __shared__ variables should not
418
  // have any non-empty initializers. This is ensured by Sema.
419
  // Whatever initializer such variable may have when it gets here is
420
  // a no-op and should not be emitted.
421
10.5k
  bool isCudaSharedVar = getLangOpts().CUDA && 
getLangOpts().CUDAIsDevice30
&&
422
30
                         D.hasAttr<CUDASharedAttr>();
423
  // If this value has an initializer, emit it.
424
10.5k
  if (D.getInit() && 
!isCudaSharedVar8.52k
)
425
8.51k
    var = AddInitializerToStaticVarDecl(D, var);
426
10.5k
427
10.5k
  var->setAlignment(alignment.getAsAlign());
428
10.5k
429
10.5k
  if (D.hasAttr<AnnotateAttr>())
430
3
    CGM.AddGlobalAnnotations(&D, var);
431
10.5k
432
10.5k
  if (auto *SA = D.getAttr<PragmaClangBSSSectionAttr>())
433
1
    var->addAttribute("bss-section", SA->getName());
434
10.5k
  if (auto *SA = D.getAttr<PragmaClangDataSectionAttr>())
435
0
    var->addAttribute("data-section", SA->getName());
436
10.5k
  if (auto *SA = D.getAttr<PragmaClangRodataSectionAttr>())
437
1
    var->addAttribute("rodata-section", SA->getName());
438
10.5k
  if (auto *SA = D.getAttr<PragmaClangRelroSectionAttr>())
439
0
    var->addAttribute("relro-section", SA->getName());
440
10.5k
441
10.5k
  if (const SectionAttr *SA = D.getAttr<SectionAttr>())
442
1
    var->setSection(SA->getName());
443
10.5k
444
10.5k
  if (D.hasAttr<UsedAttr>())
445
1
    CGM.addUsedGlobal(var);
446
10.5k
447
  // We may have to cast the constant because of the initializer
448
  // mismatch above.
449
  //
450
  // FIXME: It is really dangerous to store this in the map; if anyone
451
  // RAUW's the GV uses of this constant will be invalid.
452
10.5k
  llvm::Constant *castedAddr =
453
10.5k
    llvm::ConstantExpr::getPointerBitCastOrAddrSpaceCast(var, expectedType);
454
10.5k
  if (var != castedAddr)
455
123
    LocalDeclMap.find(&D)->second = Address(castedAddr, alignment);
456
10.5k
  CGM.setStaticLocalDeclAddress(&D, castedAddr);
457
10.5k
458
10.5k
  CGM.getSanitizerMetadata()->reportGlobalToASan(var, D);
459
10.5k
460
  // Emit global variable debug descriptor for static vars.
461
10.5k
  CGDebugInfo *DI = getDebugInfo();
462
10.5k
  if (DI && 
CGM.getCodeGenOpts().hasReducedDebugInfo()299
) {
463
299
    DI->setLocation(D.getLocation());
464
299
    DI->EmitGlobalVariable(var, &D);
465
299
  }
466
10.5k
}
467
468
namespace {
469
  struct DestroyObject final : EHScopeStack::Cleanup {
470
    DestroyObject(Address addr, QualType type,
471
                  CodeGenFunction::Destroyer *destroyer,
472
                  bool useEHCleanupForArray)
473
      : addr(addr), type(type), destroyer(destroyer),
474
13.4k
        useEHCleanupForArray(useEHCleanupForArray) {}
475
476
    Address addr;
477
    QualType type;
478
    CodeGenFunction::Destroyer *destroyer;
479
    bool useEHCleanupForArray;
480
481
16.6k
    void Emit(CodeGenFunction &CGF, Flags flags) override {
482
      // Don't use an EH cleanup recursively from an EH cleanup.
483
16.6k
      bool useEHCleanupForArray =
484
16.6k
        flags.isForNormalCleanup() && 
this->useEHCleanupForArray12.3k
;
485
16.6k
486
16.6k
      CGF.emitDestroy(addr, type, destroyer, useEHCleanupForArray);
487
16.6k
    }
488
  };
489
490
  template <class Derived>
491
  struct DestroyNRVOVariable : EHScopeStack::Cleanup {
492
    DestroyNRVOVariable(Address addr, QualType type, llvm::Value *NRVOFlag)
493
192
        : NRVOFlag(NRVOFlag), Loc(addr), Ty(type) {}
CGDecl.cpp:(anonymous namespace)::DestroyNRVOVariable<(anonymous namespace)::DestroyNRVOVariableCXX>::DestroyNRVOVariable(clang::CodeGen::Address, clang::QualType, llvm::Value*)
Line
Count
Source
493
189
        : NRVOFlag(NRVOFlag), Loc(addr), Ty(type) {}
CGDecl.cpp:(anonymous namespace)::DestroyNRVOVariable<(anonymous namespace)::DestroyNRVOVariableC>::DestroyNRVOVariable(clang::CodeGen::Address, clang::QualType, llvm::Value*)
Line
Count
Source
493
3
        : NRVOFlag(NRVOFlag), Loc(addr), Ty(type) {}
494
495
    llvm::Value *NRVOFlag;
496
    Address Loc;
497
    QualType Ty;
498
499
283
    void Emit(CodeGenFunction &CGF, Flags flags) override {
500
      // Along the exceptions path we always execute the dtor.
501
283
      bool NRVO = flags.isForNormalCleanup() && 
NRVOFlag192
;
502
283
503
283
      llvm::BasicBlock *SkipDtorBB = nullptr;
504
283
      if (NRVO) {
505
        // If we exited via NRVO, we skip the destructor call.
506
192
        llvm::BasicBlock *RunDtorBB = CGF.createBasicBlock("nrvo.unused");
507
192
        SkipDtorBB = CGF.createBasicBlock("nrvo.skipdtor");
508
192
        llvm::Value *DidNRVO =
509
192
          CGF.Builder.CreateFlagLoad(NRVOFlag, "nrvo.val");
510
192
        CGF.Builder.CreateCondBr(DidNRVO, SkipDtorBB, RunDtorBB);
511
192
        CGF.EmitBlock(RunDtorBB);
512
192
      }
513
283
514
283
      static_cast<Derived *>(this)->emitDestructorCall(CGF);
515
283
516
283
      if (NRVO) 
CGF.EmitBlock(SkipDtorBB)192
;
517
283
    }
CGDecl.cpp:(anonymous namespace)::DestroyNRVOVariable<(anonymous namespace)::DestroyNRVOVariableCXX>::Emit(clang::CodeGen::CodeGenFunction&, clang::CodeGen::EHScopeStack::Cleanup::Flags)
Line
Count
Source
499
280
    void Emit(CodeGenFunction &CGF, Flags flags) override {
500
      // Along the exceptions path we always execute the dtor.
501
280
      bool NRVO = flags.isForNormalCleanup() && 
NRVOFlag189
;
502
280
503
280
      llvm::BasicBlock *SkipDtorBB = nullptr;
504
280
      if (NRVO) {
505
        // If we exited via NRVO, we skip the destructor call.
506
189
        llvm::BasicBlock *RunDtorBB = CGF.createBasicBlock("nrvo.unused");
507
189
        SkipDtorBB = CGF.createBasicBlock("nrvo.skipdtor");
508
189
        llvm::Value *DidNRVO =
509
189
          CGF.Builder.CreateFlagLoad(NRVOFlag, "nrvo.val");
510
189
        CGF.Builder.CreateCondBr(DidNRVO, SkipDtorBB, RunDtorBB);
511
189
        CGF.EmitBlock(RunDtorBB);
512
189
      }
513
280
514
280
      static_cast<Derived *>(this)->emitDestructorCall(CGF);
515
280
516
280
      if (NRVO) 
CGF.EmitBlock(SkipDtorBB)189
;
517
280
    }
CGDecl.cpp:(anonymous namespace)::DestroyNRVOVariable<(anonymous namespace)::DestroyNRVOVariableC>::Emit(clang::CodeGen::CodeGenFunction&, clang::CodeGen::EHScopeStack::Cleanup::Flags)
Line
Count
Source
499
3
    void Emit(CodeGenFunction &CGF, Flags flags) override {
500
      // Along the exceptions path we always execute the dtor.
501
3
      bool NRVO = flags.isForNormalCleanup() && NRVOFlag;
502
3
503
3
      llvm::BasicBlock *SkipDtorBB = nullptr;
504
3
      if (NRVO) {
505
        // If we exited via NRVO, we skip the destructor call.
506
3
        llvm::BasicBlock *RunDtorBB = CGF.createBasicBlock("nrvo.unused");
507
3
        SkipDtorBB = CGF.createBasicBlock("nrvo.skipdtor");
508
3
        llvm::Value *DidNRVO =
509
3
          CGF.Builder.CreateFlagLoad(NRVOFlag, "nrvo.val");
510
3
        CGF.Builder.CreateCondBr(DidNRVO, SkipDtorBB, RunDtorBB);
511
3
        CGF.EmitBlock(RunDtorBB);
512
3
      }
513
3
514
3
      static_cast<Derived *>(this)->emitDestructorCall(CGF);
515
3
516
3
      if (NRVO) CGF.EmitBlock(SkipDtorBB);
517
3
    }
518
519
0
    virtual ~DestroyNRVOVariable() = default;
Unexecuted instantiation: CGDecl.cpp:(anonymous namespace)::DestroyNRVOVariable<(anonymous namespace)::DestroyNRVOVariableCXX>::~DestroyNRVOVariable()
Unexecuted instantiation: CGDecl.cpp:(anonymous namespace)::DestroyNRVOVariable<(anonymous namespace)::DestroyNRVOVariableC>::~DestroyNRVOVariable()
520
  };
521
522
  struct DestroyNRVOVariableCXX final
523
      : DestroyNRVOVariable<DestroyNRVOVariableCXX> {
524
    DestroyNRVOVariableCXX(Address addr, QualType type,
525
                           const CXXDestructorDecl *Dtor, llvm::Value *NRVOFlag)
526
        : DestroyNRVOVariable<DestroyNRVOVariableCXX>(addr, type, NRVOFlag),
527
189
          Dtor(Dtor) {}
528
529
    const CXXDestructorDecl *Dtor;
530
531
280
    void emitDestructorCall(CodeGenFunction &CGF) {
532
280
      CGF.EmitCXXDestructorCall(Dtor, Dtor_Complete,
533
280
                                /*ForVirtualBase=*/false,
534
280
                                /*Delegating=*/false, Loc, Ty);
535
280
    }
536
  };
537
538
  struct DestroyNRVOVariableC final
539
      : DestroyNRVOVariable<DestroyNRVOVariableC> {
540
    DestroyNRVOVariableC(Address addr, llvm::Value *NRVOFlag, QualType Ty)
541
3
        : DestroyNRVOVariable<DestroyNRVOVariableC>(addr, Ty, NRVOFlag) {}
542
543
3
    void emitDestructorCall(CodeGenFunction &CGF) {
544
3
      CGF.destroyNonTrivialCStruct(CGF, Loc, Ty);
545
3
    }
546
  };
547
548
  struct CallStackRestore final : EHScopeStack::Cleanup {
549
    Address Stack;
550
1.57k
    CallStackRestore(Address Stack) : Stack(Stack) {}
551
1.57k
    void Emit(CodeGenFunction &CGF, Flags flags) override {
552
1.57k
      llvm::Value *V = CGF.Builder.CreateLoad(Stack);
553
1.57k
      llvm::Function *F = CGF.CGM.getIntrinsic(llvm::Intrinsic::stackrestore);
554
1.57k
      CGF.Builder.CreateCall(F, V);
555
1.57k
    }
556
  };
557
558
  struct ExtendGCLifetime final : EHScopeStack::Cleanup {
559
    const VarDecl &Var;
560
1
    ExtendGCLifetime(const VarDecl *var) : Var(*var) {}
561
562
1
    void Emit(CodeGenFunction &CGF, Flags flags) override {
563
      // Compute the address of the local variable, in case it's a
564
      // byref or something.
565
1
      DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(&Var), false,
566
1
                      Var.getType(), VK_LValue, SourceLocation());
567
1
      llvm::Value *value = CGF.EmitLoadOfScalar(CGF.EmitDeclRefLValue(&DRE),
568
1
                                                SourceLocation());
569
1
      CGF.EmitExtendGCLifetime(value);
570
1
    }
571
  };
572
573
  struct CallCleanupFunction final : EHScopeStack::Cleanup {
574
    llvm::Constant *CleanupFn;
575
    const CGFunctionInfo &FnInfo;
576
    const VarDecl &Var;
577
578
    CallCleanupFunction(llvm::Constant *CleanupFn, const CGFunctionInfo *Info,
579
                        const VarDecl *Var)
580
8
      : CleanupFn(CleanupFn), FnInfo(*Info), Var(*Var) {}
581
582
10
    void Emit(CodeGenFunction &CGF, Flags flags) override {
583
10
      DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(&Var), false,
584
10
                      Var.getType(), VK_LValue, SourceLocation());
585
      // Compute the address of the local variable, in case it's a byref
586
      // or something.
587
10
      llvm::Value *Addr = CGF.EmitDeclRefLValue(&DRE).getPointer(CGF);
588
10
589
      // In some cases, the type of the function argument will be different from
590
      // the type of the pointer. An example of this is
591
      // void f(void* arg);
592
      // __attribute__((cleanup(f))) void *g;
593
      //
594
      // To fix this we insert a bitcast here.
595
10
      QualType ArgTy = FnInfo.arg_begin()->type;
596
10
      llvm::Value *Arg =
597
10
        CGF.Builder.CreateBitCast(Addr, CGF.ConvertType(ArgTy));
598
10
599
10
      CallArgList Args;
600
10
      Args.add(RValue::get(Arg),
601
10
               CGF.getContext().getPointerType(Var.getType()));
602
10
      auto Callee = CGCallee::forDirect(CleanupFn);
603
10
      CGF.EmitCall(FnInfo, Callee, ReturnValueSlot(), Args);
604
10
    }
605
  };
606
} // end anonymous namespace
607
608
/// EmitAutoVarWithLifetime - Does the setup required for an automatic
609
/// variable with lifetime.
610
static void EmitAutoVarWithLifetime(CodeGenFunction &CGF, const VarDecl &var,
611
                                    Address addr,
612
525
                                    Qualifiers::ObjCLifetime lifetime) {
613
525
  switch (lifetime) {
614
0
  case Qualifiers::OCL_None:
615
0
    llvm_unreachable("present but none");
616
0
617
262
  case Qualifiers::OCL_ExplicitNone:
618
    // nothing to do
619
262
    break;
620
0
621
255
  case Qualifiers::OCL_Strong: {
622
255
    CodeGenFunction::Destroyer *destroyer =
623
255
      (var.hasAttr<ObjCPreciseLifetimeAttr>()
624
1
       ? CodeGenFunction::destroyARCStrongPrecise
625
254
       : CodeGenFunction::destroyARCStrongImprecise);
626
255
627
255
    CleanupKind cleanupKind = CGF.getARCCleanupKind();
628
255
    CGF.pushDestroy(cleanupKind, addr, var.getType(), destroyer,
629
255
                    cleanupKind & EHCleanup);
630
255
    break;
631
0
  }
632
1
  case Qualifiers::OCL_Autoreleasing:
633
    // nothing to do
634
1
    break;
635
0
636
7
  case Qualifiers::OCL_Weak:
637
    // __weak objects always get EH cleanups; otherwise, exceptions
638
    // could cause really nasty crashes instead of mere leaks.
639
7
    CGF.pushDestroy(NormalAndEHCleanup, addr, var.getType(),
640
7
                    CodeGenFunction::destroyARCWeak,
641
7
                    /*useEHCleanup*/ true);
642
7
    break;
643
525
  }
644
525
}
645
646
1.26k
static bool isAccessedBy(const VarDecl &var, const Stmt *s) {
647
1.26k
  if (const Expr *e = dyn_cast<Expr>(s)) {
648
    // Skip the most common kinds of expressions that make
649
    // hierarchy-walking expensive.
650
1.25k
    s = e = e->IgnoreParenCasts();
651
1.25k
652
1.25k
    if (const DeclRefExpr *ref = dyn_cast<DeclRefExpr>(e))
653
523
      return (ref->getDecl() == &var);
654
730
    if (const BlockExpr *be = dyn_cast<BlockExpr>(e)) {
655
121
      const BlockDecl *block = be->getBlockDecl();
656
370
      for (const auto &I : block->captures()) {
657
370
        if (I.getVariable() == &var)
658
2
          return true;
659
370
      }
660
121
    }
661
730
  }
662
1.26k
663
744
  for (const Stmt *SubStmt : s->children())
664
    // SubStmt might be null; as in missing decl or conditional of an if-stmt.
665
364
    if (SubStmt && isAccessedBy(var, SubStmt))
666
15
      return true;
667
744
668
729
  return false;
669
744
}
670
671
1.01k
static bool isAccessedBy(const ValueDecl *decl, const Expr *e) {
672
1.01k
  if (!decl) 
return false78
;
673
938
  if (!isa<VarDecl>(decl)) 
return false33
;
674
905
  const VarDecl *var = cast<VarDecl>(decl);
675
905
  return isAccessedBy(*var, e);
676
905
}
677
678
static bool tryEmitARCCopyWeakInit(CodeGenFunction &CGF,
679
321
                                   const LValue &destLV, const Expr *init) {
680
321
  bool needsCast = false;
681
321
682
332
  while (auto castExpr = dyn_cast<CastExpr>(init->IgnoreParens())) {
683
192
    switch (castExpr->getCastKind()) {
684
    // Look through casts that don't require representation changes.
685
11
    case CK_NoOp:
686
11
    case CK_BitCast:
687
11
    case CK_BlockPointerToObjCPointerCast:
688
11
      needsCast = true;
689
11
      break;
690
11
691
    // If we find an l-value to r-value cast from a __weak variable,
692
    // emit this operation as a copy or move.
693
167
    case CK_LValueToRValue: {
694
167
      const Expr *srcExpr = castExpr->getSubExpr();
695
167
      if (srcExpr->getType().getObjCLifetime() != Qualifiers::OCL_Weak)
696
9
        return false;
697
158
698
      // Emit the source l-value.
699
158
      LValue srcLV = CGF.EmitLValue(srcExpr);
700
158
701
      // Handle a formal type change to avoid asserting.
702
158
      auto srcAddr = srcLV.getAddress(CGF);
703
158
      if (needsCast) {
704
3
        srcAddr = CGF.Builder.CreateElementBitCast(
705
3
            srcAddr, destLV.getAddress(CGF).getElementType());
706
3
      }
707
158
708
      // If it was an l-value, use objc_copyWeak.
709
158
      if (srcExpr->getValueKind() == VK_LValue) {
710
155
        CGF.EmitARCCopyWeak(destLV.getAddress(CGF), srcAddr);
711
3
      } else {
712
3
        assert(srcExpr->getValueKind() == VK_XValue);
713
3
        CGF.EmitARCMoveWeak(destLV.getAddress(CGF), srcAddr);
714
3
      }
715
158
      return true;
716
158
    }
717
158
718
    // Stop at anything else.
719
14
    default:
720
14
      return false;
721
11
    }
722
11
723
11
    init = castExpr->getSubExpr();
724
11
  }
725
140
  return false;
726
321
}
727
728
static void drillIntoBlockVariable(CodeGenFunction &CGF,
729
                                   LValue &lvalue,
730
15
                                   const VarDecl *var) {
731
15
  lvalue.setAddress(CGF.emitBlockByrefAddress(lvalue.getAddress(CGF), var));
732
15
}
733
734
void CodeGenFunction::EmitNullabilityCheck(LValue LHS, llvm::Value *RHS,
735
297k
                                           SourceLocation Loc) {
736
297k
  if (!SanOpts.has(SanitizerKind::NullabilityAssign))
737
297k
    return;
738
18
739
18
  auto Nullability = LHS.getType()->getNullability(getContext());
740
18
  if (!Nullability || 
*Nullability != NullabilityKind::NonNull14
)
741
4
    return;
742
14
743
  // Check if the right hand side of the assignment is nonnull, if the left
744
  // hand side must be nonnull.
745
14
  SanitizerScope SanScope(this);
746
14
  llvm::Value *IsNotNull = Builder.CreateIsNotNull(RHS);
747
14
  llvm::Constant *StaticData[] = {
748
14
      EmitCheckSourceLocation(Loc), EmitCheckTypeDescriptor(LHS.getType()),
749
14
      llvm::ConstantInt::get(Int8Ty, 0), // The LogAlignment info is unused.
750
14
      llvm::ConstantInt::get(Int8Ty, TCK_NonnullAssign)};
751
14
  EmitCheck({{IsNotNull, SanitizerKind::NullabilityAssign}},
752
14
            SanitizerHandler::TypeMismatch, StaticData, RHS);
753
14
}
754
755
void CodeGenFunction::EmitScalarInit(const Expr *init, const ValueDecl *D,
756
153k
                                     LValue lvalue, bool capturedByInit) {
757
153k
  Qualifiers::ObjCLifetime lifetime = lvalue.getObjCLifetime();
758
153k
  if (!lifetime) {
759
152k
    llvm::Value *value = EmitScalarExpr(init);
760
152k
    if (capturedByInit)
761
4
      drillIntoBlockVariable(*this, lvalue, cast<VarDecl>(D));
762
152k
    EmitNullabilityCheck(lvalue, value, init->getExprLoc());
763
152k
    EmitStoreThroughLValue(RValue::get(value), lvalue, true);
764
152k
    return;
765
152k
  }
766
1.11k
767
1.11k
  if (const CXXDefaultInitExpr *DIE = dyn_cast<CXXDefaultInitExpr>(init))
768
1
    init = DIE->getExpr();
769
1.11k
770
  // If we're emitting a value with lifetime, we have to do the
771
  // initialization *before* we leave the cleanup scopes.
772
1.11k
  if (const ExprWithCleanups *EWC = dyn_cast<ExprWithCleanups>(init))
773
342
    init = EWC->getSubExpr();
774
1.11k
  CodeGenFunction::RunCleanupsScope Scope(*this);
775
1.11k
776
  // We have to maintain the illusion that the variable is
777
  // zero-initialized.  If the variable might be accessed in its
778
  // initializer, zero-initialize before running the initializer, then
779
  // actually perform the initialization with an assign.
780
1.11k
  bool accessedByInit = false;
781
1.11k
  if (lifetime != Qualifiers::OCL_ExplicitNone)
782
1.01k
    accessedByInit = (capturedByInit || 
isAccessedBy(D, init)1.01k
);
783
1.11k
  if (accessedByInit) {
784
19
    LValue tempLV = lvalue;
785
    // Drill down to the __block object if necessary.
786
19
    if (capturedByInit) {
787
      // We can use a simple GEP for this because it can't have been
788
      // moved yet.
789
2
      tempLV.setAddress(emitBlockByrefAddress(tempLV.getAddress(*this),
790
2
                                              cast<VarDecl>(D),
791
2
                                              /*follow*/ false));
792
2
    }
793
19
794
19
    auto ty =
795
19
        cast<llvm::PointerType>(tempLV.getAddress(*this).getElementType());
796
19
    llvm::Value *zero = CGM.getNullPointer(ty, tempLV.getType());
797
19
798
    // If __weak, we want to use a barrier under certain conditions.
799
19
    if (lifetime == Qualifiers::OCL_Weak)
800
4
      EmitARCInitWeak(tempLV.getAddress(*this), zero);
801
15
802
    // Otherwise just do a simple store.
803
15
    else
804
15
      EmitStoreOfScalar(zero, tempLV, /* isInitialization */ true);
805
19
  }
806
1.11k
807
  // Emit the initializer.
808
1.11k
  llvm::Value *value = nullptr;
809
1.11k
810
1.11k
  switch (lifetime) {
811
0
  case Qualifiers::OCL_None:
812
0
    llvm_unreachable("present but none");
813
0
814
688
  case Qualifiers::OCL_Strong: {
815
688
    if (!D || 
!isa<VarDecl>(D)616
||
!cast<VarDecl>(D)->isARCPseudoStrong()591
) {
816
684
      value = EmitARCRetainScalarExpr(init);
817
684
      break;
818
684
    }
819
    // If D is pseudo-strong, treat it like __unsafe_unretained here. This means
820
    // that we omit the retain, and causes non-autoreleased return values to be
821
    // immediately released.
822
4
    LLVM_FALLTHROUGH;
823
4
  }
824
4
825
101
  case Qualifiers::OCL_ExplicitNone:
826
101
    value = EmitARCUnsafeUnretainedScalarExpr(init);
827
101
    break;
828
4
829
325
  case Qualifiers::OCL_Weak: {
830
    // If it's not accessed by the initializer, try to emit the
831
    // initialization with a copy or move.
832
325
    if (!accessedByInit && 
tryEmitARCCopyWeakInit(*this, lvalue, init)321
) {
833
158
      return;
834
158
    }
835
167
836
    // No way to optimize a producing initializer into this.  It's not
837
    // worth optimizing for, because the value will immediately
838
    // disappear in the common case.
839
167
    value = EmitScalarExpr(init);
840
167
841
167
    if (capturedByInit) 
drillIntoBlockVariable(*this, lvalue, cast<VarDecl>(D))0
;
842
167
    if (accessedByInit)
843
4
      EmitARCStoreWeak(lvalue.getAddress(*this), value, /*ignored*/ true);
844
163
    else
845
163
      EmitARCInitWeak(lvalue.getAddress(*this), value);
846
167
    return;
847
167
  }
848
167
849
5
  case Qualifiers::OCL_Autoreleasing:
850
5
    value = EmitARCRetainAutoreleaseScalarExpr(init);
851
5
    break;
852
790
  }
853
790
854
790
  if (capturedByInit) 
drillIntoBlockVariable(*this, lvalue, cast<VarDecl>(D))2
;
855
790
856
790
  EmitNullabilityCheck(lvalue, value, init->getExprLoc());
857
790
858
  // If the variable might have been accessed by its initializer, we
859
  // might have to initialize with a barrier.  We have to do this for
860
  // both __weak and __strong, but __weak got filtered out above.
861
790
  if (accessedByInit && 
lifetime == Qualifiers::OCL_Strong15
) {
862
11
    llvm::Value *oldValue = EmitLoadOfScalar(lvalue, init->getExprLoc());
863
11
    EmitStoreOfScalar(value, lvalue, /* isInitialization */ true);
864
11
    EmitARCRelease(oldValue, ARCImpreciseLifetime);
865
11
    return;
866
11
  }
867
779
868
779
  EmitStoreOfScalar(value, lvalue, /* isInitialization */ true);
869
779
}
870
871
/// Decide whether we can emit the non-zero parts of the specified initializer
872
/// with equal or fewer than NumStores scalar stores.
873
static bool canEmitInitWithFewStoresAfterBZero(llvm::Constant *Init,
874
102k
                                               unsigned &NumStores) {
875
  // Zero and Undef never requires any extra stores.
876
102k
  if (isa<llvm::ConstantAggregateZero>(Init) ||
877
102k
      isa<llvm::ConstantPointerNull>(Init) ||
878
102k
      isa<llvm::UndefValue>(Init))
879
56
    return true;
880
102k
  if (isa<llvm::ConstantInt>(Init) || 
isa<llvm::ConstantFP>(Init)396
||
881
322
      isa<llvm::ConstantVector>(Init) || isa<llvm::BlockAddress>(Init) ||
882
317
      isa<llvm::ConstantExpr>(Init))
883
102k
    return Init->isNullValue() || 
NumStores--1.03k
;
884
302
885
  // See if we can emit each element.
886
302
  if (isa<llvm::ConstantArray>(Init) || 
isa<llvm::ConstantStruct>(Init)254
) {
887
666
    for (unsigned i = 0, e = Init->getNumOperands(); i != e; 
++i481
) {
888
558
      llvm::Constant *Elt = cast<llvm::Constant>(Init->getOperand(i));
889
558
      if (!canEmitInitWithFewStoresAfterBZero(Elt, NumStores))
890
77
        return false;
891
558
    }
892
108
    return true;
893
117
  }
894
117
895
117
  if (llvm::ConstantDataSequential *CDS =
896
117
        dyn_cast<llvm::ConstantDataSequential>(Init)) {
897
101k
    for (unsigned i = 0, e = CDS->getNumElements(); i != e; 
++i101k
) {
898
101k
      llvm::Constant *Elt = CDS->getElementAsConstant(i);
899
101k
      if (!canEmitInitWithFewStoresAfterBZero(Elt, NumStores))
900
81
        return false;
901
101k
    }
902
36
    return true;
903
0
  }
904
0
905
  // Anything else is hard and scary.
906
0
  return false;
907
0
}
908
909
/// For inits that canEmitInitWithFewStoresAfterBZero returned true for, emit
910
/// the scalar stores that would be required.
911
static void emitStoresForInitAfterBZero(CodeGenModule &CGM,
912
                                        llvm::Constant *Init, Address Loc,
913
247
                                        bool isVolatile, CGBuilderTy &Builder) {
914
247
  assert(!Init->isNullValue() && !isa<llvm::UndefValue>(Init) &&
915
247
         "called emitStoresForInitAfterBZero for zero or undef value.");
916
247
917
247
  if (isa<llvm::ConstantInt>(Init) || 
isa<llvm::ConstantFP>(Init)111
||
918
79
      isa<llvm::ConstantVector>(Init) || isa<llvm::BlockAddress>(Init) ||
919
188
      
isa<llvm::ConstantExpr>(Init)74
) {
920
188
    Builder.CreateStore(Init, Loc, isVolatile);
921
188
    return;
922
188
  }
923
59
924
59
  if (llvm::ConstantDataSequential *CDS =
925
24
          dyn_cast<llvm::ConstantDataSequential>(Init)) {
926
101k
    for (unsigned i = 0, e = CDS->getNumElements(); i != e; 
++i101k
) {
927
101k
      llvm::Constant *Elt = CDS->getElementAsConstant(i);
928
101k
929
      // If necessary, get a pointer to the element and emit it.
930
101k
      if (!Elt->isNullValue() && 
!isa<llvm::UndefValue>(Elt)103
)
931
103
        emitStoresForInitAfterBZero(
932
103
            CGM, Elt, Builder.CreateConstInBoundsGEP2_32(Loc, 0, i), isVolatile,
933
103
            Builder);
934
101k
    }
935
24
    return;
936
24
  }
937
35
938
35
  assert((isa<llvm::ConstantStruct>(Init) || isa<llvm::ConstantArray>(Init)) &&
939
35
         "Unknown value type!");
940
35
941
178
  for (unsigned i = 0, e = Init->getNumOperands(); i != e; 
++i143
) {
942
143
    llvm::Constant *Elt = cast<llvm::Constant>(Init->getOperand(i));
943
143
944
    // If necessary, get a pointer to the element and emit it.
945
143
    if (!Elt->isNullValue() && 
!isa<llvm::UndefValue>(Elt)90
)
946
88
      emitStoresForInitAfterBZero(CGM, Elt,
947
88
                                  Builder.CreateConstInBoundsGEP2_32(Loc, 0, i),
948
88
                                  isVolatile, Builder);
949
143
  }
950
35
}
951
952
/// Decide whether we should use bzero plus some stores to initialize a local
953
/// variable instead of using a memcpy from a constant global.  It is beneficial
954
/// to use bzero if the global is all zeros, or mostly zeros and large.
955
static bool shouldUseBZeroPlusStoresToInitialize(llvm::Constant *Init,
956
4.85k
                                                 uint64_t GlobalSize) {
957
  // If a global is all zeros, always use a bzero.
958
4.85k
  if (isa<llvm::ConstantAggregateZero>(Init)) 
return true1.04k
;
959
3.80k
960
  // If a non-zero global is <= 32 bytes, always use a memcpy.  If it is large,
961
  // do it if it will require 6 or fewer scalar stores.
962
  // TODO: Should budget depends on the size?  Avoiding a large global warrants
963
  // plopping in more stores.
964
3.80k
  unsigned StoreBudget = 6;
965
3.80k
  uint64_t SizeLimit = 32;
966
3.80k
967
3.80k
  return GlobalSize > SizeLimit &&
968
177
         canEmitInitWithFewStoresAfterBZero(Init, StoreBudget);
969
3.80k
}
970
971
/// Decide whether we should use memset to initialize a local variable instead
972
/// of using a memcpy from a constant global. Assumes we've already decided to
973
/// not user bzero.
974
/// FIXME We could be more clever, as we are for bzero above, and generate
975
///       memset followed by stores. It's unclear that's worth the effort.
976
static llvm::Value *shouldUseMemSetToInitialize(llvm::Constant *Init,
977
                                                uint64_t GlobalSize,
978
3.75k
                                                const llvm::DataLayout &DL) {
979
3.75k
  uint64_t SizeLimit = 32;
980
3.75k
  if (GlobalSize <= SizeLimit)
981
3.63k
    return nullptr;
982
121
  return llvm::isBytewiseValue(Init, DL);
983
121
}
984
985
/// Decide whether we want to split a constant structure or array store into a
986
/// sequence of its fields' stores. This may cost us code size and compilation
987
/// speed, but plays better with store optimizations.
988
static bool shouldSplitConstantStore(CodeGenModule &CGM,
989
3.71k
                                     uint64_t GlobalByteSize) {
990
  // Don't break things that occupy more than one cacheline.
991
3.71k
  uint64_t ByteSizeLimit = 64;
992
3.71k
  if (CGM.getCodeGenOpts().OptimizationLevel == 0)
993
3.41k
    return false;
994
297
  if (GlobalByteSize <= ByteSizeLimit)
995
297
    return true;
996
0
  return false;
997
0
}
998
999
enum class IsPattern { No, Yes };
1000
1001
/// Generate a constant filled with either a pattern or zeroes.
1002
static llvm::Constant *patternOrZeroFor(CodeGenModule &CGM, IsPattern isPattern,
1003
355
                                        llvm::Type *Ty) {
1004
355
  if (isPattern == IsPattern::Yes)
1005
172
    return initializationPatternFor(CGM, Ty);
1006
183
  else
1007
183
    return llvm::Constant::getNullValue(Ty);
1008
355
}
1009
1010
static llvm::Constant *constWithPadding(CodeGenModule &CGM, IsPattern isPattern,
1011
                                        llvm::Constant *constant);
1012
1013
/// Helper function for constWithPadding() to deal with padding in structures.
1014
static llvm::Constant *constStructWithPadding(CodeGenModule &CGM,
1015
                                              IsPattern isPattern,
1016
                                              llvm::StructType *STy,
1017
832
                                              llvm::Constant *constant) {
1018
832
  const llvm::DataLayout &DL = CGM.getDataLayout();
1019
832
  const llvm::StructLayout *Layout = DL.getStructLayout(STy);
1020
832
  llvm::Type *Int8Ty = llvm::IntegerType::getInt8Ty(CGM.getLLVMContext());
1021
832
  unsigned SizeSoFar = 0;
1022
832
  SmallVector<llvm::Constant *, 8> Values;
1023
832
  bool NestedIntact = true;
1024
2.64k
  for (unsigned i = 0, e = STy->getNumElements(); i != e; 
i++1.81k
) {
1025
1.81k
    unsigned CurOff = Layout->getElementOffset(i);
1026
1.81k
    if (SizeSoFar < CurOff) {
1027
82
      assert(!STy->isPacked());
1028
82
      auto *PadTy = llvm::ArrayType::get(Int8Ty, CurOff - SizeSoFar);
1029
82
      Values.push_back(patternOrZeroFor(CGM, isPattern, PadTy));
1030
82
    }
1031
1.81k
    llvm::Constant *CurOp;
1032
1.81k
    if (constant->isZeroValue())
1033
422
      CurOp = llvm::Constant::getNullValue(STy->getElementType(i));
1034
1.38k
    else
1035
1.38k
      CurOp = cast<llvm::Constant>(constant->getAggregateElement(i));
1036
1.81k
    auto *NewOp = constWithPadding(CGM, isPattern, CurOp);
1037
1.81k
    if (CurOp != NewOp)
1038
35
      NestedIntact = false;
1039
1.81k
    Values.push_back(NewOp);
1040
1.81k
    SizeSoFar = CurOff + DL.getTypeAllocSize(CurOp->getType());
1041
1.81k
  }
1042
832
  unsigned TotalSize = Layout->getSizeInBytes();
1043
832
  if (SizeSoFar < TotalSize) {
1044
203
    auto *PadTy = llvm::ArrayType::get(Int8Ty, TotalSize - SizeSoFar);
1045
203
    Values.push_back(patternOrZeroFor(CGM, isPattern, PadTy));
1046
203
  }
1047
832
  if (NestedIntact && 
Values.size() == STy->getNumElements()811
)
1048
526
    return constant;
1049
306
  return llvm::ConstantStruct::getAnon(Values, STy->isPacked());
1050
306
}
1051
1052
/// Replace all padding bytes in a given constant with either a pattern byte or
1053
/// 0x00.
1054
static llvm::Constant *constWithPadding(CodeGenModule &CGM, IsPattern isPattern,
1055
69.7k
                                        llvm::Constant *constant) {
1056
69.7k
  llvm::Type *OrigTy = constant->getType();
1057
69.7k
  if (const auto STy = dyn_cast<llvm::StructType>(OrigTy))
1058
832
    return constStructWithPadding(CGM, isPattern, STy, constant);
1059
68.9k
  if (auto *ArrayTy = dyn_cast<llvm::ArrayType>(OrigTy)) {
1060
219
    llvm::SmallVector<llvm::Constant *, 8> Values;
1061
219
    uint64_t Size = ArrayTy->getNumElements();
1062
219
    if (!Size)
1063
14
      return constant;
1064
205
    llvm::Type *ElemTy = ArrayTy->getElementType();
1065
205
    bool ZeroInitializer = constant->isNullValue();
1066
205
    llvm::Constant *OpValue, *PaddedOp;
1067
205
    if (ZeroInitializer) {
1068
69
      OpValue = llvm::Constant::getNullValue(ElemTy);
1069
69
      PaddedOp = constWithPadding(CGM, isPattern, OpValue);
1070
69
    }
1071
395k
    for (unsigned Op = 0; Op != Size; 
++Op394k
) {
1072
394k
      if (!ZeroInitializer) {
1073
66.7k
        OpValue = constant->getAggregateElement(Op);
1074
66.7k
        PaddedOp = constWithPadding(CGM, isPattern, OpValue);
1075
66.7k
      }
1076
394k
      Values.push_back(PaddedOp);
1077
394k
    }
1078
205
    auto *NewElemTy = Values[0]->getType();
1079
205
    if (NewElemTy == ElemTy)
1080
169
      return constant;
1081
36
    auto *NewArrayTy = llvm::ArrayType::get(NewElemTy, Size);
1082
36
    return llvm::ConstantArray::get(NewArrayTy, Values);
1083
36
  }
1084
  // FIXME: Add handling for tail padding in vectors. Vectors don't
1085
  // have padding between or inside elements, but the total amount of
1086
  // data can be less than the allocated size.
1087
68.7k
  return constant;
1088
68.7k
}
1089
1090
Address CodeGenModule::createUnnamedGlobalFrom(const VarDecl &D,
1091
                                               llvm::Constant *Constant,
1092
3.60k
                                               CharUnits Align) {
1093
3.59k
  auto FunctionName = [&](const DeclContext *DC) -> std::string {
1094
3.59k
    if (const auto *FD = dyn_cast<FunctionDecl>(DC)) {
1095
3.58k
      if (const auto *CC = dyn_cast<CXXConstructorDecl>(FD))
1096
1
        return CC->getNameAsString();
1097
3.58k
      if (const auto *CD = dyn_cast<CXXDestructorDecl>(FD))
1098
0
        return CD->getNameAsString();
1099
3.58k
      return std::string(getMangledName(FD));
1100
14
    } else if (const auto *OM = dyn_cast<ObjCMethodDecl>(DC)) {
1101
7
      return OM->getNameAsString();
1102
7
    } else if (isa<BlockDecl>(DC)) {
1103
0
      return "<block>";
1104
7
    } else if (isa<CapturedDecl>(DC)) {
1105
7
      return "<captured>";
1106
0
    } else {
1107
0
      llvm_unreachable("expected a function or method");
1108
0
    }
1109
3.59k
  };
1110
3.60k
1111
  // Form a simple per-variable cache of these values in case we find we
1112
  // want to reuse them.
1113
3.60k
  llvm::GlobalVariable *&CacheEntry = InitializerConstants[&D];
1114
3.60k
  if (!CacheEntry || 
CacheEntry->getInitializer() != Constant6
) {
1115
3.59k
    auto *Ty = Constant->getType();
1116
3.59k
    bool isConstant = true;
1117
3.59k
    llvm::GlobalVariable *InsertBefore = nullptr;
1118
3.59k
    unsigned AS =
1119
3.59k
        getContext().getTargetAddressSpace(getStringLiteralAddressSpace());
1120
3.59k
    std::string Name;
1121
3.59k
    if (D.hasGlobalStorage())
1122
2
      Name = getMangledName(&D).str() + ".const";
1123
3.59k
    else if (const DeclContext *DC = D.getParentFunctionOrMethod())
1124
3.59k
      Name = ("__const." + FunctionName(DC) + "." + D.getName()).str();
1125
3.59k
    else
1126
0
      llvm_unreachable("local variable has no parent function or method");
1127
3.59k
    llvm::GlobalVariable *GV = new llvm::GlobalVariable(
1128
3.59k
        getModule(), Ty, isConstant, llvm::GlobalValue::PrivateLinkage,
1129
3.59k
        Constant, Name, InsertBefore, llvm::GlobalValue::NotThreadLocal, AS);
1130
3.59k
    GV->setAlignment(Align.getAsAlign());
1131
3.59k
    GV->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
1132
3.59k
    CacheEntry = GV;
1133
6
  } else if (CacheEntry->getAlignment() < Align.getQuantity()) {
1134
0
    CacheEntry->setAlignment(Align.getAsAlign());
1135
0
  }
1136
3.60k
1137
3.60k
  return Address(CacheEntry, Align);
1138
3.60k
}
1139
1140
static Address createUnnamedGlobalForMemcpyFrom(CodeGenModule &CGM,
1141
                                                const VarDecl &D,
1142
                                                CGBuilderTy &Builder,
1143
                                                llvm::Constant *Constant,
1144
3.59k
                                                CharUnits Align) {
1145
3.59k
  Address SrcPtr = CGM.createUnnamedGlobalFrom(D, Constant, Align);
1146
3.59k
  llvm::Type *BP = llvm::PointerType::getInt8PtrTy(CGM.getLLVMContext(),
1147
3.59k
                                                   SrcPtr.getAddressSpace());
1148
3.59k
  if (SrcPtr.getType() != BP)
1149
3.59k
    SrcPtr = Builder.CreateBitCast(SrcPtr, BP);
1150
3.59k
  return SrcPtr;
1151
3.59k
}
1152
1153
static void emitStoresForConstant(CodeGenModule &CGM, const VarDecl &D,
1154
                                  Address Loc, bool isVolatile,
1155
                                  CGBuilderTy &Builder,
1156
5.55k
                                  llvm::Constant *constant) {
1157
5.55k
  auto *Ty = constant->getType();
1158
5.55k
  uint64_t ConstantSize = CGM.getDataLayout().getTypeAllocSize(Ty);
1159
5.55k
  if (!ConstantSize)
1160
19
    return;
1161
5.53k
1162
5.53k
  bool canDoSingleStore = Ty->isIntOrIntVectorTy() ||
1163
5.09k
                          Ty->isPtrOrPtrVectorTy() || 
Ty->isFPOrFPVectorTy()5.00k
;
1164
5.53k
  if (canDoSingleStore) {
1165
683
    Builder.CreateStore(constant, Loc, isVolatile);
1166
683
    return;
1167
683
  }
1168
4.85k
1169
4.85k
  auto *SizeVal = llvm::ConstantInt::get(CGM.IntPtrTy, ConstantSize);
1170
4.85k
1171
  // If the initializer is all or mostly the same, codegen with bzero / memset
1172
  // then do a few stores afterward.
1173
4.85k
  if (shouldUseBZeroPlusStoresToInitialize(constant, ConstantSize)) {
1174
1.10k
    Builder.CreateMemSet(Loc, llvm::ConstantInt::get(CGM.Int8Ty, 0), SizeVal,
1175
1.10k
                         isVolatile);
1176
1.10k
1177
1.10k
    bool valueAlreadyCorrect =
1178
1.10k
        constant->isNullValue() || 
isa<llvm::UndefValue>(constant)56
;
1179
1.10k
    if (!valueAlreadyCorrect) {
1180
56
      Loc = Builder.CreateBitCast(Loc, Ty->getPointerTo(Loc.getAddressSpace()));
1181
56
      emitStoresForInitAfterBZero(CGM, constant, Loc, isVolatile, Builder);
1182
56
    }
1183
1.10k
    return;
1184
1.10k
  }
1185
3.75k
1186
  // If the initializer is a repeated byte pattern, use memset.
1187
3.75k
  llvm::Value *Pattern =
1188
3.75k
      shouldUseMemSetToInitialize(constant, ConstantSize, CGM.getDataLayout());
1189
3.75k
  if (Pattern) {
1190
42
    uint64_t Value = 0x00;
1191
42
    if (!isa<llvm::UndefValue>(Pattern)) {
1192
42
      const llvm::APInt &AP = cast<llvm::ConstantInt>(Pattern)->getValue();
1193
42
      assert(AP.getBitWidth() <= 8);
1194
42
      Value = AP.getLimitedValue();
1195
42
    }
1196
42
    Builder.CreateMemSet(Loc, llvm::ConstantInt::get(CGM.Int8Ty, Value), SizeVal,
1197
42
                         isVolatile);
1198
42
    return;
1199
42
  }
1200
3.71k
1201
  // If the initializer is small, use a handful of stores.
1202
3.71k
  if (shouldSplitConstantStore(CGM, ConstantSize)) {
1203
297
    if (auto *STy = dyn_cast<llvm::StructType>(Ty)) {
1204
      // FIXME: handle the case when STy != Loc.getElementType().
1205
256
      if (STy == Loc.getElementType()) {
1206
256
        for (unsigned i = 0; i != constant->getNumOperands(); 
i++154
) {
1207
154
          Address EltPtr = Builder.CreateStructGEP(Loc, i);
1208
154
          emitStoresForConstant(
1209
154
              CGM, D, EltPtr, isVolatile, Builder,
1210
154
              cast<llvm::Constant>(Builder.CreateExtractValue(constant, i)));
1211
154
        }
1212
102
        return;
1213
102
      }
1214
41
    } else if (auto *ATy = dyn_cast<llvm::ArrayType>(Ty)) {
1215
      // FIXME: handle the case when ATy != Loc.getElementType().
1216
41
      if (ATy == Loc.getElementType()) {
1217
64
        for (unsigned i = 0; i != ATy->getNumElements(); 
i++48
) {
1218
48
          Address EltPtr = Builder.CreateConstArrayGEP(Loc, i);
1219
48
          emitStoresForConstant(
1220
48
              CGM, D, EltPtr, isVolatile, Builder,
1221
48
              cast<llvm::Constant>(Builder.CreateExtractValue(constant, i)));
1222
48
        }
1223
16
        return;
1224
16
      }
1225
3.59k
    }
1226
297
  }
1227
3.59k
1228
  // Copy from a global.
1229
3.59k
  Builder.CreateMemCpy(Loc,
1230
3.59k
                       createUnnamedGlobalForMemcpyFrom(
1231
3.59k
                           CGM, D, Builder, constant, Loc.getAlignment()),
1232
3.59k
                       SizeVal, isVolatile);
1233
3.59k
}
1234
1235
static void emitStoresForZeroInit(CodeGenModule &CGM, const VarDecl &D,
1236
                                  Address Loc, bool isVolatile,
1237
391
                                  CGBuilderTy &Builder) {
1238
391
  llvm::Type *ElTy = Loc.getElementType();
1239
391
  llvm::Constant *constant =
1240
391
      constWithPadding(CGM, IsPattern::No, llvm::Constant::getNullValue(ElTy));
1241
391
  emitStoresForConstant(CGM, D, Loc, isVolatile, Builder, constant);
1242
391
}
1243
1244
static void emitStoresForPatternInit(CodeGenModule &CGM, const VarDecl &D,
1245
                                     Address Loc, bool isVolatile,
1246
514
                                     CGBuilderTy &Builder) {
1247
514
  llvm::Type *ElTy = Loc.getElementType();
1248
514
  llvm::Constant *constant = constWithPadding(
1249
514
      CGM, IsPattern::Yes, initializationPatternFor(CGM, ElTy));
1250
514
  assert(!isa<llvm::UndefValue>(constant));
1251
514
  emitStoresForConstant(CGM, D, Loc, isVolatile, Builder, constant);
1252
514
}
1253
1254
1.01k
static bool containsUndef(llvm::Constant *constant) {
1255
1.01k
  auto *Ty = constant->getType();
1256
1.01k
  if (isa<llvm::UndefValue>(constant))
1257
49
    return true;
1258
968
  if (Ty->isStructTy() || 
Ty->isArrayTy()687
||
Ty->isVectorTy()625
)
1259
343
    for (llvm::Use &Op : constant->operands())
1260
827
      if (containsUndef(cast<llvm::Constant>(Op)))
1261
49
        return true;
1262
919
  return false;
1263
968
}
1264
1265
static llvm::Constant *replaceUndef(CodeGenModule &CGM, IsPattern isPattern,
1266
323
                                    llvm::Constant *constant) {
1267
323
  auto *Ty = constant->getType();
1268
323
  if (isa<llvm::UndefValue>(constant))
1269
70
    return patternOrZeroFor(CGM, isPattern, Ty);
1270
253
  if (!(Ty->isStructTy() || 
Ty->isArrayTy()105
||
Ty->isVectorTy()63
))
1271
63
    return constant;
1272
190
  if (!containsUndef(constant))
1273
141
    return constant;
1274
49
  llvm::SmallVector<llvm::Constant *, 8> Values(constant->getNumOperands());
1275
175
  for (unsigned Op = 0, NumOp = constant->getNumOperands(); Op != NumOp; 
++Op126
) {
1276
126
    auto *OpValue = cast<llvm::Constant>(constant->getOperand(Op));
1277
126
    Values[Op] = replaceUndef(CGM, isPattern, OpValue);
1278
126
  }
1279
49
  if (Ty->isStructTy())
1280
49
    return llvm::ConstantStruct::get(cast<llvm::StructType>(Ty), Values);
1281
0
  if (Ty->isArrayTy())
1282
0
    return llvm::ConstantArray::get(cast<llvm::ArrayType>(Ty), Values);
1283
0
  assert(Ty->isVectorTy());
1284
0
  return llvm::ConstantVector::get(Values);
1285
0
}
1286
1287
/// EmitAutoVarDecl - Emit code and set up an entry in LocalDeclMap for a
1288
/// variable declaration with auto, register, or no storage class specifier.
1289
/// These turn into simple stack objects, or GlobalValues depending on target.
1290
191k
void CodeGenFunction::EmitAutoVarDecl(const VarDecl &D) {
1291
191k
  AutoVarEmission emission = EmitAutoVarAlloca(D);
1292
191k
  EmitAutoVarInit(emission);
1293
191k
  EmitAutoVarCleanups(emission);
1294
191k
}
1295
1296
/// Emit a lifetime.begin marker if some criteria are satisfied.
1297
/// \return a pointer to the temporary size Value if a marker was emitted, null
1298
/// otherwise
1299
llvm::Value *CodeGenFunction::EmitLifetimeStart(uint64_t Size,
1300
207k
                                                llvm::Value *Addr) {
1301
207k
  if (!ShouldEmitLifetimeMarkers)
1302
201k
    return nullptr;
1303
5.68k
1304
5.68k
  assert(Addr->getType()->getPointerAddressSpace() ==
1305
5.68k
             CGM.getDataLayout().getAllocaAddrSpace() &&
1306
5.68k
         "Pointer should be in alloca address space");
1307
5.68k
  llvm::Value *SizeV = llvm::ConstantInt::get(Int64Ty, Size);
1308
5.68k
  Addr = Builder.CreateBitCast(Addr, AllocaInt8PtrTy);
1309
5.68k
  llvm::CallInst *C =
1310
5.68k
      Builder.CreateCall(CGM.getLLVMLifetimeStartFn(), {SizeV, Addr});
1311
5.68k
  C->setDoesNotThrow();
1312
5.68k
  return SizeV;
1313
5.68k
}
1314
1315
5.96k
void CodeGenFunction::EmitLifetimeEnd(llvm::Value *Size, llvm::Value *Addr) {
1316
5.96k
  assert(Addr->getType()->getPointerAddressSpace() ==
1317
5.96k
             CGM.getDataLayout().getAllocaAddrSpace() &&
1318
5.96k
         "Pointer should be in alloca address space");
1319
5.96k
  Addr = Builder.CreateBitCast(Addr, AllocaInt8PtrTy);
1320
5.96k
  llvm::CallInst *C =
1321
5.96k
      Builder.CreateCall(CGM.getLLVMLifetimeEndFn(), {Size, Addr});
1322
5.96k
  C->setDoesNotThrow();
1323
5.96k
}
1324
1325
void CodeGenFunction::EmitAndRegisterVariableArrayDimensions(
1326
2.14k
    CGDebugInfo *DI, const VarDecl &D, bool EmitDebugInfo) {
1327
  // For each dimension stores its QualType and corresponding
1328
  // size-expression Value.
1329
2.14k
  SmallVector<CodeGenFunction::VlaSizePair, 4> Dimensions;
1330
2.14k
  SmallVector<IdentifierInfo *, 4> VLAExprNames;
1331
2.14k
1332
  // Break down the array into individual dimensions.
1333
2.14k
  QualType Type1D = D.getType();
1334
5.32k
  while (getContext().getAsVariableArrayType(Type1D)) {
1335
3.17k
    auto VlaSize = getVLAElements1D(Type1D);
1336
3.17k
    if (auto *C = dyn_cast<llvm::ConstantInt>(VlaSize.NumElts))
1337
885
      Dimensions.emplace_back(C, Type1D.getUnqualifiedType());
1338
2.29k
    else {
1339
      // Generate a locally unique name for the size expression.
1340
2.29k
      Twine Name = Twine("__vla_expr") + Twine(VLAExprCounter++);
1341
2.29k
      SmallString<12> Buffer;
1342
2.29k
      StringRef NameRef = Name.toStringRef(Buffer);
1343
2.29k
      auto &Ident = getContext().Idents.getOwn(NameRef);
1344
2.29k
      VLAExprNames.push_back(&Ident);
1345
2.29k
      auto SizeExprAddr =
1346
2.29k
          CreateDefaultAlignTempAlloca(VlaSize.NumElts->getType(), NameRef);
1347
2.29k
      Builder.CreateStore(VlaSize.NumElts, SizeExprAddr);
1348
2.29k
      Dimensions.emplace_back(SizeExprAddr.getPointer(),
1349
2.29k
                              Type1D.getUnqualifiedType());
1350
2.29k
    }
1351
3.17k
    Type1D = VlaSize.Type;
1352
3.17k
  }
1353
2.14k
1354
2.14k
  if (!EmitDebugInfo)
1355
2.10k
    return;
1356
39
1357
  // Register each dimension's size-expression with a DILocalVariable,
1358
  // so that it can be used by CGDebugInfo when instantiating a DISubrange
1359
  // to describe this array.
1360
39
  unsigned NameIdx = 0;
1361
40
  for (auto &VlaSize : Dimensions) {
1362
40
    llvm::Metadata *MD;
1363
40
    if (auto *C = dyn_cast<llvm::ConstantInt>(VlaSize.NumElts))
1364
1
      MD = llvm::ConstantAsMetadata::get(C);
1365
39
    else {
1366
      // Create an artificial VarDecl to generate debug info for.
1367
39
      IdentifierInfo *NameIdent = VLAExprNames[NameIdx++];
1368
39
      auto VlaExprTy = VlaSize.NumElts->getType()->getPointerElementType();
1369
39
      auto QT = getContext().getIntTypeForBitwidth(
1370
39
          VlaExprTy->getScalarSizeInBits(), false);
1371
39
      auto *ArtificialDecl = VarDecl::Create(
1372
39
          getContext(), const_cast<DeclContext *>(D.getDeclContext()),
1373
39
          D.getLocation(), D.getLocation(), NameIdent, QT,
1374
39
          getContext().CreateTypeSourceInfo(QT), SC_Auto);
1375
39
      ArtificialDecl->setImplicit();
1376
39
1377
39
      MD = DI->EmitDeclareOfAutoVariable(ArtificialDecl, VlaSize.NumElts,
1378
39
                                         Builder);
1379
39
    }
1380
40
    assert(MD && "No Size expression debug node created");
1381
40
    DI->registerVLASizeExpression(VlaSize.Type, MD);
1382
40
  }
1383
39
}
1384
1385
/// EmitAutoVarAlloca - Emit the alloca and debug information for a
1386
/// local variable.  Does not emit initialization or destruction.
1387
CodeGenFunction::AutoVarEmission
1388
210k
CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) {
1389
210k
  QualType Ty = D.getType();
1390
210k
  assert(
1391
210k
      Ty.getAddressSpace() == LangAS::Default ||
1392
210k
      (Ty.getAddressSpace() == LangAS::opencl_private && getLangOpts().OpenCL));
1393
210k
1394
210k
  AutoVarEmission emission(D);
1395
210k
1396
210k
  bool isEscapingByRef = D.isEscapingByref();
1397
210k
  emission.IsEscapingByRef = isEscapingByRef;
1398
210k
1399
210k
  CharUnits alignment = getContext().getDeclAlign(&D);
1400
210k
1401
  // If the type is variably-modified, emit all the VLA sizes for it.
1402
210k
  if (Ty->isVariablyModifiedType())
1403
2.22k
    EmitVariablyModifiedType(Ty);
1404
210k
1405
210k
  auto *DI = getDebugInfo();
1406
210k
  bool EmitDebugInfo = DI && 
CGM.getCodeGenOpts().hasReducedDebugInfo()65.5k
;
1407
210k
1408
210k
  Address address = Address::invalid();
1409
210k
  Address AllocaAddr = Address::invalid();
1410
210k
  Address OpenMPLocalAddr = Address::invalid();
1411
210k
  if (CGM.getLangOpts().OpenMPIRBuilder)
1412
220
    OpenMPLocalAddr = OMPBuilderCBHelpers::getAddressOfLocalVariable(*this, &D);
1413
209k
  else
1414
209k
    OpenMPLocalAddr =
1415
209k
        getLangOpts().OpenMP
1416
114k
            ? CGM.getOpenMPRuntime().getAddressOfLocalVariable(*this, &D)
1417
95.5k
            : Address::invalid();
1418
210k
1419
210k
  bool NRVO = getLangOpts().ElideConstructors && 
D.isNRVOVariable()210k
;
1420
210k
1421
210k
  if (getLangOpts().OpenMP && 
OpenMPLocalAddr.isValid()114k
) {
1422
100
    address = OpenMPLocalAddr;
1423
210k
  } else if (Ty->isConstantSizeType()) {
1424
    // If this value is an array or struct with a statically determinable
1425
    // constant initializer, there are optimizations we can do.
1426
    //
1427
    // TODO: We should constant-evaluate the initializer of any variable,
1428
    // as long as it is initialized by a constant expression. Currently,
1429
    // isConstantInitializer produces wrong answers for structs with
1430
    // reference or bitfield members, and a few other cases, and checking
1431
    // for POD-ness protects us from some of these.
1432
207k
    if (D.getInit() && 
(156k
Ty->isArrayType()156k
||
Ty->isRecordType()151k
) &&
1433
20.8k
        (D.isConstexpr() ||
1434
20.6k
         ((Ty.isPODType(getContext()) ||
1435
11.4k
           getContext().getBaseElementType(Ty)->isObjCObjectPointerType()) &&
1436
9.24k
          D.getInit()->isConstantInitializer(getContext(), false)))) {
1437
7.76k
1438
      // If the variable's a const type, and it's neither an NRVO
1439
      // candidate nor a __block variable and has no mutable members,
1440
      // emit it as a global instead.
1441
      // Exception is if a variable is located in non-constant address space
1442
      // in OpenCL.
1443
7.76k
      if ((!getLangOpts().OpenCL ||
1444
84
           Ty.getAddressSpace() == LangAS::opencl_constant) &&
1445
7.68k
          (CGM.getCodeGenOpts().MergeAllConstants && 
!NRVO86
&&
1446
86
           !isEscapingByRef && CGM.isTypeConstant(Ty, true))) {
1447
6
        EmitStaticVarDecl(D, llvm::GlobalValue::InternalLinkage);
1448
6
1449
        // Signal this condition to later callbacks.
1450
6
        emission.Addr = Address::invalid();
1451
6
        assert(emission.wasEmittedAsGlobal());
1452
6
        return emission;
1453
6
      }
1454
7.76k
1455
      // Otherwise, tell the initialization code that we're in this case.
1456
7.76k
      emission.IsConstantAggregate = true;
1457
7.76k
    }
1458
207k
1459
    // A normal fixed sized variable becomes an alloca in the entry block,
1460
    // unless:
1461
    // - it's an NRVO variable.
1462
    // - we are compiling OpenMP and it's an OpenMP local variable.
1463
207k
    if (NRVO) {
1464
      // The named return value optimization: allocate this variable in the
1465
      // return slot, so that we can elide the copy when returning this
1466
      // variable (C++0x [class.copy]p34).
1467
1.28k
      address = ReturnValue;
1468
1.28k
1469
1.28k
      if (const RecordType *RecordTy = Ty->getAs<RecordType>()) {
1470
1.28k
        const auto *RD = RecordTy->getDecl();
1471
1.28k
        const auto *CXXRD = dyn_cast<CXXRecordDecl>(RD);
1472
1.28k
        if ((CXXRD && 
!CXXRD->hasTrivialDestructor()445
) ||
1473
1.09k
            RD->isNonTrivialToPrimitiveDestroy()) {
1474
          // Create a flag that is used to indicate when the NRVO was applied
1475
          // to this variable. Set it to zero to indicate that NRVO was not
1476
          // applied.
1477
192
          llvm::Value *Zero = Builder.getFalse();
1478
192
          Address NRVOFlag =
1479
192
            CreateTempAlloca(Zero->getType(), CharUnits::One(), "nrvo");
1480
192
          EnsureInsertPoint();
1481
192
          Builder.CreateStore(Zero, NRVOFlag);
1482
192
1483
          // Record the NRVO flag for this variable.
1484
192
          NRVOFlags[&D] = NRVOFlag.getPointer();
1485
192
          emission.NRVOFlag = NRVOFlag.getPointer();
1486
192
        }
1487
1.28k
      }
1488
206k
    } else {
1489
206k
      CharUnits allocaAlignment;
1490
206k
      llvm::Type *allocaTy;
1491
206k
      if (isEscapingByRef) {
1492
205
        auto &byrefInfo = getBlockByrefInfo(&D);
1493
205
        allocaTy = byrefInfo.Type;
1494
205
        allocaAlignment = byrefInfo.ByrefAlignment;
1495
206k
      } else {
1496
206k
        allocaTy = ConvertTypeForMem(Ty);
1497
206k
        allocaAlignment = alignment;
1498
206k
      }
1499
206k
1500
      // Create the alloca.  Note that we set the name separately from
1501
      // building the instruction so that it's there even in no-asserts
1502
      // builds.
1503
206k
      address = CreateTempAlloca(allocaTy, allocaAlignment, D.getName(),
1504
206k
                                 /*ArraySize=*/nullptr, &AllocaAddr);
1505
206k
1506
      // Don't emit lifetime markers for MSVC catch parameters. The lifetime of
1507
      // the catch parameter starts in the catchpad instruction, and we can't
1508
      // insert code in those basic blocks.
1509
206k
      bool IsMSCatchParam =
1510
206k
          D.isExceptionVariable() && 
getTarget().getCXXABI().isMicrosoft()366
;
1511
206k
1512
      // Emit a lifetime intrinsic if meaningful. There's no point in doing this
1513
      // if we don't have a valid insertion point (?).
1514
206k
      if (HaveInsertPoint() && 
!IsMSCatchParam206k
) {
1515
        // If there's a jump into the lifetime of this variable, its lifetime
1516
        // gets broken up into several regions in IR, which requires more work
1517
        // to handle correctly. For now, just omit the intrinsics; this is a
1518
        // rare case, and it's better to just be conservatively correct.
1519
        // PR28267.
1520
        //
1521
        // We have to do this in all language modes if there's a jump past the
1522
        // declaration. We also have to do it in C if there's a jump to an
1523
        // earlier point in the current block because non-VLA lifetimes begin as
1524
        // soon as the containing block is entered, not when its variables
1525
        // actually come into scope; suppressing the lifetime annotations
1526
        // completely in this case is unnecessarily pessimistic, but again, this
1527
        // is rare.
1528
206k
        if (!Bypasses.IsBypassed(&D) &&
1529
206k
            !(!getLangOpts().CPlusPlus && 
hasLabelBeenSeenInCurrentScope()34.3k
)) {
1530
206k
          llvm::TypeSize size =
1531
206k
              CGM.getDataLayout().getTypeAllocSize(allocaTy);
1532
206k
          emission.SizeForLifetimeMarkers =
1533
124
              size.isScalable() ? EmitLifetimeStart(-1, AllocaAddr.getPointer())
1534
206k
                                : EmitLifetimeStart(size.getFixedSize(),
1535
206k
                                                    AllocaAddr.getPointer());
1536
206k
        }
1537
41
      } else {
1538
41
        assert(!emission.useLifetimeMarkers());
1539
41
      }
1540
206k
    }
1541
2.14k
  } else {
1542
2.14k
    EnsureInsertPoint();
1543
2.14k
1544
2.14k
    if (!DidCallStackSave) {
1545
      // Save the stack.
1546
1.57k
      Address Stack =
1547
1.57k
        CreateTempAlloca(Int8PtrTy, getPointerAlign(), "saved_stack");
1548
1.57k
1549
1.57k
      llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::stacksave);
1550
1.57k
      llvm::Value *V = Builder.CreateCall(F);
1551
1.57k
      Builder.CreateStore(V, Stack);
1552
1.57k
1553
1.57k
      DidCallStackSave = true;
1554
1.57k
1555
      // Push a cleanup block and restore the stack there.
1556
      // FIXME: in general circumstances, this should be an EH cleanup.
1557
1.57k
      pushStackRestore(NormalCleanup, Stack);
1558
1.57k
    }
1559
2.14k
1560
2.14k
    auto VlaSize = getVLASize(Ty);
1561
2.14k
    llvm::Type *llvmTy = ConvertTypeForMem(VlaSize.Type);
1562
2.14k
1563
    // Allocate memory for the array.
1564
2.14k
    address = CreateTempAlloca(llvmTy, alignment, "vla", VlaSize.NumElts,
1565
2.14k
                               &AllocaAddr);
1566
2.14k
1567
    // If we have debug info enabled, properly describe the VLA dimensions for
1568
    // this type by registering the vla size expression for each of the
1569
    // dimensions.
1570
2.14k
    EmitAndRegisterVariableArrayDimensions(DI, D, EmitDebugInfo);
1571
2.14k
  }
1572
210k
1573
210k
  setAddrOfLocalVar(&D, address);
1574
210k
  emission.Addr = address;
1575
210k
  emission.AllocaAddr = AllocaAddr;
1576
210k
1577
  // Emit debug info for local var declaration.
1578
210k
  if (EmitDebugInfo && 
HaveInsertPoint()63.9k
) {
1579
63.9k
    Address DebugAddr = address;
1580
63.9k
    bool UsePointerValue = NRVO && 
ReturnValuePointer.isValid()286
;
1581
63.9k
    DI->setLocation(D.getLocation());
1582
63.9k
1583
    // If NRVO, use a pointer to the return address.
1584
63.9k
    if (UsePointerValue)
1585
139
      DebugAddr = ReturnValuePointer;
1586
63.9k
1587
63.9k
    (void)DI->EmitDeclareOfAutoVariable(&D, DebugAddr.getPointer(), Builder,
1588
63.9k
                                        UsePointerValue);
1589
63.9k
  }
1590
210k
1591
210k
  if (D.hasAttr<AnnotateAttr>() && 
HaveInsertPoint()5
)
1592
4
    EmitVarAnnotations(&D, address.getPointer());
1593
210k
1594
  // Make sure we call @llvm.lifetime.end.
1595
210k
  if (emission.useLifetimeMarkers())
1596
5.10k
    EHStack.pushCleanup<CallLifetimeEnd>(NormalEHLifetimeMarker,
1597
5.10k
                                         emission.getOriginalAllocatedAddress(),
1598
5.10k
                                         emission.getSizeForLifetimeMarkers());
1599
210k
1600
210k
  return emission;
1601
210k
}
1602
1603
static bool isCapturedBy(const VarDecl &, const Expr *);
1604
1605
/// Determines whether the given __block variable is potentially
1606
/// captured by the given statement.
1607
24
static bool isCapturedBy(const VarDecl &Var, const Stmt *S) {
1608
24
  if (const Expr *E = dyn_cast<Expr>(S))
1609
22
    return isCapturedBy(Var, E);
1610
2
  for (const Stmt *SubStmt : S->children())
1611
2
    if (isCapturedBy(Var, SubStmt))
1612
0
      return true;
1613
2
  return false;
1614
2
}
1615
1616
/// Determines whether the given __block variable is potentially
1617
/// captured by the given expression.
1618
115
static bool isCapturedBy(const VarDecl &Var, const Expr *E) {
1619
  // Skip the most common kinds of expressions that make
1620
  // hierarchy-walking expensive.
1621
115
  E = E->IgnoreParenCasts();
1622
115
1623
115
  if (const BlockExpr *BE = dyn_cast<BlockExpr>(E)) {
1624
6
    const BlockDecl *Block = BE->getBlockDecl();
1625
6
    for (const auto &I : Block->captures()) {
1626
6
      if (I.getVariable() == &Var)
1627
6
        return true;
1628
6
    }
1629
6
1630
    // No need to walk into the subexpressions.
1631
0
    return false;
1632
109
  }
1633
109
1634
109
  if (const StmtExpr *SE = dyn_cast<StmtExpr>(E)) {
1635
0
    const CompoundStmt *CS = SE->getSubStmt();
1636
0
    for (const auto *BI : CS->body())
1637
0
      if (const auto *BIE = dyn_cast<Expr>(BI)) {
1638
0
        if (isCapturedBy(Var, BIE))
1639
0
          return true;
1640
0
      }
1641
0
      else if (const auto *DS = dyn_cast<DeclStmt>(BI)) {
1642
          // special case declarations
1643
0
          for (const auto *I : DS->decls()) {
1644
0
              if (const auto *VD = dyn_cast<VarDecl>((I))) {
1645
0
                const Expr *Init = VD->getInit();
1646
0
                if (Init && isCapturedBy(Var, Init))
1647
0
                  return true;
1648
0
              }
1649
0
          }
1650
0
      }
1651
0
      else
1652
        // FIXME. Make safe assumption assuming arbitrary statements cause capturing.
1653
        // Later, provide code to poke into statements for capture analysis.
1654
0
        return true;
1655
0
    return false;
1656
109
  }
1657
109
1658
109
  for (const Stmt *SubStmt : E->children())
1659
22
    if (isCapturedBy(Var, SubStmt))
1660
3
      return true;
1661
109
1662
106
  return false;
1663
109
}
1664
1665
/// Determine whether the given initializer is trivial in the sense
1666
/// that it requires no code to be generated.
1667
193k
bool CodeGenFunction::isTrivialInitializer(const Expr *Init) {
1668
193k
  if (!Init)
1669
37.6k
    return true;
1670
155k
1671
155k
  if (const CXXConstructExpr *Construct = dyn_cast<CXXConstructExpr>(Init))
1672
12.1k
    if (CXXConstructorDecl *Constructor = Construct->getConstructor())
1673
12.1k
      if (Constructor->isTrivial() &&
1674
4.01k
          Constructor->isDefaultConstructor() &&
1675
3.60k
          !Construct->requiresZeroInitialization())
1676
3.59k
        return true;
1677
151k
1678
151k
  return false;
1679
151k
}
1680
1681
void CodeGenFunction::emitZeroOrPatternForAutoVarInit(QualType type,
1682
                                                      const VarDecl &D,
1683
936
                                                      Address Loc) {
1684
936
  auto trivialAutoVarInit = getContext().getLangOpts().getTrivialAutoVarInit();
1685
936
  CharUnits Size = getContext().getTypeSizeInChars(type);
1686
936
  bool isVolatile = type.isVolatileQualified();
1687
936
  if (!Size.isZero()) {
1688
913
    switch (trivialAutoVarInit) {
1689
0
    case LangOptions::TrivialAutoVarInitKind::Uninitialized:
1690
0
      llvm_unreachable("Uninitialized handled by caller");
1691
395
    case LangOptions::TrivialAutoVarInitKind::Zero:
1692
395
      if (CGM.stopAutoInit())
1693
4
        return;
1694
391
      emitStoresForZeroInit(CGM, D, Loc, isVolatile, Builder);
1695
391
      break;
1696
518
    case LangOptions::TrivialAutoVarInitKind::Pattern:
1697
518
      if (CGM.stopAutoInit())
1698
4
        return;
1699
514
      emitStoresForPatternInit(CGM, D, Loc, isVolatile, Builder);
1700
514
      break;
1701
905
    }
1702
905
    return;
1703
905
  }
1704
23
1705
  // VLAs look zero-sized to getTypeInfo. We can't emit constant stores to
1706
  // them, so emit a memcpy with the VLA size to initialize each element.
1707
  // Technically zero-sized or negative-sized VLAs are undefined, and UBSan
1708
  // will catch that code, but there exists code which generates zero-sized
1709
  // VLAs. Be nice and initialize whatever they requested.
1710
23
  const auto *VlaType = getContext().getAsVariableArrayType(type);
1711
23
  if (!VlaType)
1712
9
    return;
1713
14
  auto VlaSize = getVLASize(VlaType);
1714
14
  auto SizeVal = VlaSize.NumElts;
1715
14
  CharUnits EltSize = getContext().getTypeSizeInChars(VlaSize.Type);
1716
14
  switch (trivialAutoVarInit) {
1717
0
  case LangOptions::TrivialAutoVarInitKind::Uninitialized:
1718
0
    llvm_unreachable("Uninitialized handled by caller");
1719
0
1720
7
  case LangOptions::TrivialAutoVarInitKind::Zero:
1721
7
    if (CGM.stopAutoInit())
1722
2
      return;
1723
5
    if (!EltSize.isOne())
1724
5
      SizeVal = Builder.CreateNUWMul(SizeVal, CGM.getSize(EltSize));
1725
5
    Builder.CreateMemSet(Loc, llvm::ConstantInt::get(Int8Ty, 0), SizeVal,
1726
5
                         isVolatile);
1727
5
    break;
1728
5
1729
7
  case LangOptions::TrivialAutoVarInitKind::Pattern: {
1730
7
    if (CGM.stopAutoInit())
1731
2
      return;
1732
5
    llvm::Type *ElTy = Loc.getElementType();
1733
5
    llvm::Constant *Constant = constWithPadding(
1734
5
        CGM, IsPattern::Yes, initializationPatternFor(CGM, ElTy));
1735
5
    CharUnits ConstantAlign = getContext().getTypeAlignInChars(VlaSize.Type);
1736
5
    llvm::BasicBlock *SetupBB = createBasicBlock("vla-setup.loop");
1737
5
    llvm::BasicBlock *LoopBB = createBasicBlock("vla-init.loop");
1738
5
    llvm::BasicBlock *ContBB = createBasicBlock("vla-init.cont");
1739
5
    llvm::Value *IsZeroSizedVLA = Builder.CreateICmpEQ(
1740
5
        SizeVal, llvm::ConstantInt::get(SizeVal->getType(), 0),
1741
5
        "vla.iszerosized");
1742
5
    Builder.CreateCondBr(IsZeroSizedVLA, ContBB, SetupBB);
1743
5
    EmitBlock(SetupBB);
1744
5
    if (!EltSize.isOne())
1745
5
      SizeVal = Builder.CreateNUWMul(SizeVal, CGM.getSize(EltSize));
1746
5
    llvm::Value *BaseSizeInChars =
1747
5
        llvm::ConstantInt::get(IntPtrTy, EltSize.getQuantity());
1748
5
    Address Begin = Builder.CreateElementBitCast(Loc, Int8Ty, "vla.begin");
1749
5
    llvm::Value *End =
1750
5
        Builder.CreateInBoundsGEP(Begin.getPointer(), SizeVal, "vla.end");
1751
5
    llvm::BasicBlock *OriginBB = Builder.GetInsertBlock();
1752
5
    EmitBlock(LoopBB);
1753
5
    llvm::PHINode *Cur = Builder.CreatePHI(Begin.getType(), 2, "vla.cur");
1754
5
    Cur->addIncoming(Begin.getPointer(), OriginBB);
1755
5
    CharUnits CurAlign = Loc.getAlignment().alignmentOfArrayElement(EltSize);
1756
5
    Builder.CreateMemCpy(Address(Cur, CurAlign),
1757
5
                         createUnnamedGlobalForMemcpyFrom(
1758
5
                             CGM, D, Builder, Constant, ConstantAlign),
1759
5
                         BaseSizeInChars, isVolatile);
1760
5
    llvm::Value *Next =
1761
5
        Builder.CreateInBoundsGEP(Int8Ty, Cur, BaseSizeInChars, "vla.next");
1762
5
    llvm::Value *Done = Builder.CreateICmpEQ(Next, End, "vla-init.isdone");
1763
5
    Builder.CreateCondBr(Done, ContBB, LoopBB);
1764
5
    Cur->addIncoming(Next, LoopBB);
1765
5
    EmitBlock(ContBB);
1766
5
  } break;
1767
14
  }
1768
14
}
1769
1770
192k
void CodeGenFunction::EmitAutoVarInit(const AutoVarEmission &emission) {
1771
192k
  assert(emission.Variable && "emission was not valid!");
1772
192k
1773
  // If this was emitted as a global constant, we're done.
1774
192k
  if (emission.wasEmittedAsGlobal()) 
return6
;
1775
192k
1776
192k
  const VarDecl &D = *emission.Variable;
1777
192k
  auto DL = ApplyDebugLocation::CreateDefaultArtificial(*this, D.getLocation());
1778
192k
  QualType type = D.getType();
1779
192k
1780
  // If this local has an initializer, emit it now.
1781
192k
  const Expr *Init = D.getInit();
1782
192k
1783
  // If we are at an unreachable point, we don't need to emit the initializer
1784
  // unless it contains a label.
1785
192k
  if (!HaveInsertPoint()) {
1786
19
    if (!Init || 
!ContainsLabel(Init)11
) return;
1787
0
    EnsureInsertPoint();
1788
0
  }
1789
192k
1790
  // Initialize the structure of a __block variable.
1791
192k
  if (emission.IsEscapingByRef)
1792
205
    emitByrefStructureInit(emission);
1793
192k
1794
  // Initialize the variable here if it doesn't have a initializer and it is a
1795
  // C struct that is non-trivial to initialize or an array containing such a
1796
  // struct.
1797
192k
  if (!Init &&
1798
37.6k
      type.isNonTrivialToPrimitiveDefaultInitialize() ==
1799
37
          QualType::PDIK_Struct) {
1800
37
    LValue Dst = MakeAddrLValue(emission.getAllocatedAddress(), type);
1801
37
    if (emission.IsEscapingByRef)
1802
9
      drillIntoBlockVariable(*this, Dst, &D);
1803
37
    defaultInitNonTrivialCStructVar(Dst);
1804
37
    return;
1805
37
  }
1806
192k
1807
  // Check whether this is a byref variable that's potentially
1808
  // captured and moved by its own initializer.  If so, we'll need to
1809
  // emit the initializer first, then copy into the variable.
1810
192k
  bool capturedByInit =
1811
192k
      Init && 
emission.IsEscapingByRef154k
&&
isCapturedBy(D, Init)93
;
1812
192k
1813
192k
  bool locIsByrefHeader = !capturedByInit;
1814
192k
  const Address Loc =
1815
192k
      locIsByrefHeader ? emission.getObjectAddress(*this) : 
emission.Addr6
;
1816
192k
1817
  // Note: constexpr already initializes everything correctly.
1818
192k
  LangOptions::TrivialAutoVarInitKind trivialAutoVarInit =
1819
192k
      (D.isConstexpr()
1820
344
           ? LangOptions::TrivialAutoVarInitKind::Uninitialized
1821
192k
           : (D.getAttr<UninitializedAttr>()
1822
6
                  ? LangOptions::TrivialAutoVarInitKind::Uninitialized
1823
192k
                  : getContext().getLangOpts().getTrivialAutoVarInit()));
1824
192k
1825
187k
  auto initializeWhatIsTechnicallyUninitialized = [&](Address Loc) {
1826
187k
    if (trivialAutoVarInit ==
1827
187k
        LangOptions::TrivialAutoVarInitKind::Uninitialized)
1828
186k
      return;
1829
936
1830
    // Only initialize a __block's storage: we always initialize the header.
1831
936
    if (emission.IsEscapingByRef && 
!locIsByrefHeader4
)
1832
2
      Loc = emitBlockByrefAddress(Loc, &D, /*follow=*/false);
1833
936
1834
936
    return emitZeroOrPatternForAutoVarInit(type, D, Loc);
1835
936
  };
1836
192k
1837
192k
  if (isTrivialInitializer(Init))
1838
41.1k
    return initializeWhatIsTechnicallyUninitialized(Loc);
1839
151k
1840
151k
  llvm::Constant *constant = nullptr;
1841
151k
  if (emission.IsConstantAggregate ||
1842
146k
      D.mightBeUsableInConstantExpressions(getContext())) {
1843
14.8k
    assert(!capturedByInit && "constant init contains a capturing block?");
1844
14.8k
    constant = ConstantEmitter(*this).tryEmitAbstractForInitializer(D);
1845
14.8k
    if (constant && 
!constant->isZeroValue()4.92k
&&
1846
3.99k
        (trivialAutoVarInit !=
1847
197
         LangOptions::TrivialAutoVarInitKind::Uninitialized)) {
1848
197
      IsPattern isPattern =
1849
197
          (trivialAutoVarInit == LangOptions::TrivialAutoVarInitKind::Pattern)
1850
112
              ? IsPattern::Yes
1851
85
              : IsPattern::No;
1852
      // C guarantees that brace-init with fewer initializers than members in
1853
      // the aggregate will initialize the rest of the aggregate as-if it were
1854
      // static initialization. In turn static initialization guarantees that
1855
      // padding is initialized to zero bits. We could instead pattern-init if D
1856
      // has any ImplicitValueInitExpr, but that seems to be unintuitive
1857
      // behavior.
1858
197
      constant = constWithPadding(CGM, IsPattern::No,
1859
197
                                  replaceUndef(CGM, isPattern, constant));
1860
197
    }
1861
14.8k
  }
1862
151k
1863
151k
  if (!constant) {
1864
146k
    initializeWhatIsTechnicallyUninitialized(Loc);
1865
146k
    LValue lv = MakeAddrLValue(Loc, type);
1866
146k
    lv.setNonGC(true);
1867
146k
    return EmitExprAsInit(Init, &D, lv, capturedByInit);
1868
146k
  }
1869
4.92k
1870
4.92k
  if (!emission.IsConstantAggregate) {
1871
    // For simple scalar/complex initialization, store the value directly.
1872
479
    LValue lv = MakeAddrLValue(Loc, type);
1873
479
    lv.setNonGC(true);
1874
479
    return EmitStoreThroughLValue(RValue::get(constant), lv, true);
1875
479
  }
1876
4.45k
1877
4.45k
  llvm::Type *BP = CGM.Int8Ty->getPointerTo(Loc.getAddressSpace());
1878
4.45k
  emitStoresForConstant(
1879
4.45k
      CGM, D, (Loc.getType() == BP) ? 
Loc0
: Builder.CreateBitCast(Loc, BP),
1880
4.45k
      type.isVolatileQualified(), Builder, constant);
1881
4.45k
}
1882
1883
/// Emit an expression as an initializer for an object (variable, field, etc.)
1884
/// at the given location.  The expression is not necessarily the normal
1885
/// initializer for the object, and the address is not necessarily
1886
/// its normal location.
1887
///
1888
/// \param init the initializing expression
1889
/// \param D the object to act as if we're initializing
1890
/// \param lvalue the lvalue to initialize
1891
/// \param capturedByInit true if \p D is a __block variable
1892
///   whose address is potentially changed by the initializer
1893
void CodeGenFunction::EmitExprAsInit(const Expr *init, const ValueDecl *D,
1894
161k
                                     LValue lvalue, bool capturedByInit) {
1895
161k
  QualType type = D->getType();
1896
161k
1897
161k
  if (type->isReferenceType()) {
1898
4.78k
    RValue rvalue = EmitReferenceBindingToExpr(init);
1899
4.78k
    if (capturedByInit)
1900
0
      drillIntoBlockVariable(*this, lvalue, cast<VarDecl>(D));
1901
4.78k
    EmitStoreThroughLValue(rvalue, lvalue, true);
1902
4.78k
    return;
1903
4.78k
  }
1904
156k
  switch (getEvaluationKind(type)) {
1905
143k
  case TEK_Scalar:
1906
143k
    EmitScalarInit(init, D, lvalue, capturedByInit);
1907
143k
    return;
1908
172
  case TEK_Complex: {
1909
172
    ComplexPairTy complex = EmitComplexExpr(init);
1910
172
    if (capturedByInit)
1911
0
      drillIntoBlockVariable(*this, lvalue, cast<VarDecl>(D));
1912
172
    EmitStoreOfComplex(complex, lvalue, /*init*/ true);
1913
172
    return;
1914
0
  }
1915
12.7k
  case TEK_Aggregate:
1916
12.7k
    if (type->isAtomicType()) {
1917
4
      EmitAtomicInit(const_cast<Expr*>(init), lvalue);
1918
12.7k
    } else {
1919
12.7k
      AggValueSlot::Overlap_t Overlap = AggValueSlot::MayOverlap;
1920
12.7k
      if (isa<VarDecl>(D))
1921
12.7k
        Overlap = AggValueSlot::DoesNotOverlap;
1922
0
      else if (auto *FD = dyn_cast<FieldDecl>(D))
1923
0
        Overlap = getOverlapForFieldInit(FD);
1924
      // TODO: how can we delay here if D is captured by its initializer?
1925
12.7k
      EmitAggExpr(init, AggValueSlot::forLValue(
1926
12.7k
                            lvalue, *this, AggValueSlot::IsDestructed,
1927
12.7k
                            AggValueSlot::DoesNotNeedGCBarriers,
1928
12.7k
                            AggValueSlot::IsNotAliased, Overlap));
1929
12.7k
    }
1930
12.7k
    return;
1931
0
  }
1932
0
  llvm_unreachable("bad evaluation kind");
1933
0
}
1934
1935
/// Enter a destroy cleanup for the given local variable.
1936
void CodeGenFunction::emitAutoVarTypeCleanup(
1937
                            const CodeGenFunction::AutoVarEmission &emission,
1938
9.36k
                            QualType::DestructionKind dtorKind) {
1939
9.36k
  assert(dtorKind != QualType::DK_none);
1940
9.36k
1941
  // Note that for __block variables, we want to destroy the
1942
  // original stack object, not the possibly forwarded object.
1943
9.36k
  Address addr = emission.getObjectAddress(*this);
1944
9.36k
1945
9.36k
  const VarDecl *var = emission.Variable;
1946
9.36k
  QualType type = var->getType();
1947
9.36k
1948
9.36k
  CleanupKind cleanupKind = NormalAndEHCleanup;
1949
9.36k
  CodeGenFunction::Destroyer *destroyer = nullptr;
1950
9.36k
1951
9.36k
  switch (dtorKind) {
1952
0
  case QualType::DK_none:
1953
0
    llvm_unreachable("no cleanup for trivially-destructible variable");
1954
0
1955
8.64k
  case QualType::DK_cxx_destructor:
1956
    // If there's an NRVO flag on the emission, we need a different
1957
    // cleanup.
1958
8.64k
    if (emission.NRVOFlag) {
1959
189
      assert(!type->isArrayType());
1960
189
      CXXDestructorDecl *dtor = type->getAsCXXRecordDecl()->getDestructor();
1961
189
      EHStack.pushCleanup<DestroyNRVOVariableCXX>(cleanupKind, addr, type, dtor,
1962
189
                                                  emission.NRVOFlag);
1963
189
      return;
1964
189
    }
1965
8.45k
    break;
1966
8.45k
1967
487
  case QualType::DK_objc_strong_lifetime:
1968
    // Suppress cleanups for pseudo-strong variables.
1969
487
    if (var->isARCPseudoStrong()) 
return13
;
1970
474
1971
    // Otherwise, consider whether to use an EH cleanup or not.
1972
474
    cleanupKind = getARCCleanupKind();
1973
474
1974
    // Use the imprecise destroyer by default.
1975
474
    if (!var->hasAttr<ObjCPreciseLifetimeAttr>())
1976
469
      destroyer = CodeGenFunction::destroyARCStrongImprecise;
1977
474
    break;
1978
474
1979
169
  case QualType::DK_objc_weak_lifetime:
1980
169
    break;
1981
474
1982
64
  case QualType::DK_nontrivial_c_struct:
1983
64
    destroyer = CodeGenFunction::destroyNonTrivialCStruct;
1984
64
    if (emission.NRVOFlag) {
1985
3
      assert(!type->isArrayType());
1986
3
      EHStack.pushCleanup<DestroyNRVOVariableC>(cleanupKind, addr,
1987
3
                                                emission.NRVOFlag, type);
1988
3
      return;
1989
3
    }
1990
61
    break;
1991
9.15k
  }
1992
9.15k
1993
  // If we haven't chosen a more specific destroyer, use the default.
1994
9.15k
  if (!destroyer) 
destroyer = getDestroyer(dtorKind)8.62k
;
1995
9.15k
1996
  // Use an EH cleanup in array destructors iff the destructor itself
1997
  // is being pushed as an EH cleanup.
1998
9.15k
  bool useEHCleanup = (cleanupKind & EHCleanup);
1999
9.15k
  EHStack.pushCleanup<DestroyObject>(cleanupKind, addr, type, destroyer,
2000
9.15k
                                     useEHCleanup);
2001
9.15k
}
2002
2003
210k
void CodeGenFunction::EmitAutoVarCleanups(const AutoVarEmission &emission) {
2004
210k
  assert(emission.Variable && "emission was not valid!");
2005
210k
2006
  // If this was emitted as a global constant, we're done.
2007
210k
  if (emission.wasEmittedAsGlobal()) 
return6
;
2008
210k
2009
  // If we don't have an insertion point, we're done.  Sema prevents
2010
  // us from jumping into any of these scopes anyway.
2011
210k
  if (!HaveInsertPoint()) 
return19
;
2012
210k
2013
210k
  const VarDecl &D = *emission.Variable;
2014
210k
2015
  // Check the type for a cleanup.
2016
210k
  if (QualType::DestructionKind dtorKind = D.needsDestruction(getContext()))
2017
9.36k
    emitAutoVarTypeCleanup(emission, dtorKind);
2018
210k
2019
  // In GC mode, honor objc_precise_lifetime.
2020
210k
  if (getLangOpts().getGC() != LangOptions::NonGC &&
2021
126
      D.hasAttr<ObjCPreciseLifetimeAttr>()) {
2022
1
    EHStack.pushCleanup<ExtendGCLifetime>(NormalCleanup, &D);
2023
1
  }
2024
210k
2025
  // Handle the cleanup attribute.
2026
210k
  if (const CleanupAttr *CA = D.getAttr<CleanupAttr>()) {
2027
8
    const FunctionDecl *FD = CA->getFunctionDecl();
2028
8
2029
8
    llvm::Constant *F = CGM.GetAddrOfFunction(FD);
2030
8
    assert(F && "Could not find function!");
2031
8
2032
8
    const CGFunctionInfo &Info = CGM.getTypes().arrangeFunctionDeclaration(FD);
2033
8
    EHStack.pushCleanup<CallCleanupFunction>(NormalAndEHCleanup, F, &Info, &D);
2034
8
  }
2035
210k
2036
  // If this is a block variable, call _Block_object_destroy
2037
  // (on the unforwarded address). Don't enter this cleanup if we're in pure-GC
2038
  // mode.
2039
210k
  if (emission.IsEscapingByRef &&
2040
205
      CGM.getLangOpts().getGC() != LangOptions::GCOnly) {
2041
205
    BlockFieldFlags Flags = BLOCK_FIELD_IS_BYREF;
2042
205
    if (emission.Variable->getType().isObjCGCWeak())
2043
7
      Flags |= BLOCK_FIELD_IS_WEAK;
2044
205
    enterByrefCleanup(NormalAndEHCleanup, emission.Addr, Flags,
2045
205
                      /*LoadBlockVarAddr*/ false,
2046
205
                      cxxDestructorCanThrow(emission.Variable->getType()));
2047
205
  }
2048
210k
}
2049
2050
CodeGenFunction::Destroyer *
2051
11.8k
CodeGenFunction::getDestroyer(QualType::DestructionKind kind) {
2052
11.8k
  switch (kind) {
2053
0
  case QualType::DK_none: llvm_unreachable("no destroyer for trivial dtor");
2054
11.2k
  case QualType::DK_cxx_destructor:
2055
11.2k
    return destroyCXXObject;
2056
49
  case QualType::DK_objc_strong_lifetime:
2057
49
    return destroyARCStrongPrecise;
2058
500
  case QualType::DK_objc_weak_lifetime:
2059
500
    return destroyARCWeak;
2060
60
  case QualType::DK_nontrivial_c_struct:
2061
60
    return destroyNonTrivialCStruct;
2062
0
  }
2063
0
  llvm_unreachable("Unknown DestructionKind");
2064
0
}
2065
2066
/// pushEHDestroy - Push the standard destructor for the given type as
2067
/// an EH-only cleanup.
2068
void CodeGenFunction::pushEHDestroy(QualType::DestructionKind dtorKind,
2069
845
                                    Address addr, QualType type) {
2070
845
  assert(dtorKind && "cannot push destructor for trivial type");
2071
845
  assert(needsEHCleanup(dtorKind));
2072
845
2073
845
  pushDestroy(EHCleanup, addr, type, getDestroyer(dtorKind), true);
2074
845
}
2075
2076
/// pushDestroy - Push the standard destructor for the given type as
2077
/// at least a normal cleanup.
2078
void CodeGenFunction::pushDestroy(QualType::DestructionKind dtorKind,
2079
339
                                  Address addr, QualType type) {
2080
339
  assert(dtorKind && "cannot push destructor for trivial type");
2081
339
2082
339
  CleanupKind cleanupKind = getCleanupKind(dtorKind);
2083
339
  pushDestroy(cleanupKind, addr, type, getDestroyer(dtorKind),
2084
339
              cleanupKind & EHCleanup);
2085
339
}
2086
2087
void CodeGenFunction::pushDestroy(CleanupKind cleanupKind, Address addr,
2088
                                  QualType type, Destroyer *destroyer,
2089
3.61k
                                  bool useEHCleanupForArray) {
2090
3.61k
  pushFullExprCleanup<DestroyObject>(cleanupKind, addr, type,
2091
3.61k
                                     destroyer, useEHCleanupForArray);
2092
3.61k
}
2093
2094
1.57k
void CodeGenFunction::pushStackRestore(CleanupKind Kind, Address SPMem) {
2095
1.57k
  EHStack.pushCleanup<CallStackRestore>(Kind, SPMem);
2096
1.57k
}
2097
2098
void CodeGenFunction::pushLifetimeExtendedDestroy(CleanupKind cleanupKind,
2099
                                                  Address addr, QualType type,
2100
                                                  Destroyer *destroyer,
2101
513
                                                  bool useEHCleanupForArray) {
2102
  // If we're not in a conditional branch, we don't need to bother generating a
2103
  // conditional cleanup.
2104
513
  if (!isInConditionalBranch()) {
2105
    // Push an EH-only cleanup for the object now.
2106
    // FIXME: When popping normal cleanups, we need to keep this EH cleanup
2107
    // around in case a temporary's destructor throws an exception.
2108
489
    if (cleanupKind & EHCleanup)
2109
201
      EHStack.pushCleanup<DestroyObject>(
2110
201
          static_cast<CleanupKind>(cleanupKind & ~NormalCleanup), addr, type,
2111
201
          destroyer, useEHCleanupForArray);
2112
489
2113
489
    return pushCleanupAfterFullExprWithActiveFlag<DestroyObject>(
2114
489
        cleanupKind, Address::invalid(), addr, type, destroyer, useEHCleanupForArray);
2115
489
  }
2116
24
2117
  // Otherwise, we should only destroy the object if it's been initialized.
2118
  // Re-use the active flag and saved address across both the EH and end of
2119
  // scope cleanups.
2120
24
2121
24
  using SavedType = typename DominatingValue<Address>::saved_type;
2122
24
  using ConditionalCleanupType =
2123
24
      EHScopeStack::ConditionalCleanup<DestroyObject, Address, QualType,
2124
24
                                       Destroyer *, bool>;
2125
24
2126
24
  Address ActiveFlag = createCleanupActiveFlag();
2127
24
  SavedType SavedAddr = saveValueInCond(addr);
2128
24
2129
24
  if (cleanupKind & EHCleanup) {
2130
11
    EHStack.pushCleanup<ConditionalCleanupType>(
2131
11
        static_cast<CleanupKind>(cleanupKind & ~NormalCleanup), SavedAddr, type,
2132
11
        destroyer, useEHCleanupForArray);
2133
11
    initFullExprCleanupWithFlag(ActiveFlag);
2134
11
  }
2135
24
2136
24
  pushCleanupAfterFullExprWithActiveFlag<ConditionalCleanupType>(
2137
24
      cleanupKind, ActiveFlag, SavedAddr, type, destroyer,
2138
24
      useEHCleanupForArray);
2139
24
}
2140
2141
/// emitDestroy - Immediately perform the destruction of the given
2142
/// object.
2143
///
2144
/// \param addr - the address of the object; a type*
2145
/// \param type - the type of the object; if an array type, all
2146
///   objects are destroyed in reverse order
2147
/// \param destroyer - the function to call to destroy individual
2148
///   elements
2149
/// \param useEHCleanupForArray - whether an EH cleanup should be
2150
///   used when destroying array elements, in case one of the
2151
///   destructions throws an exception
2152
void CodeGenFunction::emitDestroy(Address addr, QualType type,
2153
                                  Destroyer *destroyer,
2154
18.1k
                                  bool useEHCleanupForArray) {
2155
18.1k
  const ArrayType *arrayType = getContext().getAsArrayType(type);
2156
18.1k
  if (!arrayType)
2157
15.8k
    return destroyer(*this, addr, type);
2158
2.37k
2159
2.37k
  llvm::Value *length = emitArrayLength(arrayType, type, addr);
2160
2.37k
2161
2.37k
  CharUnits elementAlign =
2162
2.37k
    addr.getAlignment()
2163
2.37k
        .alignmentOfArrayElement(getContext().getTypeSizeInChars(type));
2164
2.37k
2165
  // Normally we have to check whether the array is zero-length.
2166
2.37k
  bool checkZeroLength = true;
2167
2.37k
2168
  // But if the array length is constant, we can suppress that.
2169
2.37k
  if (llvm::ConstantInt *constLength = dyn_cast<llvm::ConstantInt>(length)) {
2170
    // ...and if it's constant zero, we can just skip the entire thing.
2171
2.33k
    if (constLength->isZero()) 
return0
;
2172
2.33k
    checkZeroLength = false;
2173
2.33k
  }
2174
2.37k
2175
2.37k
  llvm::Value *begin = addr.getPointer();
2176
2.37k
  llvm::Value *end = Builder.CreateInBoundsGEP(begin, length);
2177
2.37k
  emitArrayDestroy(begin, end, type, elementAlign, destroyer,
2178
2.37k
                   checkZeroLength, useEHCleanupForArray);
2179
2.37k
}
2180
2181
/// emitArrayDestroy - Destroys all the elements of the given array,
2182
/// beginning from last to first.  The array cannot be zero-length.
2183
///
2184
/// \param begin - a type* denoting the first element of the array
2185
/// \param end - a type* denoting one past the end of the array
2186
/// \param elementType - the element type of the array
2187
/// \param destroyer - the function to call to destroy elements
2188
/// \param useEHCleanup - whether to push an EH cleanup to destroy
2189
///   the remaining elements in case the destruction of a single
2190
///   element throws
2191
void CodeGenFunction::emitArrayDestroy(llvm::Value *begin,
2192
                                       llvm::Value *end,
2193
                                       QualType elementType,
2194
                                       CharUnits elementAlign,
2195
                                       Destroyer *destroyer,
2196
                                       bool checkZeroLength,
2197
2.52k
                                       bool useEHCleanup) {
2198
2.52k
  assert(!elementType->isArrayType());
2199
2.52k
2200
  // The basic structure here is a do-while loop, because we don't
2201
  // need to check for the zero-element case.
2202
2.52k
  llvm::BasicBlock *bodyBB = createBasicBlock("arraydestroy.body");
2203
2.52k
  llvm::BasicBlock *doneBB = createBasicBlock("arraydestroy.done");
2204
2.52k
2205
2.52k
  if (checkZeroLength) {
2206
189
    llvm::Value *isEmpty = Builder.CreateICmpEQ(begin, end,
2207
189
                                                "arraydestroy.isempty");
2208
189
    Builder.CreateCondBr(isEmpty, doneBB, bodyBB);
2209
189
  }
2210
2.52k
2211
  // Enter the loop body, making that address the current address.
2212
2.52k
  llvm::BasicBlock *entryBB = Builder.GetInsertBlock();
2213
2.52k
  EmitBlock(bodyBB);
2214
2.52k
  llvm::PHINode *elementPast =
2215
2.52k
    Builder.CreatePHI(begin->getType(), 2, "arraydestroy.elementPast");
2216
2.52k
  elementPast->addIncoming(end, entryBB);
2217
2.52k
2218
  // Shift the address back by one element.
2219
2.52k
  llvm::Value *negativeOne = llvm::ConstantInt::get(SizeTy, -1, true);
2220
2.52k
  llvm::Value *element = Builder.CreateInBoundsGEP(elementPast, negativeOne,
2221
2.52k
                                                   "arraydestroy.element");
2222
2.52k
2223
2.52k
  if (useEHCleanup)
2224
1.84k
    pushRegularPartialArrayCleanup(begin, element, elementType, elementAlign,
2225
1.84k
                                   destroyer);
2226
2.52k
2227
  // Perform the actual destruction there.
2228
2.52k
  destroyer(*this, Address(element, elementAlign), elementType);
2229
2.52k
2230
2.52k
  if (useEHCleanup)
2231
1.84k
    PopCleanupBlock();
2232
2.52k
2233
  // Check whether we've reached the end.
2234
2.52k
  llvm::Value *done = Builder.CreateICmpEQ(element, begin, "arraydestroy.done");
2235
2.52k
  Builder.CreateCondBr(done, doneBB, bodyBB);
2236
2.52k
  elementPast->addIncoming(element, Builder.GetInsertBlock());
2237
2.52k
2238
  // Done.
2239
2.52k
  EmitBlock(doneBB);
2240
2.52k
}
2241
2242
/// Perform partial array destruction as if in an EH cleanup.  Unlike
2243
/// emitArrayDestroy, the element type here may still be an array type.
2244
static void emitPartialArrayDestroy(CodeGenFunction &CGF,
2245
                                    llvm::Value *begin, llvm::Value *end,
2246
                                    QualType type, CharUnits elementAlign,
2247
118
                                    CodeGenFunction::Destroyer *destroyer) {
2248
  // If the element type is itself an array, drill down.
2249
118
  unsigned arrayDepth = 0;
2250
137
  while (const ArrayType *arrayType = CGF.getContext().getAsArrayType(type)) {
2251
    // VLAs don't require a GEP index to walk into.
2252
19
    if (!isa<VariableArrayType>(arrayType))
2253
19
      arrayDepth++;
2254
19
    type = arrayType->getElementType();
2255
19
  }
2256
118
2257
118
  if (arrayDepth) {
2258
19
    llvm::Value *zero = llvm::ConstantInt::get(CGF.SizeTy, 0);
2259
19
2260
19
    SmallVector<llvm::Value*,4> gepIndices(arrayDepth+1, zero);
2261
19
    begin = CGF.Builder.CreateInBoundsGEP(begin, gepIndices, "pad.arraybegin");
2262
19
    end = CGF.Builder.CreateInBoundsGEP(end, gepIndices, "pad.arrayend");
2263
19
  }
2264
118
2265
  // Destroy the array.  We don't ever need an EH cleanup because we
2266
  // assume that we're in an EH cleanup ourselves, so a throwing
2267
  // destructor causes an immediate terminate.
2268
118
  CGF.emitArrayDestroy(begin, end, type, elementAlign, destroyer,
2269
118
                       /*checkZeroLength*/ true, /*useEHCleanup*/ false);
2270
118
}
2271
2272
namespace {
2273
  /// RegularPartialArrayDestroy - a cleanup which performs a partial
2274
  /// array destroy where the end pointer is regularly determined and
2275
  /// does not need to be loaded from a local.
2276
  class RegularPartialArrayDestroy final : public EHScopeStack::Cleanup {
2277
    llvm::Value *ArrayBegin;
2278
    llvm::Value *ArrayEnd;
2279
    QualType ElementType;
2280
    CodeGenFunction::Destroyer *Destroyer;
2281
    CharUnits ElementAlign;
2282
  public:
2283
    RegularPartialArrayDestroy(llvm::Value *arrayBegin, llvm::Value *arrayEnd,
2284
                               QualType elementType, CharUnits elementAlign,
2285
                               CodeGenFunction::Destroyer *destroyer)
2286
      : ArrayBegin(arrayBegin), ArrayEnd(arrayEnd),
2287
        ElementType(elementType), Destroyer(destroyer),
2288
1.88k
        ElementAlign(elementAlign) {}
2289
2290
46
    void Emit(CodeGenFunction &CGF, Flags flags) override {
2291
46
      emitPartialArrayDestroy(CGF, ArrayBegin, ArrayEnd,
2292
46
                              ElementType, ElementAlign, Destroyer);
2293
46
    }
2294
  };
2295
2296
  /// IrregularPartialArrayDestroy - a cleanup which performs a
2297
  /// partial array destroy where the end pointer is irregularly
2298
  /// determined and must be loaded from a local.
2299
  class IrregularPartialArrayDestroy final : public EHScopeStack::Cleanup {
2300
    llvm::Value *ArrayBegin;
2301
    Address ArrayEndPointer;
2302
    QualType ElementType;
2303
    CodeGenFunction::Destroyer *Destroyer;
2304
    CharUnits ElementAlign;
2305
  public:
2306
    IrregularPartialArrayDestroy(llvm::Value *arrayBegin,
2307
                                 Address arrayEndPointer,
2308
                                 QualType elementType,
2309
                                 CharUnits elementAlign,
2310
                                 CodeGenFunction::Destroyer *destroyer)
2311
      : ArrayBegin(arrayBegin), ArrayEndPointer(arrayEndPointer),
2312
        ElementType(elementType), Destroyer(destroyer),
2313
72
        ElementAlign(elementAlign) {}
2314
2315
72
    void Emit(CodeGenFunction &CGF, Flags flags) override {
2316
72
      llvm::Value *arrayEnd = CGF.Builder.CreateLoad(ArrayEndPointer);
2317
72
      emitPartialArrayDestroy(CGF, ArrayBegin, arrayEnd,
2318
72
                              ElementType, ElementAlign, Destroyer);
2319
72
    }
2320
  };
2321
} // end anonymous namespace
2322
2323
/// pushIrregularPartialArrayCleanup - Push an EH cleanup to destroy
2324
/// already-constructed elements of the given array.  The cleanup
2325
/// may be popped with DeactivateCleanupBlock or PopCleanupBlock.
2326
///
2327
/// \param elementType - the immediate element type of the array;
2328
///   possibly still an array type
2329
void CodeGenFunction::pushIrregularPartialArrayCleanup(llvm::Value *arrayBegin,
2330
                                                       Address arrayEndPointer,
2331
                                                       QualType elementType,
2332
                                                       CharUnits elementAlign,
2333
72
                                                       Destroyer *destroyer) {
2334
72
  pushFullExprCleanup<IrregularPartialArrayDestroy>(EHCleanup,
2335
72
                                                    arrayBegin, arrayEndPointer,
2336
72
                                                    elementType, elementAlign,
2337
72
                                                    destroyer);
2338
72
}
2339
2340
/// pushRegularPartialArrayCleanup - Push an EH cleanup to destroy
2341
/// already-constructed elements of the given array.  The cleanup
2342
/// may be popped with DeactivateCleanupBlock or PopCleanupBlock.
2343
///
2344
/// \param elementType - the immediate element type of the array;
2345
///   possibly still an array type
2346
void CodeGenFunction::pushRegularPartialArrayCleanup(llvm::Value *arrayBegin,
2347
                                                     llvm::Value *arrayEnd,
2348
                                                     QualType elementType,
2349
                                                     CharUnits elementAlign,
2350
1.88k
                                                     Destroyer *destroyer) {
2351
1.88k
  pushFullExprCleanup<RegularPartialArrayDestroy>(EHCleanup,
2352
1.88k
                                                  arrayBegin, arrayEnd,
2353
1.88k
                                                  elementType, elementAlign,
2354
1.88k
                                                  destroyer);
2355
1.88k
}
2356
2357
/// Lazily declare the @llvm.lifetime.start intrinsic.
2358
5.68k
llvm::Function *CodeGenModule::getLLVMLifetimeStartFn() {
2359
5.68k
  if (LifetimeStartFn)
2360
5.21k
    return LifetimeStartFn;
2361
463
  LifetimeStartFn = llvm::Intrinsic::getDeclaration(&getModule(),
2362
463
    llvm::Intrinsic::lifetime_start, AllocaInt8PtrTy);
2363
463
  return LifetimeStartFn;
2364
463
}
2365
2366
/// Lazily declare the @llvm.lifetime.end intrinsic.
2367
5.96k
llvm::Function *CodeGenModule::getLLVMLifetimeEndFn() {
2368
5.96k
  if (LifetimeEndFn)
2369
5.50k
    return LifetimeEndFn;
2370
463
  LifetimeEndFn = llvm::Intrinsic::getDeclaration(&getModule(),
2371
463
    llvm::Intrinsic::lifetime_end, AllocaInt8PtrTy);
2372
463
  return LifetimeEndFn;
2373
463
}
2374
2375
namespace {
2376
  /// A cleanup to perform a release of an object at the end of a
2377
  /// function.  This is used to balance out the incoming +1 of a
2378
  /// ns_consumed argument when we can't reasonably do that just by
2379
  /// not doing the initial retain for a __block argument.
2380
  struct ConsumeARCParameter final : EHScopeStack::Cleanup {
2381
    ConsumeARCParameter(llvm::Value *param,
2382
                        ARCPreciseLifetime_t precise)
2383
0
      : Param(param), Precise(precise) {}
2384
2385
    llvm::Value *Param;
2386
    ARCPreciseLifetime_t Precise;
2387
2388
0
    void Emit(CodeGenFunction &CGF, Flags flags) override {
2389
0
      CGF.EmitARCRelease(Param, Precise);
2390
0
    }
2391
  };
2392
} // end anonymous namespace
2393
2394
/// Emit an alloca (or GlobalValue depending on target)
2395
/// for the specified parameter and set up LocalDeclMap.
2396
void CodeGenFunction::EmitParmDecl(const VarDecl &D, ParamValue Arg,
2397
475k
                                   unsigned ArgNo) {
2398
  // FIXME: Why isn't ImplicitParamDecl a ParmVarDecl?
2399
475k
  assert((isa<ParmVarDecl>(D) || isa<ImplicitParamDecl>(D)) &&
2400
475k
         "Invalid argument to EmitParmDecl");
2401
475k
2402
475k
  Arg.getAnyValue()->setName(D.getName());
2403
475k
2404
475k
  QualType Ty = D.getType();
2405
475k
2406
  // Use better IR generation for certain implicit parameters.
2407
475k
  if (auto IPD = dyn_cast<ImplicitParamDecl>(&D)) {
2408
    // The only implicit argument a block has is its literal.
2409
    // This may be passed as an inalloca'ed value on Windows x86.
2410
180k
    if (BlockInfo) {
2411
1.04k
      llvm::Value *V = Arg.isIndirect()
2412
1
                           ? Builder.CreateLoad(Arg.getIndirectAddress())
2413
1.04k
                           : Arg.getDirectValue();
2414
1.04k
      setBlockContextParameter(IPD, ArgNo, V);
2415
1.04k
      return;
2416
1.04k
    }
2417
474k
  }
2418
474k
2419
474k
  Address DeclPtr = Address::invalid();
2420
474k
  bool DoStore = false;
2421
474k
  bool IsScalar = hasScalarEvaluationKind(Ty);
2422
  // If we already have a pointer to the argument, reuse the input pointer.
2423
474k
  if (Arg.isIndirect()) {
2424
14.6k
    DeclPtr = Arg.getIndirectAddress();
2425
    // If we have a prettier pointer type at this point, bitcast to that.
2426
14.6k
    unsigned AS = DeclPtr.getType()->getAddressSpace();
2427
14.6k
    llvm::Type *IRTy = ConvertTypeForMem(Ty)->getPointerTo(AS);
2428
14.6k
    if (DeclPtr.getType() != IRTy)
2429
2
      DeclPtr = Builder.CreateBitCast(DeclPtr, IRTy, D.getName());
2430
    // Indirect argument is in alloca address space, which may be different
2431
    // from the default address space.
2432
14.6k
    auto AllocaAS = CGM.getASTAllocaAddressSpace();
2433
14.6k
    auto *V = DeclPtr.getPointer();
2434
14.5k
    auto SrcLangAS = getLangOpts().OpenCL ? 
LangAS::opencl_private141
: AllocaAS;
2435
14.6k
    auto DestLangAS =
2436
14.5k
        getLangOpts().OpenCL ? 
LangAS::opencl_private141
: LangAS::Default;
2437
14.6k
    if (SrcLangAS != DestLangAS) {
2438
14
      assert(getContext().getTargetAddressSpace(SrcLangAS) ==
2439
14
             CGM.getDataLayout().getAllocaAddrSpace());
2440
14
      auto DestAS = getContext().getTargetAddressSpace(DestLangAS);
2441
14
      auto *T = V->getType()->getPointerElementType()->getPointerTo(DestAS);
2442
14
      DeclPtr = Address(getTargetHooks().performAddrSpaceCast(
2443
14
                            *this, V, SrcLangAS, DestLangAS, T, true),
2444
14
                        DeclPtr.getAlignment());
2445
14
    }
2446
14.6k
2447
    // Push a destructor cleanup for this parameter if the ABI requires it.
2448
    // Don't push a cleanup in a thunk for a method that will also emit a
2449
    // cleanup.
2450
14.6k
    if (hasAggregateEvaluationKind(Ty) && 
!CurFuncIsThunk13.8k
&&
2451
13.8k
        Ty->castAs<RecordType>()->getDecl()->isParamDestroyedInCallee()) {
2452
378
      if (QualType::DestructionKind DtorKind =
2453
119
              D.needsDestruction(getContext())) {
2454
119
        assert((DtorKind == QualType::DK_cxx_destructor ||
2455
119
                DtorKind == QualType::DK_nontrivial_c_struct) &&
2456
119
               "unexpected destructor type");
2457
119
        pushDestroy(DtorKind, DeclPtr, Ty);
2458
119
        CalleeDestructedParamCleanups[cast<ParmVarDecl>(&D)] =
2459
119
            EHStack.stable_begin();
2460
119
      }
2461
378
    }
2462
459k
  } else {
2463
    // Check if the parameter address is controlled by OpenMP runtime.
2464
459k
    Address OpenMPLocalAddr =
2465
459k
        getLangOpts().OpenMP
2466
110k
            ? CGM.getOpenMPRuntime().getAddressOfLocalVariable(*this, &D)
2467
349k
            : Address::invalid();
2468
459k
    if (getLangOpts().OpenMP && 
OpenMPLocalAddr.isValid()110k
) {
2469
11
      DeclPtr = OpenMPLocalAddr;
2470
459k
    } else {
2471
      // Otherwise, create a temporary to hold the value.
2472
459k
      DeclPtr = CreateMemTemp(Ty, getContext().getDeclAlign(&D),
2473
459k
                              D.getName() + ".addr");
2474
459k
    }
2475
459k
    DoStore = true;
2476
459k
  }
2477
474k
2478
459k
  llvm::Value *ArgVal = (DoStore ? Arg.getDirectValue() : 
nullptr14.6k
);
2479
474k
2480
474k
  LValue lv = MakeAddrLValue(DeclPtr, Ty);
2481
474k
  if (IsScalar) {
2482
459k
    Qualifiers qs = Ty.getQualifiers();
2483
459k
    if (Qualifiers::ObjCLifetime lt = qs.getObjCLifetime()) {
2484
      // We honor __attribute__((ns_consumed)) for types with lifetime.
2485
      // For __strong, it's handled by just skipping the initial retain;
2486
      // otherwise we have to balance out the initial +1 with an extra
2487
      // cleanup to do the release at the end of the function.
2488
525
      bool isConsumed = D.hasAttr<NSConsumedAttr>();
2489
525
2490
      // If a parameter is pseudo-strong then we can omit the implicit retain.
2491
525
      if (D.isARCPseudoStrong()) {
2492
257
        assert(lt == Qualifiers::OCL_Strong &&
2493
257
               "pseudo-strong variable isn't strong?");
2494
257
        assert(qs.hasConst() && "pseudo-strong variable should be const!");
2495
257
        lt = Qualifiers::OCL_ExplicitNone;
2496
257
      }
2497
525
2498
      // Load objects passed indirectly.
2499
525
      if (Arg.isIndirect() && 
!ArgVal8
)
2500
8
        ArgVal = Builder.CreateLoad(DeclPtr);
2501
525
2502
525
      if (lt == Qualifiers::OCL_Strong) {
2503
255
        if (!isConsumed) {
2504
225
          if (CGM.getCodeGenOpts().OptimizationLevel == 0) {
2505
            // use objc_storeStrong(&dest, value) for retaining the
2506
            // object. But first, store a null into 'dest' because
2507
            // objc_storeStrong attempts to release its old value.
2508
166
            llvm::Value *Null = CGM.EmitNullConstant(D.getType());
2509
166
            EmitStoreOfScalar(Null, lv, /* isInitialization */ true);
2510
166
            EmitARCStoreStrongCall(lv.getAddress(*this), ArgVal, true);
2511
166
            DoStore = false;
2512
166
          }
2513
59
          else
2514
          // Don't use objc_retainBlock for block pointers, because we
2515
          // don't want to Block_copy something just because we got it
2516
          // as a parameter.
2517
59
            ArgVal = EmitARCRetainNonBlock(ArgVal);
2518
225
        }
2519
270
      } else {
2520
        // Push the cleanup for a consumed parameter.
2521
270
        if (isConsumed) {
2522
0
          ARCPreciseLifetime_t precise = (D.hasAttr<ObjCPreciseLifetimeAttr>()
2523
0
                                ? ARCPreciseLifetime : ARCImpreciseLifetime);
2524
0
          EHStack.pushCleanup<ConsumeARCParameter>(getARCCleanupKind(), ArgVal,
2525
0
                                                   precise);
2526
0
        }
2527
270
2528
270
        if (lt == Qualifiers::OCL_Weak) {
2529
7
          EmitARCInitWeak(DeclPtr, ArgVal);
2530
7
          DoStore = false; // The weak init is a store, no need to do two.
2531
7
        }
2532
270
      }
2533
525
2534
      // Enter the cleanup scope.
2535
525
      EmitAutoVarWithLifetime(*this, D, DeclPtr, lt);
2536
525
    }
2537
459k
  }
2538
474k
2539
  // Store the initial value into the alloca.
2540
474k
  if (DoStore)
2541
459k
    EmitStoreOfScalar(ArgVal, lv, /* isInitialization */ true);
2542
474k
2543
474k
  setAddrOfLocalVar(&D, DeclPtr);
2544
474k
2545
  // Emit debug info for param declarations in non-thunk functions.
2546
474k
  if (CGDebugInfo *DI = getDebugInfo()) {
2547
152k
    if (CGM.getCodeGenOpts().hasReducedDebugInfo() && 
!CurFuncIsThunk151k
) {
2548
151k
      DI->EmitDeclareOfArgVariable(&D, DeclPtr.getPointer(), ArgNo, Builder);
2549
151k
    }
2550
152k
  }
2551
474k
2552
474k
  if (D.hasAttr<AnnotateAttr>())
2553
3
    EmitVarAnnotations(&D, DeclPtr.getPointer());
2554
474k
2555
  // We can only check return value nullability if all arguments to the
2556
  // function satisfy their nullability preconditions. This makes it necessary
2557
  // to emit null checks for args in the function body itself.
2558
474k
  if (requiresReturnValueNullabilityCheck()) {
2559
28
    auto Nullability = Ty->getNullability(getContext());
2560
28
    if (Nullability && 
*Nullability == NullabilityKind::NonNull10
) {
2561
8
      SanitizerScope SanScope(this);
2562
8
      RetValNullabilityPrecondition =
2563
8
          Builder.CreateAnd(RetValNullabilityPrecondition,
2564
8
                            Builder.CreateIsNotNull(Arg.getAnyValue()));
2565
8
    }
2566
28
  }
2567
474k
}
2568
2569
void CodeGenModule::EmitOMPDeclareReduction(const OMPDeclareReductionDecl *D,
2570
188
                                            CodeGenFunction *CGF) {
2571
188
  if (!LangOpts.OpenMP || (!LangOpts.EmitAllDecls && 
!D->isUsed()84
))
2572
84
    return;
2573
104
  getOpenMPRuntime().emitUserDefinedReduction(CGF, D);
2574
104
}
2575
2576
void CodeGenModule::EmitOMPDeclareMapper(const OMPDeclareMapperDecl *D,
2577
48
                                         CodeGenFunction *CGF) {
2578
48
  if (!LangOpts.OpenMP || LangOpts.OpenMPSimd ||
2579
24
      (!LangOpts.EmitAllDecls && 
!D->isUsed()0
))
2580
24
    return;
2581
24
  getOpenMPRuntime().emitUserDefinedMapper(D, CGF);
2582
24
}
2583
2584
34
void CodeGenModule::EmitOMPRequiresDecl(const OMPRequiresDecl *D) {
2585
34
  getOpenMPRuntime().processRequiresDirective(D);
2586
34
}