Coverage Report

Created: 2020-02-25 14:32

/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/CodeGen/CGDeclCXX.cpp
Line
Count
Source (jump to first uncovered line)
1
//===--- CGDeclCXX.cpp - Emit LLVM Code for C++ declarations --------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This contains code dealing with code generation of C++ declarations
10
//
11
//===----------------------------------------------------------------------===//
12
13
#include "CGCXXABI.h"
14
#include "CGObjCRuntime.h"
15
#include "CGOpenMPRuntime.h"
16
#include "CodeGenFunction.h"
17
#include "TargetInfo.h"
18
#include "clang/AST/Attr.h"
19
#include "clang/Basic/CodeGenOptions.h"
20
#include "llvm/ADT/StringExtras.h"
21
#include "llvm/IR/Intrinsics.h"
22
#include "llvm/IR/MDBuilder.h"
23
#include "llvm/Support/Path.h"
24
25
using namespace clang;
26
using namespace CodeGen;
27
28
static void EmitDeclInit(CodeGenFunction &CGF, const VarDecl &D,
29
10.3k
                         ConstantAddress DeclPtr) {
30
10.3k
  assert(
31
10.3k
      (D.hasGlobalStorage() ||
32
10.3k
       (D.hasLocalStorage() && CGF.getContext().getLangOpts().OpenCLCPlusPlus)) &&
33
10.3k
      "VarDecl must have global or local (in the case of OpenCL) storage!");
34
10.3k
  assert(!D.getType()->isReferenceType() &&
35
10.3k
         "Should not call EmitDeclInit on a reference!");
36
10.3k
37
10.3k
  QualType type = D.getType();
38
10.3k
  LValue lv = CGF.MakeAddrLValue(DeclPtr, type);
39
10.3k
40
10.3k
  const Expr *Init = D.getInit();
41
10.3k
  switch (CGF.getEvaluationKind(type)) {
42
6.99k
  case TEK_Scalar: {
43
6.99k
    CodeGenModule &CGM = CGF.CGM;
44
6.99k
    if (lv.isObjCStrong())
45
3
      CGM.getObjCRuntime().EmitObjCGlobalAssign(CGF, CGF.EmitScalarExpr(Init),
46
3
                                                DeclPtr, D.getTLSKind());
47
6.98k
    else if (lv.isObjCWeak())
48
2
      CGM.getObjCRuntime().EmitObjCWeakAssign(CGF, CGF.EmitScalarExpr(Init),
49
2
                                              DeclPtr);
50
6.98k
    else
51
6.98k
      CGF.EmitScalarInit(Init, &D, lv, false);
52
6.99k
    return;
53
0
  }
54
1
  case TEK_Complex:
55
1
    CGF.EmitComplexExprIntoLValue(Init, lv, /*isInit*/ true);
56
1
    return;
57
3.39k
  case TEK_Aggregate:
58
3.39k
    CGF.EmitAggExpr(Init,
59
3.39k
                    AggValueSlot::forLValue(lv, CGF, AggValueSlot::IsDestructed,
60
3.39k
                                            AggValueSlot::DoesNotNeedGCBarriers,
61
3.39k
                                            AggValueSlot::IsNotAliased,
62
3.39k
                                            AggValueSlot::DoesNotOverlap));
63
3.39k
    return;
64
0
  }
65
0
  llvm_unreachable("bad evaluation kind");
66
0
}
67
68
/// Emit code to cause the destruction of the given variable with
69
/// static storage duration.
70
static void EmitDeclDestroy(CodeGenFunction &CGF, const VarDecl &D,
71
10.7k
                            ConstantAddress Addr) {
72
10.7k
  // Honor __attribute__((no_destroy)) and bail instead of attempting
73
10.7k
  // to emit a reference to a possibly nonexistent destructor, which
74
10.7k
  // in turn can cause a crash. This will result in a global constructor
75
10.7k
  // that isn't balanced out by a destructor call as intended by the
76
10.7k
  // attribute. This also checks for -fno-c++-static-destructors and
77
10.7k
  // bails even if the attribute is not present.
78
10.7k
  QualType::DestructionKind DtorKind = D.needsDestruction(CGF.getContext());
79
10.7k
80
10.7k
  // FIXME:  __attribute__((cleanup)) ?
81
10.7k
82
10.7k
  switch (DtorKind) {
83
7.47k
  case QualType::DK_none:
84
7.47k
    return;
85
0
86
3.22k
  case QualType::DK_cxx_destructor:
87
3.22k
    break;
88
0
89
6
  case QualType::DK_objc_strong_lifetime:
90
6
  case QualType::DK_objc_weak_lifetime:
91
6
  case QualType::DK_nontrivial_c_struct:
92
6
    // We don't care about releasing objects during process teardown.
93
6
    assert(!D.getTLSKind() && "should have rejected this");
94
6
    return;
95
3.22k
  }
96
3.22k
97
3.22k
  llvm::FunctionCallee Func;
98
3.22k
  llvm::Constant *Argument;
99
3.22k
100
3.22k
  CodeGenModule &CGM = CGF.CGM;
101
3.22k
  QualType Type = D.getType();
102
3.22k
103
3.22k
  // Special-case non-array C++ destructors, if they have the right signature.
104
3.22k
  // Under some ABIs, destructors return this instead of void, and cannot be
105
3.22k
  // passed directly to __cxa_atexit if the target does not allow this
106
3.22k
  // mismatch.
107
3.22k
  const CXXRecordDecl *Record = Type->getAsCXXRecordDecl();
108
3.22k
  bool CanRegisterDestructor =
109
3.22k
      Record && 
(2.90k
!CGM.getCXXABI().HasThisReturn(
110
2.90k
                     GlobalDecl(Record->getDestructor(), Dtor_Complete)) ||
111
2.90k
                 
CGM.getCXXABI().canCallMismatchedFunctionType()8
);
112
3.22k
  // If __cxa_atexit is disabled via a flag, a different helper function is
113
3.22k
  // generated elsewhere which uses atexit instead, and it takes the destructor
114
3.22k
  // directly.
115
3.22k
  bool UsingExternalHelper = !CGM.getCodeGenOpts().CXAAtExit;
116
3.22k
  if (Record && 
(2.90k
CanRegisterDestructor2.90k
||
UsingExternalHelper1
)) {
117
2.90k
    assert(!Record->hasTrivialDestructor());
118
2.90k
    CXXDestructorDecl *Dtor = Record->getDestructor();
119
2.90k
120
2.90k
    Func = CGM.getAddrAndTypeOfCXXStructor(GlobalDecl(Dtor, Dtor_Complete));
121
2.90k
    if (CGF.getContext().getLangOpts().OpenCL) {
122
7
      auto DestAS =
123
7
          CGM.getTargetCodeGenInfo().getAddrSpaceOfCxaAtexitPtrParam();
124
7
      auto DestTy = CGF.getTypes().ConvertType(Type)->getPointerTo(
125
7
          CGM.getContext().getTargetAddressSpace(DestAS));
126
7
      auto SrcAS = D.getType().getQualifiers().getAddressSpace();
127
7
      if (DestAS == SrcAS)
128
0
        Argument = llvm::ConstantExpr::getBitCast(Addr.getPointer(), DestTy);
129
7
      else
130
7
        // FIXME: On addr space mismatch we are passing NULL. The generation
131
7
        // of the global destructor function should be adjusted accordingly.
132
7
        Argument = llvm::ConstantPointerNull::get(DestTy);
133
2.89k
    } else {
134
2.89k
      Argument = llvm::ConstantExpr::getBitCast(
135
2.89k
          Addr.getPointer(), CGF.getTypes().ConvertType(Type)->getPointerTo());
136
2.89k
    }
137
2.90k
  // Otherwise, the standard logic requires a helper function.
138
2.90k
  } else {
139
318
    Func = CodeGenFunction(CGM)
140
318
           .generateDestroyHelper(Addr, Type, CGF.getDestroyer(DtorKind),
141
318
                                  CGF.needsEHCleanup(DtorKind), &D);
142
318
    Argument = llvm::Constant::getNullValue(CGF.Int8PtrTy);
143
318
  }
144
3.22k
145
3.22k
  CGM.getCXXABI().registerGlobalDtor(CGF, D, Func, Argument);
146
3.22k
}
147
148
/// Emit code to cause the variable at the given address to be considered as
149
/// constant from this point onwards.
150
static void EmitDeclInvariant(CodeGenFunction &CGF, const VarDecl &D,
151
92
                              llvm::Constant *Addr) {
152
92
  return CGF.EmitInvariantStart(
153
92
      Addr, CGF.getContext().getTypeSizeInChars(D.getType()));
154
92
}
155
156
132
void CodeGenFunction::EmitInvariantStart(llvm::Constant *Addr, CharUnits Size) {
157
132
  // Do not emit the intrinsic if we're not optimizing.
158
132
  if (!CGM.getCodeGenOpts().OptimizationLevel)
159
122
    return;
160
10
161
10
  // Grab the llvm.invariant.start intrinsic.
162
10
  llvm::Intrinsic::ID InvStartID = llvm::Intrinsic::invariant_start;
163
10
  // Overloaded address space type.
164
10
  llvm::Type *ObjectPtr[1] = {Int8PtrTy};
165
10
  llvm::Function *InvariantStart = CGM.getIntrinsic(InvStartID, ObjectPtr);
166
10
167
10
  // Emit a call with the size in bytes of the object.
168
10
  uint64_t Width = Size.getQuantity();
169
10
  llvm::Value *Args[2] = { llvm::ConstantInt::getSigned(Int64Ty, Width),
170
10
                           llvm::ConstantExpr::getBitCast(Addr, Int8PtrTy)};
171
10
  Builder.CreateCall(InvariantStart, Args);
172
10
}
173
174
void CodeGenFunction::EmitCXXGlobalVarDeclInit(const VarDecl &D,
175
                                               llvm::Constant *DeclPtr,
176
10.8k
                                               bool PerformInit) {
177
10.8k
178
10.8k
  const Expr *Init = D.getInit();
179
10.8k
  QualType T = D.getType();
180
10.8k
181
10.8k
  // The address space of a static local variable (DeclPtr) may be different
182
10.8k
  // from the address space of the "this" argument of the constructor. In that
183
10.8k
  // case, we need an addrspacecast before calling the constructor.
184
10.8k
  //
185
10.8k
  // struct StructWithCtor {
186
10.8k
  //   __device__ StructWithCtor() {...}
187
10.8k
  // };
188
10.8k
  // __device__ void foo() {
189
10.8k
  //   __shared__ StructWithCtor s;
190
10.8k
  //   ...
191
10.8k
  // }
192
10.8k
  //
193
10.8k
  // For example, in the above CUDA code, the static local variable s has a
194
10.8k
  // "shared" address space qualifier, but the constructor of StructWithCtor
195
10.8k
  // expects "this" in the "generic" address space.
196
10.8k
  unsigned ExpectedAddrSpace = getContext().getTargetAddressSpace(T);
197
10.8k
  unsigned ActualAddrSpace = DeclPtr->getType()->getPointerAddressSpace();
198
10.8k
  if (ActualAddrSpace != ExpectedAddrSpace) {
199
12
    llvm::Type *LTy = CGM.getTypes().ConvertTypeForMem(T);
200
12
    llvm::PointerType *PTy = llvm::PointerType::get(LTy, ExpectedAddrSpace);
201
12
    DeclPtr = llvm::ConstantExpr::getAddrSpaceCast(DeclPtr, PTy);
202
12
  }
203
10.8k
204
10.8k
  ConstantAddress DeclAddr(DeclPtr, getContext().getDeclAlign(&D));
205
10.8k
206
10.8k
  if (!T->isReferenceType()) {
207
10.7k
    if (getLangOpts().OpenMP && 
!getLangOpts().OpenMPSimd2.50k
&&
208
10.7k
        
D.hasAttr<OMPThreadPrivateDeclAttr>()1.08k
) {
209
40
      (void)CGM.getOpenMPRuntime().emitThreadPrivateVarDefinition(
210
40
          &D, DeclAddr, D.getAttr<OMPThreadPrivateDeclAttr>()->getLocation(),
211
40
          PerformInit, this);
212
40
    }
213
10.7k
    if (PerformInit)
214
10.3k
      EmitDeclInit(*this, D, DeclAddr);
215
10.7k
    if (CGM.isTypeConstant(D.getType(), true))
216
92
      EmitDeclInvariant(*this, D, DeclPtr);
217
10.7k
    else
218
10.7k
      EmitDeclDestroy(*this, D, DeclAddr);
219
10.7k
    return;
220
10.7k
  }
221
88
222
88
  assert(PerformInit && "cannot have constant initializer which needs "
223
88
         "destruction for reference");
224
88
  RValue RV = EmitReferenceBindingToExpr(Init);
225
88
  EmitStoreOfScalar(RV.getScalarVal(), DeclAddr, false, T);
226
88
}
227
228
/// Create a stub function, suitable for being passed to atexit,
229
/// which passes the given address to the given destructor function.
230
llvm::Function *CodeGenFunction::createAtExitStub(const VarDecl &VD,
231
                                                  llvm::FunctionCallee dtor,
232
177
                                                  llvm::Constant *addr) {
233
177
  // Get the destructor function type, void(*)(void).
234
177
  llvm::FunctionType *ty = llvm::FunctionType::get(CGM.VoidTy, false);
235
177
  SmallString<256> FnName;
236
177
  {
237
177
    llvm::raw_svector_ostream Out(FnName);
238
177
    CGM.getCXXABI().getMangleContext().mangleDynamicAtExitDestructor(&VD, Out);
239
177
  }
240
177
241
177
  const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction();
242
177
  llvm::Function *fn = CGM.CreateGlobalInitOrDestructFunction(
243
177
      ty, FnName.str(), FI, VD.getLocation());
244
177
245
177
  CodeGenFunction CGF(CGM);
246
177
247
177
  CGF.StartFunction(GlobalDecl(&VD, DynamicInitKind::AtExit),
248
177
                    CGM.getContext().VoidTy, fn, FI, FunctionArgList());
249
177
250
177
  llvm::CallInst *call = CGF.Builder.CreateCall(dtor, addr);
251
177
252
177
 // Make sure the call and the callee agree on calling convention.
253
177
  if (auto *dtorFn = dyn_cast<llvm::Function>(
254
177
          dtor.getCallee()->stripPointerCastsAndAliases()))
255
177
    call->setCallingConv(dtorFn->getCallingConv());
256
177
257
177
  CGF.FinishFunction();
258
177
259
177
  return fn;
260
177
}
261
262
/// Register a global destructor using the C atexit runtime function.
263
void CodeGenFunction::registerGlobalDtorWithAtExit(const VarDecl &VD,
264
                                                   llvm::FunctionCallee dtor,
265
169
                                                   llvm::Constant *addr) {
266
169
  // Create a function which calls the destructor.
267
169
  llvm::Constant *dtorStub = createAtExitStub(VD, dtor, addr);
268
169
  registerGlobalDtorWithAtExit(dtorStub);
269
169
}
270
271
174
void CodeGenFunction::registerGlobalDtorWithAtExit(llvm::Constant *dtorStub) {
272
174
  // extern "C" int atexit(void (*f)(void));
273
174
  llvm::FunctionType *atexitTy =
274
174
    llvm::FunctionType::get(IntTy, dtorStub->getType(), false);
275
174
276
174
  llvm::FunctionCallee atexit =
277
174
      CGM.CreateRuntimeFunction(atexitTy, "atexit", llvm::AttributeList(),
278
174
                                /*Local=*/true);
279
174
  if (llvm::Function *atexitFn = dyn_cast<llvm::Function>(atexit.getCallee()))
280
174
    atexitFn->setDoesNotThrow();
281
174
282
174
  EmitNounwindRuntimeCall(atexit, dtorStub);
283
174
}
284
285
void CodeGenFunction::EmitCXXGuardedInit(const VarDecl &D,
286
                                         llvm::GlobalVariable *DeclPtr,
287
6.90k
                                         bool PerformInit) {
288
6.90k
  // If we've been asked to forbid guard variables, emit an error now.
289
6.90k
  // This diagnostic is hard-coded for Darwin's use case;  we can find
290
6.90k
  // better phrasing if someone else needs it.
291
6.90k
  if (CGM.getCodeGenOpts().ForbidGuardVariables)
292
1
    CGM.Error(D.getLocation(),
293
1
              "this initialization requires a guard variable, which "
294
1
              "the kernel does not support");
295
6.90k
296
6.90k
  CGM.getCXXABI().EmitGuardedInit(*this, D, DeclPtr, PerformInit);
297
6.90k
}
298
299
void CodeGenFunction::EmitCXXGuardedInitBranch(llvm::Value *NeedsInit,
300
                                               llvm::BasicBlock *InitBlock,
301
                                               llvm::BasicBlock *NoInitBlock,
302
                                               GuardKind Kind,
303
6.89k
                                               const VarDecl *D) {
304
6.89k
  assert((Kind == GuardKind::TlsGuard || D) && "no guarded variable");
305
6.89k
306
6.89k
  // A guess at how many times we will enter the initialization of a
307
6.89k
  // variable, depending on the kind of variable.
308
6.89k
  static const uint64_t InitsPerTLSVar = 1024;
309
6.89k
  static const uint64_t InitsPerLocalVar = 1024 * 1024;
310
6.89k
311
6.89k
  llvm::MDNode *Weights;
312
6.89k
  if (Kind == GuardKind::VariableGuard && 
!D->isLocalVarDecl()6.85k
) {
313
163
    // For non-local variables, don't apply any weighting for now. Due to our
314
163
    // use of COMDATs, we expect there to be at most one initialization of the
315
163
    // variable per DSO, but we have no way to know how many DSOs will try to
316
163
    // initialize the variable.
317
163
    Weights = nullptr;
318
6.73k
  } else {
319
6.73k
    uint64_t NumInits;
320
6.73k
    // FIXME: For the TLS case, collect and use profiling information to
321
6.73k
    // determine a more accurate brach weight.
322
6.73k
    if (Kind == GuardKind::TlsGuard || 
D->getTLSKind()6.69k
)
323
105
      NumInits = InitsPerTLSVar;
324
6.62k
    else
325
6.62k
      NumInits = InitsPerLocalVar;
326
6.73k
327
6.73k
    // The probability of us entering the initializer is
328
6.73k
    //   1 / (total number of times we attempt to initialize the variable).
329
6.73k
    llvm::MDBuilder MDHelper(CGM.getLLVMContext());
330
6.73k
    Weights = MDHelper.createBranchWeights(1, NumInits - 1);
331
6.73k
  }
332
6.89k
333
6.89k
  Builder.CreateCondBr(NeedsInit, InitBlock, NoInitBlock, Weights);
334
6.89k
}
335
336
llvm::Function *CodeGenModule::CreateGlobalInitOrDestructFunction(
337
    llvm::FunctionType *FTy, const Twine &Name, const CGFunctionInfo &FI,
338
7.62k
    SourceLocation Loc, bool TLS) {
339
7.62k
  llvm::Function *Fn =
340
7.62k
    llvm::Function::Create(FTy, llvm::GlobalValue::InternalLinkage,
341
7.62k
                           Name, &getModule());
342
7.62k
  if (!getLangOpts().AppleKext && 
!TLS7.60k
) {
343
7.56k
    // Set the section if needed.
344
7.56k
    if (const char *Section = getTarget().getStaticInitSectionSpecifier())
345
2.17k
      Fn->setSection(Section);
346
7.56k
  }
347
7.62k
348
7.62k
  SetInternalFunctionAttributes(GlobalDecl(), Fn, FI);
349
7.62k
350
7.62k
  Fn->setCallingConv(getRuntimeCC());
351
7.62k
352
7.62k
  if (!getLangOpts().Exceptions)
353
6.75k
    Fn->setDoesNotThrow();
354
7.62k
355
7.62k
  if (getLangOpts().Sanitize.has(SanitizerKind::Address) &&
356
7.62k
      
!isInSanitizerBlacklist(SanitizerKind::Address, Fn, Loc)91
)
357
86
    Fn->addFnAttr(llvm::Attribute::SanitizeAddress);
358
7.62k
359
7.62k
  if (getLangOpts().Sanitize.has(SanitizerKind::KernelAddress) &&
360
7.62k
      
!isInSanitizerBlacklist(SanitizerKind::KernelAddress, Fn, Loc)0
)
361
0
    Fn->addFnAttr(llvm::Attribute::SanitizeAddress);
362
7.62k
363
7.62k
  if (getLangOpts().Sanitize.has(SanitizerKind::HWAddress) &&
364
7.62k
      
!isInSanitizerBlacklist(SanitizerKind::HWAddress, Fn, Loc)0
)
365
0
    Fn->addFnAttr(llvm::Attribute::SanitizeHWAddress);
366
7.62k
367
7.62k
  if (getLangOpts().Sanitize.has(SanitizerKind::KernelHWAddress) &&
368
7.62k
      
!isInSanitizerBlacklist(SanitizerKind::KernelHWAddress, Fn, Loc)0
)
369
0
    Fn->addFnAttr(llvm::Attribute::SanitizeHWAddress);
370
7.62k
371
7.62k
  if (getLangOpts().Sanitize.has(SanitizerKind::MemTag) &&
372
7.62k
      
!isInSanitizerBlacklist(SanitizerKind::MemTag, Fn, Loc)0
)
373
0
    Fn->addFnAttr(llvm::Attribute::SanitizeMemTag);
374
7.62k
375
7.62k
  if (getLangOpts().Sanitize.has(SanitizerKind::Thread) &&
376
7.62k
      
!isInSanitizerBlacklist(SanitizerKind::Thread, Fn, Loc)6
)
377
3
    Fn->addFnAttr(llvm::Attribute::SanitizeThread);
378
7.62k
379
7.62k
  if (getLangOpts().Sanitize.has(SanitizerKind::Memory) &&
380
7.62k
      
!isInSanitizerBlacklist(SanitizerKind::Memory, Fn, Loc)41
)
381
41
    Fn->addFnAttr(llvm::Attribute::SanitizeMemory);
382
7.62k
383
7.62k
  if (getLangOpts().Sanitize.has(SanitizerKind::KernelMemory) &&
384
7.62k
      
!isInSanitizerBlacklist(SanitizerKind::KernelMemory, Fn, Loc)0
)
385
0
    Fn->addFnAttr(llvm::Attribute::SanitizeMemory);
386
7.62k
387
7.62k
  if (getLangOpts().Sanitize.has(SanitizerKind::SafeStack) &&
388
7.62k
      
!isInSanitizerBlacklist(SanitizerKind::SafeStack, Fn, Loc)0
)
389
0
    Fn->addFnAttr(llvm::Attribute::SafeStack);
390
7.62k
391
7.62k
  if (getLangOpts().Sanitize.has(SanitizerKind::ShadowCallStack) &&
392
7.62k
      
!isInSanitizerBlacklist(SanitizerKind::ShadowCallStack, Fn, Loc)0
)
393
0
    Fn->addFnAttr(llvm::Attribute::ShadowCallStack);
394
7.62k
395
7.62k
  auto RASignKind = getCodeGenOpts().getSignReturnAddress();
396
7.62k
  if (RASignKind != CodeGenOptions::SignReturnAddressScope::None) {
397
16
    Fn->addFnAttr("sign-return-address",
398
16
                  RASignKind == CodeGenOptions::SignReturnAddressScope::All
399
16
                      ? 
"all"8
400
16
                      : 
"non-leaf"8
);
401
16
    auto RASignKey = getCodeGenOpts().getSignReturnAddressKey();
402
16
    Fn->addFnAttr("sign-return-address-key",
403
16
                  RASignKey == CodeGenOptions::SignReturnAddressKeyValue::AKey
404
16
                      ? 
"a_key"10
405
16
                      : 
"b_key"6
);
406
16
  }
407
7.62k
408
7.62k
  if (getCodeGenOpts().BranchTargetEnforcement)
409
6
    Fn->addFnAttr("branch-target-enforcement");
410
7.62k
411
7.62k
  return Fn;
412
7.62k
}
413
414
/// Create a global pointer to a function that will initialize a global
415
/// variable.  The user has requested that this pointer be emitted in a specific
416
/// section.
417
void CodeGenModule::EmitPointerToInitFunc(const VarDecl *D,
418
                                          llvm::GlobalVariable *GV,
419
                                          llvm::Function *InitFunc,
420
6
                                          InitSegAttr *ISA) {
421
6
  llvm::GlobalVariable *PtrArray = new llvm::GlobalVariable(
422
6
      TheModule, InitFunc->getType(), /*isConstant=*/true,
423
6
      llvm::GlobalValue::PrivateLinkage, InitFunc, "__cxx_init_fn_ptr");
424
6
  PtrArray->setSection(ISA->getSection());
425
6
  addUsedGlobal(PtrArray);
426
6
427
6
  // If the GV is already in a comdat group, then we have to join it.
428
6
  if (llvm::Comdat *C = GV->getComdat())
429
3
    PtrArray->setComdat(C);
430
6
}
431
432
void
433
CodeGenModule::EmitCXXGlobalVarDeclInitFunc(const VarDecl *D,
434
                                            llvm::GlobalVariable *Addr,
435
4.28k
                                            bool PerformInit) {
436
4.28k
437
4.28k
  // According to E.2.3.1 in CUDA-7.5 Programming guide: __device__,
438
4.28k
  // __constant__ and __shared__ variables defined in namespace scope,
439
4.28k
  // that are of class type, cannot have a non-empty constructor. All
440
4.28k
  // the checks have been done in Sema by now. Whatever initializers
441
4.28k
  // are allowed are empty and we just need to ignore them here.
442
4.28k
  if (getLangOpts().CUDAIsDevice && 
!getLangOpts().GPUAllowDeviceInit25
&&
443
4.28k
      
(24
D->hasAttr<CUDADeviceAttr>()24
||
D->hasAttr<CUDAConstantAttr>()14
||
444
24
       
D->hasAttr<CUDASharedAttr>()4
))
445
24
    return;
446
4.25k
447
4.25k
  if (getLangOpts().OpenMP &&
448
4.25k
      
getOpenMPRuntime().emitDeclareTargetVarDefinition(D, Addr, PerformInit)2.49k
)
449
49
    return;
450
4.20k
451
4.20k
  // Check if we've already initialized this decl.
452
4.20k
  auto I = DelayedCXXInitPosition.find(D);
453
4.20k
  if (I != DelayedCXXInitPosition.end() && 
I->second == ~0U954
)
454
20
    return;
455
4.18k
456
4.18k
  llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy, false);
457
4.18k
  SmallString<256> FnName;
458
4.18k
  {
459
4.18k
    llvm::raw_svector_ostream Out(FnName);
460
4.18k
    getCXXABI().getMangleContext().mangleDynamicInitializer(D, Out);
461
4.18k
  }
462
4.18k
463
4.18k
  // Create a variable initialization function.
464
4.18k
  llvm::Function *Fn =
465
4.18k
      CreateGlobalInitOrDestructFunction(FTy, FnName.str(),
466
4.18k
                                         getTypes().arrangeNullaryFunction(),
467
4.18k
                                         D->getLocation());
468
4.18k
469
4.18k
  auto *ISA = D->getAttr<InitSegAttr>();
470
4.18k
  CodeGenFunction(*this).GenerateCXXGlobalVarDeclInitFunc(Fn, D, Addr,
471
4.18k
                                                          PerformInit);
472
4.18k
473
4.18k
  llvm::GlobalVariable *COMDATKey =
474
4.18k
      supportsCOMDAT() && 
D->isExternallyVisible()3.16k
?
Addr2.73k
:
nullptr1.45k
;
475
4.18k
476
4.18k
  if (D->getTLSKind()) {
477
153
    // FIXME: Should we support init_priority for thread_local?
478
153
    // FIXME: We only need to register one __cxa_thread_atexit function for the
479
153
    // entire TU.
480
153
    CXXThreadLocalInits.push_back(Fn);
481
153
    CXXThreadLocalInitVars.push_back(D);
482
4.03k
  } else if (PerformInit && 
ISA3.71k
) {
483
6
    EmitPointerToInitFunc(D, Addr, Fn, ISA);
484
4.03k
  } else if (auto *IPA = D->getAttr<InitPriorityAttr>()) {
485
489
    OrderGlobalInits Key(IPA->getPriority(), PrioritizedCXXGlobalInits.size());
486
489
    PrioritizedCXXGlobalInits.push_back(std::make_pair(Key, Fn));
487
3.54k
  } else if (isTemplateInstantiation(D->getTemplateSpecializationKind()) ||
488
3.54k
             
getContext().GetGVALinkageForVariable(D) == GVA_DiscardableODR3.43k
) {
489
122
    // C++ [basic.start.init]p2:
490
122
    //   Definitions of explicitly specialized class template static data
491
122
    //   members have ordered initialization. Other class template static data
492
122
    //   members (i.e., implicitly or explicitly instantiated specializations)
493
122
    //   have unordered initialization.
494
122
    //
495
122
    // As a consequence, we can put them into their own llvm.global_ctors entry.
496
122
    //
497
122
    // If the global is externally visible, put the initializer into a COMDAT
498
122
    // group with the global being initialized.  On most platforms, this is a
499
122
    // minor startup time optimization.  In the MS C++ ABI, there are no guard
500
122
    // variables, so this COMDAT key is required for correctness.
501
122
    AddGlobalCtor(Fn, 65535, COMDATKey);
502
122
    if (getTarget().getCXXABI().isMicrosoft() && 
COMDATKey35
) {
503
31
      // In The MS C++, MS add template static data member in the linker
504
31
      // drective.
505
31
      addUsedGlobal(COMDATKey);
506
31
    }
507
3.41k
  } else if (D->hasAttr<SelectAnyAttr>()) {
508
6
    // SelectAny globals will be comdat-folded. Put the initializer into a
509
6
    // COMDAT group associated with the global, so the initializers get folded
510
6
    // too.
511
6
    AddGlobalCtor(Fn, 65535, COMDATKey);
512
3.41k
  } else {
513
3.41k
    I = DelayedCXXInitPosition.find(D); // Re-do lookup in case of re-hash.
514
3.41k
    if (I == DelayedCXXInitPosition.end()) {
515
2.67k
      CXXGlobalInits.push_back(Fn);
516
2.67k
    } else 
if (736
I->second != ~0U736
) {
517
734
      assert(I->second < CXXGlobalInits.size() &&
518
734
             CXXGlobalInits[I->second] == nullptr);
519
734
      CXXGlobalInits[I->second] = Fn;
520
734
    }
521
3.41k
  }
522
4.18k
523
4.18k
  // Remember that we already emitted the initializer for this global.
524
4.18k
  DelayedCXXInitPosition[D] = ~0U;
525
4.18k
}
526
527
24.9k
void CodeGenModule::EmitCXXThreadLocalInitFunc() {
528
24.9k
  getCXXABI().EmitThreadLocalInitFuncs(
529
24.9k
      *this, CXXThreadLocals, CXXThreadLocalInits, CXXThreadLocalInitVars);
530
24.9k
531
24.9k
  CXXThreadLocalInits.clear();
532
24.9k
  CXXThreadLocalInitVars.clear();
533
24.9k
  CXXThreadLocals.clear();
534
24.9k
}
535
536
void
537
24.9k
CodeGenModule::EmitCXXGlobalInitFunc() {
538
38.1k
  while (!CXXGlobalInits.empty() && 
!CXXGlobalInits.back()14.2k
)
539
13.1k
    CXXGlobalInits.pop_back();
540
24.9k
541
24.9k
  if (CXXGlobalInits.empty() && 
PrioritizedCXXGlobalInits.empty()23.8k
)
542
23.8k
    return;
543
1.11k
544
1.11k
  llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy, false);
545
1.11k
  const CGFunctionInfo &FI = getTypes().arrangeNullaryFunction();
546
1.11k
547
1.11k
  // Create our global initialization function.
548
1.11k
  if (!PrioritizedCXXGlobalInits.empty()) {
549
163
    SmallVector<llvm::Function *, 8> LocalCXXGlobalInits;
550
163
    llvm::array_pod_sort(PrioritizedCXXGlobalInits.begin(),
551
163
                         PrioritizedCXXGlobalInits.end());
552
163
    // Iterate over "chunks" of ctors with same priority and emit each chunk
553
163
    // into separate function. Note - everything is sorted first by priority,
554
163
    // second - by lex order, so we emit ctor functions in proper order.
555
163
    for (SmallVectorImpl<GlobalInitData >::iterator
556
163
           I = PrioritizedCXXGlobalInits.begin(),
557
489
           E = PrioritizedCXXGlobalInits.end(); I != E; ) {
558
326
      SmallVectorImpl<GlobalInitData >::iterator
559
326
        PrioE = std::upper_bound(I + 1, E, *I, GlobalInitPriorityCmp());
560
326
561
326
      LocalCXXGlobalInits.clear();
562
326
      unsigned Priority = I->first.priority;
563
326
      // Compute the function suffix from priority. Prepend with zeroes to make
564
326
      // sure the function names are also ordered as priorities.
565
326
      std::string PrioritySuffix = llvm::utostr(Priority);
566
326
      // Priority is always <= 65535 (enforced by sema).
567
326
      PrioritySuffix = std::string(6-PrioritySuffix.size(), '0')+PrioritySuffix;
568
326
      llvm::Function *Fn = CreateGlobalInitOrDestructFunction(
569
326
          FTy, "_GLOBAL__I_" + PrioritySuffix, FI);
570
326
571
815
      for (; I < PrioE; 
++I489
)
572
489
        LocalCXXGlobalInits.push_back(I->second);
573
326
574
326
      CodeGenFunction(*this).GenerateCXXGlobalInitFunc(Fn, LocalCXXGlobalInits);
575
326
      AddGlobalCtor(Fn, Priority);
576
326
    }
577
163
    PrioritizedCXXGlobalInits.clear();
578
163
  }
579
1.11k
580
1.11k
  // Include the filename in the symbol name. Including "sub_" matches gcc and
581
1.11k
  // makes sure these symbols appear lexicographically behind the symbols with
582
1.11k
  // priority emitted above.
583
1.11k
  SmallString<128> FileName = llvm::sys::path::filename(getModule().getName());
584
1.11k
  if (FileName.empty())
585
0
    FileName = "<null>";
586
1.11k
587
36.7k
  for (size_t i = 0; i < FileName.size(); 
++i35.6k
) {
588
35.6k
    // Replace everything that's not [a-zA-Z0-9._] with a _. This set happens
589
35.6k
    // to be the set of C preprocessing numbers.
590
35.6k
    if (!isPreprocessingNumberBody(FileName[i]))
591
850
      FileName[i] = '_';
592
35.6k
  }
593
1.11k
594
1.11k
  llvm::Function *Fn = CreateGlobalInitOrDestructFunction(
595
1.11k
      FTy, llvm::Twine("_GLOBAL__sub_I_", FileName), FI);
596
1.11k
597
1.11k
  CodeGenFunction(*this).GenerateCXXGlobalInitFunc(Fn, CXXGlobalInits);
598
1.11k
  AddGlobalCtor(Fn);
599
1.11k
600
1.11k
  // In OpenCL global init functions must be converted to kernels in order to
601
1.11k
  // be able to launch them from the host.
602
1.11k
  // FIXME: Some more work might be needed to handle destructors correctly.
603
1.11k
  // Current initialization function makes use of function pointers callbacks.
604
1.11k
  // We can't support function pointers especially between host and device.
605
1.11k
  // However it seems global destruction has little meaning without any
606
1.11k
  // dynamic resource allocation on the device and program scope variables are
607
1.11k
  // destroyed by the runtime when program is released.
608
1.11k
  if (getLangOpts().OpenCL) {
609
6
    GenOpenCLArgMetadata(Fn);
610
6
    Fn->setCallingConv(llvm::CallingConv::SPIR_KERNEL);
611
6
  }
612
1.11k
613
1.11k
  if (getLangOpts().HIP) {
614
2
    Fn->setCallingConv(llvm::CallingConv::AMDGPU_KERNEL);
615
2
    Fn->addFnAttr("device-init");
616
2
  }
617
1.11k
618
1.11k
  CXXGlobalInits.clear();
619
1.11k
}
620
621
24.9k
void CodeGenModule::EmitCXXGlobalDtorFunc() {
622
24.9k
  if (CXXGlobalDtors.empty())
623
24.9k
    return;
624
3
625
3
  llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy, false);
626
3
627
3
  // Create our global destructor function.
628
3
  const CGFunctionInfo &FI = getTypes().arrangeNullaryFunction();
629
3
  llvm::Function *Fn =
630
3
      CreateGlobalInitOrDestructFunction(FTy, "_GLOBAL__D_a", FI);
631
3
632
3
  CodeGenFunction(*this).GenerateCXXGlobalDtorsFunc(Fn, CXXGlobalDtors);
633
3
  AddGlobalDtor(Fn);
634
3
}
635
636
/// Emit the code necessary to initialize the given global variable.
637
void CodeGenFunction::GenerateCXXGlobalVarDeclInitFunc(llvm::Function *Fn,
638
                                                       const VarDecl *D,
639
                                                 llvm::GlobalVariable *Addr,
640
4.18k
                                                       bool PerformInit) {
641
4.18k
  // Check if we need to emit debug info for variable initializer.
642
4.18k
  if (D->hasAttr<NoDebugAttr>())
643
1
    DebugInfo = nullptr; // disable debug info indefinitely for this function
644
4.18k
645
4.18k
  CurEHLocation = D->getBeginLoc();
646
4.18k
647
4.18k
  StartFunction(GlobalDecl(D, DynamicInitKind::Initializer),
648
4.18k
                getContext().VoidTy, Fn, getTypes().arrangeNullaryFunction(),
649
4.18k
                FunctionArgList(), D->getLocation(),
650
4.18k
                D->getInit()->getExprLoc());
651
4.18k
652
4.18k
  // Use guarded initialization if the global variable is weak. This
653
4.18k
  // occurs for, e.g., instantiated static data members and
654
4.18k
  // definitions explicitly marked weak.
655
4.18k
  //
656
4.18k
  // Also use guarded initialization for a variable with dynamic TLS and
657
4.18k
  // unordered initialization. (If the initialization is ordered, the ABI
658
4.18k
  // layer will guard the whole-TU initialization for us.)
659
4.18k
  if (Addr->hasWeakLinkage() || 
Addr->hasLinkOnceLinkage()4.12k
||
660
4.18k
      
(3.98k
D->getTLSKind() == VarDecl::TLS_Dynamic3.98k
&&
661
3.98k
       
isTemplateInstantiation(D->getTemplateSpecializationKind())76
)) {
662
208
    EmitCXXGuardedInit(*D, Addr, PerformInit);
663
3.98k
  } else {
664
3.98k
    EmitCXXGlobalVarDeclInit(*D, Addr, PerformInit);
665
3.98k
  }
666
4.18k
667
4.18k
  FinishFunction();
668
4.18k
}
669
670
void
671
CodeGenFunction::GenerateCXXGlobalInitFunc(llvm::Function *Fn,
672
                                           ArrayRef<llvm::Function *> Decls,
673
1.48k
                                           ConstantAddress Guard) {
674
1.48k
  {
675
1.48k
    auto NL = ApplyDebugLocation::CreateEmpty(*this);
676
1.48k
    StartFunction(GlobalDecl(), getContext().VoidTy, Fn,
677
1.48k
                  getTypes().arrangeNullaryFunction(), FunctionArgList());
678
1.48k
    // Emit an artificial location for this function.
679
1.48k
    auto AL = ApplyDebugLocation::CreateArtificial(*this);
680
1.48k
681
1.48k
    llvm::BasicBlock *ExitBlock = nullptr;
682
1.48k
    if (Guard.isValid()) {
683
40
      // If we have a guard variable, check whether we've already performed
684
40
      // these initializations. This happens for TLS initialization functions.
685
40
      llvm::Value *GuardVal = Builder.CreateLoad(Guard);
686
40
      llvm::Value *Uninit = Builder.CreateIsNull(GuardVal,
687
40
                                                 "guard.uninitialized");
688
40
      llvm::BasicBlock *InitBlock = createBasicBlock("init");
689
40
      ExitBlock = createBasicBlock("exit");
690
40
      EmitCXXGuardedInitBranch(Uninit, InitBlock, ExitBlock,
691
40
                               GuardKind::TlsGuard, nullptr);
692
40
      EmitBlock(InitBlock);
693
40
      // Mark as initialized before initializing anything else. If the
694
40
      // initializers use previously-initialized thread_local vars, that's
695
40
      // probably supposed to be OK, but the standard doesn't say.
696
40
      Builder.CreateStore(llvm::ConstantInt::get(GuardVal->getType(),1), Guard);
697
40
698
40
      // The guard variable can't ever change again.
699
40
      EmitInvariantStart(
700
40
          Guard.getPointer(),
701
40
          CharUnits::fromQuantity(
702
40
              CGM.getDataLayout().getTypeAllocSize(GuardVal->getType())));
703
40
    }
704
1.48k
705
1.48k
    RunCleanupsScope Scope(*this);
706
1.48k
707
1.48k
    // When building in Objective-C++ ARC mode, create an autorelease pool
708
1.48k
    // around the global initializers.
709
1.48k
    if (getLangOpts().ObjCAutoRefCount && 
getLangOpts().CPlusPlus4
) {
710
4
      llvm::Value *token = EmitObjCAutoreleasePoolPush();
711
4
      EmitObjCAutoreleasePoolCleanup(token);
712
4
    }
713
1.48k
714
8.75k
    for (unsigned i = 0, e = Decls.size(); i != e; 
++i7.27k
)
715
7.27k
      if (Decls[i])
716
3.97k
        EmitRuntimeCall(Decls[i]);
717
1.48k
718
1.48k
    Scope.ForceCleanup();
719
1.48k
720
1.48k
    if (ExitBlock) {
721
40
      Builder.CreateBr(ExitBlock);
722
40
      EmitBlock(ExitBlock);
723
40
    }
724
1.48k
  }
725
1.48k
726
1.48k
  FinishFunction();
727
1.48k
}
728
729
void CodeGenFunction::GenerateCXXGlobalDtorsFunc(
730
    llvm::Function *Fn,
731
    const std::vector<std::tuple<llvm::FunctionType *, llvm::WeakTrackingVH,
732
3
                                 llvm::Constant *>> &DtorsAndObjects) {
733
3
  {
734
3
    auto NL = ApplyDebugLocation::CreateEmpty(*this);
735
3
    StartFunction(GlobalDecl(), getContext().VoidTy, Fn,
736
3
                  getTypes().arrangeNullaryFunction(), FunctionArgList());
737
3
    // Emit an artificial location for this function.
738
3
    auto AL = ApplyDebugLocation::CreateArtificial(*this);
739
3
740
3
    // Emit the dtors, in reverse order from construction.
741
9
    for (unsigned i = 0, e = DtorsAndObjects.size(); i != e; 
++i6
) {
742
6
      llvm::FunctionType *CalleeTy;
743
6
      llvm::Value *Callee;
744
6
      llvm::Constant *Arg;
745
6
      std::tie(CalleeTy, Callee, Arg) = DtorsAndObjects[e - i - 1];
746
6
      llvm::CallInst *CI = Builder.CreateCall(CalleeTy, Callee, Arg);
747
6
      // Make sure the call and the callee agree on calling convention.
748
6
      if (llvm::Function *F = dyn_cast<llvm::Function>(Callee))
749
6
        CI->setCallingConv(F->getCallingConv());
750
6
    }
751
3
  }
752
3
753
3
  FinishFunction();
754
3
}
755
756
/// generateDestroyHelper - Generates a helper function which, when
757
/// invoked, destroys the given object.  The address of the object
758
/// should be in global memory.
759
llvm::Function *CodeGenFunction::generateDestroyHelper(
760
    Address addr, QualType type, Destroyer *destroyer,
761
322
    bool useEHCleanupForArray, const VarDecl *VD) {
762
322
  FunctionArgList args;
763
322
  ImplicitParamDecl Dst(getContext(), getContext().VoidPtrTy,
764
322
                        ImplicitParamDecl::Other);
765
322
  args.push_back(&Dst);
766
322
767
322
  const CGFunctionInfo &FI =
768
322
    CGM.getTypes().arrangeBuiltinFunctionDeclaration(getContext().VoidTy, args);
769
322
  llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
770
322
  llvm::Function *fn = CGM.CreateGlobalInitOrDestructFunction(
771
322
      FTy, "__cxx_global_array_dtor", FI, VD->getLocation());
772
322
773
322
  CurEHLocation = VD->getBeginLoc();
774
322
775
322
  StartFunction(VD, getContext().VoidTy, fn, FI, args);
776
322
777
322
  emitDestroy(addr, type, destroyer, useEHCleanupForArray);
778
322
779
322
  FinishFunction();
780
322
781
322
  return fn;
782
322
}