Coverage Report

Created: 2021-06-15 06:44

/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/CodeGen/CGDeclCXX.cpp
Line
Count
Source (jump to first uncovered line)
1
//===--- CGDeclCXX.cpp - Emit LLVM Code for C++ declarations --------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This contains code dealing with code generation of C++ declarations
10
//
11
//===----------------------------------------------------------------------===//
12
13
#include "CGCXXABI.h"
14
#include "CGObjCRuntime.h"
15
#include "CGOpenMPRuntime.h"
16
#include "CodeGenFunction.h"
17
#include "TargetInfo.h"
18
#include "clang/AST/Attr.h"
19
#include "clang/Basic/LangOptions.h"
20
#include "llvm/ADT/StringExtras.h"
21
#include "llvm/IR/Intrinsics.h"
22
#include "llvm/IR/MDBuilder.h"
23
#include "llvm/Support/Path.h"
24
25
using namespace clang;
26
using namespace CodeGen;
27
28
static void EmitDeclInit(CodeGenFunction &CGF, const VarDecl &D,
29
15.1k
                         ConstantAddress DeclPtr) {
30
15.1k
  assert(
31
15.1k
      (D.hasGlobalStorage() ||
32
15.1k
       (D.hasLocalStorage() && CGF.getContext().getLangOpts().OpenCLCPlusPlus)) &&
33
15.1k
      "VarDecl must have global or local (in the case of OpenCL) storage!");
34
0
  assert(!D.getType()->isReferenceType() &&
35
15.1k
         "Should not call EmitDeclInit on a reference!");
36
37
0
  QualType type = D.getType();
38
15.1k
  LValue lv = CGF.MakeAddrLValue(DeclPtr, type);
39
40
15.1k
  const Expr *Init = D.getInit();
41
15.1k
  switch (CGF.getEvaluationKind(type)) {
42
10.2k
  case TEK_Scalar: {
43
10.2k
    CodeGenModule &CGM = CGF.CGM;
44
10.2k
    if (lv.isObjCStrong())
45
3
      CGM.getObjCRuntime().EmitObjCGlobalAssign(CGF, CGF.EmitScalarExpr(Init),
46
3
                                                DeclPtr, D.getTLSKind());
47
10.2k
    else if (lv.isObjCWeak())
48
2
      CGM.getObjCRuntime().EmitObjCWeakAssign(CGF, CGF.EmitScalarExpr(Init),
49
2
                                              DeclPtr);
50
10.2k
    else
51
10.2k
      CGF.EmitScalarInit(Init, &D, lv, false);
52
10.2k
    return;
53
0
  }
54
3
  case TEK_Complex:
55
3
    CGF.EmitComplexExprIntoLValue(Init, lv, /*isInit*/ true);
56
3
    return;
57
4.88k
  case TEK_Aggregate:
58
4.88k
    CGF.EmitAggExpr(Init,
59
4.88k
                    AggValueSlot::forLValue(lv, CGF, AggValueSlot::IsDestructed,
60
4.88k
                                            AggValueSlot::DoesNotNeedGCBarriers,
61
4.88k
                                            AggValueSlot::IsNotAliased,
62
4.88k
                                            AggValueSlot::DoesNotOverlap));
63
4.88k
    return;
64
15.1k
  }
65
15.1k
  
llvm_unreachable0
("bad evaluation kind");
66
15.1k
}
67
68
/// Emit code to cause the destruction of the given variable with
69
/// static storage duration.
70
static void EmitDeclDestroy(CodeGenFunction &CGF, const VarDecl &D,
71
15.5k
                            ConstantAddress Addr) {
72
  // Honor __attribute__((no_destroy)) and bail instead of attempting
73
  // to emit a reference to a possibly nonexistent destructor, which
74
  // in turn can cause a crash. This will result in a global constructor
75
  // that isn't balanced out by a destructor call as intended by the
76
  // attribute. This also checks for -fno-c++-static-destructors and
77
  // bails even if the attribute is not present.
78
15.5k
  QualType::DestructionKind DtorKind = D.needsDestruction(CGF.getContext());
79
80
  // FIXME:  __attribute__((cleanup)) ?
81
82
15.5k
  switch (DtorKind) {
83
10.8k
  case QualType::DK_none:
84
10.8k
    return;
85
86
4.74k
  case QualType::DK_cxx_destructor:
87
4.74k
    break;
88
89
6
  case QualType::DK_objc_strong_lifetime:
90
6
  case QualType::DK_objc_weak_lifetime:
91
6
  case QualType::DK_nontrivial_c_struct:
92
    // We don't care about releasing objects during process teardown.
93
6
    assert(!D.getTLSKind() && "should have rejected this");
94
0
    return;
95
15.5k
  }
96
97
4.74k
  llvm::FunctionCallee Func;
98
4.74k
  llvm::Constant *Argument;
99
100
4.74k
  CodeGenModule &CGM = CGF.CGM;
101
4.74k
  QualType Type = D.getType();
102
103
  // Special-case non-array C++ destructors, if they have the right signature.
104
  // Under some ABIs, destructors return this instead of void, and cannot be
105
  // passed directly to __cxa_atexit if the target does not allow this
106
  // mismatch.
107
4.74k
  const CXXRecordDecl *Record = Type->getAsCXXRecordDecl();
108
4.74k
  bool CanRegisterDestructor =
109
4.74k
      Record && 
(4.40k
!CGM.getCXXABI().HasThisReturn(
110
4.40k
                     GlobalDecl(Record->getDestructor(), Dtor_Complete)) ||
111
4.40k
                 
CGM.getCXXABI().canCallMismatchedFunctionType()8
);
112
  // If __cxa_atexit is disabled via a flag, a different helper function is
113
  // generated elsewhere which uses atexit instead, and it takes the destructor
114
  // directly.
115
4.74k
  bool UsingExternalHelper = !CGM.getCodeGenOpts().CXAAtExit;
116
4.74k
  if (Record && 
(4.40k
CanRegisterDestructor4.40k
||
UsingExternalHelper1
)) {
117
4.40k
    assert(!Record->hasTrivialDestructor());
118
0
    CXXDestructorDecl *Dtor = Record->getDestructor();
119
120
4.40k
    Func = CGM.getAddrAndTypeOfCXXStructor(GlobalDecl(Dtor, Dtor_Complete));
121
4.40k
    if (CGF.getContext().getLangOpts().OpenCL) {
122
7
      auto DestAS =
123
7
          CGM.getTargetCodeGenInfo().getAddrSpaceOfCxaAtexitPtrParam();
124
7
      auto DestTy = CGF.getTypes().ConvertType(Type)->getPointerTo(
125
7
          CGM.getContext().getTargetAddressSpace(DestAS));
126
7
      auto SrcAS = D.getType().getQualifiers().getAddressSpace();
127
7
      if (DestAS == SrcAS)
128
0
        Argument = llvm::ConstantExpr::getBitCast(Addr.getPointer(), DestTy);
129
7
      else
130
        // FIXME: On addr space mismatch we are passing NULL. The generation
131
        // of the global destructor function should be adjusted accordingly.
132
7
        Argument = llvm::ConstantPointerNull::get(DestTy);
133
4.39k
    } else {
134
4.39k
      Argument = llvm::ConstantExpr::getBitCast(
135
4.39k
          Addr.getPointer(), CGF.getTypes().ConvertType(Type)->getPointerTo());
136
4.39k
    }
137
  // Otherwise, the standard logic requires a helper function.
138
4.40k
  } else {
139
338
    Func = CodeGenFunction(CGM)
140
338
           .generateDestroyHelper(Addr, Type, CGF.getDestroyer(DtorKind),
141
338
                                  CGF.needsEHCleanup(DtorKind), &D);
142
338
    Argument = llvm::Constant::getNullValue(CGF.Int8PtrTy);
143
338
  }
144
145
0
  CGM.getCXXABI().registerGlobalDtor(CGF, D, Func, Argument);
146
4.74k
}
147
148
/// Emit code to cause the variable at the given address to be considered as
149
/// constant from this point onwards.
150
static void EmitDeclInvariant(CodeGenFunction &CGF, const VarDecl &D,
151
92
                              llvm::Constant *Addr) {
152
92
  return CGF.EmitInvariantStart(
153
92
      Addr, CGF.getContext().getTypeSizeInChars(D.getType()));
154
92
}
155
156
141
void CodeGenFunction::EmitInvariantStart(llvm::Constant *Addr, CharUnits Size) {
157
  // Do not emit the intrinsic if we're not optimizing.
158
141
  if (!CGM.getCodeGenOpts().OptimizationLevel)
159
131
    return;
160
161
  // Grab the llvm.invariant.start intrinsic.
162
10
  llvm::Intrinsic::ID InvStartID = llvm::Intrinsic::invariant_start;
163
  // Overloaded address space type.
164
10
  llvm::Type *ObjectPtr[1] = {Int8PtrTy};
165
10
  llvm::Function *InvariantStart = CGM.getIntrinsic(InvStartID, ObjectPtr);
166
167
  // Emit a call with the size in bytes of the object.
168
10
  uint64_t Width = Size.getQuantity();
169
10
  llvm::Value *Args[2] = { llvm::ConstantInt::getSigned(Int64Ty, Width),
170
10
                           llvm::ConstantExpr::getBitCast(Addr, Int8PtrTy)};
171
10
  Builder.CreateCall(InvariantStart, Args);
172
10
}
173
174
void CodeGenFunction::EmitCXXGlobalVarDeclInit(const VarDecl &D,
175
                                               llvm::Constant *DeclPtr,
176
15.7k
                                               bool PerformInit) {
177
178
15.7k
  const Expr *Init = D.getInit();
179
15.7k
  QualType T = D.getType();
180
181
  // The address space of a static local variable (DeclPtr) may be different
182
  // from the address space of the "this" argument of the constructor. In that
183
  // case, we need an addrspacecast before calling the constructor.
184
  //
185
  // struct StructWithCtor {
186
  //   __device__ StructWithCtor() {...}
187
  // };
188
  // __device__ void foo() {
189
  //   __shared__ StructWithCtor s;
190
  //   ...
191
  // }
192
  //
193
  // For example, in the above CUDA code, the static local variable s has a
194
  // "shared" address space qualifier, but the constructor of StructWithCtor
195
  // expects "this" in the "generic" address space.
196
15.7k
  unsigned ExpectedAddrSpace = getContext().getTargetAddressSpace(T);
197
15.7k
  unsigned ActualAddrSpace = DeclPtr->getType()->getPointerAddressSpace();
198
15.7k
  if (ActualAddrSpace != ExpectedAddrSpace) {
199
12
    llvm::Type *LTy = CGM.getTypes().ConvertTypeForMem(T);
200
12
    llvm::PointerType *PTy = llvm::PointerType::get(LTy, ExpectedAddrSpace);
201
12
    DeclPtr = llvm::ConstantExpr::getAddrSpaceCast(DeclPtr, PTy);
202
12
  }
203
204
15.7k
  ConstantAddress DeclAddr(DeclPtr, getContext().getDeclAlign(&D));
205
206
15.7k
  if (!T->isReferenceType()) {
207
15.6k
    if (getLangOpts().OpenMP && 
!getLangOpts().OpenMPSimd3.87k
&&
208
15.6k
        
D.hasAttr<OMPThreadPrivateDeclAttr>()1.66k
) {
209
63
      (void)CGM.getOpenMPRuntime().emitThreadPrivateVarDefinition(
210
63
          &D, DeclAddr, D.getAttr<OMPThreadPrivateDeclAttr>()->getLocation(),
211
63
          PerformInit, this);
212
63
    }
213
15.6k
    if (PerformInit)
214
15.1k
      EmitDeclInit(*this, D, DeclAddr);
215
15.6k
    if (CGM.isTypeConstant(D.getType(), true))
216
92
      EmitDeclInvariant(*this, D, DeclPtr);
217
15.5k
    else
218
15.5k
      EmitDeclDestroy(*this, D, DeclAddr);
219
15.6k
    return;
220
15.6k
  }
221
222
97
  assert(PerformInit && "cannot have constant initializer which needs "
223
97
         "destruction for reference");
224
0
  RValue RV = EmitReferenceBindingToExpr(Init);
225
97
  EmitStoreOfScalar(RV.getScalarVal(), DeclAddr, false, T);
226
97
}
227
228
/// Create a stub function, suitable for being passed to atexit,
229
/// which passes the given address to the given destructor function.
230
llvm::Function *CodeGenFunction::createAtExitStub(const VarDecl &VD,
231
                                                  llvm::FunctionCallee dtor,
232
224
                                                  llvm::Constant *addr) {
233
  // Get the destructor function type, void(*)(void).
234
224
  llvm::FunctionType *ty = llvm::FunctionType::get(CGM.VoidTy, false);
235
224
  SmallString<256> FnName;
236
224
  {
237
224
    llvm::raw_svector_ostream Out(FnName);
238
224
    CGM.getCXXABI().getMangleContext().mangleDynamicAtExitDestructor(&VD, Out);
239
224
  }
240
241
224
  const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction();
242
224
  llvm::Function *fn = CGM.CreateGlobalInitOrCleanUpFunction(
243
224
      ty, FnName.str(), FI, VD.getLocation());
244
245
224
  CodeGenFunction CGF(CGM);
246
247
224
  CGF.StartFunction(GlobalDecl(&VD, DynamicInitKind::AtExit),
248
224
                    CGM.getContext().VoidTy, fn, FI, FunctionArgList(),
249
224
                    VD.getLocation(), VD.getInit()->getExprLoc());
250
  // Emit an artificial location for this function.
251
224
  auto AL = ApplyDebugLocation::CreateArtificial(CGF);
252
253
224
  llvm::CallInst *call = CGF.Builder.CreateCall(dtor, addr);
254
255
  // Make sure the call and the callee agree on calling convention.
256
224
  if (auto *dtorFn = dyn_cast<llvm::Function>(
257
224
          dtor.getCallee()->stripPointerCastsAndAliases()))
258
224
    call->setCallingConv(dtorFn->getCallingConv());
259
260
224
  CGF.FinishFunction();
261
262
224
  return fn;
263
224
}
264
265
/// Register a global destructor using the C atexit runtime function.
266
void CodeGenFunction::registerGlobalDtorWithAtExit(const VarDecl &VD,
267
                                                   llvm::FunctionCallee dtor,
268
178
                                                   llvm::Constant *addr) {
269
  // Create a function which calls the destructor.
270
178
  llvm::Constant *dtorStub = createAtExitStub(VD, dtor, addr);
271
178
  registerGlobalDtorWithAtExit(dtorStub);
272
178
}
273
274
231
void CodeGenFunction::registerGlobalDtorWithAtExit(llvm::Constant *dtorStub) {
275
  // extern "C" int atexit(void (*f)(void));
276
231
  assert(dtorStub->getType() ==
277
231
             llvm::PointerType::get(
278
231
                 llvm::FunctionType::get(CGM.VoidTy, false),
279
231
                 dtorStub->getType()->getPointerAddressSpace()) &&
280
231
         "Argument to atexit has a wrong type.");
281
282
0
  llvm::FunctionType *atexitTy =
283
231
      llvm::FunctionType::get(IntTy, dtorStub->getType(), false);
284
285
231
  llvm::FunctionCallee atexit =
286
231
      CGM.CreateRuntimeFunction(atexitTy, "atexit", llvm::AttributeList(),
287
231
                                /*Local=*/true);
288
231
  if (llvm::Function *atexitFn = dyn_cast<llvm::Function>(atexit.getCallee()))
289
231
    atexitFn->setDoesNotThrow();
290
291
231
  EmitNounwindRuntimeCall(atexit, dtorStub);
292
231
}
293
294
llvm::Value *
295
48
CodeGenFunction::unregisterGlobalDtorWithUnAtExit(llvm::Constant *dtorStub) {
296
  // The unatexit subroutine unregisters __dtor functions that were previously
297
  // registered by the atexit subroutine. If the referenced function is found,
298
  // it is removed from the list of functions that are called at normal program
299
  // termination and the unatexit returns a value of 0, otherwise a non-zero
300
  // value is returned.
301
  //
302
  // extern "C" int unatexit(void (*f)(void));
303
48
  assert(dtorStub->getType() ==
304
48
             llvm::PointerType::get(
305
48
                 llvm::FunctionType::get(CGM.VoidTy, false),
306
48
                 dtorStub->getType()->getPointerAddressSpace()) &&
307
48
         "Argument to unatexit has a wrong type.");
308
309
0
  llvm::FunctionType *unatexitTy =
310
48
      llvm::FunctionType::get(IntTy, {dtorStub->getType()}, /*isVarArg=*/false);
311
312
48
  llvm::FunctionCallee unatexit =
313
48
      CGM.CreateRuntimeFunction(unatexitTy, "unatexit", llvm::AttributeList());
314
315
48
  cast<llvm::Function>(unatexit.getCallee())->setDoesNotThrow();
316
317
48
  return EmitNounwindRuntimeCall(unatexit, dtorStub);
318
48
}
319
320
void CodeGenFunction::EmitCXXGuardedInit(const VarDecl &D,
321
                                         llvm::GlobalVariable *DeclPtr,
322
10.2k
                                         bool PerformInit) {
323
  // If we've been asked to forbid guard variables, emit an error now.
324
  // This diagnostic is hard-coded for Darwin's use case;  we can find
325
  // better phrasing if someone else needs it.
326
10.2k
  if (CGM.getCodeGenOpts().ForbidGuardVariables)
327
1
    CGM.Error(D.getLocation(),
328
1
              "this initialization requires a guard variable, which "
329
1
              "the kernel does not support");
330
331
10.2k
  CGM.getCXXABI().EmitGuardedInit(*this, D, DeclPtr, PerformInit);
332
10.2k
}
333
334
void CodeGenFunction::EmitCXXGuardedInitBranch(llvm::Value *NeedsInit,
335
                                               llvm::BasicBlock *InitBlock,
336
                                               llvm::BasicBlock *NoInitBlock,
337
                                               GuardKind Kind,
338
10.2k
                                               const VarDecl *D) {
339
10.2k
  assert((Kind == GuardKind::TlsGuard || D) && "no guarded variable");
340
341
  // A guess at how many times we will enter the initialization of a
342
  // variable, depending on the kind of variable.
343
0
  static const uint64_t InitsPerTLSVar = 1024;
344
10.2k
  static const uint64_t InitsPerLocalVar = 1024 * 1024;
345
346
10.2k
  llvm::MDNode *Weights;
347
10.2k
  if (Kind == GuardKind::VariableGuard && 
!D->isLocalVarDecl()10.1k
) {
348
    // For non-local variables, don't apply any weighting for now. Due to our
349
    // use of COMDATs, we expect there to be at most one initialization of the
350
    // variable per DSO, but we have no way to know how many DSOs will try to
351
    // initialize the variable.
352
176
    Weights = nullptr;
353
10.0k
  } else {
354
10.0k
    uint64_t NumInits;
355
    // FIXME: For the TLS case, collect and use profiling information to
356
    // determine a more accurate brach weight.
357
10.0k
    if (Kind == GuardKind::TlsGuard || 
D->getTLSKind()9.99k
)
358
116
      NumInits = InitsPerTLSVar;
359
9.93k
    else
360
9.93k
      NumInits = InitsPerLocalVar;
361
362
    // The probability of us entering the initializer is
363
    //   1 / (total number of times we attempt to initialize the variable).
364
10.0k
    llvm::MDBuilder MDHelper(CGM.getLLVMContext());
365
10.0k
    Weights = MDHelper.createBranchWeights(1, NumInits - 1);
366
10.0k
  }
367
368
10.2k
  Builder.CreateCondBr(NeedsInit, InitBlock, NoInitBlock, Weights);
369
10.2k
}
370
371
llvm::Function *CodeGenModule::CreateGlobalInitOrCleanUpFunction(
372
    llvm::FunctionType *FTy, const Twine &Name, const CGFunctionInfo &FI,
373
10.5k
    SourceLocation Loc, bool TLS) {
374
10.5k
  llvm::Function *Fn = llvm::Function::Create(
375
10.5k
      FTy, llvm::GlobalValue::InternalLinkage, Name, &getModule());
376
377
10.5k
  if (!getLangOpts().AppleKext && 
!TLS10.5k
) {
378
    // Set the section if needed.
379
10.5k
    if (const char *Section = getTarget().getStaticInitSectionSpecifier())
380
2.50k
      Fn->setSection(Section);
381
10.5k
  }
382
383
10.5k
  SetInternalFunctionAttributes(GlobalDecl(), Fn, FI);
384
385
10.5k
  Fn->setCallingConv(getRuntimeCC());
386
387
10.5k
  if (!getLangOpts().Exceptions)
388
9.48k
    Fn->setDoesNotThrow();
389
390
10.5k
  if (getLangOpts().Sanitize.has(SanitizerKind::Address) &&
391
10.5k
      
!isInNoSanitizeList(SanitizerKind::Address, Fn, Loc)93
)
392
88
    Fn->addFnAttr(llvm::Attribute::SanitizeAddress);
393
394
10.5k
  if (getLangOpts().Sanitize.has(SanitizerKind::KernelAddress) &&
395
10.5k
      
!isInNoSanitizeList(SanitizerKind::KernelAddress, Fn, Loc)4
)
396
2
    Fn->addFnAttr(llvm::Attribute::SanitizeAddress);
397
398
10.5k
  if (getLangOpts().Sanitize.has(SanitizerKind::HWAddress) &&
399
10.5k
      
!isInNoSanitizeList(SanitizerKind::HWAddress, Fn, Loc)0
)
400
0
    Fn->addFnAttr(llvm::Attribute::SanitizeHWAddress);
401
402
10.5k
  if (getLangOpts().Sanitize.has(SanitizerKind::KernelHWAddress) &&
403
10.5k
      
!isInNoSanitizeList(SanitizerKind::KernelHWAddress, Fn, Loc)0
)
404
0
    Fn->addFnAttr(llvm::Attribute::SanitizeHWAddress);
405
406
10.5k
  if (getLangOpts().Sanitize.has(SanitizerKind::MemTag) &&
407
10.5k
      
!isInNoSanitizeList(SanitizerKind::MemTag, Fn, Loc)0
)
408
0
    Fn->addFnAttr(llvm::Attribute::SanitizeMemTag);
409
410
10.5k
  if (getLangOpts().Sanitize.has(SanitizerKind::Thread) &&
411
10.5k
      
!isInNoSanitizeList(SanitizerKind::Thread, Fn, Loc)6
)
412
3
    Fn->addFnAttr(llvm::Attribute::SanitizeThread);
413
414
10.5k
  if (getLangOpts().Sanitize.has(SanitizerKind::Memory) &&
415
10.5k
      
!isInNoSanitizeList(SanitizerKind::Memory, Fn, Loc)93
)
416
93
    Fn->addFnAttr(llvm::Attribute::SanitizeMemory);
417
418
10.5k
  if (getLangOpts().Sanitize.has(SanitizerKind::KernelMemory) &&
419
10.5k
      
!isInNoSanitizeList(SanitizerKind::KernelMemory, Fn, Loc)0
)
420
0
    Fn->addFnAttr(llvm::Attribute::SanitizeMemory);
421
422
10.5k
  if (getLangOpts().Sanitize.has(SanitizerKind::SafeStack) &&
423
10.5k
      
!isInNoSanitizeList(SanitizerKind::SafeStack, Fn, Loc)0
)
424
0
    Fn->addFnAttr(llvm::Attribute::SafeStack);
425
426
10.5k
  if (getLangOpts().Sanitize.has(SanitizerKind::ShadowCallStack) &&
427
10.5k
      
!isInNoSanitizeList(SanitizerKind::ShadowCallStack, Fn, Loc)0
)
428
0
    Fn->addFnAttr(llvm::Attribute::ShadowCallStack);
429
430
10.5k
  return Fn;
431
10.5k
}
432
433
/// Create a global pointer to a function that will initialize a global
434
/// variable.  The user has requested that this pointer be emitted in a specific
435
/// section.
436
void CodeGenModule::EmitPointerToInitFunc(const VarDecl *D,
437
                                          llvm::GlobalVariable *GV,
438
                                          llvm::Function *InitFunc,
439
6
                                          InitSegAttr *ISA) {
440
6
  llvm::GlobalVariable *PtrArray = new llvm::GlobalVariable(
441
6
      TheModule, InitFunc->getType(), /*isConstant=*/true,
442
6
      llvm::GlobalValue::PrivateLinkage, InitFunc, "__cxx_init_fn_ptr");
443
6
  PtrArray->setSection(ISA->getSection());
444
6
  addUsedGlobal(PtrArray);
445
446
  // If the GV is already in a comdat group, then we have to join it.
447
6
  if (llvm::Comdat *C = GV->getComdat())
448
3
    PtrArray->setComdat(C);
449
6
}
450
451
void
452
CodeGenModule::EmitCXXGlobalVarDeclInitFunc(const VarDecl *D,
453
                                            llvm::GlobalVariable *Addr,
454
5.89k
                                            bool PerformInit) {
455
456
  // According to E.2.3.1 in CUDA-7.5 Programming guide: __device__,
457
  // __constant__ and __shared__ variables defined in namespace scope,
458
  // that are of class type, cannot have a non-empty constructor. All
459
  // the checks have been done in Sema by now. Whatever initializers
460
  // are allowed are empty and we just need to ignore them here.
461
5.89k
  if (getLangOpts().CUDAIsDevice && 
!getLangOpts().GPUAllowDeviceInit25
&&
462
5.89k
      
(24
D->hasAttr<CUDADeviceAttr>()24
||
D->hasAttr<CUDAConstantAttr>()14
||
463
24
       
D->hasAttr<CUDASharedAttr>()4
))
464
24
    return;
465
466
5.87k
  if (getLangOpts().OpenMP &&
467
5.87k
      
getOpenMPRuntime().emitDeclareTargetVarDefinition(D, Addr, PerformInit)3.89k
)
468
78
    return;
469
470
  // Check if we've already initialized this decl.
471
5.79k
  auto I = DelayedCXXInitPosition.find(D);
472
5.79k
  if (I != DelayedCXXInitPosition.end() && 
I->second == ~0U1.62k
)
473
22
    return;
474
475
5.77k
  llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy, false);
476
5.77k
  SmallString<256> FnName;
477
5.77k
  {
478
5.77k
    llvm::raw_svector_ostream Out(FnName);
479
5.77k
    getCXXABI().getMangleContext().mangleDynamicInitializer(D, Out);
480
5.77k
  }
481
482
  // Create a variable initialization function.
483
5.77k
  llvm::Function *Fn = CreateGlobalInitOrCleanUpFunction(
484
5.77k
      FTy, FnName.str(), getTypes().arrangeNullaryFunction(), D->getLocation());
485
486
5.77k
  auto *ISA = D->getAttr<InitSegAttr>();
487
5.77k
  CodeGenFunction(*this).GenerateCXXGlobalVarDeclInitFunc(Fn, D, Addr,
488
5.77k
                                                          PerformInit);
489
490
5.77k
  llvm::GlobalVariable *COMDATKey =
491
5.77k
      supportsCOMDAT() && 
D->isExternallyVisible()4.63k
?
Addr3.90k
:
nullptr1.86k
;
492
493
5.77k
  if (D->getTLSKind()) {
494
    // FIXME: Should we support init_priority for thread_local?
495
    // FIXME: We only need to register one __cxa_thread_atexit function for the
496
    // entire TU.
497
177
    CXXThreadLocalInits.push_back(Fn);
498
177
    CXXThreadLocalInitVars.push_back(D);
499
5.59k
  } else if (PerformInit && 
ISA5.17k
) {
500
6
    EmitPointerToInitFunc(D, Addr, Fn, ISA);
501
5.59k
  } else if (auto *IPA = D->getAttr<InitPriorityAttr>()) {
502
929
    OrderGlobalInitsOrStermFinalizers Key(IPA->getPriority(),
503
929
                                          PrioritizedCXXGlobalInits.size());
504
929
    PrioritizedCXXGlobalInits.push_back(std::make_pair(Key, Fn));
505
4.66k
  } else if (isTemplateInstantiation(D->getTemplateSpecializationKind()) ||
506
4.66k
             
getContext().GetGVALinkageForVariable(D) == GVA_DiscardableODR4.55k
) {
507
    // C++ [basic.start.init]p2:
508
    //   Definitions of explicitly specialized class template static data
509
    //   members have ordered initialization. Other class template static data
510
    //   members (i.e., implicitly or explicitly instantiated specializations)
511
    //   have unordered initialization.
512
    //
513
    // As a consequence, we can put them into their own llvm.global_ctors entry.
514
    //
515
    // If the global is externally visible, put the initializer into a COMDAT
516
    // group with the global being initialized.  On most platforms, this is a
517
    // minor startup time optimization.  In the MS C++ ABI, there are no guard
518
    // variables, so this COMDAT key is required for correctness.
519
133
    AddGlobalCtor(Fn, 65535, COMDATKey);
520
133
    if (getTarget().getCXXABI().isMicrosoft() && 
COMDATKey35
) {
521
      // In The MS C++, MS add template static data member in the linker
522
      // drective.
523
31
      addUsedGlobal(COMDATKey);
524
31
    }
525
4.52k
  } else if (D->hasAttr<SelectAnyAttr>()) {
526
    // SelectAny globals will be comdat-folded. Put the initializer into a
527
    // COMDAT group associated with the global, so the initializers get folded
528
    // too.
529
6
    AddGlobalCtor(Fn, 65535, COMDATKey);
530
4.52k
  } else {
531
4.52k
    I = DelayedCXXInitPosition.find(D); // Re-do lookup in case of re-hash.
532
4.52k
    if (I == DelayedCXXInitPosition.end()) {
533
3.29k
      CXXGlobalInits.push_back(Fn);
534
3.29k
    } else 
if (1.22k
I->second != ~0U1.22k
) {
535
1.22k
      assert(I->second < CXXGlobalInits.size() &&
536
1.22k
             CXXGlobalInits[I->second] == nullptr);
537
0
      CXXGlobalInits[I->second] = Fn;
538
1.22k
    }
539
4.52k
  }
540
541
  // Remember that we already emitted the initializer for this global.
542
0
  DelayedCXXInitPosition[D] = ~0U;
543
5.77k
}
544
545
35.8k
void CodeGenModule::EmitCXXThreadLocalInitFunc() {
546
35.8k
  getCXXABI().EmitThreadLocalInitFuncs(
547
35.8k
      *this, CXXThreadLocals, CXXThreadLocalInits, CXXThreadLocalInitVars);
548
549
35.8k
  CXXThreadLocalInits.clear();
550
35.8k
  CXXThreadLocalInitVars.clear();
551
35.8k
  CXXThreadLocals.clear();
552
35.8k
}
553
554
1.35k
static SmallString<128> getTransformedFileName(llvm::Module &M) {
555
1.35k
  SmallString<128> FileName = llvm::sys::path::filename(M.getName());
556
557
1.35k
  if (FileName.empty())
558
0
    FileName = "<null>";
559
560
45.6k
  for (size_t i = 0; i < FileName.size(); 
++i44.2k
) {
561
    // Replace everything that's not [a-zA-Z0-9._] with a _. This set happens
562
    // to be the set of C preprocessing numbers.
563
44.2k
    if (!isPreprocessingNumberBody(FileName[i]))
564
903
      FileName[i] = '_';
565
44.2k
  }
566
567
1.35k
  return FileName;
568
1.35k
}
569
570
626
static std::string getPrioritySuffix(unsigned int Priority) {
571
626
  assert(Priority <= 65535 && "Priority should always be <= 65535.");
572
573
  // Compute the function suffix from priority. Prepend with zeroes to make
574
  // sure the function names are also ordered as priorities.
575
0
  std::string PrioritySuffix = llvm::utostr(Priority);
576
626
  PrioritySuffix = std::string(6 - PrioritySuffix.size(), '0') + PrioritySuffix;
577
578
626
  return PrioritySuffix;
579
626
}
580
581
void
582
35.8k
CodeGenModule::EmitCXXGlobalInitFunc() {
583
49.4k
  while (!CXXGlobalInits.empty() && 
!CXXGlobalInits.back()14.9k
)
584
13.6k
    CXXGlobalInits.pop_back();
585
586
35.8k
  if (CXXGlobalInits.empty() && 
PrioritizedCXXGlobalInits.empty()34.4k
)
587
34.4k
    return;
588
589
1.35k
  llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy, false);
590
1.35k
  const CGFunctionInfo &FI = getTypes().arrangeNullaryFunction();
591
592
  // Create our global prioritized initialization function.
593
1.35k
  if (!PrioritizedCXXGlobalInits.empty()) {
594
309
    SmallVector<llvm::Function *, 8> LocalCXXGlobalInits;
595
309
    llvm::array_pod_sort(PrioritizedCXXGlobalInits.begin(),
596
309
                         PrioritizedCXXGlobalInits.end());
597
    // Iterate over "chunks" of ctors with same priority and emit each chunk
598
    // into separate function. Note - everything is sorted first by priority,
599
    // second - by lex order, so we emit ctor functions in proper order.
600
309
    for (SmallVectorImpl<GlobalInitData >::iterator
601
309
           I = PrioritizedCXXGlobalInits.begin(),
602
929
           E = PrioritizedCXXGlobalInits.end(); I != E; ) {
603
620
      SmallVectorImpl<GlobalInitData >::iterator
604
620
        PrioE = std::upper_bound(I + 1, E, *I, GlobalInitPriorityCmp());
605
606
620
      LocalCXXGlobalInits.clear();
607
608
620
      unsigned int Priority = I->first.priority;
609
620
      llvm::Function *Fn = CreateGlobalInitOrCleanUpFunction(
610
620
          FTy, "_GLOBAL__I_" + getPrioritySuffix(Priority), FI);
611
612
1.54k
      for (; I < PrioE; 
++I929
)
613
929
        LocalCXXGlobalInits.push_back(I->second);
614
615
620
      CodeGenFunction(*this).GenerateCXXGlobalInitFunc(Fn, LocalCXXGlobalInits);
616
620
      AddGlobalCtor(Fn, Priority);
617
620
    }
618
309
    PrioritizedCXXGlobalInits.clear();
619
309
  }
620
621
1.35k
  if (getCXXABI().useSinitAndSterm() && 
CXXGlobalInits.empty()14
)
622
0
    return;
623
624
  // Include the filename in the symbol name. Including "sub_" matches gcc
625
  // and makes sure these symbols appear lexicographically behind the symbols
626
  // with priority emitted above.
627
1.35k
  llvm::Function *Fn = CreateGlobalInitOrCleanUpFunction(
628
1.35k
      FTy, llvm::Twine("_GLOBAL__sub_I_", getTransformedFileName(getModule())),
629
1.35k
      FI);
630
631
1.35k
  CodeGenFunction(*this).GenerateCXXGlobalInitFunc(Fn, CXXGlobalInits);
632
1.35k
  AddGlobalCtor(Fn);
633
634
  // In OpenCL global init functions must be converted to kernels in order to
635
  // be able to launch them from the host.
636
  // FIXME: Some more work might be needed to handle destructors correctly.
637
  // Current initialization function makes use of function pointers callbacks.
638
  // We can't support function pointers especially between host and device.
639
  // However it seems global destruction has little meaning without any
640
  // dynamic resource allocation on the device and program scope variables are
641
  // destroyed by the runtime when program is released.
642
1.35k
  if (getLangOpts().OpenCL) {
643
7
    GenOpenCLArgMetadata(Fn);
644
7
    Fn->setCallingConv(llvm::CallingConv::SPIR_KERNEL);
645
7
  }
646
647
1.35k
  assert(!getLangOpts().CUDA || !getLangOpts().CUDAIsDevice ||
648
1.35k
         getLangOpts().GPUAllowDeviceInit);
649
1.35k
  if (getLangOpts().HIP && 
getLangOpts().CUDAIsDevice1
) {
650
1
    Fn->setCallingConv(llvm::CallingConv::AMDGPU_KERNEL);
651
1
    Fn->addFnAttr("device-init");
652
1
  }
653
654
1.35k
  CXXGlobalInits.clear();
655
1.35k
}
656
657
35.8k
void CodeGenModule::EmitCXXGlobalCleanUpFunc() {
658
35.8k
  if (CXXGlobalDtorsOrStermFinalizers.empty() &&
659
35.8k
      
PrioritizedCXXStermFinalizers.empty()35.8k
)
660
35.8k
    return;
661
662
17
  llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy, false);
663
17
  const CGFunctionInfo &FI = getTypes().arrangeNullaryFunction();
664
665
  // Create our global prioritized cleanup function.
666
17
  if (!PrioritizedCXXStermFinalizers.empty()) {
667
2
    SmallVector<CXXGlobalDtorsOrStermFinalizer_t, 8> LocalCXXStermFinalizers;
668
2
    llvm::array_pod_sort(PrioritizedCXXStermFinalizers.begin(),
669
2
                         PrioritizedCXXStermFinalizers.end());
670
    // Iterate over "chunks" of dtors with same priority and emit each chunk
671
    // into separate function. Note - everything is sorted first by priority,
672
    // second - by lex order, so we emit dtor functions in proper order.
673
2
    for (SmallVectorImpl<StermFinalizerData>::iterator
674
2
             I = PrioritizedCXXStermFinalizers.begin(),
675
2
             E = PrioritizedCXXStermFinalizers.end();
676
8
         I != E;) {
677
6
      SmallVectorImpl<StermFinalizerData>::iterator PrioE =
678
6
          std::upper_bound(I + 1, E, *I, StermFinalizerPriorityCmp());
679
680
6
      LocalCXXStermFinalizers.clear();
681
682
6
      unsigned int Priority = I->first.priority;
683
6
      llvm::Function *Fn = CreateGlobalInitOrCleanUpFunction(
684
6
          FTy, "_GLOBAL__a_" + getPrioritySuffix(Priority), FI);
685
686
14
      for (; I < PrioE; 
++I8
) {
687
8
        llvm::FunctionCallee DtorFn = I->second;
688
8
        LocalCXXStermFinalizers.emplace_back(DtorFn.getFunctionType(),
689
8
                                             DtorFn.getCallee(), nullptr);
690
8
      }
691
692
6
      CodeGenFunction(*this).GenerateCXXGlobalCleanUpFunc(
693
6
          Fn, LocalCXXStermFinalizers);
694
6
      AddGlobalDtor(Fn, Priority);
695
6
    }
696
2
    PrioritizedCXXStermFinalizers.clear();
697
2
  }
698
699
17
  if (CXXGlobalDtorsOrStermFinalizers.empty())
700
0
    return;
701
702
  // Create our global cleanup function.
703
17
  llvm::Function *Fn =
704
17
      CreateGlobalInitOrCleanUpFunction(FTy, "_GLOBAL__D_a", FI);
705
706
17
  CodeGenFunction(*this).GenerateCXXGlobalCleanUpFunc(
707
17
      Fn, CXXGlobalDtorsOrStermFinalizers);
708
17
  AddGlobalDtor(Fn);
709
17
  CXXGlobalDtorsOrStermFinalizers.clear();
710
17
}
711
712
/// Emit the code necessary to initialize the given global variable.
713
void CodeGenFunction::GenerateCXXGlobalVarDeclInitFunc(llvm::Function *Fn,
714
                                                       const VarDecl *D,
715
                                                 llvm::GlobalVariable *Addr,
716
5.77k
                                                       bool PerformInit) {
717
  // Check if we need to emit debug info for variable initializer.
718
5.77k
  if (D->hasAttr<NoDebugAttr>())
719
1
    DebugInfo = nullptr; // disable debug info indefinitely for this function
720
721
5.77k
  CurEHLocation = D->getBeginLoc();
722
723
5.77k
  StartFunction(GlobalDecl(D, DynamicInitKind::Initializer),
724
5.77k
                getContext().VoidTy, Fn, getTypes().arrangeNullaryFunction(),
725
5.77k
                FunctionArgList());
726
  // Emit an artificial location for this function.
727
5.77k
  auto AL = ApplyDebugLocation::CreateArtificial(*this);
728
729
  // Use guarded initialization if the global variable is weak. This
730
  // occurs for, e.g., instantiated static data members and
731
  // definitions explicitly marked weak.
732
  //
733
  // Also use guarded initialization for a variable with dynamic TLS and
734
  // unordered initialization. (If the initialization is ordered, the ABI
735
  // layer will guard the whole-TU initialization for us.)
736
5.77k
  if (Addr->hasWeakLinkage() || 
Addr->hasLinkOnceLinkage()5.70k
||
737
5.77k
      
(5.55k
D->getTLSKind() == VarDecl::TLS_Dynamic5.55k
&&
738
5.55k
       
isTemplateInstantiation(D->getTemplateSpecializationKind())97
)) {
739
222
    EmitCXXGuardedInit(*D, Addr, PerformInit);
740
5.55k
  } else {
741
5.55k
    EmitCXXGlobalVarDeclInit(*D, Addr, PerformInit);
742
5.55k
  }
743
744
5.77k
  FinishFunction();
745
5.77k
}
746
747
void
748
CodeGenFunction::GenerateCXXGlobalInitFunc(llvm::Function *Fn,
749
                                           ArrayRef<llvm::Function *> Decls,
750
2.03k
                                           ConstantAddress Guard) {
751
2.03k
  {
752
2.03k
    auto NL = ApplyDebugLocation::CreateEmpty(*this);
753
2.03k
    StartFunction(GlobalDecl(), getContext().VoidTy, Fn,
754
2.03k
                  getTypes().arrangeNullaryFunction(), FunctionArgList());
755
    // Emit an artificial location for this function.
756
2.03k
    auto AL = ApplyDebugLocation::CreateArtificial(*this);
757
758
2.03k
    llvm::BasicBlock *ExitBlock = nullptr;
759
2.03k
    if (Guard.isValid()) {
760
      // If we have a guard variable, check whether we've already performed
761
      // these initializations. This happens for TLS initialization functions.
762
49
      llvm::Value *GuardVal = Builder.CreateLoad(Guard);
763
49
      llvm::Value *Uninit = Builder.CreateIsNull(GuardVal,
764
49
                                                 "guard.uninitialized");
765
49
      llvm::BasicBlock *InitBlock = createBasicBlock("init");
766
49
      ExitBlock = createBasicBlock("exit");
767
49
      EmitCXXGuardedInitBranch(Uninit, InitBlock, ExitBlock,
768
49
                               GuardKind::TlsGuard, nullptr);
769
49
      EmitBlock(InitBlock);
770
      // Mark as initialized before initializing anything else. If the
771
      // initializers use previously-initialized thread_local vars, that's
772
      // probably supposed to be OK, but the standard doesn't say.
773
49
      Builder.CreateStore(llvm::ConstantInt::get(GuardVal->getType(),1), Guard);
774
775
      // The guard variable can't ever change again.
776
49
      EmitInvariantStart(
777
49
          Guard.getPointer(),
778
49
          CharUnits::fromQuantity(
779
49
              CGM.getDataLayout().getTypeAllocSize(GuardVal->getType())));
780
49
    }
781
782
2.03k
    RunCleanupsScope Scope(*this);
783
784
    // When building in Objective-C++ ARC mode, create an autorelease pool
785
    // around the global initializers.
786
2.03k
    if (getLangOpts().ObjCAutoRefCount && 
getLangOpts().CPlusPlus4
) {
787
4
      llvm::Value *token = EmitObjCAutoreleasePoolPush();
788
4
      EmitObjCAutoreleasePoolCleanup(token);
789
4
    }
790
791
10.3k
    for (unsigned i = 0, e = Decls.size(); i != e; 
++i8.34k
)
792
8.34k
      if (Decls[i])
793
5.55k
        EmitRuntimeCall(Decls[i]);
794
795
2.03k
    Scope.ForceCleanup();
796
797
2.03k
    if (ExitBlock) {
798
49
      Builder.CreateBr(ExitBlock);
799
49
      EmitBlock(ExitBlock);
800
49
    }
801
2.03k
  }
802
803
2.03k
  FinishFunction();
804
2.03k
}
805
806
void CodeGenFunction::GenerateCXXGlobalCleanUpFunc(
807
    llvm::Function *Fn,
808
    ArrayRef<std::tuple<llvm::FunctionType *, llvm::WeakTrackingVH,
809
                        llvm::Constant *>>
810
23
        DtorsOrStermFinalizers) {
811
23
  {
812
23
    auto NL = ApplyDebugLocation::CreateEmpty(*this);
813
23
    StartFunction(GlobalDecl(), getContext().VoidTy, Fn,
814
23
                  getTypes().arrangeNullaryFunction(), FunctionArgList());
815
    // Emit an artificial location for this function.
816
23
    auto AL = ApplyDebugLocation::CreateArtificial(*this);
817
818
    // Emit the cleanups, in reverse order from construction.
819
59
    for (unsigned i = 0, e = DtorsOrStermFinalizers.size(); i != e; 
++i36
) {
820
36
      llvm::FunctionType *CalleeTy;
821
36
      llvm::Value *Callee;
822
36
      llvm::Constant *Arg;
823
36
      std::tie(CalleeTy, Callee, Arg) = DtorsOrStermFinalizers[e - i - 1];
824
825
36
      llvm::CallInst *CI = nullptr;
826
36
      if (Arg == nullptr) {
827
30
        assert(
828
30
            CGM.getCXXABI().useSinitAndSterm() &&
829
30
            "Arg could not be nullptr unless using sinit and sterm functions.");
830
0
        CI = Builder.CreateCall(CalleeTy, Callee);
831
30
      } else
832
6
        CI = Builder.CreateCall(CalleeTy, Callee, Arg);
833
834
      // Make sure the call and the callee agree on calling convention.
835
36
      if (llvm::Function *F = dyn_cast<llvm::Function>(Callee))
836
36
        CI->setCallingConv(F->getCallingConv());
837
36
    }
838
23
  }
839
840
23
  FinishFunction();
841
23
}
842
843
/// generateDestroyHelper - Generates a helper function which, when
844
/// invoked, destroys the given object.  The address of the object
845
/// should be in global memory.
846
llvm::Function *CodeGenFunction::generateDestroyHelper(
847
    Address addr, QualType type, Destroyer *destroyer,
848
342
    bool useEHCleanupForArray, const VarDecl *VD) {
849
342
  FunctionArgList args;
850
342
  ImplicitParamDecl Dst(getContext(), getContext().VoidPtrTy,
851
342
                        ImplicitParamDecl::Other);
852
342
  args.push_back(&Dst);
853
854
342
  const CGFunctionInfo &FI =
855
342
    CGM.getTypes().arrangeBuiltinFunctionDeclaration(getContext().VoidTy, args);
856
342
  llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
857
342
  llvm::Function *fn = CGM.CreateGlobalInitOrCleanUpFunction(
858
342
      FTy, "__cxx_global_array_dtor", FI, VD->getLocation());
859
860
342
  CurEHLocation = VD->getBeginLoc();
861
862
342
  StartFunction(GlobalDecl(VD, DynamicInitKind::GlobalArrayDestructor),
863
342
                getContext().VoidTy, fn, FI, args);
864
  // Emit an artificial location for this function.
865
342
  auto AL = ApplyDebugLocation::CreateArtificial(*this);
866
867
342
  emitDestroy(addr, type, destroyer, useEHCleanupForArray);
868
869
342
  FinishFunction();
870
871
342
  return fn;
872
342
}