Coverage Report

Created: 2021-09-21 08:58

/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/CodeGen/CGClass.cpp
Line
Count
Source (jump to first uncovered line)
1
//===--- CGClass.cpp - Emit LLVM Code for C++ classes -----------*- C++ -*-===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This contains code dealing with C++ code generation of classes
10
//
11
//===----------------------------------------------------------------------===//
12
13
#include "CGBlocks.h"
14
#include "CGCXXABI.h"
15
#include "CGDebugInfo.h"
16
#include "CGRecordLayout.h"
17
#include "CodeGenFunction.h"
18
#include "TargetInfo.h"
19
#include "clang/AST/Attr.h"
20
#include "clang/AST/CXXInheritance.h"
21
#include "clang/AST/CharUnits.h"
22
#include "clang/AST/DeclTemplate.h"
23
#include "clang/AST/EvaluatedExprVisitor.h"
24
#include "clang/AST/RecordLayout.h"
25
#include "clang/AST/StmtCXX.h"
26
#include "clang/Basic/CodeGenOptions.h"
27
#include "clang/Basic/TargetBuiltins.h"
28
#include "clang/CodeGen/CGFunctionInfo.h"
29
#include "llvm/IR/Intrinsics.h"
30
#include "llvm/IR/Metadata.h"
31
#include "llvm/Transforms/Utils/SanitizerStats.h"
32
33
using namespace clang;
34
using namespace CodeGen;
35
36
/// Return the best known alignment for an unknown pointer to a
37
/// particular class.
38
734k
CharUnits CodeGenModule::getClassPointerAlignment(const CXXRecordDecl *RD) {
39
734k
  if (!RD->hasDefinition())
40
0
    return CharUnits::One(); // Hopefully won't be used anywhere.
41
42
734k
  auto &layout = getContext().getASTRecordLayout(RD);
43
44
  // If the class is final, then we know that the pointer points to an
45
  // object of that type and can use the full alignment.
46
734k
  if (RD->isEffectivelyFinal())
47
155
    return layout.getAlignment();
48
49
  // Otherwise, we have to assume it could be a subclass.
50
734k
  return layout.getNonVirtualAlignment();
51
734k
}
52
53
/// Return the smallest possible amount of storage that might be allocated
54
/// starting from the beginning of an object of a particular class.
55
///
56
/// This may be smaller than sizeof(RD) if RD has virtual base classes.
57
382k
CharUnits CodeGenModule::getMinimumClassObjectSize(const CXXRecordDecl *RD) {
58
382k
  if (!RD->hasDefinition())
59
2
    return CharUnits::One();
60
61
382k
  auto &layout = getContext().getASTRecordLayout(RD);
62
63
  // If the class is final, then we know that the pointer points to an
64
  // object of that type and can use the full alignment.
65
382k
  if (RD->isEffectivelyFinal())
66
70
    return layout.getSize();
67
68
  // Otherwise, we have to assume it could be a subclass.
69
382k
  return std::max(layout.getNonVirtualSize(), CharUnits::One());
70
382k
}
71
72
/// Return the best known alignment for a pointer to a virtual base,
73
/// given the alignment of a pointer to the derived class.
74
CharUnits CodeGenModule::getVBaseAlignment(CharUnits actualDerivedAlign,
75
                                           const CXXRecordDecl *derivedClass,
76
896
                                           const CXXRecordDecl *vbaseClass) {
77
  // The basic idea here is that an underaligned derived pointer might
78
  // indicate an underaligned base pointer.
79
80
896
  assert(vbaseClass->isCompleteDefinition());
81
0
  auto &baseLayout = getContext().getASTRecordLayout(vbaseClass);
82
896
  CharUnits expectedVBaseAlign = baseLayout.getNonVirtualAlignment();
83
84
896
  return getDynamicOffsetAlignment(actualDerivedAlign, derivedClass,
85
896
                                   expectedVBaseAlign);
86
896
}
87
88
CharUnits
89
CodeGenModule::getDynamicOffsetAlignment(CharUnits actualBaseAlign,
90
                                         const CXXRecordDecl *baseDecl,
91
1.07k
                                         CharUnits expectedTargetAlign) {
92
  // If the base is an incomplete type (which is, alas, possible with
93
  // member pointers), be pessimistic.
94
1.07k
  if (!baseDecl->isCompleteDefinition())
95
5
    return std::min(actualBaseAlign, expectedTargetAlign);
96
97
1.07k
  auto &baseLayout = getContext().getASTRecordLayout(baseDecl);
98
1.07k
  CharUnits expectedBaseAlign = baseLayout.getNonVirtualAlignment();
99
100
  // If the class is properly aligned, assume the target offset is, too.
101
  //
102
  // This actually isn't necessarily the right thing to do --- if the
103
  // class is a complete object, but it's only properly aligned for a
104
  // base subobject, then the alignments of things relative to it are
105
  // probably off as well.  (Note that this requires the alignment of
106
  // the target to be greater than the NV alignment of the derived
107
  // class.)
108
  //
109
  // However, our approach to this kind of under-alignment can only
110
  // ever be best effort; after all, we're never going to propagate
111
  // alignments through variables or parameters.  Note, in particular,
112
  // that constructing a polymorphic type in an address that's less
113
  // than pointer-aligned will generally trap in the constructor,
114
  // unless we someday add some sort of attribute to change the
115
  // assumed alignment of 'this'.  So our goal here is pretty much
116
  // just to allow the user to explicitly say that a pointer is
117
  // under-aligned and then safely access its fields and vtables.
118
1.07k
  if (actualBaseAlign >= expectedBaseAlign) {
119
1.07k
    return expectedTargetAlign;
120
1.07k
  }
121
122
  // Otherwise, we might be offset by an arbitrary multiple of the
123
  // actual alignment.  The correct adjustment is to take the min of
124
  // the two alignments.
125
0
  return std::min(actualBaseAlign, expectedTargetAlign);
126
1.07k
}
127
128
68.0k
Address CodeGenFunction::LoadCXXThisAddress() {
129
68.0k
  assert(CurFuncDecl && "loading 'this' without a func declaration?");
130
0
  assert(isa<CXXMethodDecl>(CurFuncDecl));
131
132
  // Lazily compute CXXThisAlignment.
133
68.0k
  if (CXXThisAlignment.isZero()) {
134
    // Just use the best known alignment for the parent.
135
    // TODO: if we're currently emitting a complete-object ctor/dtor,
136
    // we can always use the complete-object alignment.
137
52.2k
    auto RD = cast<CXXMethodDecl>(CurFuncDecl)->getParent();
138
52.2k
    CXXThisAlignment = CGM.getClassPointerAlignment(RD);
139
52.2k
  }
140
141
68.0k
  return Address(LoadCXXThis(), CXXThisAlignment);
142
68.0k
}
143
144
/// Emit the address of a field using a member data pointer.
145
///
146
/// \param E Only used for emergency diagnostics
147
Address
148
CodeGenFunction::EmitCXXMemberDataPointerAddress(const Expr *E, Address base,
149
                                                 llvm::Value *memberPtr,
150
                                      const MemberPointerType *memberPtrType,
151
                                                 LValueBaseInfo *BaseInfo,
152
81
                                                 TBAAAccessInfo *TBAAInfo) {
153
  // Ask the ABI to compute the actual address.
154
81
  llvm::Value *ptr =
155
81
    CGM.getCXXABI().EmitMemberDataPointerAddress(*this, E, base,
156
81
                                                 memberPtr, memberPtrType);
157
158
81
  QualType memberType = memberPtrType->getPointeeType();
159
81
  CharUnits memberAlign =
160
81
      CGM.getNaturalTypeAlignment(memberType, BaseInfo, TBAAInfo);
161
81
  memberAlign =
162
81
    CGM.getDynamicOffsetAlignment(base.getAlignment(),
163
81
                            memberPtrType->getClass()->getAsCXXRecordDecl(),
164
81
                                  memberAlign);
165
81
  return Address(ptr, memberAlign);
166
81
}
167
168
CharUnits CodeGenModule::computeNonVirtualBaseClassOffset(
169
    const CXXRecordDecl *DerivedClass, CastExpr::path_const_iterator Start,
170
22.8k
    CastExpr::path_const_iterator End) {
171
22.8k
  CharUnits Offset = CharUnits::Zero();
172
173
22.8k
  const ASTContext &Context = getContext();
174
22.8k
  const CXXRecordDecl *RD = DerivedClass;
175
176
47.7k
  for (CastExpr::path_const_iterator I = Start; I != End; 
++I24.9k
) {
177
24.9k
    const CXXBaseSpecifier *Base = *I;
178
24.9k
    assert(!Base->isVirtual() && "Should not see virtual bases here!");
179
180
    // Get the layout.
181
0
    const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
182
183
24.9k
    const auto *BaseDecl =
184
24.9k
        cast<CXXRecordDecl>(Base->getType()->castAs<RecordType>()->getDecl());
185
186
    // Add the offset.
187
24.9k
    Offset += Layout.getBaseClassOffset(BaseDecl);
188
189
24.9k
    RD = BaseDecl;
190
24.9k
  }
191
192
22.8k
  return Offset;
193
22.8k
}
194
195
llvm::Constant *
196
CodeGenModule::GetNonVirtualBaseClassOffset(const CXXRecordDecl *ClassDecl,
197
                                   CastExpr::path_const_iterator PathBegin,
198
787
                                   CastExpr::path_const_iterator PathEnd) {
199
787
  assert(PathBegin != PathEnd && "Base path should not be empty!");
200
201
0
  CharUnits Offset =
202
787
      computeNonVirtualBaseClassOffset(ClassDecl, PathBegin, PathEnd);
203
787
  if (Offset.isZero())
204
753
    return nullptr;
205
206
34
  llvm::Type *PtrDiffTy =
207
34
  Types.ConvertType(getContext().getPointerDiffType());
208
209
34
  return llvm::ConstantInt::get(PtrDiffTy, Offset.getQuantity());
210
787
}
211
212
/// Gets the address of a direct base class within a complete object.
213
/// This should only be used for (1) non-virtual bases or (2) virtual bases
214
/// when the type is known to be complete (e.g. in complete destructors).
215
///
216
/// The object pointed to by 'This' is assumed to be non-null.
217
Address
218
CodeGenFunction::GetAddressOfDirectBaseInCompleteClass(Address This,
219
                                                   const CXXRecordDecl *Derived,
220
                                                   const CXXRecordDecl *Base,
221
12.5k
                                                   bool BaseIsVirtual) {
222
  // 'this' must be a pointer (in some address space) to Derived.
223
12.5k
  assert(This.getElementType() == ConvertType(Derived));
224
225
  // Compute the offset of the virtual base.
226
0
  CharUnits Offset;
227
12.5k
  const ASTRecordLayout &Layout = getContext().getASTRecordLayout(Derived);
228
12.5k
  if (BaseIsVirtual)
229
1.00k
    Offset = Layout.getVBaseClassOffset(Base);
230
11.5k
  else
231
11.5k
    Offset = Layout.getBaseClassOffset(Base);
232
233
  // Shift and cast down to the base type.
234
  // TODO: for complete types, this should be possible with a GEP.
235
12.5k
  Address V = This;
236
12.5k
  if (!Offset.isZero()) {
237
1.98k
    V = Builder.CreateElementBitCast(V, Int8Ty);
238
1.98k
    V = Builder.CreateConstInBoundsByteGEP(V, Offset);
239
1.98k
  }
240
12.5k
  V = Builder.CreateElementBitCast(V, ConvertType(Base));
241
242
12.5k
  return V;
243
12.5k
}
244
245
static Address
246
ApplyNonVirtualAndVirtualOffset(CodeGenFunction &CGF, Address addr,
247
                                CharUnits nonVirtualOffset,
248
                                llvm::Value *virtualOffset,
249
                                const CXXRecordDecl *derivedClass,
250
2.08k
                                const CXXRecordDecl *nearestVBase) {
251
  // Assert that we have something to do.
252
2.08k
  assert(!nonVirtualOffset.isZero() || virtualOffset != nullptr);
253
254
  // Compute the offset from the static and dynamic components.
255
0
  llvm::Value *baseOffset;
256
2.08k
  if (!nonVirtualOffset.isZero()) {
257
1.32k
    llvm::Type *OffsetType =
258
1.32k
        (CGF.CGM.getTarget().getCXXABI().isItaniumFamily() &&
259
1.32k
         
CGF.CGM.getItaniumVTableContext().isRelativeLayout()1.18k
)
260
1.32k
            ? 
CGF.Int32Ty4
261
1.32k
            : 
CGF.PtrDiffTy1.31k
;
262
1.32k
    baseOffset =
263
1.32k
        llvm::ConstantInt::get(OffsetType, nonVirtualOffset.getQuantity());
264
1.32k
    if (virtualOffset) {
265
34
      baseOffset = CGF.Builder.CreateAdd(virtualOffset, baseOffset);
266
34
    }
267
1.32k
  } else {
268
757
    baseOffset = virtualOffset;
269
757
  }
270
271
  // Apply the base offset.
272
2.08k
  llvm::Value *ptr = addr.getPointer();
273
2.08k
  unsigned AddrSpace = ptr->getType()->getPointerAddressSpace();
274
2.08k
  ptr = CGF.Builder.CreateBitCast(ptr, CGF.Int8Ty->getPointerTo(AddrSpace));
275
2.08k
  ptr = CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, ptr, baseOffset, "add.ptr");
276
277
  // If we have a virtual component, the alignment of the result will
278
  // be relative only to the known alignment of that vbase.
279
2.08k
  CharUnits alignment;
280
2.08k
  if (virtualOffset) {
281
791
    assert(nearestVBase && "virtual offset without vbase?");
282
0
    alignment = CGF.CGM.getVBaseAlignment(addr.getAlignment(),
283
791
                                          derivedClass, nearestVBase);
284
1.28k
  } else {
285
1.28k
    alignment = addr.getAlignment();
286
1.28k
  }
287
0
  alignment = alignment.alignmentAtOffset(nonVirtualOffset);
288
289
2.08k
  return Address(ptr, alignment);
290
2.08k
}
291
292
Address CodeGenFunction::GetAddressOfBaseClass(
293
    Address Value, const CXXRecordDecl *Derived,
294
    CastExpr::path_const_iterator PathBegin,
295
    CastExpr::path_const_iterator PathEnd, bool NullCheckValue,
296
21.9k
    SourceLocation Loc) {
297
21.9k
  assert(PathBegin != PathEnd && "Base path should not be empty!");
298
299
0
  CastExpr::path_const_iterator Start = PathBegin;
300
21.9k
  const CXXRecordDecl *VBase = nullptr;
301
302
  // Sema has done some convenient canonicalization here: if the
303
  // access path involved any virtual steps, the conversion path will
304
  // *start* with a step down to the correct virtual base subobject,
305
  // and hence will not require any further steps.
306
21.9k
  if ((*Start)->isVirtual()) {
307
442
    VBase = cast<CXXRecordDecl>(
308
442
        (*Start)->getType()->castAs<RecordType>()->getDecl());
309
442
    ++Start;
310
442
  }
311
312
  // Compute the static offset of the ultimate destination within its
313
  // allocating subobject (the virtual base, if there is one, or else
314
  // the "complete" object that we see).
315
21.9k
  CharUnits NonVirtualOffset = CGM.computeNonVirtualBaseClassOffset(
316
21.9k
      VBase ? 
VBase442
:
Derived21.5k
, Start, PathEnd);
317
318
  // If there's a virtual step, we can sometimes "devirtualize" it.
319
  // For now, that's limited to when the derived type is final.
320
  // TODO: "devirtualize" this for accesses to known-complete objects.
321
21.9k
  if (VBase && 
Derived->hasAttr<FinalAttr>()442
) {
322
6
    const ASTRecordLayout &layout = getContext().getASTRecordLayout(Derived);
323
6
    CharUnits vBaseOffset = layout.getVBaseClassOffset(VBase);
324
6
    NonVirtualOffset += vBaseOffset;
325
6
    VBase = nullptr; // we no longer have a virtual step
326
6
  }
327
328
  // Get the base pointer type.
329
21.9k
  llvm::Type *BasePtrTy =
330
21.9k
      ConvertType((PathEnd[-1])->getType())
331
21.9k
          ->getPointerTo(Value.getType()->getPointerAddressSpace());
332
333
21.9k
  QualType DerivedTy = getContext().getRecordType(Derived);
334
21.9k
  CharUnits DerivedAlign = CGM.getClassPointerAlignment(Derived);
335
336
  // If the static offset is zero and we don't have a virtual step,
337
  // just do a bitcast; null checks are unnecessary.
338
21.9k
  if (NonVirtualOffset.isZero() && 
!VBase20.9k
) {
339
20.4k
    if (sanitizePerformTypeCheck()) {
340
19
      SanitizerSet SkippedChecks;
341
19
      SkippedChecks.set(SanitizerKind::Null, !NullCheckValue);
342
19
      EmitTypeCheck(TCK_Upcast, Loc, Value.getPointer(),
343
19
                    DerivedTy, DerivedAlign, SkippedChecks);
344
19
    }
345
20.4k
    return Builder.CreateBitCast(Value, BasePtrTy);
346
20.4k
  }
347
348
1.51k
  llvm::BasicBlock *origBB = nullptr;
349
1.51k
  llvm::BasicBlock *endBB = nullptr;
350
351
  // Skip over the offset (and the vtable load) if we're supposed to
352
  // null-check the pointer.
353
1.51k
  if (NullCheckValue) {
354
35
    origBB = Builder.GetInsertBlock();
355
35
    llvm::BasicBlock *notNullBB = createBasicBlock("cast.notnull");
356
35
    endBB = createBasicBlock("cast.end");
357
358
35
    llvm::Value *isNull = Builder.CreateIsNull(Value.getPointer());
359
35
    Builder.CreateCondBr(isNull, endBB, notNullBB);
360
35
    EmitBlock(notNullBB);
361
35
  }
362
363
1.51k
  if (sanitizePerformTypeCheck()) {
364
3
    SanitizerSet SkippedChecks;
365
3
    SkippedChecks.set(SanitizerKind::Null, true);
366
3
    EmitTypeCheck(VBase ? TCK_UpcastToVirtualBase : 
TCK_Upcast0
, Loc,
367
3
                  Value.getPointer(), DerivedTy, DerivedAlign, SkippedChecks);
368
3
  }
369
370
  // Compute the virtual offset.
371
1.51k
  llvm::Value *VirtualOffset = nullptr;
372
1.51k
  if (VBase) {
373
436
    VirtualOffset =
374
436
      CGM.getCXXABI().GetVirtualBaseClassOffset(*this, Value, Derived, VBase);
375
436
  }
376
377
  // Apply both offsets.
378
1.51k
  Value = ApplyNonVirtualAndVirtualOffset(*this, Value, NonVirtualOffset,
379
1.51k
                                          VirtualOffset, Derived, VBase);
380
381
  // Cast to the destination type.
382
1.51k
  Value = Builder.CreateBitCast(Value, BasePtrTy);
383
384
  // Build a phi if we needed a null check.
385
1.51k
  if (NullCheckValue) {
386
35
    llvm::BasicBlock *notNullBB = Builder.GetInsertBlock();
387
35
    Builder.CreateBr(endBB);
388
35
    EmitBlock(endBB);
389
390
35
    llvm::PHINode *PHI = Builder.CreatePHI(BasePtrTy, 2, "cast.result");
391
35
    PHI->addIncoming(Value.getPointer(), notNullBB);
392
35
    PHI->addIncoming(llvm::Constant::getNullValue(BasePtrTy), origBB);
393
35
    Value = Address(PHI, Value.getAlignment());
394
35
  }
395
396
1.51k
  return Value;
397
21.9k
}
398
399
Address
400
CodeGenFunction::GetAddressOfDerivedClass(Address BaseAddr,
401
                                          const CXXRecordDecl *Derived,
402
                                        CastExpr::path_const_iterator PathBegin,
403
                                          CastExpr::path_const_iterator PathEnd,
404
731
                                          bool NullCheckValue) {
405
731
  assert(PathBegin != PathEnd && "Base path should not be empty!");
406
407
0
  QualType DerivedTy =
408
731
    getContext().getCanonicalType(getContext().getTagDeclType(Derived));
409
731
  unsigned AddrSpace =
410
731
    BaseAddr.getPointer()->getType()->getPointerAddressSpace();
411
731
  llvm::Type *DerivedPtrTy = ConvertType(DerivedTy)->getPointerTo(AddrSpace);
412
413
731
  llvm::Value *NonVirtualOffset =
414
731
    CGM.GetNonVirtualBaseClassOffset(Derived, PathBegin, PathEnd);
415
416
731
  if (!NonVirtualOffset) {
417
    // No offset, we can just cast back.
418
720
    return Builder.CreateBitCast(BaseAddr, DerivedPtrTy);
419
720
  }
420
421
11
  llvm::BasicBlock *CastNull = nullptr;
422
11
  llvm::BasicBlock *CastNotNull = nullptr;
423
11
  llvm::BasicBlock *CastEnd = nullptr;
424
425
11
  if (NullCheckValue) {
426
5
    CastNull = createBasicBlock("cast.null");
427
5
    CastNotNull = createBasicBlock("cast.notnull");
428
5
    CastEnd = createBasicBlock("cast.end");
429
430
5
    llvm::Value *IsNull = Builder.CreateIsNull(BaseAddr.getPointer());
431
5
    Builder.CreateCondBr(IsNull, CastNull, CastNotNull);
432
5
    EmitBlock(CastNotNull);
433
5
  }
434
435
  // Apply the offset.
436
11
  llvm::Value *Value = Builder.CreateBitCast(BaseAddr.getPointer(), Int8PtrTy);
437
11
  Value = Builder.CreateInBoundsGEP(
438
11
      Int8Ty, Value, Builder.CreateNeg(NonVirtualOffset), "sub.ptr");
439
440
  // Just cast.
441
11
  Value = Builder.CreateBitCast(Value, DerivedPtrTy);
442
443
  // Produce a PHI if we had a null-check.
444
11
  if (NullCheckValue) {
445
5
    Builder.CreateBr(CastEnd);
446
5
    EmitBlock(CastNull);
447
5
    Builder.CreateBr(CastEnd);
448
5
    EmitBlock(CastEnd);
449
450
5
    llvm::PHINode *PHI = Builder.CreatePHI(Value->getType(), 2);
451
5
    PHI->addIncoming(Value, CastNotNull);
452
5
    PHI->addIncoming(llvm::Constant::getNullValue(Value->getType()), CastNull);
453
5
    Value = PHI;
454
5
  }
455
456
11
  return Address(Value, CGM.getClassPointerAlignment(Derived));
457
731
}
458
459
llvm::Value *CodeGenFunction::GetVTTParameter(GlobalDecl GD,
460
                                              bool ForVirtualBase,
461
29.3k
                                              bool Delegating) {
462
29.3k
  if (!CGM.getCXXABI().NeedsVTTParameter(GD)) {
463
    // This constructor/destructor does not need a VTT parameter.
464
29.0k
    return nullptr;
465
29.0k
  }
466
467
276
  const CXXRecordDecl *RD = cast<CXXMethodDecl>(CurCodeDecl)->getParent();
468
276
  const CXXRecordDecl *Base = cast<CXXMethodDecl>(GD.getDecl())->getParent();
469
470
276
  uint64_t SubVTTIndex;
471
472
276
  if (Delegating) {
473
    // If this is a delegating constructor call, just load the VTT.
474
2
    return LoadCXXVTT();
475
274
  } else if (RD == Base) {
476
    // If the record matches the base, this is the complete ctor/dtor
477
    // variant calling the base variant in a class with virtual bases.
478
95
    assert(!CGM.getCXXABI().NeedsVTTParameter(CurGD) &&
479
95
           "doing no-op VTT offset in base dtor/ctor?");
480
0
    assert(!ForVirtualBase && "Can't have same class as virtual base!");
481
0
    SubVTTIndex = 0;
482
179
  } else {
483
179
    const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
484
179
    CharUnits BaseOffset = ForVirtualBase ?
485
7
      Layout.getVBaseClassOffset(Base) :
486
179
      
Layout.getBaseClassOffset(Base)172
;
487
488
179
    SubVTTIndex =
489
179
      CGM.getVTables().getSubVTTIndex(RD, BaseSubobject(Base, BaseOffset));
490
179
    assert(SubVTTIndex != 0 && "Sub-VTT index must be greater than zero!");
491
179
  }
492
493
274
  if (CGM.getCXXABI().NeedsVTTParameter(CurGD)) {
494
    // A VTT parameter was passed to the constructor, use it.
495
41
    llvm::Value *VTT = LoadCXXVTT();
496
41
    return Builder.CreateConstInBoundsGEP1_64(VoidPtrTy, VTT, SubVTTIndex);
497
233
  } else {
498
    // We're the complete constructor, so get the VTT by name.
499
233
    llvm::GlobalValue *VTT = CGM.getVTables().GetAddrOfVTT(RD);
500
233
    return Builder.CreateConstInBoundsGEP2_64(
501
233
        VTT->getValueType(), VTT, 0, SubVTTIndex);
502
233
  }
503
274
}
504
505
namespace {
506
  /// Call the destructor for a direct base class.
507
  struct CallBaseDtor final : EHScopeStack::Cleanup {
508
    const CXXRecordDecl *BaseClass;
509
    bool BaseIsVirtual;
510
    CallBaseDtor(const CXXRecordDecl *Base, bool BaseIsVirtual)
511
2.62k
      : BaseClass(Base), BaseIsVirtual(BaseIsVirtual) {}
512
513
1.82k
    void Emit(CodeGenFunction &CGF, Flags flags) override {
514
1.82k
      const CXXRecordDecl *DerivedClass =
515
1.82k
        cast<CXXMethodDecl>(CGF.CurCodeDecl)->getParent();
516
517
1.82k
      const CXXDestructorDecl *D = BaseClass->getDestructor();
518
      // We are already inside a destructor, so presumably the object being
519
      // destroyed should have the expected type.
520
1.82k
      QualType ThisTy = D->getThisObjectType();
521
1.82k
      Address Addr =
522
1.82k
        CGF.GetAddressOfDirectBaseInCompleteClass(CGF.LoadCXXThisAddress(),
523
1.82k
                                                  DerivedClass, BaseClass,
524
1.82k
                                                  BaseIsVirtual);
525
1.82k
      CGF.EmitCXXDestructorCall(D, Dtor_Base, BaseIsVirtual,
526
1.82k
                                /*Delegating=*/false, Addr, ThisTy);
527
1.82k
    }
528
  };
529
530
  /// A visitor which checks whether an initializer uses 'this' in a
531
  /// way which requires the vtable to be properly set.
532
  struct DynamicThisUseChecker : ConstEvaluatedExprVisitor<DynamicThisUseChecker> {
533
    typedef ConstEvaluatedExprVisitor<DynamicThisUseChecker> super;
534
535
    bool UsesThis;
536
537
10.7k
    DynamicThisUseChecker(const ASTContext &C) : super(C), UsesThis(false) {}
538
539
    // Black-list all explicit and implicit references to 'this'.
540
    //
541
    // Do we need to worry about external references to 'this' derived
542
    // from arbitrary code?  If so, then anything which runs arbitrary
543
    // external code might potentially access the vtable.
544
12
    void VisitCXXThisExpr(const CXXThisExpr *E) { UsesThis = true; }
545
  };
546
} // end anonymous namespace
547
548
10.7k
static bool BaseInitializerUsesThis(ASTContext &C, const Expr *Init) {
549
10.7k
  DynamicThisUseChecker Checker(C);
550
10.7k
  Checker.Visit(Init);
551
10.7k
  return Checker.UsesThis;
552
10.7k
}
553
554
static void EmitBaseInitializer(CodeGenFunction &CGF,
555
                                const CXXRecordDecl *ClassDecl,
556
10.7k
                                CXXCtorInitializer *BaseInit) {
557
10.7k
  assert(BaseInit->isBaseInitializer() &&
558
10.7k
         "Must have base initializer!");
559
560
0
  Address ThisPtr = CGF.LoadCXXThisAddress();
561
562
10.7k
  const Type *BaseType = BaseInit->getBaseClass();
563
10.7k
  const auto *BaseClassDecl =
564
10.7k
      cast<CXXRecordDecl>(BaseType->castAs<RecordType>()->getDecl());
565
566
10.7k
  bool isBaseVirtual = BaseInit->isBaseVirtual();
567
568
  // If the initializer for the base (other than the constructor
569
  // itself) accesses 'this' in any way, we need to initialize the
570
  // vtables.
571
10.7k
  if (BaseInitializerUsesThis(CGF.getContext(), BaseInit->getInit()))
572
12
    CGF.InitializeVTablePointers(ClassDecl);
573
574
  // We can pretend to be a complete class because it only matters for
575
  // virtual bases, and we only do virtual bases for complete ctors.
576
10.7k
  Address V =
577
10.7k
    CGF.GetAddressOfDirectBaseInCompleteClass(ThisPtr, ClassDecl,
578
10.7k
                                              BaseClassDecl,
579
10.7k
                                              isBaseVirtual);
580
10.7k
  AggValueSlot AggSlot =
581
10.7k
      AggValueSlot::forAddr(
582
10.7k
          V, Qualifiers(),
583
10.7k
          AggValueSlot::IsDestructed,
584
10.7k
          AggValueSlot::DoesNotNeedGCBarriers,
585
10.7k
          AggValueSlot::IsNotAliased,
586
10.7k
          CGF.getOverlapForBaseInit(ClassDecl, BaseClassDecl, isBaseVirtual));
587
588
10.7k
  CGF.EmitAggExpr(BaseInit->getInit(), AggSlot);
589
590
10.7k
  if (CGF.CGM.getLangOpts().Exceptions &&
591
10.7k
      
!BaseClassDecl->hasTrivialDestructor()8.45k
)
592
1.23k
    CGF.EHStack.pushCleanup<CallBaseDtor>(EHCleanup, BaseClassDecl,
593
1.23k
                                          isBaseVirtual);
594
10.7k
}
595
596
99.7k
static bool isMemcpyEquivalentSpecialMember(const CXXMethodDecl *D) {
597
99.7k
  auto *CD = dyn_cast<CXXConstructorDecl>(D);
598
99.7k
  if (!(CD && 
CD->isCopyOrMoveConstructor()99.4k
) &&
599
99.7k
      
!D->isCopyAssignmentOperator()89.2k
&&
!D->isMoveAssignmentOperator()88.9k
)
600
88.8k
    return false;
601
602
  // We can emit a memcpy for a trivial copy or move constructor/assignment.
603
10.8k
  if (D->isTrivial() && 
!D->getParent()->mayInsertExtraPadding()6.00k
)
604
6.00k
    return true;
605
606
  // We *must* emit a memcpy for a defaulted union copy or move op.
607
4.87k
  if (D->getParent()->isUnion() && 
D->isDefaulted()4
)
608
2
    return true;
609
610
4.87k
  return false;
611
4.87k
}
612
613
static void EmitLValueForAnyFieldInitialization(CodeGenFunction &CGF,
614
                                                CXXCtorInitializer *MemberInit,
615
16.4k
                                                LValue &LHS) {
616
16.4k
  FieldDecl *Field = MemberInit->getAnyMember();
617
16.4k
  if (MemberInit->isIndirectMemberInitializer()) {
618
    // If we are initializing an anonymous union field, drill down to the field.
619
46
    IndirectFieldDecl *IndirectField = MemberInit->getIndirectMember();
620
46
    for (const auto *I : IndirectField->chain())
621
100
      LHS = CGF.EmitLValueForFieldInitialization(LHS, cast<FieldDecl>(I));
622
16.3k
  } else {
623
16.3k
    LHS = CGF.EmitLValueForFieldInitialization(LHS, Field);
624
16.3k
  }
625
16.4k
}
626
627
static void EmitMemberInitializer(CodeGenFunction &CGF,
628
                                  const CXXRecordDecl *ClassDecl,
629
                                  CXXCtorInitializer *MemberInit,
630
                                  const CXXConstructorDecl *Constructor,
631
16.4k
                                  FunctionArgList &Args) {
632
16.4k
  ApplyDebugLocation Loc(CGF, MemberInit->getSourceLocation());
633
16.4k
  assert(MemberInit->isAnyMemberInitializer() &&
634
16.4k
         "Must have member initializer!");
635
0
  assert(MemberInit->getInit() && "Must have initializer!");
636
637
  // non-static data member initializers.
638
0
  FieldDecl *Field = MemberInit->getAnyMember();
639
16.4k
  QualType FieldType = Field->getType();
640
641
16.4k
  llvm::Value *ThisPtr = CGF.LoadCXXThis();
642
16.4k
  QualType RecordTy = CGF.getContext().getTypeDeclType(ClassDecl);
643
16.4k
  LValue LHS;
644
645
  // If a base constructor is being emitted, create an LValue that has the
646
  // non-virtual alignment.
647
16.4k
  if (CGF.CurGD.getCtorType() == Ctor_Base)
648
16.2k
    LHS = CGF.MakeNaturalAlignPointeeAddrLValue(ThisPtr, RecordTy);
649
185
  else
650
185
    LHS = CGF.MakeNaturalAlignAddrLValue(ThisPtr, RecordTy);
651
652
16.4k
  EmitLValueForAnyFieldInitialization(CGF, MemberInit, LHS);
653
654
  // Special case: if we are in a copy or move constructor, and we are copying
655
  // an array of PODs or classes with trivial copy constructors, ignore the
656
  // AST and perform the copy we know is equivalent.
657
  // FIXME: This is hacky at best... if we had a bit more explicit information
658
  // in the AST, we could generalize it more easily.
659
16.4k
  const ConstantArrayType *Array
660
16.4k
    = CGF.getContext().getAsConstantArrayType(FieldType);
661
16.4k
  if (Array && 
Constructor->isDefaulted()79
&&
662
16.4k
      
Constructor->isCopyOrMoveConstructor()40
) {
663
23
    QualType BaseElementTy = CGF.getContext().getBaseElementType(Array);
664
23
    CXXConstructExpr *CE = dyn_cast<CXXConstructExpr>(MemberInit->getInit());
665
23
    if (BaseElementTy.isPODType(CGF.getContext()) ||
666
23
        
(18
CE18
&&
isMemcpyEquivalentSpecialMember(CE->getConstructor())0
)) {
667
5
      unsigned SrcArgIndex =
668
5
          CGF.CGM.getCXXABI().getSrcArgforCopyCtor(Constructor, Args);
669
5
      llvm::Value *SrcPtr
670
5
        = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(Args[SrcArgIndex]));
671
5
      LValue ThisRHSLV = CGF.MakeNaturalAlignAddrLValue(SrcPtr, RecordTy);
672
5
      LValue Src = CGF.EmitLValueForFieldInitialization(ThisRHSLV, Field);
673
674
      // Copy the aggregate.
675
5
      CGF.EmitAggregateCopy(LHS, Src, FieldType, CGF.getOverlapForFieldInit(Field),
676
5
                            LHS.isVolatileQualified());
677
      // Ensure that we destroy the objects if an exception is thrown later in
678
      // the constructor.
679
5
      QualType::DestructionKind dtorKind = FieldType.isDestructedType();
680
5
      if (CGF.needsEHCleanup(dtorKind))
681
0
        CGF.pushEHDestroy(dtorKind, LHS.getAddress(CGF), FieldType);
682
5
      return;
683
5
    }
684
23
  }
685
686
16.4k
  CGF.EmitInitializerForField(Field, LHS, MemberInit->getInit());
687
16.4k
}
688
689
void CodeGenFunction::EmitInitializerForField(FieldDecl *Field, LValue LHS,
690
17.7k
                                              Expr *Init) {
691
17.7k
  QualType FieldType = Field->getType();
692
17.7k
  switch (getEvaluationKind(FieldType)) {
693
13.0k
  case TEK_Scalar:
694
13.0k
    if (LHS.isSimple()) {
695
12.9k
      EmitExprAsInit(Init, Field, LHS, false);
696
12.9k
    } else {
697
82
      RValue RHS = RValue::get(EmitScalarExpr(Init));
698
82
      EmitStoreThroughLValue(RHS, LHS);
699
82
    }
700
13.0k
    break;
701
6
  case TEK_Complex:
702
6
    EmitComplexExprIntoLValue(Init, LHS, /*isInit*/ true);
703
6
    break;
704
4.65k
  case TEK_Aggregate: {
705
4.65k
    AggValueSlot Slot = AggValueSlot::forLValue(
706
4.65k
        LHS, *this, AggValueSlot::IsDestructed,
707
4.65k
        AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased,
708
4.65k
        getOverlapForFieldInit(Field), AggValueSlot::IsNotZeroed,
709
        // Checks are made by the code that calls constructor.
710
4.65k
        AggValueSlot::IsSanitizerChecked);
711
4.65k
    EmitAggExpr(Init, Slot);
712
4.65k
    break;
713
0
  }
714
17.7k
  }
715
716
  // Ensure that we destroy this object if an exception is thrown
717
  // later in the constructor.
718
17.7k
  QualType::DestructionKind dtorKind = FieldType.isDestructedType();
719
17.7k
  if (needsEHCleanup(dtorKind))
720
884
    pushEHDestroy(dtorKind, LHS.getAddress(*this), FieldType);
721
17.7k
}
722
723
/// Checks whether the given constructor is a valid subject for the
724
/// complete-to-base constructor delegation optimization, i.e.
725
/// emitting the complete constructor as a simple call to the base
726
/// constructor.
727
bool CodeGenFunction::IsConstructorDelegationValid(
728
17.8k
    const CXXConstructorDecl *Ctor) {
729
730
  // Currently we disable the optimization for classes with virtual
731
  // bases because (1) the addresses of parameter variables need to be
732
  // consistent across all initializers but (2) the delegate function
733
  // call necessarily creates a second copy of the parameter variable.
734
  //
735
  // The limiting example (purely theoretical AFAIK):
736
  //   struct A { A(int &c) { c++; } };
737
  //   struct B : virtual A {
738
  //     B(int count) : A(count) { printf("%d\n", count); }
739
  //   };
740
  // ...although even this example could in principle be emitted as a
741
  // delegation since the address of the parameter doesn't escape.
742
17.8k
  if (Ctor->getParent()->getNumVBases()) {
743
    // TODO: white-list trivial vbase initializers.  This case wouldn't
744
    // be subject to the restrictions below.
745
746
    // TODO: white-list cases where:
747
    //  - there are no non-reference parameters to the constructor
748
    //  - the initializers don't access any non-reference parameters
749
    //  - the initializers don't take the address of non-reference
750
    //    parameters
751
    //  - etc.
752
    // If we ever add any of the above cases, remember that:
753
    //  - function-try-blocks will always exclude this optimization
754
    //  - we need to perform the constructor prologue and cleanup in
755
    //    EmitConstructorBody.
756
757
590
    return false;
758
590
  }
759
760
  // We also disable the optimization for variadic functions because
761
  // it's impossible to "re-pass" varargs.
762
17.2k
  if (Ctor->getType()->castAs<FunctionProtoType>()->isVariadic())
763
10
    return false;
764
765
  // FIXME: Decide if we can do a delegation of a delegating constructor.
766
17.2k
  if (Ctor->isDelegatingConstructor())
767
87
    return false;
768
769
17.1k
  return true;
770
17.2k
}
771
772
// Emit code in ctor (Prologue==true) or dtor (Prologue==false)
773
// to poison the extra field paddings inserted under
774
// -fsanitize-address-field-padding=1|2.
775
56.6k
void CodeGenFunction::EmitAsanPrologueOrEpilogue(bool Prologue) {
776
56.6k
  ASTContext &Context = getContext();
777
56.6k
  const CXXRecordDecl *ClassDecl =
778
56.6k
      Prologue ? 
cast<CXXConstructorDecl>(CurGD.getDecl())->getParent()40.7k
779
56.6k
               : 
cast<CXXDestructorDecl>(CurGD.getDecl())->getParent()15.8k
;
780
56.6k
  if (!ClassDecl->mayInsertExtraPadding()) 
return56.5k
;
781
782
41
  struct SizeAndOffset {
783
41
    uint64_t Size;
784
41
    uint64_t Offset;
785
41
  };
786
787
41
  unsigned PtrSize = CGM.getDataLayout().getPointerSizeInBits();
788
41
  const ASTRecordLayout &Info = Context.getASTRecordLayout(ClassDecl);
789
790
  // Populate sizes and offsets of fields.
791
41
  SmallVector<SizeAndOffset, 16> SSV(Info.getFieldCount());
792
153
  for (unsigned i = 0, e = Info.getFieldCount(); i != e; 
++i112
)
793
112
    SSV[i].Offset =
794
112
        Context.toCharUnitsFromBits(Info.getFieldOffset(i)).getQuantity();
795
796
41
  size_t NumFields = 0;
797
112
  for (const auto *Field : ClassDecl->fields()) {
798
112
    const FieldDecl *D = Field;
799
112
    auto FieldInfo = Context.getTypeInfoInChars(D->getType());
800
112
    CharUnits FieldSize = FieldInfo.Width;
801
112
    assert(NumFields < SSV.size());
802
112
    SSV[NumFields].Size = D->isBitField() ? 
00
: FieldSize.getQuantity();
803
112
    NumFields++;
804
112
  }
805
41
  assert(NumFields == SSV.size());
806
41
  if (SSV.size() <= 1) 
return0
;
807
808
  // We will insert calls to __asan_* run-time functions.
809
  // LLVM AddressSanitizer pass may decide to inline them later.
810
41
  llvm::Type *Args[2] = {IntPtrTy, IntPtrTy};
811
41
  llvm::FunctionType *FTy =
812
41
      llvm::FunctionType::get(CGM.VoidTy, Args, false);
813
41
  llvm::FunctionCallee F = CGM.CreateRuntimeFunction(
814
41
      FTy, Prologue ? 
"__asan_poison_intra_object_redzone"22
815
41
                    : 
"__asan_unpoison_intra_object_redzone"19
);
816
817
41
  llvm::Value *ThisPtr = LoadCXXThis();
818
41
  ThisPtr = Builder.CreatePtrToInt(ThisPtr, IntPtrTy);
819
41
  uint64_t TypeSize = Info.getNonVirtualSize().getQuantity();
820
  // For each field check if it has sufficient padding,
821
  // if so (un)poison it with a call.
822
153
  for (size_t i = 0; i < SSV.size(); 
i++112
) {
823
112
    uint64_t AsanAlignment = 8;
824
112
    uint64_t NextField = i == SSV.size() - 1 ? 
TypeSize41
:
SSV[i + 1].Offset71
;
825
112
    uint64_t PoisonSize = NextField - SSV[i].Offset - SSV[i].Size;
826
112
    uint64_t EndOffset = SSV[i].Offset + SSV[i].Size;
827
112
    if (PoisonSize < AsanAlignment || 
!SSV[i].Size94
||
828
112
        
(NextField % AsanAlignment) != 094
)
829
18
      continue;
830
94
    Builder.CreateCall(
831
94
        F, {Builder.CreateAdd(ThisPtr, Builder.getIntN(PtrSize, EndOffset)),
832
94
            Builder.getIntN(PtrSize, PoisonSize)});
833
94
  }
834
41
}
835
836
/// EmitConstructorBody - Emits the body of the current constructor.
837
40.7k
void CodeGenFunction::EmitConstructorBody(FunctionArgList &Args) {
838
40.7k
  EmitAsanPrologueOrEpilogue(true);
839
40.7k
  const CXXConstructorDecl *Ctor = cast<CXXConstructorDecl>(CurGD.getDecl());
840
40.7k
  CXXCtorType CtorType = CurGD.getCtorType();
841
842
40.7k
  assert((CGM.getTarget().getCXXABI().hasConstructorVariants() ||
843
40.7k
          CtorType == Ctor_Complete) &&
844
40.7k
         "can only generate complete ctor for this ABI");
845
846
  // Before we go any further, try the complete->base constructor
847
  // delegation optimization.
848
40.7k
  if (CtorType == Ctor_Complete && 
IsConstructorDelegationValid(Ctor)17.7k
&&
849
40.7k
      
CGM.getTarget().getCXXABI().hasConstructorVariants()17.1k
) {
850
16.2k
    EmitDelegateCXXConstructorCall(Ctor, Ctor_Base, Args, Ctor->getEndLoc());
851
16.2k
    return;
852
16.2k
  }
853
854
24.5k
  const FunctionDecl *Definition = nullptr;
855
24.5k
  Stmt *Body = Ctor->getBody(Definition);
856
24.5k
  assert(Definition == Ctor && "emitting wrong constructor body");
857
858
  // Enter the function-try-block before the constructor prologue if
859
  // applicable.
860
24.5k
  bool IsTryBody = (Body && isa<CXXTryStmt>(Body));
861
24.5k
  if (IsTryBody)
862
2
    EnterCXXTryStmt(*cast<CXXTryStmt>(Body), true);
863
864
24.5k
  incrementProfileCounter(Body);
865
866
24.5k
  RunCleanupsScope RunCleanups(*this);
867
868
  // TODO: in restricted cases, we can emit the vbase initializers of
869
  // a complete ctor and then delegate to the base ctor.
870
871
  // Emit the constructor prologue, i.e. the base and member
872
  // initializers.
873
24.5k
  EmitCtorPrologue(Ctor, CtorType, Args);
874
875
  // Emit the body of the statement.
876
24.5k
  if (IsTryBody)
877
2
    EmitStmt(cast<CXXTryStmt>(Body)->getTryBlock());
878
24.5k
  else if (Body)
879
24.5k
    EmitStmt(Body);
880
881
  // Emit any cleanup blocks associated with the member or base
882
  // initializers, which includes (along the exceptional path) the
883
  // destructors for those members and bases that were fully
884
  // constructed.
885
24.5k
  RunCleanups.ForceCleanup();
886
887
24.5k
  if (IsTryBody)
888
2
    ExitCXXTryStmt(*cast<CXXTryStmt>(Body), true);
889
24.5k
}
890
891
namespace {
892
  /// RAII object to indicate that codegen is copying the value representation
893
  /// instead of the object representation. Useful when copying a struct or
894
  /// class which has uninitialized members and we're only performing
895
  /// lvalue-to-rvalue conversion on the object but not its members.
896
  class CopyingValueRepresentation {
897
  public:
898
    explicit CopyingValueRepresentation(CodeGenFunction &CGF)
899
160
        : CGF(CGF), OldSanOpts(CGF.SanOpts) {
900
160
      CGF.SanOpts.set(SanitizerKind::Bool, false);
901
160
      CGF.SanOpts.set(SanitizerKind::Enum, false);
902
160
    }
903
160
    ~CopyingValueRepresentation() {
904
160
      CGF.SanOpts = OldSanOpts;
905
160
    }
906
  private:
907
    CodeGenFunction &CGF;
908
    SanitizerSet OldSanOpts;
909
  };
910
} // end anonymous namespace
911
912
namespace {
913
  class FieldMemcpyizer {
914
  public:
915
    FieldMemcpyizer(CodeGenFunction &CGF, const CXXRecordDecl *ClassDecl,
916
                    const VarDecl *SrcRec)
917
      : CGF(CGF), ClassDecl(ClassDecl), SrcRec(SrcRec),
918
        RecLayout(CGF.getContext().getASTRecordLayout(ClassDecl)),
919
        FirstField(nullptr), LastField(nullptr), FirstFieldOffset(0),
920
25.2k
        LastFieldOffset(0), LastAddedFieldIndex(0) {}
921
922
537
    bool isMemcpyableField(FieldDecl *F) const {
923
      // Never memcpy fields when we are adding poisoned paddings.
924
537
      if (CGF.getContext().getLangOpts().SanitizeAddressFieldPadding)
925
8
        return false;
926
529
      Qualifiers Qual = F->getType().getQualifiers();
927
529
      if (Qual.hasVolatile() || 
Qual.hasObjCLifetime()525
)
928
8
        return false;
929
521
      return true;
930
529
    }
931
932
518
    void addMemcpyableField(FieldDecl *F) {
933
518
      if (F->isZeroSize(CGF.getContext()))
934
1
        return;
935
517
      if (!FirstField)
936
256
        addInitialField(F);
937
261
      else
938
261
        addNextField(F);
939
517
    }
940
941
96
    CharUnits getMemcpySize(uint64_t FirstByteOffset) const {
942
96
      ASTContext &Ctx = CGF.getContext();
943
96
      unsigned LastFieldSize =
944
96
          LastField->isBitField()
945
96
              ? 
LastField->getBitWidthValue(Ctx)6
946
96
              : Ctx.toBits(
947
90
                    Ctx.getTypeInfoDataSizeInChars(LastField->getType()).Width);
948
96
      uint64_t MemcpySizeBits = LastFieldOffset + LastFieldSize -
949
96
                                FirstByteOffset + Ctx.getCharWidth() - 1;
950
96
      CharUnits MemcpySize = Ctx.toCharUnitsFromBits(MemcpySizeBits);
951
96
      return MemcpySize;
952
96
    }
953
954
1.82k
    void emitMemcpy() {
955
      // Give the subclass a chance to bail out if it feels the memcpy isn't
956
      // worth it (e.g. Hasn't aggregated enough data).
957
1.82k
      if (!FirstField) {
958
1.72k
        return;
959
1.72k
      }
960
961
96
      uint64_t FirstByteOffset;
962
96
      if (FirstField->isBitField()) {
963
12
        const CGRecordLayout &RL =
964
12
          CGF.getTypes().getCGRecordLayout(FirstField->getParent());
965
12
        const CGBitFieldInfo &BFInfo = RL.getBitFieldInfo(FirstField);
966
        // FirstFieldOffset is not appropriate for bitfields,
967
        // we need to use the storage offset instead.
968
12
        FirstByteOffset = CGF.getContext().toBits(BFInfo.StorageOffset);
969
84
      } else {
970
84
        FirstByteOffset = FirstFieldOffset;
971
84
      }
972
973
96
      CharUnits MemcpySize = getMemcpySize(FirstByteOffset);
974
96
      QualType RecordTy = CGF.getContext().getTypeDeclType(ClassDecl);
975
96
      Address ThisPtr = CGF.LoadCXXThisAddress();
976
96
      LValue DestLV = CGF.MakeAddrLValue(ThisPtr, RecordTy);
977
96
      LValue Dest = CGF.EmitLValueForFieldInitialization(DestLV, FirstField);
978
96
      llvm::Value *SrcPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(SrcRec));
979
96
      LValue SrcLV = CGF.MakeNaturalAlignAddrLValue(SrcPtr, RecordTy);
980
96
      LValue Src = CGF.EmitLValueForFieldInitialization(SrcLV, FirstField);
981
982
96
      emitMemcpyIR(
983
96
          Dest.isBitField() ? 
Dest.getBitFieldAddress()12
:
Dest.getAddress(CGF)84
,
984
96
          Src.isBitField() ? 
Src.getBitFieldAddress()12
:
Src.getAddress(CGF)84
,
985
96
          MemcpySize);
986
96
      reset();
987
96
    }
988
989
42.6k
    void reset() {
990
42.6k
      FirstField = nullptr;
991
42.6k
    }
992
993
  protected:
994
    CodeGenFunction &CGF;
995
    const CXXRecordDecl *ClassDecl;
996
997
  private:
998
96
    void emitMemcpyIR(Address DestPtr, Address SrcPtr, CharUnits Size) {
999
96
      llvm::PointerType *DPT = DestPtr.getType();
1000
96
      llvm::Type *DBP =
1001
96
        llvm::Type::getInt8PtrTy(CGF.getLLVMContext(), DPT->getAddressSpace());
1002
96
      DestPtr = CGF.Builder.CreateBitCast(DestPtr, DBP);
1003
1004
96
      llvm::PointerType *SPT = SrcPtr.getType();
1005
96
      llvm::Type *SBP =
1006
96
        llvm::Type::getInt8PtrTy(CGF.getLLVMContext(), SPT->getAddressSpace());
1007
96
      SrcPtr = CGF.Builder.CreateBitCast(SrcPtr, SBP);
1008
1009
96
      CGF.Builder.CreateMemCpy(DestPtr, SrcPtr, Size.getQuantity());
1010
96
    }
1011
1012
256
    void addInitialField(FieldDecl *F) {
1013
256
      FirstField = F;
1014
256
      LastField = F;
1015
256
      FirstFieldOffset = RecLayout.getFieldOffset(F->getFieldIndex());
1016
256
      LastFieldOffset = FirstFieldOffset;
1017
256
      LastAddedFieldIndex = F->getFieldIndex();
1018
256
    }
1019
1020
261
    void addNextField(FieldDecl *F) {
1021
      // For the most part, the following invariant will hold:
1022
      //   F->getFieldIndex() == LastAddedFieldIndex + 1
1023
      // The one exception is that Sema won't add a copy-initializer for an
1024
      // unnamed bitfield, which will show up here as a gap in the sequence.
1025
261
      assert(F->getFieldIndex() >= LastAddedFieldIndex + 1 &&
1026
261
             "Cannot aggregate fields out of order.");
1027
0
      LastAddedFieldIndex = F->getFieldIndex();
1028
1029
      // The 'first' and 'last' fields are chosen by offset, rather than field
1030
      // index. This allows the code to support bitfields, as well as regular
1031
      // fields.
1032
261
      uint64_t FOffset = RecLayout.getFieldOffset(F->getFieldIndex());
1033
261
      if (FOffset < FirstFieldOffset) {
1034
0
        FirstField = F;
1035
0
        FirstFieldOffset = FOffset;
1036
261
      } else if (FOffset >= LastFieldOffset) {
1037
261
        LastField = F;
1038
261
        LastFieldOffset = FOffset;
1039
261
      }
1040
261
    }
1041
1042
    const VarDecl *SrcRec;
1043
    const ASTRecordLayout &RecLayout;
1044
    FieldDecl *FirstField;
1045
    FieldDecl *LastField;
1046
    uint64_t FirstFieldOffset, LastFieldOffset;
1047
    unsigned LastAddedFieldIndex;
1048
  };
1049
1050
  class ConstructorMemcpyizer : public FieldMemcpyizer {
1051
  private:
1052
    /// Get source argument for copy constructor. Returns null if not a copy
1053
    /// constructor.
1054
    static const VarDecl *getTrivialCopySource(CodeGenFunction &CGF,
1055
                                               const CXXConstructorDecl *CD,
1056
24.4k
                                               FunctionArgList &Args) {
1057
24.4k
      if (CD->isCopyOrMoveConstructor() && 
CD->isDefaulted()1.02k
)
1058
369
        return Args[CGF.CGM.getCXXABI().getSrcArgforCopyCtor(CD, Args)];
1059
24.1k
      return nullptr;
1060
24.4k
    }
1061
1062
    // Returns true if a CXXCtorInitializer represents a member initialization
1063
    // that can be rolled into a memcpy.
1064
16.6k
    bool isMemberInitMemcpyable(CXXCtorInitializer *MemberInit) const {
1065
16.6k
      if (!MemcpyableCtor)
1066
16.2k
        return false;
1067
420
      FieldDecl *Field = MemberInit->getMember();
1068
420
      assert(Field && "No field for member init.");
1069
0
      QualType FieldType = Field->getType();
1070
420
      CXXConstructExpr *CE = dyn_cast<CXXConstructExpr>(MemberInit->getInit());
1071
1072
      // Bail out on non-memcpyable, not-trivially-copyable members.
1073
420
      if (!(CE && 
isMemcpyEquivalentSpecialMember(CE->getConstructor())138
) &&
1074
420
          
!(399
FieldType.isTriviallyCopyableType(CGF.getContext())399
||
1075
399
            
FieldType->isReferenceType()167
))
1076
159
        return false;
1077
1078
      // Bail out on volatile fields.
1079
261
      if (!isMemcpyableField(Field))
1080
6
        return false;
1081
1082
      // Otherwise we're good.
1083
255
      return true;
1084
261
    }
1085
1086
  public:
1087
    ConstructorMemcpyizer(CodeGenFunction &CGF, const CXXConstructorDecl *CD,
1088
                          FunctionArgList &Args)
1089
      : FieldMemcpyizer(CGF, CD->getParent(), getTrivialCopySource(CGF, CD, Args)),
1090
        ConstructorDecl(CD),
1091
        MemcpyableCtor(CD->isDefaulted() &&
1092
                       CD->isCopyOrMoveConstructor() &&
1093
                       CGF.getLangOpts().getGC() == LangOptions::NonGC),
1094
24.4k
        Args(Args) { }
1095
1096
16.6k
    void addMemberInitializer(CXXCtorInitializer *MemberInit) {
1097
16.6k
      if (isMemberInitMemcpyable(MemberInit)) {
1098
255
        AggregatedInits.push_back(MemberInit);
1099
255
        addMemcpyableField(MemberInit->getMember());
1100
16.3k
      } else {
1101
16.3k
        emitAggregatedInits();
1102
16.3k
        EmitMemberInitializer(CGF, ConstructorDecl->getParent(), MemberInit,
1103
16.3k
                              ConstructorDecl, Args);
1104
16.3k
      }
1105
16.6k
    }
1106
1107
40.8k
    void emitAggregatedInits() {
1108
40.8k
      if (AggregatedInits.size() <= 1) {
1109
        // This memcpy is too small to be worthwhile. Fall back on default
1110
        // codegen.
1111
40.7k
        if (!AggregatedInits.empty()) {
1112
60
          CopyingValueRepresentation CVR(CGF);
1113
60
          EmitMemberInitializer(CGF, ConstructorDecl->getParent(),
1114
60
                                AggregatedInits[0], ConstructorDecl, Args);
1115
60
          AggregatedInits.clear();
1116
60
        }
1117
40.7k
        reset();
1118
40.7k
        return;
1119
40.7k
      }
1120
1121
55
      pushEHDestructors();
1122
55
      emitMemcpy();
1123
55
      AggregatedInits.clear();
1124
55
    }
1125
1126
55
    void pushEHDestructors() {
1127
55
      Address ThisPtr = CGF.LoadCXXThisAddress();
1128
55
      QualType RecordTy = CGF.getContext().getTypeDeclType(ClassDecl);
1129
55
      LValue LHS = CGF.MakeAddrLValue(ThisPtr, RecordTy);
1130
1131
250
      for (unsigned i = 0; i < AggregatedInits.size(); 
++i195
) {
1132
195
        CXXCtorInitializer *MemberInit = AggregatedInits[i];
1133
195
        QualType FieldType = MemberInit->getAnyMember()->getType();
1134
195
        QualType::DestructionKind dtorKind = FieldType.isDestructedType();
1135
195
        if (!CGF.needsEHCleanup(dtorKind))
1136
193
          continue;
1137
2
        LValue FieldLHS = LHS;
1138
2
        EmitLValueForAnyFieldInitialization(CGF, MemberInit, FieldLHS);
1139
2
        CGF.pushEHDestroy(dtorKind, FieldLHS.getAddress(CGF), FieldType);
1140
2
      }
1141
55
    }
1142
1143
24.4k
    void finish() {
1144
24.4k
      emitAggregatedInits();
1145
24.4k
    }
1146
1147
  private:
1148
    const CXXConstructorDecl *ConstructorDecl;
1149
    bool MemcpyableCtor;
1150
    FunctionArgList &Args;
1151
    SmallVector<CXXCtorInitializer*, 16> AggregatedInits;
1152
  };
1153
1154
  class AssignmentMemcpyizer : public FieldMemcpyizer {
1155
  private:
1156
    // Returns the memcpyable field copied by the given statement, if one
1157
    // exists. Otherwise returns null.
1158
1.30k
    FieldDecl *getMemcpyableField(Stmt *S) {
1159
1.30k
      if (!AssignmentsMemcpyable)
1160
11
        return nullptr;
1161
1.29k
      if (BinaryOperator *BO = dyn_cast<BinaryOperator>(S)) {
1162
        // Recognise trivial assignments.
1163
226
        if (BO->getOpcode() != BO_Assign)
1164
0
          return nullptr;
1165
226
        MemberExpr *ME = dyn_cast<MemberExpr>(BO->getLHS());
1166
226
        if (!ME)
1167
0
          return nullptr;
1168
226
        FieldDecl *Field = dyn_cast<FieldDecl>(ME->getMemberDecl());
1169
226
        if (!Field || !isMemcpyableField(Field))
1170
10
          return nullptr;
1171
216
        Stmt *RHS = BO->getRHS();
1172
216
        if (ImplicitCastExpr *EC = dyn_cast<ImplicitCastExpr>(RHS))
1173
216
          RHS = EC->getSubExpr();
1174
216
        if (!RHS)
1175
0
          return nullptr;
1176
216
        if (MemberExpr *ME2 = dyn_cast<MemberExpr>(RHS)) {
1177
214
          if (ME2->getMemberDecl() == Field)
1178
214
            return Field;
1179
214
        }
1180
2
        return nullptr;
1181
1.06k
      } else if (CXXMemberCallExpr *MCE = dyn_cast<CXXMemberCallExpr>(S)) {
1182
313
        CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(MCE->getCalleeDecl());
1183
313
        if (!(MD && isMemcpyEquivalentSpecialMember(MD)))
1184
128
          return nullptr;
1185
185
        MemberExpr *IOA = dyn_cast<MemberExpr>(MCE->getImplicitObjectArgument());
1186
185
        if (!IOA)
1187
146
          return nullptr;
1188
39
        FieldDecl *Field = dyn_cast<FieldDecl>(IOA->getMemberDecl());
1189
39
        if (!Field || !isMemcpyableField(Field))
1190
0
          return nullptr;
1191
39
        MemberExpr *Arg0 = dyn_cast<MemberExpr>(MCE->getArg(0));
1192
39
        if (!Arg0 || 
Field != dyn_cast<FieldDecl>(Arg0->getMemberDecl())38
)
1193
1
          return nullptr;
1194
38
        return Field;
1195
756
      } else if (CallExpr *CE = dyn_cast<CallExpr>(S)) {
1196
11
        FunctionDecl *FD = dyn_cast<FunctionDecl>(CE->getCalleeDecl());
1197
11
        if (!FD || FD->getBuiltinID() != Builtin::BI__builtin_memcpy)
1198
0
          return nullptr;
1199
11
        Expr *DstPtr = CE->getArg(0);
1200
11
        if (ImplicitCastExpr *DC = dyn_cast<ImplicitCastExpr>(DstPtr))
1201
11
          DstPtr = DC->getSubExpr();
1202
11
        UnaryOperator *DUO = dyn_cast<UnaryOperator>(DstPtr);
1203
11
        if (!DUO || DUO->getOpcode() != UO_AddrOf)
1204
0
          return nullptr;
1205
11
        MemberExpr *ME = dyn_cast<MemberExpr>(DUO->getSubExpr());
1206
11
        if (!ME)
1207
0
          return nullptr;
1208
11
        FieldDecl *Field = dyn_cast<FieldDecl>(ME->getMemberDecl());
1209
11
        if (!Field || !isMemcpyableField(Field))
1210
0
          return nullptr;
1211
11
        Expr *SrcPtr = CE->getArg(1);
1212
11
        if (ImplicitCastExpr *SC = dyn_cast<ImplicitCastExpr>(SrcPtr))
1213
11
          SrcPtr = SC->getSubExpr();
1214
11
        UnaryOperator *SUO = dyn_cast<UnaryOperator>(SrcPtr);
1215
11
        if (!SUO || SUO->getOpcode() != UO_AddrOf)
1216
0
          return nullptr;
1217
11
        MemberExpr *ME2 = dyn_cast<MemberExpr>(SUO->getSubExpr());
1218
11
        if (!ME2 || Field != dyn_cast<FieldDecl>(ME2->getMemberDecl()))
1219
0
          return nullptr;
1220
11
        return Field;
1221
11
      }
1222
1223
745
      return nullptr;
1224
1.29k
    }
1225
1226
    bool AssignmentsMemcpyable;
1227
    SmallVector<Stmt*, 16> AggregatedStmts;
1228
1229
  public:
1230
    AssignmentMemcpyizer(CodeGenFunction &CGF, const CXXMethodDecl *AD,
1231
                         FunctionArgList &Args)
1232
      : FieldMemcpyizer(CGF, AD->getParent(), Args[Args.size() - 1]),
1233
726
        AssignmentsMemcpyable(CGF.getLangOpts().getGC() == LangOptions::NonGC) {
1234
726
      assert(Args.size() == 2);
1235
726
    }
1236
1237
1.30k
    void emitAssignment(Stmt *S) {
1238
1.30k
      FieldDecl *F = getMemcpyableField(S);
1239
1.30k
      if (F) {
1240
263
        addMemcpyableField(F);
1241
263
        AggregatedStmts.push_back(S);
1242
1.04k
      } else {
1243
1.04k
        emitAggregatedStmts();
1244
1.04k
        CGF.EmitStmt(S);
1245
1.04k
      }
1246
1.30k
    }
1247
1248
1.76k
    void emitAggregatedStmts() {
1249
1.76k
      if (AggregatedStmts.size() <= 1) {
1250
1.72k
        if (!AggregatedStmts.empty()) {
1251
100
          CopyingValueRepresentation CVR(CGF);
1252
100
          CGF.EmitStmt(AggregatedStmts[0]);
1253
100
        }
1254
1.72k
        reset();
1255
1.72k
      }
1256
1257
1.76k
      emitMemcpy();
1258
1.76k
      AggregatedStmts.clear();
1259
1.76k
    }
1260
1261
726
    void finish() {
1262
726
      emitAggregatedStmts();
1263
726
    }
1264
  };
1265
} // end anonymous namespace
1266
1267
27
static bool isInitializerOfDynamicClass(const CXXCtorInitializer *BaseInit) {
1268
27
  const Type *BaseType = BaseInit->getBaseClass();
1269
27
  const auto *BaseClassDecl =
1270
27
      cast<CXXRecordDecl>(BaseType->castAs<RecordType>()->getDecl());
1271
27
  return BaseClassDecl->isDynamicClass();
1272
27
}
1273
1274
/// EmitCtorPrologue - This routine generates necessary code to initialize
1275
/// base classes and non-static data members belonging to this constructor.
1276
void CodeGenFunction::EmitCtorPrologue(const CXXConstructorDecl *CD,
1277
                                       CXXCtorType CtorType,
1278
24.5k
                                       FunctionArgList &Args) {
1279
24.5k
  if (CD->isDelegatingConstructor())
1280
94
    return EmitDelegatingCXXConstructorCall(CD, Args);
1281
1282
24.4k
  const CXXRecordDecl *ClassDecl = CD->getParent();
1283
1284
24.4k
  CXXConstructorDecl::init_const_iterator B = CD->init_begin(),
1285
24.4k
                                          E = CD->init_end();
1286
1287
  // Virtual base initializers first, if any. They aren't needed if:
1288
  // - This is a base ctor variant
1289
  // - There are no vbases
1290
  // - The class is abstract, so a complete object of it cannot be constructed
1291
  //
1292
  // The check for an abstract class is necessary because sema may not have
1293
  // marked virtual base destructors referenced.
1294
24.4k
  bool ConstructVBases = CtorType != Ctor_Base &&
1295
24.4k
                         
ClassDecl->getNumVBases() != 01.49k
&&
1296
24.4k
                         
!ClassDecl->isAbstract()598
;
1297
1298
  // In the Microsoft C++ ABI, there are no constructor variants. Instead, the
1299
  // constructor of a class with virtual bases takes an additional parameter to
1300
  // conditionally construct the virtual bases. Emit that check here.
1301
24.4k
  llvm::BasicBlock *BaseCtorContinueBB = nullptr;
1302
24.4k
  if (ConstructVBases &&
1303
24.4k
      
!CGM.getTarget().getCXXABI().hasConstructorVariants()595
) {
1304
381
    BaseCtorContinueBB =
1305
381
        CGM.getCXXABI().EmitCtorCompleteObjectHandler(*this, ClassDecl);
1306
381
    assert(BaseCtorContinueBB);
1307
381
  }
1308
1309
0
  llvm::Value *const OldThis = CXXThisValue;
1310
25.4k
  for (; B != E && 
(*B)->isBaseInitializer()19.8k
&&
(*B)->isBaseVirtual()8.34k
;
B++952
) {
1311
952
    if (!ConstructVBases)
1312
167
      continue;
1313
785
    if (CGM.getCodeGenOpts().StrictVTablePointers &&
1314
785
        
CGM.getCodeGenOpts().OptimizationLevel > 03
&&
1315
785
        
isInitializerOfDynamicClass(*B)3
)
1316
1
      CXXThisValue = Builder.CreateLaunderInvariantGroup(LoadCXXThis());
1317
785
    EmitBaseInitializer(*this, ClassDecl, *B);
1318
785
  }
1319
1320
24.4k
  if (BaseCtorContinueBB) {
1321
    // Complete object handler should continue to the remaining initializers.
1322
381
    Builder.CreateBr(BaseCtorContinueBB);
1323
381
    EmitBlock(BaseCtorContinueBB);
1324
381
  }
1325
1326
  // Then, non-virtual base initializers.
1327
34.4k
  for (; B != E && 
(*B)->isBaseInitializer()22.5k
;
B++9.93k
) {
1328
9.93k
    assert(!(*B)->isBaseVirtual());
1329
1330
9.93k
    if (CGM.getCodeGenOpts().StrictVTablePointers &&
1331
9.93k
        
CGM.getCodeGenOpts().OptimizationLevel > 024
&&
1332
9.93k
        
isInitializerOfDynamicClass(*B)24
)
1333
23
      CXXThisValue = Builder.CreateLaunderInvariantGroup(LoadCXXThis());
1334
9.93k
    EmitBaseInitializer(*this, ClassDecl, *B);
1335
9.93k
  }
1336
1337
24.4k
  CXXThisValue = OldThis;
1338
1339
24.4k
  InitializeVTablePointers(ClassDecl);
1340
1341
  // And finally, initialize class members.
1342
24.4k
  FieldConstructionScope FCS(*this, LoadCXXThisAddress());
1343
24.4k
  ConstructorMemcpyizer CM(*this, CD, Args);
1344
41.1k
  for (; B != E; 
B++16.6k
) {
1345
16.6k
    CXXCtorInitializer *Member = (*B);
1346
16.6k
    assert(!Member->isBaseInitializer());
1347
0
    assert(Member->isAnyMemberInitializer() &&
1348
16.6k
           "Delegating initializer on non-delegating constructor");
1349
0
    CM.addMemberInitializer(Member);
1350
16.6k
  }
1351
24.4k
  CM.finish();
1352
24.4k
}
1353
1354
static bool
1355
FieldHasTrivialDestructorBody(ASTContext &Context, const FieldDecl *Field);
1356
1357
static bool
1358
HasTrivialDestructorBody(ASTContext &Context,
1359
                         const CXXRecordDecl *BaseClassDecl,
1360
                         const CXXRecordDecl *MostDerivedClassDecl)
1361
404
{
1362
  // If the destructor is trivial we don't have to check anything else.
1363
404
  if (BaseClassDecl->hasTrivialDestructor())
1364
126
    return true;
1365
1366
278
  if (!BaseClassDecl->getDestructor()->hasTrivialBody())
1367
213
    return false;
1368
1369
  // Check fields.
1370
65
  for (const auto *Field : BaseClassDecl->fields())
1371
53
    if (!FieldHasTrivialDestructorBody(Context, Field))
1372
44
      return false;
1373
1374
  // Check non-virtual bases.
1375
21
  for (const auto &I : BaseClassDecl->bases()) {
1376
15
    if (I.isVirtual())
1377
1
      continue;
1378
1379
14
    const CXXRecordDecl *NonVirtualBase =
1380
14
      cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
1381
14
    if (!HasTrivialDestructorBody(Context, NonVirtualBase,
1382
14
                                  MostDerivedClassDecl))
1383
14
      return false;
1384
14
  }
1385
1386
7
  if (BaseClassDecl == MostDerivedClassDecl) {
1387
    // Check virtual bases.
1388
7
    for (const auto &I : BaseClassDecl->vbases()) {
1389
1
      const CXXRecordDecl *VirtualBase =
1390
1
        cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
1391
1
      if (!HasTrivialDestructorBody(Context, VirtualBase,
1392
1
                                    MostDerivedClassDecl))
1393
1
        return false;
1394
1
    }
1395
7
  }
1396
1397
6
  return true;
1398
7
}
1399
1400
static bool
1401
FieldHasTrivialDestructorBody(ASTContext &Context,
1402
                                          const FieldDecl *Field)
1403
749
{
1404
749
  QualType FieldBaseElementType = Context.getBaseElementType(Field->getType());
1405
1406
749
  const RecordType *RT = FieldBaseElementType->getAs<RecordType>();
1407
749
  if (!RT)
1408
358
    return true;
1409
1410
391
  CXXRecordDecl *FieldClassDecl = cast<CXXRecordDecl>(RT->getDecl());
1411
1412
  // The destructor for an implicit anonymous union member is never invoked.
1413
391
  if (FieldClassDecl->isUnion() && 
FieldClassDecl->isAnonymousStructOrUnion()2
)
1414
2
    return false;
1415
1416
389
  return HasTrivialDestructorBody(Context, FieldClassDecl, FieldClassDecl);
1417
391
}
1418
1419
/// CanSkipVTablePointerInitialization - Check whether we need to initialize
1420
/// any vtable pointers before calling this destructor.
1421
static bool CanSkipVTablePointerInitialization(CodeGenFunction &CGF,
1422
8.34k
                                               const CXXDestructorDecl *Dtor) {
1423
8.34k
  const CXXRecordDecl *ClassDecl = Dtor->getParent();
1424
8.34k
  if (!ClassDecl->isDynamicClass())
1425
7.38k
    return true;
1426
1427
956
  if (!Dtor->hasTrivialBody())
1428
64
    return false;
1429
1430
  // Check the fields.
1431
892
  for (const auto *Field : ClassDecl->fields())
1432
358
    if (!FieldHasTrivialDestructorBody(CGF.getContext(), Field))
1433
105
      return false;
1434
1435
787
  return true;
1436
892
}
1437
1438
/// EmitDestructorBody - Emits the body of the current destructor.
1439
16.7k
void CodeGenFunction::EmitDestructorBody(FunctionArgList &Args) {
1440
16.7k
  const CXXDestructorDecl *Dtor = cast<CXXDestructorDecl>(CurGD.getDecl());
1441
16.7k
  CXXDtorType DtorType = CurGD.getDtorType();
1442
1443
  // For an abstract class, non-base destructors are never used (and can't
1444
  // be emitted in general, because vbase dtors may not have been validated
1445
  // by Sema), but the Itanium ABI doesn't make them optional and Clang may
1446
  // in fact emit references to them from other compilations, so emit them
1447
  // as functions containing a trap instruction.
1448
16.7k
  if (DtorType != Dtor_Base && 
Dtor->getParent()->isAbstract()8.44k
) {
1449
82
    llvm::CallInst *TrapCall = EmitTrapCall(llvm::Intrinsic::trap);
1450
82
    TrapCall->setDoesNotReturn();
1451
82
    TrapCall->setDoesNotThrow();
1452
82
    Builder.CreateUnreachable();
1453
82
    Builder.ClearInsertionPoint();
1454
82
    return;
1455
82
  }
1456
1457
16.7k
  Stmt *Body = Dtor->getBody();
1458
16.7k
  if (Body)
1459
16.5k
    incrementProfileCounter(Body);
1460
1461
  // The call to operator delete in a deleting destructor happens
1462
  // outside of the function-try-block, which means it's always
1463
  // possible to delegate the destructor body to the complete
1464
  // destructor.  Do so.
1465
16.7k
  if (DtorType == Dtor_Deleting) {
1466
872
    RunCleanupsScope DtorEpilogue(*this);
1467
872
    EnterDtorCleanups(Dtor, Dtor_Deleting);
1468
872
    if (HaveInsertPoint()) {
1469
868
      QualType ThisTy = Dtor->getThisObjectType();
1470
868
      EmitCXXDestructorCall(Dtor, Dtor_Complete, /*ForVirtualBase=*/false,
1471
868
                            /*Delegating=*/false, LoadCXXThisAddress(), ThisTy);
1472
868
    }
1473
872
    return;
1474
872
  }
1475
1476
  // If the body is a function-try-block, enter the try before
1477
  // anything else.
1478
15.8k
  bool isTryBody = (Body && 
isa<CXXTryStmt>(Body)15.7k
);
1479
15.8k
  if (isTryBody)
1480
18
    EnterCXXTryStmt(*cast<CXXTryStmt>(Body), true);
1481
15.8k
  EmitAsanPrologueOrEpilogue(false);
1482
1483
  // Enter the epilogue cleanups.
1484
15.8k
  RunCleanupsScope DtorEpilogue(*this);
1485
1486
  // If this is the complete variant, just invoke the base variant;
1487
  // the epilogue will destruct the virtual bases.  But we can't do
1488
  // this optimization if the body is a function-try-block, because
1489
  // we'd introduce *two* handler blocks.  In the Microsoft ABI, we
1490
  // always delegate because we might not have a definition in this TU.
1491
15.8k
  switch (DtorType) {
1492
0
  case Dtor_Comdat: llvm_unreachable("not expecting a COMDAT");
1493
0
  case Dtor_Deleting: llvm_unreachable("already handled deleting case");
1494
1495
7.49k
  case Dtor_Complete:
1496
7.49k
    assert((Body || getTarget().getCXXABI().isMicrosoft()) &&
1497
7.49k
           "can't emit a dtor without a body for non-Microsoft ABIs");
1498
1499
    // Enter the cleanup scopes for virtual bases.
1500
0
    EnterDtorCleanups(Dtor, Dtor_Complete);
1501
1502
7.49k
    if (!isTryBody) {
1503
7.48k
      QualType ThisTy = Dtor->getThisObjectType();
1504
7.48k
      EmitCXXDestructorCall(Dtor, Dtor_Base, /*ForVirtualBase=*/false,
1505
7.48k
                            /*Delegating=*/false, LoadCXXThisAddress(), ThisTy);
1506
7.48k
      break;
1507
7.48k
    }
1508
1509
    // Fallthrough: act like we're in the base variant.
1510
7.49k
    
LLVM_FALLTHROUGH8
;8
1511
1512
8.34k
  case Dtor_Base:
1513
8.34k
    assert(Body);
1514
1515
    // Enter the cleanup scopes for fields and non-virtual bases.
1516
0
    EnterDtorCleanups(Dtor, Dtor_Base);
1517
1518
    // Initialize the vtable pointers before entering the body.
1519
8.34k
    if (!CanSkipVTablePointerInitialization(*this, Dtor)) {
1520
      // Insert the llvm.launder.invariant.group intrinsic before initializing
1521
      // the vptrs to cancel any previous assumptions we might have made.
1522
169
      if (CGM.getCodeGenOpts().StrictVTablePointers &&
1523
169
          
CGM.getCodeGenOpts().OptimizationLevel > 02
)
1524
2
        CXXThisValue = Builder.CreateLaunderInvariantGroup(LoadCXXThis());
1525
169
      InitializeVTablePointers(Dtor->getParent());
1526
169
    }
1527
1528
8.34k
    if (isTryBody)
1529
18
      EmitStmt(cast<CXXTryStmt>(Body)->getTryBlock());
1530
8.32k
    else if (Body)
1531
8.32k
      EmitStmt(Body);
1532
0
    else {
1533
0
      assert(Dtor->isImplicit() && "bodyless dtor not implicit");
1534
      // nothing to do besides what's in the epilogue
1535
0
    }
1536
    // -fapple-kext must inline any call to this dtor into
1537
    // the caller's body.
1538
8.34k
    if (getLangOpts().AppleKext)
1539
5
      CurFn->addFnAttr(llvm::Attribute::AlwaysInline);
1540
1541
8.34k
    break;
1542
15.8k
  }
1543
1544
  // Jump out through the epilogue cleanups.
1545
15.8k
  DtorEpilogue.ForceCleanup();
1546
1547
  // Exit the try if applicable.
1548
15.8k
  if (isTryBody)
1549
18
    ExitCXXTryStmt(*cast<CXXTryStmt>(Body), true);
1550
15.8k
}
1551
1552
726
void CodeGenFunction::emitImplicitAssignmentOperatorBody(FunctionArgList &Args) {
1553
726
  const CXXMethodDecl *AssignOp = cast<CXXMethodDecl>(CurGD.getDecl());
1554
726
  const Stmt *RootS = AssignOp->getBody();
1555
726
  assert(isa<CompoundStmt>(RootS) &&
1556
726
         "Body of an implicit assignment operator should be compound stmt.");
1557
0
  const CompoundStmt *RootCS = cast<CompoundStmt>(RootS);
1558
1559
726
  LexicalScope Scope(*this, RootCS->getSourceRange());
1560
1561
726
  incrementProfileCounter(RootCS);
1562
726
  AssignmentMemcpyizer AM(*this, AssignOp, Args);
1563
726
  for (auto *I : RootCS->body())
1564
1.30k
    AM.emitAssignment(I);
1565
726
  AM.finish();
1566
726
}
1567
1568
namespace {
1569
  llvm::Value *LoadThisForDtorDelete(CodeGenFunction &CGF,
1570
885
                                     const CXXDestructorDecl *DD) {
1571
885
    if (Expr *ThisArg = DD->getOperatorDeleteThisArg())
1572
12
      return CGF.EmitScalarExpr(ThisArg);
1573
873
    return CGF.LoadCXXThis();
1574
885
  }
1575
1576
  /// Call the operator delete associated with the current destructor.
1577
  struct CallDtorDelete final : EHScopeStack::Cleanup {
1578
657
    CallDtorDelete() {}
1579
1580
670
    void Emit(CodeGenFunction &CGF, Flags flags) override {
1581
670
      const CXXDestructorDecl *Dtor = cast<CXXDestructorDecl>(CGF.CurCodeDecl);
1582
670
      const CXXRecordDecl *ClassDecl = Dtor->getParent();
1583
670
      CGF.EmitDeleteCall(Dtor->getOperatorDelete(),
1584
670
                         LoadThisForDtorDelete(CGF, Dtor),
1585
670
                         CGF.getContext().getTagDeclType(ClassDecl));
1586
670
    }
1587
  };
1588
1589
  void EmitConditionalDtorDeleteCall(CodeGenFunction &CGF,
1590
                                     llvm::Value *ShouldDeleteCondition,
1591
211
                                     bool ReturnAfterDelete) {
1592
211
    llvm::BasicBlock *callDeleteBB = CGF.createBasicBlock("dtor.call_delete");
1593
211
    llvm::BasicBlock *continueBB = CGF.createBasicBlock("dtor.continue");
1594
211
    llvm::Value *ShouldCallDelete
1595
211
      = CGF.Builder.CreateIsNull(ShouldDeleteCondition);
1596
211
    CGF.Builder.CreateCondBr(ShouldCallDelete, continueBB, callDeleteBB);
1597
1598
211
    CGF.EmitBlock(callDeleteBB);
1599
211
    const CXXDestructorDecl *Dtor = cast<CXXDestructorDecl>(CGF.CurCodeDecl);
1600
211
    const CXXRecordDecl *ClassDecl = Dtor->getParent();
1601
211
    CGF.EmitDeleteCall(Dtor->getOperatorDelete(),
1602
211
                       LoadThisForDtorDelete(CGF, Dtor),
1603
211
                       CGF.getContext().getTagDeclType(ClassDecl));
1604
211
    assert(Dtor->getOperatorDelete()->isDestroyingOperatorDelete() ==
1605
211
               ReturnAfterDelete &&
1606
211
           "unexpected value for ReturnAfterDelete");
1607
211
    if (ReturnAfterDelete)
1608
8
      CGF.EmitBranchThroughCleanup(CGF.ReturnBlock);
1609
203
    else
1610
203
      CGF.Builder.CreateBr(continueBB);
1611
1612
211
    CGF.EmitBlock(continueBB);
1613
211
  }
1614
1615
  struct CallDtorDeleteConditional final : EHScopeStack::Cleanup {
1616
    llvm::Value *ShouldDeleteCondition;
1617
1618
  public:
1619
    CallDtorDeleteConditional(llvm::Value *ShouldDeleteCondition)
1620
203
        : ShouldDeleteCondition(ShouldDeleteCondition) {
1621
203
      assert(ShouldDeleteCondition != nullptr);
1622
203
    }
1623
1624
203
    void Emit(CodeGenFunction &CGF, Flags flags) override {
1625
203
      EmitConditionalDtorDeleteCall(CGF, ShouldDeleteCondition,
1626
203
                                    /*ReturnAfterDelete*/false);
1627
203
    }
1628
  };
1629
1630
  class DestroyField  final : public EHScopeStack::Cleanup {
1631
    const FieldDecl *field;
1632
    CodeGenFunction::Destroyer *destroyer;
1633
    bool useEHCleanupForArray;
1634
1635
  public:
1636
    DestroyField(const FieldDecl *field, CodeGenFunction::Destroyer *destroyer,
1637
                 bool useEHCleanupForArray)
1638
        : field(field), destroyer(destroyer),
1639
1.06k
          useEHCleanupForArray(useEHCleanupForArray) {}
1640
1641
1.10k
    void Emit(CodeGenFunction &CGF, Flags flags) override {
1642
      // Find the address of the field.
1643
1.10k
      Address thisValue = CGF.LoadCXXThisAddress();
1644
1.10k
      QualType RecordTy = CGF.getContext().getTagDeclType(field->getParent());
1645
1.10k
      LValue ThisLV = CGF.MakeAddrLValue(thisValue, RecordTy);
1646
1.10k
      LValue LV = CGF.EmitLValueForField(ThisLV, field);
1647
1.10k
      assert(LV.isSimple());
1648
1649
0
      CGF.emitDestroy(LV.getAddress(CGF), field->getType(), destroyer,
1650
1.10k
                      flags.isForNormalCleanup() && 
useEHCleanupForArray1.06k
);
1651
1.10k
    }
1652
  };
1653
1654
 static void EmitSanitizerDtorCallback(CodeGenFunction &CGF, llvm::Value *Ptr,
1655
130
             CharUnits::QuantityType PoisonSize) {
1656
130
   CodeGenFunction::SanitizerScope SanScope(&CGF);
1657
   // Pass in void pointer and size of region as arguments to runtime
1658
   // function
1659
130
   llvm::Value *Args[] = {CGF.Builder.CreateBitCast(Ptr, CGF.VoidPtrTy),
1660
130
                          llvm::ConstantInt::get(CGF.SizeTy, PoisonSize)};
1661
1662
130
   llvm::Type *ArgTypes[] = {CGF.VoidPtrTy, CGF.SizeTy};
1663
1664
130
   llvm::FunctionType *FnType =
1665
130
       llvm::FunctionType::get(CGF.VoidTy, ArgTypes, false);
1666
130
   llvm::FunctionCallee Fn =
1667
130
       CGF.CGM.CreateRuntimeFunction(FnType, "__sanitizer_dtor_callback");
1668
130
   CGF.EmitNounwindRuntimeCall(Fn, Args);
1669
130
 }
1670
1671
  class SanitizeDtorMembers final : public EHScopeStack::Cleanup {
1672
    const CXXDestructorDecl *Dtor;
1673
1674
  public:
1675
88
    SanitizeDtorMembers(const CXXDestructorDecl *Dtor) : Dtor(Dtor) {}
1676
1677
    // Generate function call for handling object poisoning.
1678
    // Disables tail call elimination, to prevent the current stack frame
1679
    // from disappearing from the stack trace.
1680
88
    void Emit(CodeGenFunction &CGF, Flags flags) override {
1681
88
      const ASTRecordLayout &Layout =
1682
88
          CGF.getContext().getASTRecordLayout(Dtor->getParent());
1683
1684
      // Nothing to poison.
1685
88
      if (Layout.getFieldCount() == 0)
1686
0
        return;
1687
1688
      // Prevent the current stack frame from disappearing from the stack trace.
1689
88
      CGF.CurFn->addFnAttr("disable-tail-calls", "true");
1690
1691
      // Construct pointer to region to begin poisoning, and calculate poison
1692
      // size, so that only members declared in this class are poisoned.
1693
88
      ASTContext &Context = CGF.getContext();
1694
1695
88
      const RecordDecl *Decl = Dtor->getParent();
1696
88
      auto Fields = Decl->fields();
1697
338
      auto IsTrivial = [&](const FieldDecl *F) {
1698
338
        return FieldHasTrivialDestructorBody(Context, F);
1699
338
      };
1700
1701
170
      auto IsZeroSize = [&](const FieldDecl *F) {
1702
170
        return F->isZeroSize(Context);
1703
170
      };
1704
1705
      // Poison blocks of fields with trivial destructors making sure that block
1706
      // begin and end do not point to zero-sized fields. They don't have
1707
      // correct offsets so can't be used to calculate poisoning range.
1708
200
      for (auto It = Fields.begin(); It != Fields.end();) {
1709
188
        It = std::find_if(It, Fields.end(), [&](const FieldDecl *F) {
1710
188
          return IsTrivial(F) && 
!IsZeroSize(F)124
;
1711
188
        });
1712
128
        if (It == Fields.end())
1713
16
          break;
1714
112
        auto Start = It++;
1715
150
        It = std::find_if(It, Fields.end(), [&](const FieldDecl *F) {
1716
150
          return !IsTrivial(F) && 
!IsZeroSize(F)46
;
1717
150
        });
1718
1719
112
        PoisonMembers(CGF, (*Start)->getFieldIndex(),
1720
112
                      It == Fields.end() ? 
-172
:
(*It)->getFieldIndex()40
);
1721
112
      }
1722
88
    }
1723
1724
  private:
1725
    /// \param layoutStartOffset index of the ASTRecordLayout field to
1726
    ///     start poisoning (inclusive)
1727
    /// \param layoutEndOffset index of the ASTRecordLayout field to
1728
    ///     end poisoning (exclusive)
1729
    void PoisonMembers(CodeGenFunction &CGF, unsigned layoutStartOffset,
1730
112
                       unsigned layoutEndOffset) {
1731
112
      ASTContext &Context = CGF.getContext();
1732
112
      const ASTRecordLayout &Layout =
1733
112
          Context.getASTRecordLayout(Dtor->getParent());
1734
1735
      // It's a first trivia field so it should be at the begining of char,
1736
      // still round up start offset just in case.
1737
112
      CharUnits PoisonStart =
1738
112
          Context.toCharUnitsFromBits(Layout.getFieldOffset(layoutStartOffset) +
1739
112
                                      Context.getCharWidth() - 1);
1740
112
      llvm::ConstantInt *OffsetSizePtr =
1741
112
          llvm::ConstantInt::get(CGF.SizeTy, PoisonStart.getQuantity());
1742
1743
112
      llvm::Value *OffsetPtr = CGF.Builder.CreateGEP(
1744
112
          CGF.Int8Ty,
1745
112
          CGF.Builder.CreateBitCast(CGF.LoadCXXThis(), CGF.Int8PtrTy),
1746
112
          OffsetSizePtr);
1747
1748
112
      CharUnits PoisonEnd;
1749
112
      if (layoutEndOffset >= Layout.getFieldCount()) {
1750
72
        PoisonEnd = Layout.getNonVirtualSize();
1751
72
      } else {
1752
40
        PoisonEnd =
1753
40
            Context.toCharUnitsFromBits(Layout.getFieldOffset(layoutEndOffset));
1754
40
      }
1755
112
      CharUnits PoisonSize = PoisonEnd - PoisonStart;
1756
112
      if (!PoisonSize.isPositive())
1757
0
        return;
1758
1759
112
      EmitSanitizerDtorCallback(CGF, OffsetPtr, PoisonSize.getQuantity());
1760
112
    }
1761
  };
1762
1763
 class SanitizeDtorVTable final : public EHScopeStack::Cleanup {
1764
    const CXXDestructorDecl *Dtor;
1765
1766
  public:
1767
18
    SanitizeDtorVTable(const CXXDestructorDecl *Dtor) : Dtor(Dtor) {}
1768
1769
    // Generate function call for handling vtable pointer poisoning.
1770
18
    void Emit(CodeGenFunction &CGF, Flags flags) override {
1771
18
      assert(Dtor->getParent()->isDynamicClass());
1772
0
      (void)Dtor;
1773
18
      ASTContext &Context = CGF.getContext();
1774
      // Poison vtable and vtable ptr if they exist for this class.
1775
18
      llvm::Value *VTablePtr = CGF.LoadCXXThis();
1776
1777
18
      CharUnits::QuantityType PoisonSize =
1778
18
          Context.toCharUnitsFromBits(CGF.PointerWidthInBits).getQuantity();
1779
      // Pass in void pointer and size of region as arguments to runtime
1780
      // function
1781
18
      EmitSanitizerDtorCallback(CGF, VTablePtr, PoisonSize);
1782
18
    }
1783
 };
1784
} // end anonymous namespace
1785
1786
/// Emit all code that comes at the end of class's
1787
/// destructor. This is to call destructors on members and base classes
1788
/// in reverse order of their construction.
1789
///
1790
/// For a deleting destructor, this also handles the case where a destroying
1791
/// operator delete completely overrides the definition.
1792
void CodeGenFunction::EnterDtorCleanups(const CXXDestructorDecl *DD,
1793
16.7k
                                        CXXDtorType DtorType) {
1794
16.7k
  assert((!DD->isTrivial() || DD->hasAttr<DLLExportAttr>()) &&
1795
16.7k
         "Should not emit dtor epilogue for non-exported trivial dtor!");
1796
1797
  // The deleting-destructor phase just needs to call the appropriate
1798
  // operator delete that Sema picked up.
1799
16.7k
  if (DtorType == Dtor_Deleting) {
1800
872
    assert(DD->getOperatorDelete() &&
1801
872
           "operator delete missing - EnterDtorCleanups");
1802
872
    if (CXXStructorImplicitParamValue) {
1803
      // If there is an implicit param to the deleting dtor, it's a boolean
1804
      // telling whether this is a deleting destructor.
1805
211
      if (DD->getOperatorDelete()->isDestroyingOperatorDelete())
1806
8
        EmitConditionalDtorDeleteCall(*this, CXXStructorImplicitParamValue,
1807
8
                                      /*ReturnAfterDelete*/true);
1808
203
      else
1809
203
        EHStack.pushCleanup<CallDtorDeleteConditional>(
1810
203
            NormalAndEHCleanup, CXXStructorImplicitParamValue);
1811
661
    } else {
1812
661
      if (DD->getOperatorDelete()->isDestroyingOperatorDelete()) {
1813
4
        const CXXRecordDecl *ClassDecl = DD->getParent();
1814
4
        EmitDeleteCall(DD->getOperatorDelete(),
1815
4
                       LoadThisForDtorDelete(*this, DD),
1816
4
                       getContext().getTagDeclType(ClassDecl));
1817
4
        EmitBranchThroughCleanup(ReturnBlock);
1818
657
      } else {
1819
657
        EHStack.pushCleanup<CallDtorDelete>(NormalAndEHCleanup);
1820
657
      }
1821
661
    }
1822
872
    return;
1823
872
  }
1824
1825
15.8k
  const CXXRecordDecl *ClassDecl = DD->getParent();
1826
1827
  // Unions have no bases and do not call field destructors.
1828
15.8k
  if (ClassDecl->isUnion())
1829
4
    return;
1830
1831
  // The complete-destructor phase just destructs all the virtual bases.
1832
15.8k
  if (DtorType == Dtor_Complete) {
1833
    // Poison the vtable pointer such that access after the base
1834
    // and member destructors are invoked is invalid.
1835
7.49k
    if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
1836
7.49k
        
SanOpts.has(SanitizerKind::Memory)89
&&
ClassDecl->getNumVBases()88
&&
1837
7.49k
        
ClassDecl->isPolymorphic()4
)
1838
4
      EHStack.pushCleanup<SanitizeDtorVTable>(NormalAndEHCleanup, DD);
1839
1840
    // We push them in the forward order so that they'll be popped in
1841
    // the reverse order.
1842
7.49k
    for (const auto &Base : ClassDecl->vbases()) {
1843
229
      auto *BaseClassDecl =
1844
229
          cast<CXXRecordDecl>(Base.getType()->castAs<RecordType>()->getDecl());
1845
1846
      // Ignore trivial destructors.
1847
229
      if (BaseClassDecl->hasTrivialDestructor())
1848
44
        continue;
1849
1850
185
      EHStack.pushCleanup<CallBaseDtor>(NormalAndEHCleanup,
1851
185
                                        BaseClassDecl,
1852
185
                                        /*BaseIsVirtual*/ true);
1853
185
    }
1854
1855
7.49k
    return;
1856
7.49k
  }
1857
1858
8.34k
  assert(DtorType == Dtor_Base);
1859
  // Poison the vtable pointer if it has no virtual bases, but inherits
1860
  // virtual functions.
1861
8.34k
  if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
1862
8.34k
      
SanOpts.has(SanitizerKind::Memory)89
&&
!ClassDecl->getNumVBases()88
&&
1863
8.34k
      
ClassDecl->isPolymorphic()84
)
1864
14
    EHStack.pushCleanup<SanitizeDtorVTable>(NormalAndEHCleanup, DD);
1865
1866
  // Destroy non-virtual bases.
1867
8.34k
  for (const auto &Base : ClassDecl->bases()) {
1868
    // Ignore virtual bases.
1869
2.02k
    if (Base.isVirtual())
1870
127
      continue;
1871
1872
1.89k
    CXXRecordDecl *BaseClassDecl = Base.getType()->getAsCXXRecordDecl();
1873
1874
    // Ignore trivial destructors.
1875
1.89k
    if (BaseClassDecl->hasTrivialDestructor())
1876
696
      continue;
1877
1878
1.19k
    EHStack.pushCleanup<CallBaseDtor>(NormalAndEHCleanup,
1879
1.19k
                                      BaseClassDecl,
1880
1.19k
                                      /*BaseIsVirtual*/ false);
1881
1.19k
  }
1882
1883
  // Poison fields such that access after their destructors are
1884
  // invoked, and before the base class destructor runs, is invalid.
1885
8.34k
  if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
1886
8.34k
      
SanOpts.has(SanitizerKind::Memory)89
)
1887
88
    EHStack.pushCleanup<SanitizeDtorMembers>(NormalAndEHCleanup, DD);
1888
1889
  // Destroy direct fields.
1890
10.2k
  for (const auto *Field : ClassDecl->fields()) {
1891
10.2k
    QualType type = Field->getType();
1892
10.2k
    QualType::DestructionKind dtorKind = type.isDestructedType();
1893
10.2k
    if (!dtorKind) 
continue9.15k
;
1894
1895
    // Anonymous union members do not have their destructors called.
1896
1.07k
    const RecordType *RT = type->getAsUnionType();
1897
1.07k
    if (RT && 
RT->getDecl()->isAnonymousStructOrUnion()9
)
continue9
;
1898
1899
1.06k
    CleanupKind cleanupKind = getCleanupKind(dtorKind);
1900
1.06k
    EHStack.pushCleanup<DestroyField>(cleanupKind, Field,
1901
1.06k
                                      getDestroyer(dtorKind),
1902
1.06k
                                      cleanupKind & EHCleanup);
1903
1.06k
  }
1904
8.34k
}
1905
1906
/// EmitCXXAggrConstructorCall - Emit a loop to call a particular
1907
/// constructor for each of several members of an array.
1908
///
1909
/// \param ctor the constructor to call for each element
1910
/// \param arrayType the type of the array to initialize
1911
/// \param arrayBegin an arrayType*
1912
/// \param zeroInitialize true if each element should be
1913
///   zero-initialized before it is constructed
1914
void CodeGenFunction::EmitCXXAggrConstructorCall(
1915
    const CXXConstructorDecl *ctor, const ArrayType *arrayType,
1916
    Address arrayBegin, const CXXConstructExpr *E, bool NewPointerIsChecked,
1917
892
    bool zeroInitialize) {
1918
892
  QualType elementType;
1919
892
  llvm::Value *numElements =
1920
892
    emitArrayLength(arrayType, elementType, arrayBegin);
1921
1922
892
  EmitCXXAggrConstructorCall(ctor, numElements, arrayBegin, E,
1923
892
                             NewPointerIsChecked, zeroInitialize);
1924
892
}
1925
1926
/// EmitCXXAggrConstructorCall - Emit a loop to call a particular
1927
/// constructor for each of several members of an array.
1928
///
1929
/// \param ctor the constructor to call for each element
1930
/// \param numElements the number of elements in the array;
1931
///   may be zero
1932
/// \param arrayBase a T*, where T is the type constructed by ctor
1933
/// \param zeroInitialize true if each element should be
1934
///   zero-initialized before it is constructed
1935
void CodeGenFunction::EmitCXXAggrConstructorCall(const CXXConstructorDecl *ctor,
1936
                                                 llvm::Value *numElements,
1937
                                                 Address arrayBase,
1938
                                                 const CXXConstructExpr *E,
1939
                                                 bool NewPointerIsChecked,
1940
957
                                                 bool zeroInitialize) {
1941
  // It's legal for numElements to be zero.  This can happen both
1942
  // dynamically, because x can be zero in 'new A[x]', and statically,
1943
  // because of GCC extensions that permit zero-length arrays.  There
1944
  // are probably legitimate places where we could assume that this
1945
  // doesn't happen, but it's not clear that it's worth it.
1946
957
  llvm::BranchInst *zeroCheckBranch = nullptr;
1947
1948
  // Optimize for a constant count.
1949
957
  llvm::ConstantInt *constantCount
1950
957
    = dyn_cast<llvm::ConstantInt>(numElements);
1951
957
  if (constantCount) {
1952
    // Just skip out if the constant count is zero.
1953
923
    if (constantCount->isZero()) 
return0
;
1954
1955
  // Otherwise, emit the check.
1956
923
  } else {
1957
34
    llvm::BasicBlock *loopBB = createBasicBlock("new.ctorloop");
1958
34
    llvm::Value *iszero = Builder.CreateIsNull(numElements, "isempty");
1959
34
    zeroCheckBranch = Builder.CreateCondBr(iszero, loopBB, loopBB);
1960
34
    EmitBlock(loopBB);
1961
34
  }
1962
1963
  // Find the end of the array.
1964
957
  llvm::Type *elementType = arrayBase.getElementType();
1965
957
  llvm::Value *arrayBegin = arrayBase.getPointer();
1966
957
  llvm::Value *arrayEnd = Builder.CreateInBoundsGEP(
1967
957
      elementType, arrayBegin, numElements, "arrayctor.end");
1968
1969
  // Enter the loop, setting up a phi for the current location to initialize.
1970
957
  llvm::BasicBlock *entryBB = Builder.GetInsertBlock();
1971
957
  llvm::BasicBlock *loopBB = createBasicBlock("arrayctor.loop");
1972
957
  EmitBlock(loopBB);
1973
957
  llvm::PHINode *cur = Builder.CreatePHI(arrayBegin->getType(), 2,
1974
957
                                         "arrayctor.cur");
1975
957
  cur->addIncoming(arrayBegin, entryBB);
1976
1977
  // Inside the loop body, emit the constructor call on the array element.
1978
1979
  // The alignment of the base, adjusted by the size of a single element,
1980
  // provides a conservative estimate of the alignment of every element.
1981
  // (This assumes we never start tracking offsetted alignments.)
1982
  //
1983
  // Note that these are complete objects and so we don't need to
1984
  // use the non-virtual size or alignment.
1985
957
  QualType type = getContext().getTypeDeclType(ctor->getParent());
1986
957
  CharUnits eltAlignment =
1987
957
    arrayBase.getAlignment()
1988
957
             .alignmentOfArrayElement(getContext().getTypeSizeInChars(type));
1989
957
  Address curAddr = Address(cur, eltAlignment);
1990
1991
  // Zero initialize the storage, if requested.
1992
957
  if (zeroInitialize)
1993
2
    EmitNullInitialization(curAddr, type);
1994
1995
  // C++ [class.temporary]p4:
1996
  // There are two contexts in which temporaries are destroyed at a different
1997
  // point than the end of the full-expression. The first context is when a
1998
  // default constructor is called to initialize an element of an array.
1999
  // If the constructor has one or more default arguments, the destruction of
2000
  // every temporary created in a default argument expression is sequenced
2001
  // before the construction of the next array element, if any.
2002
2003
957
  {
2004
957
    RunCleanupsScope Scope(*this);
2005
2006
    // Evaluate the constructor and its arguments in a regular
2007
    // partial-destroy cleanup.
2008
957
    if (getLangOpts().Exceptions &&
2009
957
        
!ctor->getParent()->hasTrivialDestructor()85
) {
2010
38
      Destroyer *destroyer = destroyCXXObject;
2011
38
      pushRegularPartialArrayCleanup(arrayBegin, cur, type, eltAlignment,
2012
38
                                     *destroyer);
2013
38
    }
2014
957
    auto currAVS = AggValueSlot::forAddr(
2015
957
        curAddr, type.getQualifiers(), AggValueSlot::IsDestructed,
2016
957
        AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased,
2017
957
        AggValueSlot::DoesNotOverlap, AggValueSlot::IsNotZeroed,
2018
957
        NewPointerIsChecked ? 
AggValueSlot::IsSanitizerChecked97
2019
957
                            : 
AggValueSlot::IsNotSanitizerChecked860
);
2020
957
    EmitCXXConstructorCall(ctor, Ctor_Complete, /*ForVirtualBase=*/false,
2021
957
                           /*Delegating=*/false, currAVS, E);
2022
957
  }
2023
2024
  // Go to the next element.
2025
957
  llvm::Value *next = Builder.CreateInBoundsGEP(
2026
957
      elementType, cur, llvm::ConstantInt::get(SizeTy, 1), "arrayctor.next");
2027
957
  cur->addIncoming(next, Builder.GetInsertBlock());
2028
2029
  // Check whether that's the end of the loop.
2030
957
  llvm::Value *done = Builder.CreateICmpEQ(next, arrayEnd, "arrayctor.done");
2031
957
  llvm::BasicBlock *contBB = createBasicBlock("arrayctor.cont");
2032
957
  Builder.CreateCondBr(done, contBB, loopBB);
2033
2034
  // Patch the earlier check to skip over the loop.
2035
957
  if (zeroCheckBranch) 
zeroCheckBranch->setSuccessor(0, contBB)34
;
2036
2037
957
  EmitBlock(contBB);
2038
957
}
2039
2040
void CodeGenFunction::destroyCXXObject(CodeGenFunction &CGF,
2041
                                       Address addr,
2042
19.3k
                                       QualType type) {
2043
19.3k
  const RecordType *rtype = type->castAs<RecordType>();
2044
19.3k
  const CXXRecordDecl *record = cast<CXXRecordDecl>(rtype->getDecl());
2045
19.3k
  const CXXDestructorDecl *dtor = record->getDestructor();
2046
19.3k
  assert(!dtor->isTrivial());
2047
0
  CGF.EmitCXXDestructorCall(dtor, Dtor_Complete, /*for vbase*/ false,
2048
19.3k
                            /*Delegating=*/false, addr, type);
2049
19.3k
}
2050
2051
void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D,
2052
                                             CXXCtorType Type,
2053
                                             bool ForVirtualBase,
2054
                                             bool Delegating,
2055
                                             AggValueSlot ThisAVS,
2056
44.3k
                                             const CXXConstructExpr *E) {
2057
44.3k
  CallArgList Args;
2058
44.3k
  Address This = ThisAVS.getAddress();
2059
44.3k
  LangAS SlotAS = ThisAVS.getQualifiers().getAddressSpace();
2060
44.3k
  QualType ThisType = D->getThisType();
2061
44.3k
  LangAS ThisAS = ThisType.getTypePtr()->getPointeeType().getAddressSpace();
2062
44.3k
  llvm::Value *ThisPtr = This.getPointer();
2063
2064
44.3k
  if (SlotAS != ThisAS) {
2065
29
    unsigned TargetThisAS = getContext().getTargetAddressSpace(ThisAS);
2066
29
    llvm::Type *NewType =
2067
29
        ThisPtr->getType()->getPointerElementType()->getPointerTo(TargetThisAS);
2068
29
    ThisPtr = getTargetHooks().performAddrSpaceCast(*this, This.getPointer(),
2069
29
                                                    ThisAS, SlotAS, NewType);
2070
29
  }
2071
2072
  // Push the this ptr.
2073
44.3k
  Args.add(RValue::get(ThisPtr), D->getThisType());
2074
2075
  // If this is a trivial constructor, emit a memcpy now before we lose
2076
  // the alignment information on the argument.
2077
  // FIXME: It would be better to preserve alignment information into CallArg.
2078
44.3k
  if (isMemcpyEquivalentSpecialMember(D)) {
2079
5.77k
    assert(E->getNumArgs() == 1 && "unexpected argcount for trivial ctor");
2080
2081
0
    const Expr *Arg = E->getArg(0);
2082
5.77k
    LValue Src = EmitLValue(Arg);
2083
5.77k
    QualType DestTy = getContext().getTypeDeclType(D->getParent());
2084
5.77k
    LValue Dest = MakeAddrLValue(This, DestTy);
2085
5.77k
    EmitAggregateCopyCtor(Dest, Src, ThisAVS.mayOverlap());
2086
5.77k
    return;
2087
5.77k
  }
2088
2089
  // Add the rest of the user-supplied arguments.
2090
38.5k
  const FunctionProtoType *FPT = D->getType()->castAs<FunctionProtoType>();
2091
38.5k
  EvaluationOrder Order = E->isListInitialization()
2092
38.5k
                              ? 
EvaluationOrder::ForceLeftToRight423
2093
38.5k
                              : 
EvaluationOrder::Default38.1k
;
2094
38.5k
  EmitCallArgs(Args, FPT, E->arguments(), E->getConstructor(),
2095
38.5k
               /*ParamsToSkip*/ 0, Order);
2096
2097
38.5k
  EmitCXXConstructorCall(D, Type, ForVirtualBase, Delegating, This, Args,
2098
38.5k
                         ThisAVS.mayOverlap(), E->getExprLoc(),
2099
38.5k
                         ThisAVS.isSanitizerChecked());
2100
38.5k
}
2101
2102
static bool canEmitDelegateCallArgs(CodeGenFunction &CGF,
2103
                                    const CXXConstructorDecl *Ctor,
2104
225
                                    CXXCtorType Type, CallArgList &Args) {
2105
  // We can't forward a variadic call.
2106
225
  if (Ctor->isVariadic())
2107
31
    return false;
2108
2109
194
  if (CGF.getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) {
2110
    // If the parameters are callee-cleanup, it's not safe to forward.
2111
32
    for (auto *P : Ctor->parameters())
2112
56
      if (P->needsDestruction(CGF.getContext()))
2113
16
        return false;
2114
2115
    // Likewise if they're inalloca.
2116
16
    const CGFunctionInfo &Info =
2117
16
        CGF.CGM.getTypes().arrangeCXXConstructorCall(Args, Ctor, Type, 0, 0);
2118
16
    if (Info.usesInAlloca())
2119
0
      return false;
2120
16
  }
2121
2122
  // Anything else should be OK.
2123
178
  return true;
2124
194
}
2125
2126
void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D,
2127
                                             CXXCtorType Type,
2128
                                             bool ForVirtualBase,
2129
                                             bool Delegating,
2130
                                             Address This,
2131
                                             CallArgList &Args,
2132
                                             AggValueSlot::Overlap_t Overlap,
2133
                                             SourceLocation Loc,
2134
55.0k
                                             bool NewPointerIsChecked) {
2135
55.0k
  const CXXRecordDecl *ClassDecl = D->getParent();
2136
2137
55.0k
  if (!NewPointerIsChecked)
2138
33.5k
    EmitTypeCheck(CodeGenFunction::TCK_ConstructorCall, Loc, This.getPointer(),
2139
33.5k
                  getContext().getRecordType(ClassDecl), CharUnits::Zero());
2140
2141
55.0k
  if (D->isTrivial() && 
D->isDefaultConstructor()54
) {
2142
33
    assert(Args.size() == 1 && "trivial default ctor with args");
2143
0
    return;
2144
33
  }
2145
2146
  // If this is a trivial constructor, just emit what's needed. If this is a
2147
  // union copy constructor, we must emit a memcpy, because the AST does not
2148
  // model that copy.
2149
55.0k
  if (isMemcpyEquivalentSpecialMember(D)) {
2150
18
    assert(Args.size() == 2 && "unexpected argcount for trivial ctor");
2151
2152
0
    QualType SrcTy = D->getParamDecl(0)->getType().getNonReferenceType();
2153
18
    Address Src(Args[1].getRValue(*this).getScalarVal(),
2154
18
                CGM.getNaturalTypeAlignment(SrcTy));
2155
18
    LValue SrcLVal = MakeAddrLValue(Src, SrcTy);
2156
18
    QualType DestTy = getContext().getTypeDeclType(ClassDecl);
2157
18
    LValue DestLVal = MakeAddrLValue(This, DestTy);
2158
18
    EmitAggregateCopyCtor(DestLVal, SrcLVal, Overlap);
2159
18
    return;
2160
18
  }
2161
2162
54.9k
  bool PassPrototypeArgs = true;
2163
  // Check whether we can actually emit the constructor before trying to do so.
2164
54.9k
  if (auto Inherited = D->getInheritedConstructor()) {
2165
235
    PassPrototypeArgs = getTypes().inheritingCtorHasParams(Inherited, Type);
2166
235
    if (PassPrototypeArgs && 
!canEmitDelegateCallArgs(*this, D, Type, Args)225
) {
2167
47
      EmitInlinedInheritingCXXConstructorCall(D, Type, ForVirtualBase,
2168
47
                                              Delegating, Args);
2169
47
      return;
2170
47
    }
2171
235
  }
2172
2173
  // Insert any ABI-specific implicit constructor arguments.
2174
54.9k
  CGCXXABI::AddedStructorArgCounts ExtraArgs =
2175
54.9k
      CGM.getCXXABI().addImplicitConstructorArgs(*this, D, Type, ForVirtualBase,
2176
54.9k
                                                 Delegating, Args);
2177
2178
  // Emit the call.
2179
54.9k
  llvm::Constant *CalleePtr = CGM.getAddrOfCXXStructor(GlobalDecl(D, Type));
2180
54.9k
  const CGFunctionInfo &Info = CGM.getTypes().arrangeCXXConstructorCall(
2181
54.9k
      Args, D, Type, ExtraArgs.Prefix, ExtraArgs.Suffix, PassPrototypeArgs);
2182
54.9k
  CGCallee Callee = CGCallee::forDirect(CalleePtr, GlobalDecl(D, Type));
2183
54.9k
  EmitCall(Info, Callee, ReturnValueSlot(), Args, nullptr, false, Loc);
2184
2185
  // Generate vtable assumptions if we're constructing a complete object
2186
  // with a vtable.  We don't do this for base subobjects for two reasons:
2187
  // first, it's incorrect for classes with virtual bases, and second, we're
2188
  // about to overwrite the vptrs anyway.
2189
  // We also have to make sure if we can refer to vtable:
2190
  // - Otherwise we can refer to vtable if it's safe to speculatively emit.
2191
  // FIXME: If vtable is used by ctor/dtor, or if vtable is external and we are
2192
  // sure that definition of vtable is not hidden,
2193
  // then we are always safe to refer to it.
2194
  // FIXME: It looks like InstCombine is very inefficient on dealing with
2195
  // assumes. Make assumption loads require -fstrict-vtable-pointers temporarily.
2196
54.9k
  if (CGM.getCodeGenOpts().OptimizationLevel > 0 &&
2197
54.9k
      
ClassDecl->isDynamicClass()1.42k
&&
Type != Ctor_Base601
&&
2198
54.9k
      
CGM.getCXXABI().canSpeculativelyEmitVTable(ClassDecl)329
&&
2199
54.9k
      
CGM.getCodeGenOpts().StrictVTablePointers116
)
2200
56
    EmitVTableAssumptionLoads(ClassDecl, This);
2201
54.9k
}
2202
2203
void CodeGenFunction::EmitInheritedCXXConstructorCall(
2204
    const CXXConstructorDecl *D, bool ForVirtualBase, Address This,
2205
200
    bool InheritedFromVBase, const CXXInheritedCtorInitExpr *E) {
2206
200
  CallArgList Args;
2207
200
  CallArg ThisArg(RValue::get(This.getPointer()), D->getThisType());
2208
2209
  // Forward the parameters.
2210
200
  if (InheritedFromVBase &&
2211
200
      
CGM.getTarget().getCXXABI().hasConstructorVariants()17
) {
2212
    // Nothing to do; this construction is not responsible for constructing
2213
    // the base class containing the inherited constructor.
2214
    // FIXME: Can we just pass undef's for the remaining arguments if we don't
2215
    // have constructor variants?
2216
10
    Args.push_back(ThisArg);
2217
190
  } else if (!CXXInheritedCtorInitExprArgs.empty()) {
2218
    // The inheriting constructor was inlined; just inject its arguments.
2219
47
    assert(CXXInheritedCtorInitExprArgs.size() >= D->getNumParams() &&
2220
47
           "wrong number of parameters for inherited constructor call");
2221
0
    Args = CXXInheritedCtorInitExprArgs;
2222
47
    Args[0] = ThisArg;
2223
143
  } else {
2224
    // The inheriting constructor was not inlined. Emit delegating arguments.
2225
143
    Args.push_back(ThisArg);
2226
143
    const auto *OuterCtor = cast<CXXConstructorDecl>(CurCodeDecl);
2227
143
    assert(OuterCtor->getNumParams() == D->getNumParams());
2228
0
    assert(!OuterCtor->isVariadic() && "should have been inlined");
2229
2230
269
    for (const auto *Param : OuterCtor->parameters()) {
2231
269
      assert(getContext().hasSameUnqualifiedType(
2232
269
          OuterCtor->getParamDecl(Param->getFunctionScopeIndex())->getType(),
2233
269
          Param->getType()));
2234
0
      EmitDelegateCallArg(Args, Param, E->getLocation());
2235
2236
      // Forward __attribute__(pass_object_size).
2237
269
      if (Param->hasAttr<PassObjectSizeAttr>()) {
2238
22
        auto *POSParam = SizeArguments[Param];
2239
22
        assert(POSParam && "missing pass_object_size value for forwarding");
2240
0
        EmitDelegateCallArg(Args, POSParam, E->getLocation());
2241
22
      }
2242
269
    }
2243
143
  }
2244
2245
0
  EmitCXXConstructorCall(D, Ctor_Base, ForVirtualBase, /*Delegating*/false,
2246
200
                         This, Args, AggValueSlot::MayOverlap,
2247
200
                         E->getLocation(), /*NewPointerIsChecked*/true);
2248
200
}
2249
2250
void CodeGenFunction::EmitInlinedInheritingCXXConstructorCall(
2251
    const CXXConstructorDecl *Ctor, CXXCtorType CtorType, bool ForVirtualBase,
2252
47
    bool Delegating, CallArgList &Args) {
2253
47
  GlobalDecl GD(Ctor, CtorType);
2254
47
  InlinedInheritingConstructorScope Scope(*this, GD);
2255
47
  ApplyInlineDebugLocation DebugScope(*this, GD);
2256
47
  RunCleanupsScope RunCleanups(*this);
2257
2258
  // Save the arguments to be passed to the inherited constructor.
2259
47
  CXXInheritedCtorInitExprArgs = Args;
2260
2261
47
  FunctionArgList Params;
2262
47
  QualType RetType = BuildFunctionArgList(CurGD, Params);
2263
47
  FnRetTy = RetType;
2264
2265
  // Insert any ABI-specific implicit constructor arguments.
2266
47
  CGM.getCXXABI().addImplicitConstructorArgs(*this, Ctor, CtorType,
2267
47
                                             ForVirtualBase, Delegating, Args);
2268
2269
  // Emit a simplified prolog. We only need to emit the implicit params.
2270
47
  assert(Args.size() >= Params.size() && "too few arguments for call");
2271
332
  for (unsigned I = 0, N = Args.size(); I != N; 
++I285
) {
2272
285
    if (I < Params.size() && 
isa<ImplicitParamDecl>(Params[I])231
) {
2273
61
      const RValue &RV = Args[I].getRValue(*this);
2274
61
      assert(!RV.isComplex() && "complex indirect params not supported");
2275
61
      ParamValue Val = RV.isScalar()
2276
61
                           ? ParamValue::forDirect(RV.getScalarVal())
2277
61
                           : 
ParamValue::forIndirect(RV.getAggregateAddress())0
;
2278
61
      EmitParmDecl(*Params[I], Val, I + 1);
2279
61
    }
2280
285
  }
2281
2282
  // Create a return value slot if the ABI implementation wants one.
2283
  // FIXME: This is dumb, we should ask the ABI not to try to set the return
2284
  // value instead.
2285
47
  if (!RetType->isVoidType())
2286
28
    ReturnValue = CreateIRTemp(RetType, "retval.inhctor");
2287
2288
47
  CGM.getCXXABI().EmitInstanceFunctionProlog(*this);
2289
47
  CXXThisValue = CXXABIThisValue;
2290
2291
  // Directly emit the constructor initializers.
2292
47
  EmitCtorPrologue(Ctor, CtorType, Params);
2293
47
}
2294
2295
62
void CodeGenFunction::EmitVTableAssumptionLoad(const VPtr &Vptr, Address This) {
2296
62
  llvm::Value *VTableGlobal =
2297
62
      CGM.getCXXABI().getVTableAddressPoint(Vptr.Base, Vptr.VTableClass);
2298
62
  if (!VTableGlobal)
2299
0
    return;
2300
2301
  // We can just use the base offset in the complete class.
2302
62
  CharUnits NonVirtualOffset = Vptr.Base.getBaseOffset();
2303
2304
62
  if (!NonVirtualOffset.isZero())
2305
4
    This =
2306
4
        ApplyNonVirtualAndVirtualOffset(*this, This, NonVirtualOffset, nullptr,
2307
4
                                        Vptr.VTableClass, Vptr.NearestVBase);
2308
2309
62
  llvm::Value *VPtrValue =
2310
62
      GetVTablePtr(This, VTableGlobal->getType(), Vptr.VTableClass);
2311
62
  llvm::Value *Cmp =
2312
62
      Builder.CreateICmpEQ(VPtrValue, VTableGlobal, "cmp.vtables");
2313
62
  Builder.CreateAssumption(Cmp);
2314
62
}
2315
2316
void CodeGenFunction::EmitVTableAssumptionLoads(const CXXRecordDecl *ClassDecl,
2317
56
                                                Address This) {
2318
56
  if (CGM.getCXXABI().doStructorsInitializeVPtrs(ClassDecl))
2319
56
    for (const VPtr &Vptr : getVTablePointers(ClassDecl))
2320
62
      EmitVTableAssumptionLoad(Vptr, This);
2321
56
}
2322
2323
void
2324
CodeGenFunction::EmitSynthesizedCXXCopyCtorCall(const CXXConstructorDecl *D,
2325
                                                Address This, Address Src,
2326
62
                                                const CXXConstructExpr *E) {
2327
62
  const FunctionProtoType *FPT = D->getType()->castAs<FunctionProtoType>();
2328
2329
62
  CallArgList Args;
2330
2331
  // Push the this ptr.
2332
62
  Args.add(RValue::get(This.getPointer()), D->getThisType());
2333
2334
  // Push the src ptr.
2335
62
  QualType QT = *(FPT->param_type_begin());
2336
62
  llvm::Type *t = CGM.getTypes().ConvertType(QT);
2337
62
  Src = Builder.CreateBitCast(Src, t);
2338
62
  Args.add(RValue::get(Src.getPointer()), QT);
2339
2340
  // Skip over first argument (Src).
2341
62
  EmitCallArgs(Args, FPT, drop_begin(E->arguments(), 1), E->getConstructor(),
2342
62
               /*ParamsToSkip*/ 1);
2343
2344
62
  EmitCXXConstructorCall(D, Ctor_Complete, /*ForVirtualBase*/false,
2345
62
                         /*Delegating*/false, This, Args,
2346
62
                         AggValueSlot::MayOverlap, E->getExprLoc(),
2347
62
                         /*NewPointerIsChecked*/false);
2348
62
}
2349
2350
void
2351
CodeGenFunction::EmitDelegateCXXConstructorCall(const CXXConstructorDecl *Ctor,
2352
                                                CXXCtorType CtorType,
2353
                                                const FunctionArgList &Args,
2354
16.2k
                                                SourceLocation Loc) {
2355
16.2k
  CallArgList DelegateArgs;
2356
2357
16.2k
  FunctionArgList::const_iterator I = Args.begin(), E = Args.end();
2358
16.2k
  assert(I != E && "no parameters to constructor");
2359
2360
  // this
2361
0
  Address This = LoadCXXThisAddress();
2362
16.2k
  DelegateArgs.add(RValue::get(This.getPointer()), (*I)->getType());
2363
16.2k
  ++I;
2364
2365
  // FIXME: The location of the VTT parameter in the parameter list is
2366
  // specific to the Itanium ABI and shouldn't be hardcoded here.
2367
16.2k
  if (CGM.getCXXABI().NeedsVTTParameter(CurGD)) {
2368
0
    assert(I != E && "cannot skip vtt parameter, already done with args");
2369
0
    assert((*I)->getType()->isPointerType() &&
2370
0
           "skipping parameter not of vtt type");
2371
0
    ++I;
2372
0
  }
2373
2374
  // Explicit arguments.
2375
34.1k
  for (; I != E; 
++I17.8k
) {
2376
17.8k
    const VarDecl *param = *I;
2377
    // FIXME: per-argument source location
2378
17.8k
    EmitDelegateCallArg(DelegateArgs, param, Loc);
2379
17.8k
  }
2380
2381
16.2k
  EmitCXXConstructorCall(Ctor, CtorType, /*ForVirtualBase=*/false,
2382
16.2k
                         /*Delegating=*/true, This, DelegateArgs,
2383
16.2k
                         AggValueSlot::MayOverlap, Loc,
2384
16.2k
                         /*NewPointerIsChecked=*/true);
2385
16.2k
}
2386
2387
namespace {
2388
  struct CallDelegatingCtorDtor final : EHScopeStack::Cleanup {
2389
    const CXXDestructorDecl *Dtor;
2390
    Address Addr;
2391
    CXXDtorType Type;
2392
2393
    CallDelegatingCtorDtor(const CXXDestructorDecl *D, Address Addr,
2394
                           CXXDtorType Type)
2395
68
      : Dtor(D), Addr(Addr), Type(Type) {}
2396
2397
4
    void Emit(CodeGenFunction &CGF, Flags flags) override {
2398
      // We are calling the destructor from within the constructor.
2399
      // Therefore, "this" should have the expected type.
2400
4
      QualType ThisTy = Dtor->getThisObjectType();
2401
4
      CGF.EmitCXXDestructorCall(Dtor, Type, /*ForVirtualBase=*/false,
2402
4
                                /*Delegating=*/true, Addr, ThisTy);
2403
4
    }
2404
  };
2405
} // end anonymous namespace
2406
2407
void
2408
CodeGenFunction::EmitDelegatingCXXConstructorCall(const CXXConstructorDecl *Ctor,
2409
94
                                                  const FunctionArgList &Args) {
2410
94
  assert(Ctor->isDelegatingConstructor());
2411
2412
0
  Address ThisPtr = LoadCXXThisAddress();
2413
2414
94
  AggValueSlot AggSlot =
2415
94
    AggValueSlot::forAddr(ThisPtr, Qualifiers(),
2416
94
                          AggValueSlot::IsDestructed,
2417
94
                          AggValueSlot::DoesNotNeedGCBarriers,
2418
94
                          AggValueSlot::IsNotAliased,
2419
94
                          AggValueSlot::MayOverlap,
2420
94
                          AggValueSlot::IsNotZeroed,
2421
                          // Checks are made by the code that calls constructor.
2422
94
                          AggValueSlot::IsSanitizerChecked);
2423
2424
94
  EmitAggExpr(Ctor->init_begin()[0]->getInit(), AggSlot);
2425
2426
94
  const CXXRecordDecl *ClassDecl = Ctor->getParent();
2427
94
  if (CGM.getLangOpts().Exceptions && 
!ClassDecl->hasTrivialDestructor()92
) {
2428
68
    CXXDtorType Type =
2429
68
      CurGD.getCtorType() == Ctor_Complete ? 
Dtor_Complete65
:
Dtor_Base3
;
2430
2431
68
    EHStack.pushCleanup<CallDelegatingCtorDtor>(EHCleanup,
2432
68
                                                ClassDecl->getDestructor(),
2433
68
                                                ThisPtr, Type);
2434
68
  }
2435
94
}
2436
2437
void CodeGenFunction::EmitCXXDestructorCall(const CXXDestructorDecl *DD,
2438
                                            CXXDtorType Type,
2439
                                            bool ForVirtualBase,
2440
                                            bool Delegating, Address This,
2441
30.1k
                                            QualType ThisTy) {
2442
30.1k
  CGM.getCXXABI().EmitDestructorCall(*this, DD, Type, ForVirtualBase,
2443
30.1k
                                     Delegating, This, ThisTy);
2444
30.1k
}
2445
2446
namespace {
2447
  struct CallLocalDtor final : EHScopeStack::Cleanup {
2448
    const CXXDestructorDecl *Dtor;
2449
    Address Addr;
2450
    QualType Ty;
2451
2452
    CallLocalDtor(const CXXDestructorDecl *D, Address Addr, QualType Ty)
2453
10
        : Dtor(D), Addr(Addr), Ty(Ty) {}
2454
2455
10
    void Emit(CodeGenFunction &CGF, Flags flags) override {
2456
10
      CGF.EmitCXXDestructorCall(Dtor, Dtor_Complete,
2457
10
                                /*ForVirtualBase=*/false,
2458
10
                                /*Delegating=*/false, Addr, Ty);
2459
10
    }
2460
  };
2461
} // end anonymous namespace
2462
2463
void CodeGenFunction::PushDestructorCleanup(const CXXDestructorDecl *D,
2464
10
                                            QualType T, Address Addr) {
2465
10
  EHStack.pushCleanup<CallLocalDtor>(NormalAndEHCleanup, D, Addr, T);
2466
10
}
2467
2468
13
void CodeGenFunction::PushDestructorCleanup(QualType T, Address Addr) {
2469
13
  CXXRecordDecl *ClassDecl = T->getAsCXXRecordDecl();
2470
13
  if (!ClassDecl) 
return0
;
2471
13
  if (ClassDecl->hasTrivialDestructor()) 
return3
;
2472
2473
10
  const CXXDestructorDecl *D = ClassDecl->getDestructor();
2474
10
  assert(D && D->isUsed() && "destructor not marked as used!");
2475
0
  PushDestructorCleanup(D, T, Addr);
2476
10
}
2477
2478
3.98k
void CodeGenFunction::InitializeVTablePointer(const VPtr &Vptr) {
2479
  // Compute the address point.
2480
3.98k
  llvm::Value *VTableAddressPoint =
2481
3.98k
      CGM.getCXXABI().getVTableAddressPointInStructor(
2482
3.98k
          *this, Vptr.VTableClass, Vptr.Base, Vptr.NearestVBase);
2483
2484
3.98k
  if (!VTableAddressPoint)
2485
719
    return;
2486
2487
  // Compute where to store the address point.
2488
3.26k
  llvm::Value *VirtualOffset = nullptr;
2489
3.26k
  CharUnits NonVirtualOffset = CharUnits::Zero();
2490
2491
3.26k
  if (CGM.getCXXABI().isVirtualOffsetNeededForVTableField(*this, Vptr)) {
2492
    // We need to use the virtual base offset offset because the virtual base
2493
    // might have a different offset in the most derived class.
2494
2495
355
    VirtualOffset = CGM.getCXXABI().GetVirtualBaseClassOffset(
2496
355
        *this, LoadCXXThisAddress(), Vptr.VTableClass, Vptr.NearestVBase);
2497
355
    NonVirtualOffset = Vptr.OffsetFromNearestVBase;
2498
2.90k
  } else {
2499
    // We can just use the base offset in the complete class.
2500
2.90k
    NonVirtualOffset = Vptr.Base.getBaseOffset();
2501
2.90k
  }
2502
2503
  // Apply the offsets.
2504
3.26k
  Address VTableField = LoadCXXThisAddress();
2505
3.26k
  unsigned ThisAddrSpace =
2506
3.26k
      VTableField.getPointer()->getType()->getPointerAddressSpace();
2507
2508
3.26k
  if (!NonVirtualOffset.isZero() || 
VirtualOffset3.02k
)
2509
561
    VTableField = ApplyNonVirtualAndVirtualOffset(
2510
561
        *this, VTableField, NonVirtualOffset, VirtualOffset, Vptr.VTableClass,
2511
561
        Vptr.NearestVBase);
2512
2513
  // Finally, store the address point. Use the same LLVM types as the field to
2514
  // support optimization.
2515
3.26k
  unsigned GlobalsAS = CGM.getDataLayout().getDefaultGlobalsAddressSpace();
2516
3.26k
  unsigned ProgAS = CGM.getDataLayout().getProgramAddressSpace();
2517
3.26k
  llvm::Type *VTablePtrTy =
2518
3.26k
      llvm::FunctionType::get(CGM.Int32Ty, /*isVarArg=*/true)
2519
3.26k
          ->getPointerTo(ProgAS)
2520
3.26k
          ->getPointerTo(GlobalsAS);
2521
  // vtable field is is derived from `this` pointer, therefore they should be in
2522
  // the same addr space. Note that this might not be LLVM address space 0.
2523
3.26k
  VTableField = Builder.CreateBitCast(VTableField,
2524
3.26k
                                      VTablePtrTy->getPointerTo(ThisAddrSpace));
2525
3.26k
  VTableAddressPoint = Builder.CreateBitCast(VTableAddressPoint, VTablePtrTy);
2526
2527
3.26k
  llvm::StoreInst *Store = Builder.CreateStore(VTableAddressPoint, VTableField);
2528
3.26k
  TBAAAccessInfo TBAAInfo = CGM.getTBAAVTablePtrAccessInfo(VTablePtrTy);
2529
3.26k
  CGM.DecorateInstructionWithTBAA(Store, TBAAInfo);
2530
3.26k
  if (CGM.getCodeGenOpts().OptimizationLevel > 0 &&
2531
3.26k
      
CGM.getCodeGenOpts().StrictVTablePointers415
)
2532
54
    CGM.DecorateInstructionWithInvariantGroup(Store, Vptr.VTableClass);
2533
3.26k
}
2534
2535
CodeGenFunction::VPtrsVector
2536
3.03k
CodeGenFunction::getVTablePointers(const CXXRecordDecl *VTableClass) {
2537
3.03k
  CodeGenFunction::VPtrsVector VPtrsResult;
2538
3.03k
  VisitedVirtualBasesSetTy VBases;
2539
3.03k
  getVTablePointers(BaseSubobject(VTableClass, CharUnits::Zero()),
2540
3.03k
                    /*NearestVBase=*/nullptr,
2541
3.03k
                    /*OffsetFromNearestVBase=*/CharUnits::Zero(),
2542
3.03k
                    /*BaseIsNonVirtualPrimaryBase=*/false, VTableClass, VBases,
2543
3.03k
                    VPtrsResult);
2544
3.03k
  return VPtrsResult;
2545
3.03k
}
2546
2547
void CodeGenFunction::getVTablePointers(BaseSubobject Base,
2548
                                        const CXXRecordDecl *NearestVBase,
2549
                                        CharUnits OffsetFromNearestVBase,
2550
                                        bool BaseIsNonVirtualPrimaryBase,
2551
                                        const CXXRecordDecl *VTableClass,
2552
                                        VisitedVirtualBasesSetTy &VBases,
2553
5.79k
                                        VPtrsVector &Vptrs) {
2554
  // If this base is a non-virtual primary base the address point has already
2555
  // been set.
2556
5.79k
  if (!BaseIsNonVirtualPrimaryBase) {
2557
    // Initialize the vtable pointer for this base.
2558
4.04k
    VPtr Vptr = {Base, NearestVBase, OffsetFromNearestVBase, VTableClass};
2559
4.04k
    Vptrs.push_back(Vptr);
2560
4.04k
  }
2561
2562
5.79k
  const CXXRecordDecl *RD = Base.getBase();
2563
2564
  // Traverse bases.
2565
5.79k
  for (const auto &I : RD->bases()) {
2566
3.52k
    auto *BaseDecl =
2567
3.52k
        cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
2568
2569
    // Ignore classes without a vtable.
2570
3.52k
    if (!BaseDecl->isDynamicClass())
2571
693
      continue;
2572
2573
2.83k
    CharUnits BaseOffset;
2574
2.83k
    CharUnits BaseOffsetFromNearestVBase;
2575
2.83k
    bool BaseDeclIsNonVirtualPrimaryBase;
2576
2577
2.83k
    if (I.isVirtual()) {
2578
      // Check if we've visited this virtual base before.
2579
678
      if (!VBases.insert(BaseDecl).second)
2580
72
        continue;
2581
2582
606
      const ASTRecordLayout &Layout =
2583
606
        getContext().getASTRecordLayout(VTableClass);
2584
2585
606
      BaseOffset = Layout.getVBaseClassOffset(BaseDecl);
2586
606
      BaseOffsetFromNearestVBase = CharUnits::Zero();
2587
606
      BaseDeclIsNonVirtualPrimaryBase = false;
2588
2.15k
    } else {
2589
2.15k
      const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
2590
2591
2.15k
      BaseOffset = Base.getBaseOffset() + Layout.getBaseClassOffset(BaseDecl);
2592
2.15k
      BaseOffsetFromNearestVBase =
2593
2.15k
        OffsetFromNearestVBase + Layout.getBaseClassOffset(BaseDecl);
2594
2.15k
      BaseDeclIsNonVirtualPrimaryBase = Layout.getPrimaryBase() == BaseDecl;
2595
2.15k
    }
2596
2597
2.76k
    getVTablePointers(
2598
2.76k
        BaseSubobject(BaseDecl, BaseOffset),
2599
2.76k
        I.isVirtual() ? 
BaseDecl606
:
NearestVBase2.15k
, BaseOffsetFromNearestVBase,
2600
2.76k
        BaseDeclIsNonVirtualPrimaryBase, VTableClass, VBases, Vptrs);
2601
2.76k
  }
2602
5.79k
}
2603
2604
24.6k
void CodeGenFunction::InitializeVTablePointers(const CXXRecordDecl *RD) {
2605
  // Ignore classes without a vtable.
2606
24.6k
  if (!RD->isDynamicClass())
2607
21.6k
    return;
2608
2609
  // Initialize the vtable pointers for this class and all of its bases.
2610
2.97k
  if (CGM.getCXXABI().doStructorsInitializeVPtrs(RD))
2611
2.97k
    for (const VPtr &Vptr : getVTablePointers(RD))
2612
3.98k
      InitializeVTablePointer(Vptr);
2613
2614
2.97k
  if (RD->getNumVBases())
2615
817
    CGM.getCXXABI().initializeHiddenVirtualInheritanceMembers(*this, RD);
2616
2.97k
}
2617
2618
llvm::Value *CodeGenFunction::GetVTablePtr(Address This,
2619
                                           llvm::Type *VTableTy,
2620
1.76k
                                           const CXXRecordDecl *RD) {
2621
1.76k
  Address VTablePtrSrc = Builder.CreateElementBitCast(This, VTableTy);
2622
1.76k
  llvm::Instruction *VTable = Builder.CreateLoad(VTablePtrSrc, "vtable");
2623
1.76k
  TBAAAccessInfo TBAAInfo = CGM.getTBAAVTablePtrAccessInfo(VTableTy);
2624
1.76k
  CGM.DecorateInstructionWithTBAA(VTable, TBAAInfo);
2625
2626
1.76k
  if (CGM.getCodeGenOpts().OptimizationLevel > 0 &&
2627
1.76k
      
CGM.getCodeGenOpts().StrictVTablePointers205
)
2628
118
    CGM.DecorateInstructionWithInvariantGroup(VTable, RD);
2629
2630
1.76k
  return VTable;
2631
1.76k
}
2632
2633
// If a class has a single non-virtual base and does not introduce or override
2634
// virtual member functions or fields, it will have the same layout as its base.
2635
// This function returns the least derived such class.
2636
//
2637
// Casting an instance of a base class to such a derived class is technically
2638
// undefined behavior, but it is a relatively common hack for introducing member
2639
// functions on class instances with specific properties (e.g. llvm::Operator)
2640
// that works under most compilers and should not have security implications, so
2641
// we allow it by default. It can be disabled with -fsanitize=cfi-cast-strict.
2642
static const CXXRecordDecl *
2643
60
LeastDerivedClassWithSameLayout(const CXXRecordDecl *RD) {
2644
60
  if (!RD->field_empty())
2645
1
    return RD;
2646
2647
59
  if (RD->getNumVBases() != 0)
2648
12
    return RD;
2649
2650
47
  if (RD->getNumBases() != 1)
2651
32
    return RD;
2652
2653
28
  
for (const CXXMethodDecl *MD : RD->methods())15
{
2654
28
    if (MD->isVirtual()) {
2655
      // Virtual member functions are only ok if they are implicit destructors
2656
      // because the implicit destructor will have the same semantics as the
2657
      // base class's destructor if no fields are added.
2658
9
      if (isa<CXXDestructorDecl>(MD) && 
MD->isImplicit()0
)
2659
0
        continue;
2660
9
      return RD;
2661
9
    }
2662
28
  }
2663
2664
6
  return LeastDerivedClassWithSameLayout(
2665
6
      RD->bases_begin()->getType()->getAsCXXRecordDecl());
2666
15
}
2667
2668
void CodeGenFunction::EmitTypeMetadataCodeForVCall(const CXXRecordDecl *RD,
2669
                                                   llvm::Value *VTable,
2670
853
                                                   SourceLocation Loc) {
2671
853
  if (SanOpts.has(SanitizerKind::CFIVCall))
2672
33
    EmitVTablePtrCheckForCall(RD, VTable, CodeGenFunction::CFITCK_VCall, Loc);
2673
820
  else if (CGM.getCodeGenOpts().WholeProgramVTables &&
2674
           // Don't insert type test assumes if we are forcing public std
2675
           // visibility.
2676
820
           
!CGM.HasLTOVisibilityPublicStd(RD)73
) {
2677
70
    llvm::Metadata *MD =
2678
70
        CGM.CreateMetadataIdentifierForType(QualType(RD->getTypeForDecl(), 0));
2679
70
    llvm::Value *TypeId =
2680
70
        llvm::MetadataAsValue::get(CGM.getLLVMContext(), MD);
2681
2682
70
    llvm::Value *CastedVTable = Builder.CreateBitCast(VTable, Int8PtrTy);
2683
70
    llvm::Value *TypeTest =
2684
70
        Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::type_test),
2685
70
                           {CastedVTable, TypeId});
2686
70
    Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::assume), TypeTest);
2687
70
  }
2688
853
}
2689
2690
void CodeGenFunction::EmitVTablePtrCheckForCall(const CXXRecordDecl *RD,
2691
                                                llvm::Value *VTable,
2692
                                                CFITypeCheckKind TCK,
2693
41
                                                SourceLocation Loc) {
2694
41
  if (!SanOpts.has(SanitizerKind::CFICastStrict))
2695
39
    RD = LeastDerivedClassWithSameLayout(RD);
2696
2697
41
  EmitVTablePtrCheck(RD, VTable, TCK, Loc);
2698
41
}
2699
2700
void CodeGenFunction::EmitVTablePtrCheckForCast(QualType T,
2701
                                                llvm::Value *Derived,
2702
                                                bool MayBeNull,
2703
                                                CFITypeCheckKind TCK,
2704
21
                                                SourceLocation Loc) {
2705
21
  if (!getLangOpts().CPlusPlus)
2706
0
    return;
2707
2708
21
  auto *ClassTy = T->getAs<RecordType>();
2709
21
  if (!ClassTy)
2710
0
    return;
2711
2712
21
  const CXXRecordDecl *ClassDecl = cast<CXXRecordDecl>(ClassTy->getDecl());
2713
2714
21
  if (!ClassDecl->isCompleteDefinition() || !ClassDecl->isDynamicClass())
2715
0
    return;
2716
2717
21
  if (!SanOpts.has(SanitizerKind::CFICastStrict))
2718
15
    ClassDecl = LeastDerivedClassWithSameLayout(ClassDecl);
2719
2720
21
  llvm::BasicBlock *ContBlock = nullptr;
2721
2722
21
  if (MayBeNull) {
2723
14
    llvm::Value *DerivedNotNull =
2724
14
        Builder.CreateIsNotNull(Derived, "cast.nonnull");
2725
2726
14
    llvm::BasicBlock *CheckBlock = createBasicBlock("cast.check");
2727
14
    ContBlock = createBasicBlock("cast.cont");
2728
2729
14
    Builder.CreateCondBr(DerivedNotNull, CheckBlock, ContBlock);
2730
2731
14
    EmitBlock(CheckBlock);
2732
14
  }
2733
2734
21
  llvm::Value *VTable;
2735
21
  std::tie(VTable, ClassDecl) = CGM.getCXXABI().LoadVTablePtr(
2736
21
      *this, Address(Derived, getPointerAlign()), ClassDecl);
2737
2738
21
  EmitVTablePtrCheck(ClassDecl, VTable, TCK, Loc);
2739
2740
21
  if (MayBeNull) {
2741
14
    Builder.CreateBr(ContBlock);
2742
14
    EmitBlock(ContBlock);
2743
14
  }
2744
21
}
2745
2746
void CodeGenFunction::EmitVTablePtrCheck(const CXXRecordDecl *RD,
2747
                                         llvm::Value *VTable,
2748
                                         CFITypeCheckKind TCK,
2749
62
                                         SourceLocation Loc) {
2750
62
  if (!CGM.getCodeGenOpts().SanitizeCfiCrossDso &&
2751
62
      
!CGM.HasHiddenLTOVisibility(RD)60
)
2752
0
    return;
2753
2754
62
  SanitizerMask M;
2755
62
  llvm::SanitizerStatKind SSK;
2756
62
  switch (TCK) {
2757
33
  case CFITCK_VCall:
2758
33
    M = SanitizerKind::CFIVCall;
2759
33
    SSK = llvm::SanStat_CFI_VCall;
2760
33
    break;
2761
8
  case CFITCK_NVCall:
2762
8
    M = SanitizerKind::CFINVCall;
2763
8
    SSK = llvm::SanStat_CFI_NVCall;
2764
8
    break;
2765
6
  case CFITCK_DerivedCast:
2766
6
    M = SanitizerKind::CFIDerivedCast;
2767
6
    SSK = llvm::SanStat_CFI_DerivedCast;
2768
6
    break;
2769
15
  case CFITCK_UnrelatedCast:
2770
15
    M = SanitizerKind::CFIUnrelatedCast;
2771
15
    SSK = llvm::SanStat_CFI_UnrelatedCast;
2772
15
    break;
2773
0
  case CFITCK_ICall:
2774
0
  case CFITCK_NVMFCall:
2775
0
  case CFITCK_VMFCall:
2776
0
    llvm_unreachable("unexpected sanitizer kind");
2777
62
  }
2778
2779
62
  std::string TypeName = RD->getQualifiedNameAsString();
2780
62
  if (getContext().getNoSanitizeList().containsType(M, TypeName))
2781
2
    return;
2782
2783
60
  SanitizerScope SanScope(this);
2784
60
  EmitSanitizerStatReport(SSK);
2785
2786
60
  llvm::Metadata *MD =
2787
60
      CGM.CreateMetadataIdentifierForType(QualType(RD->getTypeForDecl(), 0));
2788
60
  llvm::Value *TypeId = llvm::MetadataAsValue::get(getLLVMContext(), MD);
2789
2790
60
  llvm::Value *CastedVTable = Builder.CreateBitCast(VTable, Int8PtrTy);
2791
60
  llvm::Value *TypeTest = Builder.CreateCall(
2792
60
      CGM.getIntrinsic(llvm::Intrinsic::type_test), {CastedVTable, TypeId});
2793
2794
60
  llvm::Constant *StaticData[] = {
2795
60
      llvm::ConstantInt::get(Int8Ty, TCK),
2796
60
      EmitCheckSourceLocation(Loc),
2797
60
      EmitCheckTypeDescriptor(QualType(RD->getTypeForDecl(), 0)),
2798
60
  };
2799
2800
60
  auto CrossDsoTypeId = CGM.CreateCrossDsoCfiTypeId(MD);
2801
60
  if (CGM.getCodeGenOpts().SanitizeCfiCrossDso && 
CrossDsoTypeId2
) {
2802
2
    EmitCfiSlowPathCheck(M, TypeTest, CrossDsoTypeId, CastedVTable, StaticData);
2803
2
    return;
2804
2
  }
2805
2806
58
  if (CGM.getCodeGenOpts().SanitizeTrap.has(M)) {
2807
28
    EmitTrapCheck(TypeTest, SanitizerHandler::CFICheckFail);
2808
28
    return;
2809
28
  }
2810
2811
30
  llvm::Value *AllVtables = llvm::MetadataAsValue::get(
2812
30
      CGM.getLLVMContext(),
2813
30
      llvm::MDString::get(CGM.getLLVMContext(), "all-vtables"));
2814
30
  llvm::Value *ValidVtable = Builder.CreateCall(
2815
30
      CGM.getIntrinsic(llvm::Intrinsic::type_test), {CastedVTable, AllVtables});
2816
30
  EmitCheck(std::make_pair(TypeTest, M), SanitizerHandler::CFICheckFail,
2817
30
            StaticData, {CastedVTable, ValidVtable});
2818
30
}
2819
2820
1.00k
bool CodeGenFunction::ShouldEmitVTableTypeCheckedLoad(const CXXRecordDecl *RD) {
2821
1.00k
  if (!CGM.getCodeGenOpts().WholeProgramVTables ||
2822
1.00k
      
!CGM.HasHiddenLTOVisibility(RD)87
)
2823
947
    return false;
2824
2825
62
  if (CGM.getCodeGenOpts().VirtualFunctionElimination)
2826
2
    return true;
2827
2828
60
  if (!SanOpts.has(SanitizerKind::CFIVCall) ||
2829
60
      
!CGM.getCodeGenOpts().SanitizeTrap.has(SanitizerKind::CFIVCall)12
)
2830
49
    return false;
2831
2832
11
  std::string TypeName = RD->getQualifiedNameAsString();
2833
11
  return !getContext().getNoSanitizeList().containsType(SanitizerKind::CFIVCall,
2834
11
                                                        TypeName);
2835
60
}
2836
2837
llvm::Value *CodeGenFunction::EmitVTableTypeCheckedLoad(
2838
13
    const CXXRecordDecl *RD, llvm::Value *VTable, uint64_t VTableByteOffset) {
2839
13
  SanitizerScope SanScope(this);
2840
2841
13
  EmitSanitizerStatReport(llvm::SanStat_CFI_VCall);
2842
2843
13
  llvm::Metadata *MD =
2844
13
      CGM.CreateMetadataIdentifierForType(QualType(RD->getTypeForDecl(), 0));
2845
13
  llvm::Value *TypeId = llvm::MetadataAsValue::get(CGM.getLLVMContext(), MD);
2846
2847
13
  llvm::Value *CastedVTable = Builder.CreateBitCast(VTable, Int8PtrTy);
2848
13
  llvm::Value *CheckedLoad = Builder.CreateCall(
2849
13
      CGM.getIntrinsic(llvm::Intrinsic::type_checked_load),
2850
13
      {CastedVTable, llvm::ConstantInt::get(Int32Ty, VTableByteOffset),
2851
13
       TypeId});
2852
13
  llvm::Value *CheckResult = Builder.CreateExtractValue(CheckedLoad, 1);
2853
2854
13
  std::string TypeName = RD->getQualifiedNameAsString();
2855
13
  if (SanOpts.has(SanitizerKind::CFIVCall) &&
2856
13
      !getContext().getNoSanitizeList().containsType(SanitizerKind::CFIVCall,
2857
11
                                                     TypeName)) {
2858
11
    EmitCheck(std::make_pair(CheckResult, SanitizerKind::CFIVCall),
2859
11
              SanitizerHandler::CFICheckFail, {}, {});
2860
11
  }
2861
2862
13
  return Builder.CreateBitCast(
2863
13
      Builder.CreateExtractValue(CheckedLoad, 0),
2864
13
      cast<llvm::PointerType>(VTable->getType())->getElementType());
2865
13
}
2866
2867
void CodeGenFunction::EmitForwardingCallToLambda(
2868
                                      const CXXMethodDecl *callOperator,
2869
73
                                      CallArgList &callArgs) {
2870
  // Get the address of the call operator.
2871
73
  const CGFunctionInfo &calleeFnInfo =
2872
73
    CGM.getTypes().arrangeCXXMethodDeclaration(callOperator);
2873
73
  llvm::Constant *calleePtr =
2874
73
    CGM.GetAddrOfFunction(GlobalDecl(callOperator),
2875
73
                          CGM.getTypes().GetFunctionType(calleeFnInfo));
2876
2877
  // Prepare the return slot.
2878
73
  const FunctionProtoType *FPT =
2879
73
    callOperator->getType()->castAs<FunctionProtoType>();
2880
73
  QualType resultType = FPT->getReturnType();
2881
73
  ReturnValueSlot returnSlot;
2882
73
  if (!resultType->isVoidType() &&
2883
73
      
calleeFnInfo.getReturnInfo().getKind() == ABIArgInfo::Indirect37
&&
2884
73
      
!hasScalarEvaluationKind(calleeFnInfo.getReturnType())1
)
2885
1
    returnSlot =
2886
1
        ReturnValueSlot(ReturnValue, resultType.isVolatileQualified(),
2887
1
                        /*IsUnused=*/false, /*IsExternallyDestructed=*/true);
2888
2889
  // We don't need to separately arrange the call arguments because
2890
  // the call can't be variadic anyway --- it's impossible to forward
2891
  // variadic arguments.
2892
2893
  // Now emit our call.
2894
73
  auto callee = CGCallee::forDirect(calleePtr, GlobalDecl(callOperator));
2895
73
  RValue RV = EmitCall(calleeFnInfo, callee, returnSlot, callArgs);
2896
2897
  // If necessary, copy the returned value into the slot.
2898
73
  if (!resultType->isVoidType() && 
returnSlot.isNull()37
) {
2899
36
    if (getLangOpts().ObjCAutoRefCount && 
resultType->isObjCRetainableType()5
) {
2900
2
      RV = RValue::get(EmitARCRetainAutoreleasedReturnValue(RV.getScalarVal()));
2901
2
    }
2902
36
    EmitReturnOfRValue(RV, resultType);
2903
36
  } else
2904
37
    EmitBranchThroughCleanup(ReturnBlock);
2905
73
}
2906
2907
13
void CodeGenFunction::EmitLambdaBlockInvokeBody() {
2908
13
  const BlockDecl *BD = BlockInfo->getBlockDecl();
2909
13
  const VarDecl *variable = BD->capture_begin()->getVariable();
2910
13
  const CXXRecordDecl *Lambda = variable->getType()->getAsCXXRecordDecl();
2911
13
  const CXXMethodDecl *CallOp = Lambda->getLambdaCallOperator();
2912
2913
13
  if (CallOp->isVariadic()) {
2914
    // FIXME: Making this work correctly is nasty because it requires either
2915
    // cloning the body of the call operator or making the call operator
2916
    // forward.
2917
0
    CGM.ErrorUnsupported(CurCodeDecl, "lambda conversion to variadic function");
2918
0
    return;
2919
0
  }
2920
2921
  // Start building arguments for forwarding call
2922
13
  CallArgList CallArgs;
2923
2924
13
  QualType ThisType = getContext().getPointerType(getContext().getRecordType(Lambda));
2925
13
  Address ThisPtr = GetAddrOfBlockDecl(variable);
2926
13
  CallArgs.add(RValue::get(ThisPtr.getPointer()), ThisType);
2927
2928
  // Add the rest of the parameters.
2929
13
  for (auto param : BD->parameters())
2930
1
    EmitDelegateCallArg(CallArgs, param, param->getBeginLoc());
2931
2932
13
  assert(!Lambda->isGenericLambda() &&
2933
13
            "generic lambda interconversion to block not implemented");
2934
0
  EmitForwardingCallToLambda(CallOp, CallArgs);
2935
13
}
2936
2937
60
void CodeGenFunction::EmitLambdaDelegatingInvokeBody(const CXXMethodDecl *MD) {
2938
60
  const CXXRecordDecl *Lambda = MD->getParent();
2939
2940
  // Start building arguments for forwarding call
2941
60
  CallArgList CallArgs;
2942
2943
60
  QualType ThisType = getContext().getPointerType(getContext().getRecordType(Lambda));
2944
60
  llvm::Value *ThisPtr = llvm::UndefValue::get(getTypes().ConvertType(ThisType));
2945
60
  CallArgs.add(RValue::get(ThisPtr), ThisType);
2946
2947
  // Add the rest of the parameters.
2948
60
  for (auto Param : MD->parameters())
2949
140
    EmitDelegateCallArg(CallArgs, Param, Param->getBeginLoc());
2950
2951
60
  const CXXMethodDecl *CallOp = Lambda->getLambdaCallOperator();
2952
  // For a generic lambda, find the corresponding call operator specialization
2953
  // to which the call to the static-invoker shall be forwarded.
2954
60
  if (Lambda->isGenericLambda()) {
2955
0
    assert(MD->isFunctionTemplateSpecialization());
2956
0
    const TemplateArgumentList *TAL = MD->getTemplateSpecializationArgs();
2957
0
    FunctionTemplateDecl *CallOpTemplate = CallOp->getDescribedFunctionTemplate();
2958
0
    void *InsertPos = nullptr;
2959
0
    FunctionDecl *CorrespondingCallOpSpecialization =
2960
0
        CallOpTemplate->findSpecialization(TAL->asArray(), InsertPos);
2961
0
    assert(CorrespondingCallOpSpecialization);
2962
0
    CallOp = cast<CXXMethodDecl>(CorrespondingCallOpSpecialization);
2963
0
  }
2964
0
  EmitForwardingCallToLambda(CallOp, CallArgs);
2965
60
}
2966
2967
60
void CodeGenFunction::EmitLambdaStaticInvokeBody(const CXXMethodDecl *MD) {
2968
60
  if (MD->isVariadic()) {
2969
    // FIXME: Making this work correctly is nasty because it requires either
2970
    // cloning the body of the call operator or making the call operator forward.
2971
0
    CGM.ErrorUnsupported(MD, "lambda conversion to variadic function");
2972
0
    return;
2973
0
  }
2974
2975
60
  EmitLambdaDelegatingInvokeBody(MD);
2976
60
}