Coverage Report

Created: 2022-01-25 06:29

/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/CodeGen/CGClass.cpp
Line
Count
Source (jump to first uncovered line)
1
//===--- CGClass.cpp - Emit LLVM Code for C++ classes -----------*- C++ -*-===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This contains code dealing with C++ code generation of classes
10
//
11
//===----------------------------------------------------------------------===//
12
13
#include "CGBlocks.h"
14
#include "CGCXXABI.h"
15
#include "CGDebugInfo.h"
16
#include "CGRecordLayout.h"
17
#include "CodeGenFunction.h"
18
#include "TargetInfo.h"
19
#include "clang/AST/Attr.h"
20
#include "clang/AST/CXXInheritance.h"
21
#include "clang/AST/CharUnits.h"
22
#include "clang/AST/DeclTemplate.h"
23
#include "clang/AST/EvaluatedExprVisitor.h"
24
#include "clang/AST/RecordLayout.h"
25
#include "clang/AST/StmtCXX.h"
26
#include "clang/Basic/CodeGenOptions.h"
27
#include "clang/Basic/TargetBuiltins.h"
28
#include "clang/CodeGen/CGFunctionInfo.h"
29
#include "llvm/IR/Intrinsics.h"
30
#include "llvm/IR/Metadata.h"
31
#include "llvm/Transforms/Utils/SanitizerStats.h"
32
33
using namespace clang;
34
using namespace CodeGen;
35
36
/// Return the best known alignment for an unknown pointer to a
37
/// particular class.
38
751k
CharUnits CodeGenModule::getClassPointerAlignment(const CXXRecordDecl *RD) {
39
751k
  if (!RD->hasDefinition())
40
0
    return CharUnits::One(); // Hopefully won't be used anywhere.
41
42
751k
  auto &layout = getContext().getASTRecordLayout(RD);
43
44
  // If the class is final, then we know that the pointer points to an
45
  // object of that type and can use the full alignment.
46
751k
  if (RD->isEffectivelyFinal())
47
181
    return layout.getAlignment();
48
49
  // Otherwise, we have to assume it could be a subclass.
50
751k
  return layout.getNonVirtualAlignment();
51
751k
}
52
53
/// Return the smallest possible amount of storage that might be allocated
54
/// starting from the beginning of an object of a particular class.
55
///
56
/// This may be smaller than sizeof(RD) if RD has virtual base classes.
57
391k
CharUnits CodeGenModule::getMinimumClassObjectSize(const CXXRecordDecl *RD) {
58
391k
  if (!RD->hasDefinition())
59
2
    return CharUnits::One();
60
61
391k
  auto &layout = getContext().getASTRecordLayout(RD);
62
63
  // If the class is final, then we know that the pointer points to an
64
  // object of that type and can use the full alignment.
65
391k
  if (RD->isEffectivelyFinal())
66
86
    return layout.getSize();
67
68
  // Otherwise, we have to assume it could be a subclass.
69
391k
  return std::max(layout.getNonVirtualSize(), CharUnits::One());
70
391k
}
71
72
/// Return the best known alignment for a pointer to a virtual base,
73
/// given the alignment of a pointer to the derived class.
74
CharUnits CodeGenModule::getVBaseAlignment(CharUnits actualDerivedAlign,
75
                                           const CXXRecordDecl *derivedClass,
76
909
                                           const CXXRecordDecl *vbaseClass) {
77
  // The basic idea here is that an underaligned derived pointer might
78
  // indicate an underaligned base pointer.
79
80
909
  assert(vbaseClass->isCompleteDefinition());
81
0
  auto &baseLayout = getContext().getASTRecordLayout(vbaseClass);
82
909
  CharUnits expectedVBaseAlign = baseLayout.getNonVirtualAlignment();
83
84
909
  return getDynamicOffsetAlignment(actualDerivedAlign, derivedClass,
85
909
                                   expectedVBaseAlign);
86
909
}
87
88
CharUnits
89
CodeGenModule::getDynamicOffsetAlignment(CharUnits actualBaseAlign,
90
                                         const CXXRecordDecl *baseDecl,
91
1.09k
                                         CharUnits expectedTargetAlign) {
92
  // If the base is an incomplete type (which is, alas, possible with
93
  // member pointers), be pessimistic.
94
1.09k
  if (!baseDecl->isCompleteDefinition())
95
5
    return std::min(actualBaseAlign, expectedTargetAlign);
96
97
1.08k
  auto &baseLayout = getContext().getASTRecordLayout(baseDecl);
98
1.08k
  CharUnits expectedBaseAlign = baseLayout.getNonVirtualAlignment();
99
100
  // If the class is properly aligned, assume the target offset is, too.
101
  //
102
  // This actually isn't necessarily the right thing to do --- if the
103
  // class is a complete object, but it's only properly aligned for a
104
  // base subobject, then the alignments of things relative to it are
105
  // probably off as well.  (Note that this requires the alignment of
106
  // the target to be greater than the NV alignment of the derived
107
  // class.)
108
  //
109
  // However, our approach to this kind of under-alignment can only
110
  // ever be best effort; after all, we're never going to propagate
111
  // alignments through variables or parameters.  Note, in particular,
112
  // that constructing a polymorphic type in an address that's less
113
  // than pointer-aligned will generally trap in the constructor,
114
  // unless we someday add some sort of attribute to change the
115
  // assumed alignment of 'this'.  So our goal here is pretty much
116
  // just to allow the user to explicitly say that a pointer is
117
  // under-aligned and then safely access its fields and vtables.
118
1.08k
  if (actualBaseAlign >= expectedBaseAlign) {
119
1.08k
    return expectedTargetAlign;
120
1.08k
  }
121
122
  // Otherwise, we might be offset by an arbitrary multiple of the
123
  // actual alignment.  The correct adjustment is to take the min of
124
  // the two alignments.
125
0
  return std::min(actualBaseAlign, expectedTargetAlign);
126
1.08k
}
127
128
68.0k
Address CodeGenFunction::LoadCXXThisAddress() {
129
68.0k
  assert(CurFuncDecl && "loading 'this' without a func declaration?");
130
0
  auto *MD = cast<CXXMethodDecl>(CurFuncDecl);
131
132
  // Lazily compute CXXThisAlignment.
133
68.0k
  if (CXXThisAlignment.isZero()) {
134
    // Just use the best known alignment for the parent.
135
    // TODO: if we're currently emitting a complete-object ctor/dtor,
136
    // we can always use the complete-object alignment.
137
52.4k
    CXXThisAlignment = CGM.getClassPointerAlignment(MD->getParent());
138
52.4k
  }
139
140
68.0k
  llvm::Type *Ty = ConvertType(MD->getThisType()->getPointeeType());
141
68.0k
  return Address(LoadCXXThis(), Ty, CXXThisAlignment);
142
68.0k
}
143
144
/// Emit the address of a field using a member data pointer.
145
///
146
/// \param E Only used for emergency diagnostics
147
Address
148
CodeGenFunction::EmitCXXMemberDataPointerAddress(const Expr *E, Address base,
149
                                                 llvm::Value *memberPtr,
150
                                      const MemberPointerType *memberPtrType,
151
                                                 LValueBaseInfo *BaseInfo,
152
81
                                                 TBAAAccessInfo *TBAAInfo) {
153
  // Ask the ABI to compute the actual address.
154
81
  llvm::Value *ptr =
155
81
    CGM.getCXXABI().EmitMemberDataPointerAddress(*this, E, base,
156
81
                                                 memberPtr, memberPtrType);
157
158
81
  QualType memberType = memberPtrType->getPointeeType();
159
81
  CharUnits memberAlign =
160
81
      CGM.getNaturalTypeAlignment(memberType, BaseInfo, TBAAInfo);
161
81
  memberAlign =
162
81
    CGM.getDynamicOffsetAlignment(base.getAlignment(),
163
81
                            memberPtrType->getClass()->getAsCXXRecordDecl(),
164
81
                                  memberAlign);
165
81
  return Address(ptr, memberAlign);
166
81
}
167
168
CharUnits CodeGenModule::computeNonVirtualBaseClassOffset(
169
    const CXXRecordDecl *DerivedClass, CastExpr::path_const_iterator Start,
170
25.3k
    CastExpr::path_const_iterator End) {
171
25.3k
  CharUnits Offset = CharUnits::Zero();
172
173
25.3k
  const ASTContext &Context = getContext();
174
25.3k
  const CXXRecordDecl *RD = DerivedClass;
175
176
52.8k
  for (CastExpr::path_const_iterator I = Start; I != End; 
++I27.5k
) {
177
27.5k
    const CXXBaseSpecifier *Base = *I;
178
27.5k
    assert(!Base->isVirtual() && "Should not see virtual bases here!");
179
180
    // Get the layout.
181
0
    const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
182
183
27.5k
    const auto *BaseDecl =
184
27.5k
        cast<CXXRecordDecl>(Base->getType()->castAs<RecordType>()->getDecl());
185
186
    // Add the offset.
187
27.5k
    Offset += Layout.getBaseClassOffset(BaseDecl);
188
189
27.5k
    RD = BaseDecl;
190
27.5k
  }
191
192
25.3k
  return Offset;
193
25.3k
}
194
195
llvm::Constant *
196
CodeGenModule::GetNonVirtualBaseClassOffset(const CXXRecordDecl *ClassDecl,
197
                                   CastExpr::path_const_iterator PathBegin,
198
798
                                   CastExpr::path_const_iterator PathEnd) {
199
798
  assert(PathBegin != PathEnd && "Base path should not be empty!");
200
201
0
  CharUnits Offset =
202
798
      computeNonVirtualBaseClassOffset(ClassDecl, PathBegin, PathEnd);
203
798
  if (Offset.isZero())
204
764
    return nullptr;
205
206
34
  llvm::Type *PtrDiffTy =
207
34
  Types.ConvertType(getContext().getPointerDiffType());
208
209
34
  return llvm::ConstantInt::get(PtrDiffTy, Offset.getQuantity());
210
798
}
211
212
/// Gets the address of a direct base class within a complete object.
213
/// This should only be used for (1) non-virtual bases or (2) virtual bases
214
/// when the type is known to be complete (e.g. in complete destructors).
215
///
216
/// The object pointed to by 'This' is assumed to be non-null.
217
Address
218
CodeGenFunction::GetAddressOfDirectBaseInCompleteClass(Address This,
219
                                                   const CXXRecordDecl *Derived,
220
                                                   const CXXRecordDecl *Base,
221
12.4k
                                                   bool BaseIsVirtual) {
222
  // 'this' must be a pointer (in some address space) to Derived.
223
12.4k
  assert(This.getElementType() == ConvertType(Derived));
224
225
  // Compute the offset of the virtual base.
226
0
  CharUnits Offset;
227
12.4k
  const ASTRecordLayout &Layout = getContext().getASTRecordLayout(Derived);
228
12.4k
  if (BaseIsVirtual)
229
996
    Offset = Layout.getVBaseClassOffset(Base);
230
11.4k
  else
231
11.4k
    Offset = Layout.getBaseClassOffset(Base);
232
233
  // Shift and cast down to the base type.
234
  // TODO: for complete types, this should be possible with a GEP.
235
12.4k
  Address V = This;
236
12.4k
  if (!Offset.isZero()) {
237
2.00k
    V = Builder.CreateElementBitCast(V, Int8Ty);
238
2.00k
    V = Builder.CreateConstInBoundsByteGEP(V, Offset);
239
2.00k
  }
240
12.4k
  V = Builder.CreateElementBitCast(V, ConvertType(Base));
241
242
12.4k
  return V;
243
12.4k
}
244
245
static Address
246
ApplyNonVirtualAndVirtualOffset(CodeGenFunction &CGF, Address addr,
247
                                CharUnits nonVirtualOffset,
248
                                llvm::Value *virtualOffset,
249
                                const CXXRecordDecl *derivedClass,
250
2.11k
                                const CXXRecordDecl *nearestVBase) {
251
  // Assert that we have something to do.
252
2.11k
  assert(!nonVirtualOffset.isZero() || virtualOffset != nullptr);
253
254
  // Compute the offset from the static and dynamic components.
255
0
  llvm::Value *baseOffset;
256
2.11k
  if (!nonVirtualOffset.isZero()) {
257
1.34k
    llvm::Type *OffsetType =
258
1.34k
        (CGF.CGM.getTarget().getCXXABI().isItaniumFamily() &&
259
1.34k
         
CGF.CGM.getItaniumVTableContext().isRelativeLayout()1.20k
)
260
1.34k
            ? 
CGF.Int32Ty4
261
1.34k
            : 
CGF.PtrDiffTy1.34k
;
262
1.34k
    baseOffset =
263
1.34k
        llvm::ConstantInt::get(OffsetType, nonVirtualOffset.getQuantity());
264
1.34k
    if (virtualOffset) {
265
34
      baseOffset = CGF.Builder.CreateAdd(virtualOffset, baseOffset);
266
34
    }
267
1.34k
  } else {
268
770
    baseOffset = virtualOffset;
269
770
  }
270
271
  // Apply the base offset.
272
2.11k
  llvm::Value *ptr = addr.getPointer();
273
2.11k
  unsigned AddrSpace = ptr->getType()->getPointerAddressSpace();
274
2.11k
  ptr = CGF.Builder.CreateBitCast(ptr, CGF.Int8Ty->getPointerTo(AddrSpace));
275
2.11k
  ptr = CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, ptr, baseOffset, "add.ptr");
276
277
  // If we have a virtual component, the alignment of the result will
278
  // be relative only to the known alignment of that vbase.
279
2.11k
  CharUnits alignment;
280
2.11k
  if (virtualOffset) {
281
804
    assert(nearestVBase && "virtual offset without vbase?");
282
0
    alignment = CGF.CGM.getVBaseAlignment(addr.getAlignment(),
283
804
                                          derivedClass, nearestVBase);
284
1.31k
  } else {
285
1.31k
    alignment = addr.getAlignment();
286
1.31k
  }
287
0
  alignment = alignment.alignmentAtOffset(nonVirtualOffset);
288
289
2.11k
  return Address(ptr, CGF.Int8Ty, alignment);
290
2.11k
}
291
292
Address CodeGenFunction::GetAddressOfBaseClass(
293
    Address Value, const CXXRecordDecl *Derived,
294
    CastExpr::path_const_iterator PathBegin,
295
    CastExpr::path_const_iterator PathEnd, bool NullCheckValue,
296
24.4k
    SourceLocation Loc) {
297
24.4k
  assert(PathBegin != PathEnd && "Base path should not be empty!");
298
299
0
  CastExpr::path_const_iterator Start = PathBegin;
300
24.4k
  const CXXRecordDecl *VBase = nullptr;
301
302
  // Sema has done some convenient canonicalization here: if the
303
  // access path involved any virtual steps, the conversion path will
304
  // *start* with a step down to the correct virtual base subobject,
305
  // and hence will not require any further steps.
306
24.4k
  if ((*Start)->isVirtual()) {
307
457
    VBase = cast<CXXRecordDecl>(
308
457
        (*Start)->getType()->castAs<RecordType>()->getDecl());
309
457
    ++Start;
310
457
  }
311
312
  // Compute the static offset of the ultimate destination within its
313
  // allocating subobject (the virtual base, if there is one, or else
314
  // the "complete" object that we see).
315
24.4k
  CharUnits NonVirtualOffset = CGM.computeNonVirtualBaseClassOffset(
316
24.4k
      VBase ? 
VBase457
:
Derived23.9k
, Start, PathEnd);
317
318
  // If there's a virtual step, we can sometimes "devirtualize" it.
319
  // For now, that's limited to when the derived type is final.
320
  // TODO: "devirtualize" this for accesses to known-complete objects.
321
24.4k
  if (VBase && 
Derived->hasAttr<FinalAttr>()457
) {
322
6
    const ASTRecordLayout &layout = getContext().getASTRecordLayout(Derived);
323
6
    CharUnits vBaseOffset = layout.getVBaseClassOffset(VBase);
324
6
    NonVirtualOffset += vBaseOffset;
325
6
    VBase = nullptr; // we no longer have a virtual step
326
6
  }
327
328
  // Get the base pointer type.
329
24.4k
  llvm::Type *BaseValueTy = ConvertType((PathEnd[-1])->getType());
330
24.4k
  llvm::Type *BasePtrTy =
331
24.4k
      BaseValueTy->getPointerTo(Value.getType()->getPointerAddressSpace());
332
333
24.4k
  QualType DerivedTy = getContext().getRecordType(Derived);
334
24.4k
  CharUnits DerivedAlign = CGM.getClassPointerAlignment(Derived);
335
336
  // If the static offset is zero and we don't have a virtual step,
337
  // just do a bitcast; null checks are unnecessary.
338
24.4k
  if (NonVirtualOffset.isZero() && 
!VBase23.3k
) {
339
22.8k
    if (sanitizePerformTypeCheck()) {
340
19
      SanitizerSet SkippedChecks;
341
19
      SkippedChecks.set(SanitizerKind::Null, !NullCheckValue);
342
19
      EmitTypeCheck(TCK_Upcast, Loc, Value.getPointer(),
343
19
                    DerivedTy, DerivedAlign, SkippedChecks);
344
19
    }
345
22.8k
    return Builder.CreateElementBitCast(Value, BaseValueTy);
346
22.8k
  }
347
348
1.55k
  llvm::BasicBlock *origBB = nullptr;
349
1.55k
  llvm::BasicBlock *endBB = nullptr;
350
351
  // Skip over the offset (and the vtable load) if we're supposed to
352
  // null-check the pointer.
353
1.55k
  if (NullCheckValue) {
354
34
    origBB = Builder.GetInsertBlock();
355
34
    llvm::BasicBlock *notNullBB = createBasicBlock("cast.notnull");
356
34
    endBB = createBasicBlock("cast.end");
357
358
34
    llvm::Value *isNull = Builder.CreateIsNull(Value.getPointer());
359
34
    Builder.CreateCondBr(isNull, endBB, notNullBB);
360
34
    EmitBlock(notNullBB);
361
34
  }
362
363
1.55k
  if (sanitizePerformTypeCheck()) {
364
3
    SanitizerSet SkippedChecks;
365
3
    SkippedChecks.set(SanitizerKind::Null, true);
366
3
    EmitTypeCheck(VBase ? TCK_UpcastToVirtualBase : 
TCK_Upcast0
, Loc,
367
3
                  Value.getPointer(), DerivedTy, DerivedAlign, SkippedChecks);
368
3
  }
369
370
  // Compute the virtual offset.
371
1.55k
  llvm::Value *VirtualOffset = nullptr;
372
1.55k
  if (VBase) {
373
451
    VirtualOffset =
374
451
      CGM.getCXXABI().GetVirtualBaseClassOffset(*this, Value, Derived, VBase);
375
451
  }
376
377
  // Apply both offsets.
378
1.55k
  Value = ApplyNonVirtualAndVirtualOffset(*this, Value, NonVirtualOffset,
379
1.55k
                                          VirtualOffset, Derived, VBase);
380
381
  // Cast to the destination type.
382
1.55k
  Value = Builder.CreateElementBitCast(Value, BaseValueTy);
383
384
  // Build a phi if we needed a null check.
385
1.55k
  if (NullCheckValue) {
386
34
    llvm::BasicBlock *notNullBB = Builder.GetInsertBlock();
387
34
    Builder.CreateBr(endBB);
388
34
    EmitBlock(endBB);
389
390
34
    llvm::PHINode *PHI = Builder.CreatePHI(BasePtrTy, 2, "cast.result");
391
34
    PHI->addIncoming(Value.getPointer(), notNullBB);
392
34
    PHI->addIncoming(llvm::Constant::getNullValue(BasePtrTy), origBB);
393
34
    Value = Value.withPointer(PHI);
394
34
  }
395
396
1.55k
  return Value;
397
24.4k
}
398
399
Address
400
CodeGenFunction::GetAddressOfDerivedClass(Address BaseAddr,
401
                                          const CXXRecordDecl *Derived,
402
                                        CastExpr::path_const_iterator PathBegin,
403
                                          CastExpr::path_const_iterator PathEnd,
404
742
                                          bool NullCheckValue) {
405
742
  assert(PathBegin != PathEnd && "Base path should not be empty!");
406
407
0
  QualType DerivedTy =
408
742
    getContext().getCanonicalType(getContext().getTagDeclType(Derived));
409
742
  unsigned AddrSpace = BaseAddr.getAddressSpace();
410
742
  llvm::Type *DerivedValueTy = ConvertType(DerivedTy);
411
742
  llvm::Type *DerivedPtrTy = DerivedValueTy->getPointerTo(AddrSpace);
412
413
742
  llvm::Value *NonVirtualOffset =
414
742
    CGM.GetNonVirtualBaseClassOffset(Derived, PathBegin, PathEnd);
415
416
742
  if (!NonVirtualOffset) {
417
    // No offset, we can just cast back.
418
731
    return Builder.CreateElementBitCast(BaseAddr, DerivedValueTy);
419
731
  }
420
421
11
  llvm::BasicBlock *CastNull = nullptr;
422
11
  llvm::BasicBlock *CastNotNull = nullptr;
423
11
  llvm::BasicBlock *CastEnd = nullptr;
424
425
11
  if (NullCheckValue) {
426
5
    CastNull = createBasicBlock("cast.null");
427
5
    CastNotNull = createBasicBlock("cast.notnull");
428
5
    CastEnd = createBasicBlock("cast.end");
429
430
5
    llvm::Value *IsNull = Builder.CreateIsNull(BaseAddr.getPointer());
431
5
    Builder.CreateCondBr(IsNull, CastNull, CastNotNull);
432
5
    EmitBlock(CastNotNull);
433
5
  }
434
435
  // Apply the offset.
436
11
  llvm::Value *Value = Builder.CreateBitCast(BaseAddr.getPointer(), Int8PtrTy);
437
11
  Value = Builder.CreateInBoundsGEP(
438
11
      Int8Ty, Value, Builder.CreateNeg(NonVirtualOffset), "sub.ptr");
439
440
  // Just cast.
441
11
  Value = Builder.CreateBitCast(Value, DerivedPtrTy);
442
443
  // Produce a PHI if we had a null-check.
444
11
  if (NullCheckValue) {
445
5
    Builder.CreateBr(CastEnd);
446
5
    EmitBlock(CastNull);
447
5
    Builder.CreateBr(CastEnd);
448
5
    EmitBlock(CastEnd);
449
450
5
    llvm::PHINode *PHI = Builder.CreatePHI(Value->getType(), 2);
451
5
    PHI->addIncoming(Value, CastNotNull);
452
5
    PHI->addIncoming(llvm::Constant::getNullValue(Value->getType()), CastNull);
453
5
    Value = PHI;
454
5
  }
455
456
11
  return Address(Value, DerivedValueTy, CGM.getClassPointerAlignment(Derived));
457
742
}
458
459
llvm::Value *CodeGenFunction::GetVTTParameter(GlobalDecl GD,
460
                                              bool ForVirtualBase,
461
29.5k
                                              bool Delegating) {
462
29.5k
  if (!CGM.getCXXABI().NeedsVTTParameter(GD)) {
463
    // This constructor/destructor does not need a VTT parameter.
464
29.2k
    return nullptr;
465
29.2k
  }
466
467
265
  const CXXRecordDecl *RD = cast<CXXMethodDecl>(CurCodeDecl)->getParent();
468
265
  const CXXRecordDecl *Base = cast<CXXMethodDecl>(GD.getDecl())->getParent();
469
470
265
  uint64_t SubVTTIndex;
471
472
265
  if (Delegating) {
473
    // If this is a delegating constructor call, just load the VTT.
474
2
    return LoadCXXVTT();
475
263
  } else if (RD == Base) {
476
    // If the record matches the base, this is the complete ctor/dtor
477
    // variant calling the base variant in a class with virtual bases.
478
91
    assert(!CGM.getCXXABI().NeedsVTTParameter(CurGD) &&
479
91
           "doing no-op VTT offset in base dtor/ctor?");
480
0
    assert(!ForVirtualBase && "Can't have same class as virtual base!");
481
0
    SubVTTIndex = 0;
482
172
  } else {
483
172
    const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
484
172
    CharUnits BaseOffset = ForVirtualBase ?
485
7
      Layout.getVBaseClassOffset(Base) :
486
172
      
Layout.getBaseClassOffset(Base)165
;
487
488
172
    SubVTTIndex =
489
172
      CGM.getVTables().getSubVTTIndex(RD, BaseSubobject(Base, BaseOffset));
490
172
    assert(SubVTTIndex != 0 && "Sub-VTT index must be greater than zero!");
491
172
  }
492
493
263
  if (CGM.getCXXABI().NeedsVTTParameter(CurGD)) {
494
    // A VTT parameter was passed to the constructor, use it.
495
38
    llvm::Value *VTT = LoadCXXVTT();
496
38
    return Builder.CreateConstInBoundsGEP1_64(VoidPtrTy, VTT, SubVTTIndex);
497
225
  } else {
498
    // We're the complete constructor, so get the VTT by name.
499
225
    llvm::GlobalValue *VTT = CGM.getVTables().GetAddrOfVTT(RD);
500
225
    return Builder.CreateConstInBoundsGEP2_64(
501
225
        VTT->getValueType(), VTT, 0, SubVTTIndex);
502
225
  }
503
263
}
504
505
namespace {
506
  /// Call the destructor for a direct base class.
507
  struct CallBaseDtor final : EHScopeStack::Cleanup {
508
    const CXXRecordDecl *BaseClass;
509
    bool BaseIsVirtual;
510
    CallBaseDtor(const CXXRecordDecl *Base, bool BaseIsVirtual)
511
1.96k
      : BaseClass(Base), BaseIsVirtual(BaseIsVirtual) {}
512
513
1.57k
    void Emit(CodeGenFunction &CGF, Flags flags) override {
514
1.57k
      const CXXRecordDecl *DerivedClass =
515
1.57k
        cast<CXXMethodDecl>(CGF.CurCodeDecl)->getParent();
516
517
1.57k
      const CXXDestructorDecl *D = BaseClass->getDestructor();
518
      // We are already inside a destructor, so presumably the object being
519
      // destroyed should have the expected type.
520
1.57k
      QualType ThisTy = D->getThisObjectType();
521
1.57k
      Address Addr =
522
1.57k
        CGF.GetAddressOfDirectBaseInCompleteClass(CGF.LoadCXXThisAddress(),
523
1.57k
                                                  DerivedClass, BaseClass,
524
1.57k
                                                  BaseIsVirtual);
525
1.57k
      CGF.EmitCXXDestructorCall(D, Dtor_Base, BaseIsVirtual,
526
1.57k
                                /*Delegating=*/false, Addr, ThisTy);
527
1.57k
    }
528
  };
529
530
  /// A visitor which checks whether an initializer uses 'this' in a
531
  /// way which requires the vtable to be properly set.
532
  struct DynamicThisUseChecker : ConstEvaluatedExprVisitor<DynamicThisUseChecker> {
533
    typedef ConstEvaluatedExprVisitor<DynamicThisUseChecker> super;
534
535
    bool UsesThis;
536
537
10.8k
    DynamicThisUseChecker(const ASTContext &C) : super(C), UsesThis(false) {}
538
539
    // Black-list all explicit and implicit references to 'this'.
540
    //
541
    // Do we need to worry about external references to 'this' derived
542
    // from arbitrary code?  If so, then anything which runs arbitrary
543
    // external code might potentially access the vtable.
544
13
    void VisitCXXThisExpr(const CXXThisExpr *E) { UsesThis = true; }
545
  };
546
} // end anonymous namespace
547
548
10.8k
static bool BaseInitializerUsesThis(ASTContext &C, const Expr *Init) {
549
10.8k
  DynamicThisUseChecker Checker(C);
550
10.8k
  Checker.Visit(Init);
551
10.8k
  return Checker.UsesThis;
552
10.8k
}
553
554
static void EmitBaseInitializer(CodeGenFunction &CGF,
555
                                const CXXRecordDecl *ClassDecl,
556
10.8k
                                CXXCtorInitializer *BaseInit) {
557
10.8k
  assert(BaseInit->isBaseInitializer() &&
558
10.8k
         "Must have base initializer!");
559
560
0
  Address ThisPtr = CGF.LoadCXXThisAddress();
561
562
10.8k
  const Type *BaseType = BaseInit->getBaseClass();
563
10.8k
  const auto *BaseClassDecl =
564
10.8k
      cast<CXXRecordDecl>(BaseType->castAs<RecordType>()->getDecl());
565
566
10.8k
  bool isBaseVirtual = BaseInit->isBaseVirtual();
567
568
  // If the initializer for the base (other than the constructor
569
  // itself) accesses 'this' in any way, we need to initialize the
570
  // vtables.
571
10.8k
  if (BaseInitializerUsesThis(CGF.getContext(), BaseInit->getInit()))
572
13
    CGF.InitializeVTablePointers(ClassDecl);
573
574
  // We can pretend to be a complete class because it only matters for
575
  // virtual bases, and we only do virtual bases for complete ctors.
576
10.8k
  Address V =
577
10.8k
    CGF.GetAddressOfDirectBaseInCompleteClass(ThisPtr, ClassDecl,
578
10.8k
                                              BaseClassDecl,
579
10.8k
                                              isBaseVirtual);
580
10.8k
  AggValueSlot AggSlot =
581
10.8k
      AggValueSlot::forAddr(
582
10.8k
          V, Qualifiers(),
583
10.8k
          AggValueSlot::IsDestructed,
584
10.8k
          AggValueSlot::DoesNotNeedGCBarriers,
585
10.8k
          AggValueSlot::IsNotAliased,
586
10.8k
          CGF.getOverlapForBaseInit(ClassDecl, BaseClassDecl, isBaseVirtual));
587
588
10.8k
  CGF.EmitAggExpr(BaseInit->getInit(), AggSlot);
589
590
10.8k
  if (CGF.CGM.getLangOpts().Exceptions &&
591
10.8k
      
!BaseClassDecl->hasTrivialDestructor()8.74k
)
592
786
    CGF.EHStack.pushCleanup<CallBaseDtor>(EHCleanup, BaseClassDecl,
593
786
                                          isBaseVirtual);
594
10.8k
}
595
596
100k
static bool isMemcpyEquivalentSpecialMember(const CXXMethodDecl *D) {
597
100k
  auto *CD = dyn_cast<CXXConstructorDecl>(D);
598
100k
  if (!(CD && 
CD->isCopyOrMoveConstructor()100k
) &&
599
100k
      
!D->isCopyAssignmentOperator()89.8k
&&
!D->isMoveAssignmentOperator()89.6k
)
600
89.5k
    return false;
601
602
  // We can emit a memcpy for a trivial copy or move constructor/assignment.
603
11.3k
  if (D->isTrivial() && 
!D->getParent()->mayInsertExtraPadding()6.28k
)
604
6.27k
    return true;
605
606
  // We *must* emit a memcpy for a defaulted union copy or move op.
607
5.07k
  if (D->getParent()->isUnion() && 
D->isDefaulted()4
)
608
2
    return true;
609
610
5.06k
  return false;
611
5.07k
}
612
613
static void EmitLValueForAnyFieldInitialization(CodeGenFunction &CGF,
614
                                                CXXCtorInitializer *MemberInit,
615
17.0k
                                                LValue &LHS) {
616
17.0k
  FieldDecl *Field = MemberInit->getAnyMember();
617
17.0k
  if (MemberInit->isIndirectMemberInitializer()) {
618
    // If we are initializing an anonymous union field, drill down to the field.
619
46
    IndirectFieldDecl *IndirectField = MemberInit->getIndirectMember();
620
46
    for (const auto *I : IndirectField->chain())
621
100
      LHS = CGF.EmitLValueForFieldInitialization(LHS, cast<FieldDecl>(I));
622
16.9k
  } else {
623
16.9k
    LHS = CGF.EmitLValueForFieldInitialization(LHS, Field);
624
16.9k
  }
625
17.0k
}
626
627
static void EmitMemberInitializer(CodeGenFunction &CGF,
628
                                  const CXXRecordDecl *ClassDecl,
629
                                  CXXCtorInitializer *MemberInit,
630
                                  const CXXConstructorDecl *Constructor,
631
17.0k
                                  FunctionArgList &Args) {
632
17.0k
  ApplyDebugLocation Loc(CGF, MemberInit->getSourceLocation());
633
17.0k
  assert(MemberInit->isAnyMemberInitializer() &&
634
17.0k
         "Must have member initializer!");
635
0
  assert(MemberInit->getInit() && "Must have initializer!");
636
637
  // non-static data member initializers.
638
0
  FieldDecl *Field = MemberInit->getAnyMember();
639
17.0k
  QualType FieldType = Field->getType();
640
641
17.0k
  llvm::Value *ThisPtr = CGF.LoadCXXThis();
642
17.0k
  QualType RecordTy = CGF.getContext().getTypeDeclType(ClassDecl);
643
17.0k
  LValue LHS;
644
645
  // If a base constructor is being emitted, create an LValue that has the
646
  // non-virtual alignment.
647
17.0k
  if (CGF.CurGD.getCtorType() == Ctor_Base)
648
16.8k
    LHS = CGF.MakeNaturalAlignPointeeAddrLValue(ThisPtr, RecordTy);
649
196
  else
650
196
    LHS = CGF.MakeNaturalAlignAddrLValue(ThisPtr, RecordTy);
651
652
17.0k
  EmitLValueForAnyFieldInitialization(CGF, MemberInit, LHS);
653
654
  // Special case: if we are in a copy or move constructor, and we are copying
655
  // an array of PODs or classes with trivial copy constructors, ignore the
656
  // AST and perform the copy we know is equivalent.
657
  // FIXME: This is hacky at best... if we had a bit more explicit information
658
  // in the AST, we could generalize it more easily.
659
17.0k
  const ConstantArrayType *Array
660
17.0k
    = CGF.getContext().getAsConstantArrayType(FieldType);
661
17.0k
  if (Array && 
Constructor->isDefaulted()86
&&
662
17.0k
      
Constructor->isCopyOrMoveConstructor()38
) {
663
23
    QualType BaseElementTy = CGF.getContext().getBaseElementType(Array);
664
23
    CXXConstructExpr *CE = dyn_cast<CXXConstructExpr>(MemberInit->getInit());
665
23
    if (BaseElementTy.isPODType(CGF.getContext()) ||
666
23
        
(18
CE18
&&
isMemcpyEquivalentSpecialMember(CE->getConstructor())0
)) {
667
5
      unsigned SrcArgIndex =
668
5
          CGF.CGM.getCXXABI().getSrcArgforCopyCtor(Constructor, Args);
669
5
      llvm::Value *SrcPtr
670
5
        = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(Args[SrcArgIndex]));
671
5
      LValue ThisRHSLV = CGF.MakeNaturalAlignAddrLValue(SrcPtr, RecordTy);
672
5
      LValue Src = CGF.EmitLValueForFieldInitialization(ThisRHSLV, Field);
673
674
      // Copy the aggregate.
675
5
      CGF.EmitAggregateCopy(LHS, Src, FieldType, CGF.getOverlapForFieldInit(Field),
676
5
                            LHS.isVolatileQualified());
677
      // Ensure that we destroy the objects if an exception is thrown later in
678
      // the constructor.
679
5
      QualType::DestructionKind dtorKind = FieldType.isDestructedType();
680
5
      if (CGF.needsEHCleanup(dtorKind))
681
0
        CGF.pushEHDestroy(dtorKind, LHS.getAddress(CGF), FieldType);
682
5
      return;
683
5
    }
684
23
  }
685
686
17.0k
  CGF.EmitInitializerForField(Field, LHS, MemberInit->getInit());
687
17.0k
}
688
689
void CodeGenFunction::EmitInitializerForField(FieldDecl *Field, LValue LHS,
690
18.3k
                                              Expr *Init) {
691
18.3k
  QualType FieldType = Field->getType();
692
18.3k
  switch (getEvaluationKind(FieldType)) {
693
13.4k
  case TEK_Scalar:
694
13.4k
    if (LHS.isSimple()) {
695
13.3k
      EmitExprAsInit(Init, Field, LHS, false);
696
13.3k
    } else {
697
82
      RValue RHS = RValue::get(EmitScalarExpr(Init));
698
82
      EmitStoreThroughLValue(RHS, LHS);
699
82
    }
700
13.4k
    break;
701
6
  case TEK_Complex:
702
6
    EmitComplexExprIntoLValue(Init, LHS, /*isInit*/ true);
703
6
    break;
704
4.84k
  case TEK_Aggregate: {
705
4.84k
    AggValueSlot Slot = AggValueSlot::forLValue(
706
4.84k
        LHS, *this, AggValueSlot::IsDestructed,
707
4.84k
        AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased,
708
4.84k
        getOverlapForFieldInit(Field), AggValueSlot::IsNotZeroed,
709
        // Checks are made by the code that calls constructor.
710
4.84k
        AggValueSlot::IsSanitizerChecked);
711
4.84k
    EmitAggExpr(Init, Slot);
712
4.84k
    break;
713
0
  }
714
18.3k
  }
715
716
  // Ensure that we destroy this object if an exception is thrown
717
  // later in the constructor.
718
18.3k
  QualType::DestructionKind dtorKind = FieldType.isDestructedType();
719
18.3k
  if (needsEHCleanup(dtorKind))
720
949
    pushEHDestroy(dtorKind, LHS.getAddress(*this), FieldType);
721
18.3k
}
722
723
/// Checks whether the given constructor is a valid subject for the
724
/// complete-to-base constructor delegation optimization, i.e.
725
/// emitting the complete constructor as a simple call to the base
726
/// constructor.
727
bool CodeGenFunction::IsConstructorDelegationValid(
728
17.9k
    const CXXConstructorDecl *Ctor) {
729
730
  // Currently we disable the optimization for classes with virtual
731
  // bases because (1) the addresses of parameter variables need to be
732
  // consistent across all initializers but (2) the delegate function
733
  // call necessarily creates a second copy of the parameter variable.
734
  //
735
  // The limiting example (purely theoretical AFAIK):
736
  //   struct A { A(int &c) { c++; } };
737
  //   struct B : virtual A {
738
  //     B(int count) : A(count) { printf("%d\n", count); }
739
  //   };
740
  // ...although even this example could in principle be emitted as a
741
  // delegation since the address of the parameter doesn't escape.
742
17.9k
  if (Ctor->getParent()->getNumVBases()) {
743
    // TODO: white-list trivial vbase initializers.  This case wouldn't
744
    // be subject to the restrictions below.
745
746
    // TODO: white-list cases where:
747
    //  - there are no non-reference parameters to the constructor
748
    //  - the initializers don't access any non-reference parameters
749
    //  - the initializers don't take the address of non-reference
750
    //    parameters
751
    //  - etc.
752
    // If we ever add any of the above cases, remember that:
753
    //  - function-try-blocks will always exclude this optimization
754
    //  - we need to perform the constructor prologue and cleanup in
755
    //    EmitConstructorBody.
756
757
586
    return false;
758
586
  }
759
760
  // We also disable the optimization for variadic functions because
761
  // it's impossible to "re-pass" varargs.
762
17.3k
  if (Ctor->getType()->castAs<FunctionProtoType>()->isVariadic())
763
10
    return false;
764
765
  // FIXME: Decide if we can do a delegation of a delegating constructor.
766
17.3k
  if (Ctor->isDelegatingConstructor())
767
87
    return false;
768
769
17.2k
  return true;
770
17.3k
}
771
772
// Emit code in ctor (Prologue==true) or dtor (Prologue==false)
773
// to poison the extra field paddings inserted under
774
// -fsanitize-address-field-padding=1|2.
775
56.7k
void CodeGenFunction::EmitAsanPrologueOrEpilogue(bool Prologue) {
776
56.7k
  ASTContext &Context = getContext();
777
56.7k
  const CXXRecordDecl *ClassDecl =
778
56.7k
      Prologue ? 
cast<CXXConstructorDecl>(CurGD.getDecl())->getParent()41.0k
779
56.7k
               : 
cast<CXXDestructorDecl>(CurGD.getDecl())->getParent()15.7k
;
780
56.7k
  if (!ClassDecl->mayInsertExtraPadding()) 
return56.7k
;
781
782
41
  struct SizeAndOffset {
783
41
    uint64_t Size;
784
41
    uint64_t Offset;
785
41
  };
786
787
41
  unsigned PtrSize = CGM.getDataLayout().getPointerSizeInBits();
788
41
  const ASTRecordLayout &Info = Context.getASTRecordLayout(ClassDecl);
789
790
  // Populate sizes and offsets of fields.
791
41
  SmallVector<SizeAndOffset, 16> SSV(Info.getFieldCount());
792
153
  for (unsigned i = 0, e = Info.getFieldCount(); i != e; 
++i112
)
793
112
    SSV[i].Offset =
794
112
        Context.toCharUnitsFromBits(Info.getFieldOffset(i)).getQuantity();
795
796
41
  size_t NumFields = 0;
797
112
  for (const auto *Field : ClassDecl->fields()) {
798
112
    const FieldDecl *D = Field;
799
112
    auto FieldInfo = Context.getTypeInfoInChars(D->getType());
800
112
    CharUnits FieldSize = FieldInfo.Width;
801
112
    assert(NumFields < SSV.size());
802
112
    SSV[NumFields].Size = D->isBitField() ? 
00
: FieldSize.getQuantity();
803
112
    NumFields++;
804
112
  }
805
41
  assert(NumFields == SSV.size());
806
41
  if (SSV.size() <= 1) 
return0
;
807
808
  // We will insert calls to __asan_* run-time functions.
809
  // LLVM AddressSanitizer pass may decide to inline them later.
810
41
  llvm::Type *Args[2] = {IntPtrTy, IntPtrTy};
811
41
  llvm::FunctionType *FTy =
812
41
      llvm::FunctionType::get(CGM.VoidTy, Args, false);
813
41
  llvm::FunctionCallee F = CGM.CreateRuntimeFunction(
814
41
      FTy, Prologue ? 
"__asan_poison_intra_object_redzone"22
815
41
                    : 
"__asan_unpoison_intra_object_redzone"19
);
816
817
41
  llvm::Value *ThisPtr = LoadCXXThis();
818
41
  ThisPtr = Builder.CreatePtrToInt(ThisPtr, IntPtrTy);
819
41
  uint64_t TypeSize = Info.getNonVirtualSize().getQuantity();
820
  // For each field check if it has sufficient padding,
821
  // if so (un)poison it with a call.
822
153
  for (size_t i = 0; i < SSV.size(); 
i++112
) {
823
112
    uint64_t AsanAlignment = 8;
824
112
    uint64_t NextField = i == SSV.size() - 1 ? 
TypeSize41
:
SSV[i + 1].Offset71
;
825
112
    uint64_t PoisonSize = NextField - SSV[i].Offset - SSV[i].Size;
826
112
    uint64_t EndOffset = SSV[i].Offset + SSV[i].Size;
827
112
    if (PoisonSize < AsanAlignment || 
!SSV[i].Size94
||
828
112
        
(NextField % AsanAlignment) != 094
)
829
18
      continue;
830
94
    Builder.CreateCall(
831
94
        F, {Builder.CreateAdd(ThisPtr, Builder.getIntN(PtrSize, EndOffset)),
832
94
            Builder.getIntN(PtrSize, PoisonSize)});
833
94
  }
834
41
}
835
836
/// EmitConstructorBody - Emits the body of the current constructor.
837
41.0k
void CodeGenFunction::EmitConstructorBody(FunctionArgList &Args) {
838
41.0k
  EmitAsanPrologueOrEpilogue(true);
839
41.0k
  const CXXConstructorDecl *Ctor = cast<CXXConstructorDecl>(CurGD.getDecl());
840
41.0k
  CXXCtorType CtorType = CurGD.getCtorType();
841
842
41.0k
  assert((CGM.getTarget().getCXXABI().hasConstructorVariants() ||
843
41.0k
          CtorType == Ctor_Complete) &&
844
41.0k
         "can only generate complete ctor for this ABI");
845
846
  // Before we go any further, try the complete->base constructor
847
  // delegation optimization.
848
41.0k
  if (CtorType == Ctor_Complete && 
IsConstructorDelegationValid(Ctor)17.9k
&&
849
41.0k
      
CGM.getTarget().getCXXABI().hasConstructorVariants()17.2k
) {
850
16.3k
    EmitDelegateCXXConstructorCall(Ctor, Ctor_Base, Args, Ctor->getEndLoc());
851
16.3k
    return;
852
16.3k
  }
853
854
24.7k
  const FunctionDecl *Definition = nullptr;
855
24.7k
  Stmt *Body = Ctor->getBody(Definition);
856
24.7k
  assert(Definition == Ctor && "emitting wrong constructor body");
857
858
  // Enter the function-try-block before the constructor prologue if
859
  // applicable.
860
24.7k
  bool IsTryBody = (Body && isa<CXXTryStmt>(Body));
861
24.7k
  if (IsTryBody)
862
2
    EnterCXXTryStmt(*cast<CXXTryStmt>(Body), true);
863
864
24.7k
  incrementProfileCounter(Body);
865
866
24.7k
  RunCleanupsScope RunCleanups(*this);
867
868
  // TODO: in restricted cases, we can emit the vbase initializers of
869
  // a complete ctor and then delegate to the base ctor.
870
871
  // Emit the constructor prologue, i.e. the base and member
872
  // initializers.
873
24.7k
  EmitCtorPrologue(Ctor, CtorType, Args);
874
875
  // Emit the body of the statement.
876
24.7k
  if (IsTryBody)
877
2
    EmitStmt(cast<CXXTryStmt>(Body)->getTryBlock());
878
24.7k
  else if (Body)
879
24.7k
    EmitStmt(Body);
880
881
  // Emit any cleanup blocks associated with the member or base
882
  // initializers, which includes (along the exceptional path) the
883
  // destructors for those members and bases that were fully
884
  // constructed.
885
24.7k
  RunCleanups.ForceCleanup();
886
887
24.7k
  if (IsTryBody)
888
2
    ExitCXXTryStmt(*cast<CXXTryStmt>(Body), true);
889
24.7k
}
890
891
namespace {
892
  /// RAII object to indicate that codegen is copying the value representation
893
  /// instead of the object representation. Useful when copying a struct or
894
  /// class which has uninitialized members and we're only performing
895
  /// lvalue-to-rvalue conversion on the object but not its members.
896
  class CopyingValueRepresentation {
897
  public:
898
    explicit CopyingValueRepresentation(CodeGenFunction &CGF)
899
164
        : CGF(CGF), OldSanOpts(CGF.SanOpts) {
900
164
      CGF.SanOpts.set(SanitizerKind::Bool, false);
901
164
      CGF.SanOpts.set(SanitizerKind::Enum, false);
902
164
    }
903
164
    ~CopyingValueRepresentation() {
904
164
      CGF.SanOpts = OldSanOpts;
905
164
    }
906
  private:
907
    CodeGenFunction &CGF;
908
    SanitizerSet OldSanOpts;
909
  };
910
} // end anonymous namespace
911
912
namespace {
913
  class FieldMemcpyizer {
914
  public:
915
    FieldMemcpyizer(CodeGenFunction &CGF, const CXXRecordDecl *ClassDecl,
916
                    const VarDecl *SrcRec)
917
      : CGF(CGF), ClassDecl(ClassDecl), SrcRec(SrcRec),
918
        RecLayout(CGF.getContext().getASTRecordLayout(ClassDecl)),
919
        FirstField(nullptr), LastField(nullptr), FirstFieldOffset(0),
920
25.3k
        LastFieldOffset(0), LastAddedFieldIndex(0) {}
921
922
541
    bool isMemcpyableField(FieldDecl *F) const {
923
      // Never memcpy fields when we are adding poisoned paddings.
924
541
      if (CGF.getContext().getLangOpts().SanitizeAddressFieldPadding)
925
8
        return false;
926
533
      Qualifiers Qual = F->getType().getQualifiers();
927
533
      if (Qual.hasVolatile() || 
Qual.hasObjCLifetime()529
)
928
8
        return false;
929
525
      return true;
930
533
    }
931
932
522
    void addMemcpyableField(FieldDecl *F) {
933
522
      if (F->isZeroSize(CGF.getContext()))
934
1
        return;
935
521
      if (!FirstField)
936
260
        addInitialField(F);
937
261
      else
938
261
        addNextField(F);
939
521
    }
940
941
96
    CharUnits getMemcpySize(uint64_t FirstByteOffset) const {
942
96
      ASTContext &Ctx = CGF.getContext();
943
96
      unsigned LastFieldSize =
944
96
          LastField->isBitField()
945
96
              ? 
LastField->getBitWidthValue(Ctx)6
946
96
              : Ctx.toBits(
947
90
                    Ctx.getTypeInfoDataSizeInChars(LastField->getType()).Width);
948
96
      uint64_t MemcpySizeBits = LastFieldOffset + LastFieldSize -
949
96
                                FirstByteOffset + Ctx.getCharWidth() - 1;
950
96
      CharUnits MemcpySize = Ctx.toCharUnitsFromBits(MemcpySizeBits);
951
96
      return MemcpySize;
952
96
    }
953
954
1.82k
    void emitMemcpy() {
955
      // Give the subclass a chance to bail out if it feels the memcpy isn't
956
      // worth it (e.g. Hasn't aggregated enough data).
957
1.82k
      if (!FirstField) {
958
1.72k
        return;
959
1.72k
      }
960
961
96
      uint64_t FirstByteOffset;
962
96
      if (FirstField->isBitField()) {
963
12
        const CGRecordLayout &RL =
964
12
          CGF.getTypes().getCGRecordLayout(FirstField->getParent());
965
12
        const CGBitFieldInfo &BFInfo = RL.getBitFieldInfo(FirstField);
966
        // FirstFieldOffset is not appropriate for bitfields,
967
        // we need to use the storage offset instead.
968
12
        FirstByteOffset = CGF.getContext().toBits(BFInfo.StorageOffset);
969
84
      } else {
970
84
        FirstByteOffset = FirstFieldOffset;
971
84
      }
972
973
96
      CharUnits MemcpySize = getMemcpySize(FirstByteOffset);
974
96
      QualType RecordTy = CGF.getContext().getTypeDeclType(ClassDecl);
975
96
      Address ThisPtr = CGF.LoadCXXThisAddress();
976
96
      LValue DestLV = CGF.MakeAddrLValue(ThisPtr, RecordTy);
977
96
      LValue Dest = CGF.EmitLValueForFieldInitialization(DestLV, FirstField);
978
96
      llvm::Value *SrcPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(SrcRec));
979
96
      LValue SrcLV = CGF.MakeNaturalAlignAddrLValue(SrcPtr, RecordTy);
980
96
      LValue Src = CGF.EmitLValueForFieldInitialization(SrcLV, FirstField);
981
982
96
      emitMemcpyIR(
983
96
          Dest.isBitField() ? 
Dest.getBitFieldAddress()12
:
Dest.getAddress(CGF)84
,
984
96
          Src.isBitField() ? 
Src.getBitFieldAddress()12
:
Src.getAddress(CGF)84
,
985
96
          MemcpySize);
986
96
      reset();
987
96
    }
988
989
43.3k
    void reset() {
990
43.3k
      FirstField = nullptr;
991
43.3k
    }
992
993
  protected:
994
    CodeGenFunction &CGF;
995
    const CXXRecordDecl *ClassDecl;
996
997
  private:
998
96
    void emitMemcpyIR(Address DestPtr, Address SrcPtr, CharUnits Size) {
999
96
      DestPtr = CGF.Builder.CreateElementBitCast(DestPtr, CGF.Int8Ty);
1000
96
      SrcPtr = CGF.Builder.CreateElementBitCast(SrcPtr, CGF.Int8Ty);
1001
96
      CGF.Builder.CreateMemCpy(DestPtr, SrcPtr, Size.getQuantity());
1002
96
    }
1003
1004
260
    void addInitialField(FieldDecl *F) {
1005
260
      FirstField = F;
1006
260
      LastField = F;
1007
260
      FirstFieldOffset = RecLayout.getFieldOffset(F->getFieldIndex());
1008
260
      LastFieldOffset = FirstFieldOffset;
1009
260
      LastAddedFieldIndex = F->getFieldIndex();
1010
260
    }
1011
1012
261
    void addNextField(FieldDecl *F) {
1013
      // For the most part, the following invariant will hold:
1014
      //   F->getFieldIndex() == LastAddedFieldIndex + 1
1015
      // The one exception is that Sema won't add a copy-initializer for an
1016
      // unnamed bitfield, which will show up here as a gap in the sequence.
1017
261
      assert(F->getFieldIndex() >= LastAddedFieldIndex + 1 &&
1018
261
             "Cannot aggregate fields out of order.");
1019
0
      LastAddedFieldIndex = F->getFieldIndex();
1020
1021
      // The 'first' and 'last' fields are chosen by offset, rather than field
1022
      // index. This allows the code to support bitfields, as well as regular
1023
      // fields.
1024
261
      uint64_t FOffset = RecLayout.getFieldOffset(F->getFieldIndex());
1025
261
      if (FOffset < FirstFieldOffset) {
1026
0
        FirstField = F;
1027
0
        FirstFieldOffset = FOffset;
1028
261
      } else if (FOffset >= LastFieldOffset) {
1029
261
        LastField = F;
1030
261
        LastFieldOffset = FOffset;
1031
261
      }
1032
261
    }
1033
1034
    const VarDecl *SrcRec;
1035
    const ASTRecordLayout &RecLayout;
1036
    FieldDecl *FirstField;
1037
    FieldDecl *LastField;
1038
    uint64_t FirstFieldOffset, LastFieldOffset;
1039
    unsigned LastAddedFieldIndex;
1040
  };
1041
1042
  class ConstructorMemcpyizer : public FieldMemcpyizer {
1043
  private:
1044
    /// Get source argument for copy constructor. Returns null if not a copy
1045
    /// constructor.
1046
    static const VarDecl *getTrivialCopySource(CodeGenFunction &CGF,
1047
                                               const CXXConstructorDecl *CD,
1048
24.6k
                                               FunctionArgList &Args) {
1049
24.6k
      if (CD->isCopyOrMoveConstructor() && 
CD->isDefaulted()1.06k
)
1050
373
        return Args[CGF.CGM.getCXXABI().getSrcArgforCopyCtor(CD, Args)];
1051
24.2k
      return nullptr;
1052
24.6k
    }
1053
1054
    // Returns true if a CXXCtorInitializer represents a member initialization
1055
    // that can be rolled into a memcpy.
1056
17.2k
    bool isMemberInitMemcpyable(CXXCtorInitializer *MemberInit) const {
1057
17.2k
      if (!MemcpyableCtor)
1058
16.7k
        return false;
1059
424
      FieldDecl *Field = MemberInit->getMember();
1060
424
      assert(Field && "No field for member init.");
1061
0
      QualType FieldType = Field->getType();
1062
424
      CXXConstructExpr *CE = dyn_cast<CXXConstructExpr>(MemberInit->getInit());
1063
1064
      // Bail out on non-memcpyable, not-trivially-copyable members.
1065
424
      if (!(CE && 
isMemcpyEquivalentSpecialMember(CE->getConstructor())138
) &&
1066
424
          
!(403
FieldType.isTriviallyCopyableType(CGF.getContext())403
||
1067
403
            
FieldType->isReferenceType()167
))
1068
159
        return false;
1069
1070
      // Bail out on volatile fields.
1071
265
      if (!isMemcpyableField(Field))
1072
6
        return false;
1073
1074
      // Otherwise we're good.
1075
259
      return true;
1076
265
    }
1077
1078
  public:
1079
    ConstructorMemcpyizer(CodeGenFunction &CGF, const CXXConstructorDecl *CD,
1080
                          FunctionArgList &Args)
1081
      : FieldMemcpyizer(CGF, CD->getParent(), getTrivialCopySource(CGF, CD, Args)),
1082
        ConstructorDecl(CD),
1083
        MemcpyableCtor(CD->isDefaulted() &&
1084
                       CD->isCopyOrMoveConstructor() &&
1085
                       CGF.getLangOpts().getGC() == LangOptions::NonGC),
1086
24.6k
        Args(Args) { }
1087
1088
17.2k
    void addMemberInitializer(CXXCtorInitializer *MemberInit) {
1089
17.2k
      if (isMemberInitMemcpyable(MemberInit)) {
1090
259
        AggregatedInits.push_back(MemberInit);
1091
259
        addMemcpyableField(MemberInit->getMember());
1092
16.9k
      } else {
1093
16.9k
        emitAggregatedInits();
1094
16.9k
        EmitMemberInitializer(CGF, ConstructorDecl->getParent(), MemberInit,
1095
16.9k
                              ConstructorDecl, Args);
1096
16.9k
      }
1097
17.2k
    }
1098
1099
41.6k
    void emitAggregatedInits() {
1100
41.6k
      if (AggregatedInits.size() <= 1) {
1101
        // This memcpy is too small to be worthwhile. Fall back on default
1102
        // codegen.
1103
41.5k
        if (!AggregatedInits.empty()) {
1104
64
          CopyingValueRepresentation CVR(CGF);
1105
64
          EmitMemberInitializer(CGF, ConstructorDecl->getParent(),
1106
64
                                AggregatedInits[0], ConstructorDecl, Args);
1107
64
          AggregatedInits.clear();
1108
64
        }
1109
41.5k
        reset();
1110
41.5k
        return;
1111
41.5k
      }
1112
1113
55
      pushEHDestructors();
1114
55
      emitMemcpy();
1115
55
      AggregatedInits.clear();
1116
55
    }
1117
1118
55
    void pushEHDestructors() {
1119
55
      Address ThisPtr = CGF.LoadCXXThisAddress();
1120
55
      QualType RecordTy = CGF.getContext().getTypeDeclType(ClassDecl);
1121
55
      LValue LHS = CGF.MakeAddrLValue(ThisPtr, RecordTy);
1122
1123
250
      for (unsigned i = 0; i < AggregatedInits.size(); 
++i195
) {
1124
195
        CXXCtorInitializer *MemberInit = AggregatedInits[i];
1125
195
        QualType FieldType = MemberInit->getAnyMember()->getType();
1126
195
        QualType::DestructionKind dtorKind = FieldType.isDestructedType();
1127
195
        if (!CGF.needsEHCleanup(dtorKind))
1128
193
          continue;
1129
2
        LValue FieldLHS = LHS;
1130
2
        EmitLValueForAnyFieldInitialization(CGF, MemberInit, FieldLHS);
1131
2
        CGF.pushEHDestroy(dtorKind, FieldLHS.getAddress(CGF), FieldType);
1132
2
      }
1133
55
    }
1134
1135
24.6k
    void finish() {
1136
24.6k
      emitAggregatedInits();
1137
24.6k
    }
1138
1139
  private:
1140
    const CXXConstructorDecl *ConstructorDecl;
1141
    bool MemcpyableCtor;
1142
    FunctionArgList &Args;
1143
    SmallVector<CXXCtorInitializer*, 16> AggregatedInits;
1144
  };
1145
1146
  class AssignmentMemcpyizer : public FieldMemcpyizer {
1147
  private:
1148
    // Returns the memcpyable field copied by the given statement, if one
1149
    // exists. Otherwise returns null.
1150
1.30k
    FieldDecl *getMemcpyableField(Stmt *S) {
1151
1.30k
      if (!AssignmentsMemcpyable)
1152
11
        return nullptr;
1153
1.29k
      if (BinaryOperator *BO = dyn_cast<BinaryOperator>(S)) {
1154
        // Recognise trivial assignments.
1155
226
        if (BO->getOpcode() != BO_Assign)
1156
0
          return nullptr;
1157
226
        MemberExpr *ME = dyn_cast<MemberExpr>(BO->getLHS());
1158
226
        if (!ME)
1159
0
          return nullptr;
1160
226
        FieldDecl *Field = dyn_cast<FieldDecl>(ME->getMemberDecl());
1161
226
        if (!Field || !isMemcpyableField(Field))
1162
10
          return nullptr;
1163
216
        Stmt *RHS = BO->getRHS();
1164
216
        if (ImplicitCastExpr *EC = dyn_cast<ImplicitCastExpr>(RHS))
1165
216
          RHS = EC->getSubExpr();
1166
216
        if (!RHS)
1167
0
          return nullptr;
1168
216
        if (MemberExpr *ME2 = dyn_cast<MemberExpr>(RHS)) {
1169
214
          if (ME2->getMemberDecl() == Field)
1170
214
            return Field;
1171
214
        }
1172
2
        return nullptr;
1173
1.06k
      } else if (CXXMemberCallExpr *MCE = dyn_cast<CXXMemberCallExpr>(S)) {
1174
313
        CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(MCE->getCalleeDecl());
1175
313
        if (!(MD && isMemcpyEquivalentSpecialMember(MD)))
1176
128
          return nullptr;
1177
185
        MemberExpr *IOA = dyn_cast<MemberExpr>(MCE->getImplicitObjectArgument());
1178
185
        if (!IOA)
1179
146
          return nullptr;
1180
39
        FieldDecl *Field = dyn_cast<FieldDecl>(IOA->getMemberDecl());
1181
39
        if (!Field || !isMemcpyableField(Field))
1182
0
          return nullptr;
1183
39
        MemberExpr *Arg0 = dyn_cast<MemberExpr>(MCE->getArg(0));
1184
39
        if (!Arg0 || 
Field != dyn_cast<FieldDecl>(Arg0->getMemberDecl())38
)
1185
1
          return nullptr;
1186
38
        return Field;
1187
756
      } else if (CallExpr *CE = dyn_cast<CallExpr>(S)) {
1188
11
        FunctionDecl *FD = dyn_cast<FunctionDecl>(CE->getCalleeDecl());
1189
11
        if (!FD || FD->getBuiltinID() != Builtin::BI__builtin_memcpy)
1190
0
          return nullptr;
1191
11
        Expr *DstPtr = CE->getArg(0);
1192
11
        if (ImplicitCastExpr *DC = dyn_cast<ImplicitCastExpr>(DstPtr))
1193
11
          DstPtr = DC->getSubExpr();
1194
11
        UnaryOperator *DUO = dyn_cast<UnaryOperator>(DstPtr);
1195
11
        if (!DUO || DUO->getOpcode() != UO_AddrOf)
1196
0
          return nullptr;
1197
11
        MemberExpr *ME = dyn_cast<MemberExpr>(DUO->getSubExpr());
1198
11
        if (!ME)
1199
0
          return nullptr;
1200
11
        FieldDecl *Field = dyn_cast<FieldDecl>(ME->getMemberDecl());
1201
11
        if (!Field || !isMemcpyableField(Field))
1202
0
          return nullptr;
1203
11
        Expr *SrcPtr = CE->getArg(1);
1204
11
        if (ImplicitCastExpr *SC = dyn_cast<ImplicitCastExpr>(SrcPtr))
1205
11
          SrcPtr = SC->getSubExpr();
1206
11
        UnaryOperator *SUO = dyn_cast<UnaryOperator>(SrcPtr);
1207
11
        if (!SUO || SUO->getOpcode() != UO_AddrOf)
1208
0
          return nullptr;
1209
11
        MemberExpr *ME2 = dyn_cast<MemberExpr>(SUO->getSubExpr());
1210
11
        if (!ME2 || Field != dyn_cast<FieldDecl>(ME2->getMemberDecl()))
1211
0
          return nullptr;
1212
11
        return Field;
1213
11
      }
1214
1215
745
      return nullptr;
1216
1.29k
    }
1217
1218
    bool AssignmentsMemcpyable;
1219
    SmallVector<Stmt*, 16> AggregatedStmts;
1220
1221
  public:
1222
    AssignmentMemcpyizer(CodeGenFunction &CGF, const CXXMethodDecl *AD,
1223
                         FunctionArgList &Args)
1224
      : FieldMemcpyizer(CGF, AD->getParent(), Args[Args.size() - 1]),
1225
726
        AssignmentsMemcpyable(CGF.getLangOpts().getGC() == LangOptions::NonGC) {
1226
726
      assert(Args.size() == 2);
1227
726
    }
1228
1229
1.30k
    void emitAssignment(Stmt *S) {
1230
1.30k
      FieldDecl *F = getMemcpyableField(S);
1231
1.30k
      if (F) {
1232
263
        addMemcpyableField(F);
1233
263
        AggregatedStmts.push_back(S);
1234
1.04k
      } else {
1235
1.04k
        emitAggregatedStmts();
1236
1.04k
        CGF.EmitStmt(S);
1237
1.04k
      }
1238
1.30k
    }
1239
1240
1.76k
    void emitAggregatedStmts() {
1241
1.76k
      if (AggregatedStmts.size() <= 1) {
1242
1.72k
        if (!AggregatedStmts.empty()) {
1243
100
          CopyingValueRepresentation CVR(CGF);
1244
100
          CGF.EmitStmt(AggregatedStmts[0]);
1245
100
        }
1246
1.72k
        reset();
1247
1.72k
      }
1248
1249
1.76k
      emitMemcpy();
1250
1.76k
      AggregatedStmts.clear();
1251
1.76k
    }
1252
1253
726
    void finish() {
1254
726
      emitAggregatedStmts();
1255
726
    }
1256
  };
1257
} // end anonymous namespace
1258
1259
27
static bool isInitializerOfDynamicClass(const CXXCtorInitializer *BaseInit) {
1260
27
  const Type *BaseType = BaseInit->getBaseClass();
1261
27
  const auto *BaseClassDecl =
1262
27
      cast<CXXRecordDecl>(BaseType->castAs<RecordType>()->getDecl());
1263
27
  return BaseClassDecl->isDynamicClass();
1264
27
}
1265
1266
/// EmitCtorPrologue - This routine generates necessary code to initialize
1267
/// base classes and non-static data members belonging to this constructor.
1268
void CodeGenFunction::EmitCtorPrologue(const CXXConstructorDecl *CD,
1269
                                       CXXCtorType CtorType,
1270
24.7k
                                       FunctionArgList &Args) {
1271
24.7k
  if (CD->isDelegatingConstructor())
1272
94
    return EmitDelegatingCXXConstructorCall(CD, Args);
1273
1274
24.6k
  const CXXRecordDecl *ClassDecl = CD->getParent();
1275
1276
24.6k
  CXXConstructorDecl::init_const_iterator B = CD->init_begin(),
1277
24.6k
                                          E = CD->init_end();
1278
1279
  // Virtual base initializers first, if any. They aren't needed if:
1280
  // - This is a base ctor variant
1281
  // - There are no vbases
1282
  // - The class is abstract, so a complete object of it cannot be constructed
1283
  //
1284
  // The check for an abstract class is necessary because sema may not have
1285
  // marked virtual base destructors referenced.
1286
24.6k
  bool ConstructVBases = CtorType != Ctor_Base &&
1287
24.6k
                         
ClassDecl->getNumVBases() != 01.49k
&&
1288
24.6k
                         
!ClassDecl->isAbstract()594
;
1289
1290
  // In the Microsoft C++ ABI, there are no constructor variants. Instead, the
1291
  // constructor of a class with virtual bases takes an additional parameter to
1292
  // conditionally construct the virtual bases. Emit that check here.
1293
24.6k
  llvm::BasicBlock *BaseCtorContinueBB = nullptr;
1294
24.6k
  if (ConstructVBases &&
1295
24.6k
      
!CGM.getTarget().getCXXABI().hasConstructorVariants()591
) {
1296
381
    BaseCtorContinueBB =
1297
381
        CGM.getCXXABI().EmitCtorCompleteObjectHandler(*this, ClassDecl);
1298
381
    assert(BaseCtorContinueBB);
1299
381
  }
1300
1301
0
  llvm::Value *const OldThis = CXXThisValue;
1302
25.6k
  for (; B != E && 
(*B)->isBaseInitializer()20.1k
&&
(*B)->isBaseVirtual()8.34k
;
B++946
) {
1303
946
    if (!ConstructVBases)
1304
165
      continue;
1305
781
    if (CGM.getCodeGenOpts().StrictVTablePointers &&
1306
781
        
CGM.getCodeGenOpts().OptimizationLevel > 03
&&
1307
781
        
isInitializerOfDynamicClass(*B)3
)
1308
1
      CXXThisValue = Builder.CreateLaunderInvariantGroup(LoadCXXThis());
1309
781
    EmitBaseInitializer(*this, ClassDecl, *B);
1310
781
  }
1311
1312
24.6k
  if (BaseCtorContinueBB) {
1313
    // Complete object handler should continue to the remaining initializers.
1314
381
    Builder.CreateBr(BaseCtorContinueBB);
1315
381
    EmitBlock(BaseCtorContinueBB);
1316
381
  }
1317
1318
  // Then, non-virtual base initializers.
1319
34.7k
  for (; B != E && 
(*B)->isBaseInitializer()23.0k
;
B++10.0k
) {
1320
10.0k
    assert(!(*B)->isBaseVirtual());
1321
1322
10.0k
    if (CGM.getCodeGenOpts().StrictVTablePointers &&
1323
10.0k
        
CGM.getCodeGenOpts().OptimizationLevel > 024
&&
1324
10.0k
        
isInitializerOfDynamicClass(*B)24
)
1325
23
      CXXThisValue = Builder.CreateLaunderInvariantGroup(LoadCXXThis());
1326
10.0k
    EmitBaseInitializer(*this, ClassDecl, *B);
1327
10.0k
  }
1328
1329
24.6k
  CXXThisValue = OldThis;
1330
1331
24.6k
  InitializeVTablePointers(ClassDecl);
1332
1333
  // And finally, initialize class members.
1334
24.6k
  FieldConstructionScope FCS(*this, LoadCXXThisAddress());
1335
24.6k
  ConstructorMemcpyizer CM(*this, CD, Args);
1336
41.8k
  for (; B != E; 
B++17.2k
) {
1337
17.2k
    CXXCtorInitializer *Member = (*B);
1338
17.2k
    assert(!Member->isBaseInitializer());
1339
0
    assert(Member->isAnyMemberInitializer() &&
1340
17.2k
           "Delegating initializer on non-delegating constructor");
1341
0
    CM.addMemberInitializer(Member);
1342
17.2k
  }
1343
24.6k
  CM.finish();
1344
24.6k
}
1345
1346
static bool
1347
FieldHasTrivialDestructorBody(ASTContext &Context, const FieldDecl *Field);
1348
1349
static bool
1350
HasTrivialDestructorBody(ASTContext &Context,
1351
                         const CXXRecordDecl *BaseClassDecl,
1352
                         const CXXRecordDecl *MostDerivedClassDecl)
1353
404
{
1354
  // If the destructor is trivial we don't have to check anything else.
1355
404
  if (BaseClassDecl->hasTrivialDestructor())
1356
126
    return true;
1357
1358
278
  if (!BaseClassDecl->getDestructor()->hasTrivialBody())
1359
213
    return false;
1360
1361
  // Check fields.
1362
65
  for (const auto *Field : BaseClassDecl->fields())
1363
53
    if (!FieldHasTrivialDestructorBody(Context, Field))
1364
44
      return false;
1365
1366
  // Check non-virtual bases.
1367
21
  for (const auto &I : BaseClassDecl->bases()) {
1368
15
    if (I.isVirtual())
1369
1
      continue;
1370
1371
14
    const CXXRecordDecl *NonVirtualBase =
1372
14
      cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
1373
14
    if (!HasTrivialDestructorBody(Context, NonVirtualBase,
1374
14
                                  MostDerivedClassDecl))
1375
14
      return false;
1376
14
  }
1377
1378
7
  if (BaseClassDecl == MostDerivedClassDecl) {
1379
    // Check virtual bases.
1380
7
    for (const auto &I : BaseClassDecl->vbases()) {
1381
1
      const CXXRecordDecl *VirtualBase =
1382
1
        cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
1383
1
      if (!HasTrivialDestructorBody(Context, VirtualBase,
1384
1
                                    MostDerivedClassDecl))
1385
1
        return false;
1386
1
    }
1387
7
  }
1388
1389
6
  return true;
1390
7
}
1391
1392
static bool
1393
FieldHasTrivialDestructorBody(ASTContext &Context,
1394
                                          const FieldDecl *Field)
1395
760
{
1396
760
  QualType FieldBaseElementType = Context.getBaseElementType(Field->getType());
1397
1398
760
  const RecordType *RT = FieldBaseElementType->getAs<RecordType>();
1399
760
  if (!RT)
1400
369
    return true;
1401
1402
391
  CXXRecordDecl *FieldClassDecl = cast<CXXRecordDecl>(RT->getDecl());
1403
1404
  // The destructor for an implicit anonymous union member is never invoked.
1405
391
  if (FieldClassDecl->isUnion() && 
FieldClassDecl->isAnonymousStructOrUnion()2
)
1406
2
    return false;
1407
1408
389
  return HasTrivialDestructorBody(Context, FieldClassDecl, FieldClassDecl);
1409
391
}
1410
1411
/// CanSkipVTablePointerInitialization - Check whether we need to initialize
1412
/// any vtable pointers before calling this destructor.
1413
static bool CanSkipVTablePointerInitialization(CodeGenFunction &CGF,
1414
8.18k
                                               const CXXDestructorDecl *Dtor) {
1415
8.18k
  const CXXRecordDecl *ClassDecl = Dtor->getParent();
1416
8.18k
  if (!ClassDecl->isDynamicClass())
1417
7.22k
    return true;
1418
1419
  // For a final class, the vtable pointer is known to already point to the
1420
  // class's vtable.
1421
960
  if (ClassDecl->isEffectivelyFinal())
1422
6
    return true;
1423
1424
954
  if (!Dtor->hasTrivialBody())
1425
66
    return false;
1426
1427
  // Check the fields.
1428
888
  for (const auto *Field : ClassDecl->fields())
1429
369
    if (!FieldHasTrivialDestructorBody(CGF.getContext(), Field))
1430
105
      return false;
1431
1432
783
  return true;
1433
888
}
1434
1435
/// EmitDestructorBody - Emits the body of the current destructor.
1436
16.6k
void CodeGenFunction::EmitDestructorBody(FunctionArgList &Args) {
1437
16.6k
  const CXXDestructorDecl *Dtor = cast<CXXDestructorDecl>(CurGD.getDecl());
1438
16.6k
  CXXDtorType DtorType = CurGD.getDtorType();
1439
1440
  // For an abstract class, non-base destructors are never used (and can't
1441
  // be emitted in general, because vbase dtors may not have been validated
1442
  // by Sema), but the Itanium ABI doesn't make them optional and Clang may
1443
  // in fact emit references to them from other compilations, so emit them
1444
  // as functions containing a trap instruction.
1445
16.6k
  if (DtorType != Dtor_Base && 
Dtor->getParent()->isAbstract()8.49k
) {
1446
82
    llvm::CallInst *TrapCall = EmitTrapCall(llvm::Intrinsic::trap);
1447
82
    TrapCall->setDoesNotReturn();
1448
82
    TrapCall->setDoesNotThrow();
1449
82
    Builder.CreateUnreachable();
1450
82
    Builder.ClearInsertionPoint();
1451
82
    return;
1452
82
  }
1453
1454
16.5k
  Stmt *Body = Dtor->getBody();
1455
16.5k
  if (Body)
1456
16.4k
    incrementProfileCounter(Body);
1457
1458
  // The call to operator delete in a deleting destructor happens
1459
  // outside of the function-try-block, which means it's always
1460
  // possible to delegate the destructor body to the complete
1461
  // destructor.  Do so.
1462
16.5k
  if (DtorType == Dtor_Deleting) {
1463
877
    RunCleanupsScope DtorEpilogue(*this);
1464
877
    EnterDtorCleanups(Dtor, Dtor_Deleting);
1465
877
    if (HaveInsertPoint()) {
1466
873
      QualType ThisTy = Dtor->getThisObjectType();
1467
873
      EmitCXXDestructorCall(Dtor, Dtor_Complete, /*ForVirtualBase=*/false,
1468
873
                            /*Delegating=*/false, LoadCXXThisAddress(), ThisTy);
1469
873
    }
1470
877
    return;
1471
877
  }
1472
1473
  // If the body is a function-try-block, enter the try before
1474
  // anything else.
1475
15.7k
  bool isTryBody = (Body && 
isa<CXXTryStmt>(Body)15.6k
);
1476
15.7k
  if (isTryBody)
1477
18
    EnterCXXTryStmt(*cast<CXXTryStmt>(Body), true);
1478
15.7k
  EmitAsanPrologueOrEpilogue(false);
1479
1480
  // Enter the epilogue cleanups.
1481
15.7k
  RunCleanupsScope DtorEpilogue(*this);
1482
1483
  // If this is the complete variant, just invoke the base variant;
1484
  // the epilogue will destruct the virtual bases.  But we can't do
1485
  // this optimization if the body is a function-try-block, because
1486
  // we'd introduce *two* handler blocks.  In the Microsoft ABI, we
1487
  // always delegate because we might not have a definition in this TU.
1488
15.7k
  switch (DtorType) {
1489
0
  case Dtor_Comdat: llvm_unreachable("not expecting a COMDAT");
1490
0
  case Dtor_Deleting: llvm_unreachable("already handled deleting case");
1491
1492
7.53k
  case Dtor_Complete:
1493
7.53k
    assert((Body || getTarget().getCXXABI().isMicrosoft()) &&
1494
7.53k
           "can't emit a dtor without a body for non-Microsoft ABIs");
1495
1496
    // Enter the cleanup scopes for virtual bases.
1497
0
    EnterDtorCleanups(Dtor, Dtor_Complete);
1498
1499
7.53k
    if (!isTryBody) {
1500
7.52k
      QualType ThisTy = Dtor->getThisObjectType();
1501
7.52k
      EmitCXXDestructorCall(Dtor, Dtor_Base, /*ForVirtualBase=*/false,
1502
7.52k
                            /*Delegating=*/false, LoadCXXThisAddress(), ThisTy);
1503
7.52k
      break;
1504
7.52k
    }
1505
1506
    // Fallthrough: act like we're in the base variant.
1507
7.53k
    
LLVM_FALLTHROUGH8
;8
1508
1509
8.18k
  case Dtor_Base:
1510
8.18k
    assert(Body);
1511
1512
    // Enter the cleanup scopes for fields and non-virtual bases.
1513
0
    EnterDtorCleanups(Dtor, Dtor_Base);
1514
1515
    // Initialize the vtable pointers before entering the body.
1516
8.18k
    if (!CanSkipVTablePointerInitialization(*this, Dtor)) {
1517
      // Insert the llvm.launder.invariant.group intrinsic before initializing
1518
      // the vptrs to cancel any previous assumptions we might have made.
1519
171
      if (CGM.getCodeGenOpts().StrictVTablePointers &&
1520
171
          
CGM.getCodeGenOpts().OptimizationLevel > 02
)
1521
2
        CXXThisValue = Builder.CreateLaunderInvariantGroup(LoadCXXThis());
1522
171
      InitializeVTablePointers(Dtor->getParent());
1523
171
    }
1524
1525
8.18k
    if (isTryBody)
1526
18
      EmitStmt(cast<CXXTryStmt>(Body)->getTryBlock());
1527
8.16k
    else if (Body)
1528
8.16k
      EmitStmt(Body);
1529
0
    else {
1530
0
      assert(Dtor->isImplicit() && "bodyless dtor not implicit");
1531
      // nothing to do besides what's in the epilogue
1532
0
    }
1533
    // -fapple-kext must inline any call to this dtor into
1534
    // the caller's body.
1535
8.18k
    if (getLangOpts().AppleKext)
1536
5
      CurFn->addFnAttr(llvm::Attribute::AlwaysInline);
1537
1538
8.18k
    break;
1539
15.7k
  }
1540
1541
  // Jump out through the epilogue cleanups.
1542
15.7k
  DtorEpilogue.ForceCleanup();
1543
1544
  // Exit the try if applicable.
1545
15.7k
  if (isTryBody)
1546
18
    ExitCXXTryStmt(*cast<CXXTryStmt>(Body), true);
1547
15.7k
}
1548
1549
726
void CodeGenFunction::emitImplicitAssignmentOperatorBody(FunctionArgList &Args) {
1550
726
  const CXXMethodDecl *AssignOp = cast<CXXMethodDecl>(CurGD.getDecl());
1551
726
  const Stmt *RootS = AssignOp->getBody();
1552
726
  assert(isa<CompoundStmt>(RootS) &&
1553
726
         "Body of an implicit assignment operator should be compound stmt.");
1554
0
  const CompoundStmt *RootCS = cast<CompoundStmt>(RootS);
1555
1556
726
  LexicalScope Scope(*this, RootCS->getSourceRange());
1557
1558
726
  incrementProfileCounter(RootCS);
1559
726
  AssignmentMemcpyizer AM(*this, AssignOp, Args);
1560
726
  for (auto *I : RootCS->body())
1561
1.30k
    AM.emitAssignment(I);
1562
726
  AM.finish();
1563
726
}
1564
1565
namespace {
1566
  llvm::Value *LoadThisForDtorDelete(CodeGenFunction &CGF,
1567
890
                                     const CXXDestructorDecl *DD) {
1568
890
    if (Expr *ThisArg = DD->getOperatorDeleteThisArg())
1569
12
      return CGF.EmitScalarExpr(ThisArg);
1570
878
    return CGF.LoadCXXThis();
1571
890
  }
1572
1573
  /// Call the operator delete associated with the current destructor.
1574
  struct CallDtorDelete final : EHScopeStack::Cleanup {
1575
660
    CallDtorDelete() {}
1576
1577
673
    void Emit(CodeGenFunction &CGF, Flags flags) override {
1578
673
      const CXXDestructorDecl *Dtor = cast<CXXDestructorDecl>(CGF.CurCodeDecl);
1579
673
      const CXXRecordDecl *ClassDecl = Dtor->getParent();
1580
673
      CGF.EmitDeleteCall(Dtor->getOperatorDelete(),
1581
673
                         LoadThisForDtorDelete(CGF, Dtor),
1582
673
                         CGF.getContext().getTagDeclType(ClassDecl));
1583
673
    }
1584
  };
1585
1586
  void EmitConditionalDtorDeleteCall(CodeGenFunction &CGF,
1587
                                     llvm::Value *ShouldDeleteCondition,
1588
213
                                     bool ReturnAfterDelete) {
1589
213
    llvm::BasicBlock *callDeleteBB = CGF.createBasicBlock("dtor.call_delete");
1590
213
    llvm::BasicBlock *continueBB = CGF.createBasicBlock("dtor.continue");
1591
213
    llvm::Value *ShouldCallDelete
1592
213
      = CGF.Builder.CreateIsNull(ShouldDeleteCondition);
1593
213
    CGF.Builder.CreateCondBr(ShouldCallDelete, continueBB, callDeleteBB);
1594
1595
213
    CGF.EmitBlock(callDeleteBB);
1596
213
    const CXXDestructorDecl *Dtor = cast<CXXDestructorDecl>(CGF.CurCodeDecl);
1597
213
    const CXXRecordDecl *ClassDecl = Dtor->getParent();
1598
213
    CGF.EmitDeleteCall(Dtor->getOperatorDelete(),
1599
213
                       LoadThisForDtorDelete(CGF, Dtor),
1600
213
                       CGF.getContext().getTagDeclType(ClassDecl));
1601
213
    assert(Dtor->getOperatorDelete()->isDestroyingOperatorDelete() ==
1602
213
               ReturnAfterDelete &&
1603
213
           "unexpected value for ReturnAfterDelete");
1604
213
    if (ReturnAfterDelete)
1605
8
      CGF.EmitBranchThroughCleanup(CGF.ReturnBlock);
1606
205
    else
1607
205
      CGF.Builder.CreateBr(continueBB);
1608
1609
213
    CGF.EmitBlock(continueBB);
1610
213
  }
1611
1612
  struct CallDtorDeleteConditional final : EHScopeStack::Cleanup {
1613
    llvm::Value *ShouldDeleteCondition;
1614
1615
  public:
1616
    CallDtorDeleteConditional(llvm::Value *ShouldDeleteCondition)
1617
205
        : ShouldDeleteCondition(ShouldDeleteCondition) {
1618
205
      assert(ShouldDeleteCondition != nullptr);
1619
205
    }
1620
1621
205
    void Emit(CodeGenFunction &CGF, Flags flags) override {
1622
205
      EmitConditionalDtorDeleteCall(CGF, ShouldDeleteCondition,
1623
205
                                    /*ReturnAfterDelete*/false);
1624
205
    }
1625
  };
1626
1627
  class DestroyField  final : public EHScopeStack::Cleanup {
1628
    const FieldDecl *field;
1629
    CodeGenFunction::Destroyer *destroyer;
1630
    bool useEHCleanupForArray;
1631
1632
  public:
1633
    DestroyField(const FieldDecl *field, CodeGenFunction::Destroyer *destroyer,
1634
                 bool useEHCleanupForArray)
1635
        : field(field), destroyer(destroyer),
1636
1.07k
          useEHCleanupForArray(useEHCleanupForArray) {}
1637
1638
1.12k
    void Emit(CodeGenFunction &CGF, Flags flags) override {
1639
      // Find the address of the field.
1640
1.12k
      Address thisValue = CGF.LoadCXXThisAddress();
1641
1.12k
      QualType RecordTy = CGF.getContext().getTagDeclType(field->getParent());
1642
1.12k
      LValue ThisLV = CGF.MakeAddrLValue(thisValue, RecordTy);
1643
1.12k
      LValue LV = CGF.EmitLValueForField(ThisLV, field);
1644
1.12k
      assert(LV.isSimple());
1645
1646
0
      CGF.emitDestroy(LV.getAddress(CGF), field->getType(), destroyer,
1647
1.12k
                      flags.isForNormalCleanup() && 
useEHCleanupForArray1.07k
);
1648
1.12k
    }
1649
  };
1650
1651
 static void EmitSanitizerDtorCallback(CodeGenFunction &CGF, llvm::Value *Ptr,
1652
130
             CharUnits::QuantityType PoisonSize) {
1653
130
   CodeGenFunction::SanitizerScope SanScope(&CGF);
1654
   // Pass in void pointer and size of region as arguments to runtime
1655
   // function
1656
130
   llvm::Value *Args[] = {CGF.Builder.CreateBitCast(Ptr, CGF.VoidPtrTy),
1657
130
                          llvm::ConstantInt::get(CGF.SizeTy, PoisonSize)};
1658
1659
130
   llvm::Type *ArgTypes[] = {CGF.VoidPtrTy, CGF.SizeTy};
1660
1661
130
   llvm::FunctionType *FnType =
1662
130
       llvm::FunctionType::get(CGF.VoidTy, ArgTypes, false);
1663
130
   llvm::FunctionCallee Fn =
1664
130
       CGF.CGM.CreateRuntimeFunction(FnType, "__sanitizer_dtor_callback");
1665
130
   CGF.EmitNounwindRuntimeCall(Fn, Args);
1666
130
 }
1667
1668
  class SanitizeDtorMembers final : public EHScopeStack::Cleanup {
1669
    const CXXDestructorDecl *Dtor;
1670
1671
  public:
1672
88
    SanitizeDtorMembers(const CXXDestructorDecl *Dtor) : Dtor(Dtor) {}
1673
1674
    // Generate function call for handling object poisoning.
1675
    // Disables tail call elimination, to prevent the current stack frame
1676
    // from disappearing from the stack trace.
1677
88
    void Emit(CodeGenFunction &CGF, Flags flags) override {
1678
88
      const ASTRecordLayout &Layout =
1679
88
          CGF.getContext().getASTRecordLayout(Dtor->getParent());
1680
1681
      // Nothing to poison.
1682
88
      if (Layout.getFieldCount() == 0)
1683
0
        return;
1684
1685
      // Prevent the current stack frame from disappearing from the stack trace.
1686
88
      CGF.CurFn->addFnAttr("disable-tail-calls", "true");
1687
1688
      // Construct pointer to region to begin poisoning, and calculate poison
1689
      // size, so that only members declared in this class are poisoned.
1690
88
      ASTContext &Context = CGF.getContext();
1691
1692
88
      const RecordDecl *Decl = Dtor->getParent();
1693
88
      auto Fields = Decl->fields();
1694
338
      auto IsTrivial = [&](const FieldDecl *F) {
1695
338
        return FieldHasTrivialDestructorBody(Context, F);
1696
338
      };
1697
1698
170
      auto IsZeroSize = [&](const FieldDecl *F) {
1699
170
        return F->isZeroSize(Context);
1700
170
      };
1701
1702
      // Poison blocks of fields with trivial destructors making sure that block
1703
      // begin and end do not point to zero-sized fields. They don't have
1704
      // correct offsets so can't be used to calculate poisoning range.
1705
200
      for (auto It = Fields.begin(); It != Fields.end();) {
1706
188
        It = std::find_if(It, Fields.end(), [&](const FieldDecl *F) {
1707
188
          return IsTrivial(F) && 
!IsZeroSize(F)124
;
1708
188
        });
1709
128
        if (It == Fields.end())
1710
16
          break;
1711
112
        auto Start = It++;
1712
150
        It = std::find_if(It, Fields.end(), [&](const FieldDecl *F) {
1713
150
          return !IsTrivial(F) && 
!IsZeroSize(F)46
;
1714
150
        });
1715
1716
112
        PoisonMembers(CGF, (*Start)->getFieldIndex(),
1717
112
                      It == Fields.end() ? 
-172
:
(*It)->getFieldIndex()40
);
1718
112
      }
1719
88
    }
1720
1721
  private:
1722
    /// \param layoutStartOffset index of the ASTRecordLayout field to
1723
    ///     start poisoning (inclusive)
1724
    /// \param layoutEndOffset index of the ASTRecordLayout field to
1725
    ///     end poisoning (exclusive)
1726
    void PoisonMembers(CodeGenFunction &CGF, unsigned layoutStartOffset,
1727
112
                       unsigned layoutEndOffset) {
1728
112
      ASTContext &Context = CGF.getContext();
1729
112
      const ASTRecordLayout &Layout =
1730
112
          Context.getASTRecordLayout(Dtor->getParent());
1731
1732
      // It's a first trivia field so it should be at the begining of char,
1733
      // still round up start offset just in case.
1734
112
      CharUnits PoisonStart =
1735
112
          Context.toCharUnitsFromBits(Layout.getFieldOffset(layoutStartOffset) +
1736
112
                                      Context.getCharWidth() - 1);
1737
112
      llvm::ConstantInt *OffsetSizePtr =
1738
112
          llvm::ConstantInt::get(CGF.SizeTy, PoisonStart.getQuantity());
1739
1740
112
      llvm::Value *OffsetPtr = CGF.Builder.CreateGEP(
1741
112
          CGF.Int8Ty,
1742
112
          CGF.Builder.CreateBitCast(CGF.LoadCXXThis(), CGF.Int8PtrTy),
1743
112
          OffsetSizePtr);
1744
1745
112
      CharUnits PoisonEnd;
1746
112
      if (layoutEndOffset >= Layout.getFieldCount()) {
1747
72
        PoisonEnd = Layout.getNonVirtualSize();
1748
72
      } else {
1749
40
        PoisonEnd =
1750
40
            Context.toCharUnitsFromBits(Layout.getFieldOffset(layoutEndOffset));
1751
40
      }
1752
112
      CharUnits PoisonSize = PoisonEnd - PoisonStart;
1753
112
      if (!PoisonSize.isPositive())
1754
0
        return;
1755
1756
112
      EmitSanitizerDtorCallback(CGF, OffsetPtr, PoisonSize.getQuantity());
1757
112
    }
1758
  };
1759
1760
 class SanitizeDtorVTable final : public EHScopeStack::Cleanup {
1761
    const CXXDestructorDecl *Dtor;
1762
1763
  public:
1764
18
    SanitizeDtorVTable(const CXXDestructorDecl *Dtor) : Dtor(Dtor) {}
1765
1766
    // Generate function call for handling vtable pointer poisoning.
1767
18
    void Emit(CodeGenFunction &CGF, Flags flags) override {
1768
18
      assert(Dtor->getParent()->isDynamicClass());
1769
0
      (void)Dtor;
1770
18
      ASTContext &Context = CGF.getContext();
1771
      // Poison vtable and vtable ptr if they exist for this class.
1772
18
      llvm::Value *VTablePtr = CGF.LoadCXXThis();
1773
1774
18
      CharUnits::QuantityType PoisonSize =
1775
18
          Context.toCharUnitsFromBits(CGF.PointerWidthInBits).getQuantity();
1776
      // Pass in void pointer and size of region as arguments to runtime
1777
      // function
1778
18
      EmitSanitizerDtorCallback(CGF, VTablePtr, PoisonSize);
1779
18
    }
1780
 };
1781
} // end anonymous namespace
1782
1783
/// Emit all code that comes at the end of class's
1784
/// destructor. This is to call destructors on members and base classes
1785
/// in reverse order of their construction.
1786
///
1787
/// For a deleting destructor, this also handles the case where a destroying
1788
/// operator delete completely overrides the definition.
1789
void CodeGenFunction::EnterDtorCleanups(const CXXDestructorDecl *DD,
1790
16.5k
                                        CXXDtorType DtorType) {
1791
16.5k
  assert((!DD->isTrivial() || DD->hasAttr<DLLExportAttr>()) &&
1792
16.5k
         "Should not emit dtor epilogue for non-exported trivial dtor!");
1793
1794
  // The deleting-destructor phase just needs to call the appropriate
1795
  // operator delete that Sema picked up.
1796
16.5k
  if (DtorType == Dtor_Deleting) {
1797
877
    assert(DD->getOperatorDelete() &&
1798
877
           "operator delete missing - EnterDtorCleanups");
1799
877
    if (CXXStructorImplicitParamValue) {
1800
      // If there is an implicit param to the deleting dtor, it's a boolean
1801
      // telling whether this is a deleting destructor.
1802
213
      if (DD->getOperatorDelete()->isDestroyingOperatorDelete())
1803
8
        EmitConditionalDtorDeleteCall(*this, CXXStructorImplicitParamValue,
1804
8
                                      /*ReturnAfterDelete*/true);
1805
205
      else
1806
205
        EHStack.pushCleanup<CallDtorDeleteConditional>(
1807
205
            NormalAndEHCleanup, CXXStructorImplicitParamValue);
1808
664
    } else {
1809
664
      if (DD->getOperatorDelete()->isDestroyingOperatorDelete()) {
1810
4
        const CXXRecordDecl *ClassDecl = DD->getParent();
1811
4
        EmitDeleteCall(DD->getOperatorDelete(),
1812
4
                       LoadThisForDtorDelete(*this, DD),
1813
4
                       getContext().getTagDeclType(ClassDecl));
1814
4
        EmitBranchThroughCleanup(ReturnBlock);
1815
660
      } else {
1816
660
        EHStack.pushCleanup<CallDtorDelete>(NormalAndEHCleanup);
1817
660
      }
1818
664
    }
1819
877
    return;
1820
877
  }
1821
1822
15.7k
  const CXXRecordDecl *ClassDecl = DD->getParent();
1823
1824
  // Unions have no bases and do not call field destructors.
1825
15.7k
  if (ClassDecl->isUnion())
1826
4
    return;
1827
1828
  // The complete-destructor phase just destructs all the virtual bases.
1829
15.7k
  if (DtorType == Dtor_Complete) {
1830
    // Poison the vtable pointer such that access after the base
1831
    // and member destructors are invoked is invalid.
1832
7.53k
    if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
1833
7.53k
        
SanOpts.has(SanitizerKind::Memory)89
&&
ClassDecl->getNumVBases()88
&&
1834
7.53k
        
ClassDecl->isPolymorphic()4
)
1835
4
      EHStack.pushCleanup<SanitizeDtorVTable>(NormalAndEHCleanup, DD);
1836
1837
    // We push them in the forward order so that they'll be popped in
1838
    // the reverse order.
1839
7.53k
    for (const auto &Base : ClassDecl->vbases()) {
1840
225
      auto *BaseClassDecl =
1841
225
          cast<CXXRecordDecl>(Base.getType()->castAs<RecordType>()->getDecl());
1842
1843
      // Ignore trivial destructors.
1844
225
      if (BaseClassDecl->hasTrivialDestructor())
1845
44
        continue;
1846
1847
181
      EHStack.pushCleanup<CallBaseDtor>(NormalAndEHCleanup,
1848
181
                                        BaseClassDecl,
1849
181
                                        /*BaseIsVirtual*/ true);
1850
181
    }
1851
1852
7.53k
    return;
1853
7.53k
  }
1854
1855
8.18k
  assert(DtorType == Dtor_Base);
1856
  // Poison the vtable pointer if it has no virtual bases, but inherits
1857
  // virtual functions.
1858
8.18k
  if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
1859
8.18k
      
SanOpts.has(SanitizerKind::Memory)89
&&
!ClassDecl->getNumVBases()88
&&
1860
8.18k
      
ClassDecl->isPolymorphic()84
)
1861
14
    EHStack.pushCleanup<SanitizeDtorVTable>(NormalAndEHCleanup, DD);
1862
1863
  // Destroy non-virtual bases.
1864
8.18k
  for (const auto &Base : ClassDecl->bases()) {
1865
    // Ignore virtual bases.
1866
1.83k
    if (Base.isVirtual())
1867
125
      continue;
1868
1869
1.70k
    CXXRecordDecl *BaseClassDecl = Base.getType()->getAsCXXRecordDecl();
1870
1871
    // Ignore trivial destructors.
1872
1.70k
    if (BaseClassDecl->hasTrivialDestructor())
1873
711
      continue;
1874
1875
995
    EHStack.pushCleanup<CallBaseDtor>(NormalAndEHCleanup,
1876
995
                                      BaseClassDecl,
1877
995
                                      /*BaseIsVirtual*/ false);
1878
995
  }
1879
1880
  // Poison fields such that access after their destructors are
1881
  // invoked, and before the base class destructor runs, is invalid.
1882
8.18k
  if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
1883
8.18k
      
SanOpts.has(SanitizerKind::Memory)89
)
1884
88
    EHStack.pushCleanup<SanitizeDtorMembers>(NormalAndEHCleanup, DD);
1885
1886
  // Destroy direct fields.
1887
9.67k
  for (const auto *Field : ClassDecl->fields()) {
1888
9.67k
    QualType type = Field->getType();
1889
9.67k
    QualType::DestructionKind dtorKind = type.isDestructedType();
1890
9.67k
    if (!dtorKind) 
continue8.59k
;
1891
1892
    // Anonymous union members do not have their destructors called.
1893
1.08k
    const RecordType *RT = type->getAsUnionType();
1894
1.08k
    if (RT && 
RT->getDecl()->isAnonymousStructOrUnion()9
)
continue9
;
1895
1896
1.07k
    CleanupKind cleanupKind = getCleanupKind(dtorKind);
1897
1.07k
    EHStack.pushCleanup<DestroyField>(cleanupKind, Field,
1898
1.07k
                                      getDestroyer(dtorKind),
1899
1.07k
                                      cleanupKind & EHCleanup);
1900
1.07k
  }
1901
8.18k
}
1902
1903
/// EmitCXXAggrConstructorCall - Emit a loop to call a particular
1904
/// constructor for each of several members of an array.
1905
///
1906
/// \param ctor the constructor to call for each element
1907
/// \param arrayType the type of the array to initialize
1908
/// \param arrayBegin an arrayType*
1909
/// \param zeroInitialize true if each element should be
1910
///   zero-initialized before it is constructed
1911
void CodeGenFunction::EmitCXXAggrConstructorCall(
1912
    const CXXConstructorDecl *ctor, const ArrayType *arrayType,
1913
    Address arrayBegin, const CXXConstructExpr *E, bool NewPointerIsChecked,
1914
891
    bool zeroInitialize) {
1915
891
  QualType elementType;
1916
891
  llvm::Value *numElements =
1917
891
    emitArrayLength(arrayType, elementType, arrayBegin);
1918
1919
891
  EmitCXXAggrConstructorCall(ctor, numElements, arrayBegin, E,
1920
891
                             NewPointerIsChecked, zeroInitialize);
1921
891
}
1922
1923
/// EmitCXXAggrConstructorCall - Emit a loop to call a particular
1924
/// constructor for each of several members of an array.
1925
///
1926
/// \param ctor the constructor to call for each element
1927
/// \param numElements the number of elements in the array;
1928
///   may be zero
1929
/// \param arrayBase a T*, where T is the type constructed by ctor
1930
/// \param zeroInitialize true if each element should be
1931
///   zero-initialized before it is constructed
1932
void CodeGenFunction::EmitCXXAggrConstructorCall(const CXXConstructorDecl *ctor,
1933
                                                 llvm::Value *numElements,
1934
                                                 Address arrayBase,
1935
                                                 const CXXConstructExpr *E,
1936
                                                 bool NewPointerIsChecked,
1937
956
                                                 bool zeroInitialize) {
1938
  // It's legal for numElements to be zero.  This can happen both
1939
  // dynamically, because x can be zero in 'new A[x]', and statically,
1940
  // because of GCC extensions that permit zero-length arrays.  There
1941
  // are probably legitimate places where we could assume that this
1942
  // doesn't happen, but it's not clear that it's worth it.
1943
956
  llvm::BranchInst *zeroCheckBranch = nullptr;
1944
1945
  // Optimize for a constant count.
1946
956
  llvm::ConstantInt *constantCount
1947
956
    = dyn_cast<llvm::ConstantInt>(numElements);
1948
956
  if (constantCount) {
1949
    // Just skip out if the constant count is zero.
1950
922
    if (constantCount->isZero()) 
return0
;
1951
1952
  // Otherwise, emit the check.
1953
922
  } else {
1954
34
    llvm::BasicBlock *loopBB = createBasicBlock("new.ctorloop");
1955
34
    llvm::Value *iszero = Builder.CreateIsNull(numElements, "isempty");
1956
34
    zeroCheckBranch = Builder.CreateCondBr(iszero, loopBB, loopBB);
1957
34
    EmitBlock(loopBB);
1958
34
  }
1959
1960
  // Find the end of the array.
1961
956
  llvm::Type *elementType = arrayBase.getElementType();
1962
956
  llvm::Value *arrayBegin = arrayBase.getPointer();
1963
956
  llvm::Value *arrayEnd = Builder.CreateInBoundsGEP(
1964
956
      elementType, arrayBegin, numElements, "arrayctor.end");
1965
1966
  // Enter the loop, setting up a phi for the current location to initialize.
1967
956
  llvm::BasicBlock *entryBB = Builder.GetInsertBlock();
1968
956
  llvm::BasicBlock *loopBB = createBasicBlock("arrayctor.loop");
1969
956
  EmitBlock(loopBB);
1970
956
  llvm::PHINode *cur = Builder.CreatePHI(arrayBegin->getType(), 2,
1971
956
                                         "arrayctor.cur");
1972
956
  cur->addIncoming(arrayBegin, entryBB);
1973
1974
  // Inside the loop body, emit the constructor call on the array element.
1975
1976
  // The alignment of the base, adjusted by the size of a single element,
1977
  // provides a conservative estimate of the alignment of every element.
1978
  // (This assumes we never start tracking offsetted alignments.)
1979
  //
1980
  // Note that these are complete objects and so we don't need to
1981
  // use the non-virtual size or alignment.
1982
956
  QualType type = getContext().getTypeDeclType(ctor->getParent());
1983
956
  CharUnits eltAlignment =
1984
956
    arrayBase.getAlignment()
1985
956
             .alignmentOfArrayElement(getContext().getTypeSizeInChars(type));
1986
956
  Address curAddr = Address(cur, elementType, eltAlignment);
1987
1988
  // Zero initialize the storage, if requested.
1989
956
  if (zeroInitialize)
1990
2
    EmitNullInitialization(curAddr, type);
1991
1992
  // C++ [class.temporary]p4:
1993
  // There are two contexts in which temporaries are destroyed at a different
1994
  // point than the end of the full-expression. The first context is when a
1995
  // default constructor is called to initialize an element of an array.
1996
  // If the constructor has one or more default arguments, the destruction of
1997
  // every temporary created in a default argument expression is sequenced
1998
  // before the construction of the next array element, if any.
1999
2000
956
  {
2001
956
    RunCleanupsScope Scope(*this);
2002
2003
    // Evaluate the constructor and its arguments in a regular
2004
    // partial-destroy cleanup.
2005
956
    if (getLangOpts().Exceptions &&
2006
956
        
!ctor->getParent()->hasTrivialDestructor()86
) {
2007
38
      Destroyer *destroyer = destroyCXXObject;
2008
38
      pushRegularPartialArrayCleanup(arrayBegin, cur, type, eltAlignment,
2009
38
                                     *destroyer);
2010
38
    }
2011
956
    auto currAVS = AggValueSlot::forAddr(
2012
956
        curAddr, type.getQualifiers(), AggValueSlot::IsDestructed,
2013
956
        AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased,
2014
956
        AggValueSlot::DoesNotOverlap, AggValueSlot::IsNotZeroed,
2015
956
        NewPointerIsChecked ? 
AggValueSlot::IsSanitizerChecked95
2016
956
                            : 
AggValueSlot::IsNotSanitizerChecked861
);
2017
956
    EmitCXXConstructorCall(ctor, Ctor_Complete, /*ForVirtualBase=*/false,
2018
956
                           /*Delegating=*/false, currAVS, E);
2019
956
  }
2020
2021
  // Go to the next element.
2022
956
  llvm::Value *next = Builder.CreateInBoundsGEP(
2023
956
      elementType, cur, llvm::ConstantInt::get(SizeTy, 1), "arrayctor.next");
2024
956
  cur->addIncoming(next, Builder.GetInsertBlock());
2025
2026
  // Check whether that's the end of the loop.
2027
956
  llvm::Value *done = Builder.CreateICmpEQ(next, arrayEnd, "arrayctor.done");
2028
956
  llvm::BasicBlock *contBB = createBasicBlock("arrayctor.cont");
2029
956
  Builder.CreateCondBr(done, contBB, loopBB);
2030
2031
  // Patch the earlier check to skip over the loop.
2032
956
  if (zeroCheckBranch) 
zeroCheckBranch->setSuccessor(0, contBB)34
;
2033
2034
956
  EmitBlock(contBB);
2035
956
}
2036
2037
void CodeGenFunction::destroyCXXObject(CodeGenFunction &CGF,
2038
                                       Address addr,
2039
19.6k
                                       QualType type) {
2040
19.6k
  const RecordType *rtype = type->castAs<RecordType>();
2041
19.6k
  const CXXRecordDecl *record = cast<CXXRecordDecl>(rtype->getDecl());
2042
19.6k
  const CXXDestructorDecl *dtor = record->getDestructor();
2043
19.6k
  assert(!dtor->isTrivial());
2044
0
  CGF.EmitCXXDestructorCall(dtor, Dtor_Complete, /*for vbase*/ false,
2045
19.6k
                            /*Delegating=*/false, addr, type);
2046
19.6k
}
2047
2048
void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D,
2049
                                             CXXCtorType Type,
2050
                                             bool ForVirtualBase,
2051
                                             bool Delegating,
2052
                                             AggValueSlot ThisAVS,
2053
44.9k
                                             const CXXConstructExpr *E) {
2054
44.9k
  CallArgList Args;
2055
44.9k
  Address This = ThisAVS.getAddress();
2056
44.9k
  LangAS SlotAS = ThisAVS.getQualifiers().getAddressSpace();
2057
44.9k
  QualType ThisType = D->getThisType();
2058
44.9k
  LangAS ThisAS = ThisType.getTypePtr()->getPointeeType().getAddressSpace();
2059
44.9k
  llvm::Value *ThisPtr = This.getPointer();
2060
2061
44.9k
  if (SlotAS != ThisAS) {
2062
29
    unsigned TargetThisAS = getContext().getTargetAddressSpace(ThisAS);
2063
29
    llvm::Type *NewType = llvm::PointerType::getWithSamePointeeType(
2064
29
        This.getType(), TargetThisAS);
2065
29
    ThisPtr = getTargetHooks().performAddrSpaceCast(*this, This.getPointer(),
2066
29
                                                    ThisAS, SlotAS, NewType);
2067
29
  }
2068
2069
  // Push the this ptr.
2070
44.9k
  Args.add(RValue::get(ThisPtr), D->getThisType());
2071
2072
  // If this is a trivial constructor, emit a memcpy now before we lose
2073
  // the alignment information on the argument.
2074
  // FIXME: It would be better to preserve alignment information into CallArg.
2075
44.9k
  if (isMemcpyEquivalentSpecialMember(D)) {
2076
6.05k
    assert(E->getNumArgs() == 1 && "unexpected argcount for trivial ctor");
2077
2078
0
    const Expr *Arg = E->getArg(0);
2079
6.05k
    LValue Src = EmitLValue(Arg);
2080
6.05k
    QualType DestTy = getContext().getTypeDeclType(D->getParent());
2081
6.05k
    LValue Dest = MakeAddrLValue(This, DestTy);
2082
6.05k
    EmitAggregateCopyCtor(Dest, Src, ThisAVS.mayOverlap());
2083
6.05k
    return;
2084
6.05k
  }
2085
2086
  // Add the rest of the user-supplied arguments.
2087
38.8k
  const FunctionProtoType *FPT = D->getType()->castAs<FunctionProtoType>();
2088
38.8k
  EvaluationOrder Order = E->isListInitialization()
2089
38.8k
                              ? 
EvaluationOrder::ForceLeftToRight459
2090
38.8k
                              : 
EvaluationOrder::Default38.4k
;
2091
38.8k
  EmitCallArgs(Args, FPT, E->arguments(), E->getConstructor(),
2092
38.8k
               /*ParamsToSkip*/ 0, Order);
2093
2094
38.8k
  EmitCXXConstructorCall(D, Type, ForVirtualBase, Delegating, This, Args,
2095
38.8k
                         ThisAVS.mayOverlap(), E->getExprLoc(),
2096
38.8k
                         ThisAVS.isSanitizerChecked());
2097
38.8k
}
2098
2099
static bool canEmitDelegateCallArgs(CodeGenFunction &CGF,
2100
                                    const CXXConstructorDecl *Ctor,
2101
226
                                    CXXCtorType Type, CallArgList &Args) {
2102
  // We can't forward a variadic call.
2103
226
  if (Ctor->isVariadic())
2104
31
    return false;
2105
2106
195
  if (CGF.getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) {
2107
    // If the parameters are callee-cleanup, it's not safe to forward.
2108
32
    for (auto *P : Ctor->parameters())
2109
56
      if (P->needsDestruction(CGF.getContext()))
2110
16
        return false;
2111
2112
    // Likewise if they're inalloca.
2113
16
    const CGFunctionInfo &Info =
2114
16
        CGF.CGM.getTypes().arrangeCXXConstructorCall(Args, Ctor, Type, 0, 0);
2115
16
    if (Info.usesInAlloca())
2116
0
      return false;
2117
16
  }
2118
2119
  // Anything else should be OK.
2120
179
  return true;
2121
195
}
2122
2123
void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D,
2124
                                             CXXCtorType Type,
2125
                                             bool ForVirtualBase,
2126
                                             bool Delegating,
2127
                                             Address This,
2128
                                             CallArgList &Args,
2129
                                             AggValueSlot::Overlap_t Overlap,
2130
                                             SourceLocation Loc,
2131
55.5k
                                             bool NewPointerIsChecked) {
2132
55.5k
  const CXXRecordDecl *ClassDecl = D->getParent();
2133
2134
55.5k
  if (!NewPointerIsChecked)
2135
33.7k
    EmitTypeCheck(CodeGenFunction::TCK_ConstructorCall, Loc, This.getPointer(),
2136
33.7k
                  getContext().getRecordType(ClassDecl), CharUnits::Zero());
2137
2138
55.5k
  if (D->isTrivial() && 
D->isDefaultConstructor()54
) {
2139
33
    assert(Args.size() == 1 && "trivial default ctor with args");
2140
0
    return;
2141
33
  }
2142
2143
  // If this is a trivial constructor, just emit what's needed. If this is a
2144
  // union copy constructor, we must emit a memcpy, because the AST does not
2145
  // model that copy.
2146
55.4k
  if (isMemcpyEquivalentSpecialMember(D)) {
2147
18
    assert(Args.size() == 2 && "unexpected argcount for trivial ctor");
2148
2149
0
    QualType SrcTy = D->getParamDecl(0)->getType().getNonReferenceType();
2150
18
    Address Src(Args[1].getRValue(*this).getScalarVal(),
2151
18
                CGM.getNaturalTypeAlignment(SrcTy));
2152
18
    LValue SrcLVal = MakeAddrLValue(Src, SrcTy);
2153
18
    QualType DestTy = getContext().getTypeDeclType(ClassDecl);
2154
18
    LValue DestLVal = MakeAddrLValue(This, DestTy);
2155
18
    EmitAggregateCopyCtor(DestLVal, SrcLVal, Overlap);
2156
18
    return;
2157
18
  }
2158
2159
55.4k
  bool PassPrototypeArgs = true;
2160
  // Check whether we can actually emit the constructor before trying to do so.
2161
55.4k
  if (auto Inherited = D->getInheritedConstructor()) {
2162
236
    PassPrototypeArgs = getTypes().inheritingCtorHasParams(Inherited, Type);
2163
236
    if (PassPrototypeArgs && 
!canEmitDelegateCallArgs(*this, D, Type, Args)226
) {
2164
47
      EmitInlinedInheritingCXXConstructorCall(D, Type, ForVirtualBase,
2165
47
                                              Delegating, Args);
2166
47
      return;
2167
47
    }
2168
236
  }
2169
2170
  // Insert any ABI-specific implicit constructor arguments.
2171
55.4k
  CGCXXABI::AddedStructorArgCounts ExtraArgs =
2172
55.4k
      CGM.getCXXABI().addImplicitConstructorArgs(*this, D, Type, ForVirtualBase,
2173
55.4k
                                                 Delegating, Args);
2174
2175
  // Emit the call.
2176
55.4k
  llvm::Constant *CalleePtr = CGM.getAddrOfCXXStructor(GlobalDecl(D, Type));
2177
55.4k
  const CGFunctionInfo &Info = CGM.getTypes().arrangeCXXConstructorCall(
2178
55.4k
      Args, D, Type, ExtraArgs.Prefix, ExtraArgs.Suffix, PassPrototypeArgs);
2179
55.4k
  CGCallee Callee = CGCallee::forDirect(CalleePtr, GlobalDecl(D, Type));
2180
55.4k
  EmitCall(Info, Callee, ReturnValueSlot(), Args, nullptr, false, Loc);
2181
2182
  // Generate vtable assumptions if we're constructing a complete object
2183
  // with a vtable.  We don't do this for base subobjects for two reasons:
2184
  // first, it's incorrect for classes with virtual bases, and second, we're
2185
  // about to overwrite the vptrs anyway.
2186
  // We also have to make sure if we can refer to vtable:
2187
  // - Otherwise we can refer to vtable if it's safe to speculatively emit.
2188
  // FIXME: If vtable is used by ctor/dtor, or if vtable is external and we are
2189
  // sure that definition of vtable is not hidden,
2190
  // then we are always safe to refer to it.
2191
  // FIXME: It looks like InstCombine is very inefficient on dealing with
2192
  // assumes. Make assumption loads require -fstrict-vtable-pointers temporarily.
2193
55.4k
  if (CGM.getCodeGenOpts().OptimizationLevel > 0 &&
2194
55.4k
      
ClassDecl->isDynamicClass()1.24k
&&
Type != Ctor_Base628
&&
2195
55.4k
      
CGM.getCXXABI().canSpeculativelyEmitVTable(ClassDecl)355
&&
2196
55.4k
      
CGM.getCodeGenOpts().StrictVTablePointers126
)
2197
56
    EmitVTableAssumptionLoads(ClassDecl, This);
2198
55.4k
}
2199
2200
void CodeGenFunction::EmitInheritedCXXConstructorCall(
2201
    const CXXConstructorDecl *D, bool ForVirtualBase, Address This,
2202
200
    bool InheritedFromVBase, const CXXInheritedCtorInitExpr *E) {
2203
200
  CallArgList Args;
2204
200
  CallArg ThisArg(RValue::get(This.getPointer()), D->getThisType());
2205
2206
  // Forward the parameters.
2207
200
  if (InheritedFromVBase &&
2208
200
      
CGM.getTarget().getCXXABI().hasConstructorVariants()17
) {
2209
    // Nothing to do; this construction is not responsible for constructing
2210
    // the base class containing the inherited constructor.
2211
    // FIXME: Can we just pass undef's for the remaining arguments if we don't
2212
    // have constructor variants?
2213
10
    Args.push_back(ThisArg);
2214
190
  } else if (!CXXInheritedCtorInitExprArgs.empty()) {
2215
    // The inheriting constructor was inlined; just inject its arguments.
2216
47
    assert(CXXInheritedCtorInitExprArgs.size() >= D->getNumParams() &&
2217
47
           "wrong number of parameters for inherited constructor call");
2218
0
    Args = CXXInheritedCtorInitExprArgs;
2219
47
    Args[0] = ThisArg;
2220
143
  } else {
2221
    // The inheriting constructor was not inlined. Emit delegating arguments.
2222
143
    Args.push_back(ThisArg);
2223
143
    const auto *OuterCtor = cast<CXXConstructorDecl>(CurCodeDecl);
2224
143
    assert(OuterCtor->getNumParams() == D->getNumParams());
2225
0
    assert(!OuterCtor->isVariadic() && "should have been inlined");
2226
2227
269
    for (const auto *Param : OuterCtor->parameters()) {
2228
269
      assert(getContext().hasSameUnqualifiedType(
2229
269
          OuterCtor->getParamDecl(Param->getFunctionScopeIndex())->getType(),
2230
269
          Param->getType()));
2231
0
      EmitDelegateCallArg(Args, Param, E->getLocation());
2232
2233
      // Forward __attribute__(pass_object_size).
2234
269
      if (Param->hasAttr<PassObjectSizeAttr>()) {
2235
22
        auto *POSParam = SizeArguments[Param];
2236
22
        assert(POSParam && "missing pass_object_size value for forwarding");
2237
0
        EmitDelegateCallArg(Args, POSParam, E->getLocation());
2238
22
      }
2239
269
    }
2240
143
  }
2241
2242
0
  EmitCXXConstructorCall(D, Ctor_Base, ForVirtualBase, /*Delegating*/false,
2243
200
                         This, Args, AggValueSlot::MayOverlap,
2244
200
                         E->getLocation(), /*NewPointerIsChecked*/true);
2245
200
}
2246
2247
void CodeGenFunction::EmitInlinedInheritingCXXConstructorCall(
2248
    const CXXConstructorDecl *Ctor, CXXCtorType CtorType, bool ForVirtualBase,
2249
47
    bool Delegating, CallArgList &Args) {
2250
47
  GlobalDecl GD(Ctor, CtorType);
2251
47
  InlinedInheritingConstructorScope Scope(*this, GD);
2252
47
  ApplyInlineDebugLocation DebugScope(*this, GD);
2253
47
  RunCleanupsScope RunCleanups(*this);
2254
2255
  // Save the arguments to be passed to the inherited constructor.
2256
47
  CXXInheritedCtorInitExprArgs = Args;
2257
2258
47
  FunctionArgList Params;
2259
47
  QualType RetType = BuildFunctionArgList(CurGD, Params);
2260
47
  FnRetTy = RetType;
2261
2262
  // Insert any ABI-specific implicit constructor arguments.
2263
47
  CGM.getCXXABI().addImplicitConstructorArgs(*this, Ctor, CtorType,
2264
47
                                             ForVirtualBase, Delegating, Args);
2265
2266
  // Emit a simplified prolog. We only need to emit the implicit params.
2267
47
  assert(Args.size() >= Params.size() && "too few arguments for call");
2268
332
  for (unsigned I = 0, N = Args.size(); I != N; 
++I285
) {
2269
285
    if (I < Params.size() && 
isa<ImplicitParamDecl>(Params[I])231
) {
2270
61
      const RValue &RV = Args[I].getRValue(*this);
2271
61
      assert(!RV.isComplex() && "complex indirect params not supported");
2272
61
      ParamValue Val = RV.isScalar()
2273
61
                           ? ParamValue::forDirect(RV.getScalarVal())
2274
61
                           : 
ParamValue::forIndirect(RV.getAggregateAddress())0
;
2275
61
      EmitParmDecl(*Params[I], Val, I + 1);
2276
61
    }
2277
285
  }
2278
2279
  // Create a return value slot if the ABI implementation wants one.
2280
  // FIXME: This is dumb, we should ask the ABI not to try to set the return
2281
  // value instead.
2282
47
  if (!RetType->isVoidType())
2283
28
    ReturnValue = CreateIRTemp(RetType, "retval.inhctor");
2284
2285
47
  CGM.getCXXABI().EmitInstanceFunctionProlog(*this);
2286
47
  CXXThisValue = CXXABIThisValue;
2287
2288
  // Directly emit the constructor initializers.
2289
47
  EmitCtorPrologue(Ctor, CtorType, Params);
2290
47
}
2291
2292
62
void CodeGenFunction::EmitVTableAssumptionLoad(const VPtr &Vptr, Address This) {
2293
62
  llvm::Value *VTableGlobal =
2294
62
      CGM.getCXXABI().getVTableAddressPoint(Vptr.Base, Vptr.VTableClass);
2295
62
  if (!VTableGlobal)
2296
0
    return;
2297
2298
  // We can just use the base offset in the complete class.
2299
62
  CharUnits NonVirtualOffset = Vptr.Base.getBaseOffset();
2300
2301
62
  if (!NonVirtualOffset.isZero())
2302
4
    This =
2303
4
        ApplyNonVirtualAndVirtualOffset(*this, This, NonVirtualOffset, nullptr,
2304
4
                                        Vptr.VTableClass, Vptr.NearestVBase);
2305
2306
62
  llvm::Value *VPtrValue =
2307
62
      GetVTablePtr(This, VTableGlobal->getType(), Vptr.VTableClass);
2308
62
  llvm::Value *Cmp =
2309
62
      Builder.CreateICmpEQ(VPtrValue, VTableGlobal, "cmp.vtables");
2310
62
  Builder.CreateAssumption(Cmp);
2311
62
}
2312
2313
void CodeGenFunction::EmitVTableAssumptionLoads(const CXXRecordDecl *ClassDecl,
2314
56
                                                Address This) {
2315
56
  if (CGM.getCXXABI().doStructorsInitializeVPtrs(ClassDecl))
2316
56
    for (const VPtr &Vptr : getVTablePointers(ClassDecl))
2317
62
      EmitVTableAssumptionLoad(Vptr, This);
2318
56
}
2319
2320
void
2321
CodeGenFunction::EmitSynthesizedCXXCopyCtorCall(const CXXConstructorDecl *D,
2322
                                                Address This, Address Src,
2323
62
                                                const CXXConstructExpr *E) {
2324
62
  const FunctionProtoType *FPT = D->getType()->castAs<FunctionProtoType>();
2325
2326
62
  CallArgList Args;
2327
2328
  // Push the this ptr.
2329
62
  Args.add(RValue::get(This.getPointer()), D->getThisType());
2330
2331
  // Push the src ptr.
2332
62
  QualType QT = *(FPT->param_type_begin());
2333
62
  llvm::Type *t = CGM.getTypes().ConvertType(QT);
2334
62
  Src = Builder.CreateBitCast(Src, t);
2335
62
  Args.add(RValue::get(Src.getPointer()), QT);
2336
2337
  // Skip over first argument (Src).
2338
62
  EmitCallArgs(Args, FPT, drop_begin(E->arguments(), 1), E->getConstructor(),
2339
62
               /*ParamsToSkip*/ 1);
2340
2341
62
  EmitCXXConstructorCall(D, Ctor_Complete, /*ForVirtualBase*/false,
2342
62
                         /*Delegating*/false, This, Args,
2343
62
                         AggValueSlot::MayOverlap, E->getExprLoc(),
2344
62
                         /*NewPointerIsChecked*/false);
2345
62
}
2346
2347
void
2348
CodeGenFunction::EmitDelegateCXXConstructorCall(const CXXConstructorDecl *Ctor,
2349
                                                CXXCtorType CtorType,
2350
                                                const FunctionArgList &Args,
2351
16.3k
                                                SourceLocation Loc) {
2352
16.3k
  CallArgList DelegateArgs;
2353
2354
16.3k
  FunctionArgList::const_iterator I = Args.begin(), E = Args.end();
2355
16.3k
  assert(I != E && "no parameters to constructor");
2356
2357
  // this
2358
0
  Address This = LoadCXXThisAddress();
2359
16.3k
  DelegateArgs.add(RValue::get(This.getPointer()), (*I)->getType());
2360
16.3k
  ++I;
2361
2362
  // FIXME: The location of the VTT parameter in the parameter list is
2363
  // specific to the Itanium ABI and shouldn't be hardcoded here.
2364
16.3k
  if (CGM.getCXXABI().NeedsVTTParameter(CurGD)) {
2365
0
    assert(I != E && "cannot skip vtt parameter, already done with args");
2366
0
    assert((*I)->getType()->isPointerType() &&
2367
0
           "skipping parameter not of vtt type");
2368
0
    ++I;
2369
0
  }
2370
2371
  // Explicit arguments.
2372
34.5k
  for (; I != E; 
++I18.2k
) {
2373
18.2k
    const VarDecl *param = *I;
2374
    // FIXME: per-argument source location
2375
18.2k
    EmitDelegateCallArg(DelegateArgs, param, Loc);
2376
18.2k
  }
2377
2378
16.3k
  EmitCXXConstructorCall(Ctor, CtorType, /*ForVirtualBase=*/false,
2379
16.3k
                         /*Delegating=*/true, This, DelegateArgs,
2380
16.3k
                         AggValueSlot::MayOverlap, Loc,
2381
16.3k
                         /*NewPointerIsChecked=*/true);
2382
16.3k
}
2383
2384
namespace {
2385
  struct CallDelegatingCtorDtor final : EHScopeStack::Cleanup {
2386
    const CXXDestructorDecl *Dtor;
2387
    Address Addr;
2388
    CXXDtorType Type;
2389
2390
    CallDelegatingCtorDtor(const CXXDestructorDecl *D, Address Addr,
2391
                           CXXDtorType Type)
2392
68
      : Dtor(D), Addr(Addr), Type(Type) {}
2393
2394
4
    void Emit(CodeGenFunction &CGF, Flags flags) override {
2395
      // We are calling the destructor from within the constructor.
2396
      // Therefore, "this" should have the expected type.
2397
4
      QualType ThisTy = Dtor->getThisObjectType();
2398
4
      CGF.EmitCXXDestructorCall(Dtor, Type, /*ForVirtualBase=*/false,
2399
4
                                /*Delegating=*/true, Addr, ThisTy);
2400
4
    }
2401
  };
2402
} // end anonymous namespace
2403
2404
void
2405
CodeGenFunction::EmitDelegatingCXXConstructorCall(const CXXConstructorDecl *Ctor,
2406
94
                                                  const FunctionArgList &Args) {
2407
94
  assert(Ctor->isDelegatingConstructor());
2408
2409
0
  Address ThisPtr = LoadCXXThisAddress();
2410
2411
94
  AggValueSlot AggSlot =
2412
94
    AggValueSlot::forAddr(ThisPtr, Qualifiers(),
2413
94
                          AggValueSlot::IsDestructed,
2414
94
                          AggValueSlot::DoesNotNeedGCBarriers,
2415
94
                          AggValueSlot::IsNotAliased,
2416
94
                          AggValueSlot::MayOverlap,
2417
94
                          AggValueSlot::IsNotZeroed,
2418
                          // Checks are made by the code that calls constructor.
2419
94
                          AggValueSlot::IsSanitizerChecked);
2420
2421
94
  EmitAggExpr(Ctor->init_begin()[0]->getInit(), AggSlot);
2422
2423
94
  const CXXRecordDecl *ClassDecl = Ctor->getParent();
2424
94
  if (CGM.getLangOpts().Exceptions && 
!ClassDecl->hasTrivialDestructor()92
) {
2425
68
    CXXDtorType Type =
2426
68
      CurGD.getCtorType() == Ctor_Complete ? 
Dtor_Complete65
:
Dtor_Base3
;
2427
2428
68
    EHStack.pushCleanup<CallDelegatingCtorDtor>(EHCleanup,
2429
68
                                                ClassDecl->getDestructor(),
2430
68
                                                ThisPtr, Type);
2431
68
  }
2432
94
}
2433
2434
void CodeGenFunction::EmitCXXDestructorCall(const CXXDestructorDecl *DD,
2435
                                            CXXDtorType Type,
2436
                                            bool ForVirtualBase,
2437
                                            bool Delegating, Address This,
2438
30.4k
                                            QualType ThisTy) {
2439
30.4k
  CGM.getCXXABI().EmitDestructorCall(*this, DD, Type, ForVirtualBase,
2440
30.4k
                                     Delegating, This, ThisTy);
2441
30.4k
}
2442
2443
namespace {
2444
  struct CallLocalDtor final : EHScopeStack::Cleanup {
2445
    const CXXDestructorDecl *Dtor;
2446
    Address Addr;
2447
    QualType Ty;
2448
2449
    CallLocalDtor(const CXXDestructorDecl *D, Address Addr, QualType Ty)
2450
10
        : Dtor(D), Addr(Addr), Ty(Ty) {}
2451
2452
10
    void Emit(CodeGenFunction &CGF, Flags flags) override {
2453
10
      CGF.EmitCXXDestructorCall(Dtor, Dtor_Complete,
2454
10
                                /*ForVirtualBase=*/false,
2455
10
                                /*Delegating=*/false, Addr, Ty);
2456
10
    }
2457
  };
2458
} // end anonymous namespace
2459
2460
void CodeGenFunction::PushDestructorCleanup(const CXXDestructorDecl *D,
2461
10
                                            QualType T, Address Addr) {
2462
10
  EHStack.pushCleanup<CallLocalDtor>(NormalAndEHCleanup, D, Addr, T);
2463
10
}
2464
2465
13
void CodeGenFunction::PushDestructorCleanup(QualType T, Address Addr) {
2466
13
  CXXRecordDecl *ClassDecl = T->getAsCXXRecordDecl();
2467
13
  if (!ClassDecl) 
return0
;
2468
13
  if (ClassDecl->hasTrivialDestructor()) 
return3
;
2469
2470
10
  const CXXDestructorDecl *D = ClassDecl->getDestructor();
2471
10
  assert(D && D->isUsed() && "destructor not marked as used!");
2472
0
  PushDestructorCleanup(D, T, Addr);
2473
10
}
2474
2475
3.73k
void CodeGenFunction::InitializeVTablePointer(const VPtr &Vptr) {
2476
  // Compute the address point.
2477
3.73k
  llvm::Value *VTableAddressPoint =
2478
3.73k
      CGM.getCXXABI().getVTableAddressPointInStructor(
2479
3.73k
          *this, Vptr.VTableClass, Vptr.Base, Vptr.NearestVBase);
2480
2481
3.73k
  if (!VTableAddressPoint)
2482
719
    return;
2483
2484
  // Compute where to store the address point.
2485
3.02k
  llvm::Value *VirtualOffset = nullptr;
2486
3.02k
  CharUnits NonVirtualOffset = CharUnits::Zero();
2487
2488
3.02k
  if (CGM.getCXXABI().isVirtualOffsetNeededForVTableField(*this, Vptr)) {
2489
    // We need to use the virtual base offset offset because the virtual base
2490
    // might have a different offset in the most derived class.
2491
2492
353
    VirtualOffset = CGM.getCXXABI().GetVirtualBaseClassOffset(
2493
353
        *this, LoadCXXThisAddress(), Vptr.VTableClass, Vptr.NearestVBase);
2494
353
    NonVirtualOffset = Vptr.OffsetFromNearestVBase;
2495
2.66k
  } else {
2496
    // We can just use the base offset in the complete class.
2497
2.66k
    NonVirtualOffset = Vptr.Base.getBaseOffset();
2498
2.66k
  }
2499
2500
  // Apply the offsets.
2501
3.02k
  Address VTableField = LoadCXXThisAddress();
2502
3.02k
  if (!NonVirtualOffset.isZero() || 
VirtualOffset2.78k
)
2503
556
    VTableField = ApplyNonVirtualAndVirtualOffset(
2504
556
        *this, VTableField, NonVirtualOffset, VirtualOffset, Vptr.VTableClass,
2505
556
        Vptr.NearestVBase);
2506
2507
  // Finally, store the address point. Use the same LLVM types as the field to
2508
  // support optimization.
2509
3.02k
  unsigned GlobalsAS = CGM.getDataLayout().getDefaultGlobalsAddressSpace();
2510
3.02k
  unsigned ProgAS = CGM.getDataLayout().getProgramAddressSpace();
2511
3.02k
  llvm::Type *VTablePtrTy =
2512
3.02k
      llvm::FunctionType::get(CGM.Int32Ty, /*isVarArg=*/true)
2513
3.02k
          ->getPointerTo(ProgAS)
2514
3.02k
          ->getPointerTo(GlobalsAS);
2515
  // vtable field is is derived from `this` pointer, therefore they should be in
2516
  // the same addr space. Note that this might not be LLVM address space 0.
2517
3.02k
  VTableField = Builder.CreateElementBitCast(VTableField, VTablePtrTy);
2518
3.02k
  VTableAddressPoint = Builder.CreateBitCast(VTableAddressPoint, VTablePtrTy);
2519
2520
3.02k
  llvm::StoreInst *Store = Builder.CreateStore(VTableAddressPoint, VTableField);
2521
3.02k
  TBAAAccessInfo TBAAInfo = CGM.getTBAAVTablePtrAccessInfo(VTablePtrTy);
2522
3.02k
  CGM.DecorateInstructionWithTBAA(Store, TBAAInfo);
2523
3.02k
  if (CGM.getCodeGenOpts().OptimizationLevel > 0 &&
2524
3.02k
      
CGM.getCodeGenOpts().StrictVTablePointers423
)
2525
54
    CGM.DecorateInstructionWithInvariantGroup(Store, Vptr.VTableClass);
2526
3.02k
}
2527
2528
CodeGenFunction::VPtrsVector
2529
2.79k
CodeGenFunction::getVTablePointers(const CXXRecordDecl *VTableClass) {
2530
2.79k
  CodeGenFunction::VPtrsVector VPtrsResult;
2531
2.79k
  VisitedVirtualBasesSetTy VBases;
2532
2.79k
  getVTablePointers(BaseSubobject(VTableClass, CharUnits::Zero()),
2533
2.79k
                    /*NearestVBase=*/nullptr,
2534
2.79k
                    /*OffsetFromNearestVBase=*/CharUnits::Zero(),
2535
2.79k
                    /*BaseIsNonVirtualPrimaryBase=*/false, VTableClass, VBases,
2536
2.79k
                    VPtrsResult);
2537
2.79k
  return VPtrsResult;
2538
2.79k
}
2539
2540
void CodeGenFunction::getVTablePointers(BaseSubobject Base,
2541
                                        const CXXRecordDecl *NearestVBase,
2542
                                        CharUnits OffsetFromNearestVBase,
2543
                                        bool BaseIsNonVirtualPrimaryBase,
2544
                                        const CXXRecordDecl *VTableClass,
2545
                                        VisitedVirtualBasesSetTy &VBases,
2546
5.08k
                                        VPtrsVector &Vptrs) {
2547
  // If this base is a non-virtual primary base the address point has already
2548
  // been set.
2549
5.08k
  if (!BaseIsNonVirtualPrimaryBase) {
2550
    // Initialize the vtable pointer for this base.
2551
3.80k
    VPtr Vptr = {Base, NearestVBase, OffsetFromNearestVBase, VTableClass};
2552
3.80k
    Vptrs.push_back(Vptr);
2553
3.80k
  }
2554
2555
5.08k
  const CXXRecordDecl *RD = Base.getBase();
2556
2557
  // Traverse bases.
2558
5.08k
  for (const auto &I : RD->bases()) {
2559
3.05k
    auto *BaseDecl =
2560
3.05k
        cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
2561
2562
    // Ignore classes without a vtable.
2563
3.05k
    if (!BaseDecl->isDynamicClass())
2564
691
      continue;
2565
2566
2.35k
    CharUnits BaseOffset;
2567
2.35k
    CharUnits BaseOffsetFromNearestVBase;
2568
2.35k
    bool BaseDeclIsNonVirtualPrimaryBase;
2569
2570
2.35k
    if (I.isVirtual()) {
2571
      // Check if we've visited this virtual base before.
2572
673
      if (!VBases.insert(BaseDecl).second)
2573
71
        continue;
2574
2575
602
      const ASTRecordLayout &Layout =
2576
602
        getContext().getASTRecordLayout(VTableClass);
2577
2578
602
      BaseOffset = Layout.getVBaseClassOffset(BaseDecl);
2579
602
      BaseOffsetFromNearestVBase = CharUnits::Zero();
2580
602
      BaseDeclIsNonVirtualPrimaryBase = false;
2581
1.68k
    } else {
2582
1.68k
      const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
2583
2584
1.68k
      BaseOffset = Base.getBaseOffset() + Layout.getBaseClassOffset(BaseDecl);
2585
1.68k
      BaseOffsetFromNearestVBase =
2586
1.68k
        OffsetFromNearestVBase + Layout.getBaseClassOffset(BaseDecl);
2587
1.68k
      BaseDeclIsNonVirtualPrimaryBase = Layout.getPrimaryBase() == BaseDecl;
2588
1.68k
    }
2589
2590
2.28k
    getVTablePointers(
2591
2.28k
        BaseSubobject(BaseDecl, BaseOffset),
2592
2.28k
        I.isVirtual() ? 
BaseDecl602
:
NearestVBase1.68k
, BaseOffsetFromNearestVBase,
2593
2.28k
        BaseDeclIsNonVirtualPrimaryBase, VTableClass, VBases, Vptrs);
2594
2.28k
  }
2595
5.08k
}
2596
2597
24.8k
void CodeGenFunction::InitializeVTablePointers(const CXXRecordDecl *RD) {
2598
  // Ignore classes without a vtable.
2599
24.8k
  if (!RD->isDynamicClass())
2600
22.1k
    return;
2601
2602
  // Initialize the vtable pointers for this class and all of its bases.
2603
2.74k
  if (CGM.getCXXABI().doStructorsInitializeVPtrs(RD))
2604
2.73k
    for (const VPtr &Vptr : getVTablePointers(RD))
2605
3.73k
      InitializeVTablePointer(Vptr);
2606
2607
2.74k
  if (RD->getNumVBases())
2608
811
    CGM.getCXXABI().initializeHiddenVirtualInheritanceMembers(*this, RD);
2609
2.74k
}
2610
2611
llvm::Value *CodeGenFunction::GetVTablePtr(Address This,
2612
                                           llvm::Type *VTableTy,
2613
1.78k
                                           const CXXRecordDecl *RD) {
2614
1.78k
  Address VTablePtrSrc = Builder.CreateElementBitCast(This, VTableTy);
2615
1.78k
  llvm::Instruction *VTable = Builder.CreateLoad(VTablePtrSrc, "vtable");
2616
1.78k
  TBAAAccessInfo TBAAInfo = CGM.getTBAAVTablePtrAccessInfo(VTableTy);
2617
1.78k
  CGM.DecorateInstructionWithTBAA(VTable, TBAAInfo);
2618
2619
1.78k
  if (CGM.getCodeGenOpts().OptimizationLevel > 0 &&
2620
1.78k
      
CGM.getCodeGenOpts().StrictVTablePointers207
)
2621
118
    CGM.DecorateInstructionWithInvariantGroup(VTable, RD);
2622
2623
1.78k
  return VTable;
2624
1.78k
}
2625
2626
// If a class has a single non-virtual base and does not introduce or override
2627
// virtual member functions or fields, it will have the same layout as its base.
2628
// This function returns the least derived such class.
2629
//
2630
// Casting an instance of a base class to such a derived class is technically
2631
// undefined behavior, but it is a relatively common hack for introducing member
2632
// functions on class instances with specific properties (e.g. llvm::Operator)
2633
// that works under most compilers and should not have security implications, so
2634
// we allow it by default. It can be disabled with -fsanitize=cfi-cast-strict.
2635
static const CXXRecordDecl *
2636
60
LeastDerivedClassWithSameLayout(const CXXRecordDecl *RD) {
2637
60
  if (!RD->field_empty())
2638
1
    return RD;
2639
2640
59
  if (RD->getNumVBases() != 0)
2641
12
    return RD;
2642
2643
47
  if (RD->getNumBases() != 1)
2644
32
    return RD;
2645
2646
28
  
for (const CXXMethodDecl *MD : RD->methods())15
{
2647
28
    if (MD->isVirtual()) {
2648
      // Virtual member functions are only ok if they are implicit destructors
2649
      // because the implicit destructor will have the same semantics as the
2650
      // base class's destructor if no fields are added.
2651
9
      if (isa<CXXDestructorDecl>(MD) && 
MD->isImplicit()0
)
2652
0
        continue;
2653
9
      return RD;
2654
9
    }
2655
28
  }
2656
2657
6
  return LeastDerivedClassWithSameLayout(
2658
6
      RD->bases_begin()->getType()->getAsCXXRecordDecl());
2659
15
}
2660
2661
void CodeGenFunction::EmitTypeMetadataCodeForVCall(const CXXRecordDecl *RD,
2662
                                                   llvm::Value *VTable,
2663
861
                                                   SourceLocation Loc) {
2664
861
  if (SanOpts.has(SanitizerKind::CFIVCall))
2665
33
    EmitVTablePtrCheckForCall(RD, VTable, CodeGenFunction::CFITCK_VCall, Loc);
2666
828
  else if (CGM.getCodeGenOpts().WholeProgramVTables &&
2667
           // Don't insert type test assumes if we are forcing public std
2668
           // visibility.
2669
828
           
!CGM.HasLTOVisibilityPublicStd(RD)73
) {
2670
70
    llvm::Metadata *MD =
2671
70
        CGM.CreateMetadataIdentifierForType(QualType(RD->getTypeForDecl(), 0));
2672
70
    llvm::Value *TypeId =
2673
70
        llvm::MetadataAsValue::get(CGM.getLLVMContext(), MD);
2674
2675
70
    llvm::Value *CastedVTable = Builder.CreateBitCast(VTable, Int8PtrTy);
2676
70
    llvm::Value *TypeTest =
2677
70
        Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::type_test),
2678
70
                           {CastedVTable, TypeId});
2679
70
    Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::assume), TypeTest);
2680
70
  }
2681
861
}
2682
2683
void CodeGenFunction::EmitVTablePtrCheckForCall(const CXXRecordDecl *RD,
2684
                                                llvm::Value *VTable,
2685
                                                CFITypeCheckKind TCK,
2686
41
                                                SourceLocation Loc) {
2687
41
  if (!SanOpts.has(SanitizerKind::CFICastStrict))
2688
39
    RD = LeastDerivedClassWithSameLayout(RD);
2689
2690
41
  EmitVTablePtrCheck(RD, VTable, TCK, Loc);
2691
41
}
2692
2693
void CodeGenFunction::EmitVTablePtrCheckForCast(QualType T,
2694
                                                llvm::Value *Derived,
2695
                                                bool MayBeNull,
2696
                                                CFITypeCheckKind TCK,
2697
21
                                                SourceLocation Loc) {
2698
21
  if (!getLangOpts().CPlusPlus)
2699
0
    return;
2700
2701
21
  auto *ClassTy = T->getAs<RecordType>();
2702
21
  if (!ClassTy)
2703
0
    return;
2704
2705
21
  const CXXRecordDecl *ClassDecl = cast<CXXRecordDecl>(ClassTy->getDecl());
2706
2707
21
  if (!ClassDecl->isCompleteDefinition() || !ClassDecl->isDynamicClass())
2708
0
    return;
2709
2710
21
  if (!SanOpts.has(SanitizerKind::CFICastStrict))
2711
15
    ClassDecl = LeastDerivedClassWithSameLayout(ClassDecl);
2712
2713
21
  llvm::BasicBlock *ContBlock = nullptr;
2714
2715
21
  if (MayBeNull) {
2716
14
    llvm::Value *DerivedNotNull =
2717
14
        Builder.CreateIsNotNull(Derived, "cast.nonnull");
2718
2719
14
    llvm::BasicBlock *CheckBlock = createBasicBlock("cast.check");
2720
14
    ContBlock = createBasicBlock("cast.cont");
2721
2722
14
    Builder.CreateCondBr(DerivedNotNull, CheckBlock, ContBlock);
2723
2724
14
    EmitBlock(CheckBlock);
2725
14
  }
2726
2727
21
  llvm::Value *VTable;
2728
21
  std::tie(VTable, ClassDecl) = CGM.getCXXABI().LoadVTablePtr(
2729
21
      *this, Address(Derived, getPointerAlign()), ClassDecl);
2730
2731
21
  EmitVTablePtrCheck(ClassDecl, VTable, TCK, Loc);
2732
2733
21
  if (MayBeNull) {
2734
14
    Builder.CreateBr(ContBlock);
2735
14
    EmitBlock(ContBlock);
2736
14
  }
2737
21
}
2738
2739
void CodeGenFunction::EmitVTablePtrCheck(const CXXRecordDecl *RD,
2740
                                         llvm::Value *VTable,
2741
                                         CFITypeCheckKind TCK,
2742
62
                                         SourceLocation Loc) {
2743
62
  if (!CGM.getCodeGenOpts().SanitizeCfiCrossDso &&
2744
62
      
!CGM.HasHiddenLTOVisibility(RD)60
)
2745
0
    return;
2746
2747
62
  SanitizerMask M;
2748
62
  llvm::SanitizerStatKind SSK;
2749
62
  switch (TCK) {
2750
33
  case CFITCK_VCall:
2751
33
    M = SanitizerKind::CFIVCall;
2752
33
    SSK = llvm::SanStat_CFI_VCall;
2753
33
    break;
2754
8
  case CFITCK_NVCall:
2755
8
    M = SanitizerKind::CFINVCall;
2756
8
    SSK = llvm::SanStat_CFI_NVCall;
2757
8
    break;
2758
6
  case CFITCK_DerivedCast:
2759
6
    M = SanitizerKind::CFIDerivedCast;
2760
6
    SSK = llvm::SanStat_CFI_DerivedCast;
2761
6
    break;
2762
15
  case CFITCK_UnrelatedCast:
2763
15
    M = SanitizerKind::CFIUnrelatedCast;
2764
15
    SSK = llvm::SanStat_CFI_UnrelatedCast;
2765
15
    break;
2766
0
  case CFITCK_ICall:
2767
0
  case CFITCK_NVMFCall:
2768
0
  case CFITCK_VMFCall:
2769
0
    llvm_unreachable("unexpected sanitizer kind");
2770
62
  }
2771
2772
62
  std::string TypeName = RD->getQualifiedNameAsString();
2773
62
  if (getContext().getNoSanitizeList().containsType(M, TypeName))
2774
2
    return;
2775
2776
60
  SanitizerScope SanScope(this);
2777
60
  EmitSanitizerStatReport(SSK);
2778
2779
60
  llvm::Metadata *MD =
2780
60
      CGM.CreateMetadataIdentifierForType(QualType(RD->getTypeForDecl(), 0));
2781
60
  llvm::Value *TypeId = llvm::MetadataAsValue::get(getLLVMContext(), MD);
2782
2783
60
  llvm::Value *CastedVTable = Builder.CreateBitCast(VTable, Int8PtrTy);
2784
60
  llvm::Value *TypeTest = Builder.CreateCall(
2785
60
      CGM.getIntrinsic(llvm::Intrinsic::type_test), {CastedVTable, TypeId});
2786
2787
60
  llvm::Constant *StaticData[] = {
2788
60
      llvm::ConstantInt::get(Int8Ty, TCK),
2789
60
      EmitCheckSourceLocation(Loc),
2790
60
      EmitCheckTypeDescriptor(QualType(RD->getTypeForDecl(), 0)),
2791
60
  };
2792
2793
60
  auto CrossDsoTypeId = CGM.CreateCrossDsoCfiTypeId(MD);
2794
60
  if (CGM.getCodeGenOpts().SanitizeCfiCrossDso && 
CrossDsoTypeId2
) {
2795
2
    EmitCfiSlowPathCheck(M, TypeTest, CrossDsoTypeId, CastedVTable, StaticData);
2796
2
    return;
2797
2
  }
2798
2799
58
  if (CGM.getCodeGenOpts().SanitizeTrap.has(M)) {
2800
28
    EmitTrapCheck(TypeTest, SanitizerHandler::CFICheckFail);
2801
28
    return;
2802
28
  }
2803
2804
30
  llvm::Value *AllVtables = llvm::MetadataAsValue::get(
2805
30
      CGM.getLLVMContext(),
2806
30
      llvm::MDString::get(CGM.getLLVMContext(), "all-vtables"));
2807
30
  llvm::Value *ValidVtable = Builder.CreateCall(
2808
30
      CGM.getIntrinsic(llvm::Intrinsic::type_test), {CastedVTable, AllVtables});
2809
30
  EmitCheck(std::make_pair(TypeTest, M), SanitizerHandler::CFICheckFail,
2810
30
            StaticData, {CastedVTable, ValidVtable});
2811
30
}
2812
2813
1.01k
bool CodeGenFunction::ShouldEmitVTableTypeCheckedLoad(const CXXRecordDecl *RD) {
2814
1.01k
  if (!CGM.getCodeGenOpts().WholeProgramVTables ||
2815
1.01k
      
!CGM.HasHiddenLTOVisibility(RD)87
)
2816
955
    return false;
2817
2818
62
  if (CGM.getCodeGenOpts().VirtualFunctionElimination)
2819
2
    return true;
2820
2821
60
  if (!SanOpts.has(SanitizerKind::CFIVCall) ||
2822
60
      
!CGM.getCodeGenOpts().SanitizeTrap.has(SanitizerKind::CFIVCall)12
)
2823
49
    return false;
2824
2825
11
  std::string TypeName = RD->getQualifiedNameAsString();
2826
11
  return !getContext().getNoSanitizeList().containsType(SanitizerKind::CFIVCall,
2827
11
                                                        TypeName);
2828
60
}
2829
2830
llvm::Value *CodeGenFunction::EmitVTableTypeCheckedLoad(
2831
13
    const CXXRecordDecl *RD, llvm::Value *VTable, uint64_t VTableByteOffset) {
2832
13
  SanitizerScope SanScope(this);
2833
2834
13
  EmitSanitizerStatReport(llvm::SanStat_CFI_VCall);
2835
2836
13
  llvm::Metadata *MD =
2837
13
      CGM.CreateMetadataIdentifierForType(QualType(RD->getTypeForDecl(), 0));
2838
13
  llvm::Value *TypeId = llvm::MetadataAsValue::get(CGM.getLLVMContext(), MD);
2839
2840
13
  llvm::Value *CastedVTable = Builder.CreateBitCast(VTable, Int8PtrTy);
2841
13
  llvm::Value *CheckedLoad = Builder.CreateCall(
2842
13
      CGM.getIntrinsic(llvm::Intrinsic::type_checked_load),
2843
13
      {CastedVTable, llvm::ConstantInt::get(Int32Ty, VTableByteOffset),
2844
13
       TypeId});
2845
13
  llvm::Value *CheckResult = Builder.CreateExtractValue(CheckedLoad, 1);
2846
2847
13
  std::string TypeName = RD->getQualifiedNameAsString();
2848
13
  if (SanOpts.has(SanitizerKind::CFIVCall) &&
2849
13
      !getContext().getNoSanitizeList().containsType(SanitizerKind::CFIVCall,
2850
11
                                                     TypeName)) {
2851
11
    EmitCheck(std::make_pair(CheckResult, SanitizerKind::CFIVCall),
2852
11
              SanitizerHandler::CFICheckFail, {}, {});
2853
11
  }
2854
2855
13
  return Builder.CreateBitCast(Builder.CreateExtractValue(CheckedLoad, 0),
2856
13
                               VTable->getType()->getPointerElementType());
2857
13
}
2858
2859
void CodeGenFunction::EmitForwardingCallToLambda(
2860
                                      const CXXMethodDecl *callOperator,
2861
73
                                      CallArgList &callArgs) {
2862
  // Get the address of the call operator.
2863
73
  const CGFunctionInfo &calleeFnInfo =
2864
73
    CGM.getTypes().arrangeCXXMethodDeclaration(callOperator);
2865
73
  llvm::Constant *calleePtr =
2866
73
    CGM.GetAddrOfFunction(GlobalDecl(callOperator),
2867
73
                          CGM.getTypes().GetFunctionType(calleeFnInfo));
2868
2869
  // Prepare the return slot.
2870
73
  const FunctionProtoType *FPT =
2871
73
    callOperator->getType()->castAs<FunctionProtoType>();
2872
73
  QualType resultType = FPT->getReturnType();
2873
73
  ReturnValueSlot returnSlot;
2874
73
  if (!resultType->isVoidType() &&
2875
73
      
calleeFnInfo.getReturnInfo().getKind() == ABIArgInfo::Indirect37
&&
2876
73
      
!hasScalarEvaluationKind(calleeFnInfo.getReturnType())1
)
2877
1
    returnSlot =
2878
1
        ReturnValueSlot(ReturnValue, resultType.isVolatileQualified(),
2879
1
                        /*IsUnused=*/false, /*IsExternallyDestructed=*/true);
2880
2881
  // We don't need to separately arrange the call arguments because
2882
  // the call can't be variadic anyway --- it's impossible to forward
2883
  // variadic arguments.
2884
2885
  // Now emit our call.
2886
73
  auto callee = CGCallee::forDirect(calleePtr, GlobalDecl(callOperator));
2887
73
  RValue RV = EmitCall(calleeFnInfo, callee, returnSlot, callArgs);
2888
2889
  // If necessary, copy the returned value into the slot.
2890
73
  if (!resultType->isVoidType() && 
returnSlot.isNull()37
) {
2891
36
    if (getLangOpts().ObjCAutoRefCount && 
resultType->isObjCRetainableType()5
) {
2892
2
      RV = RValue::get(EmitARCRetainAutoreleasedReturnValue(RV.getScalarVal()));
2893
2
    }
2894
36
    EmitReturnOfRValue(RV, resultType);
2895
36
  } else
2896
37
    EmitBranchThroughCleanup(ReturnBlock);
2897
73
}
2898
2899
13
void CodeGenFunction::EmitLambdaBlockInvokeBody() {
2900
13
  const BlockDecl *BD = BlockInfo->getBlockDecl();
2901
13
  const VarDecl *variable = BD->capture_begin()->getVariable();
2902
13
  const CXXRecordDecl *Lambda = variable->getType()->getAsCXXRecordDecl();
2903
13
  const CXXMethodDecl *CallOp = Lambda->getLambdaCallOperator();
2904
2905
13
  if (CallOp->isVariadic()) {
2906
    // FIXME: Making this work correctly is nasty because it requires either
2907
    // cloning the body of the call operator or making the call operator
2908
    // forward.
2909
0
    CGM.ErrorUnsupported(CurCodeDecl, "lambda conversion to variadic function");
2910
0
    return;
2911
0
  }
2912
2913
  // Start building arguments for forwarding call
2914
13
  CallArgList CallArgs;
2915
2916
13
  QualType ThisType = getContext().getPointerType(getContext().getRecordType(Lambda));
2917
13
  Address ThisPtr = GetAddrOfBlockDecl(variable);
2918
13
  CallArgs.add(RValue::get(ThisPtr.getPointer()), ThisType);
2919
2920
  // Add the rest of the parameters.
2921
13
  for (auto param : BD->parameters())
2922
1
    EmitDelegateCallArg(CallArgs, param, param->getBeginLoc());
2923
2924
13
  assert(!Lambda->isGenericLambda() &&
2925
13
            "generic lambda interconversion to block not implemented");
2926
0
  EmitForwardingCallToLambda(CallOp, CallArgs);
2927
13
}
2928
2929
60
void CodeGenFunction::EmitLambdaDelegatingInvokeBody(const CXXMethodDecl *MD) {
2930
60
  const CXXRecordDecl *Lambda = MD->getParent();
2931
2932
  // Start building arguments for forwarding call
2933
60
  CallArgList CallArgs;
2934
2935
60
  QualType ThisType = getContext().getPointerType(getContext().getRecordType(Lambda));
2936
60
  llvm::Value *ThisPtr = llvm::UndefValue::get(getTypes().ConvertType(ThisType));
2937
60
  CallArgs.add(RValue::get(ThisPtr), ThisType);
2938
2939
  // Add the rest of the parameters.
2940
60
  for (auto Param : MD->parameters())
2941
140
    EmitDelegateCallArg(CallArgs, Param, Param->getBeginLoc());
2942
2943
60
  const CXXMethodDecl *CallOp = Lambda->getLambdaCallOperator();
2944
  // For a generic lambda, find the corresponding call operator specialization
2945
  // to which the call to the static-invoker shall be forwarded.
2946
60
  if (Lambda->isGenericLambda()) {
2947
0
    assert(MD->isFunctionTemplateSpecialization());
2948
0
    const TemplateArgumentList *TAL = MD->getTemplateSpecializationArgs();
2949
0
    FunctionTemplateDecl *CallOpTemplate = CallOp->getDescribedFunctionTemplate();
2950
0
    void *InsertPos = nullptr;
2951
0
    FunctionDecl *CorrespondingCallOpSpecialization =
2952
0
        CallOpTemplate->findSpecialization(TAL->asArray(), InsertPos);
2953
0
    assert(CorrespondingCallOpSpecialization);
2954
0
    CallOp = cast<CXXMethodDecl>(CorrespondingCallOpSpecialization);
2955
0
  }
2956
0
  EmitForwardingCallToLambda(CallOp, CallArgs);
2957
60
}
2958
2959
60
void CodeGenFunction::EmitLambdaStaticInvokeBody(const CXXMethodDecl *MD) {
2960
60
  if (MD->isVariadic()) {
2961
    // FIXME: Making this work correctly is nasty because it requires either
2962
    // cloning the body of the call operator or making the call operator forward.
2963
0
    CGM.ErrorUnsupported(MD, "lambda conversion to variadic function");
2964
0
    return;
2965
0
  }
2966
2967
60
  EmitLambdaDelegatingInvokeBody(MD);
2968
60
}