Coverage Report

Created: 2022-05-17 06:19

/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/CodeGen/CGClass.cpp
Line
Count
Source (jump to first uncovered line)
1
//===--- CGClass.cpp - Emit LLVM Code for C++ classes -----------*- C++ -*-===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This contains code dealing with C++ code generation of classes
10
//
11
//===----------------------------------------------------------------------===//
12
13
#include "CGBlocks.h"
14
#include "CGCXXABI.h"
15
#include "CGDebugInfo.h"
16
#include "CGRecordLayout.h"
17
#include "CodeGenFunction.h"
18
#include "TargetInfo.h"
19
#include "clang/AST/Attr.h"
20
#include "clang/AST/CXXInheritance.h"
21
#include "clang/AST/CharUnits.h"
22
#include "clang/AST/DeclTemplate.h"
23
#include "clang/AST/EvaluatedExprVisitor.h"
24
#include "clang/AST/RecordLayout.h"
25
#include "clang/AST/StmtCXX.h"
26
#include "clang/Basic/CodeGenOptions.h"
27
#include "clang/Basic/TargetBuiltins.h"
28
#include "clang/CodeGen/CGFunctionInfo.h"
29
#include "llvm/IR/Intrinsics.h"
30
#include "llvm/IR/Metadata.h"
31
#include "llvm/Transforms/Utils/SanitizerStats.h"
32
33
using namespace clang;
34
using namespace CodeGen;
35
36
/// Return the best known alignment for an unknown pointer to a
37
/// particular class.
38
726k
CharUnits CodeGenModule::getClassPointerAlignment(const CXXRecordDecl *RD) {
39
726k
  if (!RD->hasDefinition())
40
0
    return CharUnits::One(); // Hopefully won't be used anywhere.
41
42
726k
  auto &layout = getContext().getASTRecordLayout(RD);
43
44
  // If the class is final, then we know that the pointer points to an
45
  // object of that type and can use the full alignment.
46
726k
  if (RD->isEffectivelyFinal())
47
181
    return layout.getAlignment();
48
49
  // Otherwise, we have to assume it could be a subclass.
50
726k
  return layout.getNonVirtualAlignment();
51
726k
}
52
53
/// Return the smallest possible amount of storage that might be allocated
54
/// starting from the beginning of an object of a particular class.
55
///
56
/// This may be smaller than sizeof(RD) if RD has virtual base classes.
57
380k
CharUnits CodeGenModule::getMinimumClassObjectSize(const CXXRecordDecl *RD) {
58
380k
  if (!RD->hasDefinition())
59
2
    return CharUnits::One();
60
61
380k
  auto &layout = getContext().getASTRecordLayout(RD);
62
63
  // If the class is final, then we know that the pointer points to an
64
  // object of that type and can use the full alignment.
65
380k
  if (RD->isEffectivelyFinal())
66
86
    return layout.getSize();
67
68
  // Otherwise, we have to assume it could be a subclass.
69
380k
  return std::max(layout.getNonVirtualSize(), CharUnits::One());
70
380k
}
71
72
/// Return the best known alignment for a pointer to a virtual base,
73
/// given the alignment of a pointer to the derived class.
74
CharUnits CodeGenModule::getVBaseAlignment(CharUnits actualDerivedAlign,
75
                                           const CXXRecordDecl *derivedClass,
76
891
                                           const CXXRecordDecl *vbaseClass) {
77
  // The basic idea here is that an underaligned derived pointer might
78
  // indicate an underaligned base pointer.
79
80
891
  assert(vbaseClass->isCompleteDefinition());
81
0
  auto &baseLayout = getContext().getASTRecordLayout(vbaseClass);
82
891
  CharUnits expectedVBaseAlign = baseLayout.getNonVirtualAlignment();
83
84
891
  return getDynamicOffsetAlignment(actualDerivedAlign, derivedClass,
85
891
                                   expectedVBaseAlign);
86
891
}
87
88
CharUnits
89
CodeGenModule::getDynamicOffsetAlignment(CharUnits actualBaseAlign,
90
                                         const CXXRecordDecl *baseDecl,
91
1.07k
                                         CharUnits expectedTargetAlign) {
92
  // If the base is an incomplete type (which is, alas, possible with
93
  // member pointers), be pessimistic.
94
1.07k
  if (!baseDecl->isCompleteDefinition())
95
5
    return std::min(actualBaseAlign, expectedTargetAlign);
96
97
1.06k
  auto &baseLayout = getContext().getASTRecordLayout(baseDecl);
98
1.06k
  CharUnits expectedBaseAlign = baseLayout.getNonVirtualAlignment();
99
100
  // If the class is properly aligned, assume the target offset is, too.
101
  //
102
  // This actually isn't necessarily the right thing to do --- if the
103
  // class is a complete object, but it's only properly aligned for a
104
  // base subobject, then the alignments of things relative to it are
105
  // probably off as well.  (Note that this requires the alignment of
106
  // the target to be greater than the NV alignment of the derived
107
  // class.)
108
  //
109
  // However, our approach to this kind of under-alignment can only
110
  // ever be best effort; after all, we're never going to propagate
111
  // alignments through variables or parameters.  Note, in particular,
112
  // that constructing a polymorphic type in an address that's less
113
  // than pointer-aligned will generally trap in the constructor,
114
  // unless we someday add some sort of attribute to change the
115
  // assumed alignment of 'this'.  So our goal here is pretty much
116
  // just to allow the user to explicitly say that a pointer is
117
  // under-aligned and then safely access its fields and vtables.
118
1.06k
  if (actualBaseAlign >= expectedBaseAlign) {
119
1.06k
    return expectedTargetAlign;
120
1.06k
  }
121
122
  // Otherwise, we might be offset by an arbitrary multiple of the
123
  // actual alignment.  The correct adjustment is to take the min of
124
  // the two alignments.
125
0
  return std::min(actualBaseAlign, expectedTargetAlign);
126
1.06k
}
127
128
65.0k
Address CodeGenFunction::LoadCXXThisAddress() {
129
65.0k
  assert(CurFuncDecl && "loading 'this' without a func declaration?");
130
0
  auto *MD = cast<CXXMethodDecl>(CurFuncDecl);
131
132
  // Lazily compute CXXThisAlignment.
133
65.0k
  if (CXXThisAlignment.isZero()) {
134
    // Just use the best known alignment for the parent.
135
    // TODO: if we're currently emitting a complete-object ctor/dtor,
136
    // we can always use the complete-object alignment.
137
50.7k
    CXXThisAlignment = CGM.getClassPointerAlignment(MD->getParent());
138
50.7k
  }
139
140
65.0k
  llvm::Type *Ty = ConvertType(MD->getThisType()->getPointeeType());
141
65.0k
  return Address(LoadCXXThis(), Ty, CXXThisAlignment);
142
65.0k
}
143
144
/// Emit the address of a field using a member data pointer.
145
///
146
/// \param E Only used for emergency diagnostics
147
Address
148
CodeGenFunction::EmitCXXMemberDataPointerAddress(const Expr *E, Address base,
149
                                                 llvm::Value *memberPtr,
150
                                      const MemberPointerType *memberPtrType,
151
                                                 LValueBaseInfo *BaseInfo,
152
81
                                                 TBAAAccessInfo *TBAAInfo) {
153
  // Ask the ABI to compute the actual address.
154
81
  llvm::Value *ptr =
155
81
    CGM.getCXXABI().EmitMemberDataPointerAddress(*this, E, base,
156
81
                                                 memberPtr, memberPtrType);
157
158
81
  QualType memberType = memberPtrType->getPointeeType();
159
81
  CharUnits memberAlign =
160
81
      CGM.getNaturalTypeAlignment(memberType, BaseInfo, TBAAInfo);
161
81
  memberAlign =
162
81
    CGM.getDynamicOffsetAlignment(base.getAlignment(),
163
81
                            memberPtrType->getClass()->getAsCXXRecordDecl(),
164
81
                                  memberAlign);
165
81
  return Address(ptr, ConvertTypeForMem(memberPtrType->getPointeeType()),
166
81
                 memberAlign);
167
81
}
168
169
CharUnits CodeGenModule::computeNonVirtualBaseClassOffset(
170
    const CXXRecordDecl *DerivedClass, CastExpr::path_const_iterator Start,
171
18.9k
    CastExpr::path_const_iterator End) {
172
18.9k
  CharUnits Offset = CharUnits::Zero();
173
174
18.9k
  const ASTContext &Context = getContext();
175
18.9k
  const CXXRecordDecl *RD = DerivedClass;
176
177
39.8k
  for (CastExpr::path_const_iterator I = Start; I != End; 
++I20.9k
) {
178
20.9k
    const CXXBaseSpecifier *Base = *I;
179
20.9k
    assert(!Base->isVirtual() && "Should not see virtual bases here!");
180
181
    // Get the layout.
182
0
    const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
183
184
20.9k
    const auto *BaseDecl =
185
20.9k
        cast<CXXRecordDecl>(Base->getType()->castAs<RecordType>()->getDecl());
186
187
    // Add the offset.
188
20.9k
    Offset += Layout.getBaseClassOffset(BaseDecl);
189
190
20.9k
    RD = BaseDecl;
191
20.9k
  }
192
193
18.9k
  return Offset;
194
18.9k
}
195
196
llvm::Constant *
197
CodeGenModule::GetNonVirtualBaseClassOffset(const CXXRecordDecl *ClassDecl,
198
                                   CastExpr::path_const_iterator PathBegin,
199
795
                                   CastExpr::path_const_iterator PathEnd) {
200
795
  assert(PathBegin != PathEnd && "Base path should not be empty!");
201
202
0
  CharUnits Offset =
203
795
      computeNonVirtualBaseClassOffset(ClassDecl, PathBegin, PathEnd);
204
795
  if (Offset.isZero())
205
761
    return nullptr;
206
207
34
  llvm::Type *PtrDiffTy =
208
34
  Types.ConvertType(getContext().getPointerDiffType());
209
210
34
  return llvm::ConstantInt::get(PtrDiffTy, Offset.getQuantity());
211
795
}
212
213
/// Gets the address of a direct base class within a complete object.
214
/// This should only be used for (1) non-virtual bases or (2) virtual bases
215
/// when the type is known to be complete (e.g. in complete destructors).
216
///
217
/// The object pointed to by 'This' is assumed to be non-null.
218
Address
219
CodeGenFunction::GetAddressOfDirectBaseInCompleteClass(Address This,
220
                                                   const CXXRecordDecl *Derived,
221
                                                   const CXXRecordDecl *Base,
222
11.1k
                                                   bool BaseIsVirtual) {
223
  // 'this' must be a pointer (in some address space) to Derived.
224
11.1k
  assert(This.getElementType() == ConvertType(Derived));
225
226
  // Compute the offset of the virtual base.
227
0
  CharUnits Offset;
228
11.1k
  const ASTRecordLayout &Layout = getContext().getASTRecordLayout(Derived);
229
11.1k
  if (BaseIsVirtual)
230
986
    Offset = Layout.getVBaseClassOffset(Base);
231
10.1k
  else
232
10.1k
    Offset = Layout.getBaseClassOffset(Base);
233
234
  // Shift and cast down to the base type.
235
  // TODO: for complete types, this should be possible with a GEP.
236
11.1k
  Address V = This;
237
11.1k
  if (!Offset.isZero()) {
238
2.03k
    V = Builder.CreateElementBitCast(V, Int8Ty);
239
2.03k
    V = Builder.CreateConstInBoundsByteGEP(V, Offset);
240
2.03k
  }
241
11.1k
  V = Builder.CreateElementBitCast(V, ConvertType(Base));
242
243
11.1k
  return V;
244
11.1k
}
245
246
static Address
247
ApplyNonVirtualAndVirtualOffset(CodeGenFunction &CGF, Address addr,
248
                                CharUnits nonVirtualOffset,
249
                                llvm::Value *virtualOffset,
250
                                const CXXRecordDecl *derivedClass,
251
2.07k
                                const CXXRecordDecl *nearestVBase) {
252
  // Assert that we have something to do.
253
2.07k
  assert(!nonVirtualOffset.isZero() || virtualOffset != nullptr);
254
255
  // Compute the offset from the static and dynamic components.
256
0
  llvm::Value *baseOffset;
257
2.07k
  if (!nonVirtualOffset.isZero()) {
258
1.31k
    llvm::Type *OffsetType =
259
1.31k
        (CGF.CGM.getTarget().getCXXABI().isItaniumFamily() &&
260
1.31k
         
CGF.CGM.getItaniumVTableContext().isRelativeLayout()1.18k
)
261
1.31k
            ? 
CGF.Int32Ty4
262
1.31k
            : 
CGF.PtrDiffTy1.31k
;
263
1.31k
    baseOffset =
264
1.31k
        llvm::ConstantInt::get(OffsetType, nonVirtualOffset.getQuantity());
265
1.31k
    if (virtualOffset) {
266
34
      baseOffset = CGF.Builder.CreateAdd(virtualOffset, baseOffset);
267
34
    }
268
1.31k
  } else {
269
752
    baseOffset = virtualOffset;
270
752
  }
271
272
  // Apply the base offset.
273
2.07k
  llvm::Value *ptr = addr.getPointer();
274
2.07k
  unsigned AddrSpace = ptr->getType()->getPointerAddressSpace();
275
2.07k
  ptr = CGF.Builder.CreateBitCast(ptr, CGF.Int8Ty->getPointerTo(AddrSpace));
276
2.07k
  ptr = CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, ptr, baseOffset, "add.ptr");
277
278
  // If we have a virtual component, the alignment of the result will
279
  // be relative only to the known alignment of that vbase.
280
2.07k
  CharUnits alignment;
281
2.07k
  if (virtualOffset) {
282
786
    assert(nearestVBase && "virtual offset without vbase?");
283
0
    alignment = CGF.CGM.getVBaseAlignment(addr.getAlignment(),
284
786
                                          derivedClass, nearestVBase);
285
1.28k
  } else {
286
1.28k
    alignment = addr.getAlignment();
287
1.28k
  }
288
0
  alignment = alignment.alignmentAtOffset(nonVirtualOffset);
289
290
2.07k
  return Address(ptr, CGF.Int8Ty, alignment);
291
2.07k
}
292
293
Address CodeGenFunction::GetAddressOfBaseClass(
294
    Address Value, const CXXRecordDecl *Derived,
295
    CastExpr::path_const_iterator PathBegin,
296
    CastExpr::path_const_iterator PathEnd, bool NullCheckValue,
297
18.0k
    SourceLocation Loc) {
298
18.0k
  assert(PathBegin != PathEnd && "Base path should not be empty!");
299
300
0
  CastExpr::path_const_iterator Start = PathBegin;
301
18.0k
  const CXXRecordDecl *VBase = nullptr;
302
303
  // Sema has done some convenient canonicalization here: if the
304
  // access path involved any virtual steps, the conversion path will
305
  // *start* with a step down to the correct virtual base subobject,
306
  // and hence will not require any further steps.
307
18.0k
  if ((*Start)->isVirtual()) {
308
439
    VBase = cast<CXXRecordDecl>(
309
439
        (*Start)->getType()->castAs<RecordType>()->getDecl());
310
439
    ++Start;
311
439
  }
312
313
  // Compute the static offset of the ultimate destination within its
314
  // allocating subobject (the virtual base, if there is one, or else
315
  // the "complete" object that we see).
316
18.0k
  CharUnits NonVirtualOffset = CGM.computeNonVirtualBaseClassOffset(
317
18.0k
      VBase ? 
VBase439
:
Derived17.5k
, Start, PathEnd);
318
319
  // If there's a virtual step, we can sometimes "devirtualize" it.
320
  // For now, that's limited to when the derived type is final.
321
  // TODO: "devirtualize" this for accesses to known-complete objects.
322
18.0k
  if (VBase && 
Derived->hasAttr<FinalAttr>()439
) {
323
6
    const ASTRecordLayout &layout = getContext().getASTRecordLayout(Derived);
324
6
    CharUnits vBaseOffset = layout.getVBaseClassOffset(VBase);
325
6
    NonVirtualOffset += vBaseOffset;
326
6
    VBase = nullptr; // we no longer have a virtual step
327
6
  }
328
329
  // Get the base pointer type.
330
18.0k
  llvm::Type *BaseValueTy = ConvertType((PathEnd[-1])->getType());
331
18.0k
  llvm::Type *BasePtrTy =
332
18.0k
      BaseValueTy->getPointerTo(Value.getType()->getPointerAddressSpace());
333
334
18.0k
  QualType DerivedTy = getContext().getRecordType(Derived);
335
18.0k
  CharUnits DerivedAlign = CGM.getClassPointerAlignment(Derived);
336
337
  // If the static offset is zero and we don't have a virtual step,
338
  // just do a bitcast; null checks are unnecessary.
339
18.0k
  if (NonVirtualOffset.isZero() && 
!VBase16.9k
) {
340
16.5k
    if (sanitizePerformTypeCheck()) {
341
19
      SanitizerSet SkippedChecks;
342
19
      SkippedChecks.set(SanitizerKind::Null, !NullCheckValue);
343
19
      EmitTypeCheck(TCK_Upcast, Loc, Value.getPointer(),
344
19
                    DerivedTy, DerivedAlign, SkippedChecks);
345
19
    }
346
16.5k
    return Builder.CreateElementBitCast(Value, BaseValueTy);
347
16.5k
  }
348
349
1.51k
  llvm::BasicBlock *origBB = nullptr;
350
1.51k
  llvm::BasicBlock *endBB = nullptr;
351
352
  // Skip over the offset (and the vtable load) if we're supposed to
353
  // null-check the pointer.
354
1.51k
  if (NullCheckValue) {
355
35
    origBB = Builder.GetInsertBlock();
356
35
    llvm::BasicBlock *notNullBB = createBasicBlock("cast.notnull");
357
35
    endBB = createBasicBlock("cast.end");
358
359
35
    llvm::Value *isNull = Builder.CreateIsNull(Value.getPointer());
360
35
    Builder.CreateCondBr(isNull, endBB, notNullBB);
361
35
    EmitBlock(notNullBB);
362
35
  }
363
364
1.51k
  if (sanitizePerformTypeCheck()) {
365
3
    SanitizerSet SkippedChecks;
366
3
    SkippedChecks.set(SanitizerKind::Null, true);
367
3
    EmitTypeCheck(VBase ? TCK_UpcastToVirtualBase : 
TCK_Upcast0
, Loc,
368
3
                  Value.getPointer(), DerivedTy, DerivedAlign, SkippedChecks);
369
3
  }
370
371
  // Compute the virtual offset.
372
1.51k
  llvm::Value *VirtualOffset = nullptr;
373
1.51k
  if (VBase) {
374
433
    VirtualOffset =
375
433
      CGM.getCXXABI().GetVirtualBaseClassOffset(*this, Value, Derived, VBase);
376
433
  }
377
378
  // Apply both offsets.
379
1.51k
  Value = ApplyNonVirtualAndVirtualOffset(*this, Value, NonVirtualOffset,
380
1.51k
                                          VirtualOffset, Derived, VBase);
381
382
  // Cast to the destination type.
383
1.51k
  Value = Builder.CreateElementBitCast(Value, BaseValueTy);
384
385
  // Build a phi if we needed a null check.
386
1.51k
  if (NullCheckValue) {
387
35
    llvm::BasicBlock *notNullBB = Builder.GetInsertBlock();
388
35
    Builder.CreateBr(endBB);
389
35
    EmitBlock(endBB);
390
391
35
    llvm::PHINode *PHI = Builder.CreatePHI(BasePtrTy, 2, "cast.result");
392
35
    PHI->addIncoming(Value.getPointer(), notNullBB);
393
35
    PHI->addIncoming(llvm::Constant::getNullValue(BasePtrTy), origBB);
394
35
    Value = Value.withPointer(PHI);
395
35
  }
396
397
1.51k
  return Value;
398
18.0k
}
399
400
Address
401
CodeGenFunction::GetAddressOfDerivedClass(Address BaseAddr,
402
                                          const CXXRecordDecl *Derived,
403
                                        CastExpr::path_const_iterator PathBegin,
404
                                          CastExpr::path_const_iterator PathEnd,
405
739
                                          bool NullCheckValue) {
406
739
  assert(PathBegin != PathEnd && "Base path should not be empty!");
407
408
0
  QualType DerivedTy =
409
739
    getContext().getCanonicalType(getContext().getTagDeclType(Derived));
410
739
  unsigned AddrSpace = BaseAddr.getAddressSpace();
411
739
  llvm::Type *DerivedValueTy = ConvertType(DerivedTy);
412
739
  llvm::Type *DerivedPtrTy = DerivedValueTy->getPointerTo(AddrSpace);
413
414
739
  llvm::Value *NonVirtualOffset =
415
739
    CGM.GetNonVirtualBaseClassOffset(Derived, PathBegin, PathEnd);
416
417
739
  if (!NonVirtualOffset) {
418
    // No offset, we can just cast back.
419
728
    return Builder.CreateElementBitCast(BaseAddr, DerivedValueTy);
420
728
  }
421
422
11
  llvm::BasicBlock *CastNull = nullptr;
423
11
  llvm::BasicBlock *CastNotNull = nullptr;
424
11
  llvm::BasicBlock *CastEnd = nullptr;
425
426
11
  if (NullCheckValue) {
427
5
    CastNull = createBasicBlock("cast.null");
428
5
    CastNotNull = createBasicBlock("cast.notnull");
429
5
    CastEnd = createBasicBlock("cast.end");
430
431
5
    llvm::Value *IsNull = Builder.CreateIsNull(BaseAddr.getPointer());
432
5
    Builder.CreateCondBr(IsNull, CastNull, CastNotNull);
433
5
    EmitBlock(CastNotNull);
434
5
  }
435
436
  // Apply the offset.
437
11
  llvm::Value *Value = Builder.CreateBitCast(BaseAddr.getPointer(), Int8PtrTy);
438
11
  Value = Builder.CreateInBoundsGEP(
439
11
      Int8Ty, Value, Builder.CreateNeg(NonVirtualOffset), "sub.ptr");
440
441
  // Just cast.
442
11
  Value = Builder.CreateBitCast(Value, DerivedPtrTy);
443
444
  // Produce a PHI if we had a null-check.
445
11
  if (NullCheckValue) {
446
5
    Builder.CreateBr(CastEnd);
447
5
    EmitBlock(CastNull);
448
5
    Builder.CreateBr(CastEnd);
449
5
    EmitBlock(CastEnd);
450
451
5
    llvm::PHINode *PHI = Builder.CreatePHI(Value->getType(), 2);
452
5
    PHI->addIncoming(Value, CastNotNull);
453
5
    PHI->addIncoming(llvm::Constant::getNullValue(Value->getType()), CastNull);
454
5
    Value = PHI;
455
5
  }
456
457
11
  return Address(Value, DerivedValueTy, CGM.getClassPointerAlignment(Derived));
458
739
}
459
460
llvm::Value *CodeGenFunction::GetVTTParameter(GlobalDecl GD,
461
                                              bool ForVirtualBase,
462
29.0k
                                              bool Delegating) {
463
29.0k
  if (!CGM.getCXXABI().NeedsVTTParameter(GD)) {
464
    // This constructor/destructor does not need a VTT parameter.
465
28.7k
    return nullptr;
466
28.7k
  }
467
468
263
  const CXXRecordDecl *RD = cast<CXXMethodDecl>(CurCodeDecl)->getParent();
469
263
  const CXXRecordDecl *Base = cast<CXXMethodDecl>(GD.getDecl())->getParent();
470
471
263
  uint64_t SubVTTIndex;
472
473
263
  if (Delegating) {
474
    // If this is a delegating constructor call, just load the VTT.
475
2
    return LoadCXXVTT();
476
261
  } else if (RD == Base) {
477
    // If the record matches the base, this is the complete ctor/dtor
478
    // variant calling the base variant in a class with virtual bases.
479
89
    assert(!CGM.getCXXABI().NeedsVTTParameter(CurGD) &&
480
89
           "doing no-op VTT offset in base dtor/ctor?");
481
0
    assert(!ForVirtualBase && "Can't have same class as virtual base!");
482
0
    SubVTTIndex = 0;
483
172
  } else {
484
172
    const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
485
172
    CharUnits BaseOffset = ForVirtualBase ?
486
7
      Layout.getVBaseClassOffset(Base) :
487
172
      
Layout.getBaseClassOffset(Base)165
;
488
489
172
    SubVTTIndex =
490
172
      CGM.getVTables().getSubVTTIndex(RD, BaseSubobject(Base, BaseOffset));
491
172
    assert(SubVTTIndex != 0 && "Sub-VTT index must be greater than zero!");
492
172
  }
493
494
261
  if (CGM.getCXXABI().NeedsVTTParameter(CurGD)) {
495
    // A VTT parameter was passed to the constructor, use it.
496
38
    llvm::Value *VTT = LoadCXXVTT();
497
38
    return Builder.CreateConstInBoundsGEP1_64(VoidPtrTy, VTT, SubVTTIndex);
498
223
  } else {
499
    // We're the complete constructor, so get the VTT by name.
500
223
    llvm::GlobalValue *VTT = CGM.getVTables().GetAddrOfVTT(RD);
501
223
    return Builder.CreateConstInBoundsGEP2_64(
502
223
        VTT->getValueType(), VTT, 0, SubVTTIndex);
503
223
  }
504
261
}
505
506
namespace {
507
  /// Call the destructor for a direct base class.
508
  struct CallBaseDtor final : EHScopeStack::Cleanup {
509
    const CXXRecordDecl *BaseClass;
510
    bool BaseIsVirtual;
511
    CallBaseDtor(const CXXRecordDecl *Base, bool BaseIsVirtual)
512
2.08k
      : BaseClass(Base), BaseIsVirtual(BaseIsVirtual) {}
513
514
1.46k
    void Emit(CodeGenFunction &CGF, Flags flags) override {
515
1.46k
      const CXXRecordDecl *DerivedClass =
516
1.46k
        cast<CXXMethodDecl>(CGF.CurCodeDecl)->getParent();
517
518
1.46k
      const CXXDestructorDecl *D = BaseClass->getDestructor();
519
      // We are already inside a destructor, so presumably the object being
520
      // destroyed should have the expected type.
521
1.46k
      QualType ThisTy = D->getThisObjectType();
522
1.46k
      Address Addr =
523
1.46k
        CGF.GetAddressOfDirectBaseInCompleteClass(CGF.LoadCXXThisAddress(),
524
1.46k
                                                  DerivedClass, BaseClass,
525
1.46k
                                                  BaseIsVirtual);
526
1.46k
      CGF.EmitCXXDestructorCall(D, Dtor_Base, BaseIsVirtual,
527
1.46k
                                /*Delegating=*/false, Addr, ThisTy);
528
1.46k
    }
529
  };
530
531
  /// A visitor which checks whether an initializer uses 'this' in a
532
  /// way which requires the vtable to be properly set.
533
  struct DynamicThisUseChecker : ConstEvaluatedExprVisitor<DynamicThisUseChecker> {
534
    typedef ConstEvaluatedExprVisitor<DynamicThisUseChecker> super;
535
536
    bool UsesThis;
537
538
9.63k
    DynamicThisUseChecker(const ASTContext &C) : super(C), UsesThis(false) {}
539
540
    // Black-list all explicit and implicit references to 'this'.
541
    //
542
    // Do we need to worry about external references to 'this' derived
543
    // from arbitrary code?  If so, then anything which runs arbitrary
544
    // external code might potentially access the vtable.
545
13
    void VisitCXXThisExpr(const CXXThisExpr *E) { UsesThis = true; }
546
  };
547
} // end anonymous namespace
548
549
9.63k
static bool BaseInitializerUsesThis(ASTContext &C, const Expr *Init) {
550
9.63k
  DynamicThisUseChecker Checker(C);
551
9.63k
  Checker.Visit(Init);
552
9.63k
  return Checker.UsesThis;
553
9.63k
}
554
555
static void EmitBaseInitializer(CodeGenFunction &CGF,
556
                                const CXXRecordDecl *ClassDecl,
557
9.63k
                                CXXCtorInitializer *BaseInit) {
558
9.63k
  assert(BaseInit->isBaseInitializer() &&
559
9.63k
         "Must have base initializer!");
560
561
0
  Address ThisPtr = CGF.LoadCXXThisAddress();
562
563
9.63k
  const Type *BaseType = BaseInit->getBaseClass();
564
9.63k
  const auto *BaseClassDecl =
565
9.63k
      cast<CXXRecordDecl>(BaseType->castAs<RecordType>()->getDecl());
566
567
9.63k
  bool isBaseVirtual = BaseInit->isBaseVirtual();
568
569
  // If the initializer for the base (other than the constructor
570
  // itself) accesses 'this' in any way, we need to initialize the
571
  // vtables.
572
9.63k
  if (BaseInitializerUsesThis(CGF.getContext(), BaseInit->getInit()))
573
13
    CGF.InitializeVTablePointers(ClassDecl);
574
575
  // We can pretend to be a complete class because it only matters for
576
  // virtual bases, and we only do virtual bases for complete ctors.
577
9.63k
  Address V =
578
9.63k
    CGF.GetAddressOfDirectBaseInCompleteClass(ThisPtr, ClassDecl,
579
9.63k
                                              BaseClassDecl,
580
9.63k
                                              isBaseVirtual);
581
9.63k
  AggValueSlot AggSlot =
582
9.63k
      AggValueSlot::forAddr(
583
9.63k
          V, Qualifiers(),
584
9.63k
          AggValueSlot::IsDestructed,
585
9.63k
          AggValueSlot::DoesNotNeedGCBarriers,
586
9.63k
          AggValueSlot::IsNotAliased,
587
9.63k
          CGF.getOverlapForBaseInit(ClassDecl, BaseClassDecl, isBaseVirtual));
588
589
9.63k
  CGF.EmitAggExpr(BaseInit->getInit(), AggSlot);
590
591
9.63k
  if (CGF.CGM.getLangOpts().Exceptions &&
592
9.63k
      
!BaseClassDecl->hasTrivialDestructor()7.58k
)
593
914
    CGF.EHStack.pushCleanup<CallBaseDtor>(EHCleanup, BaseClassDecl,
594
914
                                          isBaseVirtual);
595
9.63k
}
596
597
98.7k
static bool isMemcpyEquivalentSpecialMember(const CXXMethodDecl *D) {
598
98.7k
  auto *CD = dyn_cast<CXXConstructorDecl>(D);
599
98.7k
  if (!(CD && 
CD->isCopyOrMoveConstructor()98.4k
) &&
600
98.7k
      
!D->isCopyAssignmentOperator()87.8k
&&
!D->isMoveAssignmentOperator()87.5k
)
601
87.5k
    return false;
602
603
  // We can emit a memcpy for a trivial copy or move constructor/assignment.
604
11.2k
  if (D->isTrivial() && 
!D->getParent()->mayInsertExtraPadding()6.03k
)
605
6.03k
    return true;
606
607
  // We *must* emit a memcpy for a defaulted union copy or move op.
608
5.20k
  if (D->getParent()->isUnion() && 
D->isDefaulted()4
)
609
2
    return true;
610
611
5.20k
  return false;
612
5.20k
}
613
614
static void EmitLValueForAnyFieldInitialization(CodeGenFunction &CGF,
615
                                                CXXCtorInitializer *MemberInit,
616
16.3k
                                                LValue &LHS) {
617
16.3k
  FieldDecl *Field = MemberInit->getAnyMember();
618
16.3k
  if (MemberInit->isIndirectMemberInitializer()) {
619
    // If we are initializing an anonymous union field, drill down to the field.
620
46
    IndirectFieldDecl *IndirectField = MemberInit->getIndirectMember();
621
46
    for (const auto *I : IndirectField->chain())
622
100
      LHS = CGF.EmitLValueForFieldInitialization(LHS, cast<FieldDecl>(I));
623
16.3k
  } else {
624
16.3k
    LHS = CGF.EmitLValueForFieldInitialization(LHS, Field);
625
16.3k
  }
626
16.3k
}
627
628
static void EmitMemberInitializer(CodeGenFunction &CGF,
629
                                  const CXXRecordDecl *ClassDecl,
630
                                  CXXCtorInitializer *MemberInit,
631
                                  const CXXConstructorDecl *Constructor,
632
16.3k
                                  FunctionArgList &Args) {
633
16.3k
  ApplyDebugLocation Loc(CGF, MemberInit->getSourceLocation());
634
16.3k
  assert(MemberInit->isAnyMemberInitializer() &&
635
16.3k
         "Must have member initializer!");
636
0
  assert(MemberInit->getInit() && "Must have initializer!");
637
638
  // non-static data member initializers.
639
0
  FieldDecl *Field = MemberInit->getAnyMember();
640
16.3k
  QualType FieldType = Field->getType();
641
642
16.3k
  llvm::Value *ThisPtr = CGF.LoadCXXThis();
643
16.3k
  QualType RecordTy = CGF.getContext().getTypeDeclType(ClassDecl);
644
16.3k
  LValue LHS;
645
646
  // If a base constructor is being emitted, create an LValue that has the
647
  // non-virtual alignment.
648
16.3k
  if (CGF.CurGD.getCtorType() == Ctor_Base)
649
16.1k
    LHS = CGF.MakeNaturalAlignPointeeAddrLValue(ThisPtr, RecordTy);
650
202
  else
651
202
    LHS = CGF.MakeNaturalAlignAddrLValue(ThisPtr, RecordTy);
652
653
16.3k
  EmitLValueForAnyFieldInitialization(CGF, MemberInit, LHS);
654
655
  // Special case: if we are in a copy or move constructor, and we are copying
656
  // an array of PODs or classes with trivial copy constructors, ignore the
657
  // AST and perform the copy we know is equivalent.
658
  // FIXME: This is hacky at best... if we had a bit more explicit information
659
  // in the AST, we could generalize it more easily.
660
16.3k
  const ConstantArrayType *Array
661
16.3k
    = CGF.getContext().getAsConstantArrayType(FieldType);
662
16.3k
  if (Array && 
Constructor->isDefaulted()86
&&
663
16.3k
      
Constructor->isCopyOrMoveConstructor()38
) {
664
23
    QualType BaseElementTy = CGF.getContext().getBaseElementType(Array);
665
23
    CXXConstructExpr *CE = dyn_cast<CXXConstructExpr>(MemberInit->getInit());
666
23
    if (BaseElementTy.isPODType(CGF.getContext()) ||
667
23
        
(18
CE18
&&
isMemcpyEquivalentSpecialMember(CE->getConstructor())0
)) {
668
5
      unsigned SrcArgIndex =
669
5
          CGF.CGM.getCXXABI().getSrcArgforCopyCtor(Constructor, Args);
670
5
      llvm::Value *SrcPtr
671
5
        = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(Args[SrcArgIndex]));
672
5
      LValue ThisRHSLV = CGF.MakeNaturalAlignAddrLValue(SrcPtr, RecordTy);
673
5
      LValue Src = CGF.EmitLValueForFieldInitialization(ThisRHSLV, Field);
674
675
      // Copy the aggregate.
676
5
      CGF.EmitAggregateCopy(LHS, Src, FieldType, CGF.getOverlapForFieldInit(Field),
677
5
                            LHS.isVolatileQualified());
678
      // Ensure that we destroy the objects if an exception is thrown later in
679
      // the constructor.
680
5
      QualType::DestructionKind dtorKind = FieldType.isDestructedType();
681
5
      if (CGF.needsEHCleanup(dtorKind))
682
0
        CGF.pushEHDestroy(dtorKind, LHS.getAddress(CGF), FieldType);
683
5
      return;
684
5
    }
685
23
  }
686
687
16.3k
  CGF.EmitInitializerForField(Field, LHS, MemberInit->getInit());
688
16.3k
}
689
690
void CodeGenFunction::EmitInitializerForField(FieldDecl *Field, LValue LHS,
691
17.7k
                                              Expr *Init) {
692
17.7k
  QualType FieldType = Field->getType();
693
17.7k
  switch (getEvaluationKind(FieldType)) {
694
13.1k
  case TEK_Scalar:
695
13.1k
    if (LHS.isSimple()) {
696
13.0k
      EmitExprAsInit(Init, Field, LHS, false);
697
13.0k
    } else {
698
82
      RValue RHS = RValue::get(EmitScalarExpr(Init));
699
82
      EmitStoreThroughLValue(RHS, LHS);
700
82
    }
701
13.1k
    break;
702
6
  case TEK_Complex:
703
6
    EmitComplexExprIntoLValue(Init, LHS, /*isInit*/ true);
704
6
    break;
705
4.54k
  case TEK_Aggregate: {
706
4.54k
    AggValueSlot Slot = AggValueSlot::forLValue(
707
4.54k
        LHS, *this, AggValueSlot::IsDestructed,
708
4.54k
        AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased,
709
4.54k
        getOverlapForFieldInit(Field), AggValueSlot::IsNotZeroed,
710
        // Checks are made by the code that calls constructor.
711
4.54k
        AggValueSlot::IsSanitizerChecked);
712
4.54k
    EmitAggExpr(Init, Slot);
713
4.54k
    break;
714
0
  }
715
17.7k
  }
716
717
  // Ensure that we destroy this object if an exception is thrown
718
  // later in the constructor.
719
17.7k
  QualType::DestructionKind dtorKind = FieldType.isDestructedType();
720
17.7k
  if (needsEHCleanup(dtorKind))
721
847
    pushEHDestroy(dtorKind, LHS.getAddress(*this), FieldType);
722
17.7k
}
723
724
/// Checks whether the given constructor is a valid subject for the
725
/// complete-to-base constructor delegation optimization, i.e.
726
/// emitting the complete constructor as a simple call to the base
727
/// constructor.
728
bool CodeGenFunction::IsConstructorDelegationValid(
729
17.5k
    const CXXConstructorDecl *Ctor) {
730
731
  // Currently we disable the optimization for classes with virtual
732
  // bases because (1) the addresses of parameter variables need to be
733
  // consistent across all initializers but (2) the delegate function
734
  // call necessarily creates a second copy of the parameter variable.
735
  //
736
  // The limiting example (purely theoretical AFAIK):
737
  //   struct A { A(int &c) { c++; } };
738
  //   struct B : virtual A {
739
  //     B(int count) : A(count) { printf("%d\n", count); }
740
  //   };
741
  // ...although even this example could in principle be emitted as a
742
  // delegation since the address of the parameter doesn't escape.
743
17.5k
  if (Ctor->getParent()->getNumVBases()) {
744
    // TODO: white-list trivial vbase initializers.  This case wouldn't
745
    // be subject to the restrictions below.
746
747
    // TODO: white-list cases where:
748
    //  - there are no non-reference parameters to the constructor
749
    //  - the initializers don't access any non-reference parameters
750
    //  - the initializers don't take the address of non-reference
751
    //    parameters
752
    //  - etc.
753
    // If we ever add any of the above cases, remember that:
754
    //  - function-try-blocks will always exclude this optimization
755
    //  - we need to perform the constructor prologue and cleanup in
756
    //    EmitConstructorBody.
757
758
582
    return false;
759
582
  }
760
761
  // We also disable the optimization for variadic functions because
762
  // it's impossible to "re-pass" varargs.
763
16.9k
  if (Ctor->getType()->castAs<FunctionProtoType>()->isVariadic())
764
10
    return false;
765
766
  // FIXME: Decide if we can do a delegation of a delegating constructor.
767
16.9k
  if (Ctor->isDelegatingConstructor())
768
78
    return false;
769
770
16.8k
  return true;
771
16.9k
}
772
773
// Emit code in ctor (Prologue==true) or dtor (Prologue==false)
774
// to poison the extra field paddings inserted under
775
// -fsanitize-address-field-padding=1|2.
776
55.0k
void CodeGenFunction::EmitAsanPrologueOrEpilogue(bool Prologue) {
777
55.0k
  ASTContext &Context = getContext();
778
55.0k
  const CXXRecordDecl *ClassDecl =
779
55.0k
      Prologue ? 
cast<CXXConstructorDecl>(CurGD.getDecl())->getParent()39.7k
780
55.0k
               : 
cast<CXXDestructorDecl>(CurGD.getDecl())->getParent()15.3k
;
781
55.0k
  if (!ClassDecl->mayInsertExtraPadding()) 
return55.0k
;
782
783
41
  struct SizeAndOffset {
784
41
    uint64_t Size;
785
41
    uint64_t Offset;
786
41
  };
787
788
41
  unsigned PtrSize = CGM.getDataLayout().getPointerSizeInBits();
789
41
  const ASTRecordLayout &Info = Context.getASTRecordLayout(ClassDecl);
790
791
  // Populate sizes and offsets of fields.
792
41
  SmallVector<SizeAndOffset, 16> SSV(Info.getFieldCount());
793
153
  for (unsigned i = 0, e = Info.getFieldCount(); i != e; 
++i112
)
794
112
    SSV[i].Offset =
795
112
        Context.toCharUnitsFromBits(Info.getFieldOffset(i)).getQuantity();
796
797
41
  size_t NumFields = 0;
798
112
  for (const auto *Field : ClassDecl->fields()) {
799
112
    const FieldDecl *D = Field;
800
112
    auto FieldInfo = Context.getTypeInfoInChars(D->getType());
801
112
    CharUnits FieldSize = FieldInfo.Width;
802
112
    assert(NumFields < SSV.size());
803
112
    SSV[NumFields].Size = D->isBitField() ? 
00
: FieldSize.getQuantity();
804
112
    NumFields++;
805
112
  }
806
41
  assert(NumFields == SSV.size());
807
41
  if (SSV.size() <= 1) 
return0
;
808
809
  // We will insert calls to __asan_* run-time functions.
810
  // LLVM AddressSanitizer pass may decide to inline them later.
811
41
  llvm::Type *Args[2] = {IntPtrTy, IntPtrTy};
812
41
  llvm::FunctionType *FTy =
813
41
      llvm::FunctionType::get(CGM.VoidTy, Args, false);
814
41
  llvm::FunctionCallee F = CGM.CreateRuntimeFunction(
815
41
      FTy, Prologue ? 
"__asan_poison_intra_object_redzone"22
816
41
                    : 
"__asan_unpoison_intra_object_redzone"19
);
817
818
41
  llvm::Value *ThisPtr = LoadCXXThis();
819
41
  ThisPtr = Builder.CreatePtrToInt(ThisPtr, IntPtrTy);
820
41
  uint64_t TypeSize = Info.getNonVirtualSize().getQuantity();
821
  // For each field check if it has sufficient padding,
822
  // if so (un)poison it with a call.
823
153
  for (size_t i = 0; i < SSV.size(); 
i++112
) {
824
112
    uint64_t AsanAlignment = 8;
825
112
    uint64_t NextField = i == SSV.size() - 1 ? 
TypeSize41
:
SSV[i + 1].Offset71
;
826
112
    uint64_t PoisonSize = NextField - SSV[i].Offset - SSV[i].Size;
827
112
    uint64_t EndOffset = SSV[i].Offset + SSV[i].Size;
828
112
    if (PoisonSize < AsanAlignment || 
!SSV[i].Size94
||
829
112
        
(NextField % AsanAlignment) != 094
)
830
18
      continue;
831
94
    Builder.CreateCall(
832
94
        F, {Builder.CreateAdd(ThisPtr, Builder.getIntN(PtrSize, EndOffset)),
833
94
            Builder.getIntN(PtrSize, PoisonSize)});
834
94
  }
835
41
}
836
837
/// EmitConstructorBody - Emits the body of the current constructor.
838
39.7k
void CodeGenFunction::EmitConstructorBody(FunctionArgList &Args) {
839
39.7k
  EmitAsanPrologueOrEpilogue(true);
840
39.7k
  const CXXConstructorDecl *Ctor = cast<CXXConstructorDecl>(CurGD.getDecl());
841
39.7k
  CXXCtorType CtorType = CurGD.getCtorType();
842
843
39.7k
  assert((CGM.getTarget().getCXXABI().hasConstructorVariants() ||
844
39.7k
          CtorType == Ctor_Complete) &&
845
39.7k
         "can only generate complete ctor for this ABI");
846
847
  // Before we go any further, try the complete->base constructor
848
  // delegation optimization.
849
39.7k
  if (CtorType == Ctor_Complete && 
IsConstructorDelegationValid(Ctor)17.5k
&&
850
39.7k
      
CGM.getTarget().getCXXABI().hasConstructorVariants()16.8k
) {
851
15.9k
    EmitDelegateCXXConstructorCall(Ctor, Ctor_Base, Args, Ctor->getEndLoc());
852
15.9k
    return;
853
15.9k
  }
854
855
23.7k
  const FunctionDecl *Definition = nullptr;
856
23.7k
  Stmt *Body = Ctor->getBody(Definition);
857
23.7k
  assert(Definition == Ctor && "emitting wrong constructor body");
858
859
  // Enter the function-try-block before the constructor prologue if
860
  // applicable.
861
23.7k
  bool IsTryBody = (Body && isa<CXXTryStmt>(Body));
862
23.7k
  if (IsTryBody)
863
2
    EnterCXXTryStmt(*cast<CXXTryStmt>(Body), true);
864
865
23.7k
  incrementProfileCounter(Body);
866
867
23.7k
  RunCleanupsScope RunCleanups(*this);
868
869
  // TODO: in restricted cases, we can emit the vbase initializers of
870
  // a complete ctor and then delegate to the base ctor.
871
872
  // Emit the constructor prologue, i.e. the base and member
873
  // initializers.
874
23.7k
  EmitCtorPrologue(Ctor, CtorType, Args);
875
876
  // Emit the body of the statement.
877
23.7k
  if (IsTryBody)
878
2
    EmitStmt(cast<CXXTryStmt>(Body)->getTryBlock());
879
23.7k
  else if (Body)
880
23.7k
    EmitStmt(Body);
881
882
  // Emit any cleanup blocks associated with the member or base
883
  // initializers, which includes (along the exceptional path) the
884
  // destructors for those members and bases that were fully
885
  // constructed.
886
23.7k
  RunCleanups.ForceCleanup();
887
888
23.7k
  if (IsTryBody)
889
2
    ExitCXXTryStmt(*cast<CXXTryStmt>(Body), true);
890
23.7k
}
891
892
namespace {
893
  /// RAII object to indicate that codegen is copying the value representation
894
  /// instead of the object representation. Useful when copying a struct or
895
  /// class which has uninitialized members and we're only performing
896
  /// lvalue-to-rvalue conversion on the object but not its members.
897
  class CopyingValueRepresentation {
898
  public:
899
    explicit CopyingValueRepresentation(CodeGenFunction &CGF)
900
161
        : CGF(CGF), OldSanOpts(CGF.SanOpts) {
901
161
      CGF.SanOpts.set(SanitizerKind::Bool, false);
902
161
      CGF.SanOpts.set(SanitizerKind::Enum, false);
903
161
    }
904
161
    ~CopyingValueRepresentation() {
905
161
      CGF.SanOpts = OldSanOpts;
906
161
    }
907
  private:
908
    CodeGenFunction &CGF;
909
    SanitizerSet OldSanOpts;
910
  };
911
} // end anonymous namespace
912
913
namespace {
914
  class FieldMemcpyizer {
915
  public:
916
    FieldMemcpyizer(CodeGenFunction &CGF, const CXXRecordDecl *ClassDecl,
917
                    const VarDecl *SrcRec)
918
      : CGF(CGF), ClassDecl(ClassDecl), SrcRec(SrcRec),
919
        RecLayout(CGF.getContext().getASTRecordLayout(ClassDecl)),
920
        FirstField(nullptr), LastField(nullptr), FirstFieldOffset(0),
921
24.4k
        LastFieldOffset(0), LastAddedFieldIndex(0) {}
922
923
538
    bool isMemcpyableField(FieldDecl *F) const {
924
      // Never memcpy fields when we are adding poisoned paddings.
925
538
      if (CGF.getContext().getLangOpts().SanitizeAddressFieldPadding)
926
8
        return false;
927
530
      Qualifiers Qual = F->getType().getQualifiers();
928
530
      if (Qual.hasVolatile() || 
Qual.hasObjCLifetime()526
)
929
8
        return false;
930
522
      return true;
931
530
    }
932
933
519
    void addMemcpyableField(FieldDecl *F) {
934
519
      if (F->isZeroSize(CGF.getContext()))
935
1
        return;
936
518
      if (!FirstField)
937
257
        addInitialField(F);
938
261
      else
939
261
        addNextField(F);
940
518
    }
941
942
96
    CharUnits getMemcpySize(uint64_t FirstByteOffset) const {
943
96
      ASTContext &Ctx = CGF.getContext();
944
96
      unsigned LastFieldSize =
945
96
          LastField->isBitField()
946
96
              ? 
LastField->getBitWidthValue(Ctx)6
947
96
              : Ctx.toBits(
948
90
                    Ctx.getTypeInfoDataSizeInChars(LastField->getType()).Width);
949
96
      uint64_t MemcpySizeBits = LastFieldOffset + LastFieldSize -
950
96
                                FirstByteOffset + Ctx.getCharWidth() - 1;
951
96
      CharUnits MemcpySize = Ctx.toCharUnitsFromBits(MemcpySizeBits);
952
96
      return MemcpySize;
953
96
    }
954
955
1.82k
    void emitMemcpy() {
956
      // Give the subclass a chance to bail out if it feels the memcpy isn't
957
      // worth it (e.g. Hasn't aggregated enough data).
958
1.82k
      if (!FirstField) {
959
1.73k
        return;
960
1.73k
      }
961
962
96
      uint64_t FirstByteOffset;
963
96
      if (FirstField->isBitField()) {
964
12
        const CGRecordLayout &RL =
965
12
          CGF.getTypes().getCGRecordLayout(FirstField->getParent());
966
12
        const CGBitFieldInfo &BFInfo = RL.getBitFieldInfo(FirstField);
967
        // FirstFieldOffset is not appropriate for bitfields,
968
        // we need to use the storage offset instead.
969
12
        FirstByteOffset = CGF.getContext().toBits(BFInfo.StorageOffset);
970
84
      } else {
971
84
        FirstByteOffset = FirstFieldOffset;
972
84
      }
973
974
96
      CharUnits MemcpySize = getMemcpySize(FirstByteOffset);
975
96
      QualType RecordTy = CGF.getContext().getTypeDeclType(ClassDecl);
976
96
      Address ThisPtr = CGF.LoadCXXThisAddress();
977
96
      LValue DestLV = CGF.MakeAddrLValue(ThisPtr, RecordTy);
978
96
      LValue Dest = CGF.EmitLValueForFieldInitialization(DestLV, FirstField);
979
96
      llvm::Value *SrcPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(SrcRec));
980
96
      LValue SrcLV = CGF.MakeNaturalAlignAddrLValue(SrcPtr, RecordTy);
981
96
      LValue Src = CGF.EmitLValueForFieldInitialization(SrcLV, FirstField);
982
983
96
      emitMemcpyIR(
984
96
          Dest.isBitField() ? 
Dest.getBitFieldAddress()12
:
Dest.getAddress(CGF)84
,
985
96
          Src.isBitField() ? 
Src.getBitFieldAddress()12
:
Src.getAddress(CGF)84
,
986
96
          MemcpySize);
987
96
      reset();
988
96
    }
989
990
41.8k
    void reset() {
991
41.8k
      FirstField = nullptr;
992
41.8k
    }
993
994
  protected:
995
    CodeGenFunction &CGF;
996
    const CXXRecordDecl *ClassDecl;
997
998
  private:
999
96
    void emitMemcpyIR(Address DestPtr, Address SrcPtr, CharUnits Size) {
1000
96
      DestPtr = CGF.Builder.CreateElementBitCast(DestPtr, CGF.Int8Ty);
1001
96
      SrcPtr = CGF.Builder.CreateElementBitCast(SrcPtr, CGF.Int8Ty);
1002
96
      CGF.Builder.CreateMemCpy(DestPtr, SrcPtr, Size.getQuantity());
1003
96
    }
1004
1005
257
    void addInitialField(FieldDecl *F) {
1006
257
      FirstField = F;
1007
257
      LastField = F;
1008
257
      FirstFieldOffset = RecLayout.getFieldOffset(F->getFieldIndex());
1009
257
      LastFieldOffset = FirstFieldOffset;
1010
257
      LastAddedFieldIndex = F->getFieldIndex();
1011
257
    }
1012
1013
261
    void addNextField(FieldDecl *F) {
1014
      // For the most part, the following invariant will hold:
1015
      //   F->getFieldIndex() == LastAddedFieldIndex + 1
1016
      // The one exception is that Sema won't add a copy-initializer for an
1017
      // unnamed bitfield, which will show up here as a gap in the sequence.
1018
261
      assert(F->getFieldIndex() >= LastAddedFieldIndex + 1 &&
1019
261
             "Cannot aggregate fields out of order.");
1020
0
      LastAddedFieldIndex = F->getFieldIndex();
1021
1022
      // The 'first' and 'last' fields are chosen by offset, rather than field
1023
      // index. This allows the code to support bitfields, as well as regular
1024
      // fields.
1025
261
      uint64_t FOffset = RecLayout.getFieldOffset(F->getFieldIndex());
1026
261
      if (FOffset < FirstFieldOffset) {
1027
0
        FirstField = F;
1028
0
        FirstFieldOffset = FOffset;
1029
261
      } else if (FOffset >= LastFieldOffset) {
1030
261
        LastField = F;
1031
261
        LastFieldOffset = FOffset;
1032
261
      }
1033
261
    }
1034
1035
    const VarDecl *SrcRec;
1036
    const ASTRecordLayout &RecLayout;
1037
    FieldDecl *FirstField;
1038
    FieldDecl *LastField;
1039
    uint64_t FirstFieldOffset, LastFieldOffset;
1040
    unsigned LastAddedFieldIndex;
1041
  };
1042
1043
  class ConstructorMemcpyizer : public FieldMemcpyizer {
1044
  private:
1045
    /// Get source argument for copy constructor. Returns null if not a copy
1046
    /// constructor.
1047
    static const VarDecl *getTrivialCopySource(CodeGenFunction &CGF,
1048
                                               const CXXConstructorDecl *CD,
1049
23.7k
                                               FunctionArgList &Args) {
1050
23.7k
      if (CD->isCopyOrMoveConstructor() && 
CD->isDefaulted()1.07k
)
1051
350
        return Args[CGF.CGM.getCXXABI().getSrcArgforCopyCtor(CD, Args)];
1052
23.3k
      return nullptr;
1053
23.7k
    }
1054
1055
    // Returns true if a CXXCtorInitializer represents a member initialization
1056
    // that can be rolled into a memcpy.
1057
16.5k
    bool isMemberInitMemcpyable(CXXCtorInitializer *MemberInit) const {
1058
16.5k
      if (!MemcpyableCtor)
1059
16.1k
        return false;
1060
403
      FieldDecl *Field = MemberInit->getMember();
1061
403
      assert(Field && "No field for member init.");
1062
0
      QualType FieldType = Field->getType();
1063
403
      CXXConstructExpr *CE = dyn_cast<CXXConstructExpr>(MemberInit->getInit());
1064
1065
      // Bail out on non-memcpyable, not-trivially-copyable members.
1066
403
      if (!(CE && 
isMemcpyEquivalentSpecialMember(CE->getConstructor())117
) &&
1067
403
          
!(385
FieldType.isTriviallyCopyableType(CGF.getContext())385
||
1068
385
            
FieldType->isReferenceType()149
))
1069
141
        return false;
1070
1071
      // Bail out on volatile fields.
1072
262
      if (!isMemcpyableField(Field))
1073
6
        return false;
1074
1075
      // Otherwise we're good.
1076
256
      return true;
1077
262
    }
1078
1079
  public:
1080
    ConstructorMemcpyizer(CodeGenFunction &CGF, const CXXConstructorDecl *CD,
1081
                          FunctionArgList &Args)
1082
      : FieldMemcpyizer(CGF, CD->getParent(), getTrivialCopySource(CGF, CD, Args)),
1083
        ConstructorDecl(CD),
1084
        MemcpyableCtor(CD->isDefaulted() &&
1085
                       CD->isCopyOrMoveConstructor() &&
1086
                       CGF.getLangOpts().getGC() == LangOptions::NonGC),
1087
23.7k
        Args(Args) { }
1088
1089
16.5k
    void addMemberInitializer(CXXCtorInitializer *MemberInit) {
1090
16.5k
      if (isMemberInitMemcpyable(MemberInit)) {
1091
256
        AggregatedInits.push_back(MemberInit);
1092
256
        addMemcpyableField(MemberInit->getMember());
1093
16.3k
      } else {
1094
16.3k
        emitAggregatedInits();
1095
16.3k
        EmitMemberInitializer(CGF, ConstructorDecl->getParent(), MemberInit,
1096
16.3k
                              ConstructorDecl, Args);
1097
16.3k
      }
1098
16.5k
    }
1099
1100
40.0k
    void emitAggregatedInits() {
1101
40.0k
      if (AggregatedInits.size() <= 1) {
1102
        // This memcpy is too small to be worthwhile. Fall back on default
1103
        // codegen.
1104
39.9k
        if (!AggregatedInits.empty()) {
1105
61
          CopyingValueRepresentation CVR(CGF);
1106
61
          EmitMemberInitializer(CGF, ConstructorDecl->getParent(),
1107
61
                                AggregatedInits[0], ConstructorDecl, Args);
1108
61
          AggregatedInits.clear();
1109
61
        }
1110
39.9k
        reset();
1111
39.9k
        return;
1112
39.9k
      }
1113
1114
55
      pushEHDestructors();
1115
55
      emitMemcpy();
1116
55
      AggregatedInits.clear();
1117
55
    }
1118
1119
55
    void pushEHDestructors() {
1120
55
      Address ThisPtr = CGF.LoadCXXThisAddress();
1121
55
      QualType RecordTy = CGF.getContext().getTypeDeclType(ClassDecl);
1122
55
      LValue LHS = CGF.MakeAddrLValue(ThisPtr, RecordTy);
1123
1124
250
      for (unsigned i = 0; i < AggregatedInits.size(); 
++i195
) {
1125
195
        CXXCtorInitializer *MemberInit = AggregatedInits[i];
1126
195
        QualType FieldType = MemberInit->getAnyMember()->getType();
1127
195
        QualType::DestructionKind dtorKind = FieldType.isDestructedType();
1128
195
        if (!CGF.needsEHCleanup(dtorKind))
1129
193
          continue;
1130
2
        LValue FieldLHS = LHS;
1131
2
        EmitLValueForAnyFieldInitialization(CGF, MemberInit, FieldLHS);
1132
2
        CGF.pushEHDestroy(dtorKind, FieldLHS.getAddress(CGF), FieldType);
1133
2
      }
1134
55
    }
1135
1136
23.7k
    void finish() {
1137
23.7k
      emitAggregatedInits();
1138
23.7k
    }
1139
1140
  private:
1141
    const CXXConstructorDecl *ConstructorDecl;
1142
    bool MemcpyableCtor;
1143
    FunctionArgList &Args;
1144
    SmallVector<CXXCtorInitializer*, 16> AggregatedInits;
1145
  };
1146
1147
  class AssignmentMemcpyizer : public FieldMemcpyizer {
1148
  private:
1149
    // Returns the memcpyable field copied by the given statement, if one
1150
    // exists. Otherwise returns null.
1151
1.30k
    FieldDecl *getMemcpyableField(Stmt *S) {
1152
1.30k
      if (!AssignmentsMemcpyable)
1153
11
        return nullptr;
1154
1.29k
      if (BinaryOperator *BO = dyn_cast<BinaryOperator>(S)) {
1155
        // Recognise trivial assignments.
1156
226
        if (BO->getOpcode() != BO_Assign)
1157
0
          return nullptr;
1158
226
        MemberExpr *ME = dyn_cast<MemberExpr>(BO->getLHS());
1159
226
        if (!ME)
1160
0
          return nullptr;
1161
226
        FieldDecl *Field = dyn_cast<FieldDecl>(ME->getMemberDecl());
1162
226
        if (!Field || !isMemcpyableField(Field))
1163
10
          return nullptr;
1164
216
        Stmt *RHS = BO->getRHS();
1165
216
        if (ImplicitCastExpr *EC = dyn_cast<ImplicitCastExpr>(RHS))
1166
216
          RHS = EC->getSubExpr();
1167
216
        if (!RHS)
1168
0
          return nullptr;
1169
216
        if (MemberExpr *ME2 = dyn_cast<MemberExpr>(RHS)) {
1170
214
          if (ME2->getMemberDecl() == Field)
1171
214
            return Field;
1172
214
        }
1173
2
        return nullptr;
1174
1.07k
      } else if (CXXMemberCallExpr *MCE = dyn_cast<CXXMemberCallExpr>(S)) {
1175
313
        CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(MCE->getCalleeDecl());
1176
313
        if (!(MD && isMemcpyEquivalentSpecialMember(MD)))
1177
128
          return nullptr;
1178
185
        MemberExpr *IOA = dyn_cast<MemberExpr>(MCE->getImplicitObjectArgument());
1179
185
        if (!IOA)
1180
146
          return nullptr;
1181
39
        FieldDecl *Field = dyn_cast<FieldDecl>(IOA->getMemberDecl());
1182
39
        if (!Field || !isMemcpyableField(Field))
1183
0
          return nullptr;
1184
39
        MemberExpr *Arg0 = dyn_cast<MemberExpr>(MCE->getArg(0));
1185
39
        if (!Arg0 || 
Field != dyn_cast<FieldDecl>(Arg0->getMemberDecl())38
)
1186
1
          return nullptr;
1187
38
        return Field;
1188
758
      } else if (CallExpr *CE = dyn_cast<CallExpr>(S)) {
1189
11
        FunctionDecl *FD = dyn_cast<FunctionDecl>(CE->getCalleeDecl());
1190
11
        if (!FD || FD->getBuiltinID() != Builtin::BI__builtin_memcpy)
1191
0
          return nullptr;
1192
11
        Expr *DstPtr = CE->getArg(0);
1193
11
        if (ImplicitCastExpr *DC = dyn_cast<ImplicitCastExpr>(DstPtr))
1194
11
          DstPtr = DC->getSubExpr();
1195
11
        UnaryOperator *DUO = dyn_cast<UnaryOperator>(DstPtr);
1196
11
        if (!DUO || DUO->getOpcode() != UO_AddrOf)
1197
0
          return nullptr;
1198
11
        MemberExpr *ME = dyn_cast<MemberExpr>(DUO->getSubExpr());
1199
11
        if (!ME)
1200
0
          return nullptr;
1201
11
        FieldDecl *Field = dyn_cast<FieldDecl>(ME->getMemberDecl());
1202
11
        if (!Field || !isMemcpyableField(Field))
1203
0
          return nullptr;
1204
11
        Expr *SrcPtr = CE->getArg(1);
1205
11
        if (ImplicitCastExpr *SC = dyn_cast<ImplicitCastExpr>(SrcPtr))
1206
11
          SrcPtr = SC->getSubExpr();
1207
11
        UnaryOperator *SUO = dyn_cast<UnaryOperator>(SrcPtr);
1208
11
        if (!SUO || SUO->getOpcode() != UO_AddrOf)
1209
0
          return nullptr;
1210
11
        MemberExpr *ME2 = dyn_cast<MemberExpr>(SUO->getSubExpr());
1211
11
        if (!ME2 || Field != dyn_cast<FieldDecl>(ME2->getMemberDecl()))
1212
0
          return nullptr;
1213
11
        return Field;
1214
11
      }
1215
1216
747
      return nullptr;
1217
1.29k
    }
1218
1219
    bool AssignmentsMemcpyable;
1220
    SmallVector<Stmt*, 16> AggregatedStmts;
1221
1222
  public:
1223
    AssignmentMemcpyizer(CodeGenFunction &CGF, const CXXMethodDecl *AD,
1224
                         FunctionArgList &Args)
1225
      : FieldMemcpyizer(CGF, AD->getParent(), Args[Args.size() - 1]),
1226
728
        AssignmentsMemcpyable(CGF.getLangOpts().getGC() == LangOptions::NonGC) {
1227
728
      assert(Args.size() == 2);
1228
728
    }
1229
1230
1.30k
    void emitAssignment(Stmt *S) {
1231
1.30k
      FieldDecl *F = getMemcpyableField(S);
1232
1.30k
      if (F) {
1233
263
        addMemcpyableField(F);
1234
263
        AggregatedStmts.push_back(S);
1235
1.04k
      } else {
1236
1.04k
        emitAggregatedStmts();
1237
1.04k
        CGF.EmitStmt(S);
1238
1.04k
      }
1239
1.30k
    }
1240
1241
1.77k
    void emitAggregatedStmts() {
1242
1.77k
      if (AggregatedStmts.size() <= 1) {
1243
1.73k
        if (!AggregatedStmts.empty()) {
1244
100
          CopyingValueRepresentation CVR(CGF);
1245
100
          CGF.EmitStmt(AggregatedStmts[0]);
1246
100
        }
1247
1.73k
        reset();
1248
1.73k
      }
1249
1250
1.77k
      emitMemcpy();
1251
1.77k
      AggregatedStmts.clear();
1252
1.77k
    }
1253
1254
728
    void finish() {
1255
728
      emitAggregatedStmts();
1256
728
    }
1257
  };
1258
} // end anonymous namespace
1259
1260
27
static bool isInitializerOfDynamicClass(const CXXCtorInitializer *BaseInit) {
1261
27
  const Type *BaseType = BaseInit->getBaseClass();
1262
27
  const auto *BaseClassDecl =
1263
27
      cast<CXXRecordDecl>(BaseType->castAs<RecordType>()->getDecl());
1264
27
  return BaseClassDecl->isDynamicClass();
1265
27
}
1266
1267
/// EmitCtorPrologue - This routine generates necessary code to initialize
1268
/// base classes and non-static data members belonging to this constructor.
1269
void CodeGenFunction::EmitCtorPrologue(const CXXConstructorDecl *CD,
1270
                                       CXXCtorType CtorType,
1271
23.8k
                                       FunctionArgList &Args) {
1272
23.8k
  if (CD->isDelegatingConstructor())
1273
85
    return EmitDelegatingCXXConstructorCall(CD, Args);
1274
1275
23.7k
  const CXXRecordDecl *ClassDecl = CD->getParent();
1276
1277
23.7k
  CXXConstructorDecl::init_const_iterator B = CD->init_begin(),
1278
23.7k
                                          E = CD->init_end();
1279
1280
  // Virtual base initializers first, if any. They aren't needed if:
1281
  // - This is a base ctor variant
1282
  // - There are no vbases
1283
  // - The class is abstract, so a complete object of it cannot be constructed
1284
  //
1285
  // The check for an abstract class is necessary because sema may not have
1286
  // marked virtual base destructors referenced.
1287
23.7k
  bool ConstructVBases = CtorType != Ctor_Base &&
1288
23.7k
                         
ClassDecl->getNumVBases() != 01.49k
&&
1289
23.7k
                         
!ClassDecl->isAbstract()590
;
1290
1291
  // In the Microsoft C++ ABI, there are no constructor variants. Instead, the
1292
  // constructor of a class with virtual bases takes an additional parameter to
1293
  // conditionally construct the virtual bases. Emit that check here.
1294
23.7k
  llvm::BasicBlock *BaseCtorContinueBB = nullptr;
1295
23.7k
  if (ConstructVBases &&
1296
23.7k
      
!CGM.getTarget().getCXXABI().hasConstructorVariants()587
) {
1297
379
    BaseCtorContinueBB =
1298
379
        CGM.getCXXABI().EmitCtorCompleteObjectHandler(*this, ClassDecl);
1299
379
    assert(BaseCtorContinueBB);
1300
379
  }
1301
1302
0
  llvm::Value *const OldThis = CXXThisValue;
1303
24.6k
  for (; B != E && 
(*B)->isBaseInitializer()19.2k
&&
(*B)->isBaseVirtual()7.28k
;
B++940
) {
1304
940
    if (!ConstructVBases)
1305
165
      continue;
1306
775
    if (CGM.getCodeGenOpts().StrictVTablePointers &&
1307
775
        
CGM.getCodeGenOpts().OptimizationLevel > 03
&&
1308
775
        
isInitializerOfDynamicClass(*B)3
)
1309
1
      CXXThisValue = Builder.CreateLaunderInvariantGroup(LoadCXXThis());
1310
775
    EmitBaseInitializer(*this, ClassDecl, *B);
1311
775
  }
1312
1313
23.7k
  if (BaseCtorContinueBB) {
1314
    // Complete object handler should continue to the remaining initializers.
1315
379
    Builder.CreateBr(BaseCtorContinueBB);
1316
379
    EmitBlock(BaseCtorContinueBB);
1317
379
  }
1318
1319
  // Then, non-virtual base initializers.
1320
32.5k
  for (; B != E && 
(*B)->isBaseInitializer()21.3k
;
B++8.86k
) {
1321
8.86k
    assert(!(*B)->isBaseVirtual());
1322
1323
8.86k
    if (CGM.getCodeGenOpts().StrictVTablePointers &&
1324
8.86k
        
CGM.getCodeGenOpts().OptimizationLevel > 024
&&
1325
8.86k
        
isInitializerOfDynamicClass(*B)24
)
1326
23
      CXXThisValue = Builder.CreateLaunderInvariantGroup(LoadCXXThis());
1327
8.86k
    EmitBaseInitializer(*this, ClassDecl, *B);
1328
8.86k
  }
1329
1330
23.7k
  CXXThisValue = OldThis;
1331
1332
23.7k
  InitializeVTablePointers(ClassDecl);
1333
1334
  // And finally, initialize class members.
1335
23.7k
  FieldConstructionScope FCS(*this, LoadCXXThisAddress());
1336
23.7k
  ConstructorMemcpyizer CM(*this, CD, Args);
1337
40.2k
  for (; B != E; 
B++16.5k
) {
1338
16.5k
    CXXCtorInitializer *Member = (*B);
1339
16.5k
    assert(!Member->isBaseInitializer());
1340
0
    assert(Member->isAnyMemberInitializer() &&
1341
16.5k
           "Delegating initializer on non-delegating constructor");
1342
0
    CM.addMemberInitializer(Member);
1343
16.5k
  }
1344
23.7k
  CM.finish();
1345
23.7k
}
1346
1347
static bool
1348
FieldHasTrivialDestructorBody(ASTContext &Context, const FieldDecl *Field);
1349
1350
static bool
1351
HasTrivialDestructorBody(ASTContext &Context,
1352
                         const CXXRecordDecl *BaseClassDecl,
1353
                         const CXXRecordDecl *MostDerivedClassDecl)
1354
257
{
1355
  // If the destructor is trivial we don't have to check anything else.
1356
257
  if (BaseClassDecl->hasTrivialDestructor())
1357
92
    return true;
1358
1359
165
  if (!BaseClassDecl->getDestructor()->hasTrivialBody())
1360
136
    return false;
1361
1362
  // Check fields.
1363
29
  for (const auto *Field : BaseClassDecl->fields())
1364
26
    if (!FieldHasTrivialDestructorBody(Context, Field))
1365
17
      return false;
1366
1367
  // Check non-virtual bases.
1368
12
  for (const auto &I : BaseClassDecl->bases()) {
1369
6
    if (I.isVirtual())
1370
1
      continue;
1371
1372
5
    const CXXRecordDecl *NonVirtualBase =
1373
5
      cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
1374
5
    if (!HasTrivialDestructorBody(Context, NonVirtualBase,
1375
5
                                  MostDerivedClassDecl))
1376
5
      return false;
1377
5
  }
1378
1379
7
  if (BaseClassDecl == MostDerivedClassDecl) {
1380
    // Check virtual bases.
1381
7
    for (const auto &I : BaseClassDecl->vbases()) {
1382
1
      const CXXRecordDecl *VirtualBase =
1383
1
        cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
1384
1
      if (!HasTrivialDestructorBody(Context, VirtualBase,
1385
1
                                    MostDerivedClassDecl))
1386
1
        return false;
1387
1
    }
1388
7
  }
1389
1390
6
  return true;
1391
7
}
1392
1393
static bool
1394
FieldHasTrivialDestructorBody(ASTContext &Context,
1395
                                          const FieldDecl *Field)
1396
618
{
1397
618
  QualType FieldBaseElementType = Context.getBaseElementType(Field->getType());
1398
1399
618
  const RecordType *RT = FieldBaseElementType->getAs<RecordType>();
1400
618
  if (!RT)
1401
365
    return true;
1402
1403
253
  CXXRecordDecl *FieldClassDecl = cast<CXXRecordDecl>(RT->getDecl());
1404
1405
  // The destructor for an implicit anonymous union member is never invoked.
1406
253
  if (FieldClassDecl->isUnion() && 
FieldClassDecl->isAnonymousStructOrUnion()2
)
1407
2
    return false;
1408
1409
251
  return HasTrivialDestructorBody(Context, FieldClassDecl, FieldClassDecl);
1410
253
}
1411
1412
/// CanSkipVTablePointerInitialization - Check whether we need to initialize
1413
/// any vtable pointers before calling this destructor.
1414
static bool CanSkipVTablePointerInitialization(CodeGenFunction &CGF,
1415
7.99k
                                               const CXXDestructorDecl *Dtor) {
1416
7.99k
  const CXXRecordDecl *ClassDecl = Dtor->getParent();
1417
7.99k
  if (!ClassDecl->isDynamicClass())
1418
7.09k
    return true;
1419
1420
  // For a final class, the vtable pointer is known to already point to the
1421
  // class's vtable.
1422
899
  if (ClassDecl->isEffectivelyFinal())
1423
6
    return true;
1424
1425
893
  if (!Dtor->hasTrivialBody())
1426
66
    return false;
1427
1428
  // Check the fields.
1429
827
  for (const auto *Field : ClassDecl->fields())
1430
330
    if (!FieldHasTrivialDestructorBody(CGF.getContext(), Field))
1431
84
      return false;
1432
1433
743
  return true;
1434
827
}
1435
1436
/// EmitDestructorBody - Emits the body of the current destructor.
1437
16.1k
void CodeGenFunction::EmitDestructorBody(FunctionArgList &Args) {
1438
16.1k
  const CXXDestructorDecl *Dtor = cast<CXXDestructorDecl>(CurGD.getDecl());
1439
16.1k
  CXXDtorType DtorType = CurGD.getDtorType();
1440
1441
  // For an abstract class, non-base destructors are never used (and can't
1442
  // be emitted in general, because vbase dtors may not have been validated
1443
  // by Sema), but the Itanium ABI doesn't make them optional and Clang may
1444
  // in fact emit references to them from other compilations, so emit them
1445
  // as functions containing a trap instruction.
1446
16.1k
  if (DtorType != Dtor_Base && 
Dtor->getParent()->isAbstract()8.20k
) {
1447
58
    llvm::CallInst *TrapCall = EmitTrapCall(llvm::Intrinsic::trap);
1448
58
    TrapCall->setDoesNotReturn();
1449
58
    TrapCall->setDoesNotThrow();
1450
58
    Builder.CreateUnreachable();
1451
58
    Builder.ClearInsertionPoint();
1452
58
    return;
1453
58
  }
1454
1455
16.1k
  Stmt *Body = Dtor->getBody();
1456
16.1k
  if (Body)
1457
16.0k
    incrementProfileCounter(Body);
1458
1459
  // The call to operator delete in a deleting destructor happens
1460
  // outside of the function-try-block, which means it's always
1461
  // possible to delegate the destructor body to the complete
1462
  // destructor.  Do so.
1463
16.1k
  if (DtorType == Dtor_Deleting) {
1464
832
    RunCleanupsScope DtorEpilogue(*this);
1465
832
    EnterDtorCleanups(Dtor, Dtor_Deleting);
1466
832
    if (HaveInsertPoint()) {
1467
828
      QualType ThisTy = Dtor->getThisObjectType();
1468
828
      EmitCXXDestructorCall(Dtor, Dtor_Complete, /*ForVirtualBase=*/false,
1469
828
                            /*Delegating=*/false, LoadCXXThisAddress(), ThisTy);
1470
828
    }
1471
832
    return;
1472
832
  }
1473
1474
  // If the body is a function-try-block, enter the try before
1475
  // anything else.
1476
15.3k
  bool isTryBody = (Body && 
isa<CXXTryStmt>(Body)15.2k
);
1477
15.3k
  if (isTryBody)
1478
18
    EnterCXXTryStmt(*cast<CXXTryStmt>(Body), true);
1479
15.3k
  EmitAsanPrologueOrEpilogue(false);
1480
1481
  // Enter the epilogue cleanups.
1482
15.3k
  RunCleanupsScope DtorEpilogue(*this);
1483
1484
  // If this is the complete variant, just invoke the base variant;
1485
  // the epilogue will destruct the virtual bases.  But we can't do
1486
  // this optimization if the body is a function-try-block, because
1487
  // we'd introduce *two* handler blocks.  In the Microsoft ABI, we
1488
  // always delegate because we might not have a definition in this TU.
1489
15.3k
  switch (DtorType) {
1490
0
  case Dtor_Comdat: llvm_unreachable("not expecting a COMDAT");
1491
0
  case Dtor_Deleting: llvm_unreachable("already handled deleting case");
1492
1493
7.31k
  case Dtor_Complete:
1494
7.31k
    assert((Body || getTarget().getCXXABI().isMicrosoft()) &&
1495
7.31k
           "can't emit a dtor without a body for non-Microsoft ABIs");
1496
1497
    // Enter the cleanup scopes for virtual bases.
1498
0
    EnterDtorCleanups(Dtor, Dtor_Complete);
1499
1500
7.31k
    if (!isTryBody) {
1501
7.30k
      QualType ThisTy = Dtor->getThisObjectType();
1502
7.30k
      EmitCXXDestructorCall(Dtor, Dtor_Base, /*ForVirtualBase=*/false,
1503
7.30k
                            /*Delegating=*/false, LoadCXXThisAddress(), ThisTy);
1504
7.30k
      break;
1505
7.30k
    }
1506
1507
    // Fallthrough: act like we're in the base variant.
1508
7.31k
    
LLVM_FALLTHROUGH8
;8
1509
1510
7.99k
  case Dtor_Base:
1511
7.99k
    assert(Body);
1512
1513
    // Enter the cleanup scopes for fields and non-virtual bases.
1514
0
    EnterDtorCleanups(Dtor, Dtor_Base);
1515
1516
    // Initialize the vtable pointers before entering the body.
1517
7.99k
    if (!CanSkipVTablePointerInitialization(*this, Dtor)) {
1518
      // Insert the llvm.launder.invariant.group intrinsic before initializing
1519
      // the vptrs to cancel any previous assumptions we might have made.
1520
150
      if (CGM.getCodeGenOpts().StrictVTablePointers &&
1521
150
          
CGM.getCodeGenOpts().OptimizationLevel > 02
)
1522
2
        CXXThisValue = Builder.CreateLaunderInvariantGroup(LoadCXXThis());
1523
150
      InitializeVTablePointers(Dtor->getParent());
1524
150
    }
1525
1526
7.99k
    if (isTryBody)
1527
18
      EmitStmt(cast<CXXTryStmt>(Body)->getTryBlock());
1528
7.97k
    else if (Body)
1529
7.97k
      EmitStmt(Body);
1530
0
    else {
1531
0
      assert(Dtor->isImplicit() && "bodyless dtor not implicit");
1532
      // nothing to do besides what's in the epilogue
1533
0
    }
1534
    // -fapple-kext must inline any call to this dtor into
1535
    // the caller's body.
1536
7.99k
    if (getLangOpts().AppleKext)
1537
5
      CurFn->addFnAttr(llvm::Attribute::AlwaysInline);
1538
1539
7.99k
    break;
1540
15.3k
  }
1541
1542
  // Jump out through the epilogue cleanups.
1543
15.3k
  DtorEpilogue.ForceCleanup();
1544
1545
  // Exit the try if applicable.
1546
15.3k
  if (isTryBody)
1547
18
    ExitCXXTryStmt(*cast<CXXTryStmt>(Body), true);
1548
15.3k
}
1549
1550
728
void CodeGenFunction::emitImplicitAssignmentOperatorBody(FunctionArgList &Args) {
1551
728
  const CXXMethodDecl *AssignOp = cast<CXXMethodDecl>(CurGD.getDecl());
1552
728
  const Stmt *RootS = AssignOp->getBody();
1553
728
  assert(isa<CompoundStmt>(RootS) &&
1554
728
         "Body of an implicit assignment operator should be compound stmt.");
1555
0
  const CompoundStmt *RootCS = cast<CompoundStmt>(RootS);
1556
1557
728
  LexicalScope Scope(*this, RootCS->getSourceRange());
1558
1559
728
  incrementProfileCounter(RootCS);
1560
728
  AssignmentMemcpyizer AM(*this, AssignOp, Args);
1561
728
  for (auto *I : RootCS->body())
1562
1.30k
    AM.emitAssignment(I);
1563
728
  AM.finish();
1564
728
}
1565
1566
namespace {
1567
  llvm::Value *LoadThisForDtorDelete(CodeGenFunction &CGF,
1568
845
                                     const CXXDestructorDecl *DD) {
1569
845
    if (Expr *ThisArg = DD->getOperatorDeleteThisArg())
1570
12
      return CGF.EmitScalarExpr(ThisArg);
1571
833
    return CGF.LoadCXXThis();
1572
845
  }
1573
1574
  /// Call the operator delete associated with the current destructor.
1575
  struct CallDtorDelete final : EHScopeStack::Cleanup {
1576
615
    CallDtorDelete() {}
1577
1578
628
    void Emit(CodeGenFunction &CGF, Flags flags) override {
1579
628
      const CXXDestructorDecl *Dtor = cast<CXXDestructorDecl>(CGF.CurCodeDecl);
1580
628
      const CXXRecordDecl *ClassDecl = Dtor->getParent();
1581
628
      CGF.EmitDeleteCall(Dtor->getOperatorDelete(),
1582
628
                         LoadThisForDtorDelete(CGF, Dtor),
1583
628
                         CGF.getContext().getTagDeclType(ClassDecl));
1584
628
    }
1585
  };
1586
1587
  void EmitConditionalDtorDeleteCall(CodeGenFunction &CGF,
1588
                                     llvm::Value *ShouldDeleteCondition,
1589
213
                                     bool ReturnAfterDelete) {
1590
213
    llvm::BasicBlock *callDeleteBB = CGF.createBasicBlock("dtor.call_delete");
1591
213
    llvm::BasicBlock *continueBB = CGF.createBasicBlock("dtor.continue");
1592
213
    llvm::Value *ShouldCallDelete
1593
213
      = CGF.Builder.CreateIsNull(ShouldDeleteCondition);
1594
213
    CGF.Builder.CreateCondBr(ShouldCallDelete, continueBB, callDeleteBB);
1595
1596
213
    CGF.EmitBlock(callDeleteBB);
1597
213
    const CXXDestructorDecl *Dtor = cast<CXXDestructorDecl>(CGF.CurCodeDecl);
1598
213
    const CXXRecordDecl *ClassDecl = Dtor->getParent();
1599
213
    CGF.EmitDeleteCall(Dtor->getOperatorDelete(),
1600
213
                       LoadThisForDtorDelete(CGF, Dtor),
1601
213
                       CGF.getContext().getTagDeclType(ClassDecl));
1602
213
    assert(Dtor->getOperatorDelete()->isDestroyingOperatorDelete() ==
1603
213
               ReturnAfterDelete &&
1604
213
           "unexpected value for ReturnAfterDelete");
1605
213
    if (ReturnAfterDelete)
1606
8
      CGF.EmitBranchThroughCleanup(CGF.ReturnBlock);
1607
205
    else
1608
205
      CGF.Builder.CreateBr(continueBB);
1609
1610
213
    CGF.EmitBlock(continueBB);
1611
213
  }
1612
1613
  struct CallDtorDeleteConditional final : EHScopeStack::Cleanup {
1614
    llvm::Value *ShouldDeleteCondition;
1615
1616
  public:
1617
    CallDtorDeleteConditional(llvm::Value *ShouldDeleteCondition)
1618
205
        : ShouldDeleteCondition(ShouldDeleteCondition) {
1619
205
      assert(ShouldDeleteCondition != nullptr);
1620
205
    }
1621
1622
205
    void Emit(CodeGenFunction &CGF, Flags flags) override {
1623
205
      EmitConditionalDtorDeleteCall(CGF, ShouldDeleteCondition,
1624
205
                                    /*ReturnAfterDelete*/false);
1625
205
    }
1626
  };
1627
1628
  class DestroyField  final : public EHScopeStack::Cleanup {
1629
    const FieldDecl *field;
1630
    CodeGenFunction::Destroyer *destroyer;
1631
    bool useEHCleanupForArray;
1632
1633
  public:
1634
    DestroyField(const FieldDecl *field, CodeGenFunction::Destroyer *destroyer,
1635
                 bool useEHCleanupForArray)
1636
        : field(field), destroyer(destroyer),
1637
1.03k
          useEHCleanupForArray(useEHCleanupForArray) {}
1638
1639
1.04k
    void Emit(CodeGenFunction &CGF, Flags flags) override {
1640
      // Find the address of the field.
1641
1.04k
      Address thisValue = CGF.LoadCXXThisAddress();
1642
1.04k
      QualType RecordTy = CGF.getContext().getTagDeclType(field->getParent());
1643
1.04k
      LValue ThisLV = CGF.MakeAddrLValue(thisValue, RecordTy);
1644
1.04k
      LValue LV = CGF.EmitLValueForField(ThisLV, field);
1645
1.04k
      assert(LV.isSimple());
1646
1647
0
      CGF.emitDestroy(LV.getAddress(CGF), field->getType(), destroyer,
1648
1.04k
                      flags.isForNormalCleanup() && 
useEHCleanupForArray1.03k
);
1649
1.04k
    }
1650
  };
1651
1652
  static void EmitSanitizerDtorCallback(CodeGenFunction &CGF, llvm::Value *Ptr,
1653
134
                                        CharUnits::QuantityType PoisonSize) {
1654
134
    CodeGenFunction::SanitizerScope SanScope(&CGF);
1655
    // Pass in void pointer and size of region as arguments to runtime
1656
    // function
1657
134
    llvm::Value *Args[] = {CGF.Builder.CreateBitCast(Ptr, CGF.VoidPtrTy),
1658
134
                           llvm::ConstantInt::get(CGF.SizeTy, PoisonSize)};
1659
1660
134
    llvm::Type *ArgTypes[] = {CGF.VoidPtrTy, CGF.SizeTy};
1661
1662
134
    llvm::FunctionType *FnType =
1663
134
        llvm::FunctionType::get(CGF.VoidTy, ArgTypes, false);
1664
134
    llvm::FunctionCallee Fn =
1665
134
        CGF.CGM.CreateRuntimeFunction(FnType, "__sanitizer_dtor_callback");
1666
134
    CGF.EmitNounwindRuntimeCall(Fn, Args);
1667
134
  }
1668
1669
  /// Poison base class with a trivial destructor.
1670
  struct SanitizeDtorTrivialBase final : EHScopeStack::Cleanup {
1671
    const CXXRecordDecl *BaseClass;
1672
    bool BaseIsVirtual;
1673
    SanitizeDtorTrivialBase(const CXXRecordDecl *Base, bool BaseIsVirtual)
1674
2
        : BaseClass(Base), BaseIsVirtual(BaseIsVirtual) {}
1675
1676
2
    void Emit(CodeGenFunction &CGF, Flags flags) override {
1677
2
      const CXXRecordDecl *DerivedClass =
1678
2
          cast<CXXMethodDecl>(CGF.CurCodeDecl)->getParent();
1679
1680
2
      Address Addr = CGF.GetAddressOfDirectBaseInCompleteClass(
1681
2
          CGF.LoadCXXThisAddress(), DerivedClass, BaseClass, BaseIsVirtual);
1682
1683
2
      const ASTRecordLayout &BaseLayout =
1684
2
          CGF.getContext().getASTRecordLayout(BaseClass);
1685
2
      CharUnits BaseSize = BaseLayout.getSize();
1686
1687
2
      if (!BaseSize.isPositive())
1688
0
        return;
1689
1690
2
      EmitSanitizerDtorCallback(CGF, Addr.getPointer(), BaseSize.getQuantity());
1691
1692
      // Prevent the current stack frame from disappearing from the stack trace.
1693
2
      CGF.CurFn->addFnAttr("disable-tail-calls", "true");
1694
2
    }
1695
  };
1696
1697
  class SanitizeDtorFieldRange final : public EHScopeStack::Cleanup {
1698
    const CXXDestructorDecl *Dtor;
1699
    unsigned StartIndex;
1700
    unsigned EndIndex;
1701
1702
  public:
1703
    SanitizeDtorFieldRange(const CXXDestructorDecl *Dtor, unsigned StartIndex,
1704
                           unsigned EndIndex)
1705
114
        : Dtor(Dtor), StartIndex(StartIndex), EndIndex(EndIndex) {}
1706
1707
    // Generate function call for handling object poisoning.
1708
    // Disables tail call elimination, to prevent the current stack frame
1709
    // from disappearing from the stack trace.
1710
114
    void Emit(CodeGenFunction &CGF, Flags flags) override {
1711
114
      const ASTContext &Context = CGF.getContext();
1712
114
      const ASTRecordLayout &Layout =
1713
114
          Context.getASTRecordLayout(Dtor->getParent());
1714
1715
      // It's a first trivial field so it should be at the begining of a char,
1716
      // still round up start offset just in case.
1717
114
      CharUnits PoisonStart = Context.toCharUnitsFromBits(
1718
114
          Layout.getFieldOffset(StartIndex) + Context.getCharWidth() - 1);
1719
114
      llvm::ConstantInt *OffsetSizePtr =
1720
114
          llvm::ConstantInt::get(CGF.SizeTy, PoisonStart.getQuantity());
1721
1722
114
      llvm::Value *OffsetPtr = CGF.Builder.CreateGEP(
1723
114
          CGF.Int8Ty,
1724
114
          CGF.Builder.CreateBitCast(CGF.LoadCXXThis(), CGF.Int8PtrTy),
1725
114
          OffsetSizePtr);
1726
1727
114
      CharUnits PoisonEnd;
1728
114
      if (EndIndex >= Layout.getFieldCount()) {
1729
74
        PoisonEnd = Layout.getNonVirtualSize();
1730
74
      } else {
1731
40
        PoisonEnd =
1732
40
            Context.toCharUnitsFromBits(Layout.getFieldOffset(EndIndex));
1733
40
      }
1734
114
      CharUnits PoisonSize = PoisonEnd - PoisonStart;
1735
114
      if (!PoisonSize.isPositive())
1736
0
        return;
1737
1738
114
      EmitSanitizerDtorCallback(CGF, OffsetPtr, PoisonSize.getQuantity());
1739
1740
      // Prevent the current stack frame from disappearing from the stack trace.
1741
114
      CGF.CurFn->addFnAttr("disable-tail-calls", "true");
1742
114
    }
1743
  };
1744
1745
 class SanitizeDtorVTable final : public EHScopeStack::Cleanup {
1746
    const CXXDestructorDecl *Dtor;
1747
1748
  public:
1749
18
    SanitizeDtorVTable(const CXXDestructorDecl *Dtor) : Dtor(Dtor) {}
1750
1751
    // Generate function call for handling vtable pointer poisoning.
1752
18
    void Emit(CodeGenFunction &CGF, Flags flags) override {
1753
18
      assert(Dtor->getParent()->isDynamicClass());
1754
0
      (void)Dtor;
1755
18
      ASTContext &Context = CGF.getContext();
1756
      // Poison vtable and vtable ptr if they exist for this class.
1757
18
      llvm::Value *VTablePtr = CGF.LoadCXXThis();
1758
1759
18
      CharUnits::QuantityType PoisonSize =
1760
18
          Context.toCharUnitsFromBits(CGF.PointerWidthInBits).getQuantity();
1761
      // Pass in void pointer and size of region as arguments to runtime
1762
      // function
1763
18
      EmitSanitizerDtorCallback(CGF, VTablePtr, PoisonSize);
1764
18
    }
1765
 };
1766
1767
 class SanitizeDtorCleanupBuilder {
1768
   ASTContext &Context;
1769
   EHScopeStack &EHStack;
1770
   const CXXDestructorDecl *DD;
1771
   llvm::Optional<unsigned> StartIndex;
1772
1773
 public:
1774
   SanitizeDtorCleanupBuilder(ASTContext &Context, EHScopeStack &EHStack,
1775
                              const CXXDestructorDecl *DD)
1776
7.99k
       : Context(Context), EHStack(EHStack), DD(DD), StartIndex(llvm::None) {}
1777
300
   void PushCleanupForField(const FieldDecl *Field) {
1778
300
     if (Field->isZeroSize(Context))
1779
38
       return;
1780
262
     unsigned FieldIndex = Field->getFieldIndex();
1781
262
     if (FieldHasTrivialDestructorBody(Context, Field)) {
1782
208
       if (!StartIndex)
1783
114
         StartIndex = FieldIndex;
1784
208
     } else 
if (54
StartIndex54
) {
1785
40
       EHStack.pushCleanup<SanitizeDtorFieldRange>(
1786
40
           NormalAndEHCleanup, DD, StartIndex.getValue(), FieldIndex);
1787
40
       StartIndex = None;
1788
40
     }
1789
262
   }
1790
90
   void End() {
1791
90
     if (StartIndex)
1792
74
       EHStack.pushCleanup<SanitizeDtorFieldRange>(NormalAndEHCleanup, DD,
1793
74
                                                   StartIndex.getValue(), -1);
1794
90
   }
1795
 };
1796
} // end anonymous namespace
1797
1798
/// Emit all code that comes at the end of class's
1799
/// destructor. This is to call destructors on members and base classes
1800
/// in reverse order of their construction.
1801
///
1802
/// For a deleting destructor, this also handles the case where a destroying
1803
/// operator delete completely overrides the definition.
1804
void CodeGenFunction::EnterDtorCleanups(const CXXDestructorDecl *DD,
1805
16.1k
                                        CXXDtorType DtorType) {
1806
16.1k
  assert((!DD->isTrivial() || DD->hasAttr<DLLExportAttr>()) &&
1807
16.1k
         "Should not emit dtor epilogue for non-exported trivial dtor!");
1808
1809
  // The deleting-destructor phase just needs to call the appropriate
1810
  // operator delete that Sema picked up.
1811
16.1k
  if (DtorType == Dtor_Deleting) {
1812
832
    assert(DD->getOperatorDelete() &&
1813
832
           "operator delete missing - EnterDtorCleanups");
1814
832
    if (CXXStructorImplicitParamValue) {
1815
      // If there is an implicit param to the deleting dtor, it's a boolean
1816
      // telling whether this is a deleting destructor.
1817
213
      if (DD->getOperatorDelete()->isDestroyingOperatorDelete())
1818
8
        EmitConditionalDtorDeleteCall(*this, CXXStructorImplicitParamValue,
1819
8
                                      /*ReturnAfterDelete*/true);
1820
205
      else
1821
205
        EHStack.pushCleanup<CallDtorDeleteConditional>(
1822
205
            NormalAndEHCleanup, CXXStructorImplicitParamValue);
1823
619
    } else {
1824
619
      if (DD->getOperatorDelete()->isDestroyingOperatorDelete()) {
1825
4
        const CXXRecordDecl *ClassDecl = DD->getParent();
1826
4
        EmitDeleteCall(DD->getOperatorDelete(),
1827
4
                       LoadThisForDtorDelete(*this, DD),
1828
4
                       getContext().getTagDeclType(ClassDecl));
1829
4
        EmitBranchThroughCleanup(ReturnBlock);
1830
615
      } else {
1831
615
        EHStack.pushCleanup<CallDtorDelete>(NormalAndEHCleanup);
1832
615
      }
1833
619
    }
1834
832
    return;
1835
832
  }
1836
1837
15.3k
  const CXXRecordDecl *ClassDecl = DD->getParent();
1838
1839
  // Unions have no bases and do not call field destructors.
1840
15.3k
  if (ClassDecl->isUnion())
1841
4
    return;
1842
1843
  // The complete-destructor phase just destructs all the virtual bases.
1844
15.3k
  if (DtorType == Dtor_Complete) {
1845
    // Poison the vtable pointer such that access after the base
1846
    // and member destructors are invoked is invalid.
1847
7.31k
    if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
1848
7.31k
        
SanOpts.has(SanitizerKind::Memory)91
&&
ClassDecl->getNumVBases()90
&&
1849
7.31k
        
ClassDecl->isPolymorphic()4
)
1850
4
      EHStack.pushCleanup<SanitizeDtorVTable>(NormalAndEHCleanup, DD);
1851
1852
    // We push them in the forward order so that they'll be popped in
1853
    // the reverse order.
1854
7.31k
    for (const auto &Base : ClassDecl->vbases()) {
1855
220
      auto *BaseClassDecl =
1856
220
          cast<CXXRecordDecl>(Base.getType()->castAs<RecordType>()->getDecl());
1857
1858
220
      if (BaseClassDecl->hasTrivialDestructor()) {
1859
        // Under SanitizeMemoryUseAfterDtor, poison the trivial base class
1860
        // memory. For non-trival base classes the same is done in the class
1861
        // destructor.
1862
43
        if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
1863
43
            
SanOpts.has(SanitizerKind::Memory)0
&&
!BaseClassDecl->isEmpty()0
)
1864
0
          EHStack.pushCleanup<SanitizeDtorTrivialBase>(NormalAndEHCleanup,
1865
0
                                                       BaseClassDecl,
1866
0
                                                       /*BaseIsVirtual*/ true);
1867
177
      } else {
1868
177
        EHStack.pushCleanup<CallBaseDtor>(NormalAndEHCleanup, BaseClassDecl,
1869
177
                                          /*BaseIsVirtual*/ true);
1870
177
      }
1871
220
    }
1872
1873
7.31k
    return;
1874
7.31k
  }
1875
1876
7.99k
  assert(DtorType == Dtor_Base);
1877
  // Poison the vtable pointer if it has no virtual bases, but inherits
1878
  // virtual functions.
1879
7.99k
  if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
1880
7.99k
      
SanOpts.has(SanitizerKind::Memory)91
&&
!ClassDecl->getNumVBases()90
&&
1881
7.99k
      
ClassDecl->isPolymorphic()86
)
1882
14
    EHStack.pushCleanup<SanitizeDtorVTable>(NormalAndEHCleanup, DD);
1883
1884
  // Destroy non-virtual bases.
1885
7.99k
  for (const auto &Base : ClassDecl->bases()) {
1886
    // Ignore virtual bases.
1887
1.48k
    if (Base.isVirtual())
1888
119
      continue;
1889
1890
1.36k
    CXXRecordDecl *BaseClassDecl = Base.getType()->getAsCXXRecordDecl();
1891
1892
1.36k
    if (BaseClassDecl->hasTrivialDestructor()) {
1893
368
      if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
1894
368
          
SanOpts.has(SanitizerKind::Memory)2
&&
!BaseClassDecl->isEmpty()2
)
1895
2
        EHStack.pushCleanup<SanitizeDtorTrivialBase>(NormalAndEHCleanup,
1896
2
                                                     BaseClassDecl,
1897
2
                                                     /*BaseIsVirtual*/ false);
1898
997
    } else {
1899
997
      EHStack.pushCleanup<CallBaseDtor>(NormalAndEHCleanup, BaseClassDecl,
1900
997
                                        /*BaseIsVirtual*/ false);
1901
997
    }
1902
1.36k
  }
1903
1904
  // Poison fields such that access after their destructors are
1905
  // invoked, and before the base class destructor runs, is invalid.
1906
7.99k
  bool SanitizeFields = CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
1907
7.99k
                        
SanOpts.has(SanitizerKind::Memory)91
;
1908
7.99k
  SanitizeDtorCleanupBuilder SanitizeBuilder(getContext(), EHStack, DD);
1909
1910
  // Destroy direct fields.
1911
10.0k
  for (const auto *Field : ClassDecl->fields()) {
1912
10.0k
    if (SanitizeFields)
1913
300
      SanitizeBuilder.PushCleanupForField(Field);
1914
1915
10.0k
    QualType type = Field->getType();
1916
10.0k
    QualType::DestructionKind dtorKind = type.isDestructedType();
1917
10.0k
    if (!dtorKind)
1918
9.00k
      continue;
1919
1920
    // Anonymous union members do not have their destructors called.
1921
1.04k
    const RecordType *RT = type->getAsUnionType();
1922
1.04k
    if (RT && 
RT->getDecl()->isAnonymousStructOrUnion()9
)
1923
9
      continue;
1924
1925
1.03k
    CleanupKind cleanupKind = getCleanupKind(dtorKind);
1926
1.03k
    EHStack.pushCleanup<DestroyField>(
1927
1.03k
        cleanupKind, Field, getDestroyer(dtorKind), cleanupKind & EHCleanup);
1928
1.03k
  }
1929
1930
7.99k
  if (SanitizeFields)
1931
90
    SanitizeBuilder.End();
1932
7.99k
}
1933
1934
/// EmitCXXAggrConstructorCall - Emit a loop to call a particular
1935
/// constructor for each of several members of an array.
1936
///
1937
/// \param ctor the constructor to call for each element
1938
/// \param arrayType the type of the array to initialize
1939
/// \param arrayBegin an arrayType*
1940
/// \param zeroInitialize true if each element should be
1941
///   zero-initialized before it is constructed
1942
void CodeGenFunction::EmitCXXAggrConstructorCall(
1943
    const CXXConstructorDecl *ctor, const ArrayType *arrayType,
1944
    Address arrayBegin, const CXXConstructExpr *E, bool NewPointerIsChecked,
1945
896
    bool zeroInitialize) {
1946
896
  QualType elementType;
1947
896
  llvm::Value *numElements =
1948
896
    emitArrayLength(arrayType, elementType, arrayBegin);
1949
1950
896
  EmitCXXAggrConstructorCall(ctor, numElements, arrayBegin, E,
1951
896
                             NewPointerIsChecked, zeroInitialize);
1952
896
}
1953
1954
/// EmitCXXAggrConstructorCall - Emit a loop to call a particular
1955
/// constructor for each of several members of an array.
1956
///
1957
/// \param ctor the constructor to call for each element
1958
/// \param numElements the number of elements in the array;
1959
///   may be zero
1960
/// \param arrayBase a T*, where T is the type constructed by ctor
1961
/// \param zeroInitialize true if each element should be
1962
///   zero-initialized before it is constructed
1963
void CodeGenFunction::EmitCXXAggrConstructorCall(const CXXConstructorDecl *ctor,
1964
                                                 llvm::Value *numElements,
1965
                                                 Address arrayBase,
1966
                                                 const CXXConstructExpr *E,
1967
                                                 bool NewPointerIsChecked,
1968
961
                                                 bool zeroInitialize) {
1969
  // It's legal for numElements to be zero.  This can happen both
1970
  // dynamically, because x can be zero in 'new A[x]', and statically,
1971
  // because of GCC extensions that permit zero-length arrays.  There
1972
  // are probably legitimate places where we could assume that this
1973
  // doesn't happen, but it's not clear that it's worth it.
1974
961
  llvm::BranchInst *zeroCheckBranch = nullptr;
1975
1976
  // Optimize for a constant count.
1977
961
  llvm::ConstantInt *constantCount
1978
961
    = dyn_cast<llvm::ConstantInt>(numElements);
1979
961
  if (constantCount) {
1980
    // Just skip out if the constant count is zero.
1981
927
    if (constantCount->isZero()) 
return0
;
1982
1983
  // Otherwise, emit the check.
1984
927
  } else {
1985
34
    llvm::BasicBlock *loopBB = createBasicBlock("new.ctorloop");
1986
34
    llvm::Value *iszero = Builder.CreateIsNull(numElements, "isempty");
1987
34
    zeroCheckBranch = Builder.CreateCondBr(iszero, loopBB, loopBB);
1988
34
    EmitBlock(loopBB);
1989
34
  }
1990
1991
  // Find the end of the array.
1992
961
  llvm::Type *elementType = arrayBase.getElementType();
1993
961
  llvm::Value *arrayBegin = arrayBase.getPointer();
1994
961
  llvm::Value *arrayEnd = Builder.CreateInBoundsGEP(
1995
961
      elementType, arrayBegin, numElements, "arrayctor.end");
1996
1997
  // Enter the loop, setting up a phi for the current location to initialize.
1998
961
  llvm::BasicBlock *entryBB = Builder.GetInsertBlock();
1999
961
  llvm::BasicBlock *loopBB = createBasicBlock("arrayctor.loop");
2000
961
  EmitBlock(loopBB);
2001
961
  llvm::PHINode *cur = Builder.CreatePHI(arrayBegin->getType(), 2,
2002
961
                                         "arrayctor.cur");
2003
961
  cur->addIncoming(arrayBegin, entryBB);
2004
2005
  // Inside the loop body, emit the constructor call on the array element.
2006
2007
  // The alignment of the base, adjusted by the size of a single element,
2008
  // provides a conservative estimate of the alignment of every element.
2009
  // (This assumes we never start tracking offsetted alignments.)
2010
  //
2011
  // Note that these are complete objects and so we don't need to
2012
  // use the non-virtual size or alignment.
2013
961
  QualType type = getContext().getTypeDeclType(ctor->getParent());
2014
961
  CharUnits eltAlignment =
2015
961
    arrayBase.getAlignment()
2016
961
             .alignmentOfArrayElement(getContext().getTypeSizeInChars(type));
2017
961
  Address curAddr = Address(cur, elementType, eltAlignment);
2018
2019
  // Zero initialize the storage, if requested.
2020
961
  if (zeroInitialize)
2021
2
    EmitNullInitialization(curAddr, type);
2022
2023
  // C++ [class.temporary]p4:
2024
  // There are two contexts in which temporaries are destroyed at a different
2025
  // point than the end of the full-expression. The first context is when a
2026
  // default constructor is called to initialize an element of an array.
2027
  // If the constructor has one or more default arguments, the destruction of
2028
  // every temporary created in a default argument expression is sequenced
2029
  // before the construction of the next array element, if any.
2030
2031
961
  {
2032
961
    RunCleanupsScope Scope(*this);
2033
2034
    // Evaluate the constructor and its arguments in a regular
2035
    // partial-destroy cleanup.
2036
961
    if (getLangOpts().Exceptions &&
2037
961
        
!ctor->getParent()->hasTrivialDestructor()86
) {
2038
38
      Destroyer *destroyer = destroyCXXObject;
2039
38
      pushRegularPartialArrayCleanup(arrayBegin, cur, type, eltAlignment,
2040
38
                                     *destroyer);
2041
38
    }
2042
961
    auto currAVS = AggValueSlot::forAddr(
2043
961
        curAddr, type.getQualifiers(), AggValueSlot::IsDestructed,
2044
961
        AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased,
2045
961
        AggValueSlot::DoesNotOverlap, AggValueSlot::IsNotZeroed,
2046
961
        NewPointerIsChecked ? 
AggValueSlot::IsSanitizerChecked95
2047
961
                            : 
AggValueSlot::IsNotSanitizerChecked866
);
2048
961
    EmitCXXConstructorCall(ctor, Ctor_Complete, /*ForVirtualBase=*/false,
2049
961
                           /*Delegating=*/false, currAVS, E);
2050
961
  }
2051
2052
  // Go to the next element.
2053
961
  llvm::Value *next = Builder.CreateInBoundsGEP(
2054
961
      elementType, cur, llvm::ConstantInt::get(SizeTy, 1), "arrayctor.next");
2055
961
  cur->addIncoming(next, Builder.GetInsertBlock());
2056
2057
  // Check whether that's the end of the loop.
2058
961
  llvm::Value *done = Builder.CreateICmpEQ(next, arrayEnd, "arrayctor.done");
2059
961
  llvm::BasicBlock *contBB = createBasicBlock("arrayctor.cont");
2060
961
  Builder.CreateCondBr(done, contBB, loopBB);
2061
2062
  // Patch the earlier check to skip over the loop.
2063
961
  if (zeroCheckBranch) 
zeroCheckBranch->setSuccessor(0, contBB)34
;
2064
2065
961
  EmitBlock(contBB);
2066
961
}
2067
2068
void CodeGenFunction::destroyCXXObject(CodeGenFunction &CGF,
2069
                                       Address addr,
2070
19.5k
                                       QualType type) {
2071
19.5k
  const RecordType *rtype = type->castAs<RecordType>();
2072
19.5k
  const CXXRecordDecl *record = cast<CXXRecordDecl>(rtype->getDecl());
2073
19.5k
  const CXXDestructorDecl *dtor = record->getDestructor();
2074
19.5k
  assert(!dtor->isTrivial());
2075
0
  CGF.EmitCXXDestructorCall(dtor, Dtor_Complete, /*for vbase*/ false,
2076
19.5k
                            /*Delegating=*/false, addr, type);
2077
19.5k
}
2078
2079
void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D,
2080
                                             CXXCtorType Type,
2081
                                             bool ForVirtualBase,
2082
                                             bool Delegating,
2083
                                             AggValueSlot ThisAVS,
2084
43.9k
                                             const CXXConstructExpr *E) {
2085
43.9k
  CallArgList Args;
2086
43.9k
  Address This = ThisAVS.getAddress();
2087
43.9k
  LangAS SlotAS = ThisAVS.getQualifiers().getAddressSpace();
2088
43.9k
  QualType ThisType = D->getThisType();
2089
43.9k
  LangAS ThisAS = ThisType.getTypePtr()->getPointeeType().getAddressSpace();
2090
43.9k
  llvm::Value *ThisPtr = This.getPointer();
2091
2092
43.9k
  if (SlotAS != ThisAS) {
2093
29
    unsigned TargetThisAS = getContext().getTargetAddressSpace(ThisAS);
2094
29
    llvm::Type *NewType = llvm::PointerType::getWithSamePointeeType(
2095
29
        This.getType(), TargetThisAS);
2096
29
    ThisPtr = getTargetHooks().performAddrSpaceCast(*this, This.getPointer(),
2097
29
                                                    ThisAS, SlotAS, NewType);
2098
29
  }
2099
2100
  // Push the this ptr.
2101
43.9k
  Args.add(RValue::get(ThisPtr), D->getThisType());
2102
2103
  // If this is a trivial constructor, emit a memcpy now before we lose
2104
  // the alignment information on the argument.
2105
  // FIXME: It would be better to preserve alignment information into CallArg.
2106
43.9k
  if (isMemcpyEquivalentSpecialMember(D)) {
2107
5.81k
    assert(E->getNumArgs() == 1 && "unexpected argcount for trivial ctor");
2108
2109
0
    const Expr *Arg = E->getArg(0);
2110
5.81k
    LValue Src = EmitLValue(Arg);
2111
5.81k
    QualType DestTy = getContext().getTypeDeclType(D->getParent());
2112
5.81k
    LValue Dest = MakeAddrLValue(This, DestTy);
2113
5.81k
    EmitAggregateCopyCtor(Dest, Src, ThisAVS.mayOverlap());
2114
5.81k
    return;
2115
5.81k
  }
2116
2117
  // Add the rest of the user-supplied arguments.
2118
38.1k
  const FunctionProtoType *FPT = D->getType()->castAs<FunctionProtoType>();
2119
38.1k
  EvaluationOrder Order = E->isListInitialization()
2120
38.1k
                              ? 
EvaluationOrder::ForceLeftToRight453
2121
38.1k
                              : 
EvaluationOrder::Default37.6k
;
2122
38.1k
  EmitCallArgs(Args, FPT, E->arguments(), E->getConstructor(),
2123
38.1k
               /*ParamsToSkip*/ 0, Order);
2124
2125
38.1k
  EmitCXXConstructorCall(D, Type, ForVirtualBase, Delegating, This, Args,
2126
38.1k
                         ThisAVS.mayOverlap(), E->getExprLoc(),
2127
38.1k
                         ThisAVS.isSanitizerChecked());
2128
38.1k
}
2129
2130
static bool canEmitDelegateCallArgs(CodeGenFunction &CGF,
2131
                                    const CXXConstructorDecl *Ctor,
2132
226
                                    CXXCtorType Type, CallArgList &Args) {
2133
  // We can't forward a variadic call.
2134
226
  if (Ctor->isVariadic())
2135
31
    return false;
2136
2137
195
  if (CGF.getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) {
2138
    // If the parameters are callee-cleanup, it's not safe to forward.
2139
32
    for (auto *P : Ctor->parameters())
2140
56
      if (P->needsDestruction(CGF.getContext()))
2141
16
        return false;
2142
2143
    // Likewise if they're inalloca.
2144
16
    const CGFunctionInfo &Info =
2145
16
        CGF.CGM.getTypes().arrangeCXXConstructorCall(Args, Ctor, Type, 0, 0);
2146
16
    if (Info.usesInAlloca())
2147
0
      return false;
2148
16
  }
2149
2150
  // Anything else should be OK.
2151
179
  return true;
2152
195
}
2153
2154
void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D,
2155
                                             CXXCtorType Type,
2156
                                             bool ForVirtualBase,
2157
                                             bool Delegating,
2158
                                             Address This,
2159
                                             CallArgList &Args,
2160
                                             AggValueSlot::Overlap_t Overlap,
2161
                                             SourceLocation Loc,
2162
54.4k
                                             bool NewPointerIsChecked) {
2163
54.4k
  const CXXRecordDecl *ClassDecl = D->getParent();
2164
2165
54.4k
  if (!NewPointerIsChecked)
2166
33.3k
    EmitTypeCheck(CodeGenFunction::TCK_ConstructorCall, Loc, This.getPointer(),
2167
33.3k
                  getContext().getRecordType(ClassDecl), CharUnits::Zero());
2168
2169
54.4k
  if (D->isTrivial() && 
D->isDefaultConstructor()56
) {
2170
35
    assert(Args.size() == 1 && "trivial default ctor with args");
2171
0
    return;
2172
35
  }
2173
2174
  // If this is a trivial constructor, just emit what's needed. If this is a
2175
  // union copy constructor, we must emit a memcpy, because the AST does not
2176
  // model that copy.
2177
54.3k
  if (isMemcpyEquivalentSpecialMember(D)) {
2178
18
    assert(Args.size() == 2 && "unexpected argcount for trivial ctor");
2179
2180
0
    QualType SrcTy = D->getParamDecl(0)->getType().getNonReferenceType();
2181
18
    Address Src = Address(Args[1].getRValue(*this).getScalarVal(), ConvertTypeForMem(SrcTy),
2182
18
                                      CGM.getNaturalTypeAlignment(SrcTy));
2183
18
    LValue SrcLVal = MakeAddrLValue(Src, SrcTy);
2184
18
    QualType DestTy = getContext().getTypeDeclType(ClassDecl);
2185
18
    LValue DestLVal = MakeAddrLValue(This, DestTy);
2186
18
    EmitAggregateCopyCtor(DestLVal, SrcLVal, Overlap);
2187
18
    return;
2188
18
  }
2189
2190
54.3k
  bool PassPrototypeArgs = true;
2191
  // Check whether we can actually emit the constructor before trying to do so.
2192
54.3k
  if (auto Inherited = D->getInheritedConstructor()) {
2193
236
    PassPrototypeArgs = getTypes().inheritingCtorHasParams(Inherited, Type);
2194
236
    if (PassPrototypeArgs && 
!canEmitDelegateCallArgs(*this, D, Type, Args)226
) {
2195
47
      EmitInlinedInheritingCXXConstructorCall(D, Type, ForVirtualBase,
2196
47
                                              Delegating, Args);
2197
47
      return;
2198
47
    }
2199
236
  }
2200
2201
  // Insert any ABI-specific implicit constructor arguments.
2202
54.3k
  CGCXXABI::AddedStructorArgCounts ExtraArgs =
2203
54.3k
      CGM.getCXXABI().addImplicitConstructorArgs(*this, D, Type, ForVirtualBase,
2204
54.3k
                                                 Delegating, Args);
2205
2206
  // Emit the call.
2207
54.3k
  llvm::Constant *CalleePtr = CGM.getAddrOfCXXStructor(GlobalDecl(D, Type));
2208
54.3k
  const CGFunctionInfo &Info = CGM.getTypes().arrangeCXXConstructorCall(
2209
54.3k
      Args, D, Type, ExtraArgs.Prefix, ExtraArgs.Suffix, PassPrototypeArgs);
2210
54.3k
  CGCallee Callee = CGCallee::forDirect(CalleePtr, GlobalDecl(D, Type));
2211
54.3k
  EmitCall(Info, Callee, ReturnValueSlot(), Args, nullptr, false, Loc);
2212
2213
  // Generate vtable assumptions if we're constructing a complete object
2214
  // with a vtable.  We don't do this for base subobjects for two reasons:
2215
  // first, it's incorrect for classes with virtual bases, and second, we're
2216
  // about to overwrite the vptrs anyway.
2217
  // We also have to make sure if we can refer to vtable:
2218
  // - Otherwise we can refer to vtable if it's safe to speculatively emit.
2219
  // FIXME: If vtable is used by ctor/dtor, or if vtable is external and we are
2220
  // sure that definition of vtable is not hidden,
2221
  // then we are always safe to refer to it.
2222
  // FIXME: It looks like InstCombine is very inefficient on dealing with
2223
  // assumes. Make assumption loads require -fstrict-vtable-pointers temporarily.
2224
54.3k
  if (CGM.getCodeGenOpts().OptimizationLevel > 0 &&
2225
54.3k
      
ClassDecl->isDynamicClass()1.12k
&&
Type != Ctor_Base604
&&
2226
54.3k
      
CGM.getCXXABI().canSpeculativelyEmitVTable(ClassDecl)341
&&
2227
54.3k
      
CGM.getCodeGenOpts().StrictVTablePointers121
)
2228
56
    EmitVTableAssumptionLoads(ClassDecl, This);
2229
54.3k
}
2230
2231
void CodeGenFunction::EmitInheritedCXXConstructorCall(
2232
    const CXXConstructorDecl *D, bool ForVirtualBase, Address This,
2233
200
    bool InheritedFromVBase, const CXXInheritedCtorInitExpr *E) {
2234
200
  CallArgList Args;
2235
200
  CallArg ThisArg(RValue::get(This.getPointer()), D->getThisType());
2236
2237
  // Forward the parameters.
2238
200
  if (InheritedFromVBase &&
2239
200
      
CGM.getTarget().getCXXABI().hasConstructorVariants()17
) {
2240
    // Nothing to do; this construction is not responsible for constructing
2241
    // the base class containing the inherited constructor.
2242
    // FIXME: Can we just pass undef's for the remaining arguments if we don't
2243
    // have constructor variants?
2244
10
    Args.push_back(ThisArg);
2245
190
  } else if (!CXXInheritedCtorInitExprArgs.empty()) {
2246
    // The inheriting constructor was inlined; just inject its arguments.
2247
47
    assert(CXXInheritedCtorInitExprArgs.size() >= D->getNumParams() &&
2248
47
           "wrong number of parameters for inherited constructor call");
2249
0
    Args = CXXInheritedCtorInitExprArgs;
2250
47
    Args[0] = ThisArg;
2251
143
  } else {
2252
    // The inheriting constructor was not inlined. Emit delegating arguments.
2253
143
    Args.push_back(ThisArg);
2254
143
    const auto *OuterCtor = cast<CXXConstructorDecl>(CurCodeDecl);
2255
143
    assert(OuterCtor->getNumParams() == D->getNumParams());
2256
0
    assert(!OuterCtor->isVariadic() && "should have been inlined");
2257
2258
269
    for (const auto *Param : OuterCtor->parameters()) {
2259
269
      assert(getContext().hasSameUnqualifiedType(
2260
269
          OuterCtor->getParamDecl(Param->getFunctionScopeIndex())->getType(),
2261
269
          Param->getType()));
2262
0
      EmitDelegateCallArg(Args, Param, E->getLocation());
2263
2264
      // Forward __attribute__(pass_object_size).
2265
269
      if (Param->hasAttr<PassObjectSizeAttr>()) {
2266
22
        auto *POSParam = SizeArguments[Param];
2267
22
        assert(POSParam && "missing pass_object_size value for forwarding");
2268
0
        EmitDelegateCallArg(Args, POSParam, E->getLocation());
2269
22
      }
2270
269
    }
2271
143
  }
2272
2273
0
  EmitCXXConstructorCall(D, Ctor_Base, ForVirtualBase, /*Delegating*/false,
2274
200
                         This, Args, AggValueSlot::MayOverlap,
2275
200
                         E->getLocation(), /*NewPointerIsChecked*/true);
2276
200
}
2277
2278
void CodeGenFunction::EmitInlinedInheritingCXXConstructorCall(
2279
    const CXXConstructorDecl *Ctor, CXXCtorType CtorType, bool ForVirtualBase,
2280
47
    bool Delegating, CallArgList &Args) {
2281
47
  GlobalDecl GD(Ctor, CtorType);
2282
47
  InlinedInheritingConstructorScope Scope(*this, GD);
2283
47
  ApplyInlineDebugLocation DebugScope(*this, GD);
2284
47
  RunCleanupsScope RunCleanups(*this);
2285
2286
  // Save the arguments to be passed to the inherited constructor.
2287
47
  CXXInheritedCtorInitExprArgs = Args;
2288
2289
47
  FunctionArgList Params;
2290
47
  QualType RetType = BuildFunctionArgList(CurGD, Params);
2291
47
  FnRetTy = RetType;
2292
2293
  // Insert any ABI-specific implicit constructor arguments.
2294
47
  CGM.getCXXABI().addImplicitConstructorArgs(*this, Ctor, CtorType,
2295
47
                                             ForVirtualBase, Delegating, Args);
2296
2297
  // Emit a simplified prolog. We only need to emit the implicit params.
2298
47
  assert(Args.size() >= Params.size() && "too few arguments for call");
2299
332
  for (unsigned I = 0, N = Args.size(); I != N; 
++I285
) {
2300
285
    if (I < Params.size() && 
isa<ImplicitParamDecl>(Params[I])231
) {
2301
61
      const RValue &RV = Args[I].getRValue(*this);
2302
61
      assert(!RV.isComplex() && "complex indirect params not supported");
2303
61
      ParamValue Val = RV.isScalar()
2304
61
                           ? ParamValue::forDirect(RV.getScalarVal())
2305
61
                           : 
ParamValue::forIndirect(RV.getAggregateAddress())0
;
2306
61
      EmitParmDecl(*Params[I], Val, I + 1);
2307
61
    }
2308
285
  }
2309
2310
  // Create a return value slot if the ABI implementation wants one.
2311
  // FIXME: This is dumb, we should ask the ABI not to try to set the return
2312
  // value instead.
2313
47
  if (!RetType->isVoidType())
2314
28
    ReturnValue = CreateIRTemp(RetType, "retval.inhctor");
2315
2316
47
  CGM.getCXXABI().EmitInstanceFunctionProlog(*this);
2317
47
  CXXThisValue = CXXABIThisValue;
2318
2319
  // Directly emit the constructor initializers.
2320
47
  EmitCtorPrologue(Ctor, CtorType, Params);
2321
47
}
2322
2323
62
void CodeGenFunction::EmitVTableAssumptionLoad(const VPtr &Vptr, Address This) {
2324
62
  llvm::Value *VTableGlobal =
2325
62
      CGM.getCXXABI().getVTableAddressPoint(Vptr.Base, Vptr.VTableClass);
2326
62
  if (!VTableGlobal)
2327
0
    return;
2328
2329
  // We can just use the base offset in the complete class.
2330
62
  CharUnits NonVirtualOffset = Vptr.Base.getBaseOffset();
2331
2332
62
  if (!NonVirtualOffset.isZero())
2333
4
    This =
2334
4
        ApplyNonVirtualAndVirtualOffset(*this, This, NonVirtualOffset, nullptr,
2335
4
                                        Vptr.VTableClass, Vptr.NearestVBase);
2336
2337
62
  llvm::Value *VPtrValue =
2338
62
      GetVTablePtr(This, VTableGlobal->getType(), Vptr.VTableClass);
2339
62
  llvm::Value *Cmp =
2340
62
      Builder.CreateICmpEQ(VPtrValue, VTableGlobal, "cmp.vtables");
2341
62
  Builder.CreateAssumption(Cmp);
2342
62
}
2343
2344
void CodeGenFunction::EmitVTableAssumptionLoads(const CXXRecordDecl *ClassDecl,
2345
56
                                                Address This) {
2346
56
  if (CGM.getCXXABI().doStructorsInitializeVPtrs(ClassDecl))
2347
56
    for (const VPtr &Vptr : getVTablePointers(ClassDecl))
2348
62
      EmitVTableAssumptionLoad(Vptr, This);
2349
56
}
2350
2351
void
2352
CodeGenFunction::EmitSynthesizedCXXCopyCtorCall(const CXXConstructorDecl *D,
2353
                                                Address This, Address Src,
2354
62
                                                const CXXConstructExpr *E) {
2355
62
  const FunctionProtoType *FPT = D->getType()->castAs<FunctionProtoType>();
2356
2357
62
  CallArgList Args;
2358
2359
  // Push the this ptr.
2360
62
  Args.add(RValue::get(This.getPointer()), D->getThisType());
2361
2362
  // Push the src ptr.
2363
62
  QualType QT = *(FPT->param_type_begin());
2364
62
  llvm::Type *t = CGM.getTypes().ConvertType(QT);
2365
62
  llvm::Value *SrcVal = Builder.CreateBitCast(Src.getPointer(), t);
2366
62
  Args.add(RValue::get(SrcVal), QT);
2367
2368
  // Skip over first argument (Src).
2369
62
  EmitCallArgs(Args, FPT, drop_begin(E->arguments(), 1), E->getConstructor(),
2370
62
               /*ParamsToSkip*/ 1);
2371
2372
62
  EmitCXXConstructorCall(D, Ctor_Complete, /*ForVirtualBase*/false,
2373
62
                         /*Delegating*/false, This, Args,
2374
62
                         AggValueSlot::MayOverlap, E->getExprLoc(),
2375
62
                         /*NewPointerIsChecked*/false);
2376
62
}
2377
2378
void
2379
CodeGenFunction::EmitDelegateCXXConstructorCall(const CXXConstructorDecl *Ctor,
2380
                                                CXXCtorType CtorType,
2381
                                                const FunctionArgList &Args,
2382
15.9k
                                                SourceLocation Loc) {
2383
15.9k
  CallArgList DelegateArgs;
2384
2385
15.9k
  FunctionArgList::const_iterator I = Args.begin(), E = Args.end();
2386
15.9k
  assert(I != E && "no parameters to constructor");
2387
2388
  // this
2389
0
  Address This = LoadCXXThisAddress();
2390
15.9k
  DelegateArgs.add(RValue::get(This.getPointer()), (*I)->getType());
2391
15.9k
  ++I;
2392
2393
  // FIXME: The location of the VTT parameter in the parameter list is
2394
  // specific to the Itanium ABI and shouldn't be hardcoded here.
2395
15.9k
  if (CGM.getCXXABI().NeedsVTTParameter(CurGD)) {
2396
0
    assert(I != E && "cannot skip vtt parameter, already done with args");
2397
0
    assert((*I)->getType()->isPointerType() &&
2398
0
           "skipping parameter not of vtt type");
2399
0
    ++I;
2400
0
  }
2401
2402
  // Explicit arguments.
2403
33.4k
  for (; I != E; 
++I17.4k
) {
2404
17.4k
    const VarDecl *param = *I;
2405
    // FIXME: per-argument source location
2406
17.4k
    EmitDelegateCallArg(DelegateArgs, param, Loc);
2407
17.4k
  }
2408
2409
15.9k
  EmitCXXConstructorCall(Ctor, CtorType, /*ForVirtualBase=*/false,
2410
15.9k
                         /*Delegating=*/true, This, DelegateArgs,
2411
15.9k
                         AggValueSlot::MayOverlap, Loc,
2412
15.9k
                         /*NewPointerIsChecked=*/true);
2413
15.9k
}
2414
2415
namespace {
2416
  struct CallDelegatingCtorDtor final : EHScopeStack::Cleanup {
2417
    const CXXDestructorDecl *Dtor;
2418
    Address Addr;
2419
    CXXDtorType Type;
2420
2421
    CallDelegatingCtorDtor(const CXXDestructorDecl *D, Address Addr,
2422
                           CXXDtorType Type)
2423
59
      : Dtor(D), Addr(Addr), Type(Type) {}
2424
2425
4
    void Emit(CodeGenFunction &CGF, Flags flags) override {
2426
      // We are calling the destructor from within the constructor.
2427
      // Therefore, "this" should have the expected type.
2428
4
      QualType ThisTy = Dtor->getThisObjectType();
2429
4
      CGF.EmitCXXDestructorCall(Dtor, Type, /*ForVirtualBase=*/false,
2430
4
                                /*Delegating=*/true, Addr, ThisTy);
2431
4
    }
2432
  };
2433
} // end anonymous namespace
2434
2435
void
2436
CodeGenFunction::EmitDelegatingCXXConstructorCall(const CXXConstructorDecl *Ctor,
2437
85
                                                  const FunctionArgList &Args) {
2438
85
  assert(Ctor->isDelegatingConstructor());
2439
2440
0
  Address ThisPtr = LoadCXXThisAddress();
2441
2442
85
  AggValueSlot AggSlot =
2443
85
    AggValueSlot::forAddr(ThisPtr, Qualifiers(),
2444
85
                          AggValueSlot::IsDestructed,
2445
85
                          AggValueSlot::DoesNotNeedGCBarriers,
2446
85
                          AggValueSlot::IsNotAliased,
2447
85
                          AggValueSlot::MayOverlap,
2448
85
                          AggValueSlot::IsNotZeroed,
2449
                          // Checks are made by the code that calls constructor.
2450
85
                          AggValueSlot::IsSanitizerChecked);
2451
2452
85
  EmitAggExpr(Ctor->init_begin()[0]->getInit(), AggSlot);
2453
2454
85
  const CXXRecordDecl *ClassDecl = Ctor->getParent();
2455
85
  if (CGM.getLangOpts().Exceptions && 
!ClassDecl->hasTrivialDestructor()83
) {
2456
59
    CXXDtorType Type =
2457
59
      CurGD.getCtorType() == Ctor_Complete ? 
Dtor_Complete56
:
Dtor_Base3
;
2458
2459
59
    EHStack.pushCleanup<CallDelegatingCtorDtor>(EHCleanup,
2460
59
                                                ClassDecl->getDestructor(),
2461
59
                                                ThisPtr, Type);
2462
59
  }
2463
85
}
2464
2465
void CodeGenFunction::EmitCXXDestructorCall(const CXXDestructorDecl *DD,
2466
                                            CXXDtorType Type,
2467
                                            bool ForVirtualBase,
2468
                                            bool Delegating, Address This,
2469
29.8k
                                            QualType ThisTy) {
2470
29.8k
  CGM.getCXXABI().EmitDestructorCall(*this, DD, Type, ForVirtualBase,
2471
29.8k
                                     Delegating, This, ThisTy);
2472
29.8k
}
2473
2474
namespace {
2475
  struct CallLocalDtor final : EHScopeStack::Cleanup {
2476
    const CXXDestructorDecl *Dtor;
2477
    Address Addr;
2478
    QualType Ty;
2479
2480
    CallLocalDtor(const CXXDestructorDecl *D, Address Addr, QualType Ty)
2481
10
        : Dtor(D), Addr(Addr), Ty(Ty) {}
2482
2483
10
    void Emit(CodeGenFunction &CGF, Flags flags) override {
2484
10
      CGF.EmitCXXDestructorCall(Dtor, Dtor_Complete,
2485
10
                                /*ForVirtualBase=*/false,
2486
10
                                /*Delegating=*/false, Addr, Ty);
2487
10
    }
2488
  };
2489
} // end anonymous namespace
2490
2491
void CodeGenFunction::PushDestructorCleanup(const CXXDestructorDecl *D,
2492
10
                                            QualType T, Address Addr) {
2493
10
  EHStack.pushCleanup<CallLocalDtor>(NormalAndEHCleanup, D, Addr, T);
2494
10
}
2495
2496
13
void CodeGenFunction::PushDestructorCleanup(QualType T, Address Addr) {
2497
13
  CXXRecordDecl *ClassDecl = T->getAsCXXRecordDecl();
2498
13
  if (!ClassDecl) 
return0
;
2499
13
  if (ClassDecl->hasTrivialDestructor()) 
return3
;
2500
2501
10
  const CXXDestructorDecl *D = ClassDecl->getDestructor();
2502
10
  assert(D && D->isUsed() && "destructor not marked as used!");
2503
0
  PushDestructorCleanup(D, T, Addr);
2504
10
}
2505
2506
3.78k
void CodeGenFunction::InitializeVTablePointer(const VPtr &Vptr) {
2507
  // Compute the address point.
2508
3.78k
  llvm::Value *VTableAddressPoint =
2509
3.78k
      CGM.getCXXABI().getVTableAddressPointInStructor(
2510
3.78k
          *this, Vptr.VTableClass, Vptr.Base, Vptr.NearestVBase);
2511
2512
3.78k
  if (!VTableAddressPoint)
2513
716
    return;
2514
2515
  // Compute where to store the address point.
2516
3.06k
  llvm::Value *VirtualOffset = nullptr;
2517
3.06k
  CharUnits NonVirtualOffset = CharUnits::Zero();
2518
2519
3.06k
  if (CGM.getCXXABI().isVirtualOffsetNeededForVTableField(*this, Vptr)) {
2520
    // We need to use the virtual base offset offset because the virtual base
2521
    // might have a different offset in the most derived class.
2522
2523
353
    VirtualOffset = CGM.getCXXABI().GetVirtualBaseClassOffset(
2524
353
        *this, LoadCXXThisAddress(), Vptr.VTableClass, Vptr.NearestVBase);
2525
353
    NonVirtualOffset = Vptr.OffsetFromNearestVBase;
2526
2.71k
  } else {
2527
    // We can just use the base offset in the complete class.
2528
2.71k
    NonVirtualOffset = Vptr.Base.getBaseOffset();
2529
2.71k
  }
2530
2531
  // Apply the offsets.
2532
3.06k
  Address VTableField = LoadCXXThisAddress();
2533
3.06k
  if (!NonVirtualOffset.isZero() || 
VirtualOffset2.83k
)
2534
554
    VTableField = ApplyNonVirtualAndVirtualOffset(
2535
554
        *this, VTableField, NonVirtualOffset, VirtualOffset, Vptr.VTableClass,
2536
554
        Vptr.NearestVBase);
2537
2538
  // Finally, store the address point. Use the same LLVM types as the field to
2539
  // support optimization.
2540
3.06k
  unsigned GlobalsAS = CGM.getDataLayout().getDefaultGlobalsAddressSpace();
2541
3.06k
  unsigned ProgAS = CGM.getDataLayout().getProgramAddressSpace();
2542
3.06k
  llvm::Type *VTablePtrTy =
2543
3.06k
      llvm::FunctionType::get(CGM.Int32Ty, /*isVarArg=*/true)
2544
3.06k
          ->getPointerTo(ProgAS)
2545
3.06k
          ->getPointerTo(GlobalsAS);
2546
  // vtable field is is derived from `this` pointer, therefore they should be in
2547
  // the same addr space. Note that this might not be LLVM address space 0.
2548
3.06k
  VTableField = Builder.CreateElementBitCast(VTableField, VTablePtrTy);
2549
3.06k
  VTableAddressPoint = Builder.CreateBitCast(VTableAddressPoint, VTablePtrTy);
2550
2551
3.06k
  llvm::StoreInst *Store = Builder.CreateStore(VTableAddressPoint, VTableField);
2552
3.06k
  TBAAAccessInfo TBAAInfo = CGM.getTBAAVTablePtrAccessInfo(VTablePtrTy);
2553
3.06k
  CGM.DecorateInstructionWithTBAA(Store, TBAAInfo);
2554
3.06k
  if (CGM.getCodeGenOpts().OptimizationLevel > 0 &&
2555
3.06k
      
CGM.getCodeGenOpts().StrictVTablePointers410
)
2556
54
    CGM.DecorateInstructionWithInvariantGroup(Store, Vptr.VTableClass);
2557
3.06k
}
2558
2559
CodeGenFunction::VPtrsVector
2560
2.84k
CodeGenFunction::getVTablePointers(const CXXRecordDecl *VTableClass) {
2561
2.84k
  CodeGenFunction::VPtrsVector VPtrsResult;
2562
2.84k
  VisitedVirtualBasesSetTy VBases;
2563
2.84k
  getVTablePointers(BaseSubobject(VTableClass, CharUnits::Zero()),
2564
2.84k
                    /*NearestVBase=*/nullptr,
2565
2.84k
                    /*OffsetFromNearestVBase=*/CharUnits::Zero(),
2566
2.84k
                    /*BaseIsNonVirtualPrimaryBase=*/false, VTableClass, VBases,
2567
2.84k
                    VPtrsResult);
2568
2.84k
  return VPtrsResult;
2569
2.84k
}
2570
2571
void CodeGenFunction::getVTablePointers(BaseSubobject Base,
2572
                                        const CXXRecordDecl *NearestVBase,
2573
                                        CharUnits OffsetFromNearestVBase,
2574
                                        bool BaseIsNonVirtualPrimaryBase,
2575
                                        const CXXRecordDecl *VTableClass,
2576
                                        VisitedVirtualBasesSetTy &VBases,
2577
5.34k
                                        VPtrsVector &Vptrs) {
2578
  // If this base is a non-virtual primary base the address point has already
2579
  // been set.
2580
5.34k
  if (!BaseIsNonVirtualPrimaryBase) {
2581
    // Initialize the vtable pointer for this base.
2582
3.84k
    VPtr Vptr = {Base, NearestVBase, OffsetFromNearestVBase, VTableClass};
2583
3.84k
    Vptrs.push_back(Vptr);
2584
3.84k
  }
2585
2586
5.34k
  const CXXRecordDecl *RD = Base.getBase();
2587
2588
  // Traverse bases.
2589
5.34k
  for (const auto &I : RD->bases()) {
2590
3.25k
    auto *BaseDecl =
2591
3.25k
        cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
2592
2593
    // Ignore classes without a vtable.
2594
3.25k
    if (!BaseDecl->isDynamicClass())
2595
685
      continue;
2596
2597
2.57k
    CharUnits BaseOffset;
2598
2.57k
    CharUnits BaseOffsetFromNearestVBase;
2599
2.57k
    bool BaseDeclIsNonVirtualPrimaryBase;
2600
2601
2.57k
    if (I.isVirtual()) {
2602
      // Check if we've visited this virtual base before.
2603
669
      if (!VBases.insert(BaseDecl).second)
2604
71
        continue;
2605
2606
598
      const ASTRecordLayout &Layout =
2607
598
        getContext().getASTRecordLayout(VTableClass);
2608
2609
598
      BaseOffset = Layout.getVBaseClassOffset(BaseDecl);
2610
598
      BaseOffsetFromNearestVBase = CharUnits::Zero();
2611
598
      BaseDeclIsNonVirtualPrimaryBase = false;
2612
1.90k
    } else {
2613
1.90k
      const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
2614
2615
1.90k
      BaseOffset = Base.getBaseOffset() + Layout.getBaseClassOffset(BaseDecl);
2616
1.90k
      BaseOffsetFromNearestVBase =
2617
1.90k
        OffsetFromNearestVBase + Layout.getBaseClassOffset(BaseDecl);
2618
1.90k
      BaseDeclIsNonVirtualPrimaryBase = Layout.getPrimaryBase() == BaseDecl;
2619
1.90k
    }
2620
2621
2.49k
    getVTablePointers(
2622
2.49k
        BaseSubobject(BaseDecl, BaseOffset),
2623
2.49k
        I.isVirtual() ? 
BaseDecl598
:
NearestVBase1.90k
, BaseOffsetFromNearestVBase,
2624
2.49k
        BaseDeclIsNonVirtualPrimaryBase, VTableClass, VBases, Vptrs);
2625
2.49k
  }
2626
5.34k
}
2627
2628
23.8k
void CodeGenFunction::InitializeVTablePointers(const CXXRecordDecl *RD) {
2629
  // Ignore classes without a vtable.
2630
23.8k
  if (!RD->isDynamicClass())
2631
21.0k
    return;
2632
2633
  // Initialize the vtable pointers for this class and all of its bases.
2634
2.79k
  if (CGM.getCXXABI().doStructorsInitializeVPtrs(RD))
2635
2.78k
    for (const VPtr &Vptr : getVTablePointers(RD))
2636
3.78k
      InitializeVTablePointer(Vptr);
2637
2638
2.79k
  if (RD->getNumVBases())
2639
807
    CGM.getCXXABI().initializeHiddenVirtualInheritanceMembers(*this, RD);
2640
2.79k
}
2641
2642
llvm::Value *CodeGenFunction::GetVTablePtr(Address This,
2643
                                           llvm::Type *VTableTy,
2644
1.71k
                                           const CXXRecordDecl *RD) {
2645
1.71k
  Address VTablePtrSrc = Builder.CreateElementBitCast(This, VTableTy);
2646
1.71k
  llvm::Instruction *VTable = Builder.CreateLoad(VTablePtrSrc, "vtable");
2647
1.71k
  TBAAAccessInfo TBAAInfo = CGM.getTBAAVTablePtrAccessInfo(VTableTy);
2648
1.71k
  CGM.DecorateInstructionWithTBAA(VTable, TBAAInfo);
2649
2650
1.71k
  if (CGM.getCodeGenOpts().OptimizationLevel > 0 &&
2651
1.71k
      
CGM.getCodeGenOpts().StrictVTablePointers207
)
2652
118
    CGM.DecorateInstructionWithInvariantGroup(VTable, RD);
2653
2654
1.71k
  return VTable;
2655
1.71k
}
2656
2657
// If a class has a single non-virtual base and does not introduce or override
2658
// virtual member functions or fields, it will have the same layout as its base.
2659
// This function returns the least derived such class.
2660
//
2661
// Casting an instance of a base class to such a derived class is technically
2662
// undefined behavior, but it is a relatively common hack for introducing member
2663
// functions on class instances with specific properties (e.g. llvm::Operator)
2664
// that works under most compilers and should not have security implications, so
2665
// we allow it by default. It can be disabled with -fsanitize=cfi-cast-strict.
2666
static const CXXRecordDecl *
2667
60
LeastDerivedClassWithSameLayout(const CXXRecordDecl *RD) {
2668
60
  if (!RD->field_empty())
2669
1
    return RD;
2670
2671
59
  if (RD->getNumVBases() != 0)
2672
12
    return RD;
2673
2674
47
  if (RD->getNumBases() != 1)
2675
32
    return RD;
2676
2677
28
  
for (const CXXMethodDecl *MD : RD->methods())15
{
2678
28
    if (MD->isVirtual()) {
2679
      // Virtual member functions are only ok if they are implicit destructors
2680
      // because the implicit destructor will have the same semantics as the
2681
      // base class's destructor if no fields are added.
2682
9
      if (isa<CXXDestructorDecl>(MD) && 
MD->isImplicit()0
)
2683
0
        continue;
2684
9
      return RD;
2685
9
    }
2686
28
  }
2687
2688
6
  return LeastDerivedClassWithSameLayout(
2689
6
      RD->bases_begin()->getType()->getAsCXXRecordDecl());
2690
15
}
2691
2692
void CodeGenFunction::EmitTypeMetadataCodeForVCall(const CXXRecordDecl *RD,
2693
                                                   llvm::Value *VTable,
2694
801
                                                   SourceLocation Loc) {
2695
801
  if (SanOpts.has(SanitizerKind::CFIVCall))
2696
33
    EmitVTablePtrCheckForCall(RD, VTable, CodeGenFunction::CFITCK_VCall, Loc);
2697
768
  else if (CGM.getCodeGenOpts().WholeProgramVTables &&
2698
           // Don't insert type test assumes if we are forcing public std
2699
           // visibility.
2700
768
           
!CGM.HasLTOVisibilityPublicStd(RD)73
) {
2701
70
    llvm::Metadata *MD =
2702
70
        CGM.CreateMetadataIdentifierForType(QualType(RD->getTypeForDecl(), 0));
2703
70
    llvm::Value *TypeId =
2704
70
        llvm::MetadataAsValue::get(CGM.getLLVMContext(), MD);
2705
2706
70
    llvm::Value *CastedVTable = Builder.CreateBitCast(VTable, Int8PtrTy);
2707
70
    llvm::Value *TypeTest =
2708
70
        Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::type_test),
2709
70
                           {CastedVTable, TypeId});
2710
70
    Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::assume), TypeTest);
2711
70
  }
2712
801
}
2713
2714
void CodeGenFunction::EmitVTablePtrCheckForCall(const CXXRecordDecl *RD,
2715
                                                llvm::Value *VTable,
2716
                                                CFITypeCheckKind TCK,
2717
41
                                                SourceLocation Loc) {
2718
41
  if (!SanOpts.has(SanitizerKind::CFICastStrict))
2719
39
    RD = LeastDerivedClassWithSameLayout(RD);
2720
2721
41
  EmitVTablePtrCheck(RD, VTable, TCK, Loc);
2722
41
}
2723
2724
void CodeGenFunction::EmitVTablePtrCheckForCast(QualType T, Address Derived,
2725
                                                bool MayBeNull,
2726
                                                CFITypeCheckKind TCK,
2727
21
                                                SourceLocation Loc) {
2728
21
  if (!getLangOpts().CPlusPlus)
2729
0
    return;
2730
2731
21
  auto *ClassTy = T->getAs<RecordType>();
2732
21
  if (!ClassTy)
2733
0
    return;
2734
2735
21
  const CXXRecordDecl *ClassDecl = cast<CXXRecordDecl>(ClassTy->getDecl());
2736
2737
21
  if (!ClassDecl->isCompleteDefinition() || !ClassDecl->isDynamicClass())
2738
0
    return;
2739
2740
21
  if (!SanOpts.has(SanitizerKind::CFICastStrict))
2741
15
    ClassDecl = LeastDerivedClassWithSameLayout(ClassDecl);
2742
2743
21
  llvm::BasicBlock *ContBlock = nullptr;
2744
2745
21
  if (MayBeNull) {
2746
14
    llvm::Value *DerivedNotNull =
2747
14
        Builder.CreateIsNotNull(Derived.getPointer(), "cast.nonnull");
2748
2749
14
    llvm::BasicBlock *CheckBlock = createBasicBlock("cast.check");
2750
14
    ContBlock = createBasicBlock("cast.cont");
2751
2752
14
    Builder.CreateCondBr(DerivedNotNull, CheckBlock, ContBlock);
2753
2754
14
    EmitBlock(CheckBlock);
2755
14
  }
2756
2757
21
  llvm::Value *VTable;
2758
21
  std::tie(VTable, ClassDecl) =
2759
21
      CGM.getCXXABI().LoadVTablePtr(*this, Derived, ClassDecl);
2760
2761
21
  EmitVTablePtrCheck(ClassDecl, VTable, TCK, Loc);
2762
2763
21
  if (MayBeNull) {
2764
14
    Builder.CreateBr(ContBlock);
2765
14
    EmitBlock(ContBlock);
2766
14
  }
2767
21
}
2768
2769
void CodeGenFunction::EmitVTablePtrCheck(const CXXRecordDecl *RD,
2770
                                         llvm::Value *VTable,
2771
                                         CFITypeCheckKind TCK,
2772
62
                                         SourceLocation Loc) {
2773
62
  if (!CGM.getCodeGenOpts().SanitizeCfiCrossDso &&
2774
62
      
!CGM.HasHiddenLTOVisibility(RD)60
)
2775
0
    return;
2776
2777
62
  SanitizerMask M;
2778
62
  llvm::SanitizerStatKind SSK;
2779
62
  switch (TCK) {
2780
33
  case CFITCK_VCall:
2781
33
    M = SanitizerKind::CFIVCall;
2782
33
    SSK = llvm::SanStat_CFI_VCall;
2783
33
    break;
2784
8
  case CFITCK_NVCall:
2785
8
    M = SanitizerKind::CFINVCall;
2786
8
    SSK = llvm::SanStat_CFI_NVCall;
2787
8
    break;
2788
6
  case CFITCK_DerivedCast:
2789
6
    M = SanitizerKind::CFIDerivedCast;
2790
6
    SSK = llvm::SanStat_CFI_DerivedCast;
2791
6
    break;
2792
15
  case CFITCK_UnrelatedCast:
2793
15
    M = SanitizerKind::CFIUnrelatedCast;
2794
15
    SSK = llvm::SanStat_CFI_UnrelatedCast;
2795
15
    break;
2796
0
  case CFITCK_ICall:
2797
0
  case CFITCK_NVMFCall:
2798
0
  case CFITCK_VMFCall:
2799
0
    llvm_unreachable("unexpected sanitizer kind");
2800
62
  }
2801
2802
62
  std::string TypeName = RD->getQualifiedNameAsString();
2803
62
  if (getContext().getNoSanitizeList().containsType(M, TypeName))
2804
2
    return;
2805
2806
60
  SanitizerScope SanScope(this);
2807
60
  EmitSanitizerStatReport(SSK);
2808
2809
60
  llvm::Metadata *MD =
2810
60
      CGM.CreateMetadataIdentifierForType(QualType(RD->getTypeForDecl(), 0));
2811
60
  llvm::Value *TypeId = llvm::MetadataAsValue::get(getLLVMContext(), MD);
2812
2813
60
  llvm::Value *CastedVTable = Builder.CreateBitCast(VTable, Int8PtrTy);
2814
60
  llvm::Value *TypeTest = Builder.CreateCall(
2815
60
      CGM.getIntrinsic(llvm::Intrinsic::type_test), {CastedVTable, TypeId});
2816
2817
60
  llvm::Constant *StaticData[] = {
2818
60
      llvm::ConstantInt::get(Int8Ty, TCK),
2819
60
      EmitCheckSourceLocation(Loc),
2820
60
      EmitCheckTypeDescriptor(QualType(RD->getTypeForDecl(), 0)),
2821
60
  };
2822
2823
60
  auto CrossDsoTypeId = CGM.CreateCrossDsoCfiTypeId(MD);
2824
60
  if (CGM.getCodeGenOpts().SanitizeCfiCrossDso && 
CrossDsoTypeId2
) {
2825
2
    EmitCfiSlowPathCheck(M, TypeTest, CrossDsoTypeId, CastedVTable, StaticData);
2826
2
    return;
2827
2
  }
2828
2829
58
  if (CGM.getCodeGenOpts().SanitizeTrap.has(M)) {
2830
28
    EmitTrapCheck(TypeTest, SanitizerHandler::CFICheckFail);
2831
28
    return;
2832
28
  }
2833
2834
30
  llvm::Value *AllVtables = llvm::MetadataAsValue::get(
2835
30
      CGM.getLLVMContext(),
2836
30
      llvm::MDString::get(CGM.getLLVMContext(), "all-vtables"));
2837
30
  llvm::Value *ValidVtable = Builder.CreateCall(
2838
30
      CGM.getIntrinsic(llvm::Intrinsic::type_test), {CastedVTable, AllVtables});
2839
30
  EmitCheck(std::make_pair(TypeTest, M), SanitizerHandler::CFICheckFail,
2840
30
            StaticData, {CastedVTable, ValidVtable});
2841
30
}
2842
2843
957
bool CodeGenFunction::ShouldEmitVTableTypeCheckedLoad(const CXXRecordDecl *RD) {
2844
957
  if (!CGM.getCodeGenOpts().WholeProgramVTables ||
2845
957
      
!CGM.HasHiddenLTOVisibility(RD)87
)
2846
895
    return false;
2847
2848
62
  if (CGM.getCodeGenOpts().VirtualFunctionElimination)
2849
2
    return true;
2850
2851
60
  if (!SanOpts.has(SanitizerKind::CFIVCall) ||
2852
60
      
!CGM.getCodeGenOpts().SanitizeTrap.has(SanitizerKind::CFIVCall)12
)
2853
49
    return false;
2854
2855
11
  std::string TypeName = RD->getQualifiedNameAsString();
2856
11
  return !getContext().getNoSanitizeList().containsType(SanitizerKind::CFIVCall,
2857
11
                                                        TypeName);
2858
60
}
2859
2860
llvm::Value *CodeGenFunction::EmitVTableTypeCheckedLoad(
2861
    const CXXRecordDecl *RD, llvm::Value *VTable, llvm::Type *VTableTy,
2862
13
    uint64_t VTableByteOffset) {
2863
13
  SanitizerScope SanScope(this);
2864
2865
13
  EmitSanitizerStatReport(llvm::SanStat_CFI_VCall);
2866
2867
13
  llvm::Metadata *MD =
2868
13
      CGM.CreateMetadataIdentifierForType(QualType(RD->getTypeForDecl(), 0));
2869
13
  llvm::Value *TypeId = llvm::MetadataAsValue::get(CGM.getLLVMContext(), MD);
2870
2871
13
  llvm::Value *CastedVTable = Builder.CreateBitCast(VTable, Int8PtrTy);
2872
13
  llvm::Value *CheckedLoad = Builder.CreateCall(
2873
13
      CGM.getIntrinsic(llvm::Intrinsic::type_checked_load),
2874
13
      {CastedVTable, llvm::ConstantInt::get(Int32Ty, VTableByteOffset),
2875
13
       TypeId});
2876
13
  llvm::Value *CheckResult = Builder.CreateExtractValue(CheckedLoad, 1);
2877
2878
13
  std::string TypeName = RD->getQualifiedNameAsString();
2879
13
  if (SanOpts.has(SanitizerKind::CFIVCall) &&
2880
13
      !getContext().getNoSanitizeList().containsType(SanitizerKind::CFIVCall,
2881
11
                                                     TypeName)) {
2882
11
    EmitCheck(std::make_pair(CheckResult, SanitizerKind::CFIVCall),
2883
11
              SanitizerHandler::CFICheckFail, {}, {});
2884
11
  }
2885
2886
13
  return Builder.CreateBitCast(Builder.CreateExtractValue(CheckedLoad, 0),
2887
13
                               VTableTy);
2888
13
}
2889
2890
void CodeGenFunction::EmitForwardingCallToLambda(
2891
                                      const CXXMethodDecl *callOperator,
2892
73
                                      CallArgList &callArgs) {
2893
  // Get the address of the call operator.
2894
73
  const CGFunctionInfo &calleeFnInfo =
2895
73
    CGM.getTypes().arrangeCXXMethodDeclaration(callOperator);
2896
73
  llvm::Constant *calleePtr =
2897
73
    CGM.GetAddrOfFunction(GlobalDecl(callOperator),
2898
73
                          CGM.getTypes().GetFunctionType(calleeFnInfo));
2899
2900
  // Prepare the return slot.
2901
73
  const FunctionProtoType *FPT =
2902
73
    callOperator->getType()->castAs<FunctionProtoType>();
2903
73
  QualType resultType = FPT->getReturnType();
2904
73
  ReturnValueSlot returnSlot;
2905
73
  if (!resultType->isVoidType() &&
2906
73
      
calleeFnInfo.getReturnInfo().getKind() == ABIArgInfo::Indirect37
&&
2907
73
      
!hasScalarEvaluationKind(calleeFnInfo.getReturnType())1
)
2908
1
    returnSlot =
2909
1
        ReturnValueSlot(ReturnValue, resultType.isVolatileQualified(),
2910
1
                        /*IsUnused=*/false, /*IsExternallyDestructed=*/true);
2911
2912
  // We don't need to separately arrange the call arguments because
2913
  // the call can't be variadic anyway --- it's impossible to forward
2914
  // variadic arguments.
2915
2916
  // Now emit our call.
2917
73
  auto callee = CGCallee::forDirect(calleePtr, GlobalDecl(callOperator));
2918
73
  RValue RV = EmitCall(calleeFnInfo, callee, returnSlot, callArgs);
2919
2920
  // If necessary, copy the returned value into the slot.
2921
73
  if (!resultType->isVoidType() && 
returnSlot.isNull()37
) {
2922
36
    if (getLangOpts().ObjCAutoRefCount && 
resultType->isObjCRetainableType()5
) {
2923
2
      RV = RValue::get(EmitARCRetainAutoreleasedReturnValue(RV.getScalarVal()));
2924
2
    }
2925
36
    EmitReturnOfRValue(RV, resultType);
2926
36
  } else
2927
37
    EmitBranchThroughCleanup(ReturnBlock);
2928
73
}
2929
2930
13
void CodeGenFunction::EmitLambdaBlockInvokeBody() {
2931
13
  const BlockDecl *BD = BlockInfo->getBlockDecl();
2932
13
  const VarDecl *variable = BD->capture_begin()->getVariable();
2933
13
  const CXXRecordDecl *Lambda = variable->getType()->getAsCXXRecordDecl();
2934
13
  const CXXMethodDecl *CallOp = Lambda->getLambdaCallOperator();
2935
2936
13
  if (CallOp->isVariadic()) {
2937
    // FIXME: Making this work correctly is nasty because it requires either
2938
    // cloning the body of the call operator or making the call operator
2939
    // forward.
2940
0
    CGM.ErrorUnsupported(CurCodeDecl, "lambda conversion to variadic function");
2941
0
    return;
2942
0
  }
2943
2944
  // Start building arguments for forwarding call
2945
13
  CallArgList CallArgs;
2946
2947
13
  QualType ThisType = getContext().getPointerType(getContext().getRecordType(Lambda));
2948
13
  Address ThisPtr = GetAddrOfBlockDecl(variable);
2949
13
  CallArgs.add(RValue::get(ThisPtr.getPointer()), ThisType);
2950
2951
  // Add the rest of the parameters.
2952
13
  for (auto param : BD->parameters())
2953
1
    EmitDelegateCallArg(CallArgs, param, param->getBeginLoc());
2954
2955
13
  assert(!Lambda->isGenericLambda() &&
2956
13
            "generic lambda interconversion to block not implemented");
2957
0
  EmitForwardingCallToLambda(CallOp, CallArgs);
2958
13
}
2959
2960
60
void CodeGenFunction::EmitLambdaDelegatingInvokeBody(const CXXMethodDecl *MD) {
2961
60
  const CXXRecordDecl *Lambda = MD->getParent();
2962
2963
  // Start building arguments for forwarding call
2964
60
  CallArgList CallArgs;
2965
2966
60
  QualType ThisType = getContext().getPointerType(getContext().getRecordType(Lambda));
2967
60
  llvm::Value *ThisPtr = llvm::UndefValue::get(getTypes().ConvertType(ThisType));
2968
60
  CallArgs.add(RValue::get(ThisPtr), ThisType);
2969
2970
  // Add the rest of the parameters.
2971
60
  for (auto Param : MD->parameters())
2972
140
    EmitDelegateCallArg(CallArgs, Param, Param->getBeginLoc());
2973
2974
60
  const CXXMethodDecl *CallOp = Lambda->getLambdaCallOperator();
2975
  // For a generic lambda, find the corresponding call operator specialization
2976
  // to which the call to the static-invoker shall be forwarded.
2977
60
  if (Lambda->isGenericLambda()) {
2978
0
    assert(MD->isFunctionTemplateSpecialization());
2979
0
    const TemplateArgumentList *TAL = MD->getTemplateSpecializationArgs();
2980
0
    FunctionTemplateDecl *CallOpTemplate = CallOp->getDescribedFunctionTemplate();
2981
0
    void *InsertPos = nullptr;
2982
0
    FunctionDecl *CorrespondingCallOpSpecialization =
2983
0
        CallOpTemplate->findSpecialization(TAL->asArray(), InsertPos);
2984
0
    assert(CorrespondingCallOpSpecialization);
2985
0
    CallOp = cast<CXXMethodDecl>(CorrespondingCallOpSpecialization);
2986
0
  }
2987
0
  EmitForwardingCallToLambda(CallOp, CallArgs);
2988
60
}
2989
2990
60
void CodeGenFunction::EmitLambdaStaticInvokeBody(const CXXMethodDecl *MD) {
2991
60
  if (MD->isVariadic()) {
2992
    // FIXME: Making this work correctly is nasty because it requires either
2993
    // cloning the body of the call operator or making the call operator forward.
2994
0
    CGM.ErrorUnsupported(MD, "lambda conversion to variadic function");
2995
0
    return;
2996
0
  }
2997
2998
60
  EmitLambdaDelegatingInvokeBody(MD);
2999
60
}