Coverage Report

Created: 2021-01-23 06:44

/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/CodeGen/CGClass.cpp
Line
Count
Source (jump to first uncovered line)
1
//===--- CGClass.cpp - Emit LLVM Code for C++ classes -----------*- C++ -*-===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This contains code dealing with C++ code generation of classes
10
//
11
//===----------------------------------------------------------------------===//
12
13
#include "CGBlocks.h"
14
#include "CGCXXABI.h"
15
#include "CGDebugInfo.h"
16
#include "CGRecordLayout.h"
17
#include "CodeGenFunction.h"
18
#include "TargetInfo.h"
19
#include "clang/AST/Attr.h"
20
#include "clang/AST/CXXInheritance.h"
21
#include "clang/AST/CharUnits.h"
22
#include "clang/AST/DeclTemplate.h"
23
#include "clang/AST/EvaluatedExprVisitor.h"
24
#include "clang/AST/RecordLayout.h"
25
#include "clang/AST/StmtCXX.h"
26
#include "clang/Basic/CodeGenOptions.h"
27
#include "clang/Basic/TargetBuiltins.h"
28
#include "clang/CodeGen/CGFunctionInfo.h"
29
#include "llvm/IR/Intrinsics.h"
30
#include "llvm/IR/Metadata.h"
31
#include "llvm/Transforms/Utils/SanitizerStats.h"
32
33
using namespace clang;
34
using namespace CodeGen;
35
36
/// Return the best known alignment for an unknown pointer to a
37
/// particular class.
38
383k
CharUnits CodeGenModule::getClassPointerAlignment(const CXXRecordDecl *RD) {
39
383k
  if (!RD->hasDefinition())
40
0
    return CharUnits::One(); // Hopefully won't be used anywhere.
41
42
383k
  auto &layout = getContext().getASTRecordLayout(RD);
43
44
  // If the class is final, then we know that the pointer points to an
45
  // object of that type and can use the full alignment.
46
383k
  if (RD->isEffectivelyFinal())
47
92
    return layout.getAlignment();
48
49
  // Otherwise, we have to assume it could be a subclass.
50
383k
  return layout.getNonVirtualAlignment();
51
383k
}
52
53
/// Return the smallest possible amount of storage that might be allocated
54
/// starting from the beginning of an object of a particular class.
55
///
56
/// This may be smaller than sizeof(RD) if RD has virtual base classes.
57
350k
CharUnits CodeGenModule::getMinimumClassObjectSize(const CXXRecordDecl *RD) {
58
350k
  if (!RD->hasDefinition())
59
2
    return CharUnits::One();
60
61
350k
  auto &layout = getContext().getASTRecordLayout(RD);
62
63
  // If the class is final, then we know that the pointer points to an
64
  // object of that type and can use the full alignment.
65
350k
  if (RD->isEffectivelyFinal())
66
73
    return layout.getSize();
67
68
  // Otherwise, we have to assume it could be a subclass.
69
350k
  return std::max(layout.getNonVirtualSize(), CharUnits::One());
70
350k
}
71
72
/// Return the best known alignment for a pointer to a virtual base,
73
/// given the alignment of a pointer to the derived class.
74
CharUnits CodeGenModule::getVBaseAlignment(CharUnits actualDerivedAlign,
75
                                           const CXXRecordDecl *derivedClass,
76
882
                                           const CXXRecordDecl *vbaseClass) {
77
  // The basic idea here is that an underaligned derived pointer might
78
  // indicate an underaligned base pointer.
79
80
882
  assert(vbaseClass->isCompleteDefinition());
81
882
  auto &baseLayout = getContext().getASTRecordLayout(vbaseClass);
82
882
  CharUnits expectedVBaseAlign = baseLayout.getNonVirtualAlignment();
83
84
882
  return getDynamicOffsetAlignment(actualDerivedAlign, derivedClass,
85
882
                                   expectedVBaseAlign);
86
882
}
87
88
CharUnits
89
CodeGenModule::getDynamicOffsetAlignment(CharUnits actualBaseAlign,
90
                                         const CXXRecordDecl *baseDecl,
91
1.05k
                                         CharUnits expectedTargetAlign) {
92
  // If the base is an incomplete type (which is, alas, possible with
93
  // member pointers), be pessimistic.
94
1.05k
  if (!baseDecl->isCompleteDefinition())
95
5
    return std::min(actualBaseAlign, expectedTargetAlign);
96
97
1.04k
  auto &baseLayout = getContext().getASTRecordLayout(baseDecl);
98
1.04k
  CharUnits expectedBaseAlign = baseLayout.getNonVirtualAlignment();
99
100
  // If the class is properly aligned, assume the target offset is, too.
101
  //
102
  // This actually isn't necessarily the right thing to do --- if the
103
  // class is a complete object, but it's only properly aligned for a
104
  // base subobject, then the alignments of things relative to it are
105
  // probably off as well.  (Note that this requires the alignment of
106
  // the target to be greater than the NV alignment of the derived
107
  // class.)
108
  //
109
  // However, our approach to this kind of under-alignment can only
110
  // ever be best effort; after all, we're never going to propagate
111
  // alignments through variables or parameters.  Note, in particular,
112
  // that constructing a polymorphic type in an address that's less
113
  // than pointer-aligned will generally trap in the constructor,
114
  // unless we someday add some sort of attribute to change the
115
  // assumed alignment of 'this'.  So our goal here is pretty much
116
  // just to allow the user to explicitly say that a pointer is
117
  // under-aligned and then safely access its fields and vtables.
118
1.04k
  if (actualBaseAlign >= expectedBaseAlign) {
119
1.04k
    return expectedTargetAlign;
120
1.04k
  }
121
122
  // Otherwise, we might be offset by an arbitrary multiple of the
123
  // actual alignment.  The correct adjustment is to take the min of
124
  // the two alignments.
125
0
  return std::min(actualBaseAlign, expectedTargetAlign);
126
0
}
127
128
64.0k
Address CodeGenFunction::LoadCXXThisAddress() {
129
64.0k
  assert(CurFuncDecl && "loading 'this' without a func declaration?");
130
64.0k
  assert(isa<CXXMethodDecl>(CurFuncDecl));
131
132
  // Lazily compute CXXThisAlignment.
133
64.0k
  if (CXXThisAlignment.isZero()) {
134
    // Just use the best known alignment for the parent.
135
    // TODO: if we're currently emitting a complete-object ctor/dtor,
136
    // we can always use the complete-object alignment.
137
49.5k
    auto RD = cast<CXXMethodDecl>(CurFuncDecl)->getParent();
138
49.5k
    CXXThisAlignment = CGM.getClassPointerAlignment(RD);
139
49.5k
  }
140
141
64.0k
  return Address(LoadCXXThis(), CXXThisAlignment);
142
64.0k
}
143
144
/// Emit the address of a field using a member data pointer.
145
///
146
/// \param E Only used for emergency diagnostics
147
Address
148
CodeGenFunction::EmitCXXMemberDataPointerAddress(const Expr *E, Address base,
149
                                                 llvm::Value *memberPtr,
150
                                      const MemberPointerType *memberPtrType,
151
                                                 LValueBaseInfo *BaseInfo,
152
81
                                                 TBAAAccessInfo *TBAAInfo) {
153
  // Ask the ABI to compute the actual address.
154
81
  llvm::Value *ptr =
155
81
    CGM.getCXXABI().EmitMemberDataPointerAddress(*this, E, base,
156
81
                                                 memberPtr, memberPtrType);
157
158
81
  QualType memberType = memberPtrType->getPointeeType();
159
81
  CharUnits memberAlign =
160
81
      CGM.getNaturalTypeAlignment(memberType, BaseInfo, TBAAInfo);
161
81
  memberAlign =
162
81
    CGM.getDynamicOffsetAlignment(base.getAlignment(),
163
81
                            memberPtrType->getClass()->getAsCXXRecordDecl(),
164
81
                                  memberAlign);
165
81
  return Address(ptr, memberAlign);
166
81
}
167
168
CharUnits CodeGenModule::computeNonVirtualBaseClassOffset(
169
    const CXXRecordDecl *DerivedClass, CastExpr::path_const_iterator Start,
170
21.4k
    CastExpr::path_const_iterator End) {
171
21.4k
  CharUnits Offset = CharUnits::Zero();
172
173
21.4k
  const ASTContext &Context = getContext();
174
21.4k
  const CXXRecordDecl *RD = DerivedClass;
175
176
45.0k
  for (CastExpr::path_const_iterator I = Start; I != End; 
++I23.6k
) {
177
23.6k
    const CXXBaseSpecifier *Base = *I;
178
23.6k
    assert(!Base->isVirtual() && "Should not see virtual bases here!");
179
180
    // Get the layout.
181
23.6k
    const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
182
183
23.6k
    const auto *BaseDecl =
184
23.6k
        cast<CXXRecordDecl>(Base->getType()->castAs<RecordType>()->getDecl());
185
186
    // Add the offset.
187
23.6k
    Offset += Layout.getBaseClassOffset(BaseDecl);
188
189
23.6k
    RD = BaseDecl;
190
23.6k
  }
191
192
21.4k
  return Offset;
193
21.4k
}
194
195
llvm::Constant *
196
CodeGenModule::GetNonVirtualBaseClassOffset(const CXXRecordDecl *ClassDecl,
197
                                   CastExpr::path_const_iterator PathBegin,
198
756
                                   CastExpr::path_const_iterator PathEnd) {
199
756
  assert(PathBegin != PathEnd && "Base path should not be empty!");
200
201
756
  CharUnits Offset =
202
756
      computeNonVirtualBaseClassOffset(ClassDecl, PathBegin, PathEnd);
203
756
  if (Offset.isZero())
204
719
    return nullptr;
205
206
37
  llvm::Type *PtrDiffTy =
207
37
  Types.ConvertType(getContext().getPointerDiffType());
208
209
37
  return llvm::ConstantInt::get(PtrDiffTy, Offset.getQuantity());
210
37
}
211
212
/// Gets the address of a direct base class within a complete object.
213
/// This should only be used for (1) non-virtual bases or (2) virtual bases
214
/// when the type is known to be complete (e.g. in complete destructors).
215
///
216
/// The object pointed to by 'This' is assumed to be non-null.
217
Address
218
CodeGenFunction::GetAddressOfDirectBaseInCompleteClass(Address This,
219
                                                   const CXXRecordDecl *Derived,
220
                                                   const CXXRecordDecl *Base,
221
11.3k
                                                   bool BaseIsVirtual) {
222
  // 'this' must be a pointer (in some address space) to Derived.
223
11.3k
  assert(This.getElementType() == ConvertType(Derived));
224
225
  // Compute the offset of the virtual base.
226
11.3k
  CharUnits Offset;
227
11.3k
  const ASTRecordLayout &Layout = getContext().getASTRecordLayout(Derived);
228
11.3k
  if (BaseIsVirtual)
229
999
    Offset = Layout.getVBaseClassOffset(Base);
230
10.3k
  else
231
10.3k
    Offset = Layout.getBaseClassOffset(Base);
232
233
  // Shift and cast down to the base type.
234
  // TODO: for complete types, this should be possible with a GEP.
235
11.3k
  Address V = This;
236
11.3k
  if (!Offset.isZero()) {
237
1.95k
    V = Builder.CreateElementBitCast(V, Int8Ty);
238
1.95k
    V = Builder.CreateConstInBoundsByteGEP(V, Offset);
239
1.95k
  }
240
11.3k
  V = Builder.CreateElementBitCast(V, ConvertType(Base));
241
242
11.3k
  return V;
243
11.3k
}
244
245
static Address
246
ApplyNonVirtualAndVirtualOffset(CodeGenFunction &CGF, Address addr,
247
                                CharUnits nonVirtualOffset,
248
                                llvm::Value *virtualOffset,
249
                                const CXXRecordDecl *derivedClass,
250
2.03k
                                const CXXRecordDecl *nearestVBase) {
251
  // Assert that we have something to do.
252
2.03k
  assert(!nonVirtualOffset.isZero() || virtualOffset != nullptr);
253
254
  // Compute the offset from the static and dynamic components.
255
2.03k
  llvm::Value *baseOffset;
256
2.03k
  if (!nonVirtualOffset.isZero()) {
257
1.29k
    llvm::Type *OffsetType =
258
1.29k
        (CGF.CGM.getTarget().getCXXABI().isItaniumFamily() &&
259
1.15k
         CGF.CGM.getItaniumVTableContext().isRelativeLayout())
260
0
            ? CGF.Int32Ty
261
1.29k
            : CGF.PtrDiffTy;
262
1.29k
    baseOffset =
263
1.29k
        llvm::ConstantInt::get(OffsetType, nonVirtualOffset.getQuantity());
264
1.29k
    if (virtualOffset) {
265
34
      baseOffset = CGF.Builder.CreateAdd(virtualOffset, baseOffset);
266
34
    }
267
743
  } else {
268
743
    baseOffset = virtualOffset;
269
743
  }
270
271
  // Apply the base offset.
272
2.03k
  llvm::Value *ptr = addr.getPointer();
273
2.03k
  unsigned AddrSpace = ptr->getType()->getPointerAddressSpace();
274
2.03k
  ptr = CGF.Builder.CreateBitCast(ptr, CGF.Int8Ty->getPointerTo(AddrSpace));
275
2.03k
  ptr = CGF.Builder.CreateInBoundsGEP(ptr, baseOffset, "add.ptr");
276
277
  // If we have a virtual component, the alignment of the result will
278
  // be relative only to the known alignment of that vbase.
279
2.03k
  CharUnits alignment;
280
2.03k
  if (virtualOffset) {
281
777
    assert(nearestVBase && "virtual offset without vbase?");
282
777
    alignment = CGF.CGM.getVBaseAlignment(addr.getAlignment(),
283
777
                                          derivedClass, nearestVBase);
284
1.26k
  } else {
285
1.26k
    alignment = addr.getAlignment();
286
1.26k
  }
287
2.03k
  alignment = alignment.alignmentAtOffset(nonVirtualOffset);
288
289
2.03k
  return Address(ptr, alignment);
290
2.03k
}
291
292
Address CodeGenFunction::GetAddressOfBaseClass(
293
    Address Value, const CXXRecordDecl *Derived,
294
    CastExpr::path_const_iterator PathBegin,
295
    CastExpr::path_const_iterator PathEnd, bool NullCheckValue,
296
20.5k
    SourceLocation Loc) {
297
20.5k
  assert(PathBegin != PathEnd && "Base path should not be empty!");
298
299
20.5k
  CastExpr::path_const_iterator Start = PathBegin;
300
20.5k
  const CXXRecordDecl *VBase = nullptr;
301
302
  // Sema has done some convenient canonicalization here: if the
303
  // access path involved any virtual steps, the conversion path will
304
  // *start* with a step down to the correct virtual base subobject,
305
  // and hence will not require any further steps.
306
20.5k
  if ((*Start)->isVirtual()) {
307
432
    VBase = cast<CXXRecordDecl>(
308
432
        (*Start)->getType()->castAs<RecordType>()->getDecl());
309
432
    ++Start;
310
432
  }
311
312
  // Compute the static offset of the ultimate destination within its
313
  // allocating subobject (the virtual base, if there is one, or else
314
  // the "complete" object that we see).
315
20.5k
  CharUnits NonVirtualOffset = CGM.computeNonVirtualBaseClassOffset(
316
20.1k
      VBase ? 
VBase432
: Derived, Start, PathEnd);
317
318
  // If there's a virtual step, we can sometimes "devirtualize" it.
319
  // For now, that's limited to when the derived type is final.
320
  // TODO: "devirtualize" this for accesses to known-complete objects.
321
20.5k
  if (VBase && 
Derived->hasAttr<FinalAttr>()432
) {
322
6
    const ASTRecordLayout &layout = getContext().getASTRecordLayout(Derived);
323
6
    CharUnits vBaseOffset = layout.getVBaseClassOffset(VBase);
324
6
    NonVirtualOffset += vBaseOffset;
325
6
    VBase = nullptr; // we no longer have a virtual step
326
6
  }
327
328
  // Get the base pointer type.
329
20.5k
  llvm::Type *BasePtrTy =
330
20.5k
      ConvertType((PathEnd[-1])->getType())
331
20.5k
          ->getPointerTo(Value.getType()->getPointerAddressSpace());
332
333
20.5k
  QualType DerivedTy = getContext().getRecordType(Derived);
334
20.5k
  CharUnits DerivedAlign = CGM.getClassPointerAlignment(Derived);
335
336
  // If the static offset is zero and we don't have a virtual step,
337
  // just do a bitcast; null checks are unnecessary.
338
20.5k
  if (NonVirtualOffset.isZero() && 
!VBase19.5k
) {
339
19.1k
    if (sanitizePerformTypeCheck()) {
340
19
      SanitizerSet SkippedChecks;
341
19
      SkippedChecks.set(SanitizerKind::Null, !NullCheckValue);
342
19
      EmitTypeCheck(TCK_Upcast, Loc, Value.getPointer(),
343
19
                    DerivedTy, DerivedAlign, SkippedChecks);
344
19
    }
345
19.1k
    return Builder.CreateBitCast(Value, BasePtrTy);
346
19.1k
  }
347
348
1.48k
  llvm::BasicBlock *origBB = nullptr;
349
1.48k
  llvm::BasicBlock *endBB = nullptr;
350
351
  // Skip over the offset (and the vtable load) if we're supposed to
352
  // null-check the pointer.
353
1.48k
  if (NullCheckValue) {
354
35
    origBB = Builder.GetInsertBlock();
355
35
    llvm::BasicBlock *notNullBB = createBasicBlock("cast.notnull");
356
35
    endBB = createBasicBlock("cast.end");
357
358
35
    llvm::Value *isNull = Builder.CreateIsNull(Value.getPointer());
359
35
    Builder.CreateCondBr(isNull, endBB, notNullBB);
360
35
    EmitBlock(notNullBB);
361
35
  }
362
363
1.48k
  if (sanitizePerformTypeCheck()) {
364
3
    SanitizerSet SkippedChecks;
365
3
    SkippedChecks.set(SanitizerKind::Null, true);
366
3
    EmitTypeCheck(VBase ? TCK_UpcastToVirtualBase : 
TCK_Upcast0
, Loc,
367
3
                  Value.getPointer(), DerivedTy, DerivedAlign, SkippedChecks);
368
3
  }
369
370
  // Compute the virtual offset.
371
1.48k
  llvm::Value *VirtualOffset = nullptr;
372
1.48k
  if (VBase) {
373
426
    VirtualOffset =
374
426
      CGM.getCXXABI().GetVirtualBaseClassOffset(*this, Value, Derived, VBase);
375
426
  }
376
377
  // Apply both offsets.
378
1.48k
  Value = ApplyNonVirtualAndVirtualOffset(*this, Value, NonVirtualOffset,
379
1.48k
                                          VirtualOffset, Derived, VBase);
380
381
  // Cast to the destination type.
382
1.48k
  Value = Builder.CreateBitCast(Value, BasePtrTy);
383
384
  // Build a phi if we needed a null check.
385
1.48k
  if (NullCheckValue) {
386
35
    llvm::BasicBlock *notNullBB = Builder.GetInsertBlock();
387
35
    Builder.CreateBr(endBB);
388
35
    EmitBlock(endBB);
389
390
35
    llvm::PHINode *PHI = Builder.CreatePHI(BasePtrTy, 2, "cast.result");
391
35
    PHI->addIncoming(Value.getPointer(), notNullBB);
392
35
    PHI->addIncoming(llvm::Constant::getNullValue(BasePtrTy), origBB);
393
35
    Value = Address(PHI, Value.getAlignment());
394
35
  }
395
396
1.48k
  return Value;
397
1.48k
}
398
399
Address
400
CodeGenFunction::GetAddressOfDerivedClass(Address BaseAddr,
401
                                          const CXXRecordDecl *Derived,
402
                                        CastExpr::path_const_iterator PathBegin,
403
                                          CastExpr::path_const_iterator PathEnd,
404
697
                                          bool NullCheckValue) {
405
697
  assert(PathBegin != PathEnd && "Base path should not be empty!");
406
407
697
  QualType DerivedTy =
408
697
    getContext().getCanonicalType(getContext().getTagDeclType(Derived));
409
697
  unsigned AddrSpace =
410
697
    BaseAddr.getPointer()->getType()->getPointerAddressSpace();
411
697
  llvm::Type *DerivedPtrTy = ConvertType(DerivedTy)->getPointerTo(AddrSpace);
412
413
697
  llvm::Value *NonVirtualOffset =
414
697
    CGM.GetNonVirtualBaseClassOffset(Derived, PathBegin, PathEnd);
415
416
697
  if (!NonVirtualOffset) {
417
    // No offset, we can just cast back.
418
686
    return Builder.CreateBitCast(BaseAddr, DerivedPtrTy);
419
686
  }
420
421
11
  llvm::BasicBlock *CastNull = nullptr;
422
11
  llvm::BasicBlock *CastNotNull = nullptr;
423
11
  llvm::BasicBlock *CastEnd = nullptr;
424
425
11
  if (NullCheckValue) {
426
5
    CastNull = createBasicBlock("cast.null");
427
5
    CastNotNull = createBasicBlock("cast.notnull");
428
5
    CastEnd = createBasicBlock("cast.end");
429
430
5
    llvm::Value *IsNull = Builder.CreateIsNull(BaseAddr.getPointer());
431
5
    Builder.CreateCondBr(IsNull, CastNull, CastNotNull);
432
5
    EmitBlock(CastNotNull);
433
5
  }
434
435
  // Apply the offset.
436
11
  llvm::Value *Value = Builder.CreateBitCast(BaseAddr.getPointer(), Int8PtrTy);
437
11
  Value = Builder.CreateInBoundsGEP(Value, Builder.CreateNeg(NonVirtualOffset),
438
11
                                    "sub.ptr");
439
440
  // Just cast.
441
11
  Value = Builder.CreateBitCast(Value, DerivedPtrTy);
442
443
  // Produce a PHI if we had a null-check.
444
11
  if (NullCheckValue) {
445
5
    Builder.CreateBr(CastEnd);
446
5
    EmitBlock(CastNull);
447
5
    Builder.CreateBr(CastEnd);
448
5
    EmitBlock(CastEnd);
449
450
5
    llvm::PHINode *PHI = Builder.CreatePHI(Value->getType(), 2);
451
5
    PHI->addIncoming(Value, CastNotNull);
452
5
    PHI->addIncoming(llvm::Constant::getNullValue(Value->getType()), CastNull);
453
5
    Value = PHI;
454
5
  }
455
456
11
  return Address(Value, CGM.getClassPointerAlignment(Derived));
457
11
}
458
459
llvm::Value *CodeGenFunction::GetVTTParameter(GlobalDecl GD,
460
                                              bool ForVirtualBase,
461
26.7k
                                              bool Delegating) {
462
26.7k
  if (!CGM.getCXXABI().NeedsVTTParameter(GD)) {
463
    // This constructor/destructor does not need a VTT parameter.
464
26.4k
    return nullptr;
465
26.4k
  }
466
467
270
  const CXXRecordDecl *RD = cast<CXXMethodDecl>(CurCodeDecl)->getParent();
468
270
  const CXXRecordDecl *Base = cast<CXXMethodDecl>(GD.getDecl())->getParent();
469
470
270
  llvm::Value *VTT;
471
472
270
  uint64_t SubVTTIndex;
473
474
270
  if (Delegating) {
475
    // If this is a delegating constructor call, just load the VTT.
476
2
    return LoadCXXVTT();
477
268
  } else if (RD == Base) {
478
    // If the record matches the base, this is the complete ctor/dtor
479
    // variant calling the base variant in a class with virtual bases.
480
94
    assert(!CGM.getCXXABI().NeedsVTTParameter(CurGD) &&
481
94
           "doing no-op VTT offset in base dtor/ctor?");
482
94
    assert(!ForVirtualBase && "Can't have same class as virtual base!");
483
94
    SubVTTIndex = 0;
484
174
  } else {
485
174
    const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
486
174
    CharUnits BaseOffset = ForVirtualBase ?
487
7
      Layout.getVBaseClassOffset(Base) :
488
167
      Layout.getBaseClassOffset(Base);
489
490
174
    SubVTTIndex =
491
174
      CGM.getVTables().getSubVTTIndex(RD, BaseSubobject(Base, BaseOffset));
492
174
    assert(SubVTTIndex != 0 && "Sub-VTT index must be greater than zero!");
493
174
  }
494
495
268
  if (CGM.getCXXABI().NeedsVTTParameter(CurGD)) {
496
    // A VTT parameter was passed to the constructor, use it.
497
40
    VTT = LoadCXXVTT();
498
40
    VTT = Builder.CreateConstInBoundsGEP1_64(VTT, SubVTTIndex);
499
228
  } else {
500
    // We're the complete constructor, so get the VTT by name.
501
228
    VTT = CGM.getVTables().GetAddrOfVTT(RD);
502
228
    VTT = Builder.CreateConstInBoundsGEP2_64(VTT, 0, SubVTTIndex);
503
228
  }
504
505
268
  return VTT;
506
270
}
507
508
namespace {
509
  /// Call the destructor for a direct base class.
510
  struct CallBaseDtor final : EHScopeStack::Cleanup {
511
    const CXXRecordDecl *BaseClass;
512
    bool BaseIsVirtual;
513
    CallBaseDtor(const CXXRecordDecl *Base, bool BaseIsVirtual)
514
2.53k
      : BaseClass(Base), BaseIsVirtual(BaseIsVirtual) {}
515
516
1.76k
    void Emit(CodeGenFunction &CGF, Flags flags) override {
517
1.76k
      const CXXRecordDecl *DerivedClass =
518
1.76k
        cast<CXXMethodDecl>(CGF.CurCodeDecl)->getParent();
519
520
1.76k
      const CXXDestructorDecl *D = BaseClass->getDestructor();
521
      // We are already inside a destructor, so presumably the object being
522
      // destroyed should have the expected type.
523
1.76k
      QualType ThisTy = D->getThisObjectType();
524
1.76k
      Address Addr =
525
1.76k
        CGF.GetAddressOfDirectBaseInCompleteClass(CGF.LoadCXXThisAddress(),
526
1.76k
                                                  DerivedClass, BaseClass,
527
1.76k
                                                  BaseIsVirtual);
528
1.76k
      CGF.EmitCXXDestructorCall(D, Dtor_Base, BaseIsVirtual,
529
1.76k
                                /*Delegating=*/false, Addr, ThisTy);
530
1.76k
    }
531
  };
532
533
  /// A visitor which checks whether an initializer uses 'this' in a
534
  /// way which requires the vtable to be properly set.
535
  struct DynamicThisUseChecker : ConstEvaluatedExprVisitor<DynamicThisUseChecker> {
536
    typedef ConstEvaluatedExprVisitor<DynamicThisUseChecker> super;
537
538
    bool UsesThis;
539
540
9.60k
    DynamicThisUseChecker(const ASTContext &C) : super(C), UsesThis(false) {}
541
542
    // Black-list all explicit and implicit references to 'this'.
543
    //
544
    // Do we need to worry about external references to 'this' derived
545
    // from arbitrary code?  If so, then anything which runs arbitrary
546
    // external code might potentially access the vtable.
547
11
    void VisitCXXThisExpr(const CXXThisExpr *E) { UsesThis = true; }
548
  };
549
} // end anonymous namespace
550
551
9.60k
static bool BaseInitializerUsesThis(ASTContext &C, const Expr *Init) {
552
9.60k
  DynamicThisUseChecker Checker(C);
553
9.60k
  Checker.Visit(Init);
554
9.60k
  return Checker.UsesThis;
555
9.60k
}
556
557
static void EmitBaseInitializer(CodeGenFunction &CGF,
558
                                const CXXRecordDecl *ClassDecl,
559
9.60k
                                CXXCtorInitializer *BaseInit) {
560
9.60k
  assert(BaseInit->isBaseInitializer() &&
561
9.60k
         "Must have base initializer!");
562
563
9.60k
  Address ThisPtr = CGF.LoadCXXThisAddress();
564
565
9.60k
  const Type *BaseType = BaseInit->getBaseClass();
566
9.60k
  const auto *BaseClassDecl =
567
9.60k
      cast<CXXRecordDecl>(BaseType->castAs<RecordType>()->getDecl());
568
569
9.60k
  bool isBaseVirtual = BaseInit->isBaseVirtual();
570
571
  // If the initializer for the base (other than the constructor
572
  // itself) accesses 'this' in any way, we need to initialize the
573
  // vtables.
574
9.60k
  if (BaseInitializerUsesThis(CGF.getContext(), BaseInit->getInit()))
575
11
    CGF.InitializeVTablePointers(ClassDecl);
576
577
  // We can pretend to be a complete class because it only matters for
578
  // virtual bases, and we only do virtual bases for complete ctors.
579
9.60k
  Address V =
580
9.60k
    CGF.GetAddressOfDirectBaseInCompleteClass(ThisPtr, ClassDecl,
581
9.60k
                                              BaseClassDecl,
582
9.60k
                                              isBaseVirtual);
583
9.60k
  AggValueSlot AggSlot =
584
9.60k
      AggValueSlot::forAddr(
585
9.60k
          V, Qualifiers(),
586
9.60k
          AggValueSlot::IsDestructed,
587
9.60k
          AggValueSlot::DoesNotNeedGCBarriers,
588
9.60k
          AggValueSlot::IsNotAliased,
589
9.60k
          CGF.getOverlapForBaseInit(ClassDecl, BaseClassDecl, isBaseVirtual));
590
591
9.60k
  CGF.EmitAggExpr(BaseInit->getInit(), AggSlot);
592
593
9.60k
  if (CGF.CGM.getLangOpts().Exceptions &&
594
7.35k
      !BaseClassDecl->hasTrivialDestructor())
595
1.18k
    CGF.EHStack.pushCleanup<CallBaseDtor>(EHCleanup, BaseClassDecl,
596
1.18k
                                          isBaseVirtual);
597
9.60k
}
598
599
92.1k
static bool isMemcpyEquivalentSpecialMember(const CXXMethodDecl *D) {
600
92.1k
  auto *CD = dyn_cast<CXXConstructorDecl>(D);
601
92.1k
  if (!(CD && 
CD->isCopyOrMoveConstructor()91.7k
) &&
602
82.1k
      !D->isCopyAssignmentOperator() && 
!D->isMoveAssignmentOperator()81.9k
)
603
81.8k
    return false;
604
605
  // We can emit a memcpy for a trivial copy or move constructor/assignment.
606
10.2k
  if (D->isTrivial() && 
!D->getParent()->mayInsertExtraPadding()5.60k
)
607
5.59k
    return true;
608
609
  // We *must* emit a memcpy for a defaulted union copy or move op.
610
4.64k
  if (D->getParent()->isUnion() && 
D->isDefaulted()4
)
611
2
    return true;
612
613
4.64k
  return false;
614
4.64k
}
615
616
static void EmitLValueForAnyFieldInitialization(CodeGenFunction &CGF,
617
                                                CXXCtorInitializer *MemberInit,
618
15.7k
                                                LValue &LHS) {
619
15.7k
  FieldDecl *Field = MemberInit->getAnyMember();
620
15.7k
  if (MemberInit->isIndirectMemberInitializer()) {
621
    // If we are initializing an anonymous union field, drill down to the field.
622
46
    IndirectFieldDecl *IndirectField = MemberInit->getIndirectMember();
623
46
    for (const auto *I : IndirectField->chain())
624
100
      LHS = CGF.EmitLValueForFieldInitialization(LHS, cast<FieldDecl>(I));
625
15.7k
  } else {
626
15.7k
    LHS = CGF.EmitLValueForFieldInitialization(LHS, Field);
627
15.7k
  }
628
15.7k
}
629
630
static void EmitMemberInitializer(CodeGenFunction &CGF,
631
                                  const CXXRecordDecl *ClassDecl,
632
                                  CXXCtorInitializer *MemberInit,
633
                                  const CXXConstructorDecl *Constructor,
634
15.7k
                                  FunctionArgList &Args) {
635
15.7k
  ApplyDebugLocation Loc(CGF, MemberInit->getSourceLocation());
636
15.7k
  assert(MemberInit->isAnyMemberInitializer() &&
637
15.7k
         "Must have member initializer!");
638
15.7k
  assert(MemberInit->getInit() && "Must have initializer!");
639
640
  // non-static data member initializers.
641
15.7k
  FieldDecl *Field = MemberInit->getAnyMember();
642
15.7k
  QualType FieldType = Field->getType();
643
644
15.7k
  llvm::Value *ThisPtr = CGF.LoadCXXThis();
645
15.7k
  QualType RecordTy = CGF.getContext().getTypeDeclType(ClassDecl);
646
15.7k
  LValue LHS;
647
648
  // If a base constructor is being emitted, create an LValue that has the
649
  // non-virtual alignment.
650
15.7k
  if (CGF.CurGD.getCtorType() == Ctor_Base)
651
15.5k
    LHS = CGF.MakeNaturalAlignPointeeAddrLValue(ThisPtr, RecordTy);
652
180
  else
653
180
    LHS = CGF.MakeNaturalAlignAddrLValue(ThisPtr, RecordTy);
654
655
15.7k
  EmitLValueForAnyFieldInitialization(CGF, MemberInit, LHS);
656
657
  // Special case: if we are in a copy or move constructor, and we are copying
658
  // an array of PODs or classes with trivial copy constructors, ignore the
659
  // AST and perform the copy we know is equivalent.
660
  // FIXME: This is hacky at best... if we had a bit more explicit information
661
  // in the AST, we could generalize it more easily.
662
15.7k
  const ConstantArrayType *Array
663
15.7k
    = CGF.getContext().getAsConstantArrayType(FieldType);
664
15.7k
  if (Array && 
Constructor->isDefaulted()77
&&
665
38
      Constructor->isCopyOrMoveConstructor()) {
666
22
    QualType BaseElementTy = CGF.getContext().getBaseElementType(Array);
667
22
    CXXConstructExpr *CE = dyn_cast<CXXConstructExpr>(MemberInit->getInit());
668
22
    if (BaseElementTy.isPODType(CGF.getContext()) ||
669
17
        (CE && 
isMemcpyEquivalentSpecialMember(CE->getConstructor())0
)) {
670
5
      unsigned SrcArgIndex =
671
5
          CGF.CGM.getCXXABI().getSrcArgforCopyCtor(Constructor, Args);
672
5
      llvm::Value *SrcPtr
673
5
        = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(Args[SrcArgIndex]));
674
5
      LValue ThisRHSLV = CGF.MakeNaturalAlignAddrLValue(SrcPtr, RecordTy);
675
5
      LValue Src = CGF.EmitLValueForFieldInitialization(ThisRHSLV, Field);
676
677
      // Copy the aggregate.
678
5
      CGF.EmitAggregateCopy(LHS, Src, FieldType, CGF.getOverlapForFieldInit(Field),
679
5
                            LHS.isVolatileQualified());
680
      // Ensure that we destroy the objects if an exception is thrown later in
681
      // the constructor.
682
5
      QualType::DestructionKind dtorKind = FieldType.isDestructedType();
683
5
      if (CGF.needsEHCleanup(dtorKind))
684
0
        CGF.pushEHDestroy(dtorKind, LHS.getAddress(CGF), FieldType);
685
5
      return;
686
5
    }
687
15.7k
  }
688
689
15.7k
  CGF.EmitInitializerForField(Field, LHS, MemberInit->getInit());
690
15.7k
}
691
692
void CodeGenFunction::EmitInitializerForField(FieldDecl *Field, LValue LHS,
693
16.9k
                                              Expr *Init) {
694
16.9k
  QualType FieldType = Field->getType();
695
16.9k
  switch (getEvaluationKind(FieldType)) {
696
12.6k
  case TEK_Scalar:
697
12.6k
    if (LHS.isSimple()) {
698
12.5k
      EmitExprAsInit(Init, Field, LHS, false);
699
82
    } else {
700
82
      RValue RHS = RValue::get(EmitScalarExpr(Init));
701
82
      EmitStoreThroughLValue(RHS, LHS);
702
82
    }
703
12.6k
    break;
704
6
  case TEK_Complex:
705
6
    EmitComplexExprIntoLValue(Init, LHS, /*isInit*/ true);
706
6
    break;
707
4.34k
  case TEK_Aggregate: {
708
4.34k
    AggValueSlot Slot = AggValueSlot::forLValue(
709
4.34k
        LHS, *this, AggValueSlot::IsDestructed,
710
4.34k
        AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased,
711
4.34k
        getOverlapForFieldInit(Field), AggValueSlot::IsNotZeroed,
712
        // Checks are made by the code that calls constructor.
713
4.34k
        AggValueSlot::IsSanitizerChecked);
714
4.34k
    EmitAggExpr(Init, Slot);
715
4.34k
    break;
716
16.9k
  }
717
16.9k
  }
718
719
  // Ensure that we destroy this object if an exception is thrown
720
  // later in the constructor.
721
16.9k
  QualType::DestructionKind dtorKind = FieldType.isDestructedType();
722
16.9k
  if (needsEHCleanup(dtorKind))
723
848
    pushEHDestroy(dtorKind, LHS.getAddress(*this), FieldType);
724
16.9k
}
725
726
/// Checks whether the given constructor is a valid subject for the
727
/// complete-to-base constructor delegation optimization, i.e.
728
/// emitting the complete constructor as a simple call to the base
729
/// constructor.
730
bool CodeGenFunction::IsConstructorDelegationValid(
731
17.1k
    const CXXConstructorDecl *Ctor) {
732
733
  // Currently we disable the optimization for classes with virtual
734
  // bases because (1) the addresses of parameter variables need to be
735
  // consistent across all initializers but (2) the delegate function
736
  // call necessarily creates a second copy of the parameter variable.
737
  //
738
  // The limiting example (purely theoretical AFAIK):
739
  //   struct A { A(int &c) { c++; } };
740
  //   struct B : virtual A {
741
  //     B(int count) : A(count) { printf("%d\n", count); }
742
  //   };
743
  // ...although even this example could in principle be emitted as a
744
  // delegation since the address of the parameter doesn't escape.
745
17.1k
  if (Ctor->getParent()->getNumVBases()) {
746
    // TODO: white-list trivial vbase initializers.  This case wouldn't
747
    // be subject to the restrictions below.
748
749
    // TODO: white-list cases where:
750
    //  - there are no non-reference parameters to the constructor
751
    //  - the initializers don't access any non-reference parameters
752
    //  - the initializers don't take the address of non-reference
753
    //    parameters
754
    //  - etc.
755
    // If we ever add any of the above cases, remember that:
756
    //  - function-try-blocks will always exclude this optimization
757
    //  - we need to perform the constructor prologue and cleanup in
758
    //    EmitConstructorBody.
759
760
586
    return false;
761
586
  }
762
763
  // We also disable the optimization for variadic functions because
764
  // it's impossible to "re-pass" varargs.
765
16.5k
  if (Ctor->getType()->castAs<FunctionProtoType>()->isVariadic())
766
10
    return false;
767
768
  // FIXME: Decide if we can do a delegation of a delegating constructor.
769
16.5k
  if (Ctor->isDelegatingConstructor())
770
83
    return false;
771
772
16.5k
  return true;
773
16.5k
}
774
775
// Emit code in ctor (Prologue==true) or dtor (Prologue==false)
776
// to poison the extra field paddings inserted under
777
// -fsanitize-address-field-padding=1|2.
778
53.7k
void CodeGenFunction::EmitAsanPrologueOrEpilogue(bool Prologue) {
779
53.7k
  ASTContext &Context = getContext();
780
53.7k
  const CXXRecordDecl *ClassDecl =
781
38.3k
      Prologue ? cast<CXXConstructorDecl>(CurGD.getDecl())->getParent()
782
15.4k
               : cast<CXXDestructorDecl>(CurGD.getDecl())->getParent();
783
53.7k
  if (!ClassDecl->mayInsertExtraPadding()) 
return53.7k
;
784
785
41
  struct SizeAndOffset {
786
41
    uint64_t Size;
787
41
    uint64_t Offset;
788
41
  };
789
790
41
  unsigned PtrSize = CGM.getDataLayout().getPointerSizeInBits();
791
41
  const ASTRecordLayout &Info = Context.getASTRecordLayout(ClassDecl);
792
793
  // Populate sizes and offsets of fields.
794
41
  SmallVector<SizeAndOffset, 16> SSV(Info.getFieldCount());
795
153
  for (unsigned i = 0, e = Info.getFieldCount(); i != e; 
++i112
)
796
112
    SSV[i].Offset =
797
112
        Context.toCharUnitsFromBits(Info.getFieldOffset(i)).getQuantity();
798
799
41
  size_t NumFields = 0;
800
112
  for (const auto *Field : ClassDecl->fields()) {
801
112
    const FieldDecl *D = Field;
802
112
    auto FieldInfo = Context.getTypeInfoInChars(D->getType());
803
112
    CharUnits FieldSize = FieldInfo.Width;
804
112
    assert(NumFields < SSV.size());
805
112
    SSV[NumFields].Size = D->isBitField() ? 
00
: FieldSize.getQuantity();
806
112
    NumFields++;
807
112
  }
808
41
  assert(NumFields == SSV.size());
809
41
  if (SSV.size() <= 1) 
return0
;
810
811
  // We will insert calls to __asan_* run-time functions.
812
  // LLVM AddressSanitizer pass may decide to inline them later.
813
41
  llvm::Type *Args[2] = {IntPtrTy, IntPtrTy};
814
41
  llvm::FunctionType *FTy =
815
41
      llvm::FunctionType::get(CGM.VoidTy, Args, false);
816
41
  llvm::FunctionCallee F = CGM.CreateRuntimeFunction(
817
22
      FTy, Prologue ? "__asan_poison_intra_object_redzone"
818
19
                    : "__asan_unpoison_intra_object_redzone");
819
820
41
  llvm::Value *ThisPtr = LoadCXXThis();
821
41
  ThisPtr = Builder.CreatePtrToInt(ThisPtr, IntPtrTy);
822
41
  uint64_t TypeSize = Info.getNonVirtualSize().getQuantity();
823
  // For each field check if it has sufficient padding,
824
  // if so (un)poison it with a call.
825
153
  for (size_t i = 0; i < SSV.size(); 
i++112
) {
826
112
    uint64_t AsanAlignment = 8;
827
71
    uint64_t NextField = i == SSV.size() - 1 ? 
TypeSize41
: SSV[i + 1].Offset;
828
112
    uint64_t PoisonSize = NextField - SSV[i].Offset - SSV[i].Size;
829
112
    uint64_t EndOffset = SSV[i].Offset + SSV[i].Size;
830
112
    if (PoisonSize < AsanAlignment || 
!SSV[i].Size94
||
831
94
        (NextField % AsanAlignment) != 0)
832
18
      continue;
833
94
    Builder.CreateCall(
834
94
        F, {Builder.CreateAdd(ThisPtr, Builder.getIntN(PtrSize, EndOffset)),
835
94
            Builder.getIntN(PtrSize, PoisonSize)});
836
94
  }
837
41
}
838
839
/// EmitConstructorBody - Emits the body of the current constructor.
840
38.3k
void CodeGenFunction::EmitConstructorBody(FunctionArgList &Args) {
841
38.3k
  EmitAsanPrologueOrEpilogue(true);
842
38.3k
  const CXXConstructorDecl *Ctor = cast<CXXConstructorDecl>(CurGD.getDecl());
843
38.3k
  CXXCtorType CtorType = CurGD.getCtorType();
844
845
38.3k
  assert((CGM.getTarget().getCXXABI().hasConstructorVariants() ||
846
38.3k
          CtorType == Ctor_Complete) &&
847
38.3k
         "can only generate complete ctor for this ABI");
848
849
  // Before we go any further, try the complete->base constructor
850
  // delegation optimization.
851
38.3k
  if (CtorType == Ctor_Complete && 
IsConstructorDelegationValid(Ctor)17.1k
&&
852
16.4k
      CGM.getTarget().getCXXABI().hasConstructorVariants()) {
853
15.6k
    EmitDelegateCXXConstructorCall(Ctor, Ctor_Base, Args, Ctor->getEndLoc());
854
15.6k
    return;
855
15.6k
  }
856
857
22.6k
  const FunctionDecl *Definition = nullptr;
858
22.6k
  Stmt *Body = Ctor->getBody(Definition);
859
22.6k
  assert(Definition == Ctor && "emitting wrong constructor body");
860
861
  // Enter the function-try-block before the constructor prologue if
862
  // applicable.
863
22.6k
  bool IsTryBody = (Body && isa<CXXTryStmt>(Body));
864
22.6k
  if (IsTryBody)
865
2
    EnterCXXTryStmt(*cast<CXXTryStmt>(Body), true);
866
867
22.6k
  incrementProfileCounter(Body);
868
869
22.6k
  RunCleanupsScope RunCleanups(*this);
870
871
  // TODO: in restricted cases, we can emit the vbase initializers of
872
  // a complete ctor and then delegate to the base ctor.
873
874
  // Emit the constructor prologue, i.e. the base and member
875
  // initializers.
876
22.6k
  EmitCtorPrologue(Ctor, CtorType, Args);
877
878
  // Emit the body of the statement.
879
22.6k
  if (IsTryBody)
880
2
    EmitStmt(cast<CXXTryStmt>(Body)->getTryBlock());
881
22.6k
  else if (Body)
882
22.6k
    EmitStmt(Body);
883
884
  // Emit any cleanup blocks associated with the member or base
885
  // initializers, which includes (along the exceptional path) the
886
  // destructors for those members and bases that were fully
887
  // constructed.
888
22.6k
  RunCleanups.ForceCleanup();
889
890
22.6k
  if (IsTryBody)
891
2
    ExitCXXTryStmt(*cast<CXXTryStmt>(Body), true);
892
22.6k
}
893
894
namespace {
895
  /// RAII object to indicate that codegen is copying the value representation
896
  /// instead of the object representation. Useful when copying a struct or
897
  /// class which has uninitialized members and we're only performing
898
  /// lvalue-to-rvalue conversion on the object but not its members.
899
  class CopyingValueRepresentation {
900
  public:
901
    explicit CopyingValueRepresentation(CodeGenFunction &CGF)
902
159
        : CGF(CGF), OldSanOpts(CGF.SanOpts) {
903
159
      CGF.SanOpts.set(SanitizerKind::Bool, false);
904
159
      CGF.SanOpts.set(SanitizerKind::Enum, false);
905
159
    }
906
159
    ~CopyingValueRepresentation() {
907
159
      CGF.SanOpts = OldSanOpts;
908
159
    }
909
  private:
910
    CodeGenFunction &CGF;
911
    SanitizerSet OldSanOpts;
912
  };
913
} // end anonymous namespace
914
915
namespace {
916
  class FieldMemcpyizer {
917
  public:
918
    FieldMemcpyizer(CodeGenFunction &CGF, const CXXRecordDecl *ClassDecl,
919
                    const VarDecl *SrcRec)
920
      : CGF(CGF), ClassDecl(ClassDecl), SrcRec(SrcRec),
921
        RecLayout(CGF.getContext().getASTRecordLayout(ClassDecl)),
922
        FirstField(nullptr), LastField(nullptr), FirstFieldOffset(0),
923
23.3k
        LastFieldOffset(0), LastAddedFieldIndex(0) {}
924
925
536
    bool isMemcpyableField(FieldDecl *F) const {
926
      // Never memcpy fields when we are adding poisoned paddings.
927
536
      if (CGF.getContext().getLangOpts().SanitizeAddressFieldPadding)
928
8
        return false;
929
528
      Qualifiers Qual = F->getType().getQualifiers();
930
528
      if (Qual.hasVolatile() || 
Qual.hasObjCLifetime()524
)
931
8
        return false;
932
520
      return true;
933
520
    }
934
935
517
    void addMemcpyableField(FieldDecl *F) {
936
517
      if (F->isZeroSize(CGF.getContext()))
937
1
        return;
938
516
      if (!FirstField)
939
255
        addInitialField(F);
940
261
      else
941
261
        addNextField(F);
942
516
    }
943
944
96
    CharUnits getMemcpySize(uint64_t FirstByteOffset) const {
945
96
      ASTContext &Ctx = CGF.getContext();
946
96
      unsigned LastFieldSize =
947
96
          LastField->isBitField()
948
6
              ? LastField->getBitWidthValue(Ctx)
949
90
              : Ctx.toBits(
950
90
                    Ctx.getTypeInfoDataSizeInChars(LastField->getType()).Width);
951
96
      uint64_t MemcpySizeBits = LastFieldOffset + LastFieldSize -
952
96
                                FirstByteOffset + Ctx.getCharWidth() - 1;
953
96
      CharUnits MemcpySize = Ctx.toCharUnitsFromBits(MemcpySizeBits);
954
96
      return MemcpySize;
955
96
    }
956
957
1.81k
    void emitMemcpy() {
958
      // Give the subclass a chance to bail out if it feels the memcpy isn't
959
      // worth it (e.g. Hasn't aggregated enough data).
960
1.81k
      if (!FirstField) {
961
1.71k
        return;
962
1.71k
      }
963
964
96
      uint64_t FirstByteOffset;
965
96
      if (FirstField->isBitField()) {
966
12
        const CGRecordLayout &RL =
967
12
          CGF.getTypes().getCGRecordLayout(FirstField->getParent());
968
12
        const CGBitFieldInfo &BFInfo = RL.getBitFieldInfo(FirstField);
969
        // FirstFieldOffset is not appropriate for bitfields,
970
        // we need to use the storage offset instead.
971
12
        FirstByteOffset = CGF.getContext().toBits(BFInfo.StorageOffset);
972
84
      } else {
973
84
        FirstByteOffset = FirstFieldOffset;
974
84
      }
975
976
96
      CharUnits MemcpySize = getMemcpySize(FirstByteOffset);
977
96
      QualType RecordTy = CGF.getContext().getTypeDeclType(ClassDecl);
978
96
      Address ThisPtr = CGF.LoadCXXThisAddress();
979
96
      LValue DestLV = CGF.MakeAddrLValue(ThisPtr, RecordTy);
980
96
      LValue Dest = CGF.EmitLValueForFieldInitialization(DestLV, FirstField);
981
96
      llvm::Value *SrcPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(SrcRec));
982
96
      LValue SrcLV = CGF.MakeNaturalAlignAddrLValue(SrcPtr, RecordTy);
983
96
      LValue Src = CGF.EmitLValueForFieldInitialization(SrcLV, FirstField);
984
985
96
      emitMemcpyIR(
986
84
          Dest.isBitField() ? 
Dest.getBitFieldAddress()12
: Dest.getAddress(CGF),
987
84
          Src.isBitField() ? 
Src.getBitFieldAddress()12
: Src.getAddress(CGF),
988
96
          MemcpySize);
989
96
      reset();
990
96
    }
991
992
40.0k
    void reset() {
993
40.0k
      FirstField = nullptr;
994
40.0k
    }
995
996
  protected:
997
    CodeGenFunction &CGF;
998
    const CXXRecordDecl *ClassDecl;
999
1000
  private:
1001
96
    void emitMemcpyIR(Address DestPtr, Address SrcPtr, CharUnits Size) {
1002
96
      llvm::PointerType *DPT = DestPtr.getType();
1003
96
      llvm::Type *DBP =
1004
96
        llvm::Type::getInt8PtrTy(CGF.getLLVMContext(), DPT->getAddressSpace());
1005
96
      DestPtr = CGF.Builder.CreateBitCast(DestPtr, DBP);
1006
1007
96
      llvm::PointerType *SPT = SrcPtr.getType();
1008
96
      llvm::Type *SBP =
1009
96
        llvm::Type::getInt8PtrTy(CGF.getLLVMContext(), SPT->getAddressSpace());
1010
96
      SrcPtr = CGF.Builder.CreateBitCast(SrcPtr, SBP);
1011
1012
96
      CGF.Builder.CreateMemCpy(DestPtr, SrcPtr, Size.getQuantity());
1013
96
    }
1014
1015
255
    void addInitialField(FieldDecl *F) {
1016
255
      FirstField = F;
1017
255
      LastField = F;
1018
255
      FirstFieldOffset = RecLayout.getFieldOffset(F->getFieldIndex());
1019
255
      LastFieldOffset = FirstFieldOffset;
1020
255
      LastAddedFieldIndex = F->getFieldIndex();
1021
255
    }
1022
1023
261
    void addNextField(FieldDecl *F) {
1024
      // For the most part, the following invariant will hold:
1025
      //   F->getFieldIndex() == LastAddedFieldIndex + 1
1026
      // The one exception is that Sema won't add a copy-initializer for an
1027
      // unnamed bitfield, which will show up here as a gap in the sequence.
1028
261
      assert(F->getFieldIndex() >= LastAddedFieldIndex + 1 &&
1029
261
             "Cannot aggregate fields out of order.");
1030
261
      LastAddedFieldIndex = F->getFieldIndex();
1031
1032
      // The 'first' and 'last' fields are chosen by offset, rather than field
1033
      // index. This allows the code to support bitfields, as well as regular
1034
      // fields.
1035
261
      uint64_t FOffset = RecLayout.getFieldOffset(F->getFieldIndex());
1036
261
      if (FOffset < FirstFieldOffset) {
1037
0
        FirstField = F;
1038
0
        FirstFieldOffset = FOffset;
1039
261
      } else if (FOffset >= LastFieldOffset) {
1040
261
        LastField = F;
1041
261
        LastFieldOffset = FOffset;
1042
261
      }
1043
261
    }
1044
1045
    const VarDecl *SrcRec;
1046
    const ASTRecordLayout &RecLayout;
1047
    FieldDecl *FirstField;
1048
    FieldDecl *LastField;
1049
    uint64_t FirstFieldOffset, LastFieldOffset;
1050
    unsigned LastAddedFieldIndex;
1051
  };
1052
1053
  class ConstructorMemcpyizer : public FieldMemcpyizer {
1054
  private:
1055
    /// Get source argument for copy constructor. Returns null if not a copy
1056
    /// constructor.
1057
    static const VarDecl *getTrivialCopySource(CodeGenFunction &CGF,
1058
                                               const CXXConstructorDecl *CD,
1059
22.6k
                                               FunctionArgList &Args) {
1060
22.6k
      if (CD->isCopyOrMoveConstructor() && 
CD->isDefaulted()1.00k
)
1061
365
        return Args[CGF.CGM.getCXXABI().getSrcArgforCopyCtor(CD, Args)];
1062
22.2k
      return nullptr;
1063
22.2k
    }
1064
1065
    // Returns true if a CXXCtorInitializer represents a member initialization
1066
    // that can be rolled into a memcpy.
1067
15.9k
    bool isMemberInitMemcpyable(CXXCtorInitializer *MemberInit) const {
1068
15.9k
      if (!MemcpyableCtor)
1069
15.5k
        return false;
1070
418
      FieldDecl *Field = MemberInit->getMember();
1071
418
      assert(Field && "No field for member init.");
1072
418
      QualType FieldType = Field->getType();
1073
418
      CXXConstructExpr *CE = dyn_cast<CXXConstructExpr>(MemberInit->getInit());
1074
1075
      // Bail out on non-memcpyable, not-trivially-copyable members.
1076
418
      if (!(CE && 
isMemcpyEquivalentSpecialMember(CE->getConstructor())138
) &&
1077
397
          !(FieldType.isTriviallyCopyableType(CGF.getContext()) ||
1078
166
            FieldType->isReferenceType()))
1079
158
        return false;
1080
1081
      // Bail out on volatile fields.
1082
260
      if (!isMemcpyableField(Field))
1083
6
        return false;
1084
1085
      // Otherwise we're good.
1086
254
      return true;
1087
254
    }
1088
1089
  public:
1090
    ConstructorMemcpyizer(CodeGenFunction &CGF, const CXXConstructorDecl *CD,
1091
                          FunctionArgList &Args)
1092
      : FieldMemcpyizer(CGF, CD->getParent(), getTrivialCopySource(CGF, CD, Args)),
1093
        ConstructorDecl(CD),
1094
        MemcpyableCtor(CD->isDefaulted() &&
1095
                       CD->isCopyOrMoveConstructor() &&
1096
                       CGF.getLangOpts().getGC() == LangOptions::NonGC),
1097
22.6k
        Args(Args) { }
1098
1099
15.9k
    void addMemberInitializer(CXXCtorInitializer *MemberInit) {
1100
15.9k
      if (isMemberInitMemcpyable(MemberInit)) {
1101
254
        AggregatedInits.push_back(MemberInit);
1102
254
        addMemcpyableField(MemberInit->getMember());
1103
15.7k
      } else {
1104
15.7k
        emitAggregatedInits();
1105
15.7k
        EmitMemberInitializer(CGF, ConstructorDecl->getParent(), MemberInit,
1106
15.7k
                              ConstructorDecl, Args);
1107
15.7k
      }
1108
15.9k
    }
1109
1110
38.3k
    void emitAggregatedInits() {
1111
38.3k
      if (AggregatedInits.size() <= 1) {
1112
        // This memcpy is too small to be worthwhile. Fall back on default
1113
        // codegen.
1114
38.2k
        if (!AggregatedInits.empty()) {
1115
59
          CopyingValueRepresentation CVR(CGF);
1116
59
          EmitMemberInitializer(CGF, ConstructorDecl->getParent(),
1117
59
                                AggregatedInits[0], ConstructorDecl, Args);
1118
59
          AggregatedInits.clear();
1119
59
        }
1120
38.2k
        reset();
1121
38.2k
        return;
1122
38.2k
      }
1123
1124
55
      pushEHDestructors();
1125
55
      emitMemcpy();
1126
55
      AggregatedInits.clear();
1127
55
    }
1128
1129
55
    void pushEHDestructors() {
1130
55
      Address ThisPtr = CGF.LoadCXXThisAddress();
1131
55
      QualType RecordTy = CGF.getContext().getTypeDeclType(ClassDecl);
1132
55
      LValue LHS = CGF.MakeAddrLValue(ThisPtr, RecordTy);
1133
1134
250
      for (unsigned i = 0; i < AggregatedInits.size(); 
++i195
) {
1135
195
        CXXCtorInitializer *MemberInit = AggregatedInits[i];
1136
195
        QualType FieldType = MemberInit->getAnyMember()->getType();
1137
195
        QualType::DestructionKind dtorKind = FieldType.isDestructedType();
1138
195
        if (!CGF.needsEHCleanup(dtorKind))
1139
193
          continue;
1140
2
        LValue FieldLHS = LHS;
1141
2
        EmitLValueForAnyFieldInitialization(CGF, MemberInit, FieldLHS);
1142
2
        CGF.pushEHDestroy(dtorKind, FieldLHS.getAddress(CGF), FieldType);
1143
2
      }
1144
55
    }
1145
1146
22.6k
    void finish() {
1147
22.6k
      emitAggregatedInits();
1148
22.6k
    }
1149
1150
  private:
1151
    const CXXConstructorDecl *ConstructorDecl;
1152
    bool MemcpyableCtor;
1153
    FunctionArgList &Args;
1154
    SmallVector<CXXCtorInitializer*, 16> AggregatedInits;
1155
  };
1156
1157
  class AssignmentMemcpyizer : public FieldMemcpyizer {
1158
  private:
1159
    // Returns the memcpyable field copied by the given statement, if one
1160
    // exists. Otherwise returns null.
1161
1.29k
    FieldDecl *getMemcpyableField(Stmt *S) {
1162
1.29k
      if (!AssignmentsMemcpyable)
1163
11
        return nullptr;
1164
1.28k
      if (BinaryOperator *BO = dyn_cast<BinaryOperator>(S)) {
1165
        // Recognise trivial assignments.
1166
226
        if (BO->getOpcode() != BO_Assign)
1167
0
          return nullptr;
1168
226
        MemberExpr *ME = dyn_cast<MemberExpr>(BO->getLHS());
1169
226
        if (!ME)
1170
0
          return nullptr;
1171
226
        FieldDecl *Field = dyn_cast<FieldDecl>(ME->getMemberDecl());
1172
226
        if (!Field || !isMemcpyableField(Field))
1173
10
          return nullptr;
1174
216
        Stmt *RHS = BO->getRHS();
1175
216
        if (ImplicitCastExpr *EC = dyn_cast<ImplicitCastExpr>(RHS))
1176
216
          RHS = EC->getSubExpr();
1177
216
        if (!RHS)
1178
0
          return nullptr;
1179
216
        if (MemberExpr *ME2 = dyn_cast<MemberExpr>(RHS)) {
1180
214
          if (ME2->getMemberDecl() == Field)
1181
214
            return Field;
1182
2
        }
1183
2
        return nullptr;
1184
1.06k
      } else if (CXXMemberCallExpr *MCE = dyn_cast<CXXMemberCallExpr>(S)) {
1185
313
        CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(MCE->getCalleeDecl());
1186
313
        if (!(MD && isMemcpyEquivalentSpecialMember(MD)))
1187
128
          return nullptr;
1188
185
        MemberExpr *IOA = dyn_cast<MemberExpr>(MCE->getImplicitObjectArgument());
1189
185
        if (!IOA)
1190
146
          return nullptr;
1191
39
        FieldDecl *Field = dyn_cast<FieldDecl>(IOA->getMemberDecl());
1192
39
        if (!Field || !isMemcpyableField(Field))
1193
0
          return nullptr;
1194
39
        MemberExpr *Arg0 = dyn_cast<MemberExpr>(MCE->getArg(0));
1195
39
        if (!Arg0 || 
Field != dyn_cast<FieldDecl>(Arg0->getMemberDecl())38
)
1196
1
          return nullptr;
1197
38
        return Field;
1198
749
      } else if (CallExpr *CE = dyn_cast<CallExpr>(S)) {
1199
11
        FunctionDecl *FD = dyn_cast<FunctionDecl>(CE->getCalleeDecl());
1200
11
        if (!FD || FD->getBuiltinID() != Builtin::BI__builtin_memcpy)
1201
0
          return nullptr;
1202
11
        Expr *DstPtr = CE->getArg(0);
1203
11
        if (ImplicitCastExpr *DC = dyn_cast<ImplicitCastExpr>(DstPtr))
1204
11
          DstPtr = DC->getSubExpr();
1205
11
        UnaryOperator *DUO = dyn_cast<UnaryOperator>(DstPtr);
1206
11
        if (!DUO || DUO->getOpcode() != UO_AddrOf)
1207
0
          return nullptr;
1208
11
        MemberExpr *ME = dyn_cast<MemberExpr>(DUO->getSubExpr());
1209
11
        if (!ME)
1210
0
          return nullptr;
1211
11
        FieldDecl *Field = dyn_cast<FieldDecl>(ME->getMemberDecl());
1212
11
        if (!Field || !isMemcpyableField(Field))
1213
0
          return nullptr;
1214
11
        Expr *SrcPtr = CE->getArg(1);
1215
11
        if (ImplicitCastExpr *SC = dyn_cast<ImplicitCastExpr>(SrcPtr))
1216
11
          SrcPtr = SC->getSubExpr();
1217
11
        UnaryOperator *SUO = dyn_cast<UnaryOperator>(SrcPtr);
1218
11
        if (!SUO || SUO->getOpcode() != UO_AddrOf)
1219
0
          return nullptr;
1220
11
        MemberExpr *ME2 = dyn_cast<MemberExpr>(SUO->getSubExpr());
1221
11
        if (!ME2 || Field != dyn_cast<FieldDecl>(ME2->getMemberDecl()))
1222
0
          return nullptr;
1223
11
        return Field;
1224
11
      }
1225
1226
738
      return nullptr;
1227
738
    }
1228
1229
    bool AssignmentsMemcpyable;
1230
    SmallVector<Stmt*, 16> AggregatedStmts;
1231
1232
  public:
1233
    AssignmentMemcpyizer(CodeGenFunction &CGF, const CXXMethodDecl *AD,
1234
                         FunctionArgList &Args)
1235
      : FieldMemcpyizer(CGF, AD->getParent(), Args[Args.size() - 1]),
1236
719
        AssignmentsMemcpyable(CGF.getLangOpts().getGC() == LangOptions::NonGC) {
1237
719
      assert(Args.size() == 2);
1238
719
    }
1239
1240
1.29k
    void emitAssignment(Stmt *S) {
1241
1.29k
      FieldDecl *F = getMemcpyableField(S);
1242
1.29k
      if (F) {
1243
263
        addMemcpyableField(F);
1244
263
        AggregatedStmts.push_back(S);
1245
1.03k
      } else {
1246
1.03k
        emitAggregatedStmts();
1247
1.03k
        CGF.EmitStmt(S);
1248
1.03k
      }
1249
1.29k
    }
1250
1251
1.75k
    void emitAggregatedStmts() {
1252
1.75k
      if (AggregatedStmts.size() <= 1) {
1253
1.71k
        if (!AggregatedStmts.empty()) {
1254
100
          CopyingValueRepresentation CVR(CGF);
1255
100
          CGF.EmitStmt(AggregatedStmts[0]);
1256
100
        }
1257
1.71k
        reset();
1258
1.71k
      }
1259
1260
1.75k
      emitMemcpy();
1261
1.75k
      AggregatedStmts.clear();
1262
1.75k
    }
1263
1264
719
    void finish() {
1265
719
      emitAggregatedStmts();
1266
719
    }
1267
  };
1268
} // end anonymous namespace
1269
1270
27
static bool isInitializerOfDynamicClass(const CXXCtorInitializer *BaseInit) {
1271
27
  const Type *BaseType = BaseInit->getBaseClass();
1272
27
  const auto *BaseClassDecl =
1273
27
      cast<CXXRecordDecl>(BaseType->castAs<RecordType>()->getDecl());
1274
27
  return BaseClassDecl->isDynamicClass();
1275
27
}
1276
1277
/// EmitCtorPrologue - This routine generates necessary code to initialize
1278
/// base classes and non-static data members belonging to this constructor.
1279
void CodeGenFunction::EmitCtorPrologue(const CXXConstructorDecl *CD,
1280
                                       CXXCtorType CtorType,
1281
22.7k
                                       FunctionArgList &Args) {
1282
22.7k
  if (CD->isDelegatingConstructor())
1283
90
    return EmitDelegatingCXXConstructorCall(CD, Args);
1284
1285
22.6k
  const CXXRecordDecl *ClassDecl = CD->getParent();
1286
1287
22.6k
  CXXConstructorDecl::init_const_iterator B = CD->init_begin(),
1288
22.6k
                                          E = CD->init_end();
1289
1290
  // Virtual base initializers first, if any. They aren't needed if:
1291
  // - This is a base ctor variant
1292
  // - There are no vbases
1293
  // - The class is abstract, so a complete object of it cannot be constructed
1294
  //
1295
  // The check for an abstract class is necessary because sema may not have
1296
  // marked virtual base destructors referenced.
1297
22.6k
  bool ConstructVBases = CtorType != Ctor_Base &&
1298
1.47k
                         ClassDecl->getNumVBases() != 0 &&
1299
594
                         !ClassDecl->isAbstract();
1300
1301
  // In the Microsoft C++ ABI, there are no constructor variants. Instead, the
1302
  // constructor of a class with virtual bases takes an additional parameter to
1303
  // conditionally construct the virtual bases. Emit that check here.
1304
22.6k
  llvm::BasicBlock *BaseCtorContinueBB = nullptr;
1305
22.6k
  if (ConstructVBases &&
1306
591
      !CGM.getTarget().getCXXABI().hasConstructorVariants()) {
1307
381
    BaseCtorContinueBB =
1308
381
        CGM.getCXXABI().EmitCtorCompleteObjectHandler(*this, ClassDecl);
1309
381
    assert(BaseCtorContinueBB);
1310
381
  }
1311
1312
22.6k
  llvm::Value *const OldThis = CXXThisValue;
1313
23.5k
  for (; B != E && 
(*B)->isBaseInitializer()18.1k
&&
(*B)->isBaseVirtual()7.37k
;
B++945
) {
1314
945
    if (!ConstructVBases)
1315
164
      continue;
1316
781
    if (CGM.getCodeGenOpts().StrictVTablePointers &&
1317
3
        CGM.getCodeGenOpts().OptimizationLevel > 0 &&
1318
3
        isInitializerOfDynamicClass(*B))
1319
1
      CXXThisValue = Builder.CreateLaunderInvariantGroup(LoadCXXThis());
1320
781
    EmitBaseInitializer(*this, ClassDecl, *B);
1321
781
  }
1322
1323
22.6k
  if (BaseCtorContinueBB) {
1324
    // Complete object handler should continue to the remaining initializers.
1325
381
    Builder.CreateBr(BaseCtorContinueBB);
1326
381
    EmitBlock(BaseCtorContinueBB);
1327
381
  }
1328
1329
  // Then, non-virtual base initializers.
1330
31.4k
  for (; B != E && 
(*B)->isBaseInitializer()20.9k
;
B++8.82k
) {
1331
8.82k
    assert(!(*B)->isBaseVirtual());
1332
1333
8.82k
    if (CGM.getCodeGenOpts().StrictVTablePointers &&
1334
24
        CGM.getCodeGenOpts().OptimizationLevel > 0 &&
1335
24
        isInitializerOfDynamicClass(*B))
1336
23
      CXXThisValue = Builder.CreateLaunderInvariantGroup(LoadCXXThis());
1337
8.82k
    EmitBaseInitializer(*this, ClassDecl, *B);
1338
8.82k
  }
1339
1340
22.6k
  CXXThisValue = OldThis;
1341
1342
22.6k
  InitializeVTablePointers(ClassDecl);
1343
1344
  // And finally, initialize class members.
1345
22.6k
  FieldConstructionScope FCS(*this, LoadCXXThisAddress());
1346
22.6k
  ConstructorMemcpyizer CM(*this, CD, Args);
1347
38.5k
  for (; B != E; 
B++15.9k
) {
1348
15.9k
    CXXCtorInitializer *Member = (*B);
1349
15.9k
    assert(!Member->isBaseInitializer());
1350
15.9k
    assert(Member->isAnyMemberInitializer() &&
1351
15.9k
           "Delegating initializer on non-delegating constructor");
1352
15.9k
    CM.addMemberInitializer(Member);
1353
15.9k
  }
1354
22.6k
  CM.finish();
1355
22.6k
}
1356
1357
static bool
1358
FieldHasTrivialDestructorBody(ASTContext &Context, const FieldDecl *Field);
1359
1360
static bool
1361
HasTrivialDestructorBody(ASTContext &Context,
1362
                         const CXXRecordDecl *BaseClassDecl,
1363
                         const CXXRecordDecl *MostDerivedClassDecl)
1364
394
{
1365
  // If the destructor is trivial we don't have to check anything else.
1366
394
  if (BaseClassDecl->hasTrivialDestructor())
1367
126
    return true;
1368
1369
268
  if (!BaseClassDecl->getDestructor()->hasTrivialBody())
1370
203
    return false;
1371
1372
  // Check fields.
1373
65
  for (const auto *Field : BaseClassDecl->fields())
1374
53
    if (!FieldHasTrivialDestructorBody(Context, Field))
1375
44
      return false;
1376
1377
  // Check non-virtual bases.
1378
21
  for (const auto &I : BaseClassDecl->bases()) {
1379
15
    if (I.isVirtual())
1380
1
      continue;
1381
1382
14
    const CXXRecordDecl *NonVirtualBase =
1383
14
      cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
1384
14
    if (!HasTrivialDestructorBody(Context, NonVirtualBase,
1385
14
                                  MostDerivedClassDecl))
1386
14
      return false;
1387
14
  }
1388
1389
7
  if (BaseClassDecl == MostDerivedClassDecl) {
1390
    // Check virtual bases.
1391
1
    for (const auto &I : BaseClassDecl->vbases()) {
1392
1
      const CXXRecordDecl *VirtualBase =
1393
1
        cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
1394
1
      if (!HasTrivialDestructorBody(Context, VirtualBase,
1395
1
                                    MostDerivedClassDecl))
1396
1
        return false;
1397
1
    }
1398
7
  }
1399
1400
6
  return true;
1401
7
}
1402
1403
static bool
1404
FieldHasTrivialDestructorBody(ASTContext &Context,
1405
                                          const FieldDecl *Field)
1406
736
{
1407
736
  QualType FieldBaseElementType = Context.getBaseElementType(Field->getType());
1408
1409
736
  const RecordType *RT = FieldBaseElementType->getAs<RecordType>();
1410
736
  if (!RT)
1411
355
    return true;
1412
1413
381
  CXXRecordDecl *FieldClassDecl = cast<CXXRecordDecl>(RT->getDecl());
1414
1415
  // The destructor for an implicit anonymous union member is never invoked.
1416
381
  if (FieldClassDecl->isUnion() && 
FieldClassDecl->isAnonymousStructOrUnion()2
)
1417
2
    return false;
1418
1419
379
  return HasTrivialDestructorBody(Context, FieldClassDecl, FieldClassDecl);
1420
379
}
1421
1422
/// CanSkipVTablePointerInitialization - Check whether we need to initialize
1423
/// any vtable pointers before calling this destructor.
1424
static bool CanSkipVTablePointerInitialization(CodeGenFunction &CGF,
1425
8.15k
                                               const CXXDestructorDecl *Dtor) {
1426
8.15k
  const CXXRecordDecl *ClassDecl = Dtor->getParent();
1427
8.15k
  if (!ClassDecl->isDynamicClass())
1428
7.21k
    return true;
1429
1430
938
  if (!Dtor->hasTrivialBody())
1431
63
    return false;
1432
1433
  // Check the fields.
1434
875
  for (const auto *Field : ClassDecl->fields())
1435
345
    if (!FieldHasTrivialDestructorBody(CGF.getContext(), Field))
1436
95
      return false;
1437
1438
780
  return true;
1439
875
}
1440
1441
/// EmitDestructorBody - Emits the body of the current destructor.
1442
16.4k
void CodeGenFunction::EmitDestructorBody(FunctionArgList &Args) {
1443
16.4k
  const CXXDestructorDecl *Dtor = cast<CXXDestructorDecl>(CurGD.getDecl());
1444
16.4k
  CXXDtorType DtorType = CurGD.getDtorType();
1445
1446
  // For an abstract class, non-base destructors are never used (and can't
1447
  // be emitted in general, because vbase dtors may not have been validated
1448
  // by Sema), but the Itanium ABI doesn't make them optional and Clang may
1449
  // in fact emit references to them from other compilations, so emit them
1450
  // as functions containing a trap instruction.
1451
16.4k
  if (DtorType != Dtor_Base && 
Dtor->getParent()->isAbstract()8.27k
) {
1452
82
    llvm::CallInst *TrapCall = EmitTrapCall(llvm::Intrinsic::trap);
1453
82
    TrapCall->setDoesNotReturn();
1454
82
    TrapCall->setDoesNotThrow();
1455
82
    Builder.CreateUnreachable();
1456
82
    Builder.ClearInsertionPoint();
1457
82
    return;
1458
82
  }
1459
1460
16.3k
  Stmt *Body = Dtor->getBody();
1461
16.3k
  if (Body)
1462
16.2k
    incrementProfileCounter(Body);
1463
1464
  // The call to operator delete in a deleting destructor happens
1465
  // outside of the function-try-block, which means it's always
1466
  // possible to delegate the destructor body to the complete
1467
  // destructor.  Do so.
1468
16.3k
  if (DtorType == Dtor_Deleting) {
1469
854
    RunCleanupsScope DtorEpilogue(*this);
1470
854
    EnterDtorCleanups(Dtor, Dtor_Deleting);
1471
854
    if (HaveInsertPoint()) {
1472
850
      QualType ThisTy = Dtor->getThisObjectType();
1473
850
      EmitCXXDestructorCall(Dtor, Dtor_Complete, /*ForVirtualBase=*/false,
1474
850
                            /*Delegating=*/false, LoadCXXThisAddress(), ThisTy);
1475
850
    }
1476
854
    return;
1477
854
  }
1478
1479
  // If the body is a function-try-block, enter the try before
1480
  // anything else.
1481
15.4k
  bool isTryBody = (Body && 
isa<CXXTryStmt>(Body)15.4k
);
1482
15.4k
  if (isTryBody)
1483
18
    EnterCXXTryStmt(*cast<CXXTryStmt>(Body), true);
1484
15.4k
  EmitAsanPrologueOrEpilogue(false);
1485
1486
  // Enter the epilogue cleanups.
1487
15.4k
  RunCleanupsScope DtorEpilogue(*this);
1488
1489
  // If this is the complete variant, just invoke the base variant;
1490
  // the epilogue will destruct the virtual bases.  But we can't do
1491
  // this optimization if the body is a function-try-block, because
1492
  // we'd introduce *two* handler blocks.  In the Microsoft ABI, we
1493
  // always delegate because we might not have a definition in this TU.
1494
15.4k
  switch (DtorType) {
1495
0
  case Dtor_Comdat: llvm_unreachable("not expecting a COMDAT");
1496
0
  case Dtor_Deleting: llvm_unreachable("already handled deleting case");
1497
1498
7.33k
  case Dtor_Complete:
1499
7.33k
    assert((Body || getTarget().getCXXABI().isMicrosoft()) &&
1500
7.33k
           "can't emit a dtor without a body for non-Microsoft ABIs");
1501
1502
    // Enter the cleanup scopes for virtual bases.
1503
7.33k
    EnterDtorCleanups(Dtor, Dtor_Complete);
1504
1505
7.33k
    if (!isTryBody) {
1506
7.32k
      QualType ThisTy = Dtor->getThisObjectType();
1507
7.32k
      EmitCXXDestructorCall(Dtor, Dtor_Base, /*ForVirtualBase=*/false,
1508
7.32k
                            /*Delegating=*/false, LoadCXXThisAddress(), ThisTy);
1509
7.32k
      break;
1510
7.32k
    }
1511
1512
    // Fallthrough: act like we're in the base variant.
1513
8
    LLVM_FALLTHROUGH;
1514
1515
8.15k
  case Dtor_Base:
1516
8.15k
    assert(Body);
1517
1518
    // Enter the cleanup scopes for fields and non-virtual bases.
1519
8.15k
    EnterDtorCleanups(Dtor, Dtor_Base);
1520
1521
    // Initialize the vtable pointers before entering the body.
1522
8.15k
    if (!CanSkipVTablePointerInitialization(*this, Dtor)) {
1523
      // Insert the llvm.launder.invariant.group intrinsic before initializing
1524
      // the vptrs to cancel any previous assumptions we might have made.
1525
158
      if (CGM.getCodeGenOpts().StrictVTablePointers &&
1526
2
          CGM.getCodeGenOpts().OptimizationLevel > 0)
1527
2
        CXXThisValue = Builder.CreateLaunderInvariantGroup(LoadCXXThis());
1528
158
      InitializeVTablePointers(Dtor->getParent());
1529
158
    }
1530
1531
8.15k
    if (isTryBody)
1532
18
      EmitStmt(cast<CXXTryStmt>(Body)->getTryBlock());
1533
8.13k
    else if (Body)
1534
8.13k
      EmitStmt(Body);
1535
0
    else {
1536
0
      assert(Dtor->isImplicit() && "bodyless dtor not implicit");
1537
      // nothing to do besides what's in the epilogue
1538
0
    }
1539
    // -fapple-kext must inline any call to this dtor into
1540
    // the caller's body.
1541
8.15k
    if (getLangOpts().AppleKext)
1542
5
      CurFn->addFnAttr(llvm::Attribute::AlwaysInline);
1543
1544
8.15k
    break;
1545
15.4k
  }
1546
1547
  // Jump out through the epilogue cleanups.
1548
15.4k
  DtorEpilogue.ForceCleanup();
1549
1550
  // Exit the try if applicable.
1551
15.4k
  if (isTryBody)
1552
18
    ExitCXXTryStmt(*cast<CXXTryStmt>(Body), true);
1553
15.4k
}
1554
1555
719
void CodeGenFunction::emitImplicitAssignmentOperatorBody(FunctionArgList &Args) {
1556
719
  const CXXMethodDecl *AssignOp = cast<CXXMethodDecl>(CurGD.getDecl());
1557
719
  const Stmt *RootS = AssignOp->getBody();
1558
719
  assert(isa<CompoundStmt>(RootS) &&
1559
719
         "Body of an implicit assignment operator should be compound stmt.");
1560
719
  const CompoundStmt *RootCS = cast<CompoundStmt>(RootS);
1561
1562
719
  LexicalScope Scope(*this, RootCS->getSourceRange());
1563
1564
719
  incrementProfileCounter(RootCS);
1565
719
  AssignmentMemcpyizer AM(*this, AssignOp, Args);
1566
719
  for (auto *I : RootCS->body())
1567
1.29k
    AM.emitAssignment(I);
1568
719
  AM.finish();
1569
719
}
1570
1571
namespace {
1572
  llvm::Value *LoadThisForDtorDelete(CodeGenFunction &CGF,
1573
867
                                     const CXXDestructorDecl *DD) {
1574
867
    if (Expr *ThisArg = DD->getOperatorDeleteThisArg())
1575
12
      return CGF.EmitScalarExpr(ThisArg);
1576
855
    return CGF.LoadCXXThis();
1577
855
  }
1578
1579
  /// Call the operator delete associated with the current destructor.
1580
  struct CallDtorDelete final : EHScopeStack::Cleanup {
1581
639
    CallDtorDelete() {}
1582
1583
652
    void Emit(CodeGenFunction &CGF, Flags flags) override {
1584
652
      const CXXDestructorDecl *Dtor = cast<CXXDestructorDecl>(CGF.CurCodeDecl);
1585
652
      const CXXRecordDecl *ClassDecl = Dtor->getParent();
1586
652
      CGF.EmitDeleteCall(Dtor->getOperatorDelete(),
1587
652
                         LoadThisForDtorDelete(CGF, Dtor),
1588
652
                         CGF.getContext().getTagDeclType(ClassDecl));
1589
652
    }
1590
  };
1591
1592
  void EmitConditionalDtorDeleteCall(CodeGenFunction &CGF,
1593
                                     llvm::Value *ShouldDeleteCondition,
1594
211
                                     bool ReturnAfterDelete) {
1595
211
    llvm::BasicBlock *callDeleteBB = CGF.createBasicBlock("dtor.call_delete");
1596
211
    llvm::BasicBlock *continueBB = CGF.createBasicBlock("dtor.continue");
1597
211
    llvm::Value *ShouldCallDelete
1598
211
      = CGF.Builder.CreateIsNull(ShouldDeleteCondition);
1599
211
    CGF.Builder.CreateCondBr(ShouldCallDelete, continueBB, callDeleteBB);
1600
1601
211
    CGF.EmitBlock(callDeleteBB);
1602
211
    const CXXDestructorDecl *Dtor = cast<CXXDestructorDecl>(CGF.CurCodeDecl);
1603
211
    const CXXRecordDecl *ClassDecl = Dtor->getParent();
1604
211
    CGF.EmitDeleteCall(Dtor->getOperatorDelete(),
1605
211
                       LoadThisForDtorDelete(CGF, Dtor),
1606
211
                       CGF.getContext().getTagDeclType(ClassDecl));
1607
211
    assert(Dtor->getOperatorDelete()->isDestroyingOperatorDelete() ==
1608
211
               ReturnAfterDelete &&
1609
211
           "unexpected value for ReturnAfterDelete");
1610
211
    if (ReturnAfterDelete)
1611
8
      CGF.EmitBranchThroughCleanup(CGF.ReturnBlock);
1612
203
    else
1613
203
      CGF.Builder.CreateBr(continueBB);
1614
1615
211
    CGF.EmitBlock(continueBB);
1616
211
  }
1617
1618
  struct CallDtorDeleteConditional final : EHScopeStack::Cleanup {
1619
    llvm::Value *ShouldDeleteCondition;
1620
1621
  public:
1622
    CallDtorDeleteConditional(llvm::Value *ShouldDeleteCondition)
1623
203
        : ShouldDeleteCondition(ShouldDeleteCondition) {
1624
203
      assert(ShouldDeleteCondition != nullptr);
1625
203
    }
1626
1627
203
    void Emit(CodeGenFunction &CGF, Flags flags) override {
1628
203
      EmitConditionalDtorDeleteCall(CGF, ShouldDeleteCondition,
1629
203
                                    /*ReturnAfterDelete*/false);
1630
203
    }
1631
  };
1632
1633
  class DestroyField  final : public EHScopeStack::Cleanup {
1634
    const FieldDecl *field;
1635
    CodeGenFunction::Destroyer *destroyer;
1636
    bool useEHCleanupForArray;
1637
1638
  public:
1639
    DestroyField(const FieldDecl *field, CodeGenFunction::Destroyer *destroyer,
1640
                 bool useEHCleanupForArray)
1641
        : field(field), destroyer(destroyer),
1642
1.02k
          useEHCleanupForArray(useEHCleanupForArray) {}
1643
1644
1.06k
    void Emit(CodeGenFunction &CGF, Flags flags) override {
1645
      // Find the address of the field.
1646
1.06k
      Address thisValue = CGF.LoadCXXThisAddress();
1647
1.06k
      QualType RecordTy = CGF.getContext().getTagDeclType(field->getParent());
1648
1.06k
      LValue ThisLV = CGF.MakeAddrLValue(thisValue, RecordTy);
1649
1.06k
      LValue LV = CGF.EmitLValueForField(ThisLV, field);
1650
1.06k
      assert(LV.isSimple());
1651
1652
1.06k
      CGF.emitDestroy(LV.getAddress(CGF), field->getType(), destroyer,
1653
1.06k
                      flags.isForNormalCleanup() && 
useEHCleanupForArray1.02k
);
1654
1.06k
    }
1655
  };
1656
1657
 static void EmitSanitizerDtorCallback(CodeGenFunction &CGF, llvm::Value *Ptr,
1658
130
             CharUnits::QuantityType PoisonSize) {
1659
130
   CodeGenFunction::SanitizerScope SanScope(&CGF);
1660
   // Pass in void pointer and size of region as arguments to runtime
1661
   // function
1662
130
   llvm::Value *Args[] = {CGF.Builder.CreateBitCast(Ptr, CGF.VoidPtrTy),
1663
130
                          llvm::ConstantInt::get(CGF.SizeTy, PoisonSize)};
1664
1665
130
   llvm::Type *ArgTypes[] = {CGF.VoidPtrTy, CGF.SizeTy};
1666
1667
130
   llvm::FunctionType *FnType =
1668
130
       llvm::FunctionType::get(CGF.VoidTy, ArgTypes, false);
1669
130
   llvm::FunctionCallee Fn =
1670
130
       CGF.CGM.CreateRuntimeFunction(FnType, "__sanitizer_dtor_callback");
1671
130
   CGF.EmitNounwindRuntimeCall(Fn, Args);
1672
130
 }
1673
1674
  class SanitizeDtorMembers final : public EHScopeStack::Cleanup {
1675
    const CXXDestructorDecl *Dtor;
1676
1677
  public:
1678
88
    SanitizeDtorMembers(const CXXDestructorDecl *Dtor) : Dtor(Dtor) {}
1679
1680
    // Generate function call for handling object poisoning.
1681
    // Disables tail call elimination, to prevent the current stack frame
1682
    // from disappearing from the stack trace.
1683
88
    void Emit(CodeGenFunction &CGF, Flags flags) override {
1684
88
      const ASTRecordLayout &Layout =
1685
88
          CGF.getContext().getASTRecordLayout(Dtor->getParent());
1686
1687
      // Nothing to poison.
1688
88
      if (Layout.getFieldCount() == 0)
1689
0
        return;
1690
1691
      // Prevent the current stack frame from disappearing from the stack trace.
1692
88
      CGF.CurFn->addFnAttr("disable-tail-calls", "true");
1693
1694
      // Construct pointer to region to begin poisoning, and calculate poison
1695
      // size, so that only members declared in this class are poisoned.
1696
88
      ASTContext &Context = CGF.getContext();
1697
1698
88
      const RecordDecl *Decl = Dtor->getParent();
1699
88
      auto Fields = Decl->fields();
1700
338
      auto IsTrivial = [&](const FieldDecl *F) {
1701
338
        return FieldHasTrivialDestructorBody(Context, F);
1702
338
      };
1703
1704
170
      auto IsZeroSize = [&](const FieldDecl *F) {
1705
170
        return F->isZeroSize(Context);
1706
170
      };
1707
1708
      // Poison blocks of fields with trivial destructors making sure that block
1709
      // begin and end do not point to zero-sized fields. They don't have
1710
      // correct offsets so can't be used to calculate poisoning range.
1711
200
      for (auto It = Fields.begin(); It != Fields.end();) {
1712
188
        It = std::find_if(It, Fields.end(), [&](const FieldDecl *F) {
1713
188
          return IsTrivial(F) && 
!IsZeroSize(F)124
;
1714
188
        });
1715
128
        if (It == Fields.end())
1716
16
          break;
1717
112
        auto Start = It++;
1718
150
        It = std::find_if(It, Fields.end(), [&](const FieldDecl *F) {
1719
150
          return !IsTrivial(F) && 
!IsZeroSize(F)46
;
1720
150
        });
1721
1722
112
        PoisonMembers(CGF, (*Start)->getFieldIndex(),
1723
72
                      It == Fields.end() ? -1 : 
(*It)->getFieldIndex()40
);
1724
112
      }
1725
88
    }
1726
1727
  private:
1728
    /// \param layoutStartOffset index of the ASTRecordLayout field to
1729
    ///     start poisoning (inclusive)
1730
    /// \param layoutEndOffset index of the ASTRecordLayout field to
1731
    ///     end poisoning (exclusive)
1732
    void PoisonMembers(CodeGenFunction &CGF, unsigned layoutStartOffset,
1733
112
                       unsigned layoutEndOffset) {
1734
112
      ASTContext &Context = CGF.getContext();
1735
112
      const ASTRecordLayout &Layout =
1736
112
          Context.getASTRecordLayout(Dtor->getParent());
1737
1738
      // It's a first trivia field so it should be at the begining of char,
1739
      // still round up start offset just in case.
1740
112
      CharUnits PoisonStart =
1741
112
          Context.toCharUnitsFromBits(Layout.getFieldOffset(layoutStartOffset) +
1742
112
                                      Context.getCharWidth() - 1);
1743
112
      llvm::ConstantInt *OffsetSizePtr =
1744
112
          llvm::ConstantInt::get(CGF.SizeTy, PoisonStart.getQuantity());
1745
1746
112
      llvm::Value *OffsetPtr = CGF.Builder.CreateGEP(
1747
112
          CGF.Builder.CreateBitCast(CGF.LoadCXXThis(), CGF.Int8PtrTy),
1748
112
          OffsetSizePtr);
1749
1750
112
      CharUnits PoisonEnd;
1751
112
      if (layoutEndOffset >= Layout.getFieldCount()) {
1752
72
        PoisonEnd = Layout.getNonVirtualSize();
1753
40
      } else {
1754
40
        PoisonEnd =
1755
40
            Context.toCharUnitsFromBits(Layout.getFieldOffset(layoutEndOffset));
1756
40
      }
1757
112
      CharUnits PoisonSize = PoisonEnd - PoisonStart;
1758
112
      if (!PoisonSize.isPositive())
1759
0
        return;
1760
1761
112
      EmitSanitizerDtorCallback(CGF, OffsetPtr, PoisonSize.getQuantity());
1762
112
    }
1763
  };
1764
1765
 class SanitizeDtorVTable final : public EHScopeStack::Cleanup {
1766
    const CXXDestructorDecl *Dtor;
1767
1768
  public:
1769
18
    SanitizeDtorVTable(const CXXDestructorDecl *Dtor) : Dtor(Dtor) {}
1770
1771
    // Generate function call for handling vtable pointer poisoning.
1772
18
    void Emit(CodeGenFunction &CGF, Flags flags) override {
1773
18
      assert(Dtor->getParent()->isDynamicClass());
1774
18
      (void)Dtor;
1775
18
      ASTContext &Context = CGF.getContext();
1776
      // Poison vtable and vtable ptr if they exist for this class.
1777
18
      llvm::Value *VTablePtr = CGF.LoadCXXThis();
1778
1779
18
      CharUnits::QuantityType PoisonSize =
1780
18
          Context.toCharUnitsFromBits(CGF.PointerWidthInBits).getQuantity();
1781
      // Pass in void pointer and size of region as arguments to runtime
1782
      // function
1783
18
      EmitSanitizerDtorCallback(CGF, VTablePtr, PoisonSize);
1784
18
    }
1785
 };
1786
} // end anonymous namespace
1787
1788
/// Emit all code that comes at the end of class's
1789
/// destructor. This is to call destructors on members and base classes
1790
/// in reverse order of their construction.
1791
///
1792
/// For a deleting destructor, this also handles the case where a destroying
1793
/// operator delete completely overrides the definition.
1794
void CodeGenFunction::EnterDtorCleanups(const CXXDestructorDecl *DD,
1795
16.3k
                                        CXXDtorType DtorType) {
1796
16.3k
  assert((!DD->isTrivial() || DD->hasAttr<DLLExportAttr>()) &&
1797
16.3k
         "Should not emit dtor epilogue for non-exported trivial dtor!");
1798
1799
  // The deleting-destructor phase just needs to call the appropriate
1800
  // operator delete that Sema picked up.
1801
16.3k
  if (DtorType == Dtor_Deleting) {
1802
854
    assert(DD->getOperatorDelete() &&
1803
854
           "operator delete missing - EnterDtorCleanups");
1804
854
    if (CXXStructorImplicitParamValue) {
1805
      // If there is an implicit param to the deleting dtor, it's a boolean
1806
      // telling whether this is a deleting destructor.
1807
211
      if (DD->getOperatorDelete()->isDestroyingOperatorDelete())
1808
8
        EmitConditionalDtorDeleteCall(*this, CXXStructorImplicitParamValue,
1809
8
                                      /*ReturnAfterDelete*/true);
1810
203
      else
1811
203
        EHStack.pushCleanup<CallDtorDeleteConditional>(
1812
203
            NormalAndEHCleanup, CXXStructorImplicitParamValue);
1813
643
    } else {
1814
643
      if (DD->getOperatorDelete()->isDestroyingOperatorDelete()) {
1815
4
        const CXXRecordDecl *ClassDecl = DD->getParent();
1816
4
        EmitDeleteCall(DD->getOperatorDelete(),
1817
4
                       LoadThisForDtorDelete(*this, DD),
1818
4
                       getContext().getTagDeclType(ClassDecl));
1819
4
        EmitBranchThroughCleanup(ReturnBlock);
1820
639
      } else {
1821
639
        EHStack.pushCleanup<CallDtorDelete>(NormalAndEHCleanup);
1822
639
      }
1823
643
    }
1824
854
    return;
1825
854
  }
1826
1827
15.4k
  const CXXRecordDecl *ClassDecl = DD->getParent();
1828
1829
  // Unions have no bases and do not call field destructors.
1830
15.4k
  if (ClassDecl->isUnion())
1831
4
    return;
1832
1833
  // The complete-destructor phase just destructs all the virtual bases.
1834
15.4k
  if (DtorType == Dtor_Complete) {
1835
    // Poison the vtable pointer such that access after the base
1836
    // and member destructors are invoked is invalid.
1837
7.33k
    if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
1838
89
        SanOpts.has(SanitizerKind::Memory) && 
ClassDecl->getNumVBases()88
&&
1839
4
        ClassDecl->isPolymorphic())
1840
4
      EHStack.pushCleanup<SanitizeDtorVTable>(NormalAndEHCleanup, DD);
1841
1842
    // We push them in the forward order so that they'll be popped in
1843
    // the reverse order.
1844
228
    for (const auto &Base : ClassDecl->vbases()) {
1845
228
      auto *BaseClassDecl =
1846
228
          cast<CXXRecordDecl>(Base.getType()->castAs<RecordType>()->getDecl());
1847
1848
      // Ignore trivial destructors.
1849
228
      if (BaseClassDecl->hasTrivialDestructor())
1850
44
        continue;
1851
1852
184
      EHStack.pushCleanup<CallBaseDtor>(NormalAndEHCleanup,
1853
184
                                        BaseClassDecl,
1854
184
                                        /*BaseIsVirtual*/ true);
1855
184
    }
1856
1857
7.33k
    return;
1858
7.33k
  }
1859
1860
8.15k
  assert(DtorType == Dtor_Base);
1861
  // Poison the vtable pointer if it has no virtual bases, but inherits
1862
  // virtual functions.
1863
8.15k
  if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
1864
89
      SanOpts.has(SanitizerKind::Memory) && 
!ClassDecl->getNumVBases()88
&&
1865
84
      ClassDecl->isPolymorphic())
1866
14
    EHStack.pushCleanup<SanitizeDtorVTable>(NormalAndEHCleanup, DD);
1867
1868
  // Destroy non-virtual bases.
1869
2.23k
  for (const auto &Base : ClassDecl->bases()) {
1870
    // Ignore virtual bases.
1871
2.23k
    if (Base.isVirtual())
1872
127
      continue;
1873
1874
2.10k
    CXXRecordDecl *BaseClassDecl = Base.getType()->getAsCXXRecordDecl();
1875
1876
    // Ignore trivial destructors.
1877
2.10k
    if (BaseClassDecl->hasTrivialDestructor())
1878
948
      continue;
1879
1880
1.16k
    EHStack.pushCleanup<CallBaseDtor>(NormalAndEHCleanup,
1881
1.16k
                                      BaseClassDecl,
1882
1.16k
                                      /*BaseIsVirtual*/ false);
1883
1.16k
  }
1884
1885
  // Poison fields such that access after their destructors are
1886
  // invoked, and before the base class destructor runs, is invalid.
1887
8.15k
  if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
1888
89
      SanOpts.has(SanitizerKind::Memory))
1889
88
    EHStack.pushCleanup<SanitizeDtorMembers>(NormalAndEHCleanup, DD);
1890
1891
  // Destroy direct fields.
1892
9.93k
  for (const auto *Field : ClassDecl->fields()) {
1893
9.93k
    QualType type = Field->getType();
1894
9.93k
    QualType::DestructionKind dtorKind = type.isDestructedType();
1895
9.93k
    if (!dtorKind) 
continue8.89k
;
1896
1897
    // Anonymous union members do not have their destructors called.
1898
1.03k
    const RecordType *RT = type->getAsUnionType();
1899
1.03k
    if (RT && 
RT->getDecl()->isAnonymousStructOrUnion()9
)
continue9
;
1900
1901
1.02k
    CleanupKind cleanupKind = getCleanupKind(dtorKind);
1902
1.02k
    EHStack.pushCleanup<DestroyField>(cleanupKind, Field,
1903
1.02k
                                      getDestroyer(dtorKind),
1904
1.02k
                                      cleanupKind & EHCleanup);
1905
1.02k
  }
1906
8.15k
}
1907
1908
/// EmitCXXAggrConstructorCall - Emit a loop to call a particular
1909
/// constructor for each of several members of an array.
1910
///
1911
/// \param ctor the constructor to call for each element
1912
/// \param arrayType the type of the array to initialize
1913
/// \param arrayBegin an arrayType*
1914
/// \param zeroInitialize true if each element should be
1915
///   zero-initialized before it is constructed
1916
void CodeGenFunction::EmitCXXAggrConstructorCall(
1917
    const CXXConstructorDecl *ctor, const ArrayType *arrayType,
1918
    Address arrayBegin, const CXXConstructExpr *E, bool NewPointerIsChecked,
1919
908
    bool zeroInitialize) {
1920
908
  QualType elementType;
1921
908
  llvm::Value *numElements =
1922
908
    emitArrayLength(arrayType, elementType, arrayBegin);
1923
1924
908
  EmitCXXAggrConstructorCall(ctor, numElements, arrayBegin, E,
1925
908
                             NewPointerIsChecked, zeroInitialize);
1926
908
}
1927
1928
/// EmitCXXAggrConstructorCall - Emit a loop to call a particular
1929
/// constructor for each of several members of an array.
1930
///
1931
/// \param ctor the constructor to call for each element
1932
/// \param numElements the number of elements in the array;
1933
///   may be zero
1934
/// \param arrayBase a T*, where T is the type constructed by ctor
1935
/// \param zeroInitialize true if each element should be
1936
///   zero-initialized before it is constructed
1937
void CodeGenFunction::EmitCXXAggrConstructorCall(const CXXConstructorDecl *ctor,
1938
                                                 llvm::Value *numElements,
1939
                                                 Address arrayBase,
1940
                                                 const CXXConstructExpr *E,
1941
                                                 bool NewPointerIsChecked,
1942
973
                                                 bool zeroInitialize) {
1943
  // It's legal for numElements to be zero.  This can happen both
1944
  // dynamically, because x can be zero in 'new A[x]', and statically,
1945
  // because of GCC extensions that permit zero-length arrays.  There
1946
  // are probably legitimate places where we could assume that this
1947
  // doesn't happen, but it's not clear that it's worth it.
1948
973
  llvm::BranchInst *zeroCheckBranch = nullptr;
1949
1950
  // Optimize for a constant count.
1951
973
  llvm::ConstantInt *constantCount
1952
973
    = dyn_cast<llvm::ConstantInt>(numElements);
1953
973
  if (constantCount) {
1954
    // Just skip out if the constant count is zero.
1955
932
    if (constantCount->isZero()) 
return0
;
1956
1957
  // Otherwise, emit the check.
1958
41
  } else {
1959
41
    llvm::BasicBlock *loopBB = createBasicBlock("new.ctorloop");
1960
41
    llvm::Value *iszero = Builder.CreateIsNull(numElements, "isempty");
1961
41
    zeroCheckBranch = Builder.CreateCondBr(iszero, loopBB, loopBB);
1962
41
    EmitBlock(loopBB);
1963
41
  }
1964
1965
  // Find the end of the array.
1966
973
  llvm::Value *arrayBegin = arrayBase.getPointer();
1967
973
  llvm::Value *arrayEnd = Builder.CreateInBoundsGEP(arrayBegin, numElements,
1968
973
                                                    "arrayctor.end");
1969
1970
  // Enter the loop, setting up a phi for the current location to initialize.
1971
973
  llvm::BasicBlock *entryBB = Builder.GetInsertBlock();
1972
973
  llvm::BasicBlock *loopBB = createBasicBlock("arrayctor.loop");
1973
973
  EmitBlock(loopBB);
1974
973
  llvm::PHINode *cur = Builder.CreatePHI(arrayBegin->getType(), 2,
1975
973
                                         "arrayctor.cur");
1976
973
  cur->addIncoming(arrayBegin, entryBB);
1977
1978
  // Inside the loop body, emit the constructor call on the array element.
1979
1980
  // The alignment of the base, adjusted by the size of a single element,
1981
  // provides a conservative estimate of the alignment of every element.
1982
  // (This assumes we never start tracking offsetted alignments.)
1983
  //
1984
  // Note that these are complete objects and so we don't need to
1985
  // use the non-virtual size or alignment.
1986
973
  QualType type = getContext().getTypeDeclType(ctor->getParent());
1987
973
  CharUnits eltAlignment =
1988
973
    arrayBase.getAlignment()
1989
973
             .alignmentOfArrayElement(getContext().getTypeSizeInChars(type));
1990
973
  Address curAddr = Address(cur, eltAlignment);
1991
1992
  // Zero initialize the storage, if requested.
1993
973
  if (zeroInitialize)
1994
2
    EmitNullInitialization(curAddr, type);
1995
1996
  // C++ [class.temporary]p4:
1997
  // There are two contexts in which temporaries are destroyed at a different
1998
  // point than the end of the full-expression. The first context is when a
1999
  // default constructor is called to initialize an element of an array.
2000
  // If the constructor has one or more default arguments, the destruction of
2001
  // every temporary created in a default argument expression is sequenced
2002
  // before the construction of the next array element, if any.
2003
2004
973
  {
2005
973
    RunCleanupsScope Scope(*this);
2006
2007
    // Evaluate the constructor and its arguments in a regular
2008
    // partial-destroy cleanup.
2009
973
    if (getLangOpts().Exceptions &&
2010
85
        !ctor->getParent()->hasTrivialDestructor()) {
2011
38
      Destroyer *destroyer = destroyCXXObject;
2012
38
      pushRegularPartialArrayCleanup(arrayBegin, cur, type, eltAlignment,
2013
38
                                     *destroyer);
2014
38
    }
2015
973
    auto currAVS = AggValueSlot::forAddr(
2016
973
        curAddr, type.getQualifiers(), AggValueSlot::IsDestructed,
2017
973
        AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased,
2018
973
        AggValueSlot::DoesNotOverlap, AggValueSlot::IsNotZeroed,
2019
97
        NewPointerIsChecked ? AggValueSlot::IsSanitizerChecked
2020
876
                            : AggValueSlot::IsNotSanitizerChecked);
2021
973
    EmitCXXConstructorCall(ctor, Ctor_Complete, /*ForVirtualBase=*/false,
2022
973
                           /*Delegating=*/false, currAVS, E);
2023
973
  }
2024
2025
  // Go to the next element.
2026
973
  llvm::Value *next =
2027
973
    Builder.CreateInBoundsGEP(cur, llvm::ConstantInt::get(SizeTy, 1),
2028
973
                              "arrayctor.next");
2029
973
  cur->addIncoming(next, Builder.GetInsertBlock());
2030
2031
  // Check whether that's the end of the loop.
2032
973
  llvm::Value *done = Builder.CreateICmpEQ(next, arrayEnd, "arrayctor.done");
2033
973
  llvm::BasicBlock *contBB = createBasicBlock("arrayctor.cont");
2034
973
  Builder.CreateCondBr(done, contBB, loopBB);
2035
2036
  // Patch the earlier check to skip over the loop.
2037
973
  if (zeroCheckBranch) 
zeroCheckBranch->setSuccessor(0, contBB)41
;
2038
2039
973
  EmitBlock(contBB);
2040
973
}
2041
2042
void CodeGenFunction::destroyCXXObject(CodeGenFunction &CGF,
2043
                                       Address addr,
2044
16.9k
                                       QualType type) {
2045
16.9k
  const RecordType *rtype = type->castAs<RecordType>();
2046
16.9k
  const CXXRecordDecl *record = cast<CXXRecordDecl>(rtype->getDecl());
2047
16.9k
  const CXXDestructorDecl *dtor = record->getDestructor();
2048
16.9k
  assert(!dtor->isTrivial());
2049
16.9k
  CGF.EmitCXXDestructorCall(dtor, Dtor_Complete, /*for vbase*/ false,
2050
16.9k
                            /*Delegating=*/false, addr, type);
2051
16.9k
}
2052
2053
void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D,
2054
                                             CXXCtorType Type,
2055
                                             bool ForVirtualBase,
2056
                                             bool Delegating,
2057
                                             AggValueSlot ThisAVS,
2058
40.5k
                                             const CXXConstructExpr *E) {
2059
40.5k
  CallArgList Args;
2060
40.5k
  Address This = ThisAVS.getAddress();
2061
40.5k
  LangAS SlotAS = ThisAVS.getQualifiers().getAddressSpace();
2062
40.5k
  QualType ThisType = D->getThisType();
2063
40.5k
  LangAS ThisAS = ThisType.getTypePtr()->getPointeeType().getAddressSpace();
2064
40.5k
  llvm::Value *ThisPtr = This.getPointer();
2065
2066
40.5k
  if (SlotAS != ThisAS) {
2067
29
    unsigned TargetThisAS = getContext().getTargetAddressSpace(ThisAS);
2068
29
    llvm::Type *NewType =
2069
29
        ThisPtr->getType()->getPointerElementType()->getPointerTo(TargetThisAS);
2070
29
    ThisPtr = getTargetHooks().performAddrSpaceCast(*this, This.getPointer(),
2071
29
                                                    ThisAS, SlotAS, NewType);
2072
29
  }
2073
2074
  // Push the this ptr.
2075
40.5k
  Args.add(RValue::get(ThisPtr), D->getThisType());
2076
2077
  // If this is a trivial constructor, emit a memcpy now before we lose
2078
  // the alignment information on the argument.
2079
  // FIXME: It would be better to preserve alignment information into CallArg.
2080
40.5k
  if (isMemcpyEquivalentSpecialMember(D)) {
2081
5.37k
    assert(E->getNumArgs() == 1 && "unexpected argcount for trivial ctor");
2082
2083
5.37k
    const Expr *Arg = E->getArg(0);
2084
5.37k
    LValue Src = EmitLValue(Arg);
2085
5.37k
    QualType DestTy = getContext().getTypeDeclType(D->getParent());
2086
5.37k
    LValue Dest = MakeAddrLValue(This, DestTy);
2087
5.37k
    EmitAggregateCopyCtor(Dest, Src, ThisAVS.mayOverlap());
2088
5.37k
    return;
2089
5.37k
  }
2090
2091
  // Add the rest of the user-supplied arguments.
2092
35.2k
  const FunctionProtoType *FPT = D->getType()->castAs<FunctionProtoType>();
2093
35.2k
  EvaluationOrder Order = E->isListInitialization()
2094
415
                              ? EvaluationOrder::ForceLeftToRight
2095
34.7k
                              : EvaluationOrder::Default;
2096
35.2k
  EmitCallArgs(Args, FPT, E->arguments(), E->getConstructor(),
2097
35.2k
               /*ParamsToSkip*/ 0, Order);
2098
2099
35.2k
  EmitCXXConstructorCall(D, Type, ForVirtualBase, Delegating, This, Args,
2100
35.2k
                         ThisAVS.mayOverlap(), E->getExprLoc(),
2101
35.2k
                         ThisAVS.isSanitizerChecked());
2102
35.2k
}
2103
2104
static bool canEmitDelegateCallArgs(CodeGenFunction &CGF,
2105
                                    const CXXConstructorDecl *Ctor,
2106
225
                                    CXXCtorType Type, CallArgList &Args) {
2107
  // We can't forward a variadic call.
2108
225
  if (Ctor->isVariadic())
2109
31
    return false;
2110
2111
194
  if (CGF.getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) {
2112
    // If the parameters are callee-cleanup, it's not safe to forward.
2113
32
    for (auto *P : Ctor->parameters())
2114
56
      if (P->needsDestruction(CGF.getContext()))
2115
16
        return false;
2116
2117
    // Likewise if they're inalloca.
2118
16
    const CGFunctionInfo &Info =
2119
16
        CGF.CGM.getTypes().arrangeCXXConstructorCall(Args, Ctor, Type, 0, 0);
2120
16
    if (Info.usesInAlloca())
2121
0
      return false;
2122
178
  }
2123
2124
  // Anything else should be OK.
2125
178
  return true;
2126
178
}
2127
2128
void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D,
2129
                                             CXXCtorType Type,
2130
                                             bool ForVirtualBase,
2131
                                             bool Delegating,
2132
                                             Address This,
2133
                                             CallArgList &Args,
2134
                                             AggValueSlot::Overlap_t Overlap,
2135
                                             SourceLocation Loc,
2136
51.1k
                                             bool NewPointerIsChecked) {
2137
51.1k
  const CXXRecordDecl *ClassDecl = D->getParent();
2138
2139
51.1k
  if (!NewPointerIsChecked)
2140
30.4k
    EmitTypeCheck(CodeGenFunction::TCK_ConstructorCall, Loc, This.getPointer(),
2141
30.4k
                  getContext().getRecordType(ClassDecl), CharUnits::Zero());
2142
2143
51.1k
  if (D->isTrivial() && 
D->isDefaultConstructor()54
) {
2144
33
    assert(Args.size() == 1 && "trivial default ctor with args");
2145
33
    return;
2146
33
  }
2147
2148
  // If this is a trivial constructor, just emit what's needed. If this is a
2149
  // union copy constructor, we must emit a memcpy, because the AST does not
2150
  // model that copy.
2151
51.0k
  if (isMemcpyEquivalentSpecialMember(D)) {
2152
18
    assert(Args.size() == 2 && "unexpected argcount for trivial ctor");
2153
2154
18
    QualType SrcTy = D->getParamDecl(0)->getType().getNonReferenceType();
2155
18
    Address Src(Args[1].getRValue(*this).getScalarVal(),
2156
18
                CGM.getNaturalTypeAlignment(SrcTy));
2157
18
    LValue SrcLVal = MakeAddrLValue(Src, SrcTy);
2158
18
    QualType DestTy = getContext().getTypeDeclType(ClassDecl);
2159
18
    LValue DestLVal = MakeAddrLValue(This, DestTy);
2160
18
    EmitAggregateCopyCtor(DestLVal, SrcLVal, Overlap);
2161
18
    return;
2162
18
  }
2163
2164
51.0k
  bool PassPrototypeArgs = true;
2165
  // Check whether we can actually emit the constructor before trying to do so.
2166
51.0k
  if (auto Inherited = D->getInheritedConstructor()) {
2167
235
    PassPrototypeArgs = getTypes().inheritingCtorHasParams(Inherited, Type);
2168
235
    if (PassPrototypeArgs && 
!canEmitDelegateCallArgs(*this, D, Type, Args)225
) {
2169
47
      EmitInlinedInheritingCXXConstructorCall(D, Type, ForVirtualBase,
2170
47
                                              Delegating, Args);
2171
47
      return;
2172
47
    }
2173
51.0k
  }
2174
2175
  // Insert any ABI-specific implicit constructor arguments.
2176
51.0k
  CGCXXABI::AddedStructorArgCounts ExtraArgs =
2177
51.0k
      CGM.getCXXABI().addImplicitConstructorArgs(*this, D, Type, ForVirtualBase,
2178
51.0k
                                                 Delegating, Args);
2179
2180
  // Emit the call.
2181
51.0k
  llvm::Constant *CalleePtr = CGM.getAddrOfCXXStructor(GlobalDecl(D, Type));
2182
51.0k
  const CGFunctionInfo &Info = CGM.getTypes().arrangeCXXConstructorCall(
2183
51.0k
      Args, D, Type, ExtraArgs.Prefix, ExtraArgs.Suffix, PassPrototypeArgs);
2184
51.0k
  CGCallee Callee = CGCallee::forDirect(CalleePtr, GlobalDecl(D, Type));
2185
51.0k
  EmitCall(Info, Callee, ReturnValueSlot(), Args, nullptr, Loc);
2186
2187
  // Generate vtable assumptions if we're constructing a complete object
2188
  // with a vtable.  We don't do this for base subobjects for two reasons:
2189
  // first, it's incorrect for classes with virtual bases, and second, we're
2190
  // about to overwrite the vptrs anyway.
2191
  // We also have to make sure if we can refer to vtable:
2192
  // - Otherwise we can refer to vtable if it's safe to speculatively emit.
2193
  // FIXME: If vtable is used by ctor/dtor, or if vtable is external and we are
2194
  // sure that definition of vtable is not hidden,
2195
  // then we are always safe to refer to it.
2196
  // FIXME: It looks like InstCombine is very inefficient on dealing with
2197
  // assumes. Make assumption loads require -fstrict-vtable-pointers temporarily.
2198
51.0k
  if (CGM.getCodeGenOpts().OptimizationLevel > 0 &&
2199
1.35k
      ClassDecl->isDynamicClass() && 
Type != Ctor_Base589
&&
2200
326
      CGM.getCXXABI().canSpeculativelyEmitVTable(ClassDecl) &&
2201
114
      CGM.getCodeGenOpts().StrictVTablePointers)
2202
56
    EmitVTableAssumptionLoads(ClassDecl, This);
2203
51.0k
}
2204
2205
void CodeGenFunction::EmitInheritedCXXConstructorCall(
2206
    const CXXConstructorDecl *D, bool ForVirtualBase, Address This,
2207
200
    bool InheritedFromVBase, const CXXInheritedCtorInitExpr *E) {
2208
200
  CallArgList Args;
2209
200
  CallArg ThisArg(RValue::get(This.getPointer()), D->getThisType());
2210
2211
  // Forward the parameters.
2212
200
  if (InheritedFromVBase &&
2213
17
      CGM.getTarget().getCXXABI().hasConstructorVariants()) {
2214
    // Nothing to do; this construction is not responsible for constructing
2215
    // the base class containing the inherited constructor.
2216
    // FIXME: Can we just pass undef's for the remaining arguments if we don't
2217
    // have constructor variants?
2218
10
    Args.push_back(ThisArg);
2219
190
  } else if (!CXXInheritedCtorInitExprArgs.empty()) {
2220
    // The inheriting constructor was inlined; just inject its arguments.
2221
47
    assert(CXXInheritedCtorInitExprArgs.size() >= D->getNumParams() &&
2222
47
           "wrong number of parameters for inherited constructor call");
2223
47
    Args = CXXInheritedCtorInitExprArgs;
2224
47
    Args[0] = ThisArg;
2225
143
  } else {
2226
    // The inheriting constructor was not inlined. Emit delegating arguments.
2227
143
    Args.push_back(ThisArg);
2228
143
    const auto *OuterCtor = cast<CXXConstructorDecl>(CurCodeDecl);
2229
143
    assert(OuterCtor->getNumParams() == D->getNumParams());
2230
143
    assert(!OuterCtor->isVariadic() && "should have been inlined");
2231
2232
269
    for (const auto *Param : OuterCtor->parameters()) {
2233
269
      assert(getContext().hasSameUnqualifiedType(
2234
269
          OuterCtor->getParamDecl(Param->getFunctionScopeIndex())->getType(),
2235
269
          Param->getType()));
2236
269
      EmitDelegateCallArg(Args, Param, E->getLocation());
2237
2238
      // Forward __attribute__(pass_object_size).
2239
269
      if (Param->hasAttr<PassObjectSizeAttr>()) {
2240
22
        auto *POSParam = SizeArguments[Param];
2241
22
        assert(POSParam && "missing pass_object_size value for forwarding");
2242
22
        EmitDelegateCallArg(Args, POSParam, E->getLocation());
2243
22
      }
2244
269
    }
2245
143
  }
2246
2247
200
  EmitCXXConstructorCall(D, Ctor_Base, ForVirtualBase, /*Delegating*/false,
2248
200
                         This, Args, AggValueSlot::MayOverlap,
2249
200
                         E->getLocation(), /*NewPointerIsChecked*/true);
2250
200
}
2251
2252
void CodeGenFunction::EmitInlinedInheritingCXXConstructorCall(
2253
    const CXXConstructorDecl *Ctor, CXXCtorType CtorType, bool ForVirtualBase,
2254
47
    bool Delegating, CallArgList &Args) {
2255
47
  GlobalDecl GD(Ctor, CtorType);
2256
47
  InlinedInheritingConstructorScope Scope(*this, GD);
2257
47
  ApplyInlineDebugLocation DebugScope(*this, GD);
2258
47
  RunCleanupsScope RunCleanups(*this);
2259
2260
  // Save the arguments to be passed to the inherited constructor.
2261
47
  CXXInheritedCtorInitExprArgs = Args;
2262
2263
47
  FunctionArgList Params;
2264
47
  QualType RetType = BuildFunctionArgList(CurGD, Params);
2265
47
  FnRetTy = RetType;
2266
2267
  // Insert any ABI-specific implicit constructor arguments.
2268
47
  CGM.getCXXABI().addImplicitConstructorArgs(*this, Ctor, CtorType,
2269
47
                                             ForVirtualBase, Delegating, Args);
2270
2271
  // Emit a simplified prolog. We only need to emit the implicit params.
2272
47
  assert(Args.size() >= Params.size() && "too few arguments for call");
2273
332
  for (unsigned I = 0, N = Args.size(); I != N; 
++I285
) {
2274
285
    if (I < Params.size() && 
isa<ImplicitParamDecl>(Params[I])231
) {
2275
61
      const RValue &RV = Args[I].getRValue(*this);
2276
61
      assert(!RV.isComplex() && "complex indirect params not supported");
2277
61
      ParamValue Val = RV.isScalar()
2278
61
                           ? ParamValue::forDirect(RV.getScalarVal())
2279
0
                           : ParamValue::forIndirect(RV.getAggregateAddress());
2280
61
      EmitParmDecl(*Params[I], Val, I + 1);
2281
61
    }
2282
285
  }
2283
2284
  // Create a return value slot if the ABI implementation wants one.
2285
  // FIXME: This is dumb, we should ask the ABI not to try to set the return
2286
  // value instead.
2287
47
  if (!RetType->isVoidType())
2288
28
    ReturnValue = CreateIRTemp(RetType, "retval.inhctor");
2289
2290
47
  CGM.getCXXABI().EmitInstanceFunctionProlog(*this);
2291
47
  CXXThisValue = CXXABIThisValue;
2292
2293
  // Directly emit the constructor initializers.
2294
47
  EmitCtorPrologue(Ctor, CtorType, Params);
2295
47
}
2296
2297
62
void CodeGenFunction::EmitVTableAssumptionLoad(const VPtr &Vptr, Address This) {
2298
62
  llvm::Value *VTableGlobal =
2299
62
      CGM.getCXXABI().getVTableAddressPoint(Vptr.Base, Vptr.VTableClass);
2300
62
  if (!VTableGlobal)
2301
0
    return;
2302
2303
  // We can just use the base offset in the complete class.
2304
62
  CharUnits NonVirtualOffset = Vptr.Base.getBaseOffset();
2305
2306
62
  if (!NonVirtualOffset.isZero())
2307
4
    This =
2308
4
        ApplyNonVirtualAndVirtualOffset(*this, This, NonVirtualOffset, nullptr,
2309
4
                                        Vptr.VTableClass, Vptr.NearestVBase);
2310
2311
62
  llvm::Value *VPtrValue =
2312
62
      GetVTablePtr(This, VTableGlobal->getType(), Vptr.VTableClass);
2313
62
  llvm::Value *Cmp =
2314
62
      Builder.CreateICmpEQ(VPtrValue, VTableGlobal, "cmp.vtables");
2315
62
  Builder.CreateAssumption(Cmp);
2316
62
}
2317
2318
void CodeGenFunction::EmitVTableAssumptionLoads(const CXXRecordDecl *ClassDecl,
2319
56
                                                Address This) {
2320
56
  if (CGM.getCXXABI().doStructorsInitializeVPtrs(ClassDecl))
2321
56
    for (const VPtr &Vptr : getVTablePointers(ClassDecl))
2322
62
      EmitVTableAssumptionLoad(Vptr, This);
2323
56
}
2324
2325
void
2326
CodeGenFunction::EmitSynthesizedCXXCopyCtorCall(const CXXConstructorDecl *D,
2327
                                                Address This, Address Src,
2328
57
                                                const CXXConstructExpr *E) {
2329
57
  const FunctionProtoType *FPT = D->getType()->castAs<FunctionProtoType>();
2330
2331
57
  CallArgList Args;
2332
2333
  // Push the this ptr.
2334
57
  Args.add(RValue::get(This.getPointer()), D->getThisType());
2335
2336
  // Push the src ptr.
2337
57
  QualType QT = *(FPT->param_type_begin());
2338
57
  llvm::Type *t = CGM.getTypes().ConvertType(QT);
2339
57
  Src = Builder.CreateBitCast(Src, t);
2340
57
  Args.add(RValue::get(Src.getPointer()), QT);
2341
2342
  // Skip over first argument (Src).
2343
57
  EmitCallArgs(Args, FPT, drop_begin(E->arguments(), 1), E->getConstructor(),
2344
57
               /*ParamsToSkip*/ 1);
2345
2346
57
  EmitCXXConstructorCall(D, Ctor_Complete, /*ForVirtualBase*/false,
2347
57
                         /*Delegating*/false, This, Args,
2348
57
                         AggValueSlot::MayOverlap, E->getExprLoc(),
2349
57
                         /*NewPointerIsChecked*/false);
2350
57
}
2351
2352
void
2353
CodeGenFunction::EmitDelegateCXXConstructorCall(const CXXConstructorDecl *Ctor,
2354
                                                CXXCtorType CtorType,
2355
                                                const FunctionArgList &Args,
2356
15.6k
                                                SourceLocation Loc) {
2357
15.6k
  CallArgList DelegateArgs;
2358
2359
15.6k
  FunctionArgList::const_iterator I = Args.begin(), E = Args.end();
2360
15.6k
  assert(I != E && "no parameters to constructor");
2361
2362
  // this
2363
15.6k
  Address This = LoadCXXThisAddress();
2364
15.6k
  DelegateArgs.add(RValue::get(This.getPointer()), (*I)->getType());
2365
15.6k
  ++I;
2366
2367
  // FIXME: The location of the VTT parameter in the parameter list is
2368
  // specific to the Itanium ABI and shouldn't be hardcoded here.
2369
15.6k
  if (CGM.getCXXABI().NeedsVTTParameter(CurGD)) {
2370
0
    assert(I != E && "cannot skip vtt parameter, already done with args");
2371
0
    assert((*I)->getType()->isPointerType() &&
2372
0
           "skipping parameter not of vtt type");
2373
0
    ++I;
2374
0
  }
2375
2376
  // Explicit arguments.
2377
32.6k
  for (; I != E; 
++I17.0k
) {
2378
17.0k
    const VarDecl *param = *I;
2379
    // FIXME: per-argument source location
2380
17.0k
    EmitDelegateCallArg(DelegateArgs, param, Loc);
2381
17.0k
  }
2382
2383
15.6k
  EmitCXXConstructorCall(Ctor, CtorType, /*ForVirtualBase=*/false,
2384
15.6k
                         /*Delegating=*/true, This, DelegateArgs,
2385
15.6k
                         AggValueSlot::MayOverlap, Loc,
2386
15.6k
                         /*NewPointerIsChecked=*/true);
2387
15.6k
}
2388
2389
namespace {
2390
  struct CallDelegatingCtorDtor final : EHScopeStack::Cleanup {
2391
    const CXXDestructorDecl *Dtor;
2392
    Address Addr;
2393
    CXXDtorType Type;
2394
2395
    CallDelegatingCtorDtor(const CXXDestructorDecl *D, Address Addr,
2396
                           CXXDtorType Type)
2397
68
      : Dtor(D), Addr(Addr), Type(Type) {}
2398
2399
4
    void Emit(CodeGenFunction &CGF, Flags flags) override {
2400
      // We are calling the destructor from within the constructor.
2401
      // Therefore, "this" should have the expected type.
2402
4
      QualType ThisTy = Dtor->getThisObjectType();
2403
4
      CGF.EmitCXXDestructorCall(Dtor, Type, /*ForVirtualBase=*/false,
2404
4
                                /*Delegating=*/true, Addr, ThisTy);
2405
4
    }
2406
  };
2407
} // end anonymous namespace
2408
2409
void
2410
CodeGenFunction::EmitDelegatingCXXConstructorCall(const CXXConstructorDecl *Ctor,
2411
90
                                                  const FunctionArgList &Args) {
2412
90
  assert(Ctor->isDelegatingConstructor());
2413
2414
90
  Address ThisPtr = LoadCXXThisAddress();
2415
2416
90
  AggValueSlot AggSlot =
2417
90
    AggValueSlot::forAddr(ThisPtr, Qualifiers(),
2418
90
                          AggValueSlot::IsDestructed,
2419
90
                          AggValueSlot::DoesNotNeedGCBarriers,
2420
90
                          AggValueSlot::IsNotAliased,
2421
90
                          AggValueSlot::MayOverlap,
2422
90
                          AggValueSlot::IsNotZeroed,
2423
                          // Checks are made by the code that calls constructor.
2424
90
                          AggValueSlot::IsSanitizerChecked);
2425
2426
90
  EmitAggExpr(Ctor->init_begin()[0]->getInit(), AggSlot);
2427
2428
90
  const CXXRecordDecl *ClassDecl = Ctor->getParent();
2429
90
  if (CGM.getLangOpts().Exceptions && 
!ClassDecl->hasTrivialDestructor()88
) {
2430
68
    CXXDtorType Type =
2431
65
      CurGD.getCtorType() == Ctor_Complete ? Dtor_Complete : 
Dtor_Base3
;
2432
2433
68
    EHStack.pushCleanup<CallDelegatingCtorDtor>(EHCleanup,
2434
68
                                                ClassDecl->getDestructor(),
2435
68
                                                ThisPtr, Type);
2436
68
  }
2437
90
}
2438
2439
void CodeGenFunction::EmitCXXDestructorCall(const CXXDestructorDecl *DD,
2440
                                            CXXDtorType Type,
2441
                                            bool ForVirtualBase,
2442
                                            bool Delegating, Address This,
2443
27.5k
                                            QualType ThisTy) {
2444
27.5k
  CGM.getCXXABI().EmitDestructorCall(*this, DD, Type, ForVirtualBase,
2445
27.5k
                                     Delegating, This, ThisTy);
2446
27.5k
}
2447
2448
namespace {
2449
  struct CallLocalDtor final : EHScopeStack::Cleanup {
2450
    const CXXDestructorDecl *Dtor;
2451
    Address Addr;
2452
    QualType Ty;
2453
2454
    CallLocalDtor(const CXXDestructorDecl *D, Address Addr, QualType Ty)
2455
9
        : Dtor(D), Addr(Addr), Ty(Ty) {}
2456
2457
9
    void Emit(CodeGenFunction &CGF, Flags flags) override {
2458
9
      CGF.EmitCXXDestructorCall(Dtor, Dtor_Complete,
2459
9
                                /*ForVirtualBase=*/false,
2460
9
                                /*Delegating=*/false, Addr, Ty);
2461
9
    }
2462
  };
2463
} // end anonymous namespace
2464
2465
void CodeGenFunction::PushDestructorCleanup(const CXXDestructorDecl *D,
2466
9
                                            QualType T, Address Addr) {
2467
9
  EHStack.pushCleanup<CallLocalDtor>(NormalAndEHCleanup, D, Addr, T);
2468
9
}
2469
2470
12
void CodeGenFunction::PushDestructorCleanup(QualType T, Address Addr) {
2471
12
  CXXRecordDecl *ClassDecl = T->getAsCXXRecordDecl();
2472
12
  if (!ClassDecl) 
return0
;
2473
12
  if (ClassDecl->hasTrivialDestructor()) 
return3
;
2474
2475
9
  const CXXDestructorDecl *D = ClassDecl->getDestructor();
2476
9
  assert(D && D->isUsed() && "destructor not marked as used!");
2477
9
  PushDestructorCleanup(D, T, Addr);
2478
9
}
2479
2480
3.87k
void CodeGenFunction::InitializeVTablePointer(const VPtr &Vptr) {
2481
  // Compute the address point.
2482
3.87k
  llvm::Value *VTableAddressPoint =
2483
3.87k
      CGM.getCXXABI().getVTableAddressPointInStructor(
2484
3.87k
          *this, Vptr.VTableClass, Vptr.Base, Vptr.NearestVBase);
2485
2486
3.87k
  if (!VTableAddressPoint)
2487
719
    return;
2488
2489
  // Compute where to store the address point.
2490
3.15k
  llvm::Value *VirtualOffset = nullptr;
2491
3.15k
  CharUnits NonVirtualOffset = CharUnits::Zero();
2492
2493
3.15k
  if (CGM.getCXXABI().isVirtualOffsetNeededForVTableField(*this, Vptr)) {
2494
    // We need to use the virtual base offset offset because the virtual base
2495
    // might have a different offset in the most derived class.
2496
2497
351
    VirtualOffset = CGM.getCXXABI().GetVirtualBaseClassOffset(
2498
351
        *this, LoadCXXThisAddress(), Vptr.VTableClass, Vptr.NearestVBase);
2499
351
    NonVirtualOffset = Vptr.OffsetFromNearestVBase;
2500
2.80k
  } else {
2501
    // We can just use the base offset in the complete class.
2502
2.80k
    NonVirtualOffset = Vptr.Base.getBaseOffset();
2503
2.80k
  }
2504
2505
  // Apply the offsets.
2506
3.15k
  Address VTableField = LoadCXXThisAddress();
2507
2508
3.15k
  if (!NonVirtualOffset.isZero() || 
VirtualOffset2.92k
)
2509
554
    VTableField = ApplyNonVirtualAndVirtualOffset(
2510
554
        *this, VTableField, NonVirtualOffset, VirtualOffset, Vptr.VTableClass,
2511
554
        Vptr.NearestVBase);
2512
2513
  // Finally, store the address point. Use the same LLVM types as the field to
2514
  // support optimization.
2515
3.15k
  unsigned GlobalsAS = CGM.getDataLayout().getDefaultGlobalsAddressSpace();
2516
3.15k
  unsigned ProgAS = CGM.getDataLayout().getProgramAddressSpace();
2517
3.15k
  llvm::Type *VTablePtrTy =
2518
3.15k
      llvm::FunctionType::get(CGM.Int32Ty, /*isVarArg=*/true)
2519
3.15k
          ->getPointerTo(ProgAS)
2520
3.15k
          ->getPointerTo(GlobalsAS);
2521
3.15k
  VTableField = Builder.CreatePointerBitCastOrAddrSpaceCast(
2522
3.15k
      VTableField, VTablePtrTy->getPointerTo(GlobalsAS));
2523
3.15k
  VTableAddressPoint = Builder.CreatePointerBitCastOrAddrSpaceCast(
2524
3.15k
      VTableAddressPoint, VTablePtrTy);
2525
2526
3.15k
  llvm::StoreInst *Store = Builder.CreateStore(VTableAddressPoint, VTableField);
2527
3.15k
  TBAAAccessInfo TBAAInfo = CGM.getTBAAVTablePtrAccessInfo(VTablePtrTy);
2528
3.15k
  CGM.DecorateInstructionWithTBAA(Store, TBAAInfo);
2529
3.15k
  if (CGM.getCodeGenOpts().OptimizationLevel > 0 &&
2530
401
      CGM.getCodeGenOpts().StrictVTablePointers)
2531
54
    CGM.DecorateInstructionWithInvariantGroup(Store, Vptr.VTableClass);
2532
3.15k
}
2533
2534
CodeGenFunction::VPtrsVector
2535
2.93k
CodeGenFunction::getVTablePointers(const CXXRecordDecl *VTableClass) {
2536
2.93k
  CodeGenFunction::VPtrsVector VPtrsResult;
2537
2.93k
  VisitedVirtualBasesSetTy VBases;
2538
2.93k
  getVTablePointers(BaseSubobject(VTableClass, CharUnits::Zero()),
2539
2.93k
                    /*NearestVBase=*/nullptr,
2540
2.93k
                    /*OffsetFromNearestVBase=*/CharUnits::Zero(),
2541
2.93k
                    /*BaseIsNonVirtualPrimaryBase=*/false, VTableClass, VBases,
2542
2.93k
                    VPtrsResult);
2543
2.93k
  return VPtrsResult;
2544
2.93k
}
2545
2546
void CodeGenFunction::getVTablePointers(BaseSubobject Base,
2547
                                        const CXXRecordDecl *NearestVBase,
2548
                                        CharUnits OffsetFromNearestVBase,
2549
                                        bool BaseIsNonVirtualPrimaryBase,
2550
                                        const CXXRecordDecl *VTableClass,
2551
                                        VisitedVirtualBasesSetTy &VBases,
2552
5.59k
                                        VPtrsVector &Vptrs) {
2553
  // If this base is a non-virtual primary base the address point has already
2554
  // been set.
2555
5.59k
  if (!BaseIsNonVirtualPrimaryBase) {
2556
    // Initialize the vtable pointer for this base.
2557
3.93k
    VPtr Vptr = {Base, NearestVBase, OffsetFromNearestVBase, VTableClass};
2558
3.93k
    Vptrs.push_back(Vptr);
2559
3.93k
  }
2560
2561
5.59k
  const CXXRecordDecl *RD = Base.getBase();
2562
2563
  // Traverse bases.
2564
3.42k
  for (const auto &I : RD->bases()) {
2565
3.42k
    auto *BaseDecl =
2566
3.42k
        cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
2567
2568
    // Ignore classes without a vtable.
2569
3.42k
    if (!BaseDecl->isDynamicClass())
2570
693
      continue;
2571
2572
2.73k
    CharUnits BaseOffset;
2573
2.73k
    CharUnits BaseOffsetFromNearestVBase;
2574
2.73k
    bool BaseDeclIsNonVirtualPrimaryBase;
2575
2576
2.73k
    if (I.isVirtual()) {
2577
      // Check if we've visited this virtual base before.
2578
668
      if (!VBases.insert(BaseDecl).second)
2579
71
        continue;
2580
2581
597
      const ASTRecordLayout &Layout =
2582
597
        getContext().getASTRecordLayout(VTableClass);
2583
2584
597
      BaseOffset = Layout.getVBaseClassOffset(BaseDecl);
2585
597
      BaseOffsetFromNearestVBase = CharUnits::Zero();
2586
597
      BaseDeclIsNonVirtualPrimaryBase = false;
2587
2.06k
    } else {
2588
2.06k
      const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
2589
2590
2.06k
      BaseOffset = Base.getBaseOffset() + Layout.getBaseClassOffset(BaseDecl);
2591
2.06k
      BaseOffsetFromNearestVBase =
2592
2.06k
        OffsetFromNearestVBase + Layout.getBaseClassOffset(BaseDecl);
2593
2.06k
      BaseDeclIsNonVirtualPrimaryBase = Layout.getPrimaryBase() == BaseDecl;
2594
2.06k
    }
2595
2596
2.66k
    getVTablePointers(
2597
2.66k
        BaseSubobject(BaseDecl, BaseOffset),
2598
2.06k
        I.isVirtual() ? 
BaseDecl597
: NearestVBase, BaseOffsetFromNearestVBase,
2599
2.66k
        BaseDeclIsNonVirtualPrimaryBase, VTableClass, VBases, Vptrs);
2600
2.66k
  }
2601
5.59k
}
2602
2603
22.8k
void CodeGenFunction::InitializeVTablePointers(const CXXRecordDecl *RD) {
2604
  // Ignore classes without a vtable.
2605
22.8k
  if (!RD->isDynamicClass())
2606
19.9k
    return;
2607
2608
  // Initialize the vtable pointers for this class and all of its bases.
2609
2.88k
  if (CGM.getCXXABI().doStructorsInitializeVPtrs(RD))
2610
2.87k
    for (const VPtr &Vptr : getVTablePointers(RD))
2611
3.87k
      InitializeVTablePointer(Vptr);
2612
2613
2.88k
  if (RD->getNumVBases())
2614
808
    CGM.getCXXABI().initializeHiddenVirtualInheritanceMembers(*this, RD);
2615
2.88k
}
2616
2617
llvm::Value *CodeGenFunction::GetVTablePtr(Address This,
2618
                                           llvm::Type *VTableTy,
2619
1.71k
                                           const CXXRecordDecl *RD) {
2620
1.71k
  Address VTablePtrSrc = Builder.CreateElementBitCast(This, VTableTy);
2621
1.71k
  llvm::Instruction *VTable = Builder.CreateLoad(VTablePtrSrc, "vtable");
2622
1.71k
  TBAAAccessInfo TBAAInfo = CGM.getTBAAVTablePtrAccessInfo(VTableTy);
2623
1.71k
  CGM.DecorateInstructionWithTBAA(VTable, TBAAInfo);
2624
2625
1.71k
  if (CGM.getCodeGenOpts().OptimizationLevel > 0 &&
2626
196
      CGM.getCodeGenOpts().StrictVTablePointers)
2627
118
    CGM.DecorateInstructionWithInvariantGroup(VTable, RD);
2628
2629
1.71k
  return VTable;
2630
1.71k
}
2631
2632
// If a class has a single non-virtual base and does not introduce or override
2633
// virtual member functions or fields, it will have the same layout as its base.
2634
// This function returns the least derived such class.
2635
//
2636
// Casting an instance of a base class to such a derived class is technically
2637
// undefined behavior, but it is a relatively common hack for introducing member
2638
// functions on class instances with specific properties (e.g. llvm::Operator)
2639
// that works under most compilers and should not have security implications, so
2640
// we allow it by default. It can be disabled with -fsanitize=cfi-cast-strict.
2641
static const CXXRecordDecl *
2642
60
LeastDerivedClassWithSameLayout(const CXXRecordDecl *RD) {
2643
60
  if (!RD->field_empty())
2644
1
    return RD;
2645
2646
59
  if (RD->getNumVBases() != 0)
2647
12
    return RD;
2648
2649
47
  if (RD->getNumBases() != 1)
2650
32
    return RD;
2651
2652
28
  
for (const CXXMethodDecl *MD : RD->methods())15
{
2653
28
    if (MD->isVirtual()) {
2654
      // Virtual member functions are only ok if they are implicit destructors
2655
      // because the implicit destructor will have the same semantics as the
2656
      // base class's destructor if no fields are added.
2657
9
      if (isa<CXXDestructorDecl>(MD) && 
MD->isImplicit()0
)
2658
0
        continue;
2659
9
      return RD;
2660
9
    }
2661
28
  }
2662
2663
6
  return LeastDerivedClassWithSameLayout(
2664
6
      RD->bases_begin()->getType()->getAsCXXRecordDecl());
2665
15
}
2666
2667
void CodeGenFunction::EmitTypeMetadataCodeForVCall(const CXXRecordDecl *RD,
2668
                                                   llvm::Value *VTable,
2669
824
                                                   SourceLocation Loc) {
2670
824
  if (SanOpts.has(SanitizerKind::CFIVCall))
2671
33
    EmitVTablePtrCheckForCall(RD, VTable, CodeGenFunction::CFITCK_VCall, Loc);
2672
791
  else if (CGM.getCodeGenOpts().WholeProgramVTables &&
2673
           // Don't insert type test assumes if we are forcing public std
2674
           // visibility.
2675
66
           !CGM.HasLTOVisibilityPublicStd(RD)) {
2676
63
    llvm::Metadata *MD =
2677
63
        CGM.CreateMetadataIdentifierForType(QualType(RD->getTypeForDecl(), 0));
2678
63
    llvm::Value *TypeId =
2679
63
        llvm::MetadataAsValue::get(CGM.getLLVMContext(), MD);
2680
2681
63
    llvm::Value *CastedVTable = Builder.CreateBitCast(VTable, Int8PtrTy);
2682
63
    llvm::Value *TypeTest =
2683
63
        Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::type_test),
2684
63
                           {CastedVTable, TypeId});
2685
63
    Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::assume), TypeTest);
2686
63
  }
2687
824
}
2688
2689
void CodeGenFunction::EmitVTablePtrCheckForCall(const CXXRecordDecl *RD,
2690
                                                llvm::Value *VTable,
2691
                                                CFITypeCheckKind TCK,
2692
41
                                                SourceLocation Loc) {
2693
41
  if (!SanOpts.has(SanitizerKind::CFICastStrict))
2694
39
    RD = LeastDerivedClassWithSameLayout(RD);
2695
2696
41
  EmitVTablePtrCheck(RD, VTable, TCK, Loc);
2697
41
}
2698
2699
void CodeGenFunction::EmitVTablePtrCheckForCast(QualType T,
2700
                                                llvm::Value *Derived,
2701
                                                bool MayBeNull,
2702
                                                CFITypeCheckKind TCK,
2703
21
                                                SourceLocation Loc) {
2704
21
  if (!getLangOpts().CPlusPlus)
2705
0
    return;
2706
2707
21
  auto *ClassTy = T->getAs<RecordType>();
2708
21
  if (!ClassTy)
2709
0
    return;
2710
2711
21
  const CXXRecordDecl *ClassDecl = cast<CXXRecordDecl>(ClassTy->getDecl());
2712
2713
21
  if (!ClassDecl->isCompleteDefinition() || !ClassDecl->isDynamicClass())
2714
0
    return;
2715
2716
21
  if (!SanOpts.has(SanitizerKind::CFICastStrict))
2717
15
    ClassDecl = LeastDerivedClassWithSameLayout(ClassDecl);
2718
2719
21
  llvm::BasicBlock *ContBlock = nullptr;
2720
2721
21
  if (MayBeNull) {
2722
14
    llvm::Value *DerivedNotNull =
2723
14
        Builder.CreateIsNotNull(Derived, "cast.nonnull");
2724
2725
14
    llvm::BasicBlock *CheckBlock = createBasicBlock("cast.check");
2726
14
    ContBlock = createBasicBlock("cast.cont");
2727
2728
14
    Builder.CreateCondBr(DerivedNotNull, CheckBlock, ContBlock);
2729
2730
14
    EmitBlock(CheckBlock);
2731
14
  }
2732
2733
21
  llvm::Value *VTable;
2734
21
  std::tie(VTable, ClassDecl) = CGM.getCXXABI().LoadVTablePtr(
2735
21
      *this, Address(Derived, getPointerAlign()), ClassDecl);
2736
2737
21
  EmitVTablePtrCheck(ClassDecl, VTable, TCK, Loc);
2738
2739
21
  if (MayBeNull) {
2740
14
    Builder.CreateBr(ContBlock);
2741
14
    EmitBlock(ContBlock);
2742
14
  }
2743
21
}
2744
2745
void CodeGenFunction::EmitVTablePtrCheck(const CXXRecordDecl *RD,
2746
                                         llvm::Value *VTable,
2747
                                         CFITypeCheckKind TCK,
2748
62
                                         SourceLocation Loc) {
2749
62
  if (!CGM.getCodeGenOpts().SanitizeCfiCrossDso &&
2750
60
      !CGM.HasHiddenLTOVisibility(RD))
2751
0
    return;
2752
2753
62
  SanitizerMask M;
2754
62
  llvm::SanitizerStatKind SSK;
2755
62
  switch (TCK) {
2756
33
  case CFITCK_VCall:
2757
33
    M = SanitizerKind::CFIVCall;
2758
33
    SSK = llvm::SanStat_CFI_VCall;
2759
33
    break;
2760
8
  case CFITCK_NVCall:
2761
8
    M = SanitizerKind::CFINVCall;
2762
8
    SSK = llvm::SanStat_CFI_NVCall;
2763
8
    break;
2764
6
  case CFITCK_DerivedCast:
2765
6
    M = SanitizerKind::CFIDerivedCast;
2766
6
    SSK = llvm::SanStat_CFI_DerivedCast;
2767
6
    break;
2768
15
  case CFITCK_UnrelatedCast:
2769
15
    M = SanitizerKind::CFIUnrelatedCast;
2770
15
    SSK = llvm::SanStat_CFI_UnrelatedCast;
2771
15
    break;
2772
0
  case CFITCK_ICall:
2773
0
  case CFITCK_NVMFCall:
2774
0
  case CFITCK_VMFCall:
2775
0
    llvm_unreachable("unexpected sanitizer kind");
2776
62
  }
2777
2778
62
  std::string TypeName = RD->getQualifiedNameAsString();
2779
62
  if (getContext().getSanitizerBlacklist().isBlacklistedType(M, TypeName))
2780
2
    return;
2781
2782
60
  SanitizerScope SanScope(this);
2783
60
  EmitSanitizerStatReport(SSK);
2784
2785
60
  llvm::Metadata *MD =
2786
60
      CGM.CreateMetadataIdentifierForType(QualType(RD->getTypeForDecl(), 0));
2787
60
  llvm::Value *TypeId = llvm::MetadataAsValue::get(getLLVMContext(), MD);
2788
2789
60
  llvm::Value *CastedVTable = Builder.CreateBitCast(VTable, Int8PtrTy);
2790
60
  llvm::Value *TypeTest = Builder.CreateCall(
2791
60
      CGM.getIntrinsic(llvm::Intrinsic::type_test), {CastedVTable, TypeId});
2792
2793
60
  llvm::Constant *StaticData[] = {
2794
60
      llvm::ConstantInt::get(Int8Ty, TCK),
2795
60
      EmitCheckSourceLocation(Loc),
2796
60
      EmitCheckTypeDescriptor(QualType(RD->getTypeForDecl(), 0)),
2797
60
  };
2798
2799
60
  auto CrossDsoTypeId = CGM.CreateCrossDsoCfiTypeId(MD);
2800
60
  if (CGM.getCodeGenOpts().SanitizeCfiCrossDso && 
CrossDsoTypeId2
) {
2801
2
    EmitCfiSlowPathCheck(M, TypeTest, CrossDsoTypeId, CastedVTable, StaticData);
2802
2
    return;
2803
2
  }
2804
2805
58
  if (CGM.getCodeGenOpts().SanitizeTrap.has(M)) {
2806
28
    EmitTrapCheck(TypeTest, SanitizerHandler::CFICheckFail);
2807
28
    return;
2808
28
  }
2809
2810
30
  llvm::Value *AllVtables = llvm::MetadataAsValue::get(
2811
30
      CGM.getLLVMContext(),
2812
30
      llvm::MDString::get(CGM.getLLVMContext(), "all-vtables"));
2813
30
  llvm::Value *ValidVtable = Builder.CreateCall(
2814
30
      CGM.getIntrinsic(llvm::Intrinsic::type_test), {CastedVTable, AllVtables});
2815
30
  EmitCheck(std::make_pair(TypeTest, M), SanitizerHandler::CFICheckFail,
2816
30
            StaticData, {CastedVTable, ValidVtable});
2817
30
}
2818
2819
979
bool CodeGenFunction::ShouldEmitVTableTypeCheckedLoad(const CXXRecordDecl *RD) {
2820
979
  if (!CGM.getCodeGenOpts().WholeProgramVTables ||
2821
80
      !CGM.HasHiddenLTOVisibility(RD))
2822
921
    return false;
2823
2824
58
  if (CGM.getCodeGenOpts().VirtualFunctionElimination)
2825
2
    return true;
2826
2827
56
  if (!SanOpts.has(SanitizerKind::CFIVCall) ||
2828
12
      !CGM.getCodeGenOpts().SanitizeTrap.has(SanitizerKind::CFIVCall))
2829
45
    return false;
2830
2831
11
  std::string TypeName = RD->getQualifiedNameAsString();
2832
11
  return !getContext().getSanitizerBlacklist().isBlacklistedType(
2833
11
      SanitizerKind::CFIVCall, TypeName);
2834
11
}
2835
2836
llvm::Value *CodeGenFunction::EmitVTableTypeCheckedLoad(
2837
13
    const CXXRecordDecl *RD, llvm::Value *VTable, uint64_t VTableByteOffset) {
2838
13
  SanitizerScope SanScope(this);
2839
2840
13
  EmitSanitizerStatReport(llvm::SanStat_CFI_VCall);
2841
2842
13
  llvm::Metadata *MD =
2843
13
      CGM.CreateMetadataIdentifierForType(QualType(RD->getTypeForDecl(), 0));
2844
13
  llvm::Value *TypeId = llvm::MetadataAsValue::get(CGM.getLLVMContext(), MD);
2845
2846
13
  llvm::Value *CastedVTable = Builder.CreateBitCast(VTable, Int8PtrTy);
2847
13
  llvm::Value *CheckedLoad = Builder.CreateCall(
2848
13
      CGM.getIntrinsic(llvm::Intrinsic::type_checked_load),
2849
13
      {CastedVTable, llvm::ConstantInt::get(Int32Ty, VTableByteOffset),
2850
13
       TypeId});
2851
13
  llvm::Value *CheckResult = Builder.CreateExtractValue(CheckedLoad, 1);
2852
2853
13
  std::string TypeName = RD->getQualifiedNameAsString();
2854
13
  if (SanOpts.has(SanitizerKind::CFIVCall) &&
2855
11
      !getContext().getSanitizerBlacklist().isBlacklistedType(
2856
11
          SanitizerKind::CFIVCall, TypeName)) {
2857
11
    EmitCheck(std::make_pair(CheckResult, SanitizerKind::CFIVCall),
2858
11
              SanitizerHandler::CFICheckFail, {}, {});
2859
11
  }
2860
2861
13
  return Builder.CreateBitCast(
2862
13
      Builder.CreateExtractValue(CheckedLoad, 0),
2863
13
      cast<llvm::PointerType>(VTable->getType())->getElementType());
2864
13
}
2865
2866
void CodeGenFunction::EmitForwardingCallToLambda(
2867
                                      const CXXMethodDecl *callOperator,
2868
69
                                      CallArgList &callArgs) {
2869
  // Get the address of the call operator.
2870
69
  const CGFunctionInfo &calleeFnInfo =
2871
69
    CGM.getTypes().arrangeCXXMethodDeclaration(callOperator);
2872
69
  llvm::Constant *calleePtr =
2873
69
    CGM.GetAddrOfFunction(GlobalDecl(callOperator),
2874
69
                          CGM.getTypes().GetFunctionType(calleeFnInfo));
2875
2876
  // Prepare the return slot.
2877
69
  const FunctionProtoType *FPT =
2878
69
    callOperator->getType()->castAs<FunctionProtoType>();
2879
69
  QualType resultType = FPT->getReturnType();
2880
69
  ReturnValueSlot returnSlot;
2881
69
  if (!resultType->isVoidType() &&
2882
33
      calleeFnInfo.getReturnInfo().getKind() == ABIArgInfo::Indirect &&
2883
1
      !hasScalarEvaluationKind(calleeFnInfo.getReturnType()))
2884
1
    returnSlot =
2885
1
        ReturnValueSlot(ReturnValue, resultType.isVolatileQualified(),
2886
1
                        /*IsUnused=*/false, /*IsExternallyDestructed=*/true);
2887
2888
  // We don't need to separately arrange the call arguments because
2889
  // the call can't be variadic anyway --- it's impossible to forward
2890
  // variadic arguments.
2891
2892
  // Now emit our call.
2893
69
  auto callee = CGCallee::forDirect(calleePtr, GlobalDecl(callOperator));
2894
69
  RValue RV = EmitCall(calleeFnInfo, callee, returnSlot, callArgs);
2895
2896
  // If necessary, copy the returned value into the slot.
2897
69
  if (!resultType->isVoidType() && 
returnSlot.isNull()33
) {
2898
32
    if (getLangOpts().ObjCAutoRefCount && 
resultType->isObjCRetainableType()5
) {
2899
2
      RV = RValue::get(EmitARCRetainAutoreleasedReturnValue(RV.getScalarVal()));
2900
2
    }
2901
32
    EmitReturnOfRValue(RV, resultType);
2902
32
  } else
2903
37
    EmitBranchThroughCleanup(ReturnBlock);
2904
69
}
2905
2906
13
void CodeGenFunction::EmitLambdaBlockInvokeBody() {
2907
13
  const BlockDecl *BD = BlockInfo->getBlockDecl();
2908
13
  const VarDecl *variable = BD->capture_begin()->getVariable();
2909
13
  const CXXRecordDecl *Lambda = variable->getType()->getAsCXXRecordDecl();
2910
13
  const CXXMethodDecl *CallOp = Lambda->getLambdaCallOperator();
2911
2912
13
  if (CallOp->isVariadic()) {
2913
    // FIXME: Making this work correctly is nasty because it requires either
2914
    // cloning the body of the call operator or making the call operator
2915
    // forward.
2916
0
    CGM.ErrorUnsupported(CurCodeDecl, "lambda conversion to variadic function");
2917
0
    return;
2918
0
  }
2919
2920
  // Start building arguments for forwarding call
2921
13
  CallArgList CallArgs;
2922
2923
13
  QualType ThisType = getContext().getPointerType(getContext().getRecordType(Lambda));
2924
13
  Address ThisPtr = GetAddrOfBlockDecl(variable);
2925
13
  CallArgs.add(RValue::get(ThisPtr.getPointer()), ThisType);
2926
2927
  // Add the rest of the parameters.
2928
13
  for (auto param : BD->parameters())
2929
1
    EmitDelegateCallArg(CallArgs, param, param->getBeginLoc());
2930
2931
13
  assert(!Lambda->isGenericLambda() &&
2932
13
            "generic lambda interconversion to block not implemented");
2933
13
  EmitForwardingCallToLambda(CallOp, CallArgs);
2934
13
}
2935
2936
56
void CodeGenFunction::EmitLambdaDelegatingInvokeBody(const CXXMethodDecl *MD) {
2937
56
  const CXXRecordDecl *Lambda = MD->getParent();
2938
2939
  // Start building arguments for forwarding call
2940
56
  CallArgList CallArgs;
2941
2942
56
  QualType ThisType = getContext().getPointerType(getContext().getRecordType(Lambda));
2943
56
  llvm::Value *ThisPtr = llvm::UndefValue::get(getTypes().ConvertType(ThisType));
2944
56
  CallArgs.add(RValue::get(ThisPtr), ThisType);
2945
2946
  // Add the rest of the parameters.
2947
56
  for (auto Param : MD->parameters())
2948
140
    EmitDelegateCallArg(CallArgs, Param, Param->getBeginLoc());
2949
2950
56
  const CXXMethodDecl *CallOp = Lambda->getLambdaCallOperator();
2951
  // For a generic lambda, find the corresponding call operator specialization
2952
  // to which the call to the static-invoker shall be forwarded.
2953
56
  if (Lambda->isGenericLambda()) {
2954
0
    assert(MD->isFunctionTemplateSpecialization());
2955
0
    const TemplateArgumentList *TAL = MD->getTemplateSpecializationArgs();
2956
0
    FunctionTemplateDecl *CallOpTemplate = CallOp->getDescribedFunctionTemplate();
2957
0
    void *InsertPos = nullptr;
2958
0
    FunctionDecl *CorrespondingCallOpSpecialization =
2959
0
        CallOpTemplate->findSpecialization(TAL->asArray(), InsertPos);
2960
0
    assert(CorrespondingCallOpSpecialization);
2961
0
    CallOp = cast<CXXMethodDecl>(CorrespondingCallOpSpecialization);
2962
0
  }
2963
56
  EmitForwardingCallToLambda(CallOp, CallArgs);
2964
56
}
2965
2966
56
void CodeGenFunction::EmitLambdaStaticInvokeBody(const CXXMethodDecl *MD) {
2967
56
  if (MD->isVariadic()) {
2968
    // FIXME: Making this work correctly is nasty because it requires either
2969
    // cloning the body of the call operator or making the call operator forward.
2970
0
    CGM.ErrorUnsupported(MD, "lambda conversion to variadic function");
2971
0
    return;
2972
0
  }
2973
2974
56
  EmitLambdaDelegatingInvokeBody(MD);
2975
56
}