Coverage Report

Created: 2020-09-22 08:39

/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/CodeGen/CGClass.cpp
Line
Count
Source (jump to first uncovered line)
1
//===--- CGClass.cpp - Emit LLVM Code for C++ classes -----------*- C++ -*-===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This contains code dealing with C++ code generation of classes
10
//
11
//===----------------------------------------------------------------------===//
12
13
#include "CGBlocks.h"
14
#include "CGCXXABI.h"
15
#include "CGDebugInfo.h"
16
#include "CGRecordLayout.h"
17
#include "CodeGenFunction.h"
18
#include "TargetInfo.h"
19
#include "clang/AST/Attr.h"
20
#include "clang/AST/CXXInheritance.h"
21
#include "clang/AST/DeclTemplate.h"
22
#include "clang/AST/EvaluatedExprVisitor.h"
23
#include "clang/AST/RecordLayout.h"
24
#include "clang/AST/StmtCXX.h"
25
#include "clang/Basic/CodeGenOptions.h"
26
#include "clang/Basic/TargetBuiltins.h"
27
#include "clang/CodeGen/CGFunctionInfo.h"
28
#include "llvm/IR/Intrinsics.h"
29
#include "llvm/IR/Metadata.h"
30
#include "llvm/Transforms/Utils/SanitizerStats.h"
31
32
using namespace clang;
33
using namespace CodeGen;
34
35
/// Return the best known alignment for an unknown pointer to a
36
/// particular class.
37
381k
CharUnits CodeGenModule::getClassPointerAlignment(const CXXRecordDecl *RD) {
38
381k
  if (!RD->hasDefinition())
39
0
    return CharUnits::One(); // Hopefully won't be used anywhere.
40
41
381k
  auto &layout = getContext().getASTRecordLayout(RD);
42
43
  // If the class is final, then we know that the pointer points to an
44
  // object of that type and can use the full alignment.
45
381k
  if (RD->isEffectivelyFinal())
46
92
    return layout.getAlignment();
47
48
  // Otherwise, we have to assume it could be a subclass.
49
381k
  return layout.getNonVirtualAlignment();
50
381k
}
51
52
/// Return the smallest possible amount of storage that might be allocated
53
/// starting from the beginning of an object of a particular class.
54
///
55
/// This may be smaller than sizeof(RD) if RD has virtual base classes.
56
100k
CharUnits CodeGenModule::getMinimumClassObjectSize(const CXXRecordDecl *RD) {
57
100k
  if (!RD->hasDefinition())
58
0
    return CharUnits::One();
59
60
100k
  auto &layout = getContext().getASTRecordLayout(RD);
61
62
  // If the class is final, then we know that the pointer points to an
63
  // object of that type and can use the full alignment.
64
100k
  if (RD->isEffectivelyFinal())
65
7
    return layout.getSize();
66
67
  // Otherwise, we have to assume it could be a subclass.
68
100k
  return std::max(layout.getNonVirtualSize(), CharUnits::One());
69
100k
}
70
71
/// Return the best known alignment for a pointer to a virtual base,
72
/// given the alignment of a pointer to the derived class.
73
CharUnits CodeGenModule::getVBaseAlignment(CharUnits actualDerivedAlign,
74
                                           const CXXRecordDecl *derivedClass,
75
863
                                           const CXXRecordDecl *vbaseClass) {
76
  // The basic idea here is that an underaligned derived pointer might
77
  // indicate an underaligned base pointer.
78
79
863
  assert(vbaseClass->isCompleteDefinition());
80
863
  auto &baseLayout = getContext().getASTRecordLayout(vbaseClass);
81
863
  CharUnits expectedVBaseAlign = baseLayout.getNonVirtualAlignment();
82
83
863
  return getDynamicOffsetAlignment(actualDerivedAlign, derivedClass,
84
863
                                   expectedVBaseAlign);
85
863
}
86
87
CharUnits
88
CodeGenModule::getDynamicOffsetAlignment(CharUnits actualBaseAlign,
89
                                         const CXXRecordDecl *baseDecl,
90
1.03k
                                         CharUnits expectedTargetAlign) {
91
  // If the base is an incomplete type (which is, alas, possible with
92
  // member pointers), be pessimistic.
93
1.03k
  if (!baseDecl->isCompleteDefinition())
94
5
    return std::min(actualBaseAlign, expectedTargetAlign);
95
96
1.02k
  auto &baseLayout = getContext().getASTRecordLayout(baseDecl);
97
1.02k
  CharUnits expectedBaseAlign = baseLayout.getNonVirtualAlignment();
98
99
  // If the class is properly aligned, assume the target offset is, too.
100
  //
101
  // This actually isn't necessarily the right thing to do --- if the
102
  // class is a complete object, but it's only properly aligned for a
103
  // base subobject, then the alignments of things relative to it are
104
  // probably off as well.  (Note that this requires the alignment of
105
  // the target to be greater than the NV alignment of the derived
106
  // class.)
107
  //
108
  // However, our approach to this kind of under-alignment can only
109
  // ever be best effort; after all, we're never going to propagate
110
  // alignments through variables or parameters.  Note, in particular,
111
  // that constructing a polymorphic type in an address that's less
112
  // than pointer-aligned will generally trap in the constructor,
113
  // unless we someday add some sort of attribute to change the
114
  // assumed alignment of 'this'.  So our goal here is pretty much
115
  // just to allow the user to explicitly say that a pointer is
116
  // under-aligned and then safely access its fields and vtables.
117
1.02k
  if (actualBaseAlign >= expectedBaseAlign) {
118
1.02k
    return expectedTargetAlign;
119
1.02k
  }
120
121
  // Otherwise, we might be offset by an arbitrary multiple of the
122
  // actual alignment.  The correct adjustment is to take the min of
123
  // the two alignments.
124
0
  return std::min(actualBaseAlign, expectedTargetAlign);
125
0
}
126
127
63.2k
Address CodeGenFunction::LoadCXXThisAddress() {
128
63.2k
  assert(CurFuncDecl && "loading 'this' without a func declaration?");
129
63.2k
  assert(isa<CXXMethodDecl>(CurFuncDecl));
130
131
  // Lazily compute CXXThisAlignment.
132
63.2k
  if (CXXThisAlignment.isZero()) {
133
    // Just use the best known alignment for the parent.
134
    // TODO: if we're currently emitting a complete-object ctor/dtor,
135
    // we can always use the complete-object alignment.
136
48.9k
    auto RD = cast<CXXMethodDecl>(CurFuncDecl)->getParent();
137
48.9k
    CXXThisAlignment = CGM.getClassPointerAlignment(RD);
138
48.9k
  }
139
140
63.2k
  return Address(LoadCXXThis(), CXXThisAlignment);
141
63.2k
}
142
143
/// Emit the address of a field using a member data pointer.
144
///
145
/// \param E Only used for emergency diagnostics
146
Address
147
CodeGenFunction::EmitCXXMemberDataPointerAddress(const Expr *E, Address base,
148
                                                 llvm::Value *memberPtr,
149
                                      const MemberPointerType *memberPtrType,
150
                                                 LValueBaseInfo *BaseInfo,
151
81
                                                 TBAAAccessInfo *TBAAInfo) {
152
  // Ask the ABI to compute the actual address.
153
81
  llvm::Value *ptr =
154
81
    CGM.getCXXABI().EmitMemberDataPointerAddress(*this, E, base,
155
81
                                                 memberPtr, memberPtrType);
156
157
81
  QualType memberType = memberPtrType->getPointeeType();
158
81
  CharUnits memberAlign =
159
81
      CGM.getNaturalTypeAlignment(memberType, BaseInfo, TBAAInfo);
160
81
  memberAlign =
161
81
    CGM.getDynamicOffsetAlignment(base.getAlignment(),
162
81
                            memberPtrType->getClass()->getAsCXXRecordDecl(),
163
81
                                  memberAlign);
164
81
  return Address(ptr, memberAlign);
165
81
}
166
167
CharUnits CodeGenModule::computeNonVirtualBaseClassOffset(
168
    const CXXRecordDecl *DerivedClass, CastExpr::path_const_iterator Start,
169
21.7k
    CastExpr::path_const_iterator End) {
170
21.7k
  CharUnits Offset = CharUnits::Zero();
171
172
21.7k
  const ASTContext &Context = getContext();
173
21.7k
  const CXXRecordDecl *RD = DerivedClass;
174
175
45.6k
  for (CastExpr::path_const_iterator I = Start; I != End; 
++I23.9k
) {
176
23.9k
    const CXXBaseSpecifier *Base = *I;
177
23.9k
    assert(!Base->isVirtual() && "Should not see virtual bases here!");
178
179
    // Get the layout.
180
23.9k
    const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
181
182
23.9k
    const auto *BaseDecl =
183
23.9k
        cast<CXXRecordDecl>(Base->getType()->castAs<RecordType>()->getDecl());
184
185
    // Add the offset.
186
23.9k
    Offset += Layout.getBaseClassOffset(BaseDecl);
187
188
23.9k
    RD = BaseDecl;
189
23.9k
  }
190
191
21.7k
  return Offset;
192
21.7k
}
193
194
llvm::Constant *
195
CodeGenModule::GetNonVirtualBaseClassOffset(const CXXRecordDecl *ClassDecl,
196
                                   CastExpr::path_const_iterator PathBegin,
197
748
                                   CastExpr::path_const_iterator PathEnd) {
198
748
  assert(PathBegin != PathEnd && "Base path should not be empty!");
199
200
748
  CharUnits Offset =
201
748
      computeNonVirtualBaseClassOffset(ClassDecl, PathBegin, PathEnd);
202
748
  if (Offset.isZero())
203
711
    return nullptr;
204
205
37
  llvm::Type *PtrDiffTy =
206
37
  Types.ConvertType(getContext().getPointerDiffType());
207
208
37
  return llvm::ConstantInt::get(PtrDiffTy, Offset.getQuantity());
209
37
}
210
211
/// Gets the address of a direct base class within a complete object.
212
/// This should only be used for (1) non-virtual bases or (2) virtual bases
213
/// when the type is known to be complete (e.g. in complete destructors).
214
///
215
/// The object pointed to by 'This' is assumed to be non-null.
216
Address
217
CodeGenFunction::GetAddressOfDirectBaseInCompleteClass(Address This,
218
                                                   const CXXRecordDecl *Derived,
219
                                                   const CXXRecordDecl *Base,
220
11.2k
                                                   bool BaseIsVirtual) {
221
  // 'this' must be a pointer (in some address space) to Derived.
222
11.2k
  assert(This.getElementType() == ConvertType(Derived));
223
224
  // Compute the offset of the virtual base.
225
11.2k
  CharUnits Offset;
226
11.2k
  const ASTRecordLayout &Layout = getContext().getASTRecordLayout(Derived);
227
11.2k
  if (BaseIsVirtual)
228
997
    Offset = Layout.getVBaseClassOffset(Base);
229
10.2k
  else
230
10.2k
    Offset = Layout.getBaseClassOffset(Base);
231
232
  // Shift and cast down to the base type.
233
  // TODO: for complete types, this should be possible with a GEP.
234
11.2k
  Address V = This;
235
11.2k
  if (!Offset.isZero()) {
236
1.95k
    V = Builder.CreateElementBitCast(V, Int8Ty);
237
1.95k
    V = Builder.CreateConstInBoundsByteGEP(V, Offset);
238
1.95k
  }
239
11.2k
  V = Builder.CreateElementBitCast(V, ConvertType(Base));
240
241
11.2k
  return V;
242
11.2k
}
243
244
static Address
245
ApplyNonVirtualAndVirtualOffset(CodeGenFunction &CGF, Address addr,
246
                                CharUnits nonVirtualOffset,
247
                                llvm::Value *virtualOffset,
248
                                const CXXRecordDecl *derivedClass,
249
2.00k
                                const CXXRecordDecl *nearestVBase) {
250
  // Assert that we have something to do.
251
2.00k
  assert(!nonVirtualOffset.isZero() || virtualOffset != nullptr);
252
253
  // Compute the offset from the static and dynamic components.
254
2.00k
  llvm::Value *baseOffset;
255
2.00k
  if (!nonVirtualOffset.isZero()) {
256
1.28k
    llvm::Type *OffsetType =
257
1.28k
        (CGF.CGM.getTarget().getCXXABI().isItaniumFamily() &&
258
1.14k
         CGF.CGM.getItaniumVTableContext().isRelativeLayout())
259
0
            ? CGF.Int32Ty
260
1.28k
            : CGF.PtrDiffTy;
261
1.28k
    baseOffset =
262
1.28k
        llvm::ConstantInt::get(OffsetType, nonVirtualOffset.getQuantity());
263
1.28k
    if (virtualOffset) {
264
34
      baseOffset = CGF.Builder.CreateAdd(virtualOffset, baseOffset);
265
34
    }
266
724
  } else {
267
724
    baseOffset = virtualOffset;
268
724
  }
269
270
  // Apply the base offset.
271
2.00k
  llvm::Value *ptr = addr.getPointer();
272
2.00k
  unsigned AddrSpace = ptr->getType()->getPointerAddressSpace();
273
2.00k
  ptr = CGF.Builder.CreateBitCast(ptr, CGF.Int8Ty->getPointerTo(AddrSpace));
274
2.00k
  ptr = CGF.Builder.CreateInBoundsGEP(ptr, baseOffset, "add.ptr");
275
276
  // If we have a virtual component, the alignment of the result will
277
  // be relative only to the known alignment of that vbase.
278
2.00k
  CharUnits alignment;
279
2.00k
  if (virtualOffset) {
280
758
    assert(nearestVBase && "virtual offset without vbase?");
281
758
    alignment = CGF.CGM.getVBaseAlignment(addr.getAlignment(),
282
758
                                          derivedClass, nearestVBase);
283
1.25k
  } else {
284
1.25k
    alignment = addr.getAlignment();
285
1.25k
  }
286
2.00k
  alignment = alignment.alignmentAtOffset(nonVirtualOffset);
287
288
2.00k
  return Address(ptr, alignment);
289
2.00k
}
290
291
Address CodeGenFunction::GetAddressOfBaseClass(
292
    Address Value, const CXXRecordDecl *Derived,
293
    CastExpr::path_const_iterator PathBegin,
294
    CastExpr::path_const_iterator PathEnd, bool NullCheckValue,
295
20.9k
    SourceLocation Loc) {
296
20.9k
  assert(PathBegin != PathEnd && "Base path should not be empty!");
297
298
20.9k
  CastExpr::path_const_iterator Start = PathBegin;
299
20.9k
  const CXXRecordDecl *VBase = nullptr;
300
301
  // Sema has done some convenient canonicalization here: if the
302
  // access path involved any virtual steps, the conversion path will
303
  // *start* with a step down to the correct virtual base subobject,
304
  // and hence will not require any further steps.
305
20.9k
  if ((*Start)->isVirtual()) {
306
415
    VBase = cast<CXXRecordDecl>(
307
415
        (*Start)->getType()->castAs<RecordType>()->getDecl());
308
415
    ++Start;
309
415
  }
310
311
  // Compute the static offset of the ultimate destination within its
312
  // allocating subobject (the virtual base, if there is one, or else
313
  // the "complete" object that we see).
314
20.9k
  CharUnits NonVirtualOffset = CGM.computeNonVirtualBaseClassOffset(
315
20.4k
      VBase ? 
VBase415
: Derived, Start, PathEnd);
316
317
  // If there's a virtual step, we can sometimes "devirtualize" it.
318
  // For now, that's limited to when the derived type is final.
319
  // TODO: "devirtualize" this for accesses to known-complete objects.
320
20.9k
  if (VBase && 
Derived->hasAttr<FinalAttr>()415
) {
321
6
    const ASTRecordLayout &layout = getContext().getASTRecordLayout(Derived);
322
6
    CharUnits vBaseOffset = layout.getVBaseClassOffset(VBase);
323
6
    NonVirtualOffset += vBaseOffset;
324
6
    VBase = nullptr; // we no longer have a virtual step
325
6
  }
326
327
  // Get the base pointer type.
328
20.9k
  llvm::Type *BasePtrTy =
329
20.9k
      ConvertType((PathEnd[-1])->getType())
330
20.9k
          ->getPointerTo(Value.getType()->getPointerAddressSpace());
331
332
20.9k
  QualType DerivedTy = getContext().getRecordType(Derived);
333
20.9k
  CharUnits DerivedAlign = CGM.getClassPointerAlignment(Derived);
334
335
  // If the static offset is zero and we don't have a virtual step,
336
  // just do a bitcast; null checks are unnecessary.
337
20.9k
  if (NonVirtualOffset.isZero() && 
!VBase19.8k
) {
338
19.4k
    if (sanitizePerformTypeCheck()) {
339
19
      SanitizerSet SkippedChecks;
340
19
      SkippedChecks.set(SanitizerKind::Null, !NullCheckValue);
341
19
      EmitTypeCheck(TCK_Upcast, Loc, Value.getPointer(),
342
19
                    DerivedTy, DerivedAlign, SkippedChecks);
343
19
    }
344
19.4k
    return Builder.CreateBitCast(Value, BasePtrTy);
345
19.4k
  }
346
347
1.45k
  llvm::BasicBlock *origBB = nullptr;
348
1.45k
  llvm::BasicBlock *endBB = nullptr;
349
350
  // Skip over the offset (and the vtable load) if we're supposed to
351
  // null-check the pointer.
352
1.45k
  if (NullCheckValue) {
353
35
    origBB = Builder.GetInsertBlock();
354
35
    llvm::BasicBlock *notNullBB = createBasicBlock("cast.notnull");
355
35
    endBB = createBasicBlock("cast.end");
356
357
35
    llvm::Value *isNull = Builder.CreateIsNull(Value.getPointer());
358
35
    Builder.CreateCondBr(isNull, endBB, notNullBB);
359
35
    EmitBlock(notNullBB);
360
35
  }
361
362
1.45k
  if (sanitizePerformTypeCheck()) {
363
3
    SanitizerSet SkippedChecks;
364
3
    SkippedChecks.set(SanitizerKind::Null, true);
365
3
    EmitTypeCheck(VBase ? TCK_UpcastToVirtualBase : 
TCK_Upcast0
, Loc,
366
3
                  Value.getPointer(), DerivedTy, DerivedAlign, SkippedChecks);
367
3
  }
368
369
  // Compute the virtual offset.
370
1.45k
  llvm::Value *VirtualOffset = nullptr;
371
1.45k
  if (VBase) {
372
409
    VirtualOffset =
373
409
      CGM.getCXXABI().GetVirtualBaseClassOffset(*this, Value, Derived, VBase);
374
409
  }
375
376
  // Apply both offsets.
377
1.45k
  Value = ApplyNonVirtualAndVirtualOffset(*this, Value, NonVirtualOffset,
378
1.45k
                                          VirtualOffset, Derived, VBase);
379
380
  // Cast to the destination type.
381
1.45k
  Value = Builder.CreateBitCast(Value, BasePtrTy);
382
383
  // Build a phi if we needed a null check.
384
1.45k
  if (NullCheckValue) {
385
35
    llvm::BasicBlock *notNullBB = Builder.GetInsertBlock();
386
35
    Builder.CreateBr(endBB);
387
35
    EmitBlock(endBB);
388
389
35
    llvm::PHINode *PHI = Builder.CreatePHI(BasePtrTy, 2, "cast.result");
390
35
    PHI->addIncoming(Value.getPointer(), notNullBB);
391
35
    PHI->addIncoming(llvm::Constant::getNullValue(BasePtrTy), origBB);
392
35
    Value = Address(PHI, Value.getAlignment());
393
35
  }
394
395
1.45k
  return Value;
396
1.45k
}
397
398
Address
399
CodeGenFunction::GetAddressOfDerivedClass(Address BaseAddr,
400
                                          const CXXRecordDecl *Derived,
401
                                        CastExpr::path_const_iterator PathBegin,
402
                                          CastExpr::path_const_iterator PathEnd,
403
689
                                          bool NullCheckValue) {
404
689
  assert(PathBegin != PathEnd && "Base path should not be empty!");
405
406
689
  QualType DerivedTy =
407
689
    getContext().getCanonicalType(getContext().getTagDeclType(Derived));
408
689
  unsigned AddrSpace =
409
689
    BaseAddr.getPointer()->getType()->getPointerAddressSpace();
410
689
  llvm::Type *DerivedPtrTy = ConvertType(DerivedTy)->getPointerTo(AddrSpace);
411
412
689
  llvm::Value *NonVirtualOffset =
413
689
    CGM.GetNonVirtualBaseClassOffset(Derived, PathBegin, PathEnd);
414
415
689
  if (!NonVirtualOffset) {
416
    // No offset, we can just cast back.
417
678
    return Builder.CreateBitCast(BaseAddr, DerivedPtrTy);
418
678
  }
419
420
11
  llvm::BasicBlock *CastNull = nullptr;
421
11
  llvm::BasicBlock *CastNotNull = nullptr;
422
11
  llvm::BasicBlock *CastEnd = nullptr;
423
424
11
  if (NullCheckValue) {
425
5
    CastNull = createBasicBlock("cast.null");
426
5
    CastNotNull = createBasicBlock("cast.notnull");
427
5
    CastEnd = createBasicBlock("cast.end");
428
429
5
    llvm::Value *IsNull = Builder.CreateIsNull(BaseAddr.getPointer());
430
5
    Builder.CreateCondBr(IsNull, CastNull, CastNotNull);
431
5
    EmitBlock(CastNotNull);
432
5
  }
433
434
  // Apply the offset.
435
11
  llvm::Value *Value = Builder.CreateBitCast(BaseAddr.getPointer(), Int8PtrTy);
436
11
  Value = Builder.CreateInBoundsGEP(Value, Builder.CreateNeg(NonVirtualOffset),
437
11
                                    "sub.ptr");
438
439
  // Just cast.
440
11
  Value = Builder.CreateBitCast(Value, DerivedPtrTy);
441
442
  // Produce a PHI if we had a null-check.
443
11
  if (NullCheckValue) {
444
5
    Builder.CreateBr(CastEnd);
445
5
    EmitBlock(CastNull);
446
5
    Builder.CreateBr(CastEnd);
447
5
    EmitBlock(CastEnd);
448
449
5
    llvm::PHINode *PHI = Builder.CreatePHI(Value->getType(), 2);
450
5
    PHI->addIncoming(Value, CastNotNull);
451
5
    PHI->addIncoming(llvm::Constant::getNullValue(Value->getType()), CastNull);
452
5
    Value = PHI;
453
5
  }
454
455
11
  return Address(Value, CGM.getClassPointerAlignment(Derived));
456
11
}
457
458
llvm::Value *CodeGenFunction::GetVTTParameter(GlobalDecl GD,
459
                                              bool ForVirtualBase,
460
26.2k
                                              bool Delegating) {
461
26.2k
  if (!CGM.getCXXABI().NeedsVTTParameter(GD)) {
462
    // This constructor/destructor does not need a VTT parameter.
463
25.9k
    return nullptr;
464
25.9k
  }
465
466
270
  const CXXRecordDecl *RD = cast<CXXMethodDecl>(CurCodeDecl)->getParent();
467
270
  const CXXRecordDecl *Base = cast<CXXMethodDecl>(GD.getDecl())->getParent();
468
469
270
  llvm::Value *VTT;
470
471
270
  uint64_t SubVTTIndex;
472
473
270
  if (Delegating) {
474
    // If this is a delegating constructor call, just load the VTT.
475
2
    return LoadCXXVTT();
476
268
  } else if (RD == Base) {
477
    // If the record matches the base, this is the complete ctor/dtor
478
    // variant calling the base variant in a class with virtual bases.
479
94
    assert(!CGM.getCXXABI().NeedsVTTParameter(CurGD) &&
480
94
           "doing no-op VTT offset in base dtor/ctor?");
481
94
    assert(!ForVirtualBase && "Can't have same class as virtual base!");
482
94
    SubVTTIndex = 0;
483
174
  } else {
484
174
    const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
485
174
    CharUnits BaseOffset = ForVirtualBase ?
486
7
      Layout.getVBaseClassOffset(Base) :
487
167
      Layout.getBaseClassOffset(Base);
488
489
174
    SubVTTIndex =
490
174
      CGM.getVTables().getSubVTTIndex(RD, BaseSubobject(Base, BaseOffset));
491
174
    assert(SubVTTIndex != 0 && "Sub-VTT index must be greater than zero!");
492
174
  }
493
494
268
  if (CGM.getCXXABI().NeedsVTTParameter(CurGD)) {
495
    // A VTT parameter was passed to the constructor, use it.
496
40
    VTT = LoadCXXVTT();
497
40
    VTT = Builder.CreateConstInBoundsGEP1_64(VTT, SubVTTIndex);
498
228
  } else {
499
    // We're the complete constructor, so get the VTT by name.
500
228
    VTT = CGM.getVTables().GetAddrOfVTT(RD);
501
228
    VTT = Builder.CreateConstInBoundsGEP2_64(VTT, 0, SubVTTIndex);
502
228
  }
503
504
268
  return VTT;
505
270
}
506
507
namespace {
508
  /// Call the destructor for a direct base class.
509
  struct CallBaseDtor final : EHScopeStack::Cleanup {
510
    const CXXRecordDecl *BaseClass;
511
    bool BaseIsVirtual;
512
    CallBaseDtor(const CXXRecordDecl *Base, bool BaseIsVirtual)
513
2.50k
      : BaseClass(Base), BaseIsVirtual(BaseIsVirtual) {}
514
515
1.74k
    void Emit(CodeGenFunction &CGF, Flags flags) override {
516
1.74k
      const CXXRecordDecl *DerivedClass =
517
1.74k
        cast<CXXMethodDecl>(CGF.CurCodeDecl)->getParent();
518
519
1.74k
      const CXXDestructorDecl *D = BaseClass->getDestructor();
520
      // We are already inside a destructor, so presumably the object being
521
      // destroyed should have the expected type.
522
1.74k
      QualType ThisTy = D->getThisObjectType();
523
1.74k
      Address Addr =
524
1.74k
        CGF.GetAddressOfDirectBaseInCompleteClass(CGF.LoadCXXThisAddress(),
525
1.74k
                                                  DerivedClass, BaseClass,
526
1.74k
                                                  BaseIsVirtual);
527
1.74k
      CGF.EmitCXXDestructorCall(D, Dtor_Base, BaseIsVirtual,
528
1.74k
                                /*Delegating=*/false, Addr, ThisTy);
529
1.74k
    }
530
  };
531
532
  /// A visitor which checks whether an initializer uses 'this' in a
533
  /// way which requires the vtable to be properly set.
534
  struct DynamicThisUseChecker : ConstEvaluatedExprVisitor<DynamicThisUseChecker> {
535
    typedef ConstEvaluatedExprVisitor<DynamicThisUseChecker> super;
536
537
    bool UsesThis;
538
539
9.49k
    DynamicThisUseChecker(const ASTContext &C) : super(C), UsesThis(false) {}
540
541
    // Black-list all explicit and implicit references to 'this'.
542
    //
543
    // Do we need to worry about external references to 'this' derived
544
    // from arbitrary code?  If so, then anything which runs arbitrary
545
    // external code might potentially access the vtable.
546
11
    void VisitCXXThisExpr(const CXXThisExpr *E) { UsesThis = true; }
547
  };
548
} // end anonymous namespace
549
550
9.49k
static bool BaseInitializerUsesThis(ASTContext &C, const Expr *Init) {
551
9.49k
  DynamicThisUseChecker Checker(C);
552
9.49k
  Checker.Visit(Init);
553
9.49k
  return Checker.UsesThis;
554
9.49k
}
555
556
static void EmitBaseInitializer(CodeGenFunction &CGF,
557
                                const CXXRecordDecl *ClassDecl,
558
9.49k
                                CXXCtorInitializer *BaseInit) {
559
9.49k
  assert(BaseInit->isBaseInitializer() &&
560
9.49k
         "Must have base initializer!");
561
562
9.49k
  Address ThisPtr = CGF.LoadCXXThisAddress();
563
564
9.49k
  const Type *BaseType = BaseInit->getBaseClass();
565
9.49k
  const auto *BaseClassDecl =
566
9.49k
      cast<CXXRecordDecl>(BaseType->castAs<RecordType>()->getDecl());
567
568
9.49k
  bool isBaseVirtual = BaseInit->isBaseVirtual();
569
570
  // If the initializer for the base (other than the constructor
571
  // itself) accesses 'this' in any way, we need to initialize the
572
  // vtables.
573
9.49k
  if (BaseInitializerUsesThis(CGF.getContext(), BaseInit->getInit()))
574
11
    CGF.InitializeVTablePointers(ClassDecl);
575
576
  // We can pretend to be a complete class because it only matters for
577
  // virtual bases, and we only do virtual bases for complete ctors.
578
9.49k
  Address V =
579
9.49k
    CGF.GetAddressOfDirectBaseInCompleteClass(ThisPtr, ClassDecl,
580
9.49k
                                              BaseClassDecl,
581
9.49k
                                              isBaseVirtual);
582
9.49k
  AggValueSlot AggSlot =
583
9.49k
      AggValueSlot::forAddr(
584
9.49k
          V, Qualifiers(),
585
9.49k
          AggValueSlot::IsDestructed,
586
9.49k
          AggValueSlot::DoesNotNeedGCBarriers,
587
9.49k
          AggValueSlot::IsNotAliased,
588
9.49k
          CGF.getOverlapForBaseInit(ClassDecl, BaseClassDecl, isBaseVirtual));
589
590
9.49k
  CGF.EmitAggExpr(BaseInit->getInit(), AggSlot);
591
592
9.49k
  if (CGF.CGM.getLangOpts().Exceptions &&
593
7.27k
      !BaseClassDecl->hasTrivialDestructor())
594
1.16k
    CGF.EHStack.pushCleanup<CallBaseDtor>(EHCleanup, BaseClassDecl,
595
1.16k
                                          isBaseVirtual);
596
9.49k
}
597
598
92.2k
static bool isMemcpyEquivalentSpecialMember(const CXXMethodDecl *D) {
599
92.2k
  auto *CD = dyn_cast<CXXConstructorDecl>(D);
600
92.2k
  if (!(CD && 
CD->isCopyOrMoveConstructor()91.9k
) &&
601
81.3k
      !D->isCopyAssignmentOperator() && 
!D->isMoveAssignmentOperator()81.1k
)
602
81.0k
    return false;
603
604
  // We can emit a memcpy for a trivial copy or move constructor/assignment.
605
11.1k
  if (D->isTrivial() && 
!D->getParent()->mayInsertExtraPadding()6.58k
)
606
6.57k
    return true;
607
608
  // We *must* emit a memcpy for a defaulted union copy or move op.
609
4.58k
  if (D->getParent()->isUnion() && 
D->isDefaulted()4
)
610
2
    return true;
611
612
4.58k
  return false;
613
4.58k
}
614
615
static void EmitLValueForAnyFieldInitialization(CodeGenFunction &CGF,
616
                                                CXXCtorInitializer *MemberInit,
617
15.4k
                                                LValue &LHS) {
618
15.4k
  FieldDecl *Field = MemberInit->getAnyMember();
619
15.4k
  if (MemberInit->isIndirectMemberInitializer()) {
620
    // If we are initializing an anonymous union field, drill down to the field.
621
45
    IndirectFieldDecl *IndirectField = MemberInit->getIndirectMember();
622
45
    for (const auto *I : IndirectField->chain())
623
98
      LHS = CGF.EmitLValueForFieldInitialization(LHS, cast<FieldDecl>(I));
624
15.4k
  } else {
625
15.4k
    LHS = CGF.EmitLValueForFieldInitialization(LHS, Field);
626
15.4k
  }
627
15.4k
}
628
629
static void EmitMemberInitializer(CodeGenFunction &CGF,
630
                                  const CXXRecordDecl *ClassDecl,
631
                                  CXXCtorInitializer *MemberInit,
632
                                  const CXXConstructorDecl *Constructor,
633
15.4k
                                  FunctionArgList &Args) {
634
15.4k
  ApplyDebugLocation Loc(CGF, MemberInit->getSourceLocation());
635
15.4k
  assert(MemberInit->isAnyMemberInitializer() &&
636
15.4k
         "Must have member initializer!");
637
15.4k
  assert(MemberInit->getInit() && "Must have initializer!");
638
639
  // non-static data member initializers.
640
15.4k
  FieldDecl *Field = MemberInit->getAnyMember();
641
15.4k
  QualType FieldType = Field->getType();
642
643
15.4k
  llvm::Value *ThisPtr = CGF.LoadCXXThis();
644
15.4k
  QualType RecordTy = CGF.getContext().getTypeDeclType(ClassDecl);
645
15.4k
  LValue LHS;
646
647
  // If a base constructor is being emitted, create an LValue that has the
648
  // non-virtual alignment.
649
15.4k
  if (CGF.CurGD.getCtorType() == Ctor_Base)
650
15.2k
    LHS = CGF.MakeNaturalAlignPointeeAddrLValue(ThisPtr, RecordTy);
651
167
  else
652
167
    LHS = CGF.MakeNaturalAlignAddrLValue(ThisPtr, RecordTy);
653
654
15.4k
  EmitLValueForAnyFieldInitialization(CGF, MemberInit, LHS);
655
656
  // Special case: if we are in a copy or move constructor, and we are copying
657
  // an array of PODs or classes with trivial copy constructors, ignore the
658
  // AST and perform the copy we know is equivalent.
659
  // FIXME: This is hacky at best... if we had a bit more explicit information
660
  // in the AST, we could generalize it more easily.
661
15.4k
  const ConstantArrayType *Array
662
15.4k
    = CGF.getContext().getAsConstantArrayType(FieldType);
663
15.4k
  if (Array && 
Constructor->isDefaulted()77
&&
664
38
      Constructor->isCopyOrMoveConstructor()) {
665
22
    QualType BaseElementTy = CGF.getContext().getBaseElementType(Array);
666
22
    CXXConstructExpr *CE = dyn_cast<CXXConstructExpr>(MemberInit->getInit());
667
22
    if (BaseElementTy.isPODType(CGF.getContext()) ||
668
17
        (CE && 
isMemcpyEquivalentSpecialMember(CE->getConstructor())0
)) {
669
5
      unsigned SrcArgIndex =
670
5
          CGF.CGM.getCXXABI().getSrcArgforCopyCtor(Constructor, Args);
671
5
      llvm::Value *SrcPtr
672
5
        = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(Args[SrcArgIndex]));
673
5
      LValue ThisRHSLV = CGF.MakeNaturalAlignAddrLValue(SrcPtr, RecordTy);
674
5
      LValue Src = CGF.EmitLValueForFieldInitialization(ThisRHSLV, Field);
675
676
      // Copy the aggregate.
677
5
      CGF.EmitAggregateCopy(LHS, Src, FieldType, CGF.getOverlapForFieldInit(Field),
678
5
                            LHS.isVolatileQualified());
679
      // Ensure that we destroy the objects if an exception is thrown later in
680
      // the constructor.
681
5
      QualType::DestructionKind dtorKind = FieldType.isDestructedType();
682
5
      if (CGF.needsEHCleanup(dtorKind))
683
0
        CGF.pushEHDestroy(dtorKind, LHS.getAddress(CGF), FieldType);
684
5
      return;
685
5
    }
686
15.4k
  }
687
688
15.4k
  CGF.EmitInitializerForField(Field, LHS, MemberInit->getInit());
689
15.4k
}
690
691
void CodeGenFunction::EmitInitializerForField(FieldDecl *Field, LValue LHS,
692
16.4k
                                              Expr *Init) {
693
16.4k
  QualType FieldType = Field->getType();
694
16.4k
  switch (getEvaluationKind(FieldType)) {
695
12.1k
  case TEK_Scalar:
696
12.1k
    if (LHS.isSimple()) {
697
12.1k
      EmitExprAsInit(Init, Field, LHS, false);
698
82
    } else {
699
82
      RValue RHS = RValue::get(EmitScalarExpr(Init));
700
82
      EmitStoreThroughLValue(RHS, LHS);
701
82
    }
702
12.1k
    break;
703
6
  case TEK_Complex:
704
6
    EmitComplexExprIntoLValue(Init, LHS, /*isInit*/ true);
705
6
    break;
706
4.27k
  case TEK_Aggregate: {
707
4.27k
    AggValueSlot Slot = AggValueSlot::forLValue(
708
4.27k
        LHS, *this, AggValueSlot::IsDestructed,
709
4.27k
        AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased,
710
4.27k
        getOverlapForFieldInit(Field), AggValueSlot::IsNotZeroed,
711
        // Checks are made by the code that calls constructor.
712
4.27k
        AggValueSlot::IsSanitizerChecked);
713
4.27k
    EmitAggExpr(Init, Slot);
714
4.27k
    break;
715
16.4k
  }
716
16.4k
  }
717
718
  // Ensure that we destroy this object if an exception is thrown
719
  // later in the constructor.
720
16.4k
  QualType::DestructionKind dtorKind = FieldType.isDestructedType();
721
16.4k
  if (needsEHCleanup(dtorKind))
722
843
    pushEHDestroy(dtorKind, LHS.getAddress(*this), FieldType);
723
16.4k
}
724
725
/// Checks whether the given constructor is a valid subject for the
726
/// complete-to-base constructor delegation optimization, i.e.
727
/// emitting the complete constructor as a simple call to the base
728
/// constructor.
729
bool CodeGenFunction::IsConstructorDelegationValid(
730
16.9k
    const CXXConstructorDecl *Ctor) {
731
732
  // Currently we disable the optimization for classes with virtual
733
  // bases because (1) the addresses of parameter variables need to be
734
  // consistent across all initializers but (2) the delegate function
735
  // call necessarily creates a second copy of the parameter variable.
736
  //
737
  // The limiting example (purely theoretical AFAIK):
738
  //   struct A { A(int &c) { c++; } };
739
  //   struct B : virtual A {
740
  //     B(int count) : A(count) { printf("%d\n", count); }
741
  //   };
742
  // ...although even this example could in principle be emitted as a
743
  // delegation since the address of the parameter doesn't escape.
744
16.9k
  if (Ctor->getParent()->getNumVBases()) {
745
    // TODO: white-list trivial vbase initializers.  This case wouldn't
746
    // be subject to the restrictions below.
747
748
    // TODO: white-list cases where:
749
    //  - there are no non-reference parameters to the constructor
750
    //  - the initializers don't access any non-reference parameters
751
    //  - the initializers don't take the address of non-reference
752
    //    parameters
753
    //  - etc.
754
    // If we ever add any of the above cases, remember that:
755
    //  - function-try-blocks will always exclude this optimization
756
    //  - we need to perform the constructor prologue and cleanup in
757
    //    EmitConstructorBody.
758
759
584
    return false;
760
584
  }
761
762
  // We also disable the optimization for variadic functions because
763
  // it's impossible to "re-pass" varargs.
764
16.3k
  if (Ctor->getType()->castAs<FunctionProtoType>()->isVariadic())
765
10
    return false;
766
767
  // FIXME: Decide if we can do a delegation of a delegating constructor.
768
16.3k
  if (Ctor->isDelegatingConstructor())
769
68
    return false;
770
771
16.3k
  return true;
772
16.3k
}
773
774
// Emit code in ctor (Prologue==true) or dtor (Prologue==false)
775
// to poison the extra field paddings inserted under
776
// -fsanitize-address-field-padding=1|2.
777
53.1k
void CodeGenFunction::EmitAsanPrologueOrEpilogue(bool Prologue) {
778
53.1k
  ASTContext &Context = getContext();
779
53.1k
  const CXXRecordDecl *ClassDecl =
780
37.9k
      Prologue ? cast<CXXConstructorDecl>(CurGD.getDecl())->getParent()
781
15.2k
               : cast<CXXDestructorDecl>(CurGD.getDecl())->getParent();
782
53.1k
  if (!ClassDecl->mayInsertExtraPadding()) 
return53.0k
;
783
784
41
  struct SizeAndOffset {
785
41
    uint64_t Size;
786
41
    uint64_t Offset;
787
41
  };
788
789
41
  unsigned PtrSize = CGM.getDataLayout().getPointerSizeInBits();
790
41
  const ASTRecordLayout &Info = Context.getASTRecordLayout(ClassDecl);
791
792
  // Populate sizes and offsets of fields.
793
41
  SmallVector<SizeAndOffset, 16> SSV(Info.getFieldCount());
794
153
  for (unsigned i = 0, e = Info.getFieldCount(); i != e; 
++i112
)
795
112
    SSV[i].Offset =
796
112
        Context.toCharUnitsFromBits(Info.getFieldOffset(i)).getQuantity();
797
798
41
  size_t NumFields = 0;
799
112
  for (const auto *Field : ClassDecl->fields()) {
800
112
    const FieldDecl *D = Field;
801
112
    std::pair<CharUnits, CharUnits> FieldInfo =
802
112
        Context.getTypeInfoInChars(D->getType());
803
112
    CharUnits FieldSize = FieldInfo.first;
804
112
    assert(NumFields < SSV.size());
805
112
    SSV[NumFields].Size = D->isBitField() ? 
00
: FieldSize.getQuantity();
806
112
    NumFields++;
807
112
  }
808
41
  assert(NumFields == SSV.size());
809
41
  if (SSV.size() <= 1) 
return0
;
810
811
  // We will insert calls to __asan_* run-time functions.
812
  // LLVM AddressSanitizer pass may decide to inline them later.
813
41
  llvm::Type *Args[2] = {IntPtrTy, IntPtrTy};
814
41
  llvm::FunctionType *FTy =
815
41
      llvm::FunctionType::get(CGM.VoidTy, Args, false);
816
41
  llvm::FunctionCallee F = CGM.CreateRuntimeFunction(
817
22
      FTy, Prologue ? "__asan_poison_intra_object_redzone"
818
19
                    : "__asan_unpoison_intra_object_redzone");
819
820
41
  llvm::Value *ThisPtr = LoadCXXThis();
821
41
  ThisPtr = Builder.CreatePtrToInt(ThisPtr, IntPtrTy);
822
41
  uint64_t TypeSize = Info.getNonVirtualSize().getQuantity();
823
  // For each field check if it has sufficient padding,
824
  // if so (un)poison it with a call.
825
153
  for (size_t i = 0; i < SSV.size(); 
i++112
) {
826
112
    uint64_t AsanAlignment = 8;
827
71
    uint64_t NextField = i == SSV.size() - 1 ? 
TypeSize41
: SSV[i + 1].Offset;
828
112
    uint64_t PoisonSize = NextField - SSV[i].Offset - SSV[i].Size;
829
112
    uint64_t EndOffset = SSV[i].Offset + SSV[i].Size;
830
112
    if (PoisonSize < AsanAlignment || 
!SSV[i].Size94
||
831
94
        (NextField % AsanAlignment) != 0)
832
18
      continue;
833
94
    Builder.CreateCall(
834
94
        F, {Builder.CreateAdd(ThisPtr, Builder.getIntN(PtrSize, EndOffset)),
835
94
            Builder.getIntN(PtrSize, PoisonSize)});
836
94
  }
837
41
}
838
839
/// EmitConstructorBody - Emits the body of the current constructor.
840
37.9k
void CodeGenFunction::EmitConstructorBody(FunctionArgList &Args) {
841
37.9k
  EmitAsanPrologueOrEpilogue(true);
842
37.9k
  const CXXConstructorDecl *Ctor = cast<CXXConstructorDecl>(CurGD.getDecl());
843
37.9k
  CXXCtorType CtorType = CurGD.getCtorType();
844
845
37.9k
  assert((CGM.getTarget().getCXXABI().hasConstructorVariants() ||
846
37.9k
          CtorType == Ctor_Complete) &&
847
37.9k
         "can only generate complete ctor for this ABI");
848
849
  // Before we go any further, try the complete->base constructor
850
  // delegation optimization.
851
37.9k
  if (CtorType == Ctor_Complete && 
IsConstructorDelegationValid(Ctor)16.9k
&&
852
16.2k
      CGM.getTarget().getCXXABI().hasConstructorVariants()) {
853
15.4k
    EmitDelegateCXXConstructorCall(Ctor, Ctor_Base, Args, Ctor->getEndLoc());
854
15.4k
    return;
855
15.4k
  }
856
857
22.4k
  const FunctionDecl *Definition = nullptr;
858
22.4k
  Stmt *Body = Ctor->getBody(Definition);
859
22.4k
  assert(Definition == Ctor && "emitting wrong constructor body");
860
861
  // Enter the function-try-block before the constructor prologue if
862
  // applicable.
863
22.4k
  bool IsTryBody = (Body && isa<CXXTryStmt>(Body));
864
22.4k
  if (IsTryBody)
865
2
    EnterCXXTryStmt(*cast<CXXTryStmt>(Body), true);
866
867
22.4k
  incrementProfileCounter(Body);
868
869
22.4k
  RunCleanupsScope RunCleanups(*this);
870
871
  // TODO: in restricted cases, we can emit the vbase initializers of
872
  // a complete ctor and then delegate to the base ctor.
873
874
  // Emit the constructor prologue, i.e. the base and member
875
  // initializers.
876
22.4k
  EmitCtorPrologue(Ctor, CtorType, Args);
877
878
  // Emit the body of the statement.
879
22.4k
  if (IsTryBody)
880
2
    EmitStmt(cast<CXXTryStmt>(Body)->getTryBlock());
881
22.4k
  else if (Body)
882
22.4k
    EmitStmt(Body);
883
884
  // Emit any cleanup blocks associated with the member or base
885
  // initializers, which includes (along the exceptional path) the
886
  // destructors for those members and bases that were fully
887
  // constructed.
888
22.4k
  RunCleanups.ForceCleanup();
889
890
22.4k
  if (IsTryBody)
891
2
    ExitCXXTryStmt(*cast<CXXTryStmt>(Body), true);
892
22.4k
}
893
894
namespace {
895
  /// RAII object to indicate that codegen is copying the value representation
896
  /// instead of the object representation. Useful when copying a struct or
897
  /// class which has uninitialized members and we're only performing
898
  /// lvalue-to-rvalue conversion on the object but not its members.
899
  class CopyingValueRepresentation {
900
  public:
901
    explicit CopyingValueRepresentation(CodeGenFunction &CGF)
902
139
        : CGF(CGF), OldSanOpts(CGF.SanOpts) {
903
139
      CGF.SanOpts.set(SanitizerKind::Bool, false);
904
139
      CGF.SanOpts.set(SanitizerKind::Enum, false);
905
139
    }
906
139
    ~CopyingValueRepresentation() {
907
139
      CGF.SanOpts = OldSanOpts;
908
139
    }
909
  private:
910
    CodeGenFunction &CGF;
911
    SanitizerSet OldSanOpts;
912
  };
913
} // end anonymous namespace
914
915
namespace {
916
  class FieldMemcpyizer {
917
  public:
918
    FieldMemcpyizer(CodeGenFunction &CGF, const CXXRecordDecl *ClassDecl,
919
                    const VarDecl *SrcRec)
920
      : CGF(CGF), ClassDecl(ClassDecl), SrcRec(SrcRec),
921
        RecLayout(CGF.getContext().getASTRecordLayout(ClassDecl)),
922
        FirstField(nullptr), LastField(nullptr), FirstFieldOffset(0),
923
23.1k
        LastFieldOffset(0), LastAddedFieldIndex(0) {}
924
925
516
    bool isMemcpyableField(FieldDecl *F) const {
926
      // Never memcpy fields when we are adding poisoned paddings.
927
516
      if (CGF.getContext().getLangOpts().SanitizeAddressFieldPadding)
928
8
        return false;
929
508
      Qualifiers Qual = F->getType().getQualifiers();
930
508
      if (Qual.hasVolatile() || 
Qual.hasObjCLifetime()504
)
931
8
        return false;
932
500
      return true;
933
500
    }
934
935
497
    void addMemcpyableField(FieldDecl *F) {
936
497
      if (F->isZeroSize(CGF.getContext()))
937
1
        return;
938
496
      if (!FirstField)
939
235
        addInitialField(F);
940
261
      else
941
261
        addNextField(F);
942
496
    }
943
944
96
    CharUnits getMemcpySize(uint64_t FirstByteOffset) const {
945
96
      ASTContext &Ctx = CGF.getContext();
946
96
      unsigned LastFieldSize =
947
96
          LastField->isBitField()
948
6
              ? LastField->getBitWidthValue(Ctx)
949
90
              : Ctx.toBits(
950
90
                    Ctx.getTypeInfoDataSizeInChars(LastField->getType()).first);
951
96
      uint64_t MemcpySizeBits = LastFieldOffset + LastFieldSize -
952
96
                                FirstByteOffset + Ctx.getCharWidth() - 1;
953
96
      CharUnits MemcpySize = Ctx.toCharUnitsFromBits(MemcpySizeBits);
954
96
      return MemcpySize;
955
96
    }
956
957
1.74k
    void emitMemcpy() {
958
      // Give the subclass a chance to bail out if it feels the memcpy isn't
959
      // worth it (e.g. Hasn't aggregated enough data).
960
1.74k
      if (!FirstField) {
961
1.65k
        return;
962
1.65k
      }
963
964
96
      uint64_t FirstByteOffset;
965
96
      if (FirstField->isBitField()) {
966
12
        const CGRecordLayout &RL =
967
12
          CGF.getTypes().getCGRecordLayout(FirstField->getParent());
968
12
        const CGBitFieldInfo &BFInfo = RL.getBitFieldInfo(FirstField);
969
        // FirstFieldOffset is not appropriate for bitfields,
970
        // we need to use the storage offset instead.
971
12
        FirstByteOffset = CGF.getContext().toBits(BFInfo.StorageOffset);
972
84
      } else {
973
84
        FirstByteOffset = FirstFieldOffset;
974
84
      }
975
976
96
      CharUnits MemcpySize = getMemcpySize(FirstByteOffset);
977
96
      QualType RecordTy = CGF.getContext().getTypeDeclType(ClassDecl);
978
96
      Address ThisPtr = CGF.LoadCXXThisAddress();
979
96
      LValue DestLV = CGF.MakeAddrLValue(ThisPtr, RecordTy);
980
96
      LValue Dest = CGF.EmitLValueForFieldInitialization(DestLV, FirstField);
981
96
      llvm::Value *SrcPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(SrcRec));
982
96
      LValue SrcLV = CGF.MakeNaturalAlignAddrLValue(SrcPtr, RecordTy);
983
96
      LValue Src = CGF.EmitLValueForFieldInitialization(SrcLV, FirstField);
984
985
96
      emitMemcpyIR(
986
84
          Dest.isBitField() ? 
Dest.getBitFieldAddress()12
: Dest.getAddress(CGF),
987
84
          Src.isBitField() ? 
Src.getBitFieldAddress()12
: Src.getAddress(CGF),
988
96
          MemcpySize);
989
96
      reset();
990
96
    }
991
992
39.5k
    void reset() {
993
39.5k
      FirstField = nullptr;
994
39.5k
    }
995
996
  protected:
997
    CodeGenFunction &CGF;
998
    const CXXRecordDecl *ClassDecl;
999
1000
  private:
1001
96
    void emitMemcpyIR(Address DestPtr, Address SrcPtr, CharUnits Size) {
1002
96
      llvm::PointerType *DPT = DestPtr.getType();
1003
96
      llvm::Type *DBP =
1004
96
        llvm::Type::getInt8PtrTy(CGF.getLLVMContext(), DPT->getAddressSpace());
1005
96
      DestPtr = CGF.Builder.CreateBitCast(DestPtr, DBP);
1006
1007
96
      llvm::PointerType *SPT = SrcPtr.getType();
1008
96
      llvm::Type *SBP =
1009
96
        llvm::Type::getInt8PtrTy(CGF.getLLVMContext(), SPT->getAddressSpace());
1010
96
      SrcPtr = CGF.Builder.CreateBitCast(SrcPtr, SBP);
1011
1012
96
      CGF.Builder.CreateMemCpy(DestPtr, SrcPtr, Size.getQuantity());
1013
96
    }
1014
1015
235
    void addInitialField(FieldDecl *F) {
1016
235
      FirstField = F;
1017
235
      LastField = F;
1018
235
      FirstFieldOffset = RecLayout.getFieldOffset(F->getFieldIndex());
1019
235
      LastFieldOffset = FirstFieldOffset;
1020
235
      LastAddedFieldIndex = F->getFieldIndex();
1021
235
    }
1022
1023
261
    void addNextField(FieldDecl *F) {
1024
      // For the most part, the following invariant will hold:
1025
      //   F->getFieldIndex() == LastAddedFieldIndex + 1
1026
      // The one exception is that Sema won't add a copy-initializer for an
1027
      // unnamed bitfield, which will show up here as a gap in the sequence.
1028
261
      assert(F->getFieldIndex() >= LastAddedFieldIndex + 1 &&
1029
261
             "Cannot aggregate fields out of order.");
1030
261
      LastAddedFieldIndex = F->getFieldIndex();
1031
1032
      // The 'first' and 'last' fields are chosen by offset, rather than field
1033
      // index. This allows the code to support bitfields, as well as regular
1034
      // fields.
1035
261
      uint64_t FOffset = RecLayout.getFieldOffset(F->getFieldIndex());
1036
261
      if (FOffset < FirstFieldOffset) {
1037
0
        FirstField = F;
1038
0
        FirstFieldOffset = FOffset;
1039
261
      } else if (FOffset >= LastFieldOffset) {
1040
261
        LastField = F;
1041
261
        LastFieldOffset = FOffset;
1042
261
      }
1043
261
    }
1044
1045
    const VarDecl *SrcRec;
1046
    const ASTRecordLayout &RecLayout;
1047
    FieldDecl *FirstField;
1048
    FieldDecl *LastField;
1049
    uint64_t FirstFieldOffset, LastFieldOffset;
1050
    unsigned LastAddedFieldIndex;
1051
  };
1052
1053
  class ConstructorMemcpyizer : public FieldMemcpyizer {
1054
  private:
1055
    /// Get source argument for copy constructor. Returns null if not a copy
1056
    /// constructor.
1057
    static const VarDecl *getTrivialCopySource(CodeGenFunction &CGF,
1058
                                               const CXXConstructorDecl *CD,
1059
22.4k
                                               FunctionArgList &Args) {
1060
22.4k
      if (CD->isCopyOrMoveConstructor() && 
CD->isDefaulted()988
)
1061
356
        return Args[CGF.CGM.getCXXABI().getSrcArgforCopyCtor(CD, Args)];
1062
22.0k
      return nullptr;
1063
22.0k
    }
1064
1065
    // Returns true if a CXXCtorInitializer represents a member initialization
1066
    // that can be rolled into a memcpy.
1067
15.6k
    bool isMemberInitMemcpyable(CXXCtorInitializer *MemberInit) const {
1068
15.6k
      if (!MemcpyableCtor)
1069
15.2k
        return false;
1070
415
      FieldDecl *Field = MemberInit->getMember();
1071
415
      assert(Field && "No field for member init.");
1072
415
      QualType FieldType = Field->getType();
1073
415
      CXXConstructExpr *CE = dyn_cast<CXXConstructExpr>(MemberInit->getInit());
1074
1075
      // Bail out on non-memcpyable, not-trivially-copyable members.
1076
415
      if (!(CE && 
isMemcpyEquivalentSpecialMember(CE->getConstructor())135
) &&
1077
394
          !(FieldType.isTriviallyCopyableType(CGF.getContext()) ||
1078
163
            FieldType->isReferenceType()))
1079
155
        return false;
1080
1081
      // Bail out on volatile fields.
1082
260
      if (!isMemcpyableField(Field))
1083
6
        return false;
1084
1085
      // Otherwise we're good.
1086
254
      return true;
1087
254
    }
1088
1089
  public:
1090
    ConstructorMemcpyizer(CodeGenFunction &CGF, const CXXConstructorDecl *CD,
1091
                          FunctionArgList &Args)
1092
      : FieldMemcpyizer(CGF, CD->getParent(), getTrivialCopySource(CGF, CD, Args)),
1093
        ConstructorDecl(CD),
1094
        MemcpyableCtor(CD->isDefaulted() &&
1095
                       CD->isCopyOrMoveConstructor() &&
1096
                       CGF.getLangOpts().getGC() == LangOptions::NonGC),
1097
22.4k
        Args(Args) { }
1098
1099
15.6k
    void addMemberInitializer(CXXCtorInitializer *MemberInit) {
1100
15.6k
      if (isMemberInitMemcpyable(MemberInit)) {
1101
254
        AggregatedInits.push_back(MemberInit);
1102
254
        addMemcpyableField(MemberInit->getMember());
1103
15.3k
      } else {
1104
15.3k
        emitAggregatedInits();
1105
15.3k
        EmitMemberInitializer(CGF, ConstructorDecl->getParent(), MemberInit,
1106
15.3k
                              ConstructorDecl, Args);
1107
15.3k
      }
1108
15.6k
    }
1109
1110
37.8k
    void emitAggregatedInits() {
1111
37.8k
      if (AggregatedInits.size() <= 1) {
1112
        // This memcpy is too small to be worthwhile. Fall back on default
1113
        // codegen.
1114
37.7k
        if (!AggregatedInits.empty()) {
1115
59
          CopyingValueRepresentation CVR(CGF);
1116
59
          EmitMemberInitializer(CGF, ConstructorDecl->getParent(),
1117
59
                                AggregatedInits[0], ConstructorDecl, Args);
1118
59
          AggregatedInits.clear();
1119
59
        }
1120
37.7k
        reset();
1121
37.7k
        return;
1122
37.7k
      }
1123
1124
55
      pushEHDestructors();
1125
55
      emitMemcpy();
1126
55
      AggregatedInits.clear();
1127
55
    }
1128
1129
55
    void pushEHDestructors() {
1130
55
      Address ThisPtr = CGF.LoadCXXThisAddress();
1131
55
      QualType RecordTy = CGF.getContext().getTypeDeclType(ClassDecl);
1132
55
      LValue LHS = CGF.MakeAddrLValue(ThisPtr, RecordTy);
1133
1134
250
      for (unsigned i = 0; i < AggregatedInits.size(); 
++i195
) {
1135
195
        CXXCtorInitializer *MemberInit = AggregatedInits[i];
1136
195
        QualType FieldType = MemberInit->getAnyMember()->getType();
1137
195
        QualType::DestructionKind dtorKind = FieldType.isDestructedType();
1138
195
        if (!CGF.needsEHCleanup(dtorKind))
1139
193
          continue;
1140
2
        LValue FieldLHS = LHS;
1141
2
        EmitLValueForAnyFieldInitialization(CGF, MemberInit, FieldLHS);
1142
2
        CGF.pushEHDestroy(dtorKind, FieldLHS.getAddress(CGF), FieldType);
1143
2
      }
1144
55
    }
1145
1146
22.4k
    void finish() {
1147
22.4k
      emitAggregatedInits();
1148
22.4k
    }
1149
1150
  private:
1151
    const CXXConstructorDecl *ConstructorDecl;
1152
    bool MemcpyableCtor;
1153
    FunctionArgList &Args;
1154
    SmallVector<CXXCtorInitializer*, 16> AggregatedInits;
1155
  };
1156
1157
  class AssignmentMemcpyizer : public FieldMemcpyizer {
1158
  private:
1159
    // Returns the memcpyable field copied by the given statement, if one
1160
    // exists. Otherwise returns null.
1161
1.24k
    FieldDecl *getMemcpyableField(Stmt *S) {
1162
1.24k
      if (!AssignmentsMemcpyable)
1163
11
        return nullptr;
1164
1.23k
      if (BinaryOperator *BO = dyn_cast<BinaryOperator>(S)) {
1165
        // Recognise trivial assignments.
1166
226
        if (BO->getOpcode() != BO_Assign)
1167
0
          return nullptr;
1168
226
        MemberExpr *ME = dyn_cast<MemberExpr>(BO->getLHS());
1169
226
        if (!ME)
1170
0
          return nullptr;
1171
226
        FieldDecl *Field = dyn_cast<FieldDecl>(ME->getMemberDecl());
1172
226
        if (!Field || !isMemcpyableField(Field))
1173
10
          return nullptr;
1174
216
        Stmt *RHS = BO->getRHS();
1175
216
        if (ImplicitCastExpr *EC = dyn_cast<ImplicitCastExpr>(RHS))
1176
216
          RHS = EC->getSubExpr();
1177
216
        if (!RHS)
1178
0
          return nullptr;
1179
216
        if (MemberExpr *ME2 = dyn_cast<MemberExpr>(RHS)) {
1180
214
          if (ME2->getMemberDecl() == Field)
1181
214
            return Field;
1182
2
        }
1183
2
        return nullptr;
1184
1.00k
      } else if (CXXMemberCallExpr *MCE = dyn_cast<CXXMemberCallExpr>(S)) {
1185
290
        CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(MCE->getCalleeDecl());
1186
290
        if (!(MD && isMemcpyEquivalentSpecialMember(MD)))
1187
125
          return nullptr;
1188
165
        MemberExpr *IOA = dyn_cast<MemberExpr>(MCE->getImplicitObjectArgument());
1189
165
        if (!IOA)
1190
146
          return nullptr;
1191
19
        FieldDecl *Field = dyn_cast<FieldDecl>(IOA->getMemberDecl());
1192
19
        if (!Field || !isMemcpyableField(Field))
1193
0
          return nullptr;
1194
19
        MemberExpr *Arg0 = dyn_cast<MemberExpr>(MCE->getArg(0));
1195
19
        if (!Arg0 || 
Field != dyn_cast<FieldDecl>(Arg0->getMemberDecl())18
)
1196
1
          return nullptr;
1197
18
        return Field;
1198
719
      } else if (CallExpr *CE = dyn_cast<CallExpr>(S)) {
1199
11
        FunctionDecl *FD = dyn_cast<FunctionDecl>(CE->getCalleeDecl());
1200
11
        if (!FD || FD->getBuiltinID() != Builtin::BI__builtin_memcpy)
1201
0
          return nullptr;
1202
11
        Expr *DstPtr = CE->getArg(0);
1203
11
        if (ImplicitCastExpr *DC = dyn_cast<ImplicitCastExpr>(DstPtr))
1204
11
          DstPtr = DC->getSubExpr();
1205
11
        UnaryOperator *DUO = dyn_cast<UnaryOperator>(DstPtr);
1206
11
        if (!DUO || DUO->getOpcode() != UO_AddrOf)
1207
0
          return nullptr;
1208
11
        MemberExpr *ME = dyn_cast<MemberExpr>(DUO->getSubExpr());
1209
11
        if (!ME)
1210
0
          return nullptr;
1211
11
        FieldDecl *Field = dyn_cast<FieldDecl>(ME->getMemberDecl());
1212
11
        if (!Field || !isMemcpyableField(Field))
1213
0
          return nullptr;
1214
11
        Expr *SrcPtr = CE->getArg(1);
1215
11
        if (ImplicitCastExpr *SC = dyn_cast<ImplicitCastExpr>(SrcPtr))
1216
11
          SrcPtr = SC->getSubExpr();
1217
11
        UnaryOperator *SUO = dyn_cast<UnaryOperator>(SrcPtr);
1218
11
        if (!SUO || SUO->getOpcode() != UO_AddrOf)
1219
0
          return nullptr;
1220
11
        MemberExpr *ME2 = dyn_cast<MemberExpr>(SUO->getSubExpr());
1221
11
        if (!ME2 || Field != dyn_cast<FieldDecl>(ME2->getMemberDecl()))
1222
0
          return nullptr;
1223
11
        return Field;
1224
11
      }
1225
1226
708
      return nullptr;
1227
708
    }
1228
1229
    bool AssignmentsMemcpyable;
1230
    SmallVector<Stmt*, 16> AggregatedStmts;
1231
1232
  public:
1233
    AssignmentMemcpyizer(CodeGenFunction &CGF, const CXXMethodDecl *AD,
1234
                         FunctionArgList &Args)
1235
      : FieldMemcpyizer(CGF, AD->getParent(), Args[Args.size() - 1]),
1236
689
        AssignmentsMemcpyable(CGF.getLangOpts().getGC() == LangOptions::NonGC) {
1237
689
      assert(Args.size() == 2);
1238
689
    }
1239
1240
1.24k
    void emitAssignment(Stmt *S) {
1241
1.24k
      FieldDecl *F = getMemcpyableField(S);
1242
1.24k
      if (F) {
1243
243
        addMemcpyableField(F);
1244
243
        AggregatedStmts.push_back(S);
1245
1.00k
      } else {
1246
1.00k
        emitAggregatedStmts();
1247
1.00k
        CGF.EmitStmt(S);
1248
1.00k
      }
1249
1.24k
    }
1250
1251
1.69k
    void emitAggregatedStmts() {
1252
1.69k
      if (AggregatedStmts.size() <= 1) {
1253
1.65k
        if (!AggregatedStmts.empty()) {
1254
80
          CopyingValueRepresentation CVR(CGF);
1255
80
          CGF.EmitStmt(AggregatedStmts[0]);
1256
80
        }
1257
1.65k
        reset();
1258
1.65k
      }
1259
1260
1.69k
      emitMemcpy();
1261
1.69k
      AggregatedStmts.clear();
1262
1.69k
    }
1263
1264
689
    void finish() {
1265
689
      emitAggregatedStmts();
1266
689
    }
1267
  };
1268
} // end anonymous namespace
1269
1270
27
static bool isInitializerOfDynamicClass(const CXXCtorInitializer *BaseInit) {
1271
27
  const Type *BaseType = BaseInit->getBaseClass();
1272
27
  const auto *BaseClassDecl =
1273
27
      cast<CXXRecordDecl>(BaseType->castAs<RecordType>()->getDecl());
1274
27
  return BaseClassDecl->isDynamicClass();
1275
27
}
1276
1277
/// EmitCtorPrologue - This routine generates necessary code to initialize
1278
/// base classes and non-static data members belonging to this constructor.
1279
void CodeGenFunction::EmitCtorPrologue(const CXXConstructorDecl *CD,
1280
                                       CXXCtorType CtorType,
1281
22.4k
                                       FunctionArgList &Args) {
1282
22.4k
  if (CD->isDelegatingConstructor())
1283
75
    return EmitDelegatingCXXConstructorCall(CD, Args);
1284
1285
22.4k
  const CXXRecordDecl *ClassDecl = CD->getParent();
1286
1287
22.4k
  CXXConstructorDecl::init_const_iterator B = CD->init_begin(),
1288
22.4k
                                          E = CD->init_end();
1289
1290
  // Virtual base initializers first, if any. They aren't needed if:
1291
  // - This is a base ctor variant
1292
  // - There are no vbases
1293
  // - The class is abstract, so a complete object of it cannot be constructed
1294
  //
1295
  // The check for an abstract class is necessary because sema may not have
1296
  // marked virtual base destructors referenced.
1297
22.4k
  bool ConstructVBases = CtorType != Ctor_Base &&
1298
1.45k
                         ClassDecl->getNumVBases() != 0 &&
1299
592
                         !ClassDecl->isAbstract();
1300
1301
  // In the Microsoft C++ ABI, there are no constructor variants. Instead, the
1302
  // constructor of a class with virtual bases takes an additional parameter to
1303
  // conditionally construct the virtual bases. Emit that check here.
1304
22.4k
  llvm::BasicBlock *BaseCtorContinueBB = nullptr;
1305
22.4k
  if (ConstructVBases &&
1306
589
      !CGM.getTarget().getCXXABI().hasConstructorVariants()) {
1307
381
    BaseCtorContinueBB =
1308
381
        CGM.getCXXABI().EmitCtorCompleteObjectHandler(*this, ClassDecl);
1309
381
    assert(BaseCtorContinueBB);
1310
381
  }
1311
1312
22.4k
  llvm::Value *const OldThis = CXXThisValue;
1313
23.3k
  for (; B != E && 
(*B)->isBaseInitializer()17.9k
&&
(*B)->isBaseVirtual()7.27k
;
B++941
) {
1314
941
    if (!ConstructVBases)
1315
162
      continue;
1316
779
    if (CGM.getCodeGenOpts().StrictVTablePointers &&
1317
3
        CGM.getCodeGenOpts().OptimizationLevel > 0 &&
1318
3
        isInitializerOfDynamicClass(*B))
1319
1
      CXXThisValue = Builder.CreateLaunderInvariantGroup(LoadCXXThis());
1320
779
    EmitBaseInitializer(*this, ClassDecl, *B);
1321
779
  }
1322
1323
22.4k
  if (BaseCtorContinueBB) {
1324
    // Complete object handler should continue to the remaining initializers.
1325
381
    Builder.CreateBr(BaseCtorContinueBB);
1326
381
    EmitBlock(BaseCtorContinueBB);
1327
381
  }
1328
1329
  // Then, non-virtual base initializers.
1330
31.1k
  for (; B != E && 
(*B)->isBaseInitializer()20.6k
;
B++8.71k
) {
1331
8.71k
    assert(!(*B)->isBaseVirtual());
1332
1333
8.71k
    if (CGM.getCodeGenOpts().StrictVTablePointers &&
1334
24
        CGM.getCodeGenOpts().OptimizationLevel > 0 &&
1335
24
        isInitializerOfDynamicClass(*B))
1336
23
      CXXThisValue = Builder.CreateLaunderInvariantGroup(LoadCXXThis());
1337
8.71k
    EmitBaseInitializer(*this, ClassDecl, *B);
1338
8.71k
  }
1339
1340
22.4k
  CXXThisValue = OldThis;
1341
1342
22.4k
  InitializeVTablePointers(ClassDecl);
1343
1344
  // And finally, initialize class members.
1345
22.4k
  FieldConstructionScope FCS(*this, LoadCXXThisAddress());
1346
22.4k
  ConstructorMemcpyizer CM(*this, CD, Args);
1347
38.0k
  for (; B != E; 
B++15.6k
) {
1348
15.6k
    CXXCtorInitializer *Member = (*B);
1349
15.6k
    assert(!Member->isBaseInitializer());
1350
15.6k
    assert(Member->isAnyMemberInitializer() &&
1351
15.6k
           "Delegating initializer on non-delegating constructor");
1352
15.6k
    CM.addMemberInitializer(Member);
1353
15.6k
  }
1354
22.4k
  CM.finish();
1355
22.4k
}
1356
1357
static bool
1358
FieldHasTrivialDestructorBody(ASTContext &Context, const FieldDecl *Field);
1359
1360
static bool
1361
HasTrivialDestructorBody(ASTContext &Context,
1362
                         const CXXRecordDecl *BaseClassDecl,
1363
                         const CXXRecordDecl *MostDerivedClassDecl)
1364
264
{
1365
  // If the destructor is trivial we don't have to check anything else.
1366
264
  if (BaseClassDecl->hasTrivialDestructor())
1367
75
    return true;
1368
1369
189
  if (!BaseClassDecl->getDestructor()->hasTrivialBody())
1370
97
    return false;
1371
1372
  // Check fields.
1373
92
  for (const auto *Field : BaseClassDecl->fields())
1374
71
    if (!FieldHasTrivialDestructorBody(Context, Field))
1375
62
      return false;
1376
1377
  // Check non-virtual bases.
1378
33
  
for (const auto &I : BaseClassDecl->bases())30
{
1379
33
    if (I.isVirtual())
1380
1
      continue;
1381
1382
32
    const CXXRecordDecl *NonVirtualBase =
1383
32
      cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
1384
32
    if (!HasTrivialDestructorBody(Context, NonVirtualBase,
1385
32
                                  MostDerivedClassDecl))
1386
23
      return false;
1387
32
  }
1388
1389
7
  if (BaseClassDecl == MostDerivedClassDecl) {
1390
    // Check virtual bases.
1391
1
    for (const auto &I : BaseClassDecl->vbases()) {
1392
1
      const CXXRecordDecl *VirtualBase =
1393
1
        cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
1394
1
      if (!HasTrivialDestructorBody(Context, VirtualBase,
1395
1
                                    MostDerivedClassDecl))
1396
1
        return false;
1397
1
    }
1398
7
  }
1399
1400
6
  return true;
1401
7
}
1402
1403
static bool
1404
FieldHasTrivialDestructorBody(ASTContext &Context,
1405
                                          const FieldDecl *Field)
1406
488
{
1407
488
  QualType FieldBaseElementType = Context.getBaseElementType(Field->getType());
1408
1409
488
  const RecordType *RT = FieldBaseElementType->getAs<RecordType>();
1410
488
  if (!RT)
1411
255
    return true;
1412
1413
233
  CXXRecordDecl *FieldClassDecl = cast<CXXRecordDecl>(RT->getDecl());
1414
1415
  // The destructor for an implicit anonymous union member is never invoked.
1416
233
  if (FieldClassDecl->isUnion() && 
FieldClassDecl->isAnonymousStructOrUnion()2
)
1417
2
    return false;
1418
1419
231
  return HasTrivialDestructorBody(Context, FieldClassDecl, FieldClassDecl);
1420
231
}
1421
1422
/// CanSkipVTablePointerInitialization - Check whether we need to initialize
1423
/// any vtable pointers before calling this destructor.
1424
static bool CanSkipVTablePointerInitialization(CodeGenFunction &CGF,
1425
8.03k
                                               const CXXDestructorDecl *Dtor) {
1426
8.03k
  const CXXRecordDecl *ClassDecl = Dtor->getParent();
1427
8.03k
  if (!ClassDecl->isDynamicClass())
1428
7.08k
    return true;
1429
1430
943
  if (!Dtor->hasTrivialBody())
1431
63
    return false;
1432
1433
  // Check the fields.
1434
880
  for (const auto *Field : ClassDecl->fields())
1435
349
    if (!FieldHasTrivialDestructorBody(CGF.getContext(), Field))
1436
93
      return false;
1437
1438
787
  return true;
1439
880
}
1440
1441
/// EmitDestructorBody - Emits the body of the current destructor.
1442
16.1k
void CodeGenFunction::EmitDestructorBody(FunctionArgList &Args) {
1443
16.1k
  const CXXDestructorDecl *Dtor = cast<CXXDestructorDecl>(CurGD.getDecl());
1444
16.1k
  CXXDtorType DtorType = CurGD.getDtorType();
1445
1446
  // For an abstract class, non-base destructors are never used (and can't
1447
  // be emitted in general, because vbase dtors may not have been validated
1448
  // by Sema), but the Itanium ABI doesn't make them optional and Clang may
1449
  // in fact emit references to them from other compilations, so emit them
1450
  // as functions containing a trap instruction.
1451
16.1k
  if (DtorType != Dtor_Base && 
Dtor->getParent()->isAbstract()8.15k
) {
1452
82
    llvm::CallInst *TrapCall = EmitTrapCall(llvm::Intrinsic::trap);
1453
82
    TrapCall->setDoesNotReturn();
1454
82
    TrapCall->setDoesNotThrow();
1455
82
    Builder.CreateUnreachable();
1456
82
    Builder.ClearInsertionPoint();
1457
82
    return;
1458
82
  }
1459
1460
16.0k
  Stmt *Body = Dtor->getBody();
1461
16.0k
  if (Body)
1462
15.9k
    incrementProfileCounter(Body);
1463
1464
  // The call to operator delete in a deleting destructor happens
1465
  // outside of the function-try-block, which means it's always
1466
  // possible to delegate the destructor body to the complete
1467
  // destructor.  Do so.
1468
16.0k
  if (DtorType == Dtor_Deleting) {
1469
860
    RunCleanupsScope DtorEpilogue(*this);
1470
860
    EnterDtorCleanups(Dtor, Dtor_Deleting);
1471
860
    if (HaveInsertPoint()) {
1472
856
      QualType ThisTy = Dtor->getThisObjectType();
1473
856
      EmitCXXDestructorCall(Dtor, Dtor_Complete, /*ForVirtualBase=*/false,
1474
856
                            /*Delegating=*/false, LoadCXXThisAddress(), ThisTy);
1475
856
    }
1476
860
    return;
1477
860
  }
1478
1479
  // If the body is a function-try-block, enter the try before
1480
  // anything else.
1481
15.2k
  bool isTryBody = (Body && 
isa<CXXTryStmt>(Body)15.1k
);
1482
15.2k
  if (isTryBody)
1483
18
    EnterCXXTryStmt(*cast<CXXTryStmt>(Body), true);
1484
15.2k
  EmitAsanPrologueOrEpilogue(false);
1485
1486
  // Enter the epilogue cleanups.
1487
15.2k
  RunCleanupsScope DtorEpilogue(*this);
1488
1489
  // If this is the complete variant, just invoke the base variant;
1490
  // the epilogue will destruct the virtual bases.  But we can't do
1491
  // this optimization if the body is a function-try-block, because
1492
  // we'd introduce *two* handler blocks.  In the Microsoft ABI, we
1493
  // always delegate because we might not have a definition in this TU.
1494
15.2k
  switch (DtorType) {
1495
0
  case Dtor_Comdat: llvm_unreachable("not expecting a COMDAT");
1496
0
  case Dtor_Deleting: llvm_unreachable("already handled deleting case");
1497
1498
7.20k
  case Dtor_Complete:
1499
7.20k
    assert((Body || getTarget().getCXXABI().isMicrosoft()) &&
1500
7.20k
           "can't emit a dtor without a body for non-Microsoft ABIs");
1501
1502
    // Enter the cleanup scopes for virtual bases.
1503
7.20k
    EnterDtorCleanups(Dtor, Dtor_Complete);
1504
1505
7.20k
    if (!isTryBody) {
1506
7.20k
      QualType ThisTy = Dtor->getThisObjectType();
1507
7.20k
      EmitCXXDestructorCall(Dtor, Dtor_Base, /*ForVirtualBase=*/false,
1508
7.20k
                            /*Delegating=*/false, LoadCXXThisAddress(), ThisTy);
1509
7.20k
      break;
1510
7.20k
    }
1511
1512
    // Fallthrough: act like we're in the base variant.
1513
8
    LLVM_FALLTHROUGH;
1514
1515
8.03k
  case Dtor_Base:
1516
8.03k
    assert(Body);
1517
1518
    // Enter the cleanup scopes for fields and non-virtual bases.
1519
8.03k
    EnterDtorCleanups(Dtor, Dtor_Base);
1520
1521
    // Initialize the vtable pointers before entering the body.
1522
8.03k
    if (!CanSkipVTablePointerInitialization(*this, Dtor)) {
1523
      // Insert the llvm.launder.invariant.group intrinsic before initializing
1524
      // the vptrs to cancel any previous assumptions we might have made.
1525
156
      if (CGM.getCodeGenOpts().StrictVTablePointers &&
1526
2
          CGM.getCodeGenOpts().OptimizationLevel > 0)
1527
2
        CXXThisValue = Builder.CreateLaunderInvariantGroup(LoadCXXThis());
1528
156
      InitializeVTablePointers(Dtor->getParent());
1529
156
    }
1530
1531
8.03k
    if (isTryBody)
1532
18
      EmitStmt(cast<CXXTryStmt>(Body)->getTryBlock());
1533
8.01k
    else if (Body)
1534
8.01k
      EmitStmt(Body);
1535
0
    else {
1536
0
      assert(Dtor->isImplicit() && "bodyless dtor not implicit");
1537
      // nothing to do besides what's in the epilogue
1538
0
    }
1539
    // -fapple-kext must inline any call to this dtor into
1540
    // the caller's body.
1541
8.03k
    if (getLangOpts().AppleKext)
1542
5
      CurFn->addFnAttr(llvm::Attribute::AlwaysInline);
1543
1544
8.03k
    break;
1545
15.2k
  }
1546
1547
  // Jump out through the epilogue cleanups.
1548
15.2k
  DtorEpilogue.ForceCleanup();
1549
1550
  // Exit the try if applicable.
1551
15.2k
  if (isTryBody)
1552
18
    ExitCXXTryStmt(*cast<CXXTryStmt>(Body), true);
1553
15.2k
}
1554
1555
689
void CodeGenFunction::emitImplicitAssignmentOperatorBody(FunctionArgList &Args) {
1556
689
  const CXXMethodDecl *AssignOp = cast<CXXMethodDecl>(CurGD.getDecl());
1557
689
  const Stmt *RootS = AssignOp->getBody();
1558
689
  assert(isa<CompoundStmt>(RootS) &&
1559
689
         "Body of an implicit assignment operator should be compound stmt.");
1560
689
  const CompoundStmt *RootCS = cast<CompoundStmt>(RootS);
1561
1562
689
  LexicalScope Scope(*this, RootCS->getSourceRange());
1563
1564
689
  incrementProfileCounter(RootCS);
1565
689
  AssignmentMemcpyizer AM(*this, AssignOp, Args);
1566
689
  for (auto *I : RootCS->body())
1567
1.24k
    AM.emitAssignment(I);
1568
689
  AM.finish();
1569
689
}
1570
1571
namespace {
1572
  llvm::Value *LoadThisForDtorDelete(CodeGenFunction &CGF,
1573
873
                                     const CXXDestructorDecl *DD) {
1574
873
    if (Expr *ThisArg = DD->getOperatorDeleteThisArg())
1575
12
      return CGF.EmitScalarExpr(ThisArg);
1576
861
    return CGF.LoadCXXThis();
1577
861
  }
1578
1579
  /// Call the operator delete associated with the current destructor.
1580
  struct CallDtorDelete final : EHScopeStack::Cleanup {
1581
645
    CallDtorDelete() {}
1582
1583
658
    void Emit(CodeGenFunction &CGF, Flags flags) override {
1584
658
      const CXXDestructorDecl *Dtor = cast<CXXDestructorDecl>(CGF.CurCodeDecl);
1585
658
      const CXXRecordDecl *ClassDecl = Dtor->getParent();
1586
658
      CGF.EmitDeleteCall(Dtor->getOperatorDelete(),
1587
658
                         LoadThisForDtorDelete(CGF, Dtor),
1588
658
                         CGF.getContext().getTagDeclType(ClassDecl));
1589
658
    }
1590
  };
1591
1592
  void EmitConditionalDtorDeleteCall(CodeGenFunction &CGF,
1593
                                     llvm::Value *ShouldDeleteCondition,
1594
211
                                     bool ReturnAfterDelete) {
1595
211
    llvm::BasicBlock *callDeleteBB = CGF.createBasicBlock("dtor.call_delete");
1596
211
    llvm::BasicBlock *continueBB = CGF.createBasicBlock("dtor.continue");
1597
211
    llvm::Value *ShouldCallDelete
1598
211
      = CGF.Builder.CreateIsNull(ShouldDeleteCondition);
1599
211
    CGF.Builder.CreateCondBr(ShouldCallDelete, continueBB, callDeleteBB);
1600
1601
211
    CGF.EmitBlock(callDeleteBB);
1602
211
    const CXXDestructorDecl *Dtor = cast<CXXDestructorDecl>(CGF.CurCodeDecl);
1603
211
    const CXXRecordDecl *ClassDecl = Dtor->getParent();
1604
211
    CGF.EmitDeleteCall(Dtor->getOperatorDelete(),
1605
211
                       LoadThisForDtorDelete(CGF, Dtor),
1606
211
                       CGF.getContext().getTagDeclType(ClassDecl));
1607
211
    assert(Dtor->getOperatorDelete()->isDestroyingOperatorDelete() ==
1608
211
               ReturnAfterDelete &&
1609
211
           "unexpected value for ReturnAfterDelete");
1610
211
    if (ReturnAfterDelete)
1611
8
      CGF.EmitBranchThroughCleanup(CGF.ReturnBlock);
1612
203
    else
1613
203
      CGF.Builder.CreateBr(continueBB);
1614
1615
211
    CGF.EmitBlock(continueBB);
1616
211
  }
1617
1618
  struct CallDtorDeleteConditional final : EHScopeStack::Cleanup {
1619
    llvm::Value *ShouldDeleteCondition;
1620
1621
  public:
1622
    CallDtorDeleteConditional(llvm::Value *ShouldDeleteCondition)
1623
203
        : ShouldDeleteCondition(ShouldDeleteCondition) {
1624
203
      assert(ShouldDeleteCondition != nullptr);
1625
203
    }
1626
1627
203
    void Emit(CodeGenFunction &CGF, Flags flags) override {
1628
203
      EmitConditionalDtorDeleteCall(CGF, ShouldDeleteCondition,
1629
203
                                    /*ReturnAfterDelete*/false);
1630
203
    }
1631
  };
1632
1633
  class DestroyField  final : public EHScopeStack::Cleanup {
1634
    const FieldDecl *field;
1635
    CodeGenFunction::Destroyer *destroyer;
1636
    bool useEHCleanupForArray;
1637
1638
  public:
1639
    DestroyField(const FieldDecl *field, CodeGenFunction::Destroyer *destroyer,
1640
                 bool useEHCleanupForArray)
1641
        : field(field), destroyer(destroyer),
1642
955
          useEHCleanupForArray(useEHCleanupForArray) {}
1643
1644
995
    void Emit(CodeGenFunction &CGF, Flags flags) override {
1645
      // Find the address of the field.
1646
995
      Address thisValue = CGF.LoadCXXThisAddress();
1647
995
      QualType RecordTy = CGF.getContext().getTagDeclType(field->getParent());
1648
995
      LValue ThisLV = CGF.MakeAddrLValue(thisValue, RecordTy);
1649
995
      LValue LV = CGF.EmitLValueForField(ThisLV, field);
1650
995
      assert(LV.isSimple());
1651
1652
995
      CGF.emitDestroy(LV.getAddress(CGF), field->getType(), destroyer,
1653
995
                      flags.isForNormalCleanup() && 
useEHCleanupForArray955
);
1654
995
    }
1655
  };
1656
1657
 static void EmitSanitizerDtorCallback(CodeGenFunction &CGF, llvm::Value *Ptr,
1658
56
             CharUnits::QuantityType PoisonSize) {
1659
56
   CodeGenFunction::SanitizerScope SanScope(&CGF);
1660
   // Pass in void pointer and size of region as arguments to runtime
1661
   // function
1662
56
   llvm::Value *Args[] = {CGF.Builder.CreateBitCast(Ptr, CGF.VoidPtrTy),
1663
56
                          llvm::ConstantInt::get(CGF.SizeTy, PoisonSize)};
1664
1665
56
   llvm::Type *ArgTypes[] = {CGF.VoidPtrTy, CGF.SizeTy};
1666
1667
56
   llvm::FunctionType *FnType =
1668
56
       llvm::FunctionType::get(CGF.VoidTy, ArgTypes, false);
1669
56
   llvm::FunctionCallee Fn =
1670
56
       CGF.CGM.CreateRuntimeFunction(FnType, "__sanitizer_dtor_callback");
1671
56
   CGF.EmitNounwindRuntimeCall(Fn, Args);
1672
56
 }
1673
1674
  class SanitizeDtorMembers final : public EHScopeStack::Cleanup {
1675
    const CXXDestructorDecl *Dtor;
1676
1677
  public:
1678
38
    SanitizeDtorMembers(const CXXDestructorDecl *Dtor) : Dtor(Dtor) {}
1679
1680
    // Generate function call for handling object poisoning.
1681
    // Disables tail call elimination, to prevent the current stack frame
1682
    // from disappearing from the stack trace.
1683
38
    void Emit(CodeGenFunction &CGF, Flags flags) override {
1684
38
      const ASTRecordLayout &Layout =
1685
38
          CGF.getContext().getASTRecordLayout(Dtor->getParent());
1686
1687
      // Nothing to poison.
1688
38
      if (Layout.getFieldCount() == 0)
1689
0
        return;
1690
1691
      // Prevent the current stack frame from disappearing from the stack trace.
1692
38
      CGF.CurFn->addFnAttr("disable-tail-calls", "true");
1693
1694
      // Construct pointer to region to begin poisoning, and calculate poison
1695
      // size, so that only members declared in this class are poisoned.
1696
38
      ASTContext &Context = CGF.getContext();
1697
38
      unsigned fieldIndex = 0;
1698
38
      int startIndex = -1;
1699
      // RecordDecl::field_iterator Field;
1700
68
      for (const FieldDecl *Field : Dtor->getParent()->fields()) {
1701
        // Poison field if it is trivial
1702
68
        if (FieldHasTrivialDestructorBody(Context, Field)) {
1703
          // Start sanitizing at this field
1704
62
          if (startIndex < 0)
1705
40
            startIndex = fieldIndex;
1706
1707
          // Currently on the last field, and it must be poisoned with the
1708
          // current block.
1709
62
          if (fieldIndex == Layout.getFieldCount() - 1) {
1710
36
            PoisonMembers(CGF, startIndex, Layout.getFieldCount());
1711
36
          }
1712
6
        } else if (startIndex >= 0) {
1713
          // No longer within a block of memory to poison, so poison the block
1714
4
          PoisonMembers(CGF, startIndex, fieldIndex);
1715
          // Re-set the start index
1716
4
          startIndex = -1;
1717
4
        }
1718
68
        fieldIndex += 1;
1719
68
      }
1720
38
    }
1721
1722
  private:
1723
    /// \param layoutStartOffset index of the ASTRecordLayout field to
1724
    ///     start poisoning (inclusive)
1725
    /// \param layoutEndOffset index of the ASTRecordLayout field to
1726
    ///     end poisoning (exclusive)
1727
    void PoisonMembers(CodeGenFunction &CGF, unsigned layoutStartOffset,
1728
40
                     unsigned layoutEndOffset) {
1729
40
      ASTContext &Context = CGF.getContext();
1730
40
      const ASTRecordLayout &Layout =
1731
40
          Context.getASTRecordLayout(Dtor->getParent());
1732
1733
40
      llvm::ConstantInt *OffsetSizePtr = llvm::ConstantInt::get(
1734
40
          CGF.SizeTy,
1735
40
          Context.toCharUnitsFromBits(Layout.getFieldOffset(layoutStartOffset))
1736
40
              .getQuantity());
1737
1738
40
      llvm::Value *OffsetPtr = CGF.Builder.CreateGEP(
1739
40
          CGF.Builder.CreateBitCast(CGF.LoadCXXThis(), CGF.Int8PtrTy),
1740
40
          OffsetSizePtr);
1741
1742
40
      CharUnits::QuantityType PoisonSize;
1743
40
      if (layoutEndOffset >= Layout.getFieldCount()) {
1744
36
        PoisonSize = Layout.getNonVirtualSize().getQuantity() -
1745
36
                     Context.toCharUnitsFromBits(
1746
36
                                Layout.getFieldOffset(layoutStartOffset))
1747
36
                         .getQuantity();
1748
4
      } else {
1749
4
        PoisonSize = Context.toCharUnitsFromBits(
1750
4
                                Layout.getFieldOffset(layoutEndOffset) -
1751
4
                                Layout.getFieldOffset(layoutStartOffset))
1752
4
                         .getQuantity();
1753
4
      }
1754
1755
40
      if (PoisonSize == 0)
1756
2
        return;
1757
1758
38
      EmitSanitizerDtorCallback(CGF, OffsetPtr, PoisonSize);
1759
38
    }
1760
  };
1761
1762
 class SanitizeDtorVTable final : public EHScopeStack::Cleanup {
1763
    const CXXDestructorDecl *Dtor;
1764
1765
  public:
1766
18
    SanitizeDtorVTable(const CXXDestructorDecl *Dtor) : Dtor(Dtor) {}
1767
1768
    // Generate function call for handling vtable pointer poisoning.
1769
18
    void Emit(CodeGenFunction &CGF, Flags flags) override {
1770
18
      assert(Dtor->getParent()->isDynamicClass());
1771
18
      (void)Dtor;
1772
18
      ASTContext &Context = CGF.getContext();
1773
      // Poison vtable and vtable ptr if they exist for this class.
1774
18
      llvm::Value *VTablePtr = CGF.LoadCXXThis();
1775
1776
18
      CharUnits::QuantityType PoisonSize =
1777
18
          Context.toCharUnitsFromBits(CGF.PointerWidthInBits).getQuantity();
1778
      // Pass in void pointer and size of region as arguments to runtime
1779
      // function
1780
18
      EmitSanitizerDtorCallback(CGF, VTablePtr, PoisonSize);
1781
18
    }
1782
 };
1783
} // end anonymous namespace
1784
1785
/// Emit all code that comes at the end of class's
1786
/// destructor. This is to call destructors on members and base classes
1787
/// in reverse order of their construction.
1788
///
1789
/// For a deleting destructor, this also handles the case where a destroying
1790
/// operator delete completely overrides the definition.
1791
void CodeGenFunction::EnterDtorCleanups(const CXXDestructorDecl *DD,
1792
16.0k
                                        CXXDtorType DtorType) {
1793
16.0k
  assert((!DD->isTrivial() || DD->hasAttr<DLLExportAttr>()) &&
1794
16.0k
         "Should not emit dtor epilogue for non-exported trivial dtor!");
1795
1796
  // The deleting-destructor phase just needs to call the appropriate
1797
  // operator delete that Sema picked up.
1798
16.0k
  if (DtorType == Dtor_Deleting) {
1799
860
    assert(DD->getOperatorDelete() &&
1800
860
           "operator delete missing - EnterDtorCleanups");
1801
860
    if (CXXStructorImplicitParamValue) {
1802
      // If there is an implicit param to the deleting dtor, it's a boolean
1803
      // telling whether this is a deleting destructor.
1804
211
      if (DD->getOperatorDelete()->isDestroyingOperatorDelete())
1805
8
        EmitConditionalDtorDeleteCall(*this, CXXStructorImplicitParamValue,
1806
8
                                      /*ReturnAfterDelete*/true);
1807
203
      else
1808
203
        EHStack.pushCleanup<CallDtorDeleteConditional>(
1809
203
            NormalAndEHCleanup, CXXStructorImplicitParamValue);
1810
649
    } else {
1811
649
      if (DD->getOperatorDelete()->isDestroyingOperatorDelete()) {
1812
4
        const CXXRecordDecl *ClassDecl = DD->getParent();
1813
4
        EmitDeleteCall(DD->getOperatorDelete(),
1814
4
                       LoadThisForDtorDelete(*this, DD),
1815
4
                       getContext().getTagDeclType(ClassDecl));
1816
4
        EmitBranchThroughCleanup(ReturnBlock);
1817
645
      } else {
1818
645
        EHStack.pushCleanup<CallDtorDelete>(NormalAndEHCleanup);
1819
645
      }
1820
649
    }
1821
860
    return;
1822
860
  }
1823
1824
15.2k
  const CXXRecordDecl *ClassDecl = DD->getParent();
1825
1826
  // Unions have no bases and do not call field destructors.
1827
15.2k
  if (ClassDecl->isUnion())
1828
4
    return;
1829
1830
  // The complete-destructor phase just destructs all the virtual bases.
1831
15.2k
  if (DtorType == Dtor_Complete) {
1832
    // Poison the vtable pointer such that access after the base
1833
    // and member destructors are invoked is invalid.
1834
7.20k
    if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
1835
39
        SanOpts.has(SanitizerKind::Memory) && 
ClassDecl->getNumVBases()38
&&
1836
4
        ClassDecl->isPolymorphic())
1837
4
      EHStack.pushCleanup<SanitizeDtorVTable>(NormalAndEHCleanup, DD);
1838
1839
    // We push them in the forward order so that they'll be popped in
1840
    // the reverse order.
1841
228
    for (const auto &Base : ClassDecl->vbases()) {
1842
228
      auto *BaseClassDecl =
1843
228
          cast<CXXRecordDecl>(Base.getType()->castAs<RecordType>()->getDecl());
1844
1845
      // Ignore trivial destructors.
1846
228
      if (BaseClassDecl->hasTrivialDestructor())
1847
44
        continue;
1848
1849
184
      EHStack.pushCleanup<CallBaseDtor>(NormalAndEHCleanup,
1850
184
                                        BaseClassDecl,
1851
184
                                        /*BaseIsVirtual*/ true);
1852
184
    }
1853
1854
7.20k
    return;
1855
7.20k
  }
1856
1857
8.02k
  assert(DtorType == Dtor_Base);
1858
  // Poison the vtable pointer if it has no virtual bases, but inherits
1859
  // virtual functions.
1860
8.02k
  if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
1861
39
      SanOpts.has(SanitizerKind::Memory) && 
!ClassDecl->getNumVBases()38
&&
1862
34
      ClassDecl->isPolymorphic())
1863
14
    EHStack.pushCleanup<SanitizeDtorVTable>(NormalAndEHCleanup, DD);
1864
1865
  // Destroy non-virtual bases.
1866
2.20k
  for (const auto &Base : ClassDecl->bases()) {
1867
    // Ignore virtual bases.
1868
2.20k
    if (Base.isVirtual())
1869
127
      continue;
1870
1871
2.08k
    CXXRecordDecl *BaseClassDecl = Base.getType()->getAsCXXRecordDecl();
1872
1873
    // Ignore trivial destructors.
1874
2.08k
    if (BaseClassDecl->hasTrivialDestructor())
1875
925
      continue;
1876
1877
1.15k
    EHStack.pushCleanup<CallBaseDtor>(NormalAndEHCleanup,
1878
1.15k
                                      BaseClassDecl,
1879
1.15k
                                      /*BaseIsVirtual*/ false);
1880
1.15k
  }
1881
1882
  // Poison fields such that access after their destructors are
1883
  // invoked, and before the base class destructor runs, is invalid.
1884
8.02k
  if (CGM.getCodeGenOpts().SanitizeMemoryUseAfterDtor &&
1885
39
      SanOpts.has(SanitizerKind::Memory))
1886
38
    EHStack.pushCleanup<SanitizeDtorMembers>(NormalAndEHCleanup, DD);
1887
1888
  // Destroy direct fields.
1889
9.45k
  for (const auto *Field : ClassDecl->fields()) {
1890
9.45k
    QualType type = Field->getType();
1891
9.45k
    QualType::DestructionKind dtorKind = type.isDestructedType();
1892
9.45k
    if (!dtorKind) 
continue8.49k
;
1893
1894
    // Anonymous union members do not have their destructors called.
1895
964
    const RecordType *RT = type->getAsUnionType();
1896
964
    if (RT && 
RT->getDecl()->isAnonymousStructOrUnion()9
)
continue9
;
1897
1898
955
    CleanupKind cleanupKind = getCleanupKind(dtorKind);
1899
955
    EHStack.pushCleanup<DestroyField>(cleanupKind, Field,
1900
955
                                      getDestroyer(dtorKind),
1901
955
                                      cleanupKind & EHCleanup);
1902
955
  }
1903
8.02k
}
1904
1905
/// EmitCXXAggrConstructorCall - Emit a loop to call a particular
1906
/// constructor for each of several members of an array.
1907
///
1908
/// \param ctor the constructor to call for each element
1909
/// \param arrayType the type of the array to initialize
1910
/// \param arrayBegin an arrayType*
1911
/// \param zeroInitialize true if each element should be
1912
///   zero-initialized before it is constructed
1913
void CodeGenFunction::EmitCXXAggrConstructorCall(
1914
    const CXXConstructorDecl *ctor, const ArrayType *arrayType,
1915
    Address arrayBegin, const CXXConstructExpr *E, bool NewPointerIsChecked,
1916
908
    bool zeroInitialize) {
1917
908
  QualType elementType;
1918
908
  llvm::Value *numElements =
1919
908
    emitArrayLength(arrayType, elementType, arrayBegin);
1920
1921
908
  EmitCXXAggrConstructorCall(ctor, numElements, arrayBegin, E,
1922
908
                             NewPointerIsChecked, zeroInitialize);
1923
908
}
1924
1925
/// EmitCXXAggrConstructorCall - Emit a loop to call a particular
1926
/// constructor for each of several members of an array.
1927
///
1928
/// \param ctor the constructor to call for each element
1929
/// \param numElements the number of elements in the array;
1930
///   may be zero
1931
/// \param arrayBase a T*, where T is the type constructed by ctor
1932
/// \param zeroInitialize true if each element should be
1933
///   zero-initialized before it is constructed
1934
void CodeGenFunction::EmitCXXAggrConstructorCall(const CXXConstructorDecl *ctor,
1935
                                                 llvm::Value *numElements,
1936
                                                 Address arrayBase,
1937
                                                 const CXXConstructExpr *E,
1938
                                                 bool NewPointerIsChecked,
1939
973
                                                 bool zeroInitialize) {
1940
  // It's legal for numElements to be zero.  This can happen both
1941
  // dynamically, because x can be zero in 'new A[x]', and statically,
1942
  // because of GCC extensions that permit zero-length arrays.  There
1943
  // are probably legitimate places where we could assume that this
1944
  // doesn't happen, but it's not clear that it's worth it.
1945
973
  llvm::BranchInst *zeroCheckBranch = nullptr;
1946
1947
  // Optimize for a constant count.
1948
973
  llvm::ConstantInt *constantCount
1949
973
    = dyn_cast<llvm::ConstantInt>(numElements);
1950
973
  if (constantCount) {
1951
    // Just skip out if the constant count is zero.
1952
932
    if (constantCount->isZero()) 
return0
;
1953
1954
  // Otherwise, emit the check.
1955
41
  } else {
1956
41
    llvm::BasicBlock *loopBB = createBasicBlock("new.ctorloop");
1957
41
    llvm::Value *iszero = Builder.CreateIsNull(numElements, "isempty");
1958
41
    zeroCheckBranch = Builder.CreateCondBr(iszero, loopBB, loopBB);
1959
41
    EmitBlock(loopBB);
1960
41
  }
1961
1962
  // Find the end of the array.
1963
973
  llvm::Value *arrayBegin = arrayBase.getPointer();
1964
973
  llvm::Value *arrayEnd = Builder.CreateInBoundsGEP(arrayBegin, numElements,
1965
973
                                                    "arrayctor.end");
1966
1967
  // Enter the loop, setting up a phi for the current location to initialize.
1968
973
  llvm::BasicBlock *entryBB = Builder.GetInsertBlock();
1969
973
  llvm::BasicBlock *loopBB = createBasicBlock("arrayctor.loop");
1970
973
  EmitBlock(loopBB);
1971
973
  llvm::PHINode *cur = Builder.CreatePHI(arrayBegin->getType(), 2,
1972
973
                                         "arrayctor.cur");
1973
973
  cur->addIncoming(arrayBegin, entryBB);
1974
1975
  // Inside the loop body, emit the constructor call on the array element.
1976
1977
  // The alignment of the base, adjusted by the size of a single element,
1978
  // provides a conservative estimate of the alignment of every element.
1979
  // (This assumes we never start tracking offsetted alignments.)
1980
  //
1981
  // Note that these are complete objects and so we don't need to
1982
  // use the non-virtual size or alignment.
1983
973
  QualType type = getContext().getTypeDeclType(ctor->getParent());
1984
973
  CharUnits eltAlignment =
1985
973
    arrayBase.getAlignment()
1986
973
             .alignmentOfArrayElement(getContext().getTypeSizeInChars(type));
1987
973
  Address curAddr = Address(cur, eltAlignment);
1988
1989
  // Zero initialize the storage, if requested.
1990
973
  if (zeroInitialize)
1991
2
    EmitNullInitialization(curAddr, type);
1992
1993
  // C++ [class.temporary]p4:
1994
  // There are two contexts in which temporaries are destroyed at a different
1995
  // point than the end of the full-expression. The first context is when a
1996
  // default constructor is called to initialize an element of an array.
1997
  // If the constructor has one or more default arguments, the destruction of
1998
  // every temporary created in a default argument expression is sequenced
1999
  // before the construction of the next array element, if any.
2000
2001
973
  {
2002
973
    RunCleanupsScope Scope(*this);
2003
2004
    // Evaluate the constructor and its arguments in a regular
2005
    // partial-destroy cleanup.
2006
973
    if (getLangOpts().Exceptions &&
2007
85
        !ctor->getParent()->hasTrivialDestructor()) {
2008
38
      Destroyer *destroyer = destroyCXXObject;
2009
38
      pushRegularPartialArrayCleanup(arrayBegin, cur, type, eltAlignment,
2010
38
                                     *destroyer);
2011
38
    }
2012
973
    auto currAVS = AggValueSlot::forAddr(
2013
973
        curAddr, type.getQualifiers(), AggValueSlot::IsDestructed,
2014
973
        AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased,
2015
973
        AggValueSlot::DoesNotOverlap, AggValueSlot::IsNotZeroed,
2016
97
        NewPointerIsChecked ? AggValueSlot::IsSanitizerChecked
2017
876
                            : AggValueSlot::IsNotSanitizerChecked);
2018
973
    EmitCXXConstructorCall(ctor, Ctor_Complete, /*ForVirtualBase=*/false,
2019
973
                           /*Delegating=*/false, currAVS, E);
2020
973
  }
2021
2022
  // Go to the next element.
2023
973
  llvm::Value *next =
2024
973
    Builder.CreateInBoundsGEP(cur, llvm::ConstantInt::get(SizeTy, 1),
2025
973
                              "arrayctor.next");
2026
973
  cur->addIncoming(next, Builder.GetInsertBlock());
2027
2028
  // Check whether that's the end of the loop.
2029
973
  llvm::Value *done = Builder.CreateICmpEQ(next, arrayEnd, "arrayctor.done");
2030
973
  llvm::BasicBlock *contBB = createBasicBlock("arrayctor.cont");
2031
973
  Builder.CreateCondBr(done, contBB, loopBB);
2032
2033
  // Patch the earlier check to skip over the loop.
2034
973
  if (zeroCheckBranch) 
zeroCheckBranch->setSuccessor(0, contBB)41
;
2035
2036
973
  EmitBlock(contBB);
2037
973
}
2038
2039
void CodeGenFunction::destroyCXXObject(CodeGenFunction &CGF,
2040
                                       Address addr,
2041
16.5k
                                       QualType type) {
2042
16.5k
  const RecordType *rtype = type->castAs<RecordType>();
2043
16.5k
  const CXXRecordDecl *record = cast<CXXRecordDecl>(rtype->getDecl());
2044
16.5k
  const CXXDestructorDecl *dtor = record->getDestructor();
2045
16.5k
  assert(!dtor->isTrivial());
2046
16.5k
  CGF.EmitCXXDestructorCall(dtor, Dtor_Complete, /*for vbase*/ false,
2047
16.5k
                            /*Delegating=*/false, addr, type);
2048
16.5k
}
2049
2050
void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D,
2051
                                             CXXCtorType Type,
2052
                                             bool ForVirtualBase,
2053
                                             bool Delegating,
2054
                                             AggValueSlot ThisAVS,
2055
41.2k
                                             const CXXConstructExpr *E) {
2056
41.2k
  CallArgList Args;
2057
41.2k
  Address This = ThisAVS.getAddress();
2058
41.2k
  LangAS SlotAS = ThisAVS.getQualifiers().getAddressSpace();
2059
41.2k
  QualType ThisType = D->getThisType();
2060
41.2k
  LangAS ThisAS = ThisType.getTypePtr()->getPointeeType().getAddressSpace();
2061
41.2k
  llvm::Value *ThisPtr = This.getPointer();
2062
2063
41.2k
  if (SlotAS != ThisAS) {
2064
29
    unsigned TargetThisAS = getContext().getTargetAddressSpace(ThisAS);
2065
29
    llvm::Type *NewType =
2066
29
        ThisPtr->getType()->getPointerElementType()->getPointerTo(TargetThisAS);
2067
29
    ThisPtr = getTargetHooks().performAddrSpaceCast(*this, This.getPointer(),
2068
29
                                                    ThisAS, SlotAS, NewType);
2069
29
  }
2070
2071
  // Push the this ptr.
2072
41.2k
  Args.add(RValue::get(ThisPtr), D->getThisType());
2073
2074
  // If this is a trivial constructor, emit a memcpy now before we lose
2075
  // the alignment information on the argument.
2076
  // FIXME: It would be better to preserve alignment information into CallArg.
2077
41.2k
  if (isMemcpyEquivalentSpecialMember(D)) {
2078
6.37k
    assert(E->getNumArgs() == 1 && "unexpected argcount for trivial ctor");
2079
2080
6.37k
    const Expr *Arg = E->getArg(0);
2081
6.37k
    LValue Src = EmitLValue(Arg);
2082
6.37k
    QualType DestTy = getContext().getTypeDeclType(D->getParent());
2083
6.37k
    LValue Dest = MakeAddrLValue(This, DestTy);
2084
6.37k
    EmitAggregateCopyCtor(Dest, Src, ThisAVS.mayOverlap());
2085
6.37k
    return;
2086
6.37k
  }
2087
2088
  // Add the rest of the user-supplied arguments.
2089
34.8k
  const FunctionProtoType *FPT = D->getType()->castAs<FunctionProtoType>();
2090
34.8k
  EvaluationOrder Order = E->isListInitialization()
2091
395
                              ? EvaluationOrder::ForceLeftToRight
2092
34.4k
                              : EvaluationOrder::Default;
2093
34.8k
  EmitCallArgs(Args, FPT, E->arguments(), E->getConstructor(),
2094
34.8k
               /*ParamsToSkip*/ 0, Order);
2095
2096
34.8k
  EmitCXXConstructorCall(D, Type, ForVirtualBase, Delegating, This, Args,
2097
34.8k
                         ThisAVS.mayOverlap(), E->getExprLoc(),
2098
34.8k
                         ThisAVS.isSanitizerChecked());
2099
34.8k
}
2100
2101
static bool canEmitDelegateCallArgs(CodeGenFunction &CGF,
2102
                                    const CXXConstructorDecl *Ctor,
2103
225
                                    CXXCtorType Type, CallArgList &Args) {
2104
  // We can't forward a variadic call.
2105
225
  if (Ctor->isVariadic())
2106
31
    return false;
2107
2108
194
  if (CGF.getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) {
2109
    // If the parameters are callee-cleanup, it's not safe to forward.
2110
32
    for (auto *P : Ctor->parameters())
2111
56
      if (P->needsDestruction(CGF.getContext()))
2112
16
        return false;
2113
2114
    // Likewise if they're inalloca.
2115
16
    const CGFunctionInfo &Info =
2116
16
        CGF.CGM.getTypes().arrangeCXXConstructorCall(Args, Ctor, Type, 0, 0);
2117
16
    if (Info.usesInAlloca())
2118
0
      return false;
2119
178
  }
2120
2121
  // Anything else should be OK.
2122
178
  return true;
2123
178
}
2124
2125
void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D,
2126
                                             CXXCtorType Type,
2127
                                             bool ForVirtualBase,
2128
                                             bool Delegating,
2129
                                             Address This,
2130
                                             CallArgList &Args,
2131
                                             AggValueSlot::Overlap_t Overlap,
2132
                                             SourceLocation Loc,
2133
50.6k
                                             bool NewPointerIsChecked) {
2134
50.6k
  const CXXRecordDecl *ClassDecl = D->getParent();
2135
2136
50.6k
  if (!NewPointerIsChecked)
2137
30.1k
    EmitTypeCheck(CodeGenFunction::TCK_ConstructorCall, Loc, This.getPointer(),
2138
30.1k
                  getContext().getRecordType(ClassDecl), CharUnits::Zero());
2139
2140
50.6k
  if (D->isTrivial() && 
D->isDefaultConstructor()54
) {
2141
33
    assert(Args.size() == 1 && "trivial default ctor with args");
2142
33
    return;
2143
33
  }
2144
2145
  // If this is a trivial constructor, just emit what's needed. If this is a
2146
  // union copy constructor, we must emit a memcpy, because the AST does not
2147
  // model that copy.
2148
50.5k
  if (isMemcpyEquivalentSpecialMember(D)) {
2149
18
    assert(Args.size() == 2 && "unexpected argcount for trivial ctor");
2150
2151
18
    QualType SrcTy = D->getParamDecl(0)->getType().getNonReferenceType();
2152
18
    Address Src(Args[1].getRValue(*this).getScalarVal(),
2153
18
                CGM.getNaturalTypeAlignment(SrcTy));
2154
18
    LValue SrcLVal = MakeAddrLValue(Src, SrcTy);
2155
18
    QualType DestTy = getContext().getTypeDeclType(ClassDecl);
2156
18
    LValue DestLVal = MakeAddrLValue(This, DestTy);
2157
18
    EmitAggregateCopyCtor(DestLVal, SrcLVal, Overlap);
2158
18
    return;
2159
18
  }
2160
2161
50.5k
  bool PassPrototypeArgs = true;
2162
  // Check whether we can actually emit the constructor before trying to do so.
2163
50.5k
  if (auto Inherited = D->getInheritedConstructor()) {
2164
235
    PassPrototypeArgs = getTypes().inheritingCtorHasParams(Inherited, Type);
2165
235
    if (PassPrototypeArgs && 
!canEmitDelegateCallArgs(*this, D, Type, Args)225
) {
2166
47
      EmitInlinedInheritingCXXConstructorCall(D, Type, ForVirtualBase,
2167
47
                                              Delegating, Args);
2168
47
      return;
2169
47
    }
2170
50.5k
  }
2171
2172
  // Insert any ABI-specific implicit constructor arguments.
2173
50.5k
  CGCXXABI::AddedStructorArgCounts ExtraArgs =
2174
50.5k
      CGM.getCXXABI().addImplicitConstructorArgs(*this, D, Type, ForVirtualBase,
2175
50.5k
                                                 Delegating, Args);
2176
2177
  // Emit the call.
2178
50.5k
  llvm::Constant *CalleePtr = CGM.getAddrOfCXXStructor(GlobalDecl(D, Type));
2179
50.5k
  const CGFunctionInfo &Info = CGM.getTypes().arrangeCXXConstructorCall(
2180
50.5k
      Args, D, Type, ExtraArgs.Prefix, ExtraArgs.Suffix, PassPrototypeArgs);
2181
50.5k
  CGCallee Callee = CGCallee::forDirect(CalleePtr, GlobalDecl(D, Type));
2182
50.5k
  EmitCall(Info, Callee, ReturnValueSlot(), Args, nullptr, Loc);
2183
2184
  // Generate vtable assumptions if we're constructing a complete object
2185
  // with a vtable.  We don't do this for base subobjects for two reasons:
2186
  // first, it's incorrect for classes with virtual bases, and second, we're
2187
  // about to overwrite the vptrs anyway.
2188
  // We also have to make sure if we can refer to vtable:
2189
  // - Otherwise we can refer to vtable if it's safe to speculatively emit.
2190
  // FIXME: If vtable is used by ctor/dtor, or if vtable is external and we are
2191
  // sure that definition of vtable is not hidden,
2192
  // then we are always safe to refer to it.
2193
  // FIXME: It looks like InstCombine is very inefficient on dealing with
2194
  // assumes. Make assumption loads require -fstrict-vtable-pointers temporarily.
2195
50.5k
  if (CGM.getCodeGenOpts().OptimizationLevel > 0 &&
2196
1.33k
      ClassDecl->isDynamicClass() && 
Type != Ctor_Base589
&&
2197
326
      CGM.getCXXABI().canSpeculativelyEmitVTable(ClassDecl) &&
2198
114
      CGM.getCodeGenOpts().StrictVTablePointers)
2199
56
    EmitVTableAssumptionLoads(ClassDecl, This);
2200
50.5k
}
2201
2202
void CodeGenFunction::EmitInheritedCXXConstructorCall(
2203
    const CXXConstructorDecl *D, bool ForVirtualBase, Address This,
2204
200
    bool InheritedFromVBase, const CXXInheritedCtorInitExpr *E) {
2205
200
  CallArgList Args;
2206
200
  CallArg ThisArg(RValue::get(This.getPointer()), D->getThisType());
2207
2208
  // Forward the parameters.
2209
200
  if (InheritedFromVBase &&
2210
17
      CGM.getTarget().getCXXABI().hasConstructorVariants()) {
2211
    // Nothing to do; this construction is not responsible for constructing
2212
    // the base class containing the inherited constructor.
2213
    // FIXME: Can we just pass undef's for the remaining arguments if we don't
2214
    // have constructor variants?
2215
10
    Args.push_back(ThisArg);
2216
190
  } else if (!CXXInheritedCtorInitExprArgs.empty()) {
2217
    // The inheriting constructor was inlined; just inject its arguments.
2218
47
    assert(CXXInheritedCtorInitExprArgs.size() >= D->getNumParams() &&
2219
47
           "wrong number of parameters for inherited constructor call");
2220
47
    Args = CXXInheritedCtorInitExprArgs;
2221
47
    Args[0] = ThisArg;
2222
143
  } else {
2223
    // The inheriting constructor was not inlined. Emit delegating arguments.
2224
143
    Args.push_back(ThisArg);
2225
143
    const auto *OuterCtor = cast<CXXConstructorDecl>(CurCodeDecl);
2226
143
    assert(OuterCtor->getNumParams() == D->getNumParams());
2227
143
    assert(!OuterCtor->isVariadic() && "should have been inlined");
2228
2229
269
    for (const auto *Param : OuterCtor->parameters()) {
2230
269
      assert(getContext().hasSameUnqualifiedType(
2231
269
          OuterCtor->getParamDecl(Param->getFunctionScopeIndex())->getType(),
2232
269
          Param->getType()));
2233
269
      EmitDelegateCallArg(Args, Param, E->getLocation());
2234
2235
      // Forward __attribute__(pass_object_size).
2236
269
      if (Param->hasAttr<PassObjectSizeAttr>()) {
2237
22
        auto *POSParam = SizeArguments[Param];
2238
22
        assert(POSParam && "missing pass_object_size value for forwarding");
2239
22
        EmitDelegateCallArg(Args, POSParam, E->getLocation());
2240
22
      }
2241
269
    }
2242
143
  }
2243
2244
200
  EmitCXXConstructorCall(D, Ctor_Base, ForVirtualBase, /*Delegating*/false,
2245
200
                         This, Args, AggValueSlot::MayOverlap,
2246
200
                         E->getLocation(), /*NewPointerIsChecked*/true);
2247
200
}
2248
2249
void CodeGenFunction::EmitInlinedInheritingCXXConstructorCall(
2250
    const CXXConstructorDecl *Ctor, CXXCtorType CtorType, bool ForVirtualBase,
2251
47
    bool Delegating, CallArgList &Args) {
2252
47
  GlobalDecl GD(Ctor, CtorType);
2253
47
  InlinedInheritingConstructorScope Scope(*this, GD);
2254
47
  ApplyInlineDebugLocation DebugScope(*this, GD);
2255
47
  RunCleanupsScope RunCleanups(*this);
2256
2257
  // Save the arguments to be passed to the inherited constructor.
2258
47
  CXXInheritedCtorInitExprArgs = Args;
2259
2260
47
  FunctionArgList Params;
2261
47
  QualType RetType = BuildFunctionArgList(CurGD, Params);
2262
47
  FnRetTy = RetType;
2263
2264
  // Insert any ABI-specific implicit constructor arguments.
2265
47
  CGM.getCXXABI().addImplicitConstructorArgs(*this, Ctor, CtorType,
2266
47
                                             ForVirtualBase, Delegating, Args);
2267
2268
  // Emit a simplified prolog. We only need to emit the implicit params.
2269
47
  assert(Args.size() >= Params.size() && "too few arguments for call");
2270
332
  for (unsigned I = 0, N = Args.size(); I != N; 
++I285
) {
2271
285
    if (I < Params.size() && 
isa<ImplicitParamDecl>(Params[I])231
) {
2272
61
      const RValue &RV = Args[I].getRValue(*this);
2273
61
      assert(!RV.isComplex() && "complex indirect params not supported");
2274
61
      ParamValue Val = RV.isScalar()
2275
61
                           ? ParamValue::forDirect(RV.getScalarVal())
2276
0
                           : ParamValue::forIndirect(RV.getAggregateAddress());
2277
61
      EmitParmDecl(*Params[I], Val, I + 1);
2278
61
    }
2279
285
  }
2280
2281
  // Create a return value slot if the ABI implementation wants one.
2282
  // FIXME: This is dumb, we should ask the ABI not to try to set the return
2283
  // value instead.
2284
47
  if (!RetType->isVoidType())
2285
28
    ReturnValue = CreateIRTemp(RetType, "retval.inhctor");
2286
2287
47
  CGM.getCXXABI().EmitInstanceFunctionProlog(*this);
2288
47
  CXXThisValue = CXXABIThisValue;
2289
2290
  // Directly emit the constructor initializers.
2291
47
  EmitCtorPrologue(Ctor, CtorType, Params);
2292
47
}
2293
2294
62
void CodeGenFunction::EmitVTableAssumptionLoad(const VPtr &Vptr, Address This) {
2295
62
  llvm::Value *VTableGlobal =
2296
62
      CGM.getCXXABI().getVTableAddressPoint(Vptr.Base, Vptr.VTableClass);
2297
62
  if (!VTableGlobal)
2298
0
    return;
2299
2300
  // We can just use the base offset in the complete class.
2301
62
  CharUnits NonVirtualOffset = Vptr.Base.getBaseOffset();
2302
2303
62
  if (!NonVirtualOffset.isZero())
2304
4
    This =
2305
4
        ApplyNonVirtualAndVirtualOffset(*this, This, NonVirtualOffset, nullptr,
2306
4
                                        Vptr.VTableClass, Vptr.NearestVBase);
2307
2308
62
  llvm::Value *VPtrValue =
2309
62
      GetVTablePtr(This, VTableGlobal->getType(), Vptr.VTableClass);
2310
62
  llvm::Value *Cmp =
2311
62
      Builder.CreateICmpEQ(VPtrValue, VTableGlobal, "cmp.vtables");
2312
62
  Builder.CreateAssumption(Cmp);
2313
62
}
2314
2315
void CodeGenFunction::EmitVTableAssumptionLoads(const CXXRecordDecl *ClassDecl,
2316
56
                                                Address This) {
2317
56
  if (CGM.getCXXABI().doStructorsInitializeVPtrs(ClassDecl))
2318
56
    for (const VPtr &Vptr : getVTablePointers(ClassDecl))
2319
62
      EmitVTableAssumptionLoad(Vptr, This);
2320
56
}
2321
2322
void
2323
CodeGenFunction::EmitSynthesizedCXXCopyCtorCall(const CXXConstructorDecl *D,
2324
                                                Address This, Address Src,
2325
57
                                                const CXXConstructExpr *E) {
2326
57
  const FunctionProtoType *FPT = D->getType()->castAs<FunctionProtoType>();
2327
2328
57
  CallArgList Args;
2329
2330
  // Push the this ptr.
2331
57
  Args.add(RValue::get(This.getPointer()), D->getThisType());
2332
2333
  // Push the src ptr.
2334
57
  QualType QT = *(FPT->param_type_begin());
2335
57
  llvm::Type *t = CGM.getTypes().ConvertType(QT);
2336
57
  Src = Builder.CreateBitCast(Src, t);
2337
57
  Args.add(RValue::get(Src.getPointer()), QT);
2338
2339
  // Skip over first argument (Src).
2340
57
  EmitCallArgs(Args, FPT, drop_begin(E->arguments(), 1), E->getConstructor(),
2341
57
               /*ParamsToSkip*/ 1);
2342
2343
57
  EmitCXXConstructorCall(D, Ctor_Complete, /*ForVirtualBase*/false,
2344
57
                         /*Delegating*/false, This, Args,
2345
57
                         AggValueSlot::MayOverlap, E->getExprLoc(),
2346
57
                         /*NewPointerIsChecked*/false);
2347
57
}
2348
2349
void
2350
CodeGenFunction::EmitDelegateCXXConstructorCall(const CXXConstructorDecl *Ctor,
2351
                                                CXXCtorType CtorType,
2352
                                                const FunctionArgList &Args,
2353
15.4k
                                                SourceLocation Loc) {
2354
15.4k
  CallArgList DelegateArgs;
2355
2356
15.4k
  FunctionArgList::const_iterator I = Args.begin(), E = Args.end();
2357
15.4k
  assert(I != E && "no parameters to constructor");
2358
2359
  // this
2360
15.4k
  Address This = LoadCXXThisAddress();
2361
15.4k
  DelegateArgs.add(RValue::get(This.getPointer()), (*I)->getType());
2362
15.4k
  ++I;
2363
2364
  // FIXME: The location of the VTT parameter in the parameter list is
2365
  // specific to the Itanium ABI and shouldn't be hardcoded here.
2366
15.4k
  if (CGM.getCXXABI().NeedsVTTParameter(CurGD)) {
2367
0
    assert(I != E && "cannot skip vtt parameter, already done with args");
2368
0
    assert((*I)->getType()->isPointerType() &&
2369
0
           "skipping parameter not of vtt type");
2370
0
    ++I;
2371
0
  }
2372
2373
  // Explicit arguments.
2374
32.3k
  for (; I != E; 
++I16.9k
) {
2375
16.9k
    const VarDecl *param = *I;
2376
    // FIXME: per-argument source location
2377
16.9k
    EmitDelegateCallArg(DelegateArgs, param, Loc);
2378
16.9k
  }
2379
2380
15.4k
  EmitCXXConstructorCall(Ctor, CtorType, /*ForVirtualBase=*/false,
2381
15.4k
                         /*Delegating=*/true, This, DelegateArgs,
2382
15.4k
                         AggValueSlot::MayOverlap, Loc,
2383
15.4k
                         /*NewPointerIsChecked=*/true);
2384
15.4k
}
2385
2386
namespace {
2387
  struct CallDelegatingCtorDtor final : EHScopeStack::Cleanup {
2388
    const CXXDestructorDecl *Dtor;
2389
    Address Addr;
2390
    CXXDtorType Type;
2391
2392
    CallDelegatingCtorDtor(const CXXDestructorDecl *D, Address Addr,
2393
                           CXXDtorType Type)
2394
65
      : Dtor(D), Addr(Addr), Type(Type) {}
2395
2396
4
    void Emit(CodeGenFunction &CGF, Flags flags) override {
2397
      // We are calling the destructor from within the constructor.
2398
      // Therefore, "this" should have the expected type.
2399
4
      QualType ThisTy = Dtor->getThisObjectType();
2400
4
      CGF.EmitCXXDestructorCall(Dtor, Type, /*ForVirtualBase=*/false,
2401
4
                                /*Delegating=*/true, Addr, ThisTy);
2402
4
    }
2403
  };
2404
} // end anonymous namespace
2405
2406
void
2407
CodeGenFunction::EmitDelegatingCXXConstructorCall(const CXXConstructorDecl *Ctor,
2408
75
                                                  const FunctionArgList &Args) {
2409
75
  assert(Ctor->isDelegatingConstructor());
2410
2411
75
  Address ThisPtr = LoadCXXThisAddress();
2412
2413
75
  AggValueSlot AggSlot =
2414
75
    AggValueSlot::forAddr(ThisPtr, Qualifiers(),
2415
75
                          AggValueSlot::IsDestructed,
2416
75
                          AggValueSlot::DoesNotNeedGCBarriers,
2417
75
                          AggValueSlot::IsNotAliased,
2418
75
                          AggValueSlot::MayOverlap,
2419
75
                          AggValueSlot::IsNotZeroed,
2420
                          // Checks are made by the code that calls constructor.
2421
75
                          AggValueSlot::IsSanitizerChecked);
2422
2423
75
  EmitAggExpr(Ctor->init_begin()[0]->getInit(), AggSlot);
2424
2425
75
  const CXXRecordDecl *ClassDecl = Ctor->getParent();
2426
75
  if (CGM.getLangOpts().Exceptions && 
!ClassDecl->hasTrivialDestructor()73
) {
2427
65
    CXXDtorType Type =
2428
62
      CurGD.getCtorType() == Ctor_Complete ? Dtor_Complete : 
Dtor_Base3
;
2429
2430
65
    EHStack.pushCleanup<CallDelegatingCtorDtor>(EHCleanup,
2431
65
                                                ClassDecl->getDestructor(),
2432
65
                                                ThisPtr, Type);
2433
65
  }
2434
75
}
2435
2436
void CodeGenFunction::EmitCXXDestructorCall(const CXXDestructorDecl *DD,
2437
                                            CXXDtorType Type,
2438
                                            bool ForVirtualBase,
2439
                                            bool Delegating, Address This,
2440
27.0k
                                            QualType ThisTy) {
2441
27.0k
  CGM.getCXXABI().EmitDestructorCall(*this, DD, Type, ForVirtualBase,
2442
27.0k
                                     Delegating, This, ThisTy);
2443
27.0k
}
2444
2445
namespace {
2446
  struct CallLocalDtor final : EHScopeStack::Cleanup {
2447
    const CXXDestructorDecl *Dtor;
2448
    Address Addr;
2449
    QualType Ty;
2450
2451
    CallLocalDtor(const CXXDestructorDecl *D, Address Addr, QualType Ty)
2452
9
        : Dtor(D), Addr(Addr), Ty(Ty) {}
2453
2454
9
    void Emit(CodeGenFunction &CGF, Flags flags) override {
2455
9
      CGF.EmitCXXDestructorCall(Dtor, Dtor_Complete,
2456
9
                                /*ForVirtualBase=*/false,
2457
9
                                /*Delegating=*/false, Addr, Ty);
2458
9
    }
2459
  };
2460
} // end anonymous namespace
2461
2462
void CodeGenFunction::PushDestructorCleanup(const CXXDestructorDecl *D,
2463
9
                                            QualType T, Address Addr) {
2464
9
  EHStack.pushCleanup<CallLocalDtor>(NormalAndEHCleanup, D, Addr, T);
2465
9
}
2466
2467
12
void CodeGenFunction::PushDestructorCleanup(QualType T, Address Addr) {
2468
12
  CXXRecordDecl *ClassDecl = T->getAsCXXRecordDecl();
2469
12
  if (!ClassDecl) 
return0
;
2470
12
  if (ClassDecl->hasTrivialDestructor()) 
return3
;
2471
2472
9
  const CXXDestructorDecl *D = ClassDecl->getDestructor();
2473
9
  assert(D && D->isUsed() && "destructor not marked as used!");
2474
9
  PushDestructorCleanup(D, T, Addr);
2475
9
}
2476
2477
3.85k
void CodeGenFunction::InitializeVTablePointer(const VPtr &Vptr) {
2478
  // Compute the address point.
2479
3.85k
  llvm::Value *VTableAddressPoint =
2480
3.85k
      CGM.getCXXABI().getVTableAddressPointInStructor(
2481
3.85k
          *this, Vptr.VTableClass, Vptr.Base, Vptr.NearestVBase);
2482
2483
3.85k
  if (!VTableAddressPoint)
2484
719
    return;
2485
2486
  // Compute where to store the address point.
2487
3.13k
  llvm::Value *VirtualOffset = nullptr;
2488
3.13k
  CharUnits NonVirtualOffset = CharUnits::Zero();
2489
2490
3.13k
  if (CGM.getCXXABI().isVirtualOffsetNeededForVTableField(*this, Vptr)) {
2491
    // We need to use the virtual base offset offset because the virtual base
2492
    // might have a different offset in the most derived class.
2493
2494
349
    VirtualOffset = CGM.getCXXABI().GetVirtualBaseClassOffset(
2495
349
        *this, LoadCXXThisAddress(), Vptr.VTableClass, Vptr.NearestVBase);
2496
349
    NonVirtualOffset = Vptr.OffsetFromNearestVBase;
2497
2.78k
  } else {
2498
    // We can just use the base offset in the complete class.
2499
2.78k
    NonVirtualOffset = Vptr.Base.getBaseOffset();
2500
2.78k
  }
2501
2502
  // Apply the offsets.
2503
3.13k
  Address VTableField = LoadCXXThisAddress();
2504
2505
3.13k
  if (!NonVirtualOffset.isZero() || 
VirtualOffset2.90k
)
2506
550
    VTableField = ApplyNonVirtualAndVirtualOffset(
2507
550
        *this, VTableField, NonVirtualOffset, VirtualOffset, Vptr.VTableClass,
2508
550
        Vptr.NearestVBase);
2509
2510
  // Finally, store the address point. Use the same LLVM types as the field to
2511
  // support optimization.
2512
3.13k
  llvm::Type *VTablePtrTy =
2513
3.13k
      llvm::FunctionType::get(CGM.Int32Ty, /*isVarArg=*/true)
2514
3.13k
          ->getPointerTo()
2515
3.13k
          ->getPointerTo();
2516
3.13k
  VTableField = Builder.CreateBitCast(VTableField, VTablePtrTy->getPointerTo());
2517
3.13k
  VTableAddressPoint = Builder.CreateBitCast(VTableAddressPoint, VTablePtrTy);
2518
2519
3.13k
  llvm::StoreInst *Store = Builder.CreateStore(VTableAddressPoint, VTableField);
2520
3.13k
  TBAAAccessInfo TBAAInfo = CGM.getTBAAVTablePtrAccessInfo(VTablePtrTy);
2521
3.13k
  CGM.DecorateInstructionWithTBAA(Store, TBAAInfo);
2522
3.13k
  if (CGM.getCodeGenOpts().OptimizationLevel > 0 &&
2523
401
      CGM.getCodeGenOpts().StrictVTablePointers)
2524
54
    CGM.DecorateInstructionWithInvariantGroup(Store, Vptr.VTableClass);
2525
3.13k
}
2526
2527
CodeGenFunction::VPtrsVector
2528
2.92k
CodeGenFunction::getVTablePointers(const CXXRecordDecl *VTableClass) {
2529
2.92k
  CodeGenFunction::VPtrsVector VPtrsResult;
2530
2.92k
  VisitedVirtualBasesSetTy VBases;
2531
2.92k
  getVTablePointers(BaseSubobject(VTableClass, CharUnits::Zero()),
2532
2.92k
                    /*NearestVBase=*/nullptr,
2533
2.92k
                    /*OffsetFromNearestVBase=*/CharUnits::Zero(),
2534
2.92k
                    /*BaseIsNonVirtualPrimaryBase=*/false, VTableClass, VBases,
2535
2.92k
                    VPtrsResult);
2536
2.92k
  return VPtrsResult;
2537
2.92k
}
2538
2539
void CodeGenFunction::getVTablePointers(BaseSubobject Base,
2540
                                        const CXXRecordDecl *NearestVBase,
2541
                                        CharUnits OffsetFromNearestVBase,
2542
                                        bool BaseIsNonVirtualPrimaryBase,
2543
                                        const CXXRecordDecl *VTableClass,
2544
                                        VisitedVirtualBasesSetTy &VBases,
2545
5.56k
                                        VPtrsVector &Vptrs) {
2546
  // If this base is a non-virtual primary base the address point has already
2547
  // been set.
2548
5.56k
  if (!BaseIsNonVirtualPrimaryBase) {
2549
    // Initialize the vtable pointer for this base.
2550
3.91k
    VPtr Vptr = {Base, NearestVBase, OffsetFromNearestVBase, VTableClass};
2551
3.91k
    Vptrs.push_back(Vptr);
2552
3.91k
  }
2553
2554
5.56k
  const CXXRecordDecl *RD = Base.getBase();
2555
2556
  // Traverse bases.
2557
3.41k
  for (const auto &I : RD->bases()) {
2558
3.41k
    auto *BaseDecl =
2559
3.41k
        cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
2560
2561
    // Ignore classes without a vtable.
2562
3.41k
    if (!BaseDecl->isDynamicClass())
2563
693
      continue;
2564
2565
2.71k
    CharUnits BaseOffset;
2566
2.71k
    CharUnits BaseOffsetFromNearestVBase;
2567
2.71k
    bool BaseDeclIsNonVirtualPrimaryBase;
2568
2569
2.71k
    if (I.isVirtual()) {
2570
      // Check if we've visited this virtual base before.
2571
664
      if (!VBases.insert(BaseDecl).second)
2572
71
        continue;
2573
2574
593
      const ASTRecordLayout &Layout =
2575
593
        getContext().getASTRecordLayout(VTableClass);
2576
2577
593
      BaseOffset = Layout.getVBaseClassOffset(BaseDecl);
2578
593
      BaseOffsetFromNearestVBase = CharUnits::Zero();
2579
593
      BaseDeclIsNonVirtualPrimaryBase = false;
2580
2.05k
    } else {
2581
2.05k
      const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
2582
2583
2.05k
      BaseOffset = Base.getBaseOffset() + Layout.getBaseClassOffset(BaseDecl);
2584
2.05k
      BaseOffsetFromNearestVBase =
2585
2.05k
        OffsetFromNearestVBase + Layout.getBaseClassOffset(BaseDecl);
2586
2.05k
      BaseDeclIsNonVirtualPrimaryBase = Layout.getPrimaryBase() == BaseDecl;
2587
2.05k
    }
2588
2589
2.64k
    getVTablePointers(
2590
2.64k
        BaseSubobject(BaseDecl, BaseOffset),
2591
2.05k
        I.isVirtual() ? 
BaseDecl593
: NearestVBase, BaseOffsetFromNearestVBase,
2592
2.64k
        BaseDeclIsNonVirtualPrimaryBase, VTableClass, VBases, Vptrs);
2593
2.64k
  }
2594
5.56k
}
2595
2596
22.5k
void CodeGenFunction::InitializeVTablePointers(const CXXRecordDecl *RD) {
2597
  // Ignore classes without a vtable.
2598
22.5k
  if (!RD->isDynamicClass())
2599
19.7k
    return;
2600
2601
  // Initialize the vtable pointers for this class and all of its bases.
2602
2.86k
  if (CGM.getCXXABI().doStructorsInitializeVPtrs(RD))
2603
2.86k
    for (const VPtr &Vptr : getVTablePointers(RD))
2604
3.85k
      InitializeVTablePointer(Vptr);
2605
2606
2.86k
  if (RD->getNumVBases())
2607
804
    CGM.getCXXABI().initializeHiddenVirtualInheritanceMembers(*this, RD);
2608
2.86k
}
2609
2610
llvm::Value *CodeGenFunction::GetVTablePtr(Address This,
2611
                                           llvm::Type *VTableTy,
2612
1.68k
                                           const CXXRecordDecl *RD) {
2613
1.68k
  Address VTablePtrSrc = Builder.CreateElementBitCast(This, VTableTy);
2614
1.68k
  llvm::Instruction *VTable = Builder.CreateLoad(VTablePtrSrc, "vtable");
2615
1.68k
  TBAAAccessInfo TBAAInfo = CGM.getTBAAVTablePtrAccessInfo(VTableTy);
2616
1.68k
  CGM.DecorateInstructionWithTBAA(VTable, TBAAInfo);
2617
2618
1.68k
  if (CGM.getCodeGenOpts().OptimizationLevel > 0 &&
2619
197
      CGM.getCodeGenOpts().StrictVTablePointers)
2620
118
    CGM.DecorateInstructionWithInvariantGroup(VTable, RD);
2621
2622
1.68k
  return VTable;
2623
1.68k
}
2624
2625
// If a class has a single non-virtual base and does not introduce or override
2626
// virtual member functions or fields, it will have the same layout as its base.
2627
// This function returns the least derived such class.
2628
//
2629
// Casting an instance of a base class to such a derived class is technically
2630
// undefined behavior, but it is a relatively common hack for introducing member
2631
// functions on class instances with specific properties (e.g. llvm::Operator)
2632
// that works under most compilers and should not have security implications, so
2633
// we allow it by default. It can be disabled with -fsanitize=cfi-cast-strict.
2634
static const CXXRecordDecl *
2635
60
LeastDerivedClassWithSameLayout(const CXXRecordDecl *RD) {
2636
60
  if (!RD->field_empty())
2637
1
    return RD;
2638
2639
59
  if (RD->getNumVBases() != 0)
2640
12
    return RD;
2641
2642
47
  if (RD->getNumBases() != 1)
2643
32
    return RD;
2644
2645
28
  
for (const CXXMethodDecl *MD : RD->methods())15
{
2646
28
    if (MD->isVirtual()) {
2647
      // Virtual member functions are only ok if they are implicit destructors
2648
      // because the implicit destructor will have the same semantics as the
2649
      // base class's destructor if no fields are added.
2650
9
      if (isa<CXXDestructorDecl>(MD) && 
MD->isImplicit()0
)
2651
0
        continue;
2652
9
      return RD;
2653
9
    }
2654
28
  }
2655
2656
6
  return LeastDerivedClassWithSameLayout(
2657
6
      RD->bases_begin()->getType()->getAsCXXRecordDecl());
2658
15
}
2659
2660
void CodeGenFunction::EmitTypeMetadataCodeForVCall(const CXXRecordDecl *RD,
2661
                                                   llvm::Value *VTable,
2662
822
                                                   SourceLocation Loc) {
2663
822
  if (SanOpts.has(SanitizerKind::CFIVCall))
2664
33
    EmitVTablePtrCheckForCall(RD, VTable, CodeGenFunction::CFITCK_VCall, Loc);
2665
789
  else if (CGM.getCodeGenOpts().WholeProgramVTables &&
2666
           // Don't insert type test assumes if we are forcing public std
2667
           // visibility.
2668
62
           !CGM.HasLTOVisibilityPublicStd(RD)) {
2669
59
    llvm::Metadata *MD =
2670
59
        CGM.CreateMetadataIdentifierForType(QualType(RD->getTypeForDecl(), 0));
2671
59
    llvm::Value *TypeId =
2672
59
        llvm::MetadataAsValue::get(CGM.getLLVMContext(), MD);
2673
2674
59
    llvm::Value *CastedVTable = Builder.CreateBitCast(VTable, Int8PtrTy);
2675
59
    llvm::Value *TypeTest =
2676
59
        Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::type_test),
2677
59
                           {CastedVTable, TypeId});
2678
59
    Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::assume), TypeTest);
2679
59
  }
2680
822
}
2681
2682
void CodeGenFunction::EmitVTablePtrCheckForCall(const CXXRecordDecl *RD,
2683
                                                llvm::Value *VTable,
2684
                                                CFITypeCheckKind TCK,
2685
41
                                                SourceLocation Loc) {
2686
41
  if (!SanOpts.has(SanitizerKind::CFICastStrict))
2687
39
    RD = LeastDerivedClassWithSameLayout(RD);
2688
2689
41
  EmitVTablePtrCheck(RD, VTable, TCK, Loc);
2690
41
}
2691
2692
void CodeGenFunction::EmitVTablePtrCheckForCast(QualType T,
2693
                                                llvm::Value *Derived,
2694
                                                bool MayBeNull,
2695
                                                CFITypeCheckKind TCK,
2696
21
                                                SourceLocation Loc) {
2697
21
  if (!getLangOpts().CPlusPlus)
2698
0
    return;
2699
2700
21
  auto *ClassTy = T->getAs<RecordType>();
2701
21
  if (!ClassTy)
2702
0
    return;
2703
2704
21
  const CXXRecordDecl *ClassDecl = cast<CXXRecordDecl>(ClassTy->getDecl());
2705
2706
21
  if (!ClassDecl->isCompleteDefinition() || !ClassDecl->isDynamicClass())
2707
0
    return;
2708
2709
21
  if (!SanOpts.has(SanitizerKind::CFICastStrict))
2710
15
    ClassDecl = LeastDerivedClassWithSameLayout(ClassDecl);
2711
2712
21
  llvm::BasicBlock *ContBlock = nullptr;
2713
2714
21
  if (MayBeNull) {
2715
14
    llvm::Value *DerivedNotNull =
2716
14
        Builder.CreateIsNotNull(Derived, "cast.nonnull");
2717
2718
14
    llvm::BasicBlock *CheckBlock = createBasicBlock("cast.check");
2719
14
    ContBlock = createBasicBlock("cast.cont");
2720
2721
14
    Builder.CreateCondBr(DerivedNotNull, CheckBlock, ContBlock);
2722
2723
14
    EmitBlock(CheckBlock);
2724
14
  }
2725
2726
21
  llvm::Value *VTable;
2727
21
  std::tie(VTable, ClassDecl) = CGM.getCXXABI().LoadVTablePtr(
2728
21
      *this, Address(Derived, getPointerAlign()), ClassDecl);
2729
2730
21
  EmitVTablePtrCheck(ClassDecl, VTable, TCK, Loc);
2731
2732
21
  if (MayBeNull) {
2733
14
    Builder.CreateBr(ContBlock);
2734
14
    EmitBlock(ContBlock);
2735
14
  }
2736
21
}
2737
2738
void CodeGenFunction::EmitVTablePtrCheck(const CXXRecordDecl *RD,
2739
                                         llvm::Value *VTable,
2740
                                         CFITypeCheckKind TCK,
2741
62
                                         SourceLocation Loc) {
2742
62
  if (!CGM.getCodeGenOpts().SanitizeCfiCrossDso &&
2743
60
      !CGM.HasHiddenLTOVisibility(RD))
2744
0
    return;
2745
2746
62
  SanitizerMask M;
2747
62
  llvm::SanitizerStatKind SSK;
2748
62
  switch (TCK) {
2749
33
  case CFITCK_VCall:
2750
33
    M = SanitizerKind::CFIVCall;
2751
33
    SSK = llvm::SanStat_CFI_VCall;
2752
33
    break;
2753
8
  case CFITCK_NVCall:
2754
8
    M = SanitizerKind::CFINVCall;
2755
8
    SSK = llvm::SanStat_CFI_NVCall;
2756
8
    break;
2757
6
  case CFITCK_DerivedCast:
2758
6
    M = SanitizerKind::CFIDerivedCast;
2759
6
    SSK = llvm::SanStat_CFI_DerivedCast;
2760
6
    break;
2761
15
  case CFITCK_UnrelatedCast:
2762
15
    M = SanitizerKind::CFIUnrelatedCast;
2763
15
    SSK = llvm::SanStat_CFI_UnrelatedCast;
2764
15
    break;
2765
0
  case CFITCK_ICall:
2766
0
  case CFITCK_NVMFCall:
2767
0
  case CFITCK_VMFCall:
2768
0
    llvm_unreachable("unexpected sanitizer kind");
2769
62
  }
2770
2771
62
  std::string TypeName = RD->getQualifiedNameAsString();
2772
62
  if (getContext().getSanitizerBlacklist().isBlacklistedType(M, TypeName))
2773
2
    return;
2774
2775
60
  SanitizerScope SanScope(this);
2776
60
  EmitSanitizerStatReport(SSK);
2777
2778
60
  llvm::Metadata *MD =
2779
60
      CGM.CreateMetadataIdentifierForType(QualType(RD->getTypeForDecl(), 0));
2780
60
  llvm::Value *TypeId = llvm::MetadataAsValue::get(getLLVMContext(), MD);
2781
2782
60
  llvm::Value *CastedVTable = Builder.CreateBitCast(VTable, Int8PtrTy);
2783
60
  llvm::Value *TypeTest = Builder.CreateCall(
2784
60
      CGM.getIntrinsic(llvm::Intrinsic::type_test), {CastedVTable, TypeId});
2785
2786
60
  llvm::Constant *StaticData[] = {
2787
60
      llvm::ConstantInt::get(Int8Ty, TCK),
2788
60
      EmitCheckSourceLocation(Loc),
2789
60
      EmitCheckTypeDescriptor(QualType(RD->getTypeForDecl(), 0)),
2790
60
  };
2791
2792
60
  auto CrossDsoTypeId = CGM.CreateCrossDsoCfiTypeId(MD);
2793
60
  if (CGM.getCodeGenOpts().SanitizeCfiCrossDso && 
CrossDsoTypeId2
) {
2794
2
    EmitCfiSlowPathCheck(M, TypeTest, CrossDsoTypeId, CastedVTable, StaticData);
2795
2
    return;
2796
2
  }
2797
2798
58
  if (CGM.getCodeGenOpts().SanitizeTrap.has(M)) {
2799
28
    EmitTrapCheck(TypeTest);
2800
28
    return;
2801
28
  }
2802
2803
30
  llvm::Value *AllVtables = llvm::MetadataAsValue::get(
2804
30
      CGM.getLLVMContext(),
2805
30
      llvm::MDString::get(CGM.getLLVMContext(), "all-vtables"));
2806
30
  llvm::Value *ValidVtable = Builder.CreateCall(
2807
30
      CGM.getIntrinsic(llvm::Intrinsic::type_test), {CastedVTable, AllVtables});
2808
30
  EmitCheck(std::make_pair(TypeTest, M), SanitizerHandler::CFICheckFail,
2809
30
            StaticData, {CastedVTable, ValidVtable});
2810
30
}
2811
2812
971
bool CodeGenFunction::ShouldEmitVTableTypeCheckedLoad(const CXXRecordDecl *RD) {
2813
971
  if (!CGM.getCodeGenOpts().WholeProgramVTables ||
2814
76
      !CGM.HasHiddenLTOVisibility(RD))
2815
915
    return false;
2816
2817
56
  if (CGM.getCodeGenOpts().VirtualFunctionElimination)
2818
2
    return true;
2819
2820
54
  if (!SanOpts.has(SanitizerKind::CFIVCall) ||
2821
12
      !CGM.getCodeGenOpts().SanitizeTrap.has(SanitizerKind::CFIVCall))
2822
43
    return false;
2823
2824
11
  std::string TypeName = RD->getQualifiedNameAsString();
2825
11
  return !getContext().getSanitizerBlacklist().isBlacklistedType(
2826
11
      SanitizerKind::CFIVCall, TypeName);
2827
11
}
2828
2829
llvm::Value *CodeGenFunction::EmitVTableTypeCheckedLoad(
2830
13
    const CXXRecordDecl *RD, llvm::Value *VTable, uint64_t VTableByteOffset) {
2831
13
  SanitizerScope SanScope(this);
2832
2833
13
  EmitSanitizerStatReport(llvm::SanStat_CFI_VCall);
2834
2835
13
  llvm::Metadata *MD =
2836
13
      CGM.CreateMetadataIdentifierForType(QualType(RD->getTypeForDecl(), 0));
2837
13
  llvm::Value *TypeId = llvm::MetadataAsValue::get(CGM.getLLVMContext(), MD);
2838
2839
13
  llvm::Value *CastedVTable = Builder.CreateBitCast(VTable, Int8PtrTy);
2840
13
  llvm::Value *CheckedLoad = Builder.CreateCall(
2841
13
      CGM.getIntrinsic(llvm::Intrinsic::type_checked_load),
2842
13
      {CastedVTable, llvm::ConstantInt::get(Int32Ty, VTableByteOffset),
2843
13
       TypeId});
2844
13
  llvm::Value *CheckResult = Builder.CreateExtractValue(CheckedLoad, 1);
2845
2846
13
  std::string TypeName = RD->getQualifiedNameAsString();
2847
13
  if (SanOpts.has(SanitizerKind::CFIVCall) &&
2848
11
      !getContext().getSanitizerBlacklist().isBlacklistedType(
2849
11
          SanitizerKind::CFIVCall, TypeName)) {
2850
11
    EmitCheck(std::make_pair(CheckResult, SanitizerKind::CFIVCall),
2851
11
              SanitizerHandler::CFICheckFail, {}, {});
2852
11
  }
2853
2854
13
  return Builder.CreateBitCast(
2855
13
      Builder.CreateExtractValue(CheckedLoad, 0),
2856
13
      cast<llvm::PointerType>(VTable->getType())->getElementType());
2857
13
}
2858
2859
void CodeGenFunction::EmitForwardingCallToLambda(
2860
                                      const CXXMethodDecl *callOperator,
2861
62
                                      CallArgList &callArgs) {
2862
  // Get the address of the call operator.
2863
62
  const CGFunctionInfo &calleeFnInfo =
2864
62
    CGM.getTypes().arrangeCXXMethodDeclaration(callOperator);
2865
62
  llvm::Constant *calleePtr =
2866
62
    CGM.GetAddrOfFunction(GlobalDecl(callOperator),
2867
62
                          CGM.getTypes().GetFunctionType(calleeFnInfo));
2868
2869
  // Prepare the return slot.
2870
62
  const FunctionProtoType *FPT =
2871
62
    callOperator->getType()->castAs<FunctionProtoType>();
2872
62
  QualType resultType = FPT->getReturnType();
2873
62
  ReturnValueSlot returnSlot;
2874
62
  if (!resultType->isVoidType() &&
2875
26
      calleeFnInfo.getReturnInfo().getKind() == ABIArgInfo::Indirect &&
2876
1
      !hasScalarEvaluationKind(calleeFnInfo.getReturnType()))
2877
1
    returnSlot =
2878
1
        ReturnValueSlot(ReturnValue, resultType.isVolatileQualified(),
2879
1
                        /*IsUnused=*/false, /*IsExternallyDestructed=*/true);
2880
2881
  // We don't need to separately arrange the call arguments because
2882
  // the call can't be variadic anyway --- it's impossible to forward
2883
  // variadic arguments.
2884
2885
  // Now emit our call.
2886
62
  auto callee = CGCallee::forDirect(calleePtr, GlobalDecl(callOperator));
2887
62
  RValue RV = EmitCall(calleeFnInfo, callee, returnSlot, callArgs);
2888
2889
  // If necessary, copy the returned value into the slot.
2890
62
  if (!resultType->isVoidType() && 
returnSlot.isNull()26
) {
2891
25
    if (getLangOpts().ObjCAutoRefCount && 
resultType->isObjCRetainableType()5
) {
2892
2
      RV = RValue::get(EmitARCRetainAutoreleasedReturnValue(RV.getScalarVal()));
2893
2
    }
2894
25
    EmitReturnOfRValue(RV, resultType);
2895
25
  } else
2896
37
    EmitBranchThroughCleanup(ReturnBlock);
2897
62
}
2898
2899
13
void CodeGenFunction::EmitLambdaBlockInvokeBody() {
2900
13
  const BlockDecl *BD = BlockInfo->getBlockDecl();
2901
13
  const VarDecl *variable = BD->capture_begin()->getVariable();
2902
13
  const CXXRecordDecl *Lambda = variable->getType()->getAsCXXRecordDecl();
2903
13
  const CXXMethodDecl *CallOp = Lambda->getLambdaCallOperator();
2904
2905
13
  if (CallOp->isVariadic()) {
2906
    // FIXME: Making this work correctly is nasty because it requires either
2907
    // cloning the body of the call operator or making the call operator
2908
    // forward.
2909
0
    CGM.ErrorUnsupported(CurCodeDecl, "lambda conversion to variadic function");
2910
0
    return;
2911
0
  }
2912
2913
  // Start building arguments for forwarding call
2914
13
  CallArgList CallArgs;
2915
2916
13
  QualType ThisType = getContext().getPointerType(getContext().getRecordType(Lambda));
2917
13
  Address ThisPtr = GetAddrOfBlockDecl(variable);
2918
13
  CallArgs.add(RValue::get(ThisPtr.getPointer()), ThisType);
2919
2920
  // Add the rest of the parameters.
2921
13
  for (auto param : BD->parameters())
2922
1
    EmitDelegateCallArg(CallArgs, param, param->getBeginLoc());
2923
2924
13
  assert(!Lambda->isGenericLambda() &&
2925
13
            "generic lambda interconversion to block not implemented");
2926
13
  EmitForwardingCallToLambda(CallOp, CallArgs);
2927
13
}
2928
2929
49
void CodeGenFunction::EmitLambdaDelegatingInvokeBody(const CXXMethodDecl *MD) {
2930
49
  const CXXRecordDecl *Lambda = MD->getParent();
2931
2932
  // Start building arguments for forwarding call
2933
49
  CallArgList CallArgs;
2934
2935
49
  QualType ThisType = getContext().getPointerType(getContext().getRecordType(Lambda));
2936
49
  llvm::Value *ThisPtr = llvm::UndefValue::get(getTypes().ConvertType(ThisType));
2937
49
  CallArgs.add(RValue::get(ThisPtr), ThisType);
2938
2939
  // Add the rest of the parameters.
2940
49
  for (auto Param : MD->parameters())
2941
119
    EmitDelegateCallArg(CallArgs, Param, Param->getBeginLoc());
2942
2943
49
  const CXXMethodDecl *CallOp = Lambda->getLambdaCallOperator();
2944
  // For a generic lambda, find the corresponding call operator specialization
2945
  // to which the call to the static-invoker shall be forwarded.
2946
49
  if (Lambda->isGenericLambda()) {
2947
0
    assert(MD->isFunctionTemplateSpecialization());
2948
0
    const TemplateArgumentList *TAL = MD->getTemplateSpecializationArgs();
2949
0
    FunctionTemplateDecl *CallOpTemplate = CallOp->getDescribedFunctionTemplate();
2950
0
    void *InsertPos = nullptr;
2951
0
    FunctionDecl *CorrespondingCallOpSpecialization =
2952
0
        CallOpTemplate->findSpecialization(TAL->asArray(), InsertPos);
2953
0
    assert(CorrespondingCallOpSpecialization);
2954
0
    CallOp = cast<CXXMethodDecl>(CorrespondingCallOpSpecialization);
2955
0
  }
2956
49
  EmitForwardingCallToLambda(CallOp, CallArgs);
2957
49
}
2958
2959
49
void CodeGenFunction::EmitLambdaStaticInvokeBody(const CXXMethodDecl *MD) {
2960
49
  if (MD->isVariadic()) {
2961
    // FIXME: Making this work correctly is nasty because it requires either
2962
    // cloning the body of the call operator or making the call operator forward.
2963
0
    CGM.ErrorUnsupported(MD, "lambda conversion to variadic function");
2964
0
    return;
2965
0
  }
2966
2967
49
  EmitLambdaDelegatingInvokeBody(MD);
2968
49
}