Coverage Report

Created: 2020-03-28 08:29

/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/CodeGen/CodeGenFunction.cpp
Line
Count
Source (jump to first uncovered line)
1
//===--- CodeGenFunction.cpp - Emit LLVM Code from ASTs for a Function ----===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This coordinates the per-function state used while generating code.
10
//
11
//===----------------------------------------------------------------------===//
12
13
#include "CodeGenFunction.h"
14
#include "CGBlocks.h"
15
#include "CGCUDARuntime.h"
16
#include "CGCXXABI.h"
17
#include "CGCleanup.h"
18
#include "CGDebugInfo.h"
19
#include "CGOpenMPRuntime.h"
20
#include "CodeGenModule.h"
21
#include "CodeGenPGO.h"
22
#include "TargetInfo.h"
23
#include "clang/AST/ASTContext.h"
24
#include "clang/AST/ASTLambda.h"
25
#include "clang/AST/Attr.h"
26
#include "clang/AST/Decl.h"
27
#include "clang/AST/DeclCXX.h"
28
#include "clang/AST/StmtCXX.h"
29
#include "clang/AST/StmtObjC.h"
30
#include "clang/Basic/Builtins.h"
31
#include "clang/Basic/CodeGenOptions.h"
32
#include "clang/Basic/TargetInfo.h"
33
#include "clang/CodeGen/CGFunctionInfo.h"
34
#include "clang/Frontend/FrontendDiagnostic.h"
35
#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
36
#include "llvm/IR/DataLayout.h"
37
#include "llvm/IR/Dominators.h"
38
#include "llvm/IR/FPEnv.h"
39
#include "llvm/IR/IntrinsicInst.h"
40
#include "llvm/IR/Intrinsics.h"
41
#include "llvm/IR/MDBuilder.h"
42
#include "llvm/IR/Operator.h"
43
#include "llvm/Transforms/Utils/PromoteMemToReg.h"
44
using namespace clang;
45
using namespace CodeGen;
46
47
/// shouldEmitLifetimeMarkers - Decide whether we need emit the life-time
48
/// markers.
49
static bool shouldEmitLifetimeMarkers(const CodeGenOptions &CGOpts,
50
246k
                                      const LangOptions &LangOpts) {
51
246k
  if (CGOpts.DisableLifetimeMarkers)
52
7
    return false;
53
246k
54
246k
  // Sanitizers may use markers.
55
246k
  if (CGOpts.SanitizeAddressUseAfterScope ||
56
246k
      
LangOpts.Sanitize.has(SanitizerKind::HWAddress)245k
||
57
246k
      
LangOpts.Sanitize.has(SanitizerKind::Memory)245k
)
58
1.03k
    return true;
59
245k
60
245k
  // For now, only in optimized builds.
61
245k
  return CGOpts.OptimizationLevel != 0;
62
245k
}
63
64
CodeGenFunction::CodeGenFunction(CodeGenModule &cgm, bool suppressNewContext)
65
    : CodeGenTypeCache(cgm), CGM(cgm), Target(cgm.getTarget()),
66
      Builder(cgm, cgm.getModule().getContext(), llvm::ConstantFolder(),
67
              CGBuilderInserterTy(this)),
68
      SanOpts(CGM.getLangOpts().Sanitize), DebugInfo(CGM.getModuleDebugInfo()),
69
      PGO(cgm), ShouldEmitLifetimeMarkers(shouldEmitLifetimeMarkers(
70
246k
                    CGM.getCodeGenOpts(), CGM.getLangOpts())) {
71
246k
  if (!suppressNewContext)
72
229k
    CGM.getCXXABI().getMangleContext().startNewFunction();
73
246k
74
246k
  llvm::FastMathFlags FMF;
75
246k
  if (CGM.getLangOpts().FastMath)
76
87
    FMF.setFast();
77
246k
  if (CGM.getLangOpts().FiniteMathOnly) {
78
3
    FMF.setNoNaNs();
79
3
    FMF.setNoInfs();
80
3
  }
81
246k
  if (CGM.getCodeGenOpts().NoNaNsFPMath) {
82
19
    FMF.setNoNaNs();
83
19
  }
84
246k
  if (CGM.getCodeGenOpts().NoSignedZeros) {
85
5
    FMF.setNoSignedZeros();
86
5
  }
87
246k
  if (CGM.getCodeGenOpts().ReciprocalMath) {
88
1
    FMF.setAllowReciprocal();
89
1
  }
90
246k
  if (CGM.getCodeGenOpts().Reassociate) {
91
1
    FMF.setAllowReassoc();
92
1
  }
93
246k
  Builder.setFastMathFlags(FMF);
94
246k
  SetFPModel();
95
246k
}
96
97
246k
CodeGenFunction::~CodeGenFunction() {
98
246k
  assert(LifetimeExtendedCleanupStack.empty() && "failed to emit a cleanup");
99
246k
100
246k
  // If there are any unclaimed block infos, go ahead and destroy them
101
246k
  // now.  This can happen if IR-gen gets clever and skips evaluating
102
246k
  // something.
103
246k
  if (FirstBlockInfo)
104
0
    destroyBlockInfos(FirstBlockInfo);
105
246k
106
246k
  if (getLangOpts().OpenMP && 
CurFn47.8k
)
107
47.8k
    CGM.getOpenMPRuntime().functionFinished(*this);
108
246k
109
246k
  // If we have an OpenMPIRBuilder we want to finalize functions (incl.
110
246k
  // outlining etc) at some point. Doing it once the function codegen is done
111
246k
  // seems to be a reasonable spot. We do it here, as opposed to the deletion
112
246k
  // time of the CodeGenModule, because we have to ensure the IR has not yet
113
246k
  // been "emitted" to the outside, thus, modifications are still sensible.
114
246k
  if (llvm::OpenMPIRBuilder *OMPBuilder = CGM.getOpenMPIRBuilder())
115
138
    OMPBuilder->finalize();
116
246k
}
117
118
// Map the LangOption for rounding mode into
119
// the corresponding enum in the IR.
120
static llvm::fp::RoundingMode ToConstrainedRoundingMD(
121
246k
  LangOptions::FPRoundingModeKind Kind) {
122
246k
123
246k
  switch (Kind) {
124
246k
  case LangOptions::FPR_ToNearest:  return llvm::fp::rmToNearest;
125
0
  case LangOptions::FPR_Downward:   return llvm::fp::rmDownward;
126
0
  case LangOptions::FPR_Upward:     return llvm::fp::rmUpward;
127
0
  case LangOptions::FPR_TowardZero: return llvm::fp::rmTowardZero;
128
78
  case LangOptions::FPR_Dynamic:    return llvm::fp::rmDynamic;
129
0
  }
130
0
  llvm_unreachable("Unsupported FP RoundingMode");
131
0
}
132
133
// Map the LangOption for exception behavior into
134
// the corresponding enum in the IR.
135
static llvm::fp::ExceptionBehavior ToConstrainedExceptMD(
136
246k
  LangOptions::FPExceptionModeKind Kind) {
137
246k
138
246k
  switch (Kind) {
139
244k
  case LangOptions::FPE_Ignore:  return llvm::fp::ebIgnore;
140
54
  case LangOptions::FPE_MayTrap: return llvm::fp::ebMayTrap;
141
1.38k
  case LangOptions::FPE_Strict:  return llvm::fp::ebStrict;
142
0
  }
143
0
  llvm_unreachable("Unsupported FP Exception Behavior");
144
0
}
145
146
246k
void CodeGenFunction::SetFPModel() {
147
246k
  auto fpRoundingMode = ToConstrainedRoundingMD(
148
246k
                          getLangOpts().getFPRoundingMode());
149
246k
  auto fpExceptionBehavior = ToConstrainedExceptMD(
150
246k
                               getLangOpts().getFPExceptionMode());
151
246k
152
246k
  if (fpExceptionBehavior == llvm::fp::ebIgnore &&
153
246k
      
fpRoundingMode == llvm::fp::rmToNearest244k
)
154
244k
    // Constrained intrinsics are not used.
155
244k
    ;
156
1.46k
  else {
157
1.46k
    Builder.setIsFPConstrained(true);
158
1.46k
    Builder.setDefaultConstrainedRounding(fpRoundingMode);
159
1.46k
    Builder.setDefaultConstrainedExcept(fpExceptionBehavior);
160
1.46k
  }
161
246k
}
162
163
CharUnits CodeGenFunction::getNaturalPointeeTypeAlignment(QualType T,
164
                                                    LValueBaseInfo *BaseInfo,
165
167k
                                                    TBAAAccessInfo *TBAAInfo) {
166
167k
  return getNaturalTypeAlignment(T->getPointeeType(), BaseInfo, TBAAInfo,
167
167k
                                 /* forPointeeType= */ true);
168
167k
}
169
170
CharUnits CodeGenFunction::getNaturalTypeAlignment(QualType T,
171
                                                   LValueBaseInfo *BaseInfo,
172
                                                   TBAAAccessInfo *TBAAInfo,
173
346k
                                                   bool forPointeeType) {
174
346k
  if (TBAAInfo)
175
317k
    *TBAAInfo = CGM.getTBAAAccessInfo(T);
176
346k
177
346k
  // Honor alignment typedef attributes even on incomplete types.
178
346k
  // We also honor them straight for C++ class types, even as pointees;
179
346k
  // there's an expressivity gap here.
180
346k
  if (auto TT = T->getAs<TypedefType>()) {
181
34.6k
    if (auto Align = TT->getDecl()->getMaxAlignment()) {
182
343
      if (BaseInfo)
183
343
        *BaseInfo = LValueBaseInfo(AlignmentSource::AttributedType);
184
343
      return getContext().toCharUnitsFromBits(Align);
185
343
    }
186
345k
  }
187
345k
188
345k
  if (BaseInfo)
189
327k
    *BaseInfo = LValueBaseInfo(AlignmentSource::Type);
190
345k
191
345k
  CharUnits Alignment;
192
345k
  if (T->isIncompleteType()) {
193
960
    Alignment = CharUnits::One(); // Shouldn't be used, but pessimistic is best.
194
345k
  } else {
195
345k
    // For C++ class pointees, we don't know whether we're pointing at a
196
345k
    // base or a complete object, so we generally need to use the
197
345k
    // non-virtual alignment.
198
345k
    const CXXRecordDecl *RD;
199
345k
    if (forPointeeType && 
(RD = T->getAsCXXRecordDecl())314k
) {
200
205k
      Alignment = CGM.getClassPointerAlignment(RD);
201
205k
    } else {
202
139k
      Alignment = getContext().getTypeAlignInChars(T);
203
139k
      if (T.getQualifiers().hasUnaligned())
204
5
        Alignment = CharUnits::One();
205
139k
    }
206
345k
207
345k
    // Cap to the global maximum type alignment unless the alignment
208
345k
    // was somehow explicit on the type.
209
345k
    if (unsigned MaxAlign = getLangOpts().MaxTypeAlign) {
210
159k
      if (Alignment.getQuantity() > MaxAlign &&
211
159k
          
!getContext().isAlignmentRequired(T)6
)
212
3
        Alignment = CharUnits::fromQuantity(MaxAlign);
213
159k
    }
214
345k
  }
215
345k
  return Alignment;
216
345k
}
217
218
29.9k
LValue CodeGenFunction::MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T) {
219
29.9k
  LValueBaseInfo BaseInfo;
220
29.9k
  TBAAAccessInfo TBAAInfo;
221
29.9k
  CharUnits Alignment = getNaturalTypeAlignment(T, &BaseInfo, &TBAAInfo);
222
29.9k
  return LValue::MakeAddr(Address(V, Alignment), T, getContext(), BaseInfo,
223
29.9k
                          TBAAInfo);
224
29.9k
}
225
226
/// Given a value of type T* that may not be to a complete object,
227
/// construct an l-value with the natural pointee alignment of T.
228
LValue
229
58.8k
CodeGenFunction::MakeNaturalAlignPointeeAddrLValue(llvm::Value *V, QualType T) {
230
58.8k
  LValueBaseInfo BaseInfo;
231
58.8k
  TBAAAccessInfo TBAAInfo;
232
58.8k
  CharUnits Align = getNaturalTypeAlignment(T, &BaseInfo, &TBAAInfo,
233
58.8k
                                            /* forPointeeType= */ true);
234
58.8k
  return MakeAddrLValue(Address(V, Align), T, BaseInfo, TBAAInfo);
235
58.8k
}
236
237
238
738k
llvm::Type *CodeGenFunction::ConvertTypeForMem(QualType T) {
239
738k
  return CGM.getTypes().ConvertTypeForMem(T);
240
738k
}
241
242
2.15M
llvm::Type *CodeGenFunction::ConvertType(QualType T) {
243
2.15M
  return CGM.getTypes().ConvertType(T);
244
2.15M
}
245
246
4.49M
TypeEvaluationKind CodeGenFunction::getEvaluationKind(QualType type) {
247
4.49M
  type = type.getCanonicalType();
248
4.49M
  while (true) {
249
4.49M
    switch (type->getTypeClass()) {
250
0
#define TYPE(name, parent)
251
0
#define ABSTRACT_TYPE(name, parent)
252
0
#define NON_CANONICAL_TYPE(name, parent) case Type::name:
253
0
#define DEPENDENT_TYPE(name, parent) case Type::name:
254
0
#define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(name, parent) case Type::name:
255
0
#include "clang/AST/TypeNodes.inc"
256
0
      llvm_unreachable("non-canonical or dependent type in IR-generation");
257
0
258
0
    case Type::Auto:
259
0
    case Type::DeducedTemplateSpecialization:
260
0
      llvm_unreachable("undeduced type in IR-generation");
261
0
262
0
    // Various scalar types.
263
4.25M
    case Type::Builtin:
264
4.25M
    case Type::Pointer:
265
4.25M
    case Type::BlockPointer:
266
4.25M
    case Type::LValueReference:
267
4.25M
    case Type::RValueReference:
268
4.25M
    case Type::MemberPointer:
269
4.25M
    case Type::Vector:
270
4.25M
    case Type::ExtVector:
271
4.25M
    case Type::FunctionProto:
272
4.25M
    case Type::FunctionNoProto:
273
4.25M
    case Type::Enum:
274
4.25M
    case Type::ObjCObjectPointer:
275
4.25M
    case Type::Pipe:
276
4.25M
      return TEK_Scalar;
277
4.25M
278
4.25M
    // Complexes.
279
4.25M
    case Type::Complex:
280
6.95k
      return TEK_Complex;
281
4.25M
282
4.25M
    // Arrays, records, and Objective-C objects.
283
4.25M
    case Type::ConstantArray:
284
233k
    case Type::IncompleteArray:
285
233k
    case Type::VariableArray:
286
233k
    case Type::Record:
287
233k
    case Type::ObjCObject:
288
233k
    case Type::ObjCInterface:
289
233k
      return TEK_Aggregate;
290
233k
291
233k
    // We operate on atomic values according to their underlying type.
292
233k
    case Type::Atomic:
293
280
      type = cast<AtomicType>(type)->getValueType();
294
280
      continue;
295
0
    }
296
0
    llvm_unreachable("unknown type kind!");
297
0
  }
298
4.49M
}
299
300
245k
llvm::DebugLoc CodeGenFunction::EmitReturnBlock() {
301
245k
  // For cleanliness, we try to avoid emitting the return block for
302
245k
  // simple cases.
303
245k
  llvm::BasicBlock *CurBB = Builder.GetInsertBlock();
304
245k
305
245k
  if (CurBB) {
306
127k
    assert(!CurBB->getTerminator() && "Unexpected terminated block.");
307
127k
308
127k
    // We have a valid insert point, reuse it if it is empty or there are no
309
127k
    // explicit jumps to the return block.
310
127k
    if (CurBB->empty() || 
ReturnBlock.getBlock()->use_empty()105k
) {
311
127k
      ReturnBlock.getBlock()->replaceAllUsesWith(CurBB);
312
127k
      delete ReturnBlock.getBlock();
313
127k
      ReturnBlock = JumpDest();
314
127k
    } else
315
22
      EmitBlock(ReturnBlock.getBlock());
316
127k
    return llvm::DebugLoc();
317
127k
  }
318
117k
319
117k
  // Otherwise, if the return block is the target of a single direct
320
117k
  // branch then we can just put the code in that block instead. This
321
117k
  // cleans up functions which started with a unified return block.
322
117k
  if (ReturnBlock.getBlock()->hasOneUse()) {
323
114k
    llvm::BranchInst *BI =
324
114k
      dyn_cast<llvm::BranchInst>(*ReturnBlock.getBlock()->user_begin());
325
114k
    if (BI && 
BI->isUnconditional()114k
&&
326
114k
        
BI->getSuccessor(0) == ReturnBlock.getBlock()114k
) {
327
114k
      // Record/return the DebugLoc of the simple 'return' expression to be used
328
114k
      // later by the actual 'ret' instruction.
329
114k
      llvm::DebugLoc Loc = BI->getDebugLoc();
330
114k
      Builder.SetInsertPoint(BI->getParent());
331
114k
      BI->eraseFromParent();
332
114k
      delete ReturnBlock.getBlock();
333
114k
      ReturnBlock = JumpDest();
334
114k
      return Loc;
335
114k
    }
336
2.44k
  }
337
2.44k
338
2.44k
  // FIXME: We are at an unreachable point, there is no reason to emit the block
339
2.44k
  // unless it has uses. However, we still need a place to put the debug
340
2.44k
  // region.end for now.
341
2.44k
342
2.44k
  EmitBlock(ReturnBlock.getBlock());
343
2.44k
  return llvm::DebugLoc();
344
2.44k
}
345
346
980k
static void EmitIfUsed(CodeGenFunction &CGF, llvm::BasicBlock *BB) {
347
980k
  if (!BB) 
return972k
;
348
7.78k
  if (!BB->use_empty())
349
7.77k
    return CGF.CurFn->getBasicBlockList().push_back(BB);
350
8
  delete BB;
351
8
}
352
353
245k
void CodeGenFunction::FinishFunction(SourceLocation EndLoc) {
354
245k
  assert(BreakContinueStack.empty() &&
355
245k
         "mismatched push/pop in break/continue stack!");
356
245k
357
245k
  bool OnlySimpleReturnStmts = NumSimpleReturnExprs > 0
358
245k
    && 
NumSimpleReturnExprs == NumReturnExprs12.3k
359
245k
    && 
ReturnBlock.getBlock()->use_empty()11.1k
;
360
245k
  // Usually the return expression is evaluated before the cleanup
361
245k
  // code.  If the function contains only a simple return statement,
362
245k
  // such as a constant, the location before the cleanup code becomes
363
245k
  // the last useful breakpoint in the function, because the simple
364
245k
  // return expression will be evaluated after the cleanup code. To be
365
245k
  // safe, set the debug location for cleanup code to the location of
366
245k
  // the return statement.  Otherwise the cleanup code should be at the
367
245k
  // end of the function's lexical scope.
368
245k
  //
369
245k
  // If there are multiple branches to the return block, the branch
370
245k
  // instructions will get the location of the return statements and
371
245k
  // all will be fine.
372
245k
  if (CGDebugInfo *DI = getDebugInfo()) {
373
89.3k
    if (OnlySimpleReturnStmts)
374
580
      DI->EmitLocation(Builder, LastStopPoint);
375
88.7k
    else
376
88.7k
      DI->EmitLocation(Builder, EndLoc);
377
89.3k
  }
378
245k
379
245k
  // Pop any cleanups that might have been associated with the
380
245k
  // parameters.  Do this in whatever block we're currently in; it's
381
245k
  // important to do this before we enter the return block or return
382
245k
  // edges will be *really* confused.
383
245k
  bool HasCleanups = EHStack.stable_begin() != PrologueCleanupDepth;
384
245k
  bool HasOnlyLifetimeMarkers =
385
245k
      HasCleanups && 
EHStack.containsOnlyLifetimeMarkers(PrologueCleanupDepth)7.59k
;
386
245k
  bool EmitRetDbgLoc = !HasCleanups || 
HasOnlyLifetimeMarkers7.59k
;
387
245k
  if (HasCleanups) {
388
7.59k
    // Make sure the line table doesn't jump back into the body for
389
7.59k
    // the ret after it's been at EndLoc.
390
7.59k
    Optional<ApplyDebugLocation> AL;
391
7.59k
    if (CGDebugInfo *DI = getDebugInfo()) {
392
1.71k
      if (OnlySimpleReturnStmts)
393
580
        DI->EmitLocation(Builder, EndLoc);
394
1.13k
      else
395
1.13k
        // We may not have a valid end location. Try to apply it anyway, and
396
1.13k
        // fall back to an artificial location if needed.
397
1.13k
        AL = ApplyDebugLocation::CreateDefaultArtificial(*this, EndLoc);
398
1.71k
    }
399
7.59k
400
7.59k
    PopCleanupBlocks(PrologueCleanupDepth);
401
7.59k
  }
402
245k
403
245k
  // Emit function epilog (to return).
404
245k
  llvm::DebugLoc Loc = EmitReturnBlock();
405
245k
406
245k
  if (ShouldInstrumentFunction()) {
407
11
    if (CGM.getCodeGenOpts().InstrumentFunctions)
408
6
      CurFn->addFnAttr("instrument-function-exit", "__cyg_profile_func_exit");
409
11
    if (CGM.getCodeGenOpts().InstrumentFunctionsAfterInlining)
410
4
      CurFn->addFnAttr("instrument-function-exit-inlined",
411
4
                       "__cyg_profile_func_exit");
412
11
  }
413
245k
414
245k
  // Emit debug descriptor for function end.
415
245k
  if (CGDebugInfo *DI = getDebugInfo())
416
89.3k
    DI->EmitFunctionEnd(Builder, CurFn);
417
245k
418
245k
  // Reset the debug location to that of the simple 'return' expression, if any
419
245k
  // rather than that of the end of the function's scope '}'.
420
245k
  ApplyDebugLocation AL(*this, Loc);
421
245k
  EmitFunctionEpilog(*CurFnInfo, EmitRetDbgLoc, EndLoc);
422
245k
  EmitEndEHSpec(CurCodeDecl);
423
245k
424
245k
  assert(EHStack.empty() &&
425
245k
         "did not remove all scopes from cleanup stack!");
426
245k
427
245k
  // If someone did an indirect goto, emit the indirect goto block at the end of
428
245k
  // the function.
429
245k
  if (IndirectBranch) {
430
35
    EmitBlock(IndirectBranch->getParent());
431
35
    Builder.ClearInsertionPoint();
432
35
  }
433
245k
434
245k
  // If some of our locals escaped, insert a call to llvm.localescape in the
435
245k
  // entry block.
436
245k
  if (!EscapedLocals.empty()) {
437
32
    // Invert the map from local to index into a simple vector. There should be
438
32
    // no holes.
439
32
    SmallVector<llvm::Value *, 4> EscapeArgs;
440
32
    EscapeArgs.resize(EscapedLocals.size());
441
32
    for (auto &Pair : EscapedLocals)
442
37
      EscapeArgs[Pair.second] = Pair.first;
443
32
    llvm::Function *FrameEscapeFn = llvm::Intrinsic::getDeclaration(
444
32
        &CGM.getModule(), llvm::Intrinsic::localescape);
445
32
    CGBuilderTy(*this, AllocaInsertPt).CreateCall(FrameEscapeFn, EscapeArgs);
446
32
  }
447
245k
448
245k
  // Remove the AllocaInsertPt instruction, which is just a convenience for us.
449
245k
  llvm::Instruction *Ptr = AllocaInsertPt;
450
245k
  AllocaInsertPt = nullptr;
451
245k
  Ptr->eraseFromParent();
452
245k
453
245k
  // If someone took the address of a label but never did an indirect goto, we
454
245k
  // made a zero entry PHI node, which is illegal, zap it now.
455
245k
  if (IndirectBranch) {
456
35
    llvm::PHINode *PN = cast<llvm::PHINode>(IndirectBranch->getAddress());
457
35
    if (PN->getNumIncomingValues() == 0) {
458
15
      PN->replaceAllUsesWith(llvm::UndefValue::get(PN->getType()));
459
15
      PN->eraseFromParent();
460
15
    }
461
35
  }
462
245k
463
245k
  EmitIfUsed(*this, EHResumeBlock);
464
245k
  EmitIfUsed(*this, TerminateLandingPad);
465
245k
  EmitIfUsed(*this, TerminateHandler);
466
245k
  EmitIfUsed(*this, UnreachableBlock);
467
245k
468
245k
  for (const auto &FuncletAndParent : TerminateFunclets)
469
19
    EmitIfUsed(*this, FuncletAndParent.second);
470
245k
471
245k
  if (CGM.getCodeGenOpts().EmitDeclMetadata)
472
17.5k
    EmitDeclMetadata();
473
245k
474
245k
  for (SmallVectorImpl<std::pair<llvm::Instruction *, llvm::Value *> >::iterator
475
245k
           I = DeferredReplacements.begin(),
476
245k
           E = DeferredReplacements.end();
477
245k
       I != E; 
++I97
) {
478
97
    I->first->replaceAllUsesWith(I->second);
479
97
    I->first->eraseFromParent();
480
97
  }
481
245k
482
245k
  // Eliminate CleanupDestSlot alloca by replacing it with SSA values and
483
245k
  // PHIs if the current function is a coroutine. We don't do it for all
484
245k
  // functions as it may result in slight increase in numbers of instructions
485
245k
  // if compiled with no optimizations. We do it for coroutine as the lifetime
486
245k
  // of CleanupDestSlot alloca make correct coroutine frame building very
487
245k
  // difficult.
488
245k
  if (NormalCleanupDest.isValid() && 
isCoroutine()280
) {
489
49
    llvm::DominatorTree DT(*CurFn);
490
49
    llvm::PromoteMemToReg(
491
49
        cast<llvm::AllocaInst>(NormalCleanupDest.getPointer()), DT);
492
49
    NormalCleanupDest = Address::invalid();
493
49
  }
494
245k
495
245k
  // Scan function arguments for vector width.
496
245k
  for (llvm::Argument &A : CurFn->args())
497
393k
    if (auto *VT = dyn_cast<llvm::VectorType>(A.getType()))
498
75.6k
      LargestVectorWidth =
499
75.6k
          std::max((uint64_t)LargestVectorWidth,
500
75.6k
                   VT->getPrimitiveSizeInBits().getKnownMinSize());
501
245k
502
245k
  // Update vector width based on return type.
503
245k
  if (auto *VT = dyn_cast<llvm::VectorType>(CurFn->getReturnType()))
504
36.1k
    LargestVectorWidth =
505
36.1k
        std::max((uint64_t)LargestVectorWidth,
506
36.1k
                 VT->getPrimitiveSizeInBits().getKnownMinSize());
507
245k
508
245k
  // Add the required-vector-width attribute. This contains the max width from:
509
245k
  // 1. min-vector-width attribute used in the source program.
510
245k
  // 2. Any builtins used that have a vector width specified.
511
245k
  // 3. Values passed in and out of inline assembly.
512
245k
  // 4. Width of vector arguments and return types for this function.
513
245k
  // 5. Width of vector aguments and return types for functions called by this
514
245k
  //    function.
515
245k
  CurFn->addFnAttr("min-legal-vector-width", llvm::utostr(LargestVectorWidth));
516
245k
517
245k
  // If we generated an unreachable return block, delete it now.
518
245k
  if (ReturnBlock.isValid() && 
ReturnBlock.getBlock()->use_empty()2.46k
) {
519
931
    Builder.ClearInsertionPoint();
520
931
    ReturnBlock.getBlock()->eraseFromParent();
521
931
  }
522
245k
  if (ReturnValue.isValid()) {
523
118k
    auto *RetAlloca = dyn_cast<llvm::AllocaInst>(ReturnValue.getPointer());
524
118k
    if (RetAlloca && 
RetAlloca->use_empty()115k
) {
525
103k
      RetAlloca->eraseFromParent();
526
103k
      ReturnValue = Address::invalid();
527
103k
    }
528
118k
  }
529
245k
}
530
531
/// ShouldInstrumentFunction - Return true if the current function should be
532
/// instrumented with __cyg_profile_func_* calls
533
490k
bool CodeGenFunction::ShouldInstrumentFunction() {
534
490k
  if (!CGM.getCodeGenOpts().InstrumentFunctions &&
535
490k
      
!CGM.getCodeGenOpts().InstrumentFunctionsAfterInlining490k
&&
536
490k
      
!CGM.getCodeGenOpts().InstrumentFunctionEntryBare490k
)
537
490k
    return false;
538
32
  if (!CurFuncDecl || 
CurFuncDecl->hasAttr<NoInstrumentFunctionAttr>()28
)
539
10
    return false;
540
22
  return true;
541
22
}
542
543
/// ShouldXRayInstrument - Return true if the current function should be
544
/// instrumented with XRay nop sleds.
545
478k
bool CodeGenFunction::ShouldXRayInstrumentFunction() const {
546
478k
  return CGM.getCodeGenOpts().XRayInstrumentFunctions;
547
478k
}
548
549
/// AlwaysEmitXRayCustomEvents - Return true if we should emit IR for calls to
550
/// the __xray_customevent(...) builtin calls, when doing XRay instrumentation.
551
2
bool CodeGenFunction::AlwaysEmitXRayCustomEvents() const {
552
2
  return CGM.getCodeGenOpts().XRayInstrumentFunctions &&
553
2
         (CGM.getCodeGenOpts().XRayAlwaysEmitCustomEvents ||
554
2
          CGM.getCodeGenOpts().XRayInstrumentationBundle.Mask ==
555
1
              XRayInstrKind::Custom);
556
2
}
557
558
2
bool CodeGenFunction::AlwaysEmitXRayTypedEvents() const {
559
2
  return CGM.getCodeGenOpts().XRayInstrumentFunctions &&
560
2
         (CGM.getCodeGenOpts().XRayAlwaysEmitTypedEvents ||
561
2
          CGM.getCodeGenOpts().XRayInstrumentationBundle.Mask ==
562
1
              XRayInstrKind::Typed);
563
2
}
564
565
llvm::Constant *
566
CodeGenFunction::EncodeAddrForUseInPrologue(llvm::Function *F,
567
113
                                            llvm::Constant *Addr) {
568
113
  // Addresses stored in prologue data can't require run-time fixups and must
569
113
  // be PC-relative. Run-time fixups are undesirable because they necessitate
570
113
  // writable text segments, which are unsafe. And absolute addresses are
571
113
  // undesirable because they break PIE mode.
572
113
573
113
  // Add a layer of indirection through a private global. Taking its address
574
113
  // won't result in a run-time fixup, even if Addr has linkonce_odr linkage.
575
113
  auto *GV = new llvm::GlobalVariable(CGM.getModule(), Addr->getType(),
576
113
                                      /*isConstant=*/true,
577
113
                                      llvm::GlobalValue::PrivateLinkage, Addr);
578
113
579
113
  // Create a PC-relative address.
580
113
  auto *GOTAsInt = llvm::ConstantExpr::getPtrToInt(GV, IntPtrTy);
581
113
  auto *FuncAsInt = llvm::ConstantExpr::getPtrToInt(F, IntPtrTy);
582
113
  auto *PCRelAsInt = llvm::ConstantExpr::getSub(GOTAsInt, FuncAsInt);
583
113
  return (IntPtrTy == Int32Ty)
584
113
             ? 
PCRelAsInt70
585
113
             : 
llvm::ConstantExpr::getTrunc(PCRelAsInt, Int32Ty)43
;
586
113
}
587
588
llvm::Value *
589
CodeGenFunction::DecodeAddrUsedInPrologue(llvm::Value *F,
590
8
                                          llvm::Value *EncodedAddr) {
591
8
  // Reconstruct the address of the global.
592
8
  auto *PCRelAsInt = Builder.CreateSExt(EncodedAddr, IntPtrTy);
593
8
  auto *FuncAsInt = Builder.CreatePtrToInt(F, IntPtrTy, "func_addr.int");
594
8
  auto *GOTAsInt = Builder.CreateAdd(PCRelAsInt, FuncAsInt, "global_addr.int");
595
8
  auto *GOTAddr = Builder.CreateIntToPtr(GOTAsInt, Int8PtrPtrTy, "global_addr");
596
8
597
8
  // Load the original pointer through the global.
598
8
  return Builder.CreateLoad(Address(GOTAddr, getPointerAlign()),
599
8
                            "decoded_addr");
600
8
}
601
602
void CodeGenFunction::EmitOpenCLKernelMetadata(const FunctionDecl *FD,
603
                                               llvm::Function *Fn)
604
1.59k
{
605
1.59k
  if (!FD->hasAttr<OpenCLKernelAttr>())
606
1.18k
    return;
607
408
608
408
  llvm::LLVMContext &Context = getLLVMContext();
609
408
610
408
  CGM.GenOpenCLArgMetadata(Fn, FD, this);
611
408
612
408
  if (const VecTypeHintAttr *A = FD->getAttr<VecTypeHintAttr>()) {
613
2
    QualType HintQTy = A->getTypeHint();
614
2
    const ExtVectorType *HintEltQTy = HintQTy->getAs<ExtVectorType>();
615
2
    bool IsSignedInteger =
616
2
        HintQTy->isSignedIntegerType() ||
617
2
        
(1
HintEltQTy1
&&
HintEltQTy->getElementType()->isSignedIntegerType()1
);
618
2
    llvm::Metadata *AttrMDArgs[] = {
619
2
        llvm::ConstantAsMetadata::get(llvm::UndefValue::get(
620
2
            CGM.getTypes().ConvertType(A->getTypeHint()))),
621
2
        llvm::ConstantAsMetadata::get(llvm::ConstantInt::get(
622
2
            llvm::IntegerType::get(Context, 32),
623
2
            llvm::APInt(32, (uint64_t)(IsSignedInteger ? 
11
:
01
))))};
624
2
    Fn->setMetadata("vec_type_hint", llvm::MDNode::get(Context, AttrMDArgs));
625
2
  }
626
408
627
408
  if (const WorkGroupSizeHintAttr *A = FD->getAttr<WorkGroupSizeHintAttr>()) {
628
1
    llvm::Metadata *AttrMDArgs[] = {
629
1
        llvm::ConstantAsMetadata::get(Builder.getInt32(A->getXDim())),
630
1
        llvm::ConstantAsMetadata::get(Builder.getInt32(A->getYDim())),
631
1
        llvm::ConstantAsMetadata::get(Builder.getInt32(A->getZDim()))};
632
1
    Fn->setMetadata("work_group_size_hint", llvm::MDNode::get(Context, AttrMDArgs));
633
1
  }
634
408
635
408
  if (const ReqdWorkGroupSizeAttr *A = FD->getAttr<ReqdWorkGroupSizeAttr>()) {
636
7
    llvm::Metadata *AttrMDArgs[] = {
637
7
        llvm::ConstantAsMetadata::get(Builder.getInt32(A->getXDim())),
638
7
        llvm::ConstantAsMetadata::get(Builder.getInt32(A->getYDim())),
639
7
        llvm::ConstantAsMetadata::get(Builder.getInt32(A->getZDim()))};
640
7
    Fn->setMetadata("reqd_work_group_size", llvm::MDNode::get(Context, AttrMDArgs));
641
7
  }
642
408
643
408
  if (const OpenCLIntelReqdSubGroupSizeAttr *A =
644
1
          FD->getAttr<OpenCLIntelReqdSubGroupSizeAttr>()) {
645
1
    llvm::Metadata *AttrMDArgs[] = {
646
1
        llvm::ConstantAsMetadata::get(Builder.getInt32(A->getSubGroupSize()))};
647
1
    Fn->setMetadata("intel_reqd_sub_group_size",
648
1
                    llvm::MDNode::get(Context, AttrMDArgs));
649
1
  }
650
408
}
651
652
/// Determine whether the function F ends with a return stmt.
653
126k
static bool endsWithReturn(const Decl* F) {
654
126k
  const Stmt *Body = nullptr;
655
126k
  if (auto *FD = dyn_cast_or_null<FunctionDecl>(F))
656
99.4k
    Body = FD->getBody();
657
27.3k
  else if (auto *OMD = dyn_cast_or_null<ObjCMethodDecl>(F))
658
1.21k
    Body = OMD->getBody();
659
126k
660
126k
  if (auto *CS = dyn_cast_or_null<CompoundStmt>(Body)) {
661
99.1k
    auto LastStmt = CS->body_rbegin();
662
99.1k
    if (LastStmt != CS->body_rend())
663
57.2k
      return isa<ReturnStmt>(*LastStmt);
664
69.6k
  }
665
69.6k
  return false;
666
69.6k
}
667
668
327
void CodeGenFunction::markAsIgnoreThreadCheckingAtRuntime(llvm::Function *Fn) {
669
327
  if (SanOpts.has(SanitizerKind::Thread)) {
670
4
    Fn->addFnAttr("sanitize_thread_no_checking_at_run_time");
671
4
    Fn->removeFnAttr(llvm::Attribute::SanitizeThread);
672
4
  }
673
327
}
674
675
/// Check if the return value of this function requires sanitization.
676
363k
bool CodeGenFunction::requiresReturnValueCheck() const {
677
363k
  return requiresReturnValueNullabilityCheck() ||
678
363k
         
(363k
SanOpts.has(SanitizerKind::ReturnsNonnullAttribute)363k
&&
CurCodeDecl108
&&
679
363k
          
CurCodeDecl->getAttr<ReturnsNonNullAttr>()108
);
680
363k
}
681
682
35
static bool matchesStlAllocatorFn(const Decl *D, const ASTContext &Ctx) {
683
35
  auto *MD = dyn_cast_or_null<CXXMethodDecl>(D);
684
35
  if (!MD || 
!MD->getDeclName().getAsIdentifierInfo()3
||
685
35
      
!MD->getDeclName().getAsIdentifierInfo()->isStr("allocate")3
||
686
35
      
(2
MD->getNumParams() != 12
&&
MD->getNumParams() != 21
))
687
33
    return false;
688
2
689
2
  if (MD->parameters()[0]->getType().getCanonicalType() != Ctx.getSizeType())
690
0
    return false;
691
2
692
2
  if (MD->getNumParams() == 2) {
693
1
    auto *PT = MD->parameters()[1]->getType()->getAs<PointerType>();
694
1
    if (!PT || !PT->isVoidPointerType() ||
695
1
        !PT->getPointeeType().isConstQualified())
696
0
      return false;
697
2
  }
698
2
699
2
  return true;
700
2
}
701
702
/// Return the UBSan prologue signature for \p FD if one is available.
703
static llvm::Constant *getPrologueSignature(CodeGenModule &CGM,
704
203
                                            const FunctionDecl *FD) {
705
203
  if (const auto *MD = dyn_cast<CXXMethodDecl>(FD))
706
96
    if (!MD->isStatic())
707
90
      return nullptr;
708
113
  return CGM.getTargetCodeGenInfo().getUBSanFunctionSignature(CGM);
709
113
}
710
711
void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy,
712
                                    llvm::Function *Fn,
713
                                    const CGFunctionInfo &FnInfo,
714
                                    const FunctionArgList &Args,
715
                                    SourceLocation Loc,
716
245k
                                    SourceLocation StartLoc) {
717
245k
  assert(!CurFn &&
718
245k
         "Do not use a CodeGenFunction object for more than one function");
719
245k
720
245k
  const Decl *D = GD.getDecl();
721
245k
722
245k
  DidCallStackSave = false;
723
245k
  CurCodeDecl = D;
724
245k
  if (const auto *FD = dyn_cast_or_null<FunctionDecl>(D))
725
215k
    if (FD->usesSEHTry())
726
100
      CurSEHParent = FD;
727
245k
  CurFuncDecl = (D ? 
D->getNonClosureContext()239k
:
nullptr5.90k
);
728
245k
  FnRetTy = RetTy;
729
245k
  CurFn = Fn;
730
245k
  CurFnInfo = &FnInfo;
731
245k
  assert(CurFn->isDeclaration() && "Function already has body?");
732
245k
733
245k
  // If this function has been blacklisted for any of the enabled sanitizers,
734
245k
  // disable the sanitizer for the function.
735
245k
  do {
736
245k
#define SANITIZER(NAME, ID)                                                    \
737
701k
  if (
SanOpts.empty()460k
) \
738
701k
    
break240k
; \
739
701k
  
if (219k
SanOpts.has(SanitizerKind::ID)219k
) \
740
219k
    
if (7.23k
CGM.isInSanitizerBlacklist(SanitizerKind::ID, Fn, Loc)7.23k
) \
741
7.23k
      
SanOpts.set(SanitizerKind::ID, false)49
;
742
245k
743
245k
#include "clang/Basic/Sanitizers.def"
744
219k
#undef SANITIZER
745
219k
  } while (
04.13k
);
746
245k
747
245k
  if (D) {
748
239k
    // Apply the no_sanitize* attributes to SanOpts.
749
239k
    for (auto Attr : D->specific_attrs<NoSanitizeAttr>()) {
750
935
      SanitizerMask mask = Attr->getMask();
751
935
      SanOpts.Mask &= ~mask;
752
935
      if (mask & SanitizerKind::Address)
753
52
        SanOpts.set(SanitizerKind::KernelAddress, false);
754
935
      if (mask & SanitizerKind::KernelAddress)
755
5
        SanOpts.set(SanitizerKind::Address, false);
756
935
      if (mask & SanitizerKind::HWAddress)
757
5
        SanOpts.set(SanitizerKind::KernelHWAddress, false);
758
935
      if (mask & SanitizerKind::KernelHWAddress)
759
5
        SanOpts.set(SanitizerKind::HWAddress, false);
760
935
    }
761
239k
  }
762
245k
763
245k
  // Apply sanitizer attributes to the function.
764
245k
  if (SanOpts.hasOneOf(SanitizerKind::Address | SanitizerKind::KernelAddress))
765
1.03k
    Fn->addFnAttr(llvm::Attribute::SanitizeAddress);
766
245k
  if (SanOpts.hasOneOf(SanitizerKind::HWAddress | SanitizerKind::KernelHWAddress))
767
30
    Fn->addFnAttr(llvm::Attribute::SanitizeHWAddress);
768
245k
  if (SanOpts.has(SanitizerKind::MemTag))
769
17
    Fn->addFnAttr(llvm::Attribute::SanitizeMemTag);
770
245k
  if (SanOpts.has(SanitizerKind::Thread))
771
62
    Fn->addFnAttr(llvm::Attribute::SanitizeThread);
772
245k
  if (SanOpts.hasOneOf(SanitizerKind::Memory | SanitizerKind::KernelMemory))
773
205
    Fn->addFnAttr(llvm::Attribute::SanitizeMemory);
774
245k
  if (SanOpts.has(SanitizerKind::SafeStack))
775
14
    Fn->addFnAttr(llvm::Attribute::SafeStack);
776
245k
  if (SanOpts.has(SanitizerKind::ShadowCallStack))
777
1
    Fn->addFnAttr(llvm::Attribute::ShadowCallStack);
778
245k
779
245k
  // Apply fuzzing attribute to the function.
780
245k
  if (SanOpts.hasOneOf(SanitizerKind::Fuzzer | SanitizerKind::FuzzerNoLink))
781
6
    Fn->addFnAttr(llvm::Attribute::OptForFuzzing);
782
245k
783
245k
  // Ignore TSan memory acesses from within ObjC/ObjC++ dealloc, initialize,
784
245k
  // .cxx_destruct, __destroy_helper_block_ and all of their calees at run time.
785
245k
  if (SanOpts.has(SanitizerKind::Thread)) {
786
62
    if (const auto *OMD = dyn_cast_or_null<ObjCMethodDecl>(D)) {
787
3
      IdentifierInfo *II = OMD->getSelector().getIdentifierInfoForSlot(0);
788
3
      if (OMD->getMethodFamily() == OMF_dealloc ||
789
3
          
OMD->getMethodFamily() == OMF_initialize2
||
790
3
          
(1
OMD->getSelector().isUnarySelector()1
&&
II->isStr(".cxx_destruct")1
)) {
791
3
        markAsIgnoreThreadCheckingAtRuntime(Fn);
792
3
      }
793
3
    }
794
62
  }
795
245k
796
245k
  // Ignore unrelated casts in STL allocate() since the allocator must cast
797
245k
  // from void* to T* before object initialization completes. Don't match on the
798
245k
  // namespace because not all allocators are in std::
799
245k
  if (D && 
SanOpts.has(SanitizerKind::CFIUnrelatedCast)239k
) {
800
35
    if (matchesStlAllocatorFn(D, getContext()))
801
2
      SanOpts.Mask &= ~SanitizerKind::CFIUnrelatedCast;
802
35
  }
803
245k
804
245k
  // Ignore null checks in coroutine functions since the coroutines passes
805
245k
  // are not aware of how to move the extra UBSan instructions across the split
806
245k
  // coroutine boundaries.
807
245k
  if (D && 
SanOpts.has(SanitizerKind::Null)239k
)
808
242
    if (const auto *FD = dyn_cast<FunctionDecl>(D))
809
242
      if (FD->getBody() &&
810
242
          FD->getBody()->getStmtClass() == Stmt::CoroutineBodyStmtClass)
811
1
        SanOpts.Mask &= ~SanitizerKind::Null;
812
245k
813
245k
  if (D) {
814
239k
    // Apply xray attributes to the function (as a string, for now)
815
239k
    if (const auto *XRayAttr = D->getAttr<XRayInstrumentAttr>()) {
816
80
      if (CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
817
80
              XRayInstrKind::FunctionEntry) ||
818
80
          CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
819
76
              XRayInstrKind::FunctionExit)) {
820
76
        if (XRayAttr->alwaysXRayInstrument() && 
ShouldXRayInstrumentFunction()42
)
821
35
          Fn->addFnAttr("function-instrument", "xray-always");
822
76
        if (XRayAttr->neverXRayInstrument())
823
34
          Fn->addFnAttr("function-instrument", "xray-never");
824
76
        if (const auto *LogArgs = D->getAttr<XRayLogArgsAttr>())
825
1
          if (ShouldXRayInstrumentFunction())
826
1
            Fn->addFnAttr("xray-log-args",
827
1
                          llvm::utostr(LogArgs->getArgumentCount()));
828
76
      }
829
239k
    } else {
830
239k
      if (ShouldXRayInstrumentFunction() && 
!CGM.imbueXRayAttrs(Fn, Loc)20
)
831
8
        Fn->addFnAttr(
832
8
            "xray-instruction-threshold",
833
8
            llvm::itostr(CGM.getCodeGenOpts().XRayInstructionThreshold));
834
239k
    }
835
239k
836
239k
    if (ShouldXRayInstrumentFunction()) {
837
86
      if (CGM.getCodeGenOpts().XRayIgnoreLoops)
838
1
        Fn->addFnAttr("xray-ignore-loops");
839
86
840
86
      if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
841
86
              XRayInstrKind::FunctionExit))
842
6
        Fn->addFnAttr("xray-skip-exit");
843
86
844
86
      if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
845
86
              XRayInstrKind::FunctionEntry))
846
6
        Fn->addFnAttr("xray-skip-entry");
847
86
    }
848
239k
849
239k
    unsigned Count, Offset;
850
239k
    if (const auto *Attr = D->getAttr<PatchableFunctionEntryAttr>()) {
851
14
      Count = Attr->getCount();
852
14
      Offset = Attr->getOffset();
853
239k
    } else {
854
239k
      Count = CGM.getCodeGenOpts().PatchableFunctionEntryCount;
855
239k
      Offset = CGM.getCodeGenOpts().PatchableFunctionEntryOffset;
856
239k
    }
857
239k
    if (Count && 
Offset <= Count11
) {
858
11
      Fn->addFnAttr("patchable-function-entry", std::to_string(Count - Offset));
859
11
      if (Offset)
860
4
        Fn->addFnAttr("patchable-function-prefix", std::to_string(Offset));
861
11
    }
862
239k
  }
863
245k
864
245k
  // Add no-jump-tables value.
865
245k
  Fn->addFnAttr("no-jump-tables",
866
245k
                llvm::toStringRef(CGM.getCodeGenOpts().NoUseJumpTables));
867
245k
868
245k
  // Add no-inline-line-tables value.
869
245k
  if (CGM.getCodeGenOpts().NoInlineLineTables)
870
4
    Fn->addFnAttr("no-inline-line-tables");
871
245k
872
245k
  // Add profile-sample-accurate value.
873
245k
  if (CGM.getCodeGenOpts().ProfileSampleAccurate)
874
2
    Fn->addFnAttr("profile-sample-accurate");
875
245k
876
245k
  if (D && 
D->hasAttr<CFICanonicalJumpTableAttr>()239k
)
877
2
    Fn->addFnAttr("cfi-canonical-jump-table");
878
245k
879
245k
  if (getLangOpts().OpenCL) {
880
1.69k
    // Add metadata for a kernel function.
881
1.69k
    if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D))
882
1.59k
      EmitOpenCLKernelMetadata(FD, Fn);
883
1.69k
  }
884
245k
885
245k
  // If we are checking function types, emit a function type signature as
886
245k
  // prologue data.
887
245k
  if (getLangOpts().CPlusPlus && 
SanOpts.has(SanitizerKind::Function)172k
) {
888
210
    if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) {
889
203
      if (llvm::Constant *PrologueSig = getPrologueSignature(CGM, FD)) {
890
113
        // Remove any (C++17) exception specifications, to allow calling e.g. a
891
113
        // noexcept function through a non-noexcept pointer.
892
113
        auto ProtoTy =
893
113
          getContext().getFunctionTypeWithExceptionSpec(FD->getType(),
894
113
                                                        EST_None);
895
113
        llvm::Constant *FTRTTIConst =
896
113
            CGM.GetAddrOfRTTIDescriptor(ProtoTy, /*ForEH=*/true);
897
113
        llvm::Constant *FTRTTIConstEncoded =
898
113
            EncodeAddrForUseInPrologue(Fn, FTRTTIConst);
899
113
        llvm::Constant *PrologueStructElems[] = {PrologueSig,
900
113
                                                 FTRTTIConstEncoded};
901
113
        llvm::Constant *PrologueStructConst =
902
113
            llvm::ConstantStruct::getAnon(PrologueStructElems, /*Packed=*/true);
903
113
        Fn->setPrologueData(PrologueStructConst);
904
113
      }
905
203
    }
906
210
  }
907
245k
908
245k
  // If we're checking nullability, we need to know whether we can check the
909
245k
  // return value. Initialize the flag to 'true' and refine it in EmitParmDecl.
910
245k
  if (SanOpts.has(SanitizerKind::NullabilityReturn)) {
911
60
    auto Nullability = FnRetTy->getNullability(getContext());
912
60
    if (Nullability && 
*Nullability == NullabilityKind::NonNull17
) {
913
15
      if (!(SanOpts.has(SanitizerKind::ReturnsNonnullAttribute) &&
914
15
            
CurCodeDecl1
&&
CurCodeDecl->getAttr<ReturnsNonNullAttr>()1
))
915
14
        RetValNullabilityPrecondition =
916
14
            llvm::ConstantInt::getTrue(getLLVMContext());
917
15
    }
918
60
  }
919
245k
920
245k
  // If we're in C++ mode and the function name is "main", it is guaranteed
921
245k
  // to be norecurse by the standard (3.6.1.3 "The function main shall not be
922
245k
  // used within a program").
923
245k
  //
924
245k
  // OpenCL C 2.0 v2.2-11 s6.9.i:
925
245k
  //     Recursion is not supported.
926
245k
  //
927
245k
  // SYCL v1.2.1 s3.10:
928
245k
  //     kernels cannot include RTTI information, exception classes,
929
245k
  //     recursive code, virtual functions or make use of C++ libraries that
930
245k
  //     are not compiled for the device.
931
245k
  if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) {
932
215k
    if ((getLangOpts().CPlusPlus && 
FD->isMain()146k
) ||
getLangOpts().OpenCL211k
||
933
215k
        
getLangOpts().SYCLIsDevice210k
||
934
215k
        
(210k
getLangOpts().CUDA210k
&&
FD->hasAttr<CUDAGlobalAttr>()514
))
935
5.07k
      Fn->addFnAttr(llvm::Attribute::NoRecurse);
936
215k
  }
937
245k
938
245k
  if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D))
939
215k
    if (FD->usesFPIntrin())
940
55
      Fn->addFnAttr(llvm::Attribute::StrictFP);
941
245k
942
245k
  // If a custom alignment is used, force realigning to this alignment on
943
245k
  // any main function which certainly will need it.
944
245k
  if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D))
945
215k
    if ((FD->isMain() || 
FD->isMSVCRTEntryPoint()210k
) &&
946
215k
        
CGM.getCodeGenOpts().StackAlignment4.84k
)
947
1
      Fn->addFnAttr("stackrealign");
948
245k
949
245k
  llvm::BasicBlock *EntryBB = createBasicBlock("entry", CurFn);
950
245k
951
245k
  // Create a marker to make it easy to insert allocas into the entryblock
952
245k
  // later.  Don't create this with the builder, because we don't want it
953
245k
  // folded.
954
245k
  llvm::Value *Undef = llvm::UndefValue::get(Int32Ty);
955
245k
  AllocaInsertPt = new llvm::BitCastInst(Undef, Int32Ty, "allocapt", EntryBB);
956
245k
957
245k
  ReturnBlock = getJumpDestInCurrentScope("return");
958
245k
959
245k
  Builder.SetInsertPoint(EntryBB);
960
245k
961
245k
  // If we're checking the return value, allocate space for a pointer to a
962
245k
  // precise source location of the checked return statement.
963
245k
  if (requiresReturnValueCheck()) {
964
19
    ReturnLocation = CreateDefaultAlignTempAlloca(Int8PtrTy, "return.sloc.ptr");
965
19
    InitTempAlloca(ReturnLocation, llvm::ConstantPointerNull::get(Int8PtrTy));
966
19
  }
967
245k
968
245k
  // Emit subprogram debug descriptor.
969
245k
  if (CGDebugInfo *DI = getDebugInfo()) {
970
89.3k
    // Reconstruct the type from the argument list so that implicit parameters,
971
89.3k
    // such as 'this' and 'vtt', show up in the debug info. Preserve the calling
972
89.3k
    // convention.
973
89.3k
    CallingConv CC = CallingConv::CC_C;
974
89.3k
    if (auto *FD = dyn_cast_or_null<FunctionDecl>(D))
975
86.7k
      if (const auto *SrcFnTy = FD->getType()->getAs<FunctionType>())
976
86.7k
        CC = SrcFnTy->getCallConv();
977
89.3k
    SmallVector<QualType, 16> ArgTypes;
978
89.3k
    for (const VarDecl *VD : Args)
979
151k
      ArgTypes.push_back(VD->getType());
980
89.3k
    QualType FnType = getContext().getFunctionType(
981
89.3k
        RetTy, ArgTypes, FunctionProtoType::ExtProtoInfo(CC));
982
89.3k
    DI->EmitFunctionStart(GD, Loc, StartLoc, FnType, CurFn, CurFuncIsThunk,
983
89.3k
                          Builder);
984
89.3k
  }
985
245k
986
245k
  if (ShouldInstrumentFunction()) {
987
11
    if (CGM.getCodeGenOpts().InstrumentFunctions)
988
6
      CurFn->addFnAttr("instrument-function-entry", "__cyg_profile_func_enter");
989
11
    if (CGM.getCodeGenOpts().InstrumentFunctionsAfterInlining)
990
4
      CurFn->addFnAttr("instrument-function-entry-inlined",
991
4
                       "__cyg_profile_func_enter");
992
11
    if (CGM.getCodeGenOpts().InstrumentFunctionEntryBare)
993
1
      CurFn->addFnAttr("instrument-function-entry-inlined",
994
1
                       "__cyg_profile_func_enter_bare");
995
11
  }
996
245k
997
245k
  // Since emitting the mcount call here impacts optimizations such as function
998
245k
  // inlining, we just add an attribute to insert a mcount call in backend.
999
245k
  // The attribute "counting-function" is set to mcount function name which is
1000
245k
  // architecture dependent.
1001
245k
  if (CGM.getCodeGenOpts().InstrumentForProfiling) {
1002
119
    // Calls to fentry/mcount should not be generated if function has
1003
119
    // the no_instrument_function attribute.
1004
119
    if (!CurFuncDecl || !CurFuncDecl->hasAttr<NoInstrumentFunctionAttr>()) {
1005
96
      if (CGM.getCodeGenOpts().CallFEntry)
1006
4
        Fn->addFnAttr("fentry-call", "true");
1007
92
      else {
1008
92
        Fn->addFnAttr("instrument-function-entry-inlined",
1009
92
                      getTarget().getMCountName());
1010
92
      }
1011
96
      if (CGM.getCodeGenOpts().MNopMCount) {
1012
2
        if (!CGM.getCodeGenOpts().CallFEntry)
1013
1
          CGM.getDiags().Report(diag::err_opt_not_valid_without_opt)
1014
1
            << "-mnop-mcount" << "-mfentry";
1015
2
        Fn->addFnAttr("mnop-mcount");
1016
2
      }
1017
96
1018
96
      if (CGM.getCodeGenOpts().RecordMCount) {
1019
2
        if (!CGM.getCodeGenOpts().CallFEntry)
1020
1
          CGM.getDiags().Report(diag::err_opt_not_valid_without_opt)
1021
1
            << "-mrecord-mcount" << "-mfentry";
1022
2
        Fn->addFnAttr("mrecord-mcount");
1023
2
      }
1024
96
    }
1025
119
  }
1026
245k
1027
245k
  if (CGM.getCodeGenOpts().PackedStack) {
1028
2
    if (getContext().getTargetInfo().getTriple().getArch() !=
1029
2
        llvm::Triple::systemz)
1030
1
      CGM.getDiags().Report(diag::err_opt_not_valid_on_target)
1031
1
        << "-mpacked-stack";
1032
2
    Fn->addFnAttr("packed-stack");
1033
2
  }
1034
245k
1035
245k
  if (RetTy->isVoidType()) {
1036
126k
    // Void type; nothing to return.
1037
126k
    ReturnValue = Address::invalid();
1038
126k
1039
126k
    // Count the implicit return.
1040
126k
    if (!endsWithReturn(D))
1041
125k
      ++NumReturnExprs;
1042
126k
  } else 
if (118k
CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::Indirect118k
) {
1043
2.25k
    // Indirect return; emit returned value directly into sret slot.
1044
2.25k
    // This reduces code size, and affects correctness in C++.
1045
2.25k
    auto AI = CurFn->arg_begin();
1046
2.25k
    if (CurFnInfo->getReturnInfo().isSRetAfterThis())
1047
49
      ++AI;
1048
2.25k
    ReturnValue = Address(&*AI, CurFnInfo->getReturnInfo().getIndirectAlign());
1049
2.25k
    if (!CurFnInfo->getReturnInfo().getIndirectByVal()) {
1050
929
      ReturnValuePointer =
1051
929
          CreateDefaultAlignTempAlloca(Int8PtrTy, "result.ptr");
1052
929
      Builder.CreateStore(Builder.CreatePointerBitCastOrAddrSpaceCast(
1053
929
                              ReturnValue.getPointer(), Int8PtrTy),
1054
929
                          ReturnValuePointer);
1055
929
    }
1056
116k
  } else if (CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::InAlloca &&
1057
116k
             
!hasScalarEvaluationKind(CurFnInfo->getReturnType())5
) {
1058
5
    // Load the sret pointer from the argument struct and return into that.
1059
5
    unsigned Idx = CurFnInfo->getReturnInfo().getInAllocaFieldIndex();
1060
5
    llvm::Function::arg_iterator EI = CurFn->arg_end();
1061
5
    --EI;
1062
5
    llvm::Value *Addr = Builder.CreateStructGEP(nullptr, &*EI, Idx);
1063
5
    ReturnValuePointer = Address(Addr, getPointerAlign());
1064
5
    Addr = Builder.CreateAlignedLoad(Addr, getPointerAlign(), "agg.result");
1065
5
    ReturnValue = Address(Addr, getNaturalTypeAlignment(RetTy));
1066
116k
  } else {
1067
116k
    ReturnValue = CreateIRTemp(RetTy, "retval");
1068
116k
1069
116k
    // Tell the epilog emitter to autorelease the result.  We do this
1070
116k
    // now so that various specialized functions can suppress it
1071
116k
    // during their IR-generation.
1072
116k
    if (getLangOpts().ObjCAutoRefCount &&
1073
116k
        
!CurFnInfo->isReturnsRetained()372
&&
1074
116k
        
RetTy->isObjCRetainableType()338
)
1075
136
      AutoreleaseResult = true;
1076
116k
  }
1077
245k
1078
245k
  EmitStartEHSpec(CurCodeDecl);
1079
245k
1080
245k
  PrologueCleanupDepth = EHStack.stable_begin();
1081
245k
1082
245k
  // Emit OpenMP specific initialization of the device functions.
1083
245k
  if (getLangOpts().OpenMP && 
CurCodeDecl47.8k
)
1084
43.3k
    CGM.getOpenMPRuntime().emitFunctionProlog(*this, CurCodeDecl);
1085
245k
1086
245k
  EmitFunctionProlog(*CurFnInfo, CurFn, Args);
1087
245k
1088
245k
  if (D && 
isa<CXXMethodDecl>(D)239k
&&
cast<CXXMethodDecl>(D)->isInstance()95.5k
) {
1089
88.3k
    CGM.getCXXABI().EmitInstanceFunctionProlog(*this);
1090
88.3k
    const CXXMethodDecl *MD = cast<CXXMethodDecl>(D);
1091
88.3k
    if (MD->getParent()->isLambda() &&
1092
88.3k
        
MD->getOverloadedOperator() == OO_Call1.58k
) {
1093
1.43k
      // We're in a lambda; figure out the captures.
1094
1.43k
      MD->getParent()->getCaptureFields(LambdaCaptureFields,
1095
1.43k
                                        LambdaThisCaptureField);
1096
1.43k
      if (LambdaThisCaptureField) {
1097
68
        // If the lambda captures the object referred to by '*this' - either by
1098
68
        // value or by reference, make sure CXXThisValue points to the correct
1099
68
        // object.
1100
68
1101
68
        // Get the lvalue for the field (which is a copy of the enclosing object
1102
68
        // or contains the address of the enclosing object).
1103
68
        LValue ThisFieldLValue = EmitLValueForLambdaField(LambdaThisCaptureField);
1104
68
        if (!LambdaThisCaptureField->getType()->isPointerType()) {
1105
1
          // If the enclosing object was captured by value, just use its address.
1106
1
          CXXThisValue = ThisFieldLValue.getAddress(*this).getPointer();
1107
67
        } else {
1108
67
          // Load the lvalue pointed to by the field, since '*this' was captured
1109
67
          // by reference.
1110
67
          CXXThisValue =
1111
67
              EmitLoadOfLValue(ThisFieldLValue, SourceLocation()).getScalarVal();
1112
67
        }
1113
68
      }
1114
2.45k
      for (auto *FD : MD->getParent()->fields()) {
1115
2.45k
        if (FD->hasCapturedVLAType()) {
1116
24
          auto *ExprArg = EmitLoadOfLValue(EmitLValueForLambdaField(FD),
1117
24
                                           SourceLocation()).getScalarVal();
1118
24
          auto VAT = FD->getCapturedVLAType();
1119
24
          VLASizeMap[VAT->getSizeExpr()] = ExprArg;
1120
24
        }
1121
2.45k
      }
1122
86.8k
    } else {
1123
86.8k
      // Not in a lambda; just use 'this' from the method.
1124
86.8k
      // FIXME: Should we generate a new load for each use of 'this'?  The
1125
86.8k
      // fast register allocator would be happier...
1126
86.8k
      CXXThisValue = CXXABIThisValue;
1127
86.8k
    }
1128
88.3k
1129
88.3k
    // Check the 'this' pointer once per function, if it's available.
1130
88.3k
    if (CXXABIThisValue) {
1131
88.3k
      SanitizerSet SkippedChecks;
1132
88.3k
      SkippedChecks.set(SanitizerKind::ObjectSize, true);
1133
88.3k
      QualType ThisTy = MD->getThisType();
1134
88.3k
1135
88.3k
      // If this is the call operator of a lambda with no capture-default, it
1136
88.3k
      // may have a static invoker function, which may call this operator with
1137
88.3k
      // a null 'this' pointer.
1138
88.3k
      if (isLambdaCallOperator(MD) &&
1139
88.3k
          
MD->getParent()->getLambdaCaptureDefault() == LCD_None1.43k
)
1140
287
        SkippedChecks.set(SanitizerKind::Null, true);
1141
88.3k
1142
88.3k
      EmitTypeCheck(isa<CXXConstructorDecl>(MD) ? 
TCK_ConstructorCall35.2k
1143
88.3k
                                                : 
TCK_MemberCall53.0k
,
1144
88.3k
                    Loc, CXXABIThisValue, ThisTy,
1145
88.3k
                    getContext().getTypeAlignInChars(ThisTy->getPointeeType()),
1146
88.3k
                    SkippedChecks);
1147
88.3k
    }
1148
88.3k
  }
1149
245k
1150
245k
  // If any of the arguments have a variably modified type, make sure to
1151
245k
  // emit the type size.
1152
245k
  for (FunctionArgList::const_iterator i = Args.begin(), e = Args.end();
1153
642k
       i != e; 
++i396k
) {
1154
396k
    const VarDecl *VD = *i;
1155
396k
1156
396k
    // Dig out the type as written from ParmVarDecls; it's unclear whether
1157
396k
    // the standard (C99 6.9.1p10) requires this, but we're following the
1158
396k
    // precedent set by gcc.
1159
396k
    QualType Ty;
1160
396k
    if (const ParmVarDecl *PVD = dyn_cast<ParmVarDecl>(VD))
1161
243k
      Ty = PVD->getOriginalType();
1162
153k
    else
1163
153k
      Ty = VD->getType();
1164
396k
1165
396k
    if (Ty->isVariablyModifiedType())
1166
121
      EmitVariablyModifiedType(Ty);
1167
396k
  }
1168
245k
  // Emit a location at the end of the prologue.
1169
245k
  if (CGDebugInfo *DI = getDebugInfo())
1170
89.3k
    DI->EmitLocation(Builder, StartLoc);
1171
245k
1172
245k
  // TODO: Do we need to handle this in two places like we do with
1173
245k
  // target-features/target-cpu?
1174
245k
  if (CurFuncDecl)
1175
234k
    if (const auto *VecWidth = CurFuncDecl->getAttr<MinVectorWidthAttr>())
1176
7.34k
      LargestVectorWidth = VecWidth->getVectorWidth();
1177
245k
}
1178
1179
164k
void CodeGenFunction::EmitFunctionBody(const Stmt *Body) {
1180
164k
  incrementProfileCounter(Body);
1181
164k
  if (const CompoundStmt *S = dyn_cast<CompoundStmt>(Body))
1182
164k
    EmitCompoundStmtWithoutScope(*S);
1183
53
  else
1184
53
    EmitStmt(Body);
1185
164k
}
1186
1187
/// When instrumenting to collect profile data, the counts for some blocks
1188
/// such as switch cases need to not include the fall-through counts, so
1189
/// emit a branch around the instrumentation code. When not instrumenting,
1190
/// this just calls EmitBlock().
1191
void CodeGenFunction::EmitBlockWithFallThrough(llvm::BasicBlock *BB,
1192
1.63k
                                               const Stmt *S) {
1193
1.63k
  llvm::BasicBlock *SkipCountBB = nullptr;
1194
1.63k
  if (HaveInsertPoint() && 
CGM.getCodeGenOpts().hasProfileClangInstr()594
) {
1195
22
    // When instrumenting for profiling, the fallthrough to certain
1196
22
    // statements needs to skip over the instrumentation code so that we
1197
22
    // get an accurate count.
1198
22
    SkipCountBB = createBasicBlock("skipcount");
1199
22
    EmitBranch(SkipCountBB);
1200
22
  }
1201
1.63k
  EmitBlock(BB);
1202
1.63k
  uint64_t CurrentCount = getCurrentProfileCount();
1203
1.63k
  incrementProfileCounter(S);
1204
1.63k
  setCurrentProfileCount(getCurrentProfileCount() + CurrentCount);
1205
1.63k
  if (SkipCountBB)
1206
22
    EmitBlock(SkipCountBB);
1207
1.63k
}
1208
1209
/// Tries to mark the given function nounwind based on the
1210
/// non-existence of any throwing calls within it.  We believe this is
1211
/// lightweight enough to do at -O0.
1212
155k
static void TryMarkNoThrow(llvm::Function *F) {
1213
155k
  // LLVM treats 'nounwind' on a function as part of the type, so we
1214
155k
  // can't do this on functions that can be overwritten.
1215
155k
  if (F->isInterposable()) 
return23
;
1216
155k
1217
155k
  for (llvm::BasicBlock &BB : *F)
1218
235k
    for (llvm::Instruction &I : BB)
1219
2.07M
      if (I.mayThrow())
1220
65.7k
        return;
1221
155k
1222
155k
  F->setDoesNotThrow();
1223
90.1k
}
1224
1225
QualType CodeGenFunction::BuildFunctionArgList(GlobalDecl GD,
1226
214k
                                               FunctionArgList &Args) {
1227
214k
  const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
1228
214k
  QualType ResTy = FD->getReturnType();
1229
214k
1230
214k
  const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(FD);
1231
214k
  if (MD && 
MD->isInstance()95.5k
) {
1232
88.3k
    if (CGM.getCXXABI().HasThisReturn(GD))
1233
1.57k
      ResTy = MD->getThisType();
1234
86.7k
    else if (CGM.getCXXABI().hasMostDerivedReturn(GD))
1235
204
      ResTy = CGM.getContext().VoidPtrTy;
1236
88.3k
    CGM.getCXXABI().buildThisParam(*this, Args);
1237
88.3k
  }
1238
214k
1239
214k
  // The base version of an inheriting constructor whose constructed base is a
1240
214k
  // virtual base is not passed any arguments (because it doesn't actually call
1241
214k
  // the inherited constructor).
1242
214k
  bool PassedParams = true;
1243
214k
  if (const CXXConstructorDecl *CD = dyn_cast<CXXConstructorDecl>(FD))
1244
35.2k
    if (auto Inherited = CD->getInheritedConstructor())
1245
225
      PassedParams =
1246
225
          getTypes().inheritingCtorHasParams(Inherited, GD.getCtorType());
1247
214k
1248
214k
  if (PassedParams) {
1249
241k
    for (auto *Param : FD->parameters()) {
1250
241k
      Args.push_back(Param);
1251
241k
      if (!Param->hasAttr<PassObjectSizeAttr>())
1252
241k
        continue;
1253
88
1254
88
      auto *Implicit = ImplicitParamDecl::Create(
1255
88
          getContext(), Param->getDeclContext(), Param->getLocation(),
1256
88
          /*Id=*/nullptr, getContext().getSizeType(), ImplicitParamDecl::Other);
1257
88
      SizeArguments[Param] = Implicit;
1258
88
      Args.push_back(Implicit);
1259
88
    }
1260
214k
  }
1261
214k
1262
214k
  if (MD && 
(95.5k
isa<CXXConstructorDecl>(MD)95.5k
||
isa<CXXDestructorDecl>(MD)60.2k
))
1263
49.0k
    CGM.getCXXABI().addImplicitStructorParams(*this, ResTy, Args);
1264
214k
1265
214k
  return ResTy;
1266
214k
}
1267
1268
static bool
1269
shouldUseUndefinedBehaviorReturnOptimization(const FunctionDecl *FD,
1270
21
                                             const ASTContext &Context) {
1271
21
  QualType T = FD->getReturnType();
1272
21
  // Avoid the optimization for functions that return a record type with a
1273
21
  // trivial destructor or another trivially copyable type.
1274
21
  if (const RecordType *RT = T.getCanonicalType()->getAs<RecordType>()) {
1275
12
    if (const auto *ClassDecl = dyn_cast<CXXRecordDecl>(RT->getDecl()))
1276
12
      return !ClassDecl->hasTrivialDestructor();
1277
9
  }
1278
9
  return !T.isTriviallyCopyableType(Context);
1279
9
}
1280
1281
void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn,
1282
214k
                                   const CGFunctionInfo &FnInfo) {
1283
214k
  const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
1284
214k
  CurGD = GD;
1285
214k
1286
214k
  FunctionArgList Args;
1287
214k
  QualType ResTy = BuildFunctionArgList(GD, Args);
1288
214k
1289
214k
  // Check if we should generate debug info for this function.
1290
214k
  if (FD->hasAttr<NoDebugAttr>())
1291
11.8k
    DebugInfo = nullptr; // disable debug info indefinitely for this function
1292
214k
1293
214k
  // The function might not have a body if we're generating thunks for a
1294
214k
  // function declaration.
1295
214k
  SourceRange BodyRange;
1296
214k
  if (Stmt *Body = FD->getBody())
1297
214k
    BodyRange = Body->getSourceRange();
1298
125
  else
1299
125
    BodyRange = FD->getLocation();
1300
214k
  CurEHLocation = BodyRange.getEnd();
1301
214k
1302
214k
  // Use the location of the start of the function to determine where
1303
214k
  // the function definition is located. By default use the location
1304
214k
  // of the declaration as the location for the subprogram. A function
1305
214k
  // may lack a declaration in the source code if it is created by code
1306
214k
  // gen. (examples: _GLOBAL__I_a, __cxx_global_array_dtor, thunk).
1307
214k
  SourceLocation Loc = FD->getLocation();
1308
214k
1309
214k
  // If this is a function specialization then use the pattern body
1310
214k
  // as the location for the function.
1311
214k
  if (const FunctionDecl *SpecDecl = FD->getTemplateInstantiationPattern())
1312
84.9k
    if (SpecDecl->hasBody(SpecDecl))
1313
84.8k
      Loc = SpecDecl->getLocation();
1314
214k
1315
214k
  Stmt *Body = FD->getBody();
1316
214k
1317
214k
  // Initialize helper which will detect jumps which can cause invalid lifetime
1318
214k
  // markers.
1319
214k
  if (Body && 
ShouldEmitLifetimeMarkers214k
)
1320
13.5k
    Bypasses.Init(Body);
1321
214k
1322
214k
  // Emit the standard function prologue.
1323
214k
  StartFunction(GD, ResTy, Fn, FnInfo, Args, Loc, BodyRange.getBegin());
1324
214k
1325
214k
  // Generate the body of the function.
1326
214k
  PGO.assignRegionCounters(GD, CurFn);
1327
214k
  if (isa<CXXDestructorDecl>(FD))
1328
13.7k
    EmitDestructorBody(Args);
1329
200k
  else if (isa<CXXConstructorDecl>(FD))
1330
35.2k
    EmitConstructorBody(Args);
1331
165k
  else if (getLangOpts().CUDA &&
1332
165k
           
!getLangOpts().CUDAIsDevice373
&&
1333
165k
           
FD->hasAttr<CUDAGlobalAttr>()117
)
1334
33
    CGM.getCUDARuntime().emitDeviceStub(*this, Args);
1335
165k
  else if (isa<CXXMethodDecl>(FD) &&
1336
165k
           
cast<CXXMethodDecl>(FD)->isLambdaStaticInvoker()46.5k
) {
1337
37
    // The lambda static invoker function is special, because it forwards or
1338
37
    // clones the body of the function call operator (but is actually static).
1339
37
    EmitLambdaStaticInvokeBody(cast<CXXMethodDecl>(FD));
1340
165k
  } else if (FD->isDefaulted() && 
isa<CXXMethodDecl>(FD)706
&&
1341
165k
             
(702
cast<CXXMethodDecl>(FD)->isCopyAssignmentOperator()702
||
1342
702
              
cast<CXXMethodDecl>(FD)->isMoveAssignmentOperator()202
)) {
1343
686
    // Implicit copy-assignment gets the same special treatment as implicit
1344
686
    // copy-constructors.
1345
686
    emitImplicitAssignmentOperatorBody(Args);
1346
164k
  } else if (Body) {
1347
164k
    EmitFunctionBody(Body);
1348
164k
  } else
1349
164k
    
llvm_unreachable0
("no definition for emitted function");
1350
214k
1351
214k
  // C++11 [stmt.return]p2:
1352
214k
  //   Flowing off the end of a function [...] results in undefined behavior in
1353
214k
  //   a value-returning function.
1354
214k
  // C11 6.9.1p12:
1355
214k
  //   If the '}' that terminates a function is reached, and the value of the
1356
214k
  //   function call is used by the caller, the behavior is undefined.
1357
214k
  if (getLangOpts().CPlusPlus && 
!FD->hasImplicitReturnZero()145k
&&
!SawAsmBlock142k
&&
1358
214k
      
!FD->getReturnType()->isVoidType()142k
&&
Builder.GetInsertBlock()54.7k
) {
1359
167
    bool ShouldEmitUnreachable =
1360
167
        CGM.getCodeGenOpts().StrictReturn ||
1361
167
        
shouldUseUndefinedBehaviorReturnOptimization(FD, getContext())21
;
1362
167
    if (SanOpts.has(SanitizerKind::Return)) {
1363
1
      SanitizerScope SanScope(this);
1364
1
      llvm::Value *IsFalse = Builder.getFalse();
1365
1
      EmitCheck(std::make_pair(IsFalse, SanitizerKind::Return),
1366
1
                SanitizerHandler::MissingReturn,
1367
1
                EmitCheckSourceLocation(FD->getLocation()), None);
1368
166
    } else if (ShouldEmitUnreachable) {
1369
148
      if (CGM.getCodeGenOpts().OptimizationLevel == 0)
1370
117
        EmitTrapCall(llvm::Intrinsic::trap);
1371
148
    }
1372
167
    if (SanOpts.has(SanitizerKind::Return) || 
ShouldEmitUnreachable166
) {
1373
149
      Builder.CreateUnreachable();
1374
149
      Builder.ClearInsertionPoint();
1375
149
    }
1376
167
  }
1377
214k
1378
214k
  // Emit the standard function epilogue.
1379
214k
  FinishFunction(BodyRange.getEnd());
1380
214k
1381
214k
  // If we haven't marked the function nothrow through other means, do
1382
214k
  // a quick pass now to see if we can.
1383
214k
  if (!CurFn->doesNotThrow())
1384
155k
    TryMarkNoThrow(CurFn);
1385
214k
}
1386
1387
/// ContainsLabel - Return true if the statement contains a label in it.  If
1388
/// this statement is not executed normally, it not containing a label means
1389
/// that we can just remove the code.
1390
38.6k
bool CodeGenFunction::ContainsLabel(const Stmt *S, bool IgnoreCaseStmts) {
1391
38.6k
  // Null statement, not a label!
1392
38.6k
  if (!S) 
return false83
;
1393
38.5k
1394
38.5k
  // If this is a label, we have to emit the code, consider something like:
1395
38.5k
  // if (0) {  ...  foo:  bar(); }  goto foo;
1396
38.5k
  //
1397
38.5k
  // TODO: If anyone cared, we could track __label__'s, since we know that you
1398
38.5k
  // can't jump to one from outside their declared region.
1399
38.5k
  if (isa<LabelStmt>(S))
1400
5
    return true;
1401
38.5k
1402
38.5k
  // If this is a case/default statement, and we haven't seen a switch, we have
1403
38.5k
  // to emit the code.
1404
38.5k
  if (isa<SwitchCase>(S) && 
!IgnoreCaseStmts68
)
1405
0
    return true;
1406
38.5k
1407
38.5k
  // If this is a switch statement, we want to ignore cases below it.
1408
38.5k
  if (isa<SwitchStmt>(S))
1409
10
    IgnoreCaseStmts = true;
1410
38.5k
1411
38.5k
  // Scan subexpressions for verboten labels.
1412
38.5k
  for (const Stmt *SubStmt : S->children())
1413
28.4k
    if (ContainsLabel(SubStmt, IgnoreCaseStmts))
1414
11
      return true;
1415
38.5k
1416
38.5k
  
return false38.5k
;
1417
38.5k
}
1418
1419
/// containsBreak - Return true if the statement contains a break out of it.
1420
/// If the statement (recursively) contains a switch or loop with a break
1421
/// inside of it, this is fine.
1422
952
bool CodeGenFunction::containsBreak(const Stmt *S) {
1423
952
  // Null statement, not a label!
1424
952
  if (!S) 
return false0
;
1425
952
1426
952
  // If this is a switch or loop that defines its own break scope, then we can
1427
952
  // include it and anything inside of it.
1428
952
  if (isa<SwitchStmt>(S) || isa<WhileStmt>(S) || isa<DoStmt>(S) ||
1429
952
      
isa<ForStmt>(S)949
)
1430
3
    return false;
1431
949
1432
949
  if (isa<BreakStmt>(S))
1433
1
    return true;
1434
948
1435
948
  // Scan subexpressions for verboten breaks.
1436
948
  for (const Stmt *SubStmt : S->children())
1437
870
    if (containsBreak(SubStmt))
1438
1
      return true;
1439
948
1440
948
  
return false947
;
1441
948
}
1442
1443
2.61k
bool CodeGenFunction::mightAddDeclToScope(const Stmt *S) {
1444
2.61k
  if (!S) 
return false0
;
1445
2.61k
1446
2.61k
  // Some statement kinds add a scope and thus never add a decl to the current
1447
2.61k
  // scope. Note, this list is longer than the list of statements that might
1448
2.61k
  // have an unscoped decl nested within them, but this way is conservatively
1449
2.61k
  // correct even if more statement kinds are added.
1450
2.61k
  if (isa<IfStmt>(S) || 
isa<SwitchStmt>(S)2.61k
||
isa<WhileStmt>(S)2.61k
||
1451
2.61k
      
isa<DoStmt>(S)2.61k
||
isa<ForStmt>(S)2.61k
||
isa<CompoundStmt>(S)2.61k
||
1452
2.61k
      
isa<CXXForRangeStmt>(S)2.60k
||
isa<CXXTryStmt>(S)2.60k
||
1453
2.61k
      
isa<ObjCForCollectionStmt>(S)2.60k
||
isa<ObjCAtTryStmt>(S)2.60k
)
1454
15
    return false;
1455
2.60k
1456
2.60k
  if (isa<DeclStmt>(S))
1457
7
    return true;
1458
2.59k
1459
2.59k
  for (const Stmt *SubStmt : S->children())
1460
2.34k
    if (mightAddDeclToScope(SubStmt))
1461
2
      return true;
1462
2.59k
1463
2.59k
  
return false2.59k
;
1464
2.59k
}
1465
1466
/// ConstantFoldsToSimpleInteger - If the specified expression does not fold
1467
/// to a constant, or if it does but contains a label, return false.  If it
1468
/// constant folds return true and set the boolean result in Result.
1469
bool CodeGenFunction::ConstantFoldsToSimpleInteger(const Expr *Cond,
1470
                                                   bool &ResultBool,
1471
94.9k
                                                   bool AllowLabels) {
1472
94.9k
  llvm::APSInt ResultInt;
1473
94.9k
  if (!ConstantFoldsToSimpleInteger(Cond, ResultInt, AllowLabels))
1474
85.5k
    return false;
1475
9.42k
1476
9.42k
  ResultBool = ResultInt.getBoolValue();
1477
9.42k
  return true;
1478
9.42k
}
1479
1480
/// ConstantFoldsToSimpleInteger - If the specified expression does not fold
1481
/// to a constant, or if it does but contains a label, return false.  If it
1482
/// constant folds return true and set the folded value.
1483
bool CodeGenFunction::ConstantFoldsToSimpleInteger(const Expr *Cond,
1484
                                                   llvm::APSInt &ResultInt,
1485
95.3k
                                                   bool AllowLabels) {
1486
95.3k
  // FIXME: Rename and handle conversion of other evaluatable things
1487
95.3k
  // to bool.
1488
95.3k
  Expr::EvalResult Result;
1489
95.3k
  if (!Cond->EvaluateAsInt(Result, getContext()))
1490
85.8k
    return false;  // Not foldable, not integer or not fully evaluatable.
1491
9.51k
1492
9.51k
  llvm::APSInt Int = Result.Val.getInt();
1493
9.51k
  if (!AllowLabels && 
CodeGenFunction::ContainsLabel(Cond)9.51k
)
1494
0
    return false;  // Contains a label.
1495
9.51k
1496
9.51k
  ResultInt = Int;
1497
9.51k
  return true;
1498
9.51k
}
1499
1500
1501
1502
/// EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g. for an if
1503
/// statement) to the specified blocks.  Based on the condition, this might try
1504
/// to simplify the codegen of the conditional based on the branch.
1505
///
1506
void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond,
1507
                                           llvm::BasicBlock *TrueBlock,
1508
                                           llvm::BasicBlock *FalseBlock,
1509
96.3k
                                           uint64_t TrueCount) {
1510
96.3k
  Cond = Cond->IgnoreParens();
1511
96.3k
1512
96.3k
  if (const BinaryOperator *CondBOp = dyn_cast<BinaryOperator>(Cond)) {
1513
46.5k
1514
46.5k
    // Handle X && Y in a condition.
1515
46.5k
    if (CondBOp->getOpcode() == BO_LAnd) {
1516
2.92k
      // If we have "1 && X", simplify the code.  "0 && X" would have constant
1517
2.92k
      // folded if the case was simple enough.
1518
2.92k
      bool ConstantBool = false;
1519
2.92k
      if (ConstantFoldsToSimpleInteger(CondBOp->getLHS(), ConstantBool) &&
1520
2.92k
          
ConstantBool58
) {
1521
58
        // br(1 && X) -> br(X).
1522
58
        incrementProfileCounter(CondBOp);
1523
58
        return EmitBranchOnBoolExpr(CondBOp->getRHS(), TrueBlock, FalseBlock,
1524
58
                                    TrueCount);
1525
58
      }
1526
2.87k
1527
2.87k
      // If we have "X && 1", simplify the code to use an uncond branch.
1528
2.87k
      // "X && 0" would have been constant folded to 0.
1529
2.87k
      if (ConstantFoldsToSimpleInteger(CondBOp->getRHS(), ConstantBool) &&
1530
2.87k
          
ConstantBool14
) {
1531
14
        // br(X && 1) -> br(X).
1532
14
        return EmitBranchOnBoolExpr(CondBOp->getLHS(), TrueBlock, FalseBlock,
1533
14
                                    TrueCount);
1534
14
      }
1535
2.85k
1536
2.85k
      // Emit the LHS as a conditional.  If the LHS conditional is false, we
1537
2.85k
      // want to jump to the FalseBlock.
1538
2.85k
      llvm::BasicBlock *LHSTrue = createBasicBlock("land.lhs.true");
1539
2.85k
      // The counter tells us how often we evaluate RHS, and all of TrueCount
1540
2.85k
      // can be propagated to that branch.
1541
2.85k
      uint64_t RHSCount = getProfileCount(CondBOp->getRHS());
1542
2.85k
1543
2.85k
      ConditionalEvaluation eval(*this);
1544
2.85k
      {
1545
2.85k
        ApplyDebugLocation DL(*this, Cond);
1546
2.85k
        EmitBranchOnBoolExpr(CondBOp->getLHS(), LHSTrue, FalseBlock, RHSCount);
1547
2.85k
        EmitBlock(LHSTrue);
1548
2.85k
      }
1549
2.85k
1550
2.85k
      incrementProfileCounter(CondBOp);
1551
2.85k
      setCurrentProfileCount(getProfileCount(CondBOp->getRHS()));
1552
2.85k
1553
2.85k
      // Any temporaries created here are conditional.
1554
2.85k
      eval.begin(*this);
1555
2.85k
      EmitBranchOnBoolExpr(CondBOp->getRHS(), TrueBlock, FalseBlock, TrueCount);
1556
2.85k
      eval.end(*this);
1557
2.85k
1558
2.85k
      return;
1559
2.85k
    }
1560
43.5k
1561
43.5k
    if (CondBOp->getOpcode() == BO_LOr) {
1562
3.45k
      // If we have "0 || X", simplify the code.  "1 || X" would have constant
1563
3.45k
      // folded if the case was simple enough.
1564
3.45k
      bool ConstantBool = false;
1565
3.45k
      if (ConstantFoldsToSimpleInteger(CondBOp->getLHS(), ConstantBool) &&
1566
3.45k
          
!ConstantBool5
) {
1567
5
        // br(0 || X) -> br(X).
1568
5
        incrementProfileCounter(CondBOp);
1569
5
        return EmitBranchOnBoolExpr(CondBOp->getRHS(), TrueBlock, FalseBlock,
1570
5
                                    TrueCount);
1571
5
      }
1572
3.44k
1573
3.44k
      // If we have "X || 0", simplify the code to use an uncond branch.
1574
3.44k
      // "X || 1" would have been constant folded to 1.
1575
3.44k
      if (ConstantFoldsToSimpleInteger(CondBOp->getRHS(), ConstantBool) &&
1576
3.44k
          
!ConstantBool0
) {
1577
0
        // br(X || 0) -> br(X).
1578
0
        return EmitBranchOnBoolExpr(CondBOp->getLHS(), TrueBlock, FalseBlock,
1579
0
                                    TrueCount);
1580
0
      }
1581
3.44k
1582
3.44k
      // Emit the LHS as a conditional.  If the LHS conditional is true, we
1583
3.44k
      // want to jump to the TrueBlock.
1584
3.44k
      llvm::BasicBlock *LHSFalse = createBasicBlock("lor.lhs.false");
1585
3.44k
      // We have the count for entry to the RHS and for the whole expression
1586
3.44k
      // being true, so we can divy up True count between the short circuit and
1587
3.44k
      // the RHS.
1588
3.44k
      uint64_t LHSCount =
1589
3.44k
          getCurrentProfileCount() - getProfileCount(CondBOp->getRHS());
1590
3.44k
      uint64_t RHSCount = TrueCount - LHSCount;
1591
3.44k
1592
3.44k
      ConditionalEvaluation eval(*this);
1593
3.44k
      {
1594
3.44k
        ApplyDebugLocation DL(*this, Cond);
1595
3.44k
        EmitBranchOnBoolExpr(CondBOp->getLHS(), TrueBlock, LHSFalse, LHSCount);
1596
3.44k
        EmitBlock(LHSFalse);
1597
3.44k
      }
1598
3.44k
1599
3.44k
      incrementProfileCounter(CondBOp);
1600
3.44k
      setCurrentProfileCount(getProfileCount(CondBOp->getRHS()));
1601
3.44k
1602
3.44k
      // Any temporaries created here are conditional.
1603
3.44k
      eval.begin(*this);
1604
3.44k
      EmitBranchOnBoolExpr(CondBOp->getRHS(), TrueBlock, FalseBlock, RHSCount);
1605
3.44k
1606
3.44k
      eval.end(*this);
1607
3.44k
1608
3.44k
      return;
1609
3.44k
    }
1610
43.5k
  }
1611
90.0k
1612
90.0k
  if (const UnaryOperator *CondUOp = dyn_cast<UnaryOperator>(Cond)) {
1613
1.13k
    // br(!x, t, f) -> br(x, f, t)
1614
1.13k
    if (CondUOp->getOpcode() == UO_LNot) {
1615
1.13k
      // Negate the count.
1616
1.13k
      uint64_t FalseCount = getCurrentProfileCount() - TrueCount;
1617
1.13k
      // Negate the condition and swap the destination blocks.
1618
1.13k
      return EmitBranchOnBoolExpr(CondUOp->getSubExpr(), FalseBlock, TrueBlock,
1619
1.13k
                                  FalseCount);
1620
1.13k
    }
1621
88.8k
  }
1622
88.8k
1623
88.8k
  if (const ConditionalOperator *CondOp = dyn_cast<ConditionalOperator>(Cond)) {
1624
0
    // br(c ? x : y, t, f) -> br(c, br(x, t, f), br(y, t, f))
1625
0
    llvm::BasicBlock *LHSBlock = createBasicBlock("cond.true");
1626
0
    llvm::BasicBlock *RHSBlock = createBasicBlock("cond.false");
1627
0
1628
0
    ConditionalEvaluation cond(*this);
1629
0
    EmitBranchOnBoolExpr(CondOp->getCond(), LHSBlock, RHSBlock,
1630
0
                         getProfileCount(CondOp));
1631
0
1632
0
    // When computing PGO branch weights, we only know the overall count for
1633
0
    // the true block. This code is essentially doing tail duplication of the
1634
0
    // naive code-gen, introducing new edges for which counts are not
1635
0
    // available. Divide the counts proportionally between the LHS and RHS of
1636
0
    // the conditional operator.
1637
0
    uint64_t LHSScaledTrueCount = 0;
1638
0
    if (TrueCount) {
1639
0
      double LHSRatio =
1640
0
          getProfileCount(CondOp) / (double)getCurrentProfileCount();
1641
0
      LHSScaledTrueCount = TrueCount * LHSRatio;
1642
0
    }
1643
0
1644
0
    cond.begin(*this);
1645
0
    EmitBlock(LHSBlock);
1646
0
    incrementProfileCounter(CondOp);
1647
0
    {
1648
0
      ApplyDebugLocation DL(*this, Cond);
1649
0
      EmitBranchOnBoolExpr(CondOp->getLHS(), TrueBlock, FalseBlock,
1650
0
                           LHSScaledTrueCount);
1651
0
    }
1652
0
    cond.end(*this);
1653
0
1654
0
    cond.begin(*this);
1655
0
    EmitBlock(RHSBlock);
1656
0
    EmitBranchOnBoolExpr(CondOp->getRHS(), TrueBlock, FalseBlock,
1657
0
                         TrueCount - LHSScaledTrueCount);
1658
0
    cond.end(*this);
1659
0
1660
0
    return;
1661
0
  }
1662
88.8k
1663
88.8k
  if (const CXXThrowExpr *Throw = dyn_cast<CXXThrowExpr>(Cond)) {
1664
0
    // Conditional operator handling can give us a throw expression as a
1665
0
    // condition for a case like:
1666
0
    //   br(c ? throw x : y, t, f) -> br(c, br(throw x, t, f), br(y, t, f)
1667
0
    // Fold this to:
1668
0
    //   br(c, throw x, br(y, t, f))
1669
0
    EmitCXXThrowExpr(Throw, /*KeepInsertionPoint*/false);
1670
0
    return;
1671
0
  }
1672
88.8k
1673
88.8k
  // If the branch has a condition wrapped by __builtin_unpredictable,
1674
88.8k
  // create metadata that specifies that the branch is unpredictable.
1675
88.8k
  // Don't bother if not optimizing because that metadata would not be used.
1676
88.8k
  llvm::MDNode *Unpredictable = nullptr;
1677
88.8k
  auto *Call = dyn_cast<CallExpr>(Cond->IgnoreImpCasts());
1678
88.8k
  if (Call && 
CGM.getCodeGenOpts().OptimizationLevel != 04.77k
) {
1679
52
    auto *FD = dyn_cast_or_null<FunctionDecl>(Call->getCalleeDecl());
1680
52
    if (FD && FD->getBuiltinID() == Builtin::BI__builtin_unpredictable) {
1681
3
      llvm::MDBuilder MDHelper(getLLVMContext());
1682
3
      Unpredictable = MDHelper.createUnpredictable();
1683
3
    }
1684
52
  }
1685
88.8k
1686
88.8k
  // Create branch weights based on the number of times we get here and the
1687
88.8k
  // number of times the condition should be true.
1688
88.8k
  uint64_t CurrentCount = std::max(getCurrentProfileCount(), TrueCount);
1689
88.8k
  llvm::MDNode *Weights =
1690
88.8k
      createProfileWeights(TrueCount, CurrentCount - TrueCount);
1691
88.8k
1692
88.8k
  // Emit the code with the fully general case.
1693
88.8k
  llvm::Value *CondV;
1694
88.8k
  {
1695
88.8k
    ApplyDebugLocation DL(*this, Cond);
1696
88.8k
    CondV = EvaluateExprAsBool(Cond);
1697
88.8k
  }
1698
88.8k
  Builder.CreateCondBr(CondV, TrueBlock, FalseBlock, Weights, Unpredictable);
1699
88.8k
}
1700
1701
/// ErrorUnsupported - Print out an error that codegen doesn't support the
1702
/// specified stmt yet.
1703
1
void CodeGenFunction::ErrorUnsupported(const Stmt *S, const char *Type) {
1704
1
  CGM.ErrorUnsupported(S, Type);
1705
1
}
1706
1707
/// emitNonZeroVLAInit - Emit the "zero" initialization of a
1708
/// variable-length array whose elements have a non-zero bit-pattern.
1709
///
1710
/// \param baseType the inner-most element type of the array
1711
/// \param src - a char* pointing to the bit-pattern for a single
1712
/// base element of the array
1713
/// \param sizeInChars - the total size of the VLA, in chars
1714
static void emitNonZeroVLAInit(CodeGenFunction &CGF, QualType baseType,
1715
                               Address dest, Address src,
1716
0
                               llvm::Value *sizeInChars) {
1717
0
  CGBuilderTy &Builder = CGF.Builder;
1718
0
1719
0
  CharUnits baseSize = CGF.getContext().getTypeSizeInChars(baseType);
1720
0
  llvm::Value *baseSizeInChars
1721
0
    = llvm::ConstantInt::get(CGF.IntPtrTy, baseSize.getQuantity());
1722
0
1723
0
  Address begin =
1724
0
    Builder.CreateElementBitCast(dest, CGF.Int8Ty, "vla.begin");
1725
0
  llvm::Value *end =
1726
0
    Builder.CreateInBoundsGEP(begin.getPointer(), sizeInChars, "vla.end");
1727
0
1728
0
  llvm::BasicBlock *originBB = CGF.Builder.GetInsertBlock();
1729
0
  llvm::BasicBlock *loopBB = CGF.createBasicBlock("vla-init.loop");
1730
0
  llvm::BasicBlock *contBB = CGF.createBasicBlock("vla-init.cont");
1731
0
1732
0
  // Make a loop over the VLA.  C99 guarantees that the VLA element
1733
0
  // count must be nonzero.
1734
0
  CGF.EmitBlock(loopBB);
1735
0
1736
0
  llvm::PHINode *cur = Builder.CreatePHI(begin.getType(), 2, "vla.cur");
1737
0
  cur->addIncoming(begin.getPointer(), originBB);
1738
0
1739
0
  CharUnits curAlign =
1740
0
    dest.getAlignment().alignmentOfArrayElement(baseSize);
1741
0
1742
0
  // memcpy the individual element bit-pattern.
1743
0
  Builder.CreateMemCpy(Address(cur, curAlign), src, baseSizeInChars,
1744
0
                       /*volatile*/ false);
1745
0
1746
0
  // Go to the next element.
1747
0
  llvm::Value *next =
1748
0
    Builder.CreateInBoundsGEP(CGF.Int8Ty, cur, baseSizeInChars, "vla.next");
1749
0
1750
0
  // Leave if that's the end of the VLA.
1751
0
  llvm::Value *done = Builder.CreateICmpEQ(next, end, "vla-init.isdone");
1752
0
  Builder.CreateCondBr(done, contBB, loopBB);
1753
0
  cur->addIncoming(next, loopBB);
1754
0
1755
0
  CGF.EmitBlock(contBB);
1756
0
}
1757
1758
void
1759
7.50k
CodeGenFunction::EmitNullInitialization(Address DestPtr, QualType Ty) {
1760
7.50k
  // Ignore empty classes in C++.
1761
7.50k
  if (getLangOpts().CPlusPlus) {
1762
7.35k
    if (const RecordType *RT = Ty->getAs<RecordType>()) {
1763
7.05k
      if (cast<CXXRecordDecl>(RT->getDecl())->isEmpty())
1764
6.35k
        return;
1765
1.14k
    }
1766
7.35k
  }
1767
1.14k
1768
1.14k
  // Cast the dest ptr to the appropriate i8 pointer type.
1769
1.14k
  if (DestPtr.getElementType() != Int8Ty)
1770
1.14k
    DestPtr = Builder.CreateElementBitCast(DestPtr, Int8Ty);
1771
1.14k
1772
1.14k
  // Get size and alignment info for this aggregate.
1773
1.14k
  CharUnits size = getContext().getTypeSizeInChars(Ty);
1774
1.14k
1775
1.14k
  llvm::Value *SizeVal;
1776
1.14k
  const VariableArrayType *vla;
1777
1.14k
1778
1.14k
  // Don't bother emitting a zero-byte memset.
1779
1.14k
  if (size.isZero()) {
1780
11
    // But note that getTypeInfo returns 0 for a VLA.
1781
11
    if (const VariableArrayType *vlaType =
1782
8
          dyn_cast_or_null<VariableArrayType>(
1783
8
                                          getContext().getAsArrayType(Ty))) {
1784
8
      auto VlaSize = getVLASize(vlaType);
1785
8
      SizeVal = VlaSize.NumElts;
1786
8
      CharUnits eltSize = getContext().getTypeSizeInChars(VlaSize.Type);
1787
8
      if (!eltSize.isOne())
1788
8
        SizeVal = Builder.CreateNUWMul(SizeVal, CGM.getSize(eltSize));
1789
8
      vla = vlaType;
1790
8
    } else {
1791
3
      return;
1792
3
    }
1793
1.13k
  } else {
1794
1.13k
    SizeVal = CGM.getSize(size);
1795
1.13k
    vla = nullptr;
1796
1.13k
  }
1797
1.14k
1798
1.14k
  // If the type contains a pointer to data member we can't memset it to zero.
1799
1.14k
  // Instead, create a null constant and copy it to the destination.
1800
1.14k
  // TODO: there are other patterns besides zero that we can usefully memset,
1801
1.14k
  // like -1, which happens to be the pattern used by member-pointers.
1802
1.14k
  
if (1.14k
!CGM.getTypes().isZeroInitializable(Ty)1.14k
) {
1803
9
    // For a VLA, emit a single element, then splat that over the VLA.
1804
9
    if (vla) 
Ty = getContext().getBaseElementType(vla)0
;
1805
9
1806
9
    llvm::Constant *NullConstant = CGM.EmitNullConstant(Ty);
1807
9
1808
9
    llvm::GlobalVariable *NullVariable =
1809
9
      new llvm::GlobalVariable(CGM.getModule(), NullConstant->getType(),
1810
9
                               /*isConstant=*/true,
1811
9
                               llvm::GlobalVariable::PrivateLinkage,
1812
9
                               NullConstant, Twine());
1813
9
    CharUnits NullAlign = DestPtr.getAlignment();
1814
9
    NullVariable->setAlignment(NullAlign.getAsAlign());
1815
9
    Address SrcPtr(Builder.CreateBitCast(NullVariable, Builder.getInt8PtrTy()),
1816
9
                   NullAlign);
1817
9
1818
9
    if (vla) 
return emitNonZeroVLAInit(*this, Ty, DestPtr, SrcPtr, SizeVal)0
;
1819
9
1820
9
    // Get and call the appropriate llvm.memcpy overload.
1821
9
    Builder.CreateMemCpy(DestPtr, SrcPtr, SizeVal, false);
1822
9
    return;
1823
9
  }
1824
1.13k
1825
1.13k
  // Otherwise, just memset the whole thing to zero.  This is legal
1826
1.13k
  // because in LLVM, all default initializers (other than the ones we just
1827
1.13k
  // handled above) are guaranteed to have a bit pattern of all zeros.
1828
1.13k
  Builder.CreateMemSet(DestPtr, Builder.getInt8(0), SizeVal, false);
1829
1.13k
}
1830
1831
57
llvm::BlockAddress *CodeGenFunction::GetAddrOfLabel(const LabelDecl *L) {
1832
57
  // Make sure that there is a block for the indirect goto.
1833
57
  if (!IndirectBranch)
1834
34
    GetIndirectGotoBlock();
1835
57
1836
57
  llvm::BasicBlock *BB = getJumpDestForLabel(L).getBlock();
1837
57
1838
57
  // Make sure the indirect branch includes all of the address-taken blocks.
1839
57
  IndirectBranch->addDestination(BB);
1840
57
  return llvm::BlockAddress::get(CurFn, BB);
1841
57
}
1842
1843
56
llvm::BasicBlock *CodeGenFunction::GetIndirectGotoBlock() {
1844
56
  // If we already made the indirect branch for indirect goto, return its block.
1845
56
  if (IndirectBranch) 
return IndirectBranch->getParent()21
;
1846
35
1847
35
  CGBuilderTy TmpBuilder(*this, createBasicBlock("indirectgoto"));
1848
35
1849
35
  // Create the PHI node that indirect gotos will add entries to.
1850
35
  llvm::Value *DestVal = TmpBuilder.CreatePHI(Int8PtrTy, 0,
1851
35
                                              "indirect.goto.dest");
1852
35
1853
35
  // Create the indirect branch instruction.
1854
35
  IndirectBranch = TmpBuilder.CreateIndirectBr(DestVal);
1855
35
  return IndirectBranch->getParent();
1856
35
}
1857
1858
/// Computes the length of an array in elements, as well as the base
1859
/// element type and a properly-typed first element pointer.
1860
llvm::Value *CodeGenFunction::emitArrayLength(const ArrayType *origArrayType,
1861
                                              QualType &baseType,
1862
4.42k
                                              Address &addr) {
1863
4.42k
  const ArrayType *arrayType = origArrayType;
1864
4.42k
1865
4.42k
  // If it's a VLA, we have to load the stored size.  Note that
1866
4.42k
  // this is the size of the VLA in bytes, not its size in elements.
1867
4.42k
  llvm::Value *numVLAElements = nullptr;
1868
4.42k
  if (isa<VariableArrayType>(arrayType)) {
1869
344
    numVLAElements = getVLASize(cast<VariableArrayType>(arrayType)).NumElts;
1870
344
1871
344
    // Walk into all VLAs.  This doesn't require changes to addr,
1872
344
    // which has type T* where T is the first non-VLA element type.
1873
381
    do {
1874
381
      QualType elementType = arrayType->getElementType();
1875
381
      arrayType = getContext().getAsArrayType(elementType);
1876
381
1877
381
      // If we only have VLA components, 'addr' requires no adjustment.
1878
381
      if (!arrayType) {
1879
340
        baseType = elementType;
1880
340
        return numVLAElements;
1881
340
      }
1882
41
    } while (isa<VariableArrayType>(arrayType));
1883
344
1884
344
    // We get out here only if we find a constant array type
1885
344
    // inside the VLA.
1886
344
  }
1887
4.42k
1888
4.42k
  // We have some number of constant-length arrays, so addr should
1889
4.42k
  // have LLVM type [M x [N x [...]]]*.  Build a GEP that walks
1890
4.42k
  // down to the first element of addr.
1891
4.42k
  SmallVector<llvm::Value*, 8> gepIndices;
1892
4.08k
1893
4.08k
  // GEP down to the array type.
1894
4.08k
  llvm::ConstantInt *zero = Builder.getInt32(0);
1895
4.08k
  gepIndices.push_back(zero);
1896
4.08k
1897
4.08k
  uint64_t countFromCLAs = 1;
1898
4.08k
  QualType eltType;
1899
4.08k
1900
4.08k
  llvm::ArrayType *llvmArrayType =
1901
4.08k
    dyn_cast<llvm::ArrayType>(addr.getElementType());
1902
7.96k
  while (llvmArrayType) {
1903
3.88k
    assert(isa<ConstantArrayType>(arrayType));
1904
3.88k
    assert(cast<ConstantArrayType>(arrayType)->getSize().getZExtValue()
1905
3.88k
             == llvmArrayType->getNumElements());
1906
3.88k
1907
3.88k
    gepIndices.push_back(zero);
1908
3.88k
    countFromCLAs *= llvmArrayType->getNumElements();
1909
3.88k
    eltType = arrayType->getElementType();
1910
3.88k
1911
3.88k
    llvmArrayType =
1912
3.88k
      dyn_cast<llvm::ArrayType>(llvmArrayType->getElementType());
1913
3.88k
    arrayType = getContext().getAsArrayType(arrayType->getElementType());
1914
3.88k
    assert((!llvmArrayType || arrayType) &&
1915
3.88k
           "LLVM and Clang types are out-of-synch");
1916
3.88k
  }
1917
4.08k
1918
4.08k
  if (arrayType) {
1919
293
    // From this point onwards, the Clang array type has been emitted
1920
293
    // as some other type (probably a packed struct). Compute the array
1921
293
    // size, and just emit the 'begin' expression as a bitcast.
1922
626
    while (arrayType) {
1923
333
      countFromCLAs *=
1924
333
          cast<ConstantArrayType>(arrayType)->getSize().getZExtValue();
1925
333
      eltType = arrayType->getElementType();
1926
333
      arrayType = getContext().getAsArrayType(eltType);
1927
333
    }
1928
293
1929
293
    llvm::Type *baseType = ConvertType(eltType);
1930
293
    addr = Builder.CreateElementBitCast(addr, baseType, "array.begin");
1931
3.79k
  } else {
1932
3.79k
    // Create the actual GEP.
1933
3.79k
    addr = Address(Builder.CreateInBoundsGEP(addr.getPointer(),
1934
3.79k
                                             gepIndices, "array.begin"),
1935
3.79k
                   addr.getAlignment());
1936
3.79k
  }
1937
4.08k
1938
4.08k
  baseType = eltType;
1939
4.08k
1940
4.08k
  llvm::Value *numElements
1941
4.08k
    = llvm::ConstantInt::get(SizeTy, countFromCLAs);
1942
4.08k
1943
4.08k
  // If we had any VLA dimensions, factor them in.
1944
4.08k
  if (numVLAElements)
1945
4
    numElements = Builder.CreateNUWMul(numVLAElements, numElements);
1946
4.08k
1947
4.08k
  return numElements;
1948
4.42k
}
1949
1950
1.66k
CodeGenFunction::VlaSizePair CodeGenFunction::getVLASize(QualType type) {
1951
1.66k
  const VariableArrayType *vla = getContext().getAsVariableArrayType(type);
1952
1.66k
  assert(vla && "type was not a variable array type!");
1953
1.66k
  return getVLASize(vla);
1954
1.66k
}
1955
1956
CodeGenFunction::VlaSizePair
1957
4.09k
CodeGenFunction::getVLASize(const VariableArrayType *type) {
1958
4.09k
  // The number of elements so far; always size_t.
1959
4.09k
  llvm::Value *numElements = nullptr;
1960
4.09k
1961
4.09k
  QualType elementType;
1962
5.18k
  do {
1963
5.18k
    elementType = type->getElementType();
1964
5.18k
    llvm::Value *vlaSize = VLASizeMap[type->getSizeExpr()];
1965
5.18k
    assert(vlaSize && "no size for VLA!");
1966
5.18k
    assert(vlaSize->getType() == SizeTy);
1967
5.18k
1968
5.18k
    if (!numElements) {
1969
4.09k
      numElements = vlaSize;
1970
4.09k
    } else {
1971
1.09k
      // It's undefined behavior if this wraps around, so mark it that way.
1972
1.09k
      // FIXME: Teach -fsanitize=undefined to trap this.
1973
1.09k
      numElements = Builder.CreateNUWMul(numElements, vlaSize);
1974
1.09k
    }
1975
5.18k
  } while ((type = getContext().getAsVariableArrayType(elementType)));
1976
4.09k
1977
4.09k
  return { numElements, elementType };
1978
4.09k
}
1979
1980
CodeGenFunction::VlaSizePair
1981
2.45k
CodeGenFunction::getVLAElements1D(QualType type) {
1982
2.45k
  const VariableArrayType *vla = getContext().getAsVariableArrayType(type);
1983
2.45k
  assert(vla && "type was not a variable array type!");
1984
2.45k
  return getVLAElements1D(vla);
1985
2.45k
}
1986
1987
CodeGenFunction::VlaSizePair
1988
2.45k
CodeGenFunction::getVLAElements1D(const VariableArrayType *Vla) {
1989
2.45k
  llvm::Value *VlaSize = VLASizeMap[Vla->getSizeExpr()];
1990
2.45k
  assert(VlaSize && "no size for VLA!");
1991
2.45k
  assert(VlaSize->getType() == SizeTy);
1992
2.45k
  return { VlaSize, Vla->getElementType() };
1993
2.45k
}
1994
1995
2.10k
void CodeGenFunction::EmitVariablyModifiedType(QualType type) {
1996
2.10k
  assert(type->isVariablyModifiedType() &&
1997
2.10k
         "Must pass variably modified type to EmitVLASizes!");
1998
2.10k
1999
2.10k
  EnsureInsertPoint();
2000
2.10k
2001
2.10k
  // We're going to walk down into the type and look for VLA
2002
2.10k
  // expressions.
2003
3.07k
  do {
2004
3.07k
    assert(type->isVariablyModifiedType());
2005
3.07k
2006
3.07k
    const Type *ty = type.getTypePtr();
2007
3.07k
    switch (ty->getTypeClass()) {
2008
0
2009
0
#define TYPE(Class, Base)
2010
0
#define ABSTRACT_TYPE(Class, Base)
2011
0
#define NON_CANONICAL_TYPE(Class, Base)
2012
0
#define DEPENDENT_TYPE(Class, Base) case Type::Class:
2013
0
#define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(Class, Base)
2014
0
#include "clang/AST/TypeNodes.inc"
2015
0
      llvm_unreachable("unexpected dependent type!");
2016
0
2017
0
    // These types are never variably-modified.
2018
0
    case Type::Builtin:
2019
0
    case Type::Complex:
2020
0
    case Type::Vector:
2021
0
    case Type::ExtVector:
2022
0
    case Type::Record:
2023
0
    case Type::Enum:
2024
0
    case Type::Elaborated:
2025
0
    case Type::TemplateSpecialization:
2026
0
    case Type::ObjCTypeParam:
2027
0
    case Type::ObjCObject:
2028
0
    case Type::ObjCInterface:
2029
0
    case Type::ObjCObjectPointer:
2030
0
      llvm_unreachable("type class is never variably-modified!");
2031
0
2032
0
    case Type::Adjusted:
2033
0
      type = cast<AdjustedType>(ty)->getAdjustedType();
2034
0
      break;
2035
0
2036
50
    case Type::Decayed:
2037
50
      type = cast<DecayedType>(ty)->getPointeeType();
2038
50
      break;
2039
0
2040
90
    case Type::Pointer:
2041
90
      type = cast<PointerType>(ty)->getPointeeType();
2042
90
      break;
2043
0
2044
0
    case Type::BlockPointer:
2045
0
      type = cast<BlockPointerType>(ty)->getPointeeType();
2046
0
      break;
2047
0
2048
3
    case Type::LValueReference:
2049
3
    case Type::RValueReference:
2050
3
      type = cast<ReferenceType>(ty)->getPointeeType();
2051
3
      break;
2052
3
2053
3
    case Type::MemberPointer:
2054
0
      type = cast<MemberPointerType>(ty)->getPointeeType();
2055
0
      break;
2056
3
2057
12
    case Type::ConstantArray:
2058
12
    case Type::IncompleteArray:
2059
12
      // Losing element qualification here is fine.
2060
12
      type = cast<ArrayType>(ty)->getElementType();
2061
12
      break;
2062
12
2063
2.82k
    case Type::VariableArray: {
2064
2.82k
      // Losing element qualification here is fine.
2065
2.82k
      const VariableArrayType *vat = cast<VariableArrayType>(ty);
2066
2.82k
2067
2.82k
      // Unknown size indication requires no size computation.
2068
2.82k
      // Otherwise, evaluate and record it.
2069
2.82k
      if (const Expr *size = vat->getSizeExpr()) {
2070
2.82k
        // It's possible that we might have emitted this already,
2071
2.82k
        // e.g. with a typedef and a pointer to it.
2072
2.82k
        llvm::Value *&entry = VLASizeMap[size];
2073
2.82k
        if (!entry) {
2074
2.62k
          llvm::Value *Size = EmitScalarExpr(size);
2075
2.62k
2076
2.62k
          // C11 6.7.6.2p5:
2077
2.62k
          //   If the size is an expression that is not an integer constant
2078
2.62k
          //   expression [...] each time it is evaluated it shall have a value
2079
2.62k
          //   greater than zero.
2080
2.62k
          if (SanOpts.has(SanitizerKind::VLABound) &&
2081
2.62k
              
size->getType()->isSignedIntegerType()2
) {
2082
2
            SanitizerScope SanScope(this);
2083
2
            llvm::Value *Zero = llvm::Constant::getNullValue(Size->getType());
2084
2
            llvm::Constant *StaticArgs[] = {
2085
2
                EmitCheckSourceLocation(size->getBeginLoc()),
2086
2
                EmitCheckTypeDescriptor(size->getType())};
2087
2
            EmitCheck(std::make_pair(Builder.CreateICmpSGT(Size, Zero),
2088
2
                                     SanitizerKind::VLABound),
2089
2
                      SanitizerHandler::VLABoundNotPositive, StaticArgs, Size);
2090
2
          }
2091
2.62k
2092
2.62k
          // Always zexting here would be wrong if it weren't
2093
2.62k
          // undefined behavior to have a negative bound.
2094
2.62k
          entry = Builder.CreateIntCast(Size, SizeTy, /*signed*/ false);
2095
2.62k
        }
2096
2.82k
      }
2097
2.82k
      type = vat->getElementType();
2098
2.82k
      break;
2099
12
    }
2100
12
2101
12
    case Type::FunctionProto:
2102
1
    case Type::FunctionNoProto:
2103
1
      type = cast<FunctionType>(ty)->getReturnType();
2104
1
      break;
2105
1
2106
44
    case Type::Paren:
2107
44
    case Type::TypeOf:
2108
44
    case Type::UnaryTransform:
2109
44
    case Type::Attributed:
2110
44
    case Type::SubstTemplateTypeParm:
2111
44
    case Type::PackExpansion:
2112
44
    case Type::MacroQualified:
2113
44
      // Keep walking after single level desugaring.
2114
44
      type = type.getSingleStepDesugaredType(getContext());
2115
44
      break;
2116
44
2117
44
    case Type::Typedef:
2118
28
    case Type::Decltype:
2119
28
    case Type::Auto:
2120
28
    case Type::DeducedTemplateSpecialization:
2121
28
      // Stop walking: nothing to do.
2122
28
      return;
2123
28
2124
28
    case Type::TypeOfExpr:
2125
13
      // Stop walking: emit typeof expression.
2126
13
      EmitIgnoredExpr(cast<TypeOfExprType>(ty)->getUnderlyingExpr());
2127
13
      return;
2128
28
2129
28
    case Type::Atomic:
2130
1
      type = cast<AtomicType>(ty)->getValueType();
2131
1
      break;
2132
28
2133
28
    case Type::Pipe:
2134
0
      type = cast<PipeType>(ty)->getElementType();
2135
0
      break;
2136
3.03k
    }
2137
3.03k
  } while (type->isVariablyModifiedType());
2138
2.10k
}
2139
2140
1.08k
Address CodeGenFunction::EmitVAListRef(const Expr* E) {
2141
1.08k
  if (getContext().getBuiltinVaListType()->isArrayType())
2142
464
    return EmitPointerWithAlignment(E);
2143
623
  return EmitLValue(E).getAddress(*this);
2144
623
}
2145
2146
29
Address CodeGenFunction::EmitMSVAListRef(const Expr *E) {
2147
29
  return EmitLValue(E).getAddress(*this);
2148
29
}
2149
2150
void CodeGenFunction::EmitDeclRefExprDbgValue(const DeclRefExpr *E,
2151
7.44k
                                              const APValue &Init) {
2152
7.44k
  assert(Init.hasValue() && "Invalid DeclRefExpr initializer!");
2153
7.44k
  if (CGDebugInfo *Dbg = getDebugInfo())
2154
5.04k
    if (CGM.getCodeGenOpts().hasReducedDebugInfo())
2155
5.04k
      Dbg->EmitGlobalVariable(E->getDecl(), Init);
2156
7.44k
}
2157
2158
CodeGenFunction::PeepholeProtection
2159
1.03k
CodeGenFunction::protectFromPeepholes(RValue rvalue) {
2160
1.03k
  // At the moment, the only aggressive peephole we do in IR gen
2161
1.03k
  // is trunc(zext) folding, but if we add more, we can easily
2162
1.03k
  // extend this protection.
2163
1.03k
2164
1.03k
  if (!rvalue.isScalar()) 
return PeepholeProtection()55
;
2165
984
  llvm::Value *value = rvalue.getScalarVal();
2166
984
  if (!isa<llvm::ZExtInst>(value)) 
return PeepholeProtection()965
;
2167
19
2168
19
  // Just make an extra bitcast.
2169
19
  assert(HaveInsertPoint());
2170
19
  llvm::Instruction *inst = new llvm::BitCastInst(value, value->getType(), "",
2171
19
                                                  Builder.GetInsertBlock());
2172
19
2173
19
  PeepholeProtection protection;
2174
19
  protection.Inst = inst;
2175
19
  return protection;
2176
19
}
2177
2178
1.03k
void CodeGenFunction::unprotectFromPeepholes(PeepholeProtection protection) {
2179
1.03k
  if (!protection.Inst) 
return1.02k
;
2180
19
2181
19
  // In theory, we could try to duplicate the peepholes now, but whatever.
2182
19
  protection.Inst->eraseFromParent();
2183
19
}
2184
2185
void CodeGenFunction::emitAlignmentAssumption(llvm::Value *PtrValue,
2186
                                              QualType Ty, SourceLocation Loc,
2187
                                              SourceLocation AssumptionLoc,
2188
                                              llvm::Value *Alignment,
2189
429
                                              llvm::Value *OffsetValue) {
2190
429
  llvm::Value *TheCheck;
2191
429
  llvm::Instruction *Assumption = Builder.CreateAlignmentAssumption(
2192
429
      CGM.getDataLayout(), PtrValue, Alignment, OffsetValue, &TheCheck);
2193
429
  if (SanOpts.has(SanitizerKind::Alignment)) {
2194
33
    emitAlignmentAssumptionCheck(PtrValue, Ty, Loc, AssumptionLoc, Alignment,
2195
33
                                 OffsetValue, TheCheck, Assumption);
2196
33
  }
2197
429
}
2198
2199
void CodeGenFunction::emitAlignmentAssumption(llvm::Value *PtrValue,
2200
                                              const Expr *E,
2201
                                              SourceLocation AssumptionLoc,
2202
                                              llvm::Value *Alignment,
2203
402
                                              llvm::Value *OffsetValue) {
2204
402
  if (auto *CE = dyn_cast<CastExpr>(E))
2205
168
    E = CE->getSubExprAsWritten();
2206
402
  QualType Ty = E->getType();
2207
402
  SourceLocation Loc = E->getExprLoc();
2208
402
2209
402
  emitAlignmentAssumption(PtrValue, Ty, Loc, AssumptionLoc, Alignment,
2210
402
                          OffsetValue);
2211
402
}
2212
2213
llvm::Value *CodeGenFunction::EmitAnnotationCall(llvm::Function *AnnotationFn,
2214
                                                 llvm::Value *AnnotatedVal,
2215
                                                 StringRef AnnotationStr,
2216
22
                                                 SourceLocation Location) {
2217
22
  llvm::Value *Args[4] = {
2218
22
    AnnotatedVal,
2219
22
    Builder.CreateBitCast(CGM.EmitAnnotationString(AnnotationStr), Int8PtrTy),
2220
22
    Builder.CreateBitCast(CGM.EmitAnnotationUnit(Location), Int8PtrTy),
2221
22
    CGM.EmitAnnotationLineNo(Location)
2222
22
  };
2223
22
  return Builder.CreateCall(AnnotationFn, Args);
2224
22
}
2225
2226
7
void CodeGenFunction::EmitVarAnnotations(const VarDecl *D, llvm::Value *V) {
2227
7
  assert(D->hasAttr<AnnotateAttr>() && "no annotate attribute");
2228
7
  // FIXME We create a new bitcast for every annotation because that's what
2229
7
  // llvm-gcc was doing.
2230
7
  for (const auto *I : D->specific_attrs<AnnotateAttr>())
2231
11
    EmitAnnotationCall(CGM.getIntrinsic(llvm::Intrinsic::var_annotation),
2232
11
                       Builder.CreateBitCast(V, CGM.Int8PtrTy, V->getName()),
2233
11
                       I->getAnnotation(), D->getLocation());
2234
7
}
2235
2236
Address CodeGenFunction::EmitFieldAnnotations(const FieldDecl *D,
2237
2
                                              Address Addr) {
2238
2
  assert(D->hasAttr<AnnotateAttr>() && "no annotate attribute");
2239
2
  llvm::Value *V = Addr.getPointer();
2240
2
  llvm::Type *VTy = V->getType();
2241
2
  llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::ptr_annotation,
2242
2
                                    CGM.Int8PtrTy);
2243
2
2244
4
  for (const auto *I : D->specific_attrs<AnnotateAttr>()) {
2245
4
    // FIXME Always emit the cast inst so we can differentiate between
2246
4
    // annotation on the first field of a struct and annotation on the struct
2247
4
    // itself.
2248
4
    if (VTy != CGM.Int8PtrTy)
2249
4
      V = Builder.CreateBitCast(V, CGM.Int8PtrTy);
2250
4
    V = EmitAnnotationCall(F, V, I->getAnnotation(), D->getLocation());
2251
4
    V = Builder.CreateBitCast(V, VTy);
2252
4
  }
2253
2
2254
2
  return Address(V, Addr.getAlignment());
2255
2
}
2256
2257
45.9k
CodeGenFunction::CGCapturedStmtInfo::~CGCapturedStmtInfo() { }
2258
2259
CodeGenFunction::SanitizerScope::SanitizerScope(CodeGenFunction *CGF)
2260
42.8k
    : CGF(CGF) {
2261
42.8k
  assert(!CGF->IsSanitizerScope);
2262
42.8k
  CGF->IsSanitizerScope = true;
2263
42.8k
}
2264
2265
42.8k
CodeGenFunction::SanitizerScope::~SanitizerScope() {
2266
42.8k
  CGF->IsSanitizerScope = false;
2267
42.8k
}
2268
2269
void CodeGenFunction::InsertHelper(llvm::Instruction *I,
2270
                                   const llvm::Twine &Name,
2271
                                   llvm::BasicBlock *BB,
2272
4.21M
                                   llvm::BasicBlock::iterator InsertPt) const {
2273
4.21M
  LoopStack.InsertHelper(I);
2274
4.21M
  if (IsSanitizerScope)
2275
14.9k
    CGM.getSanitizerMetadata()->disableSanitizerForInstruction(I);
2276
4.21M
}
2277
2278
void CGBuilderInserter::InsertHelper(
2279
    llvm::Instruction *I, const llvm::Twine &Name, llvm::BasicBlock *BB,
2280
4.22M
    llvm::BasicBlock::iterator InsertPt) const {
2281
4.22M
  llvm::IRBuilderDefaultInserter::InsertHelper(I, Name, BB, InsertPt);
2282
4.22M
  if (CGF)
2283
4.21M
    CGF->InsertHelper(I, Name, BB, InsertPt);
2284
4.22M
}
2285
2286
static bool hasRequiredFeatures(const SmallVectorImpl<StringRef> &ReqFeatures,
2287
                                CodeGenModule &CGM, const FunctionDecl *FD,
2288
29.9k
                                std::string &FirstMissing) {
2289
29.9k
  // If there aren't any required features listed then go ahead and return.
2290
29.9k
  if (ReqFeatures.empty())
2291
0
    return false;
2292
29.9k
2293
29.9k
  // Now build up the set of caller features and verify that all the required
2294
29.9k
  // features are there.
2295
29.9k
  llvm::StringMap<bool> CallerFeatureMap;
2296
29.9k
  CGM.getContext().getFunctionFeatureMap(CallerFeatureMap, FD);
2297
29.9k
2298
29.9k
  // If we have at least one of the features in the feature list return
2299
29.9k
  // true, otherwise return false.
2300
29.9k
  return std::all_of(
2301
217k
      ReqFeatures.begin(), ReqFeatures.end(), [&](StringRef Feature) {
2302
217k
        SmallVector<StringRef, 1> OrFeatures;
2303
217k
        Feature.split(OrFeatures, '|');
2304
224k
        return llvm::any_of(OrFeatures, [&](StringRef Feature) {
2305
224k
          if (!CallerFeatureMap.lookup(Feature)) {
2306
7.14k
            FirstMissing = Feature.str();
2307
7.14k
            return false;
2308
7.14k
          }
2309
217k
          return true;
2310
217k
        });
2311
217k
      });
2312
29.9k
}
2313
2314
// Emits an error if we don't have a valid set of target features for the
2315
// called function.
2316
void CodeGenFunction::checkTargetFeatures(const CallExpr *E,
2317
34.4k
                                          const FunctionDecl *TargetDecl) {
2318
34.4k
  return checkTargetFeatures(E->getBeginLoc(), TargetDecl);
2319
34.4k
}
2320
2321
// Emits an error if we don't have a valid set of target features for the
2322
// called function.
2323
void CodeGenFunction::checkTargetFeatures(SourceLocation Loc,
2324
46.3k
                                          const FunctionDecl *TargetDecl) {
2325
46.3k
  // Early exit if this is an indirect call.
2326
46.3k
  if (!TargetDecl)
2327
0
    return;
2328
46.3k
2329
46.3k
  // Get the current enclosing function if it exists. If it doesn't
2330
46.3k
  // we can't check the target features anyhow.
2331
46.3k
  const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(CurCodeDecl);
2332
46.3k
  if (!FD)
2333
0
    return;
2334
46.3k
2335
46.3k
  // Grab the required features for the call. For a builtin this is listed in
2336
46.3k
  // the td file with the default cpu, for an always_inline function this is any
2337
46.3k
  // listed cpu and any listed features.
2338
46.3k
  unsigned BuiltinID = TargetDecl->getBuiltinID();
2339
46.3k
  std::string MissingFeature;
2340
46.3k
  if (BuiltinID) {
2341
34.4k
    SmallVector<StringRef, 1> ReqFeatures;
2342
34.4k
    const char *FeatureList =
2343
34.4k
        CGM.getContext().BuiltinInfo.getRequiredFeatures(BuiltinID);
2344
34.4k
    // Return if the builtin doesn't have any required features.
2345
34.4k
    if (!FeatureList || 
StringRef(FeatureList) == ""23.0k
)
2346
16.4k
      return;
2347
17.9k
    StringRef(FeatureList).split(ReqFeatures, ',');
2348
17.9k
    if (!hasRequiredFeatures(ReqFeatures, CGM, FD, MissingFeature))
2349
542
      CGM.getDiags().Report(Loc, diag::err_builtin_needs_feature)
2350
542
          << TargetDecl->getDeclName()
2351
542
          << CGM.getContext().BuiltinInfo.getRequiredFeatures(BuiltinID);
2352
17.9k
2353
17.9k
  } else 
if (11.9k
!TargetDecl->isMultiVersion()11.9k
&&
2354
11.9k
             TargetDecl->hasAttr<TargetAttr>()) {
2355
11.9k
    // Get the required features for the callee.
2356
11.9k
2357
11.9k
    const TargetAttr *TD = TargetDecl->getAttr<TargetAttr>();
2358
11.9k
    ParsedTargetAttr ParsedAttr =
2359
11.9k
        CGM.getContext().filterFunctionTargetAttrs(TD);
2360
11.9k
2361
11.9k
    SmallVector<StringRef, 1> ReqFeatures;
2362
11.9k
    llvm::StringMap<bool> CalleeFeatureMap;
2363
11.9k
    CGM.getContext().getFunctionFeatureMap(CalleeFeatureMap,
2364
11.9k
                                           GlobalDecl(TargetDecl));
2365
11.9k
2366
13.2k
    for (const auto &F : ParsedAttr.Features) {
2367
13.2k
      if (F[0] == '+' && CalleeFeatureMap.lookup(F.substr(1)))
2368
13.2k
        ReqFeatures.push_back(StringRef(F).substr(1));
2369
13.2k
    }
2370
11.9k
2371
184k
    for (const auto &F : CalleeFeatureMap) {
2372
184k
      // Only positive features are "required".
2373
184k
      if (F.getValue())
2374
184k
        ReqFeatures.push_back(F.getKey());
2375
184k
    }
2376
11.9k
    if (!hasRequiredFeatures(ReqFeatures, CGM, FD, MissingFeature))
2377
14
      CGM.getDiags().Report(Loc, diag::err_function_needs_feature)
2378
14
          << FD->getDeclName() << TargetDecl->getDeclName() << MissingFeature;
2379
11.9k
  }
2380
46.3k
}
2381
2382
87
void CodeGenFunction::EmitSanitizerStatReport(llvm::SanitizerStatKind SSK) {
2383
87
  if (!CGM.getCodeGenOpts().SanitizeStats)
2384
77
    return;
2385
10
2386
10
  llvm::IRBuilder<> IRB(Builder.GetInsertBlock(), Builder.GetInsertPoint());
2387
10
  IRB.SetCurrentDebugLocation(Builder.getCurrentDebugLocation());
2388
10
  CGM.getSanStats().create(IRB, SSK);
2389
10
}
2390
2391
llvm::Value *
2392
212
CodeGenFunction::FormResolverCondition(const MultiVersionResolverOption &RO) {
2393
212
  llvm::Value *Condition = nullptr;
2394
212
2395
212
  if (!RO.Conditions.Architecture.empty())
2396
68
    Condition = EmitX86CpuIs(RO.Conditions.Architecture);
2397
212
2398
212
  if (!RO.Conditions.Features.empty()) {
2399
91
    llvm::Value *FeatureCond = EmitX86CpuSupports(RO.Conditions.Features);
2400
91
    Condition =
2401
91
        Condition ? 
Builder.CreateAnd(Condition, FeatureCond)2
:
FeatureCond89
;
2402
91
  }
2403
212
  return Condition;
2404
212
}
2405
2406
static void CreateMultiVersionResolverReturn(CodeGenModule &CGM,
2407
                                             llvm::Function *Resolver,
2408
                                             CGBuilderTy &Builder,
2409
                                             llvm::Function *FuncToReturn,
2410
212
                                             bool SupportsIFunc) {
2411
212
  if (SupportsIFunc) {
2412
104
    Builder.CreateRet(FuncToReturn);
2413
104
    return;
2414
104
  }
2415
108
2416
108
  llvm::SmallVector<llvm::Value *, 10> Args;
2417
108
  llvm::for_each(Resolver->args(),
2418
116
                 [&](llvm::Argument &Arg) { Args.push_back(&Arg); });
2419
108
2420
108
  llvm::CallInst *Result = Builder.CreateCall(FuncToReturn, Args);
2421
108
  Result->setTailCallKind(llvm::CallInst::TCK_MustTail);
2422
108
2423
108
  if (Resolver->getReturnType()->isVoidTy())
2424
29
    Builder.CreateRetVoid();
2425
79
  else
2426
79
    Builder.CreateRet(Result);
2427
108
}
2428
2429
void CodeGenFunction::EmitMultiVersionResolver(
2430
65
    llvm::Function *Resolver, ArrayRef<MultiVersionResolverOption> Options) {
2431
65
  assert(getContext().getTargetInfo().getTriple().isX86() &&
2432
65
         "Only implemented for x86 targets");
2433
65
2434
65
  bool SupportsIFunc = getContext().getTargetInfo().supportsIFunc();
2435
65
2436
65
  // Main function's basic block.
2437
65
  llvm::BasicBlock *CurBlock = createBasicBlock("resolver_entry", Resolver);
2438
65
  Builder.SetInsertPoint(CurBlock);
2439
65
  EmitX86CpuInit();
2440
65
2441
212
  for (const MultiVersionResolverOption &RO : Options) {
2442
212
    Builder.SetInsertPoint(CurBlock);
2443
212
    llvm::Value *Condition = FormResolverCondition(RO);
2444
212
2445
212
    // The 'default' or 'generic' case.
2446
212
    if (!Condition) {
2447
55
      assert(&RO == Options.end() - 1 &&
2448
55
             "Default or Generic case must be last");
2449
55
      CreateMultiVersionResolverReturn(CGM, Resolver, Builder, RO.Function,
2450
55
                                       SupportsIFunc);
2451
55
      return;
2452
55
    }
2453
157
2454
157
    llvm::BasicBlock *RetBlock = createBasicBlock("resolver_return", Resolver);
2455
157
    CGBuilderTy RetBuilder(*this, RetBlock);
2456
157
    CreateMultiVersionResolverReturn(CGM, Resolver, RetBuilder, RO.Function,
2457
157
                                     SupportsIFunc);
2458
157
    CurBlock = createBasicBlock("resolver_else", Resolver);
2459
157
    Builder.CreateCondBr(Condition, RetBlock, CurBlock);
2460
157
  }
2461
65
2462
65
  // If no generic/default, emit an unreachable.
2463
65
  Builder.SetInsertPoint(CurBlock);
2464
10
  llvm::CallInst *TrapCall = EmitTrapCall(llvm::Intrinsic::trap);
2465
10
  TrapCall->setDoesNotReturn();
2466
10
  TrapCall->setDoesNotThrow();
2467
10
  Builder.CreateUnreachable();
2468
10
  Builder.ClearInsertionPoint();
2469
10
}
2470
2471
// Loc - where the diagnostic will point, where in the source code this
2472
//  alignment has failed.
2473
// SecondaryLoc - if present (will be present if sufficiently different from
2474
//  Loc), the diagnostic will additionally point a "Note:" to this location.
2475
//  It should be the location where the __attribute__((assume_aligned))
2476
//  was written e.g.
2477
void CodeGenFunction::emitAlignmentAssumptionCheck(
2478
    llvm::Value *Ptr, QualType Ty, SourceLocation Loc,
2479
    SourceLocation SecondaryLoc, llvm::Value *Alignment,
2480
    llvm::Value *OffsetValue, llvm::Value *TheCheck,
2481
33
    llvm::Instruction *Assumption) {
2482
33
  assert(Assumption && isa<llvm::CallInst>(Assumption) &&
2483
33
         cast<llvm::CallInst>(Assumption)->getCalledValue() ==
2484
33
             llvm::Intrinsic::getDeclaration(
2485
33
                 Builder.GetInsertBlock()->getParent()->getParent(),
2486
33
                 llvm::Intrinsic::assume) &&
2487
33
         "Assumption should be a call to llvm.assume().");
2488
33
  assert(&(Builder.GetInsertBlock()->back()) == Assumption &&
2489
33
         "Assumption should be the last instruction of the basic block, "
2490
33
         "since the basic block is still being generated.");
2491
33
2492
33
  if (!SanOpts.has(SanitizerKind::Alignment))
2493
0
    return;
2494
33
2495
33
  // Don't check pointers to volatile data. The behavior here is implementation-
2496
33
  // defined.
2497
33
  if (Ty->getPointeeType().isVolatileQualified())
2498
1
    return;
2499
32
2500
32
  // We need to temorairly remove the assumption so we can insert the
2501
32
  // sanitizer check before it, else the check will be dropped by optimizations.
2502
32
  Assumption->removeFromParent();
2503
32
2504
32
  {
2505
32
    SanitizerScope SanScope(this);
2506
32
2507
32
    if (!OffsetValue)
2508
23
      OffsetValue = Builder.getInt1(0); // no offset.
2509
32
2510
32
    llvm::Constant *StaticData[] = {EmitCheckSourceLocation(Loc),
2511
32
                                    EmitCheckSourceLocation(SecondaryLoc),
2512
32
                                    EmitCheckTypeDescriptor(Ty)};
2513
32
    llvm::Value *DynamicData[] = {EmitCheckValue(Ptr),
2514
32
                                  EmitCheckValue(Alignment),
2515
32
                                  EmitCheckValue(OffsetValue)};
2516
32
    EmitCheck({std::make_pair(TheCheck, SanitizerKind::Alignment)},
2517
32
              SanitizerHandler::AlignmentAssumption, StaticData, DynamicData);
2518
32
  }
2519
32
2520
32
  // We are now in the (new, empty) "cont" basic block.
2521
32
  // Reintroduce the assumption.
2522
32
  Builder.Insert(Assumption);
2523
32
  // FIXME: Assumption still has it's original basic block as it's Parent.
2524
32
}
2525
2526
43.2k
llvm::DebugLoc CodeGenFunction::SourceLocToDebugLoc(SourceLocation Location) {
2527
43.2k
  if (CGDebugInfo *DI = getDebugInfo())
2528
16.0k
    return DI->SourceLocToDebugLoc(Location);
2529
27.1k
2530
27.1k
  return llvm::DebugLoc();
2531
27.1k
}