Coverage Report

Created: 2020-11-24 06:42

/Users/buildslave/jenkins/workspace/coverage/llvm-project/clang/lib/CodeGen/CodeGenFunction.cpp
Line
Count
Source (jump to first uncovered line)
1
//===--- CodeGenFunction.cpp - Emit LLVM Code from ASTs for a Function ----===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This coordinates the per-function state used while generating code.
10
//
11
//===----------------------------------------------------------------------===//
12
13
#include "CodeGenFunction.h"
14
#include "CGBlocks.h"
15
#include "CGCUDARuntime.h"
16
#include "CGCXXABI.h"
17
#include "CGCleanup.h"
18
#include "CGDebugInfo.h"
19
#include "CGOpenMPRuntime.h"
20
#include "CodeGenModule.h"
21
#include "CodeGenPGO.h"
22
#include "TargetInfo.h"
23
#include "clang/AST/ASTContext.h"
24
#include "clang/AST/ASTLambda.h"
25
#include "clang/AST/Attr.h"
26
#include "clang/AST/Decl.h"
27
#include "clang/AST/DeclCXX.h"
28
#include "clang/AST/Expr.h"
29
#include "clang/AST/StmtCXX.h"
30
#include "clang/AST/StmtObjC.h"
31
#include "clang/Basic/Builtins.h"
32
#include "clang/Basic/CodeGenOptions.h"
33
#include "clang/Basic/TargetInfo.h"
34
#include "clang/CodeGen/CGFunctionInfo.h"
35
#include "clang/Frontend/FrontendDiagnostic.h"
36
#include "llvm/ADT/ArrayRef.h"
37
#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
38
#include "llvm/IR/DataLayout.h"
39
#include "llvm/IR/Dominators.h"
40
#include "llvm/IR/FPEnv.h"
41
#include "llvm/IR/IntrinsicInst.h"
42
#include "llvm/IR/Intrinsics.h"
43
#include "llvm/IR/MDBuilder.h"
44
#include "llvm/IR/Operator.h"
45
#include "llvm/Support/CRC.h"
46
#include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h"
47
#include "llvm/Transforms/Utils/PromoteMemToReg.h"
48
using namespace clang;
49
using namespace CodeGen;
50
51
/// shouldEmitLifetimeMarkers - Decide whether we need emit the life-time
52
/// markers.
53
static bool shouldEmitLifetimeMarkers(const CodeGenOptions &CGOpts,
54
294k
                                      const LangOptions &LangOpts) {
55
294k
  if (CGOpts.DisableLifetimeMarkers)
56
7
    return false;
57
58
  // Sanitizers may use markers.
59
294k
  if (CGOpts.SanitizeAddressUseAfterScope ||
60
293k
      LangOpts.Sanitize.has(SanitizerKind::HWAddress) ||
61
293k
      LangOpts.Sanitize.has(SanitizerKind::Memory))
62
1.26k
    return true;
63
64
  // For now, only in optimized builds.
65
293k
  return CGOpts.OptimizationLevel != 0;
66
293k
}
67
68
CodeGenFunction::CodeGenFunction(CodeGenModule &cgm, bool suppressNewContext)
69
    : CodeGenTypeCache(cgm), CGM(cgm), Target(cgm.getTarget()),
70
      Builder(cgm, cgm.getModule().getContext(), llvm::ConstantFolder(),
71
              CGBuilderInserterTy(this)),
72
      SanOpts(CGM.getLangOpts().Sanitize), CurFPFeatures(CGM.getLangOpts()),
73
      DebugInfo(CGM.getModuleDebugInfo()), PGO(cgm),
74
      ShouldEmitLifetimeMarkers(
75
294k
          shouldEmitLifetimeMarkers(CGM.getCodeGenOpts(), CGM.getLangOpts())) {
76
294k
  if (!suppressNewContext)
77
268k
    CGM.getCXXABI().getMangleContext().startNewFunction();
78
79
294k
  SetFastMathFlags(CurFPFeatures);
80
294k
  SetFPModel();
81
294k
}
82
83
294k
CodeGenFunction::~CodeGenFunction() {
84
294k
  assert(LifetimeExtendedCleanupStack.empty() && "failed to emit a cleanup");
85
86
294k
  if (getLangOpts().OpenMP && 
CurFn68.3k
)
87
68.3k
    CGM.getOpenMPRuntime().functionFinished(*this);
88
89
  // If we have an OpenMPIRBuilder we want to finalize functions (incl.
90
  // outlining etc) at some point. Doing it once the function codegen is done
91
  // seems to be a reasonable spot. We do it here, as opposed to the deletion
92
  // time of the CodeGenModule, because we have to ensure the IR has not yet
93
  // been "emitted" to the outside, thus, modifications are still sensible.
94
294k
  if (CGM.getLangOpts().OpenMPIRBuilder)
95
165
    CGM.getOpenMPRuntime().getOMPBuilder().finalize();
96
294k
}
97
98
// Map the LangOption for exception behavior into
99
// the corresponding enum in the IR.
100
llvm::fp::ExceptionBehavior
101
295k
clang::ToConstrainedExceptMD(LangOptions::FPExceptionModeKind Kind) {
102
103
295k
  switch (Kind) {
104
292k
  case LangOptions::FPE_Ignore:  return llvm::fp::ebIgnore;
105
142
  case LangOptions::FPE_MayTrap: return llvm::fp::ebMayTrap;
106
2.79k
  case LangOptions::FPE_Strict:  return llvm::fp::ebStrict;
107
0
  }
108
0
  llvm_unreachable("Unsupported FP Exception Behavior");
109
0
}
110
111
294k
void CodeGenFunction::SetFPModel() {
112
294k
  llvm::RoundingMode RM = getLangOpts().getFPRoundingMode();
113
294k
  auto fpExceptionBehavior = ToConstrainedExceptMD(
114
294k
                               getLangOpts().getFPExceptionMode());
115
116
294k
  Builder.setDefaultConstrainedRounding(RM);
117
294k
  Builder.setDefaultConstrainedExcept(fpExceptionBehavior);
118
294k
  Builder.setIsFPConstrained(fpExceptionBehavior != llvm::fp::ebIgnore ||
119
292k
                             RM != llvm::RoundingMode::NearestTiesToEven);
120
294k
}
121
122
295k
void CodeGenFunction::SetFastMathFlags(FPOptions FPFeatures) {
123
295k
  llvm::FastMathFlags FMF;
124
295k
  FMF.setAllowReassoc(FPFeatures.getAllowFPReassociate());
125
295k
  FMF.setNoNaNs(FPFeatures.getNoHonorNaNs());
126
295k
  FMF.setNoInfs(FPFeatures.getNoHonorInfs());
127
295k
  FMF.setNoSignedZeros(FPFeatures.getNoSignedZero());
128
295k
  FMF.setAllowReciprocal(FPFeatures.getAllowReciprocal());
129
295k
  FMF.setApproxFunc(FPFeatures.getAllowApproxFunc());
130
295k
  FMF.setAllowContract(FPFeatures.allowFPContractAcrossStatement());
131
295k
  Builder.setFastMathFlags(FMF);
132
295k
}
133
134
CodeGenFunction::CGFPOptionsRAII::CGFPOptionsRAII(CodeGenFunction &CGF,
135
                                                  const Expr *E)
136
131k
    : CGF(CGF) {
137
131k
  ConstructorHelper(E->getFPFeaturesInEffect(CGF.getLangOpts()));
138
131k
}
139
140
CodeGenFunction::CGFPOptionsRAII::CGFPOptionsRAII(CodeGenFunction &CGF,
141
                                                  FPOptions FPFeatures)
142
9.99k
    : CGF(CGF) {
143
9.99k
  ConstructorHelper(FPFeatures);
144
9.99k
}
145
146
141k
void CodeGenFunction::CGFPOptionsRAII::ConstructorHelper(FPOptions FPFeatures) {
147
141k
  OldFPFeatures = CGF.CurFPFeatures;
148
141k
  CGF.CurFPFeatures = FPFeatures;
149
150
141k
  OldExcept = CGF.Builder.getDefaultConstrainedExcept();
151
141k
  OldRounding = CGF.Builder.getDefaultConstrainedRounding();
152
153
141k
  if (OldFPFeatures == FPFeatures)
154
139k
    return;
155
156
1.51k
  FMFGuard.emplace(CGF.Builder);
157
158
1.51k
  llvm::RoundingMode NewRoundingBehavior =
159
1.51k
      static_cast<llvm::RoundingMode>(FPFeatures.getRoundingMode());
160
1.51k
  CGF.Builder.setDefaultConstrainedRounding(NewRoundingBehavior);
161
1.51k
  auto NewExceptionBehavior =
162
1.51k
      ToConstrainedExceptMD(static_cast<LangOptions::FPExceptionModeKind>(
163
1.51k
          FPFeatures.getFPExceptionMode()));
164
1.51k
  CGF.Builder.setDefaultConstrainedExcept(NewExceptionBehavior);
165
166
1.51k
  CGF.SetFastMathFlags(FPFeatures);
167
168
1.51k
  assert((CGF.CurFuncDecl == nullptr || CGF.Builder.getIsFPConstrained() ||
169
1.51k
          isa<CXXConstructorDecl>(CGF.CurFuncDecl) ||
170
1.51k
          isa<CXXDestructorDecl>(CGF.CurFuncDecl) ||
171
1.51k
          (NewExceptionBehavior == llvm::fp::ebIgnore &&
172
1.51k
           NewRoundingBehavior == llvm::RoundingMode::NearestTiesToEven)) &&
173
1.51k
         "FPConstrained should be enabled on entire function");
174
175
6.05k
  auto mergeFnAttrValue = [&](StringRef Name, bool Value) {
176
6.05k
    auto OldValue =
177
6.05k
        CGF.CurFn->getFnAttribute(Name).getValueAsString() == "true";
178
6.05k
    auto NewValue = OldValue & Value;
179
6.05k
    if (OldValue != NewValue)
180
29
      CGF.CurFn->addFnAttr(Name, llvm::toStringRef(NewValue));
181
6.05k
  };
182
1.51k
  mergeFnAttrValue("no-infs-fp-math", FPFeatures.getNoHonorInfs());
183
1.51k
  mergeFnAttrValue("no-nans-fp-math", FPFeatures.getNoHonorNaNs());
184
1.51k
  mergeFnAttrValue("no-signed-zeros-fp-math", FPFeatures.getNoSignedZero());
185
1.51k
  mergeFnAttrValue("unsafe-fp-math", FPFeatures.getAllowFPReassociate() &&
186
85
                                         FPFeatures.getAllowReciprocal() &&
187
72
                                         FPFeatures.getAllowApproxFunc() &&
188
72
                                         FPFeatures.getNoSignedZero());
189
1.51k
}
190
191
141k
CodeGenFunction::CGFPOptionsRAII::~CGFPOptionsRAII() {
192
141k
  CGF.CurFPFeatures = OldFPFeatures;
193
141k
  CGF.Builder.setDefaultConstrainedExcept(OldExcept);
194
141k
  CGF.Builder.setDefaultConstrainedRounding(OldRounding);
195
141k
}
196
197
41.7k
LValue CodeGenFunction::MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T) {
198
41.7k
  LValueBaseInfo BaseInfo;
199
41.7k
  TBAAAccessInfo TBAAInfo;
200
41.7k
  CharUnits Alignment = CGM.getNaturalTypeAlignment(T, &BaseInfo, &TBAAInfo);
201
41.7k
  return LValue::MakeAddr(Address(V, Alignment), T, getContext(), BaseInfo,
202
41.7k
                          TBAAInfo);
203
41.7k
}
204
205
/// Given a value of type T* that may not be to a complete object,
206
/// construct an l-value with the natural pointee alignment of T.
207
LValue
208
60.3k
CodeGenFunction::MakeNaturalAlignPointeeAddrLValue(llvm::Value *V, QualType T) {
209
60.3k
  LValueBaseInfo BaseInfo;
210
60.3k
  TBAAAccessInfo TBAAInfo;
211
60.3k
  CharUnits Align = CGM.getNaturalTypeAlignment(T, &BaseInfo, &TBAAInfo,
212
60.3k
                                                /* forPointeeType= */ true);
213
60.3k
  return MakeAddrLValue(Address(V, Align), T, BaseInfo, TBAAInfo);
214
60.3k
}
215
216
217
895k
llvm::Type *CodeGenFunction::ConvertTypeForMem(QualType T) {
218
895k
  return CGM.getTypes().ConvertTypeForMem(T);
219
895k
}
220
221
2.94M
llvm::Type *CodeGenFunction::ConvertType(QualType T) {
222
2.94M
  return CGM.getTypes().ConvertType(T);
223
2.94M
}
224
225
5.22M
TypeEvaluationKind CodeGenFunction::getEvaluationKind(QualType type) {
226
5.22M
  type = type.getCanonicalType();
227
5.22M
  while (true) {
228
5.22M
    switch (type->getTypeClass()) {
229
0
#define TYPE(name, parent)
230
0
#define ABSTRACT_TYPE(name, parent)
231
0
#define NON_CANONICAL_TYPE(name, parent) case Type::name:
232
0
#define DEPENDENT_TYPE(name, parent) case Type::name:
233
0
#define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(name, parent) case Type::name:
234
0
#include "clang/AST/TypeNodes.inc"
235
0
      llvm_unreachable("non-canonical or dependent type in IR-generation");
236
237
0
    case Type::Auto:
238
0
    case Type::DeducedTemplateSpecialization:
239
0
      llvm_unreachable("undeduced type in IR-generation");
240
241
    // Various scalar types.
242
4.97M
    case Type::Builtin:
243
4.97M
    case Type::Pointer:
244
4.97M
    case Type::BlockPointer:
245
4.97M
    case Type::LValueReference:
246
4.97M
    case Type::RValueReference:
247
4.97M
    case Type::MemberPointer:
248
4.97M
    case Type::Vector:
249
4.97M
    case Type::ExtVector:
250
4.97M
    case Type::ConstantMatrix:
251
4.97M
    case Type::FunctionProto:
252
4.97M
    case Type::FunctionNoProto:
253
4.97M
    case Type::Enum:
254
4.97M
    case Type::ObjCObjectPointer:
255
4.97M
    case Type::Pipe:
256
4.97M
    case Type::ExtInt:
257
4.97M
      return TEK_Scalar;
258
259
    // Complexes.
260
7.63k
    case Type::Complex:
261
7.63k
      return TEK_Complex;
262
263
    // Arrays, records, and Objective-C objects.
264
247k
    case Type::ConstantArray:
265
247k
    case Type::IncompleteArray:
266
247k
    case Type::VariableArray:
267
247k
    case Type::Record:
268
247k
    case Type::ObjCObject:
269
247k
    case Type::ObjCInterface:
270
247k
      return TEK_Aggregate;
271
272
    // We operate on atomic values according to their underlying type.
273
286
    case Type::Atomic:
274
286
      type = cast<AtomicType>(type)->getValueType();
275
286
      continue;
276
0
    }
277
0
    llvm_unreachable("unknown type kind!");
278
0
  }
279
5.22M
}
280
281
293k
llvm::DebugLoc CodeGenFunction::EmitReturnBlock() {
282
  // For cleanliness, we try to avoid emitting the return block for
283
  // simple cases.
284
293k
  llvm::BasicBlock *CurBB = Builder.GetInsertBlock();
285
286
293k
  if (CurBB) {
287
151k
    assert(!CurBB->getTerminator() && "Unexpected terminated block.");
288
289
    // We have a valid insert point, reuse it if it is empty or there are no
290
    // explicit jumps to the return block.
291
151k
    if (CurBB->empty() || 
ReturnBlock.getBlock()->use_empty()125k
) {
292
151k
      ReturnBlock.getBlock()->replaceAllUsesWith(CurBB);
293
151k
      delete ReturnBlock.getBlock();
294
151k
      ReturnBlock = JumpDest();
295
151k
    } else
296
30
      EmitBlock(ReturnBlock.getBlock());
297
151k
    return llvm::DebugLoc();
298
151k
  }
299
300
  // Otherwise, if the return block is the target of a single direct
301
  // branch then we can just put the code in that block instead. This
302
  // cleans up functions which started with a unified return block.
303
141k
  if (ReturnBlock.getBlock()->hasOneUse()) {
304
138k
    llvm::BranchInst *BI =
305
138k
      dyn_cast<llvm::BranchInst>(*ReturnBlock.getBlock()->user_begin());
306
138k
    if (BI && 
BI->isUnconditional()138k
&&
307
138k
        BI->getSuccessor(0) == ReturnBlock.getBlock()) {
308
      // Record/return the DebugLoc of the simple 'return' expression to be used
309
      // later by the actual 'ret' instruction.
310
138k
      llvm::DebugLoc Loc = BI->getDebugLoc();
311
138k
      Builder.SetInsertPoint(BI->getParent());
312
138k
      BI->eraseFromParent();
313
138k
      delete ReturnBlock.getBlock();
314
138k
      ReturnBlock = JumpDest();
315
138k
      return Loc;
316
138k
    }
317
2.57k
  }
318
319
  // FIXME: We are at an unreachable point, there is no reason to emit the block
320
  // unless it has uses. However, we still need a place to put the debug
321
  // region.end for now.
322
323
2.57k
  EmitBlock(ReturnBlock.getBlock());
324
2.57k
  return llvm::DebugLoc();
325
2.57k
}
326
327
1.17M
static void EmitIfUsed(CodeGenFunction &CGF, llvm::BasicBlock *BB) {
328
1.17M
  if (!BB) 
return1.16M
;
329
7.93k
  if (!BB->use_empty())
330
7.93k
    return CGF.CurFn->getBasicBlockList().push_back(BB);
331
8
  delete BB;
332
8
}
333
334
293k
void CodeGenFunction::FinishFunction(SourceLocation EndLoc) {
335
293k
  assert(BreakContinueStack.empty() &&
336
293k
         "mismatched push/pop in break/continue stack!");
337
338
293k
  bool OnlySimpleReturnStmts = NumSimpleReturnExprs > 0
339
13.9k
    && NumSimpleReturnExprs == NumReturnExprs
340
12.6k
    && ReturnBlock.getBlock()->use_empty();
341
  // Usually the return expression is evaluated before the cleanup
342
  // code.  If the function contains only a simple return statement,
343
  // such as a constant, the location before the cleanup code becomes
344
  // the last useful breakpoint in the function, because the simple
345
  // return expression will be evaluated after the cleanup code. To be
346
  // safe, set the debug location for cleanup code to the location of
347
  // the return statement.  Otherwise the cleanup code should be at the
348
  // end of the function's lexical scope.
349
  //
350
  // If there are multiple branches to the return block, the branch
351
  // instructions will get the location of the return statements and
352
  // all will be fine.
353
293k
  if (CGDebugInfo *DI = getDebugInfo()) {
354
92.2k
    if (OnlySimpleReturnStmts)
355
589
      DI->EmitLocation(Builder, LastStopPoint);
356
91.6k
    else
357
91.6k
      DI->EmitLocation(Builder, EndLoc);
358
92.2k
  }
359
360
  // Pop any cleanups that might have been associated with the
361
  // parameters.  Do this in whatever block we're currently in; it's
362
  // important to do this before we enter the return block or return
363
  // edges will be *really* confused.
364
293k
  bool HasCleanups = EHStack.stable_begin() != PrologueCleanupDepth;
365
293k
  bool HasOnlyLifetimeMarkers =
366
293k
      HasCleanups && 
EHStack.containsOnlyLifetimeMarkers(PrologueCleanupDepth)8.14k
;
367
293k
  bool EmitRetDbgLoc = !HasCleanups || 
HasOnlyLifetimeMarkers8.14k
;
368
293k
  if (HasCleanups) {
369
    // Make sure the line table doesn't jump back into the body for
370
    // the ret after it's been at EndLoc.
371
8.14k
    Optional<ApplyDebugLocation> AL;
372
8.14k
    if (CGDebugInfo *DI = getDebugInfo()) {
373
1.75k
      if (OnlySimpleReturnStmts)
374
589
        DI->EmitLocation(Builder, EndLoc);
375
1.16k
      else
376
        // We may not have a valid end location. Try to apply it anyway, and
377
        // fall back to an artificial location if needed.
378
1.16k
        AL = ApplyDebugLocation::CreateDefaultArtificial(*this, EndLoc);
379
1.75k
    }
380
381
8.14k
    PopCleanupBlocks(PrologueCleanupDepth);
382
8.14k
  }
383
384
  // Emit function epilog (to return).
385
293k
  llvm::DebugLoc Loc = EmitReturnBlock();
386
387
293k
  if (ShouldInstrumentFunction()) {
388
11
    if (CGM.getCodeGenOpts().InstrumentFunctions)
389
6
      CurFn->addFnAttr("instrument-function-exit", "__cyg_profile_func_exit");
390
11
    if (CGM.getCodeGenOpts().InstrumentFunctionsAfterInlining)
391
4
      CurFn->addFnAttr("instrument-function-exit-inlined",
392
4
                       "__cyg_profile_func_exit");
393
11
  }
394
395
  // Emit debug descriptor for function end.
396
293k
  if (CGDebugInfo *DI = getDebugInfo())
397
92.2k
    DI->EmitFunctionEnd(Builder, CurFn);
398
399
  // Reset the debug location to that of the simple 'return' expression, if any
400
  // rather than that of the end of the function's scope '}'.
401
293k
  ApplyDebugLocation AL(*this, Loc);
402
293k
  EmitFunctionEpilog(*CurFnInfo, EmitRetDbgLoc, EndLoc);
403
293k
  EmitEndEHSpec(CurCodeDecl);
404
405
293k
  assert(EHStack.empty() &&
406
293k
         "did not remove all scopes from cleanup stack!");
407
408
  // If someone did an indirect goto, emit the indirect goto block at the end of
409
  // the function.
410
293k
  if (IndirectBranch) {
411
35
    EmitBlock(IndirectBranch->getParent());
412
35
    Builder.ClearInsertionPoint();
413
35
  }
414
415
  // If some of our locals escaped, insert a call to llvm.localescape in the
416
  // entry block.
417
293k
  if (!EscapedLocals.empty()) {
418
    // Invert the map from local to index into a simple vector. There should be
419
    // no holes.
420
35
    SmallVector<llvm::Value *, 4> EscapeArgs;
421
35
    EscapeArgs.resize(EscapedLocals.size());
422
35
    for (auto &Pair : EscapedLocals)
423
40
      EscapeArgs[Pair.second] = Pair.first;
424
35
    llvm::Function *FrameEscapeFn = llvm::Intrinsic::getDeclaration(
425
35
        &CGM.getModule(), llvm::Intrinsic::localescape);
426
35
    CGBuilderTy(*this, AllocaInsertPt).CreateCall(FrameEscapeFn, EscapeArgs);
427
35
  }
428
429
  // Remove the AllocaInsertPt instruction, which is just a convenience for us.
430
293k
  llvm::Instruction *Ptr = AllocaInsertPt;
431
293k
  AllocaInsertPt = nullptr;
432
293k
  Ptr->eraseFromParent();
433
434
  // If someone took the address of a label but never did an indirect goto, we
435
  // made a zero entry PHI node, which is illegal, zap it now.
436
293k
  if (IndirectBranch) {
437
35
    llvm::PHINode *PN = cast<llvm::PHINode>(IndirectBranch->getAddress());
438
35
    if (PN->getNumIncomingValues() == 0) {
439
15
      PN->replaceAllUsesWith(llvm::UndefValue::get(PN->getType()));
440
15
      PN->eraseFromParent();
441
15
    }
442
35
  }
443
444
293k
  EmitIfUsed(*this, EHResumeBlock);
445
293k
  EmitIfUsed(*this, TerminateLandingPad);
446
293k
  EmitIfUsed(*this, TerminateHandler);
447
293k
  EmitIfUsed(*this, UnreachableBlock);
448
449
293k
  for (const auto &FuncletAndParent : TerminateFunclets)
450
30
    EmitIfUsed(*this, FuncletAndParent.second);
451
452
293k
  if (CGM.getCodeGenOpts().EmitDeclMetadata)
453
19.1k
    EmitDeclMetadata();
454
455
293k
  for (SmallVectorImpl<std::pair<llvm::Instruction *, llvm::Value *> >::iterator
456
293k
           I = DeferredReplacements.begin(),
457
293k
           E = DeferredReplacements.end();
458
293k
       I != E; 
++I99
) {
459
99
    I->first->replaceAllUsesWith(I->second);
460
99
    I->first->eraseFromParent();
461
99
  }
462
463
  // Eliminate CleanupDestSlot alloca by replacing it with SSA values and
464
  // PHIs if the current function is a coroutine. We don't do it for all
465
  // functions as it may result in slight increase in numbers of instructions
466
  // if compiled with no optimizations. We do it for coroutine as the lifetime
467
  // of CleanupDestSlot alloca make correct coroutine frame building very
468
  // difficult.
469
293k
  if (NormalCleanupDest.isValid() && 
isCoroutine()288
) {
470
53
    llvm::DominatorTree DT(*CurFn);
471
53
    llvm::PromoteMemToReg(
472
53
        cast<llvm::AllocaInst>(NormalCleanupDest.getPointer()), DT);
473
53
    NormalCleanupDest = Address::invalid();
474
53
  }
475
476
  // Scan function arguments for vector width.
477
293k
  for (llvm::Argument &A : CurFn->args())
478
479k
    if (auto *VT = dyn_cast<llvm::VectorType>(A.getType()))
479
113k
      LargestVectorWidth =
480
113k
          std::max((uint64_t)LargestVectorWidth,
481
113k
                   VT->getPrimitiveSizeInBits().getKnownMinSize());
482
483
  // Update vector width based on return type.
484
293k
  if (auto *VT = dyn_cast<llvm::VectorType>(CurFn->getReturnType()))
485
52.5k
    LargestVectorWidth =
486
52.5k
        std::max((uint64_t)LargestVectorWidth,
487
52.5k
                 VT->getPrimitiveSizeInBits().getKnownMinSize());
488
489
  // Add the required-vector-width attribute. This contains the max width from:
490
  // 1. min-vector-width attribute used in the source program.
491
  // 2. Any builtins used that have a vector width specified.
492
  // 3. Values passed in and out of inline assembly.
493
  // 4. Width of vector arguments and return types for this function.
494
  // 5. Width of vector aguments and return types for functions called by this
495
  //    function.
496
293k
  CurFn->addFnAttr("min-legal-vector-width", llvm::utostr(LargestVectorWidth));
497
498
  // If we generated an unreachable return block, delete it now.
499
293k
  if (ReturnBlock.isValid() && 
ReturnBlock.getBlock()->use_empty()2.60k
) {
500
993
    Builder.ClearInsertionPoint();
501
993
    ReturnBlock.getBlock()->eraseFromParent();
502
993
  }
503
293k
  if (ReturnValue.isValid()) {
504
142k
    auto *RetAlloca = dyn_cast<llvm::AllocaInst>(ReturnValue.getPointer());
505
142k
    if (RetAlloca && 
RetAlloca->use_empty()139k
) {
506
126k
      RetAlloca->eraseFromParent();
507
126k
      ReturnValue = Address::invalid();
508
126k
    }
509
142k
  }
510
293k
}
511
512
/// ShouldInstrumentFunction - Return true if the current function should be
513
/// instrumented with __cyg_profile_func_* calls
514
586k
bool CodeGenFunction::ShouldInstrumentFunction() {
515
586k
  if (!CGM.getCodeGenOpts().InstrumentFunctions &&
516
586k
      !CGM.getCodeGenOpts().InstrumentFunctionsAfterInlining &&
517
586k
      !CGM.getCodeGenOpts().InstrumentFunctionEntryBare)
518
586k
    return false;
519
32
  if (!CurFuncDecl || 
CurFuncDecl->hasAttr<NoInstrumentFunctionAttr>()28
)
520
10
    return false;
521
22
  return true;
522
22
}
523
524
/// ShouldXRayInstrument - Return true if the current function should be
525
/// instrumented with XRay nop sleds.
526
586k
bool CodeGenFunction::ShouldXRayInstrumentFunction() const {
527
586k
  return CGM.getCodeGenOpts().XRayInstrumentFunctions;
528
586k
}
529
530
/// AlwaysEmitXRayCustomEvents - Return true if we should emit IR for calls to
531
/// the __xray_customevent(...) builtin calls, when doing XRay instrumentation.
532
2
bool CodeGenFunction::AlwaysEmitXRayCustomEvents() const {
533
2
  return CGM.getCodeGenOpts().XRayInstrumentFunctions &&
534
2
         (CGM.getCodeGenOpts().XRayAlwaysEmitCustomEvents ||
535
1
          CGM.getCodeGenOpts().XRayInstrumentationBundle.Mask ==
536
1
              XRayInstrKind::Custom);
537
2
}
538
539
2
bool CodeGenFunction::AlwaysEmitXRayTypedEvents() const {
540
2
  return CGM.getCodeGenOpts().XRayInstrumentFunctions &&
541
2
         (CGM.getCodeGenOpts().XRayAlwaysEmitTypedEvents ||
542
1
          CGM.getCodeGenOpts().XRayInstrumentationBundle.Mask ==
543
1
              XRayInstrKind::Typed);
544
2
}
545
546
llvm::Constant *
547
CodeGenFunction::EncodeAddrForUseInPrologue(llvm::Function *F,
548
113
                                            llvm::Constant *Addr) {
549
  // Addresses stored in prologue data can't require run-time fixups and must
550
  // be PC-relative. Run-time fixups are undesirable because they necessitate
551
  // writable text segments, which are unsafe. And absolute addresses are
552
  // undesirable because they break PIE mode.
553
554
  // Add a layer of indirection through a private global. Taking its address
555
  // won't result in a run-time fixup, even if Addr has linkonce_odr linkage.
556
113
  auto *GV = new llvm::GlobalVariable(CGM.getModule(), Addr->getType(),
557
113
                                      /*isConstant=*/true,
558
113
                                      llvm::GlobalValue::PrivateLinkage, Addr);
559
560
  // Create a PC-relative address.
561
113
  auto *GOTAsInt = llvm::ConstantExpr::getPtrToInt(GV, IntPtrTy);
562
113
  auto *FuncAsInt = llvm::ConstantExpr::getPtrToInt(F, IntPtrTy);
563
113
  auto *PCRelAsInt = llvm::ConstantExpr::getSub(GOTAsInt, FuncAsInt);
564
113
  return (IntPtrTy == Int32Ty)
565
70
             ? PCRelAsInt
566
43
             : llvm::ConstantExpr::getTrunc(PCRelAsInt, Int32Ty);
567
113
}
568
569
llvm::Value *
570
CodeGenFunction::DecodeAddrUsedInPrologue(llvm::Value *F,
571
8
                                          llvm::Value *EncodedAddr) {
572
  // Reconstruct the address of the global.
573
8
  auto *PCRelAsInt = Builder.CreateSExt(EncodedAddr, IntPtrTy);
574
8
  auto *FuncAsInt = Builder.CreatePtrToInt(F, IntPtrTy, "func_addr.int");
575
8
  auto *GOTAsInt = Builder.CreateAdd(PCRelAsInt, FuncAsInt, "global_addr.int");
576
8
  auto *GOTAddr = Builder.CreateIntToPtr(GOTAsInt, Int8PtrPtrTy, "global_addr");
577
578
  // Load the original pointer through the global.
579
8
  return Builder.CreateLoad(Address(GOTAddr, getPointerAlign()),
580
8
                            "decoded_addr");
581
8
}
582
583
void CodeGenFunction::EmitOpenCLKernelMetadata(const FunctionDecl *FD,
584
                                               llvm::Function *Fn)
585
1.62k
{
586
1.62k
  if (!FD->hasAttr<OpenCLKernelAttr>())
587
1.20k
    return;
588
589
420
  llvm::LLVMContext &Context = getLLVMContext();
590
591
420
  CGM.GenOpenCLArgMetadata(Fn, FD, this);
592
593
420
  if (const VecTypeHintAttr *A = FD->getAttr<VecTypeHintAttr>()) {
594
2
    QualType HintQTy = A->getTypeHint();
595
2
    const ExtVectorType *HintEltQTy = HintQTy->getAs<ExtVectorType>();
596
2
    bool IsSignedInteger =
597
2
        HintQTy->isSignedIntegerType() ||
598
1
        (HintEltQTy && HintEltQTy->getElementType()->isSignedIntegerType());
599
2
    llvm::Metadata *AttrMDArgs[] = {
600
2
        llvm::ConstantAsMetadata::get(llvm::UndefValue::get(
601
2
            CGM.getTypes().ConvertType(A->getTypeHint()))),
602
2
        llvm::ConstantAsMetadata::get(llvm::ConstantInt::get(
603
2
            llvm::IntegerType::get(Context, 32),
604
1
            llvm::APInt(32, (uint64_t)(IsSignedInteger ? 1 : 0))))};
605
2
    Fn->setMetadata("vec_type_hint", llvm::MDNode::get(Context, AttrMDArgs));
606
2
  }
607
608
420
  if (const WorkGroupSizeHintAttr *A = FD->getAttr<WorkGroupSizeHintAttr>()) {
609
1
    llvm::Metadata *AttrMDArgs[] = {
610
1
        llvm::ConstantAsMetadata::get(Builder.getInt32(A->getXDim())),
611
1
        llvm::ConstantAsMetadata::get(Builder.getInt32(A->getYDim())),
612
1
        llvm::ConstantAsMetadata::get(Builder.getInt32(A->getZDim()))};
613
1
    Fn->setMetadata("work_group_size_hint", llvm::MDNode::get(Context, AttrMDArgs));
614
1
  }
615
616
420
  if (const ReqdWorkGroupSizeAttr *A = FD->getAttr<ReqdWorkGroupSizeAttr>()) {
617
7
    llvm::Metadata *AttrMDArgs[] = {
618
7
        llvm::ConstantAsMetadata::get(Builder.getInt32(A->getXDim())),
619
7
        llvm::ConstantAsMetadata::get(Builder.getInt32(A->getYDim())),
620
7
        llvm::ConstantAsMetadata::get(Builder.getInt32(A->getZDim()))};
621
7
    Fn->setMetadata("reqd_work_group_size", llvm::MDNode::get(Context, AttrMDArgs));
622
7
  }
623
624
420
  if (const OpenCLIntelReqdSubGroupSizeAttr *A =
625
1
          FD->getAttr<OpenCLIntelReqdSubGroupSizeAttr>()) {
626
1
    llvm::Metadata *AttrMDArgs[] = {
627
1
        llvm::ConstantAsMetadata::get(Builder.getInt32(A->getSubGroupSize()))};
628
1
    Fn->setMetadata("intel_reqd_sub_group_size",
629
1
                    llvm::MDNode::get(Context, AttrMDArgs));
630
1
  }
631
420
}
632
633
/// Determine whether the function F ends with a return stmt.
634
150k
static bool endsWithReturn(const Decl* F) {
635
150k
  const Stmt *Body = nullptr;
636
150k
  if (auto *FD = dyn_cast_or_null<FunctionDecl>(F))
637
111k
    Body = FD->getBody();
638
39.2k
  else if (auto *OMD = dyn_cast_or_null<ObjCMethodDecl>(F))
639
1.23k
    Body = OMD->getBody();
640
641
150k
  if (auto *CS = dyn_cast_or_null<CompoundStmt>(Body)) {
642
111k
    auto LastStmt = CS->body_rbegin();
643
111k
    if (LastStmt != CS->body_rend())
644
67.5k
      return isa<ReturnStmt>(*LastStmt);
645
83.3k
  }
646
83.3k
  return false;
647
83.3k
}
648
649
329
void CodeGenFunction::markAsIgnoreThreadCheckingAtRuntime(llvm::Function *Fn) {
650
329
  if (SanOpts.has(SanitizerKind::Thread)) {
651
4
    Fn->addFnAttr("sanitize_thread_no_checking_at_run_time");
652
4
    Fn->removeFnAttr(llvm::Attribute::SanitizeThread);
653
4
  }
654
329
}
655
656
/// Check if the return value of this function requires sanitization.
657
436k
bool CodeGenFunction::requiresReturnValueCheck() const {
658
436k
  return requiresReturnValueNullabilityCheck() ||
659
436k
         (SanOpts.has(SanitizerKind::ReturnsNonnullAttribute) && 
CurCodeDecl109
&&
660
109
          CurCodeDecl->getAttr<ReturnsNonNullAttr>());
661
436k
}
662
663
35
static bool matchesStlAllocatorFn(const Decl *D, const ASTContext &Ctx) {
664
35
  auto *MD = dyn_cast_or_null<CXXMethodDecl>(D);
665
35
  if (!MD || 
!MD->getDeclName().getAsIdentifierInfo()3
||
666
3
      !MD->getDeclName().getAsIdentifierInfo()->isStr("allocate") ||
667
2
      (MD->getNumParams() != 1 && 
MD->getNumParams() != 21
))
668
33
    return false;
669
670
2
  if (MD->parameters()[0]->getType().getCanonicalType() != Ctx.getSizeType())
671
0
    return false;
672
673
2
  if (MD->getNumParams() == 2) {
674
1
    auto *PT = MD->parameters()[1]->getType()->getAs<PointerType>();
675
1
    if (!PT || !PT->isVoidPointerType() ||
676
1
        !PT->getPointeeType().isConstQualified())
677
0
      return false;
678
2
  }
679
680
2
  return true;
681
2
}
682
683
/// Return the UBSan prologue signature for \p FD if one is available.
684
static llvm::Constant *getPrologueSignature(CodeGenModule &CGM,
685
203
                                            const FunctionDecl *FD) {
686
203
  if (const auto *MD = dyn_cast<CXXMethodDecl>(FD))
687
96
    if (!MD->isStatic())
688
90
      return nullptr;
689
113
  return CGM.getTargetCodeGenInfo().getUBSanFunctionSignature(CGM);
690
113
}
691
692
void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy,
693
                                    llvm::Function *Fn,
694
                                    const CGFunctionInfo &FnInfo,
695
                                    const FunctionArgList &Args,
696
                                    SourceLocation Loc,
697
293k
                                    SourceLocation StartLoc) {
698
293k
  assert(!CurFn &&
699
293k
         "Do not use a CodeGenFunction object for more than one function");
700
701
293k
  const Decl *D = GD.getDecl();
702
703
293k
  DidCallStackSave = false;
704
293k
  CurCodeDecl = D;
705
293k
  if (const auto *FD = dyn_cast_or_null<FunctionDecl>(D))
706
251k
    if (FD->usesSEHTry())
707
102
      CurSEHParent = FD;
708
285k
  CurFuncDecl = (D ? D->getNonClosureContext() : 
nullptr7.96k
);
709
293k
  FnRetTy = RetTy;
710
293k
  CurFn = Fn;
711
293k
  CurFnInfo = &FnInfo;
712
293k
  assert(CurFn->isDeclaration() && "Function already has body?");
713
714
  // If this function has been blacklisted for any of the enabled sanitizers,
715
  // disable the sanitizer for the function.
716
293k
  do {
717
293k
#define SANITIZER(NAME, ID)                                                    \
718
809k
  if (
SanOpts.empty()520k
) \
719
288k
    break;                                                                     \
720
231k
  if (SanOpts.has(SanitizerKind::ID))                                          \
721
7.41k
    if (CGM.isInSanitizerBlacklist(SanitizerKind::ID, Fn, Loc))                \
722
55
      SanOpts.set(SanitizerKind::ID, false);
723
724
293k
#include "clang/Basic/Sanitizers.def"
725
231k
#undef SANITIZER
726
4.19k
  } while (0);
727
728
293k
  if (D) {
729
    // Apply the no_sanitize* attributes to SanOpts.
730
954
    for (auto Attr : D->specific_attrs<NoSanitizeAttr>()) {
731
954
      SanitizerMask mask = Attr->getMask();
732
954
      SanOpts.Mask &= ~mask;
733
954
      if (mask & SanitizerKind::Address)
734
52
        SanOpts.set(SanitizerKind::KernelAddress, false);
735
954
      if (mask & SanitizerKind::KernelAddress)
736
5
        SanOpts.set(SanitizerKind::Address, false);
737
954
      if (mask & SanitizerKind::HWAddress)
738
5
        SanOpts.set(SanitizerKind::KernelHWAddress, false);
739
954
      if (mask & SanitizerKind::KernelHWAddress)
740
5
        SanOpts.set(SanitizerKind::HWAddress, false);
741
954
    }
742
285k
  }
743
744
  // Apply sanitizer attributes to the function.
745
293k
  if (SanOpts.hasOneOf(SanitizerKind::Address | SanitizerKind::KernelAddress))
746
1.04k
    Fn->addFnAttr(llvm::Attribute::SanitizeAddress);
747
293k
  if (SanOpts.hasOneOf(SanitizerKind::HWAddress | SanitizerKind::KernelHWAddress))
748
24
    Fn->addFnAttr(llvm::Attribute::SanitizeHWAddress);
749
293k
  if (SanOpts.has(SanitizerKind::MemTag))
750
37
    Fn->addFnAttr(llvm::Attribute::SanitizeMemTag);
751
293k
  if (SanOpts.has(SanitizerKind::Thread))
752
64
    Fn->addFnAttr(llvm::Attribute::SanitizeThread);
753
293k
  if (SanOpts.hasOneOf(SanitizerKind::Memory | SanitizerKind::KernelMemory))
754
208
    Fn->addFnAttr(llvm::Attribute::SanitizeMemory);
755
293k
  if (SanOpts.has(SanitizerKind::SafeStack))
756
14
    Fn->addFnAttr(llvm::Attribute::SafeStack);
757
293k
  if (SanOpts.has(SanitizerKind::ShadowCallStack))
758
3
    Fn->addFnAttr(llvm::Attribute::ShadowCallStack);
759
760
  // Apply fuzzing attribute to the function.
761
293k
  if (SanOpts.hasOneOf(SanitizerKind::Fuzzer | SanitizerKind::FuzzerNoLink))
762
2
    Fn->addFnAttr(llvm::Attribute::OptForFuzzing);
763
764
  // Ignore TSan memory acesses from within ObjC/ObjC++ dealloc, initialize,
765
  // .cxx_destruct, __destroy_helper_block_ and all of their calees at run time.
766
293k
  if (SanOpts.has(SanitizerKind::Thread)) {
767
64
    if (const auto *OMD = dyn_cast_or_null<ObjCMethodDecl>(D)) {
768
3
      IdentifierInfo *II = OMD->getSelector().getIdentifierInfoForSlot(0);
769
3
      if (OMD->getMethodFamily() == OMF_dealloc ||
770
2
          OMD->getMethodFamily() == OMF_initialize ||
771
3
          
(1
OMD->getSelector().isUnarySelector()1
&&
II->isStr(".cxx_destruct")1
)) {
772
3
        markAsIgnoreThreadCheckingAtRuntime(Fn);
773
3
      }
774
3
    }
775
64
  }
776
777
  // Ignore unrelated casts in STL allocate() since the allocator must cast
778
  // from void* to T* before object initialization completes. Don't match on the
779
  // namespace because not all allocators are in std::
780
293k
  if (D && 
SanOpts.has(SanitizerKind::CFIUnrelatedCast)285k
) {
781
35
    if (matchesStlAllocatorFn(D, getContext()))
782
2
      SanOpts.Mask &= ~SanitizerKind::CFIUnrelatedCast;
783
35
  }
784
785
  // Ignore null checks in coroutine functions since the coroutines passes
786
  // are not aware of how to move the extra UBSan instructions across the split
787
  // coroutine boundaries.
788
293k
  if (D && 
SanOpts.has(SanitizerKind::Null)285k
)
789
243
    if (const auto *FD = dyn_cast<FunctionDecl>(D))
790
243
      if (FD->getBody() &&
791
243
          FD->getBody()->getStmtClass() == Stmt::CoroutineBodyStmtClass)
792
1
        SanOpts.Mask &= ~SanitizerKind::Null;
793
794
  // Apply xray attributes to the function (as a string, for now)
795
293k
  bool AlwaysXRayAttr = false;
796
293k
  if (const auto *XRayAttr = D ? D->getAttr<XRayInstrumentAttr>() : nullptr) {
797
86
    if (CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
798
86
            XRayInstrKind::FunctionEntry) ||
799
5
        CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
800
82
            XRayInstrKind::FunctionExit)) {
801
82
      if (XRayAttr->alwaysXRayInstrument() && 
ShouldXRayInstrumentFunction()45
) {
802
38
        Fn->addFnAttr("function-instrument", "xray-always");
803
38
        AlwaysXRayAttr = true;
804
38
      }
805
82
      if (XRayAttr->neverXRayInstrument())
806
37
        Fn->addFnAttr("function-instrument", "xray-never");
807
82
      if (const auto *LogArgs = D->getAttr<XRayLogArgsAttr>())
808
1
        if (ShouldXRayInstrumentFunction())
809
1
          Fn->addFnAttr("xray-log-args",
810
1
                        llvm::utostr(LogArgs->getArgumentCount()));
811
82
    }
812
293k
  } else {
813
293k
    if (ShouldXRayInstrumentFunction() && 
!CGM.imbueXRayAttrs(Fn, Loc)31
)
814
19
      Fn->addFnAttr(
815
19
          "xray-instruction-threshold",
816
19
          llvm::itostr(CGM.getCodeGenOpts().XRayInstructionThreshold));
817
293k
  }
818
819
293k
  if (ShouldXRayInstrumentFunction()) {
820
103
    if (CGM.getCodeGenOpts().XRayIgnoreLoops)
821
1
      Fn->addFnAttr("xray-ignore-loops");
822
823
103
    if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
824
103
            XRayInstrKind::FunctionExit))
825
6
      Fn->addFnAttr("xray-skip-exit");
826
827
103
    if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
828
103
            XRayInstrKind::FunctionEntry))
829
6
      Fn->addFnAttr("xray-skip-entry");
830
831
103
    auto FuncGroups = CGM.getCodeGenOpts().XRayTotalFunctionGroups;
832
103
    if (FuncGroups > 1) {
833
15
      auto FuncName = llvm::makeArrayRef<uint8_t>(
834
15
          CurFn->getName().bytes_begin(), CurFn->getName().bytes_end());
835
15
      auto Group = crc32(FuncName) % FuncGroups;
836
15
      if (Group != CGM.getCodeGenOpts().XRaySelectedFunctionGroup &&
837
10
          !AlwaysXRayAttr)
838
8
        Fn->addFnAttr("function-instrument", "xray-never");
839
15
    }
840
103
  }
841
842
293k
  unsigned Count, Offset;
843
293k
  if (const auto *Attr =
844
14
          D ? D->getAttr<PatchableFunctionEntryAttr>() : nullptr) {
845
14
    Count = Attr->getCount();
846
14
    Offset = Attr->getOffset();
847
293k
  } else {
848
293k
    Count = CGM.getCodeGenOpts().PatchableFunctionEntryCount;
849
293k
    Offset = CGM.getCodeGenOpts().PatchableFunctionEntryOffset;
850
293k
  }
851
293k
  if (Count && 
Offset <= Count11
) {
852
11
    Fn->addFnAttr("patchable-function-entry", std::to_string(Count - Offset));
853
11
    if (Offset)
854
4
      Fn->addFnAttr("patchable-function-prefix", std::to_string(Offset));
855
11
  }
856
857
  // Add no-jump-tables value.
858
293k
  Fn->addFnAttr("no-jump-tables",
859
293k
                llvm::toStringRef(CGM.getCodeGenOpts().NoUseJumpTables));
860
861
  // Add no-inline-line-tables value.
862
293k
  if (CGM.getCodeGenOpts().NoInlineLineTables)
863
4
    Fn->addFnAttr("no-inline-line-tables");
864
865
  // Add profile-sample-accurate value.
866
293k
  if (CGM.getCodeGenOpts().ProfileSampleAccurate)
867
2
    Fn->addFnAttr("profile-sample-accurate");
868
869
293k
  if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
870
29
    Fn->addFnAttr("use-sample-profile");
871
872
293k
  if (D && 
D->hasAttr<CFICanonicalJumpTableAttr>()285k
)
873
2
    Fn->addFnAttr("cfi-canonical-jump-table");
874
875
293k
  if (getLangOpts().OpenCL) {
876
    // Add metadata for a kernel function.
877
1.71k
    if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D))
878
1.62k
      EmitOpenCLKernelMetadata(FD, Fn);
879
1.71k
  }
880
881
  // If we are checking function types, emit a function type signature as
882
  // prologue data.
883
293k
  if (getLangOpts().CPlusPlus && 
SanOpts.has(SanitizerKind::Function)199k
) {
884
210
    if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) {
885
203
      if (llvm::Constant *PrologueSig = getPrologueSignature(CGM, FD)) {
886
        // Remove any (C++17) exception specifications, to allow calling e.g. a
887
        // noexcept function through a non-noexcept pointer.
888
113
        auto ProtoTy =
889
113
          getContext().getFunctionTypeWithExceptionSpec(FD->getType(),
890
113
                                                        EST_None);
891
113
        llvm::Constant *FTRTTIConst =
892
113
            CGM.GetAddrOfRTTIDescriptor(ProtoTy, /*ForEH=*/true);
893
113
        llvm::Constant *FTRTTIConstEncoded =
894
113
            EncodeAddrForUseInPrologue(Fn, FTRTTIConst);
895
113
        llvm::Constant *PrologueStructElems[] = {PrologueSig,
896
113
                                                 FTRTTIConstEncoded};
897
113
        llvm::Constant *PrologueStructConst =
898
113
            llvm::ConstantStruct::getAnon(PrologueStructElems, /*Packed=*/true);
899
113
        Fn->setPrologueData(PrologueStructConst);
900
113
      }
901
203
    }
902
210
  }
903
904
  // If we're checking nullability, we need to know whether we can check the
905
  // return value. Initialize the flag to 'true' and refine it in EmitParmDecl.
906
293k
  if (SanOpts.has(SanitizerKind::NullabilityReturn)) {
907
60
    auto Nullability = FnRetTy->getNullability(getContext());
908
60
    if (Nullability && 
*Nullability == NullabilityKind::NonNull17
) {
909
15
      if (!(SanOpts.has(SanitizerKind::ReturnsNonnullAttribute) &&
910
1
            CurCodeDecl && CurCodeDecl->getAttr<ReturnsNonNullAttr>()))
911
14
        RetValNullabilityPrecondition =
912
14
            llvm::ConstantInt::getTrue(getLLVMContext());
913
15
    }
914
60
  }
915
916
  // If we're in C++ mode and the function name is "main", it is guaranteed
917
  // to be norecurse by the standard (3.6.1.3 "The function main shall not be
918
  // used within a program").
919
  //
920
  // OpenCL C 2.0 v2.2-11 s6.9.i:
921
  //     Recursion is not supported.
922
  //
923
  // SYCL v1.2.1 s3.10:
924
  //     kernels cannot include RTTI information, exception classes,
925
  //     recursive code, virtual functions or make use of C++ libraries that
926
  //     are not compiled for the device.
927
293k
  if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) {
928
251k
    if ((getLangOpts().CPlusPlus && 
FD->isMain()160k
) ||
getLangOpts().OpenCL247k
||
929
245k
        getLangOpts().SYCLIsDevice ||
930
245k
        (getLangOpts().CUDA && 
FD->hasAttr<CUDAGlobalAttr>()606
))
931
5.49k
      Fn->addFnAttr(llvm::Attribute::NoRecurse);
932
251k
  }
933
934
293k
  if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) {
935
251k
    Builder.setIsFPConstrained(FD->hasAttr<StrictFPAttr>());
936
251k
    if (FD->hasAttr<StrictFPAttr>())
937
1.66k
      Fn->addFnAttr(llvm::Attribute::StrictFP);
938
251k
  }
939
940
  // If a custom alignment is used, force realigning to this alignment on
941
  // any main function which certainly will need it.
942
293k
  if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D))
943
251k
    if ((FD->isMain() || 
FD->isMSVCRTEntryPoint()245k
) &&
944
5.30k
        CGM.getCodeGenOpts().StackAlignment)
945
1
      Fn->addFnAttr("stackrealign");
946
947
293k
  llvm::BasicBlock *EntryBB = createBasicBlock("entry", CurFn);
948
949
  // Create a marker to make it easy to insert allocas into the entryblock
950
  // later.  Don't create this with the builder, because we don't want it
951
  // folded.
952
293k
  llvm::Value *Undef = llvm::UndefValue::get(Int32Ty);
953
293k
  AllocaInsertPt = new llvm::BitCastInst(Undef, Int32Ty, "allocapt", EntryBB);
954
955
293k
  ReturnBlock = getJumpDestInCurrentScope("return");
956
957
293k
  Builder.SetInsertPoint(EntryBB);
958
959
  // If we're checking the return value, allocate space for a pointer to a
960
  // precise source location of the checked return statement.
961
293k
  if (requiresReturnValueCheck()) {
962
19
    ReturnLocation = CreateDefaultAlignTempAlloca(Int8PtrTy, "return.sloc.ptr");
963
19
    InitTempAlloca(ReturnLocation, llvm::ConstantPointerNull::get(Int8PtrTy));
964
19
  }
965
966
  // Emit subprogram debug descriptor.
967
293k
  if (CGDebugInfo *DI = getDebugInfo()) {
968
    // Reconstruct the type from the argument list so that implicit parameters,
969
    // such as 'this' and 'vtt', show up in the debug info. Preserve the calling
970
    // convention.
971
92.2k
    CallingConv CC = CallingConv::CC_C;
972
92.2k
    if (auto *FD = dyn_cast_or_null<FunctionDecl>(D))
973
89.3k
      if (const auto *SrcFnTy = FD->getType()->getAs<FunctionType>())
974
89.3k
        CC = SrcFnTy->getCallConv();
975
92.2k
    SmallVector<QualType, 16> ArgTypes;
976
92.2k
    for (const VarDecl *VD : Args)
977
154k
      ArgTypes.push_back(VD->getType());
978
92.2k
    QualType FnType = getContext().getFunctionType(
979
92.2k
        RetTy, ArgTypes, FunctionProtoType::ExtProtoInfo(CC));
980
92.2k
    DI->EmitFunctionStart(GD, Loc, StartLoc, FnType, CurFn, CurFuncIsThunk,
981
92.2k
                          Builder);
982
92.2k
  }
983
984
293k
  if (ShouldInstrumentFunction()) {
985
11
    if (CGM.getCodeGenOpts().InstrumentFunctions)
986
6
      CurFn->addFnAttr("instrument-function-entry", "__cyg_profile_func_enter");
987
11
    if (CGM.getCodeGenOpts().InstrumentFunctionsAfterInlining)
988
4
      CurFn->addFnAttr("instrument-function-entry-inlined",
989
4
                       "__cyg_profile_func_enter");
990
11
    if (CGM.getCodeGenOpts().InstrumentFunctionEntryBare)
991
1
      CurFn->addFnAttr("instrument-function-entry-inlined",
992
1
                       "__cyg_profile_func_enter_bare");
993
11
  }
994
995
  // Since emitting the mcount call here impacts optimizations such as function
996
  // inlining, we just add an attribute to insert a mcount call in backend.
997
  // The attribute "counting-function" is set to mcount function name which is
998
  // architecture dependent.
999
293k
  if (CGM.getCodeGenOpts().InstrumentForProfiling) {
1000
    // Calls to fentry/mcount should not be generated if function has
1001
    // the no_instrument_function attribute.
1002
119
    if (!CurFuncDecl || !CurFuncDecl->hasAttr<NoInstrumentFunctionAttr>()) {
1003
96
      if (CGM.getCodeGenOpts().CallFEntry)
1004
4
        Fn->addFnAttr("fentry-call", "true");
1005
92
      else {
1006
92
        Fn->addFnAttr("instrument-function-entry-inlined",
1007
92
                      getTarget().getMCountName());
1008
92
      }
1009
96
      if (CGM.getCodeGenOpts().MNopMCount) {
1010
2
        if (!CGM.getCodeGenOpts().CallFEntry)
1011
1
          CGM.getDiags().Report(diag::err_opt_not_valid_without_opt)
1012
1
            << "-mnop-mcount" << "-mfentry";
1013
2
        Fn->addFnAttr("mnop-mcount");
1014
2
      }
1015
1016
96
      if (CGM.getCodeGenOpts().RecordMCount) {
1017
2
        if (!CGM.getCodeGenOpts().CallFEntry)
1018
1
          CGM.getDiags().Report(diag::err_opt_not_valid_without_opt)
1019
1
            << "-mrecord-mcount" << "-mfentry";
1020
2
        Fn->addFnAttr("mrecord-mcount");
1021
2
      }
1022
96
    }
1023
119
  }
1024
1025
293k
  if (CGM.getCodeGenOpts().PackedStack) {
1026
2
    if (getContext().getTargetInfo().getTriple().getArch() !=
1027
2
        llvm::Triple::systemz)
1028
1
      CGM.getDiags().Report(diag::err_opt_not_valid_on_target)
1029
1
        << "-mpacked-stack";
1030
2
    Fn->addFnAttr("packed-stack");
1031
2
  }
1032
1033
293k
  if (RetTy->isVoidType()) {
1034
    // Void type; nothing to return.
1035
150k
    ReturnValue = Address::invalid();
1036
1037
    // Count the implicit return.
1038
150k
    if (!endsWithReturn(D))
1039
148k
      ++NumReturnExprs;
1040
142k
  } else if (CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::Indirect) {
1041
    // Indirect return; emit returned value directly into sret slot.
1042
    // This reduces code size, and affects correctness in C++.
1043
2.53k
    auto AI = CurFn->arg_begin();
1044
2.53k
    if (CurFnInfo->getReturnInfo().isSRetAfterThis())
1045
50
      ++AI;
1046
2.53k
    ReturnValue = Address(&*AI, CurFnInfo->getReturnInfo().getIndirectAlign());
1047
2.53k
    if (!CurFnInfo->getReturnInfo().getIndirectByVal()) {
1048
1.00k
      ReturnValuePointer =
1049
1.00k
          CreateDefaultAlignTempAlloca(Int8PtrTy, "result.ptr");
1050
1.00k
      Builder.CreateStore(Builder.CreatePointerBitCastOrAddrSpaceCast(
1051
1.00k
                              ReturnValue.getPointer(), Int8PtrTy),
1052
1.00k
                          ReturnValuePointer);
1053
1.00k
    }
1054
139k
  } else if (CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::InAlloca &&
1055
5
             !hasScalarEvaluationKind(CurFnInfo->getReturnType())) {
1056
    // Load the sret pointer from the argument struct and return into that.
1057
5
    unsigned Idx = CurFnInfo->getReturnInfo().getInAllocaFieldIndex();
1058
5
    llvm::Function::arg_iterator EI = CurFn->arg_end();
1059
5
    --EI;
1060
5
    llvm::Value *Addr = Builder.CreateStructGEP(nullptr, &*EI, Idx);
1061
5
    ReturnValuePointer = Address(Addr, getPointerAlign());
1062
5
    Addr = Builder.CreateAlignedLoad(Addr, getPointerAlign(), "agg.result");
1063
5
    ReturnValue = Address(Addr, CGM.getNaturalTypeAlignment(RetTy));
1064
139k
  } else {
1065
139k
    ReturnValue = CreateIRTemp(RetTy, "retval");
1066
1067
    // Tell the epilog emitter to autorelease the result.  We do this
1068
    // now so that various specialized functions can suppress it
1069
    // during their IR-generation.
1070
139k
    if (getLangOpts().ObjCAutoRefCount &&
1071
389
        !CurFnInfo->isReturnsRetained() &&
1072
355
        RetTy->isObjCRetainableType())
1073
148
      AutoreleaseResult = true;
1074
139k
  }
1075
1076
293k
  EmitStartEHSpec(CurCodeDecl);
1077
1078
293k
  PrologueCleanupDepth = EHStack.stable_begin();
1079
1080
  // Emit OpenMP specific initialization of the device functions.
1081
293k
  if (getLangOpts().OpenMP && 
CurCodeDecl68.3k
)
1082
61.8k
    CGM.getOpenMPRuntime().emitFunctionProlog(*this, CurCodeDecl);
1083
1084
293k
  EmitFunctionProlog(*CurFnInfo, CurFn, Args);
1085
1086
293k
  if (D && 
isa<CXXMethodDecl>(D)285k
&&
cast<CXXMethodDecl>(D)->isInstance()103k
) {
1087
96.5k
    CGM.getCXXABI().EmitInstanceFunctionProlog(*this);
1088
96.5k
    const CXXMethodDecl *MD = cast<CXXMethodDecl>(D);
1089
96.5k
    if (MD->getParent()->isLambda() &&
1090
1.67k
        MD->getOverloadedOperator() == OO_Call) {
1091
      // We're in a lambda; figure out the captures.
1092
1.49k
      MD->getParent()->getCaptureFields(LambdaCaptureFields,
1093
1.49k
                                        LambdaThisCaptureField);
1094
1.49k
      if (LambdaThisCaptureField) {
1095
        // If the lambda captures the object referred to by '*this' - either by
1096
        // value or by reference, make sure CXXThisValue points to the correct
1097
        // object.
1098
1099
        // Get the lvalue for the field (which is a copy of the enclosing object
1100
        // or contains the address of the enclosing object).
1101
82
        LValue ThisFieldLValue = EmitLValueForLambdaField(LambdaThisCaptureField);
1102
82
        if (!LambdaThisCaptureField->getType()->isPointerType()) {
1103
          // If the enclosing object was captured by value, just use its address.
1104
1
          CXXThisValue = ThisFieldLValue.getAddress(*this).getPointer();
1105
81
        } else {
1106
          // Load the lvalue pointed to by the field, since '*this' was captured
1107
          // by reference.
1108
81
          CXXThisValue =
1109
81
              EmitLoadOfLValue(ThisFieldLValue, SourceLocation()).getScalarVal();
1110
81
        }
1111
82
      }
1112
2.52k
      for (auto *FD : MD->getParent()->fields()) {
1113
2.52k
        if (FD->hasCapturedVLAType()) {
1114
24
          auto *ExprArg = EmitLoadOfLValue(EmitLValueForLambdaField(FD),
1115
24
                                           SourceLocation()).getScalarVal();
1116
24
          auto VAT = FD->getCapturedVLAType();
1117
24
          VLASizeMap[VAT->getSizeExpr()] = ExprArg;
1118
24
        }
1119
2.52k
      }
1120
95.0k
    } else {
1121
      // Not in a lambda; just use 'this' from the method.
1122
      // FIXME: Should we generate a new load for each use of 'this'?  The
1123
      // fast register allocator would be happier...
1124
95.0k
      CXXThisValue = CXXABIThisValue;
1125
95.0k
    }
1126
1127
    // Check the 'this' pointer once per function, if it's available.
1128
96.5k
    if (CXXABIThisValue) {
1129
96.5k
      SanitizerSet SkippedChecks;
1130
96.5k
      SkippedChecks.set(SanitizerKind::ObjectSize, true);
1131
96.5k
      QualType ThisTy = MD->getThisType();
1132
1133
      // If this is the call operator of a lambda with no capture-default, it
1134
      // may have a static invoker function, which may call this operator with
1135
      // a null 'this' pointer.
1136
96.5k
      if (isLambdaCallOperator(MD) &&
1137
1.49k
          MD->getParent()->getLambdaCaptureDefault() == LCD_None)
1138
317
        SkippedChecks.set(SanitizerKind::Null, true);
1139
1140
38.2k
      EmitTypeCheck(isa<CXXConstructorDecl>(MD) ? TCK_ConstructorCall
1141
58.3k
                                                : TCK_MemberCall,
1142
96.5k
                    Loc, CXXABIThisValue, ThisTy,
1143
96.5k
                    getContext().getTypeAlignInChars(ThisTy->getPointeeType()),
1144
96.5k
                    SkippedChecks);
1145
96.5k
    }
1146
96.5k
  }
1147
1148
  // If any of the arguments have a variably modified type, make sure to
1149
  // emit the type size.
1150
293k
  for (FunctionArgList::const_iterator i = Args.begin(), e = Args.end();
1151
776k
       i != e; 
++i483k
) {
1152
483k
    const VarDecl *VD = *i;
1153
1154
    // Dig out the type as written from ParmVarDecls; it's unclear whether
1155
    // the standard (C99 6.9.1p10) requires this, but we're following the
1156
    // precedent set by gcc.
1157
483k
    QualType Ty;
1158
483k
    if (const ParmVarDecl *PVD = dyn_cast<ParmVarDecl>(VD))
1159
298k
      Ty = PVD->getOriginalType();
1160
184k
    else
1161
184k
      Ty = VD->getType();
1162
1163
483k
    if (Ty->isVariablyModifiedType())
1164
137
      EmitVariablyModifiedType(Ty);
1165
483k
  }
1166
  // Emit a location at the end of the prologue.
1167
293k
  if (CGDebugInfo *DI = getDebugInfo())
1168
92.2k
    DI->EmitLocation(Builder, StartLoc);
1169
1170
  // TODO: Do we need to handle this in two places like we do with
1171
  // target-features/target-cpu?
1172
293k
  if (CurFuncDecl)
1173
278k
    if (const auto *VecWidth = CurFuncDecl->getAttr<MinVectorWidthAttr>())
1174
7.37k
      LargestVectorWidth = VecWidth->getVectorWidth();
1175
293k
}
1176
1177
194k
void CodeGenFunction::EmitFunctionBody(const Stmt *Body) {
1178
194k
  incrementProfileCounter(Body);
1179
194k
  if (CPlusPlusWithProgress())
1180
103k
    FnIsMustProgress = true;
1181
1182
194k
  if (const CompoundStmt *S = dyn_cast<CompoundStmt>(Body))
1183
194k
    EmitCompoundStmtWithoutScope(*S);
1184
57
  else
1185
57
    EmitStmt(Body);
1186
1187
  // This is checked after emitting the function body so we know if there
1188
  // are any permitted infinite loops.
1189
194k
  if (FnIsMustProgress)
1190
103k
    CurFn->addFnAttr(llvm::Attribute::MustProgress);
1191
194k
}
1192
1193
/// When instrumenting to collect profile data, the counts for some blocks
1194
/// such as switch cases need to not include the fall-through counts, so
1195
/// emit a branch around the instrumentation code. When not instrumenting,
1196
/// this just calls EmitBlock().
1197
void CodeGenFunction::EmitBlockWithFallThrough(llvm::BasicBlock *BB,
1198
1.76k
                                               const Stmt *S) {
1199
1.76k
  llvm::BasicBlock *SkipCountBB = nullptr;
1200
1.76k
  if (HaveInsertPoint() && 
CGM.getCodeGenOpts().hasProfileClangInstr()692
) {
1201
    // When instrumenting for profiling, the fallthrough to certain
1202
    // statements needs to skip over the instrumentation code so that we
1203
    // get an accurate count.
1204
22
    SkipCountBB = createBasicBlock("skipcount");
1205
22
    EmitBranch(SkipCountBB);
1206
22
  }
1207
1.76k
  EmitBlock(BB);
1208
1.76k
  uint64_t CurrentCount = getCurrentProfileCount();
1209
1.76k
  incrementProfileCounter(S);
1210
1.76k
  setCurrentProfileCount(getCurrentProfileCount() + CurrentCount);
1211
1.76k
  if (SkipCountBB)
1212
22
    EmitBlock(SkipCountBB);
1213
1.76k
}
1214
1215
/// Tries to mark the given function nounwind based on the
1216
/// non-existence of any throwing calls within it.  We believe this is
1217
/// lightweight enough to do at -O0.
1218
187k
static void TryMarkNoThrow(llvm::Function *F) {
1219
  // LLVM treats 'nounwind' on a function as part of the type, so we
1220
  // can't do this on functions that can be overwritten.
1221
187k
  if (F->isInterposable()) 
return23
;
1222
1223
187k
  for (llvm::BasicBlock &BB : *F)
1224
290k
    for (llvm::Instruction &I : BB)
1225
2.65M
      if (I.mayThrow())
1226
67.5k
        return;
1227
1228
119k
  F->setDoesNotThrow();
1229
119k
}
1230
1231
QualType CodeGenFunction::BuildFunctionArgList(GlobalDecl GD,
1232
250k
                                               FunctionArgList &Args) {
1233
250k
  const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
1234
250k
  QualType ResTy = FD->getReturnType();
1235
1236
250k
  const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(FD);
1237
250k
  if (MD && 
MD->isInstance()103k
) {
1238
96.5k
    if (CGM.getCXXABI().HasThisReturn(GD))
1239
1.65k
      ResTy = MD->getThisType();
1240
94.9k
    else if (CGM.getCXXABI().hasMostDerivedReturn(GD))
1241
214
      ResTy = CGM.getContext().VoidPtrTy;
1242
96.5k
    CGM.getCXXABI().buildThisParam(*this, Args);
1243
96.5k
  }
1244
1245
  // The base version of an inheriting constructor whose constructed base is a
1246
  // virtual base is not passed any arguments (because it doesn't actually call
1247
  // the inherited constructor).
1248
250k
  bool PassedParams = true;
1249
250k
  if (const CXXConstructorDecl *CD = dyn_cast<CXXConstructorDecl>(FD))
1250
38.2k
    if (auto Inherited = CD->getInheritedConstructor())
1251
225
      PassedParams =
1252
225
          getTypes().inheritingCtorHasParams(Inherited, GD.getCtorType());
1253
1254
250k
  if (PassedParams) {
1255
296k
    for (auto *Param : FD->parameters()) {
1256
296k
      Args.push_back(Param);
1257
296k
      if (!Param->hasAttr<PassObjectSizeAttr>())
1258
296k
        continue;
1259
1260
88
      auto *Implicit = ImplicitParamDecl::Create(
1261
88
          getContext(), Param->getDeclContext(), Param->getLocation(),
1262
88
          /*Id=*/nullptr, getContext().getSizeType(), ImplicitParamDecl::Other);
1263
88
      SizeArguments[Param] = Implicit;
1264
88
      Args.push_back(Implicit);
1265
88
    }
1266
250k
  }
1267
1268
250k
  if (MD && 
(103k
isa<CXXConstructorDecl>(MD)103k
||
isa<CXXDestructorDecl>(MD)64.9k
))
1269
54.5k
    CGM.getCXXABI().addImplicitStructorParams(*this, ResTy, Args);
1270
1271
250k
  return ResTy;
1272
250k
}
1273
1274
static bool
1275
shouldUseUndefinedBehaviorReturnOptimization(const FunctionDecl *FD,
1276
21
                                             const ASTContext &Context) {
1277
21
  QualType T = FD->getReturnType();
1278
  // Avoid the optimization for functions that return a record type with a
1279
  // trivial destructor or another trivially copyable type.
1280
21
  if (const RecordType *RT = T.getCanonicalType()->getAs<RecordType>()) {
1281
12
    if (const auto *ClassDecl = dyn_cast<CXXRecordDecl>(RT->getDecl()))
1282
12
      return !ClassDecl->hasTrivialDestructor();
1283
9
  }
1284
9
  return !T.isTriviallyCopyableType(Context);
1285
9
}
1286
1287
void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn,
1288
250k
                                   const CGFunctionInfo &FnInfo) {
1289
250k
  const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
1290
250k
  CurGD = GD;
1291
1292
250k
  FunctionArgList Args;
1293
250k
  QualType ResTy = BuildFunctionArgList(GD, Args);
1294
1295
  // Check if we should generate debug info for this function.
1296
250k
  if (FD->hasAttr<NoDebugAttr>())
1297
12.4k
    DebugInfo = nullptr; // disable debug info indefinitely for this function
1298
1299
  // The function might not have a body if we're generating thunks for a
1300
  // function declaration.
1301
250k
  SourceRange BodyRange;
1302
250k
  if (Stmt *Body = FD->getBody())
1303
249k
    BodyRange = Body->getSourceRange();
1304
128
  else
1305
128
    BodyRange = FD->getLocation();
1306
250k
  CurEHLocation = BodyRange.getEnd();
1307
1308
  // Use the location of the start of the function to determine where
1309
  // the function definition is located. By default use the location
1310
  // of the declaration as the location for the subprogram. A function
1311
  // may lack a declaration in the source code if it is created by code
1312
  // gen. (examples: _GLOBAL__I_a, __cxx_global_array_dtor, thunk).
1313
250k
  SourceLocation Loc = FD->getLocation();
1314
1315
  // If this is a function specialization then use the pattern body
1316
  // as the location for the function.
1317
250k
  if (const FunctionDecl *SpecDecl = FD->getTemplateInstantiationPattern())
1318
90.2k
    if (SpecDecl->hasBody(SpecDecl))
1319
90.1k
      Loc = SpecDecl->getLocation();
1320
1321
250k
  Stmt *Body = FD->getBody();
1322
1323
  // Initialize helper which will detect jumps which can cause invalid lifetime
1324
  // markers.
1325
250k
  if (Body && 
ShouldEmitLifetimeMarkers249k
)
1326
32.1k
    Bypasses.Init(Body);
1327
1328
  // Emit the standard function prologue.
1329
250k
  StartFunction(GD, ResTy, Fn, FnInfo, Args, Loc, BodyRange.getBegin());
1330
1331
  // Generate the body of the function.
1332
250k
  PGO.assignRegionCounters(GD, CurFn);
1333
250k
  if (isa<CXXDestructorDecl>(FD))
1334
16.2k
    EmitDestructorBody(Args);
1335
233k
  else if (isa<CXXConstructorDecl>(FD))
1336
38.2k
    EmitConstructorBody(Args);
1337
195k
  else if (getLangOpts().CUDA &&
1338
449
           !getLangOpts().CUDAIsDevice &&
1339
145
           FD->hasAttr<CUDAGlobalAttr>())
1340
46
    CGM.getCUDARuntime().emitDeviceStub(*this, Args);
1341
195k
  else if (isa<CXXMethodDecl>(FD) &&
1342
48.7k
           cast<CXXMethodDecl>(FD)->isLambdaStaticInvoker()) {
1343
    // The lambda static invoker function is special, because it forwards or
1344
    // clones the body of the function call operator (but is actually static).
1345
56
    EmitLambdaStaticInvokeBody(cast<CXXMethodDecl>(FD));
1346
195k
  } else if (FD->isDefaulted() && 
isa<CXXMethodDecl>(FD)731
&&
1347
727
             (cast<CXXMethodDecl>(FD)->isCopyAssignmentOperator() ||
1348
711
              
cast<CXXMethodDecl>(FD)->isMoveAssignmentOperator()209
)) {
1349
    // Implicit copy-assignment gets the same special treatment as implicit
1350
    // copy-constructors.
1351
711
    emitImplicitAssignmentOperatorBody(Args);
1352
194k
  } else if (Body) {
1353
194k
    EmitFunctionBody(Body);
1354
194k
  } else
1355
0
    llvm_unreachable("no definition for emitted function");
1356
1357
  // C++11 [stmt.return]p2:
1358
  //   Flowing off the end of a function [...] results in undefined behavior in
1359
  //   a value-returning function.
1360
  // C11 6.9.1p12:
1361
  //   If the '}' that terminates a function is reached, and the value of the
1362
  //   function call is used by the caller, the behavior is undefined.
1363
250k
  if (getLangOpts().CPlusPlus && 
!FD->hasImplicitReturnZero()160k
&&
!SawAsmBlock156k
&&
1364
156k
      !FD->getReturnType()->isVoidType() && 
Builder.GetInsertBlock()59.5k
) {
1365
178
    bool ShouldEmitUnreachable =
1366
178
        CGM.getCodeGenOpts().StrictReturn ||
1367
21
        shouldUseUndefinedBehaviorReturnOptimization(FD, getContext());
1368
178
    if (SanOpts.has(SanitizerKind::Return)) {
1369
1
      SanitizerScope SanScope(this);
1370
1
      llvm::Value *IsFalse = Builder.getFalse();
1371
1
      EmitCheck(std::make_pair(IsFalse, SanitizerKind::Return),
1372
1
                SanitizerHandler::MissingReturn,
1373
1
                EmitCheckSourceLocation(FD->getLocation()), None);
1374
177
    } else if (ShouldEmitUnreachable) {
1375
159
      if (CGM.getCodeGenOpts().OptimizationLevel == 0)
1376
128
        EmitTrapCall(llvm::Intrinsic::trap);
1377
159
    }
1378
178
    if (SanOpts.has(SanitizerKind::Return) || 
ShouldEmitUnreachable177
) {
1379
160
      Builder.CreateUnreachable();
1380
160
      Builder.ClearInsertionPoint();
1381
160
    }
1382
178
  }
1383
1384
  // Emit the standard function epilogue.
1385
250k
  FinishFunction(BodyRange.getEnd());
1386
1387
  // If we haven't marked the function nothrow through other means, do
1388
  // a quick pass now to see if we can.
1389
250k
  if (!CurFn->doesNotThrow())
1390
187k
    TryMarkNoThrow(CurFn);
1391
250k
}
1392
1393
/// ContainsLabel - Return true if the statement contains a label in it.  If
1394
/// this statement is not executed normally, it not containing a label means
1395
/// that we can just remove the code.
1396
62.1k
bool CodeGenFunction::ContainsLabel(const Stmt *S, bool IgnoreCaseStmts) {
1397
  // Null statement, not a label!
1398
62.1k
  if (!S) 
return false87
;
1399
1400
  // If this is a label, we have to emit the code, consider something like:
1401
  // if (0) {  ...  foo:  bar(); }  goto foo;
1402
  //
1403
  // TODO: If anyone cared, we could track __label__'s, since we know that you
1404
  // can't jump to one from outside their declared region.
1405
62.0k
  if (isa<LabelStmt>(S))
1406
6
    return true;
1407
1408
  // If this is a case/default statement, and we haven't seen a switch, we have
1409
  // to emit the code.
1410
62.0k
  if (isa<SwitchCase>(S) && 
!IgnoreCaseStmts68
)
1411
0
    return true;
1412
1413
  // If this is a switch statement, we want to ignore cases below it.
1414
62.0k
  if (isa<SwitchStmt>(S))
1415
10
    IgnoreCaseStmts = true;
1416
1417
  // Scan subexpressions for verboten labels.
1418
62.0k
  for (const Stmt *SubStmt : S->children())
1419
46.0k
    if (ContainsLabel(SubStmt, IgnoreCaseStmts))
1420
13
      return true;
1421
1422
62.0k
  return false;
1423
62.0k
}
1424
1425
/// containsBreak - Return true if the statement contains a break out of it.
1426
/// If the statement (recursively) contains a switch or loop with a break
1427
/// inside of it, this is fine.
1428
952
bool CodeGenFunction::containsBreak(const Stmt *S) {
1429
  // Null statement, not a label!
1430
952
  if (!S) 
return false0
;
1431
1432
  // If this is a switch or loop that defines its own break scope, then we can
1433
  // include it and anything inside of it.
1434
952
  if (isa<SwitchStmt>(S) || isa<WhileStmt>(S) || isa<DoStmt>(S) ||
1435
949
      isa<ForStmt>(S))
1436
3
    return false;
1437
1438
949
  if (isa<BreakStmt>(S))
1439
1
    return true;
1440
1441
  // Scan subexpressions for verboten breaks.
1442
948
  for (const Stmt *SubStmt : S->children())
1443
870
    if (containsBreak(SubStmt))
1444
1
      return true;
1445
1446
947
  return false;
1447
948
}
1448
1449
2.61k
bool CodeGenFunction::mightAddDeclToScope(const Stmt *S) {
1450
2.61k
  if (!S) 
return false0
;
1451
1452
  // Some statement kinds add a scope and thus never add a decl to the current
1453
  // scope. Note, this list is longer than the list of statements that might
1454
  // have an unscoped decl nested within them, but this way is conservatively
1455
  // correct even if more statement kinds are added.
1456
2.61k
  if (isa<IfStmt>(S) || 
isa<SwitchStmt>(S)2.61k
||
isa<WhileStmt>(S)2.61k
||
1457
2.61k
      isa<DoStmt>(S) || 
isa<ForStmt>(S)2.61k
||
isa<CompoundStmt>(S)2.61k
||
1458
2.60k
      isa<CXXForRangeStmt>(S) || isa<CXXTryStmt>(S) ||
1459
2.60k
      isa<ObjCForCollectionStmt>(S) || isa<ObjCAtTryStmt>(S))
1460
15
    return false;
1461
1462
2.60k
  if (isa<DeclStmt>(S))
1463
7
    return true;
1464
1465
2.59k
  for (const Stmt *SubStmt : S->children())
1466
2.34k
    if (mightAddDeclToScope(SubStmt))
1467
2
      return true;
1468
1469
2.59k
  return false;
1470
2.59k
}
1471
1472
/// ConstantFoldsToSimpleInteger - If the specified expression does not fold
1473
/// to a constant, or if it does but contains a label, return false.  If it
1474
/// constant folds return true and set the boolean result in Result.
1475
bool CodeGenFunction::ConstantFoldsToSimpleInteger(const Expr *Cond,
1476
                                                   bool &ResultBool,
1477
108k
                                                   bool AllowLabels) {
1478
108k
  llvm::APSInt ResultInt;
1479
108k
  if (!ConstantFoldsToSimpleInteger(Cond, ResultInt, AllowLabels))
1480
93.9k
    return false;
1481
1482
14.3k
  ResultBool = ResultInt.getBoolValue();
1483
14.3k
  return true;
1484
14.3k
}
1485
1486
/// ConstantFoldsToSimpleInteger - If the specified expression does not fold
1487
/// to a constant, or if it does but contains a label, return false.  If it
1488
/// constant folds return true and set the folded value.
1489
bool CodeGenFunction::ConstantFoldsToSimpleInteger(const Expr *Cond,
1490
                                                   llvm::APSInt &ResultInt,
1491
108k
                                                   bool AllowLabels) {
1492
  // FIXME: Rename and handle conversion of other evaluatable things
1493
  // to bool.
1494
108k
  Expr::EvalResult Result;
1495
108k
  if (!Cond->EvaluateAsInt(Result, getContext()))
1496
94.3k
    return false;  // Not foldable, not integer or not fully evaluatable.
1497
1498
14.4k
  llvm::APSInt Int = Result.Val.getInt();
1499
14.4k
  if (!AllowLabels && 
CodeGenFunction::ContainsLabel(Cond)14.4k
)
1500
0
    return false;  // Contains a label.
1501
1502
14.4k
  ResultInt = Int;
1503
14.4k
  return true;
1504
14.4k
}
1505
1506
/// EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g. for an if
1507
/// statement) to the specified blocks.  Based on the condition, this might try
1508
/// to simplify the codegen of the conditional based on the branch.
1509
/// \param LH The value of the likelihood attribute on the True branch.
1510
void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond,
1511
                                           llvm::BasicBlock *TrueBlock,
1512
                                           llvm::BasicBlock *FalseBlock,
1513
                                           uint64_t TrueCount,
1514
108k
                                           Stmt::Likelihood LH) {
1515
108k
  Cond = Cond->IgnoreParens();
1516
1517
108k
  if (const BinaryOperator *CondBOp = dyn_cast<BinaryOperator>(Cond)) {
1518
1519
    // Handle X && Y in a condition.
1520
55.1k
    if (CondBOp->getOpcode() == BO_LAnd) {
1521
      // If we have "1 && X", simplify the code.  "0 && X" would have constant
1522
      // folded if the case was simple enough.
1523
3.20k
      bool ConstantBool = false;
1524
3.20k
      if (ConstantFoldsToSimpleInteger(CondBOp->getLHS(), ConstantBool) &&
1525
59
          ConstantBool) {
1526
        // br(1 && X) -> br(X).
1527
59
        incrementProfileCounter(CondBOp);
1528
59
        return EmitBranchOnBoolExpr(CondBOp->getRHS(), TrueBlock, FalseBlock,
1529
59
                                    TrueCount, LH);
1530
59
      }
1531
1532
      // If we have "X && 1", simplify the code to use an uncond branch.
1533
      // "X && 0" would have been constant folded to 0.
1534
3.14k
      if (ConstantFoldsToSimpleInteger(CondBOp->getRHS(), ConstantBool) &&
1535
20
          ConstantBool) {
1536
        // br(X && 1) -> br(X).
1537
20
        return EmitBranchOnBoolExpr(CondBOp->getLHS(), TrueBlock, FalseBlock,
1538
20
                                    TrueCount, LH);
1539
20
      }
1540
1541
      // Emit the LHS as a conditional.  If the LHS conditional is false, we
1542
      // want to jump to the FalseBlock.
1543
3.12k
      llvm::BasicBlock *LHSTrue = createBasicBlock("land.lhs.true");
1544
      // The counter tells us how often we evaluate RHS, and all of TrueCount
1545
      // can be propagated to that branch.
1546
3.12k
      uint64_t RHSCount = getProfileCount(CondBOp->getRHS());
1547
1548
3.12k
      ConditionalEvaluation eval(*this);
1549
3.12k
      {
1550
3.12k
        ApplyDebugLocation DL(*this, Cond);
1551
        // Propagate the likelihood attribute like __builtin_expect
1552
        // __builtin_expect(X && Y, 1) -> X and Y are likely
1553
        // __builtin_expect(X && Y, 0) -> only Y is unlikely
1554
3.12k
        EmitBranchOnBoolExpr(CondBOp->getLHS(), LHSTrue, FalseBlock, RHSCount,
1555
3.12k
                             LH == Stmt::LH_Unlikely ? 
Stmt::LH_None1
: LH);
1556
3.12k
        EmitBlock(LHSTrue);
1557
3.12k
      }
1558
1559
3.12k
      incrementProfileCounter(CondBOp);
1560
3.12k
      setCurrentProfileCount(getProfileCount(CondBOp->getRHS()));
1561
1562
      // Any temporaries created here are conditional.
1563
3.12k
      eval.begin(*this);
1564
3.12k
      EmitBranchOnBoolExpr(CondBOp->getRHS(), TrueBlock, FalseBlock, TrueCount,
1565
3.12k
                           LH);
1566
3.12k
      eval.end(*this);
1567
1568
3.12k
      return;
1569
3.12k
    }
1570
1571
51.9k
    if (CondBOp->getOpcode() == BO_LOr) {
1572
      // If we have "0 || X", simplify the code.  "1 || X" would have constant
1573
      // folded if the case was simple enough.
1574
3.74k
      bool ConstantBool = false;
1575
3.74k
      if (ConstantFoldsToSimpleInteger(CondBOp->getLHS(), ConstantBool) &&
1576
6
          !ConstantBool) {
1577
        // br(0 || X) -> br(X).
1578
6
        incrementProfileCounter(CondBOp);
1579
6
        return EmitBranchOnBoolExpr(CondBOp->getRHS(), TrueBlock, FalseBlock,
1580
6
                                    TrueCount, LH);
1581
6
      }
1582
1583
      // If we have "X || 0", simplify the code to use an uncond branch.
1584
      // "X || 1" would have been constant folded to 1.
1585
3.73k
      if (ConstantFoldsToSimpleInteger(CondBOp->getRHS(), ConstantBool) &&
1586
0
          !ConstantBool) {
1587
        // br(X || 0) -> br(X).
1588
0
        return EmitBranchOnBoolExpr(CondBOp->getLHS(), TrueBlock, FalseBlock,
1589
0
                                    TrueCount, LH);
1590
0
      }
1591
1592
      // Emit the LHS as a conditional.  If the LHS conditional is true, we
1593
      // want to jump to the TrueBlock.
1594
3.73k
      llvm::BasicBlock *LHSFalse = createBasicBlock("lor.lhs.false");
1595
      // We have the count for entry to the RHS and for the whole expression
1596
      // being true, so we can divy up True count between the short circuit and
1597
      // the RHS.
1598
3.73k
      uint64_t LHSCount =
1599
3.73k
          getCurrentProfileCount() - getProfileCount(CondBOp->getRHS());
1600
3.73k
      uint64_t RHSCount = TrueCount - LHSCount;
1601
1602
3.73k
      ConditionalEvaluation eval(*this);
1603
3.73k
      {
1604
        // Propagate the likelihood attribute like __builtin_expect
1605
        // __builtin_expect(X || Y, 1) -> only Y is likely
1606
        // __builtin_expect(X || Y, 0) -> both X and Y are unlikely
1607
3.73k
        ApplyDebugLocation DL(*this, Cond);
1608
3.73k
        EmitBranchOnBoolExpr(CondBOp->getLHS(), TrueBlock, LHSFalse, LHSCount,
1609
3.73k
                             LH == Stmt::LH_Likely ? 
Stmt::LH_None1
: LH);
1610
3.73k
        EmitBlock(LHSFalse);
1611
3.73k
      }
1612
1613
3.73k
      incrementProfileCounter(CondBOp);
1614
3.73k
      setCurrentProfileCount(getProfileCount(CondBOp->getRHS()));
1615
1616
      // Any temporaries created here are conditional.
1617
3.73k
      eval.begin(*this);
1618
3.73k
      EmitBranchOnBoolExpr(CondBOp->getRHS(), TrueBlock, FalseBlock, RHSCount,
1619
3.73k
                           LH);
1620
1621
3.73k
      eval.end(*this);
1622
1623
3.73k
      return;
1624
3.73k
    }
1625
51.9k
  }
1626
1627
101k
  if (const UnaryOperator *CondUOp = dyn_cast<UnaryOperator>(Cond)) {
1628
    // br(!x, t, f) -> br(x, f, t)
1629
1.25k
    if (CondUOp->getOpcode() == UO_LNot) {
1630
      // Negate the count.
1631
1.25k
      uint64_t FalseCount = getCurrentProfileCount() - TrueCount;
1632
      // The values of the enum are chosen to make this negation possible.
1633
1.25k
      LH = static_cast<Stmt::Likelihood>(-LH);
1634
      // Negate the condition and swap the destination blocks.
1635
1.25k
      return EmitBranchOnBoolExpr(CondUOp->getSubExpr(), FalseBlock, TrueBlock,
1636
1.25k
                                  FalseCount, LH);
1637
1.25k
    }
1638
100k
  }
1639
1640
100k
  if (const ConditionalOperator *CondOp = dyn_cast<ConditionalOperator>(Cond)) {
1641
    // br(c ? x : y, t, f) -> br(c, br(x, t, f), br(y, t, f))
1642
2
    llvm::BasicBlock *LHSBlock = createBasicBlock("cond.true");
1643
2
    llvm::BasicBlock *RHSBlock = createBasicBlock("cond.false");
1644
1645
    // The ConditionalOperator itself has no likelihood information for its
1646
    // true and false branches. This matches the behavior of __builtin_expect.
1647
2
    ConditionalEvaluation cond(*this);
1648
2
    EmitBranchOnBoolExpr(CondOp->getCond(), LHSBlock, RHSBlock,
1649
2
                         getProfileCount(CondOp), Stmt::LH_None);
1650
1651
    // When computing PGO branch weights, we only know the overall count for
1652
    // the true block. This code is essentially doing tail duplication of the
1653
    // naive code-gen, introducing new edges for which counts are not
1654
    // available. Divide the counts proportionally between the LHS and RHS of
1655
    // the conditional operator.
1656
2
    uint64_t LHSScaledTrueCount = 0;
1657
2
    if (TrueCount) {
1658
0
      double LHSRatio =
1659
0
          getProfileCount(CondOp) / (double)getCurrentProfileCount();
1660
0
      LHSScaledTrueCount = TrueCount * LHSRatio;
1661
0
    }
1662
1663
2
    cond.begin(*this);
1664
2
    EmitBlock(LHSBlock);
1665
2
    incrementProfileCounter(CondOp);
1666
2
    {
1667
2
      ApplyDebugLocation DL(*this, Cond);
1668
2
      EmitBranchOnBoolExpr(CondOp->getLHS(), TrueBlock, FalseBlock,
1669
2
                           LHSScaledTrueCount, LH);
1670
2
    }
1671
2
    cond.end(*this);
1672
1673
2
    cond.begin(*this);
1674
2
    EmitBlock(RHSBlock);
1675
2
    EmitBranchOnBoolExpr(CondOp->getRHS(), TrueBlock, FalseBlock,
1676
2
                         TrueCount - LHSScaledTrueCount, LH);
1677
2
    cond.end(*this);
1678
1679
2
    return;
1680
2
  }
1681
1682
100k
  if (const CXXThrowExpr *Throw = dyn_cast<CXXThrowExpr>(Cond)) {
1683
    // Conditional operator handling can give us a throw expression as a
1684
    // condition for a case like:
1685
    //   br(c ? throw x : y, t, f) -> br(c, br(throw x, t, f), br(y, t, f)
1686
    // Fold this to:
1687
    //   br(c, throw x, br(y, t, f))
1688
0
    EmitCXXThrowExpr(Throw, /*KeepInsertionPoint*/false);
1689
0
    return;
1690
0
  }
1691
1692
  // If the branch has a condition wrapped by __builtin_unpredictable,
1693
  // create metadata that specifies that the branch is unpredictable.
1694
  // Don't bother if not optimizing because that metadata would not be used.
1695
100k
  llvm::MDNode *Unpredictable = nullptr;
1696
100k
  auto *Call = dyn_cast<CallExpr>(Cond->IgnoreImpCasts());
1697
100k
  if (Call && 
CGM.getCodeGenOpts().OptimizationLevel != 05.16k
) {
1698
103
    auto *FD = dyn_cast_or_null<FunctionDecl>(Call->getCalleeDecl());
1699
103
    if (FD && FD->getBuiltinID() == Builtin::BI__builtin_unpredictable) {
1700
2
      llvm::MDBuilder MDHelper(getLLVMContext());
1701
2
      Unpredictable = MDHelper.createUnpredictable();
1702
2
    }
1703
103
  }
1704
1705
100k
  llvm::MDNode *Weights = createBranchWeights(LH);
1706
100k
  if (!Weights) {
1707
100k
    uint64_t CurrentCount = std::max(getCurrentProfileCount(), TrueCount);
1708
100k
    Weights = createProfileWeights(TrueCount, CurrentCount - TrueCount);
1709
100k
  }
1710
1711
  // Emit the code with the fully general case.
1712
100k
  llvm::Value *CondV;
1713
100k
  {
1714
100k
    ApplyDebugLocation DL(*this, Cond);
1715
100k
    CondV = EvaluateExprAsBool(Cond);
1716
100k
  }
1717
100k
  Builder.CreateCondBr(CondV, TrueBlock, FalseBlock, Weights, Unpredictable);
1718
100k
}
1719
1720
/// ErrorUnsupported - Print out an error that codegen doesn't support the
1721
/// specified stmt yet.
1722
1
void CodeGenFunction::ErrorUnsupported(const Stmt *S, const char *Type) {
1723
1
  CGM.ErrorUnsupported(S, Type);
1724
1
}
1725
1726
/// emitNonZeroVLAInit - Emit the "zero" initialization of a
1727
/// variable-length array whose elements have a non-zero bit-pattern.
1728
///
1729
/// \param baseType the inner-most element type of the array
1730
/// \param src - a char* pointing to the bit-pattern for a single
1731
/// base element of the array
1732
/// \param sizeInChars - the total size of the VLA, in chars
1733
static void emitNonZeroVLAInit(CodeGenFunction &CGF, QualType baseType,
1734
                               Address dest, Address src,
1735
0
                               llvm::Value *sizeInChars) {
1736
0
  CGBuilderTy &Builder = CGF.Builder;
1737
1738
0
  CharUnits baseSize = CGF.getContext().getTypeSizeInChars(baseType);
1739
0
  llvm::Value *baseSizeInChars
1740
0
    = llvm::ConstantInt::get(CGF.IntPtrTy, baseSize.getQuantity());
1741
1742
0
  Address begin =
1743
0
    Builder.CreateElementBitCast(dest, CGF.Int8Ty, "vla.begin");
1744
0
  llvm::Value *end =
1745
0
    Builder.CreateInBoundsGEP(begin.getPointer(), sizeInChars, "vla.end");
1746
1747
0
  llvm::BasicBlock *originBB = CGF.Builder.GetInsertBlock();
1748
0
  llvm::BasicBlock *loopBB = CGF.createBasicBlock("vla-init.loop");
1749
0
  llvm::BasicBlock *contBB = CGF.createBasicBlock("vla-init.cont");
1750
1751
  // Make a loop over the VLA.  C99 guarantees that the VLA element
1752
  // count must be nonzero.
1753
0
  CGF.EmitBlock(loopBB);
1754
1755
0
  llvm::PHINode *cur = Builder.CreatePHI(begin.getType(), 2, "vla.cur");
1756
0
  cur->addIncoming(begin.getPointer(), originBB);
1757
1758
0
  CharUnits curAlign =
1759
0
    dest.getAlignment().alignmentOfArrayElement(baseSize);
1760
1761
  // memcpy the individual element bit-pattern.
1762
0
  Builder.CreateMemCpy(Address(cur, curAlign), src, baseSizeInChars,
1763
0
                       /*volatile*/ false);
1764
1765
  // Go to the next element.
1766
0
  llvm::Value *next =
1767
0
    Builder.CreateInBoundsGEP(CGF.Int8Ty, cur, baseSizeInChars, "vla.next");
1768
1769
  // Leave if that's the end of the VLA.
1770
0
  llvm::Value *done = Builder.CreateICmpEQ(next, end, "vla-init.isdone");
1771
0
  Builder.CreateCondBr(done, contBB, loopBB);
1772
0
  cur->addIncoming(next, loopBB);
1773
1774
0
  CGF.EmitBlock(contBB);
1775
0
}
1776
1777
void
1778
7.88k
CodeGenFunction::EmitNullInitialization(Address DestPtr, QualType Ty) {
1779
  // Ignore empty classes in C++.
1780
7.88k
  if (getLangOpts().CPlusPlus) {
1781
7.74k
    if (const RecordType *RT = Ty->getAs<RecordType>()) {
1782
7.41k
      if (cast<CXXRecordDecl>(RT->getDecl())->isEmpty())
1783
6.70k
        return;
1784
1.18k
    }
1785
7.74k
  }
1786
1787
  // Cast the dest ptr to the appropriate i8 pointer type.
1788
1.18k
  if (DestPtr.getElementType() != Int8Ty)
1789
1.18k
    DestPtr = Builder.CreateElementBitCast(DestPtr, Int8Ty);
1790
1791
  // Get size and alignment info for this aggregate.
1792
1.18k
  CharUnits size = getContext().getTypeSizeInChars(Ty);
1793
1794
1.18k
  llvm::Value *SizeVal;
1795
1.18k
  const VariableArrayType *vla;
1796
1797
  // Don't bother emitting a zero-byte memset.
1798
1.18k
  if (size.isZero()) {
1799
    // But note that getTypeInfo returns 0 for a VLA.
1800
11
    if (const VariableArrayType *vlaType =
1801
8
          dyn_cast_or_null<VariableArrayType>(
1802
8
                                          getContext().getAsArrayType(Ty))) {
1803
8
      auto VlaSize = getVLASize(vlaType);
1804
8
      SizeVal = VlaSize.NumElts;
1805
8
      CharUnits eltSize = getContext().getTypeSizeInChars(VlaSize.Type);
1806
8
      if (!eltSize.isOne())
1807
8
        SizeVal = Builder.CreateNUWMul(SizeVal, CGM.getSize(eltSize));
1808
8
      vla = vlaType;
1809
3
    } else {
1810
3
      return;
1811
3
    }
1812
1.17k
  } else {
1813
1.17k
    SizeVal = CGM.getSize(size);
1814
1.17k
    vla = nullptr;
1815
1.17k
  }
1816
1817
  // If the type contains a pointer to data member we can't memset it to zero.
1818
  // Instead, create a null constant and copy it to the destination.
1819
  // TODO: there are other patterns besides zero that we can usefully memset,
1820
  // like -1, which happens to be the pattern used by member-pointers.
1821
1.18k
  if (!CGM.getTypes().isZeroInitializable(Ty)) {
1822
    // For a VLA, emit a single element, then splat that over the VLA.
1823
9
    if (vla) 
Ty = getContext().getBaseElementType(vla)0
;
1824
1825
9
    llvm::Constant *NullConstant = CGM.EmitNullConstant(Ty);
1826
1827
9
    llvm::GlobalVariable *NullVariable =
1828
9
      new llvm::GlobalVariable(CGM.getModule(), NullConstant->getType(),
1829
9
                               /*isConstant=*/true,
1830
9
                               llvm::GlobalVariable::PrivateLinkage,
1831
9
                               NullConstant, Twine());
1832
9
    CharUnits NullAlign = DestPtr.getAlignment();
1833
9
    NullVariable->setAlignment(NullAlign.getAsAlign());
1834
9
    Address SrcPtr(Builder.CreateBitCast(NullVariable, Builder.getInt8PtrTy()),
1835
9
                   NullAlign);
1836
1837
9
    if (vla) 
return emitNonZeroVLAInit(*this, Ty, DestPtr, SrcPtr, SizeVal)0
;
1838
1839
    // Get and call the appropriate llvm.memcpy overload.
1840
9
    Builder.CreateMemCpy(DestPtr, SrcPtr, SizeVal, false);
1841
9
    return;
1842
9
  }
1843
1844
  // Otherwise, just memset the whole thing to zero.  This is legal
1845
  // because in LLVM, all default initializers (other than the ones we just
1846
  // handled above) are guaranteed to have a bit pattern of all zeros.
1847
1.17k
  Builder.CreateMemSet(DestPtr, Builder.getInt8(0), SizeVal, false);
1848
1.17k
}
1849
1850
57
llvm::BlockAddress *CodeGenFunction::GetAddrOfLabel(const LabelDecl *L) {
1851
  // Make sure that there is a block for the indirect goto.
1852
57
  if (!IndirectBranch)
1853
34
    GetIndirectGotoBlock();
1854
1855
57
  llvm::BasicBlock *BB = getJumpDestForLabel(L).getBlock();
1856
1857
  // Make sure the indirect branch includes all of the address-taken blocks.
1858
57
  IndirectBranch->addDestination(BB);
1859
57
  return llvm::BlockAddress::get(CurFn, BB);
1860
57
}
1861
1862
56
llvm::BasicBlock *CodeGenFunction::GetIndirectGotoBlock() {
1863
  // If we already made the indirect branch for indirect goto, return its block.
1864
56
  if (IndirectBranch) 
return IndirectBranch->getParent()21
;
1865
1866
35
  CGBuilderTy TmpBuilder(*this, createBasicBlock("indirectgoto"));
1867
1868
  // Create the PHI node that indirect gotos will add entries to.
1869
35
  llvm::Value *DestVal = TmpBuilder.CreatePHI(Int8PtrTy, 0,
1870
35
                                              "indirect.goto.dest");
1871
1872
  // Create the indirect branch instruction.
1873
35
  IndirectBranch = TmpBuilder.CreateIndirectBr(DestVal);
1874
35
  return IndirectBranch->getParent();
1875
35
}
1876
1877
/// Computes the length of an array in elements, as well as the base
1878
/// element type and a properly-typed first element pointer.
1879
llvm::Value *CodeGenFunction::emitArrayLength(const ArrayType *origArrayType,
1880
                                              QualType &baseType,
1881
4.78k
                                              Address &addr) {
1882
4.78k
  const ArrayType *arrayType = origArrayType;
1883
1884
  // If it's a VLA, we have to load the stored size.  Note that
1885
  // this is the size of the VLA in bytes, not its size in elements.
1886
4.78k
  llvm::Value *numVLAElements = nullptr;
1887
4.78k
  if (isa<VariableArrayType>(arrayType)) {
1888
615
    numVLAElements = getVLASize(cast<VariableArrayType>(arrayType)).NumElts;
1889
1890
    // Walk into all VLAs.  This doesn't require changes to addr,
1891
    // which has type T* where T is the first non-VLA element type.
1892
652
    do {
1893
652
      QualType elementType = arrayType->getElementType();
1894
652
      arrayType = getContext().getAsArrayType(elementType);
1895
1896
      // If we only have VLA components, 'addr' requires no adjustment.
1897
652
      if (!arrayType) {
1898
611
        baseType = elementType;
1899
611
        return numVLAElements;
1900
611
      }
1901
41
    } while (isa<VariableArrayType>(arrayType));
1902
1903
    // We get out here only if we find a constant array type
1904
    // inside the VLA.
1905
615
  }
1906
1907
  // We have some number of constant-length arrays, so addr should
1908
  // have LLVM type [M x [N x [...]]]*.  Build a GEP that walks
1909
  // down to the first element of addr.
1910
4.17k
  SmallVector<llvm::Value*, 8> gepIndices;
1911
1912
  // GEP down to the array type.
1913
4.17k
  llvm::ConstantInt *zero = Builder.getInt32(0);
1914
4.17k
  gepIndices.push_back(zero);
1915
1916
4.17k
  uint64_t countFromCLAs = 1;
1917
4.17k
  QualType eltType;
1918
1919
4.17k
  llvm::ArrayType *llvmArrayType =
1920
4.17k
    dyn_cast<llvm::ArrayType>(addr.getElementType());
1921
8.14k
  while (llvmArrayType) {
1922
3.96k
    assert(isa<ConstantArrayType>(arrayType));
1923
3.96k
    assert(cast<ConstantArrayType>(arrayType)->getSize().getZExtValue()
1924
3.96k
             == llvmArrayType->getNumElements());
1925
1926
3.96k
    gepIndices.push_back(zero);
1927
3.96k
    countFromCLAs *= llvmArrayType->getNumElements();
1928
3.96k
    eltType = arrayType->getElementType();
1929
1930
3.96k
    llvmArrayType =
1931
3.96k
      dyn_cast<llvm::ArrayType>(llvmArrayType->getElementType());
1932
3.96k
    arrayType = getContext().getAsArrayType(arrayType->getElementType());
1933
3.96k
    assert((!llvmArrayType || arrayType) &&
1934
3.96k
           "LLVM and Clang types are out-of-synch");
1935
3.96k
  }
1936
1937
4.17k
  if (arrayType) {
1938
    // From this point onwards, the Clang array type has been emitted
1939
    // as some other type (probably a packed struct). Compute the array
1940
    // size, and just emit the 'begin' expression as a bitcast.
1941
668
    while (arrayType) {
1942
355
      countFromCLAs *=
1943
355
          cast<ConstantArrayType>(arrayType)->getSize().getZExtValue();
1944
355
      eltType = arrayType->getElementType();
1945
355
      arrayType = getContext().getAsArrayType(eltType);
1946
355
    }
1947
1948
313
    llvm::Type *baseType = ConvertType(eltType);
1949
313
    addr = Builder.CreateElementBitCast(addr, baseType, "array.begin");
1950
3.86k
  } else {
1951
    // Create the actual GEP.
1952
3.86k
    addr = Address(Builder.CreateInBoundsGEP(addr.getPointer(),
1953
3.86k
                                             gepIndices, "array.begin"),
1954
3.86k
                   addr.getAlignment());
1955
3.86k
  }
1956
1957
4.17k
  baseType = eltType;
1958
1959
4.17k
  llvm::Value *numElements
1960
4.17k
    = llvm::ConstantInt::get(SizeTy, countFromCLAs);
1961
1962
  // If we had any VLA dimensions, factor them in.
1963
4.17k
  if (numVLAElements)
1964
4
    numElements = Builder.CreateNUWMul(numVLAElements, numElements);
1965
1966
4.17k
  return numElements;
1967
4.78k
}
1968
1969
2.14k
CodeGenFunction::VlaSizePair CodeGenFunction::getVLASize(QualType type) {
1970
2.14k
  const VariableArrayType *vla = getContext().getAsVariableArrayType(type);
1971
2.14k
  assert(vla && "type was not a variable array type!");
1972
2.14k
  return getVLASize(vla);
1973
2.14k
}
1974
1975
CodeGenFunction::VlaSizePair
1976
5.74k
CodeGenFunction::getVLASize(const VariableArrayType *type) {
1977
  // The number of elements so far; always size_t.
1978
5.74k
  llvm::Value *numElements = nullptr;
1979
1980
5.74k
  QualType elementType;
1981
7.28k
  do {
1982
7.28k
    elementType = type->getElementType();
1983
7.28k
    llvm::Value *vlaSize = VLASizeMap[type->getSizeExpr()];
1984
7.28k
    assert(vlaSize && "no size for VLA!");
1985
7.28k
    assert(vlaSize->getType() == SizeTy);
1986
1987
7.28k
    if (!numElements) {
1988
5.74k
      numElements = vlaSize;
1989
1.54k
    } else {
1990
      // It's undefined behavior if this wraps around, so mark it that way.
1991
      // FIXME: Teach -fsanitize=undefined to trap this.
1992
1.54k
      numElements = Builder.CreateNUWMul(numElements, vlaSize);
1993
1.54k
    }
1994
7.28k
  } while ((type = getContext().getAsVariableArrayType(elementType)));
1995
1996
5.74k
  return { numElements, elementType };
1997
5.74k
}
1998
1999
CodeGenFunction::VlaSizePair
2000
3.18k
CodeGenFunction::getVLAElements1D(QualType type) {
2001
3.18k
  const VariableArrayType *vla = getContext().getAsVariableArrayType(type);
2002
3.18k
  assert(vla && "type was not a variable array type!");
2003
3.18k
  return getVLAElements1D(vla);
2004
3.18k
}
2005
2006
CodeGenFunction::VlaSizePair
2007
3.18k
CodeGenFunction::getVLAElements1D(const VariableArrayType *Vla) {
2008
3.18k
  llvm::Value *VlaSize = VLASizeMap[Vla->getSizeExpr()];
2009
3.18k
  assert(VlaSize && "no size for VLA!");
2010
3.18k
  assert(VlaSize->getType() == SizeTy);
2011
3.18k
  return { VlaSize, Vla->getElementType() };
2012
3.18k
}
2013
2014
2.84k
void CodeGenFunction::EmitVariablyModifiedType(QualType type) {
2015
2.84k
  assert(type->isVariablyModifiedType() &&
2016
2.84k
         "Must pass variably modified type to EmitVLASizes!");
2017
2018
2.84k
  EnsureInsertPoint();
2019
2020
  // We're going to walk down into the type and look for VLA
2021
  // expressions.
2022
4.04k
  do {
2023
4.04k
    assert(type->isVariablyModifiedType());
2024
2025
4.04k
    const Type *ty = type.getTypePtr();
2026
4.04k
    switch (ty->getTypeClass()) {
2027
2028
0
#define TYPE(Class, Base)
2029
0
#define ABSTRACT_TYPE(Class, Base)
2030
0
#define NON_CANONICAL_TYPE(Class, Base)
2031
0
#define DEPENDENT_TYPE(Class, Base) case Type::Class:
2032
0
#define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(Class, Base)
2033
0
#include "clang/AST/TypeNodes.inc"
2034
0
      llvm_unreachable("unexpected dependent type!");
2035
2036
    // These types are never variably-modified.
2037
0
    case Type::Builtin:
2038
0
    case Type::Complex:
2039
0
    case Type::Vector:
2040
0
    case Type::ExtVector:
2041
0
    case Type::ConstantMatrix:
2042
0
    case Type::Record:
2043
0
    case Type::Enum:
2044
0
    case Type::Elaborated:
2045
0
    case Type::TemplateSpecialization:
2046
0
    case Type::ObjCTypeParam:
2047
0
    case Type::ObjCObject:
2048
0
    case Type::ObjCInterface:
2049
0
    case Type::ObjCObjectPointer:
2050
0
    case Type::ExtInt:
2051
0
      llvm_unreachable("type class is never variably-modified!");
2052
2053
0
    case Type::Adjusted:
2054
0
      type = cast<AdjustedType>(ty)->getAdjustedType();
2055
0
      break;
2056
2057
70
    case Type::Decayed:
2058
70
      type = cast<DecayedType>(ty)->getPointeeType();
2059
70
      break;
2060
2061
90
    case Type::Pointer:
2062
90
      type = cast<PointerType>(ty)->getPointeeType();
2063
90
      break;
2064
2065
0
    case Type::BlockPointer:
2066
0
      type = cast<BlockPointerType>(ty)->getPointeeType();
2067
0
      break;
2068
2069
3
    case Type::LValueReference:
2070
3
    case Type::RValueReference:
2071
3
      type = cast<ReferenceType>(ty)->getPointeeType();
2072
3
      break;
2073
2074
0
    case Type::MemberPointer:
2075
0
      type = cast<MemberPointerType>(ty)->getPointeeType();
2076
0
      break;
2077
2078
12
    case Type::ConstantArray:
2079
12
    case Type::IncompleteArray:
2080
      // Losing element qualification here is fine.
2081
12
      type = cast<ArrayType>(ty)->getElementType();
2082
12
      break;
2083
2084
3.78k
    case Type::VariableArray: {
2085
      // Losing element qualification here is fine.
2086
3.78k
      const VariableArrayType *vat = cast<VariableArrayType>(ty);
2087
2088
      // Unknown size indication requires no size computation.
2089
      // Otherwise, evaluate and record it.
2090
3.78k
      if (const Expr *size = vat->getSizeExpr()) {
2091
        // It's possible that we might have emitted this already,
2092
        // e.g. with a typedef and a pointer to it.
2093
3.78k
        llvm::Value *&entry = VLASizeMap[size];
2094
3.78k
        if (!entry) {
2095
3.42k
          llvm::Value *Size = EmitScalarExpr(size);
2096
2097
          // C11 6.7.6.2p5:
2098
          //   If the size is an expression that is not an integer constant
2099
          //   expression [...] each time it is evaluated it shall have a value
2100
          //   greater than zero.
2101
3.42k
          if (SanOpts.has(SanitizerKind::VLABound) &&
2102
2
              size->getType()->isSignedIntegerType()) {
2103
2
            SanitizerScope SanScope(this);
2104
2
            llvm::Value *Zero = llvm::Constant::getNullValue(Size->getType());
2105
2
            llvm::Constant *StaticArgs[] = {
2106
2
                EmitCheckSourceLocation(size->getBeginLoc()),
2107
2
                EmitCheckTypeDescriptor(size->getType())};
2108
2
            EmitCheck(std::make_pair(Builder.CreateICmpSGT(Size, Zero),
2109
2
                                     SanitizerKind::VLABound),
2110
2
                      SanitizerHandler::VLABoundNotPositive, StaticArgs, Size);
2111
2
          }
2112
2113
          // Always zexting here would be wrong if it weren't
2114
          // undefined behavior to have a negative bound.
2115
3.42k
          entry = Builder.CreateIntCast(Size, SizeTy, /*signed*/ false);
2116
3.42k
        }
2117
3.78k
      }
2118
3.78k
      type = vat->getElementType();
2119
3.78k
      break;
2120
12
    }
2121
2122
1
    case Type::FunctionProto:
2123
1
    case Type::FunctionNoProto:
2124
1
      type = cast<FunctionType>(ty)->getReturnType();
2125
1
      break;
2126
2127
44
    case Type::Paren:
2128
44
    case Type::TypeOf:
2129
44
    case Type::UnaryTransform:
2130
44
    case Type::Attributed:
2131
44
    case Type::SubstTemplateTypeParm:
2132
44
    case Type::MacroQualified:
2133
      // Keep walking after single level desugaring.
2134
44
      type = type.getSingleStepDesugaredType(getContext());
2135
44
      break;
2136
2137
28
    case Type::Typedef:
2138
28
    case Type::Decltype:
2139
28
    case Type::Auto:
2140
28
    case Type::DeducedTemplateSpecialization:
2141
      // Stop walking: nothing to do.
2142
28
      return;
2143
2144
13
    case Type::TypeOfExpr:
2145
      // Stop walking: emit typeof expression.
2146
13
      EmitIgnoredExpr(cast<TypeOfExprType>(ty)->getUnderlyingExpr());
2147
13
      return;
2148
2149
1
    case Type::Atomic:
2150
1
      type = cast<AtomicType>(ty)->getValueType();
2151
1
      break;
2152
2153
0
    case Type::Pipe:
2154
0
      type = cast<PipeType>(ty)->getElementType();
2155
0
      break;
2156
4.00k
    }
2157
4.00k
  } while (type->isVariablyModifiedType());
2158
2.84k
}
2159
2160
1.14k
Address CodeGenFunction::EmitVAListRef(const Expr* E) {
2161
1.14k
  if (getContext().getBuiltinVaListType()->isArrayType())
2162
478
    return EmitPointerWithAlignment(E);
2163
667
  return EmitLValue(E).getAddress(*this);
2164
667
}
2165
2166
29
Address CodeGenFunction::EmitMSVAListRef(const Expr *E) {
2167
29
  return EmitLValue(E).getAddress(*this);
2168
29
}
2169
2170
void CodeGenFunction::EmitDeclRefExprDbgValue(const DeclRefExpr *E,
2171
7.64k
                                              const APValue &Init) {
2172
7.64k
  assert(Init.hasValue() && "Invalid DeclRefExpr initializer!");
2173
7.64k
  if (CGDebugInfo *Dbg = getDebugInfo())
2174
5.12k
    if (CGM.getCodeGenOpts().hasReducedDebugInfo())
2175
5.12k
      Dbg->EmitGlobalVariable(E->getDecl(), Init);
2176
7.64k
}
2177
2178
CodeGenFunction::PeepholeProtection
2179
1.44k
CodeGenFunction::protectFromPeepholes(RValue rvalue) {
2180
  // At the moment, the only aggressive peephole we do in IR gen
2181
  // is trunc(zext) folding, but if we add more, we can easily
2182
  // extend this protection.
2183
2184
1.44k
  if (!rvalue.isScalar()) 
return PeepholeProtection()55
;
2185
1.39k
  llvm::Value *value = rvalue.getScalarVal();
2186
1.39k
  if (!isa<llvm::ZExtInst>(value)) 
return PeepholeProtection()1.32k
;
2187
2188
  // Just make an extra bitcast.
2189
67
  assert(HaveInsertPoint());
2190
67
  llvm::Instruction *inst = new llvm::BitCastInst(value, value->getType(), "",
2191
67
                                                  Builder.GetInsertBlock());
2192
2193
67
  PeepholeProtection protection;
2194
67
  protection.Inst = inst;
2195
67
  return protection;
2196
67
}
2197
2198
1.44k
void CodeGenFunction::unprotectFromPeepholes(PeepholeProtection protection) {
2199
1.44k
  if (!protection.Inst) 
return1.38k
;
2200
2201
  // In theory, we could try to duplicate the peepholes now, but whatever.
2202
67
  protection.Inst->eraseFromParent();
2203
67
}
2204
2205
void CodeGenFunction::emitAlignmentAssumption(llvm::Value *PtrValue,
2206
                                              QualType Ty, SourceLocation Loc,
2207
                                              SourceLocation AssumptionLoc,
2208
                                              llvm::Value *Alignment,
2209
445
                                              llvm::Value *OffsetValue) {
2210
445
  if (Alignment->getType() != IntPtrTy)
2211
221
    Alignment =
2212
221
        Builder.CreateIntCast(Alignment, IntPtrTy, false, "casted.align");
2213
445
  if (OffsetValue && 
OffsetValue->getType() != IntPtrTy16
)
2214
5
    OffsetValue =
2215
5
        Builder.CreateIntCast(OffsetValue, IntPtrTy, true, "casted.offset");
2216
445
  llvm::Value *TheCheck = nullptr;
2217
445
  if (SanOpts.has(SanitizerKind::Alignment)) {
2218
33
    llvm::Value *PtrIntValue =
2219
33
        Builder.CreatePtrToInt(PtrValue, IntPtrTy, "ptrint");
2220
2221
33
    if (OffsetValue) {
2222
9
      bool IsOffsetZero = false;
2223
9
      if (const auto *CI = dyn_cast<llvm::ConstantInt>(OffsetValue))
2224
6
        IsOffsetZero = CI->isZero();
2225
2226
9
      if (!IsOffsetZero)
2227
9
        PtrIntValue = Builder.CreateSub(PtrIntValue, OffsetValue, "offsetptr");
2228
9
    }
2229
2230
33
    llvm::Value *Zero = llvm::ConstantInt::get(IntPtrTy, 0);
2231
33
    llvm::Value *Mask =
2232
33
        Builder.CreateSub(Alignment, llvm::ConstantInt::get(IntPtrTy, 1));
2233
33
    llvm::Value *MaskedPtr = Builder.CreateAnd(PtrIntValue, Mask, "maskedptr");
2234
33
    TheCheck = Builder.CreateICmpEQ(MaskedPtr, Zero, "maskcond");
2235
33
  }
2236
445
  llvm::Instruction *Assumption = Builder.CreateAlignmentAssumption(
2237
445
      CGM.getDataLayout(), PtrValue, Alignment, OffsetValue);
2238
2239
445
  if (!SanOpts.has(SanitizerKind::Alignment))
2240
412
    return;
2241
33
  emitAlignmentAssumptionCheck(PtrValue, Ty, Loc, AssumptionLoc, Alignment,
2242
33
                               OffsetValue, TheCheck, Assumption);
2243
33
}
2244
2245
void CodeGenFunction::emitAlignmentAssumption(llvm::Value *PtrValue,
2246
                                              const Expr *E,
2247
                                              SourceLocation AssumptionLoc,
2248
                                              llvm::Value *Alignment,
2249
418
                                              llvm::Value *OffsetValue) {
2250
418
  if (auto *CE = dyn_cast<CastExpr>(E))
2251
168
    E = CE->getSubExprAsWritten();
2252
418
  QualType Ty = E->getType();
2253
418
  SourceLocation Loc = E->getExprLoc();
2254
2255
418
  emitAlignmentAssumption(PtrValue, Ty, Loc, AssumptionLoc, Alignment,
2256
418
                          OffsetValue);
2257
418
}
2258
2259
llvm::Value *CodeGenFunction::EmitAnnotationCall(llvm::Function *AnnotationFn,
2260
                                                 llvm::Value *AnnotatedVal,
2261
                                                 StringRef AnnotationStr,
2262
                                                 SourceLocation Location,
2263
27
                                                 const AnnotateAttr *Attr) {
2264
27
  SmallVector<llvm::Value *, 5> Args = {
2265
27
      AnnotatedVal,
2266
27
      Builder.CreateBitCast(CGM.EmitAnnotationString(AnnotationStr), Int8PtrTy),
2267
27
      Builder.CreateBitCast(CGM.EmitAnnotationUnit(Location), Int8PtrTy),
2268
27
      CGM.EmitAnnotationLineNo(Location),
2269
27
  };
2270
27
  if (Attr)
2271
20
    Args.push_back(CGM.EmitAnnotationArgs(Attr));
2272
27
  return Builder.CreateCall(AnnotationFn, Args);
2273
27
}
2274
2275
8
void CodeGenFunction::EmitVarAnnotations(const VarDecl *D, llvm::Value *V) {
2276
8
  assert(D->hasAttr<AnnotateAttr>() && "no annotate attribute");
2277
  // FIXME We create a new bitcast for every annotation because that's what
2278
  // llvm-gcc was doing.
2279
8
  for (const auto *I : D->specific_attrs<AnnotateAttr>())
2280
12
    EmitAnnotationCall(CGM.getIntrinsic(llvm::Intrinsic::var_annotation),
2281
12
                       Builder.CreateBitCast(V, CGM.Int8PtrTy, V->getName()),
2282
12
                       I->getAnnotation(), D->getLocation(), I);
2283
8
}
2284
2285
Address CodeGenFunction::EmitFieldAnnotations(const FieldDecl *D,
2286
4
                                              Address Addr) {
2287
4
  assert(D->hasAttr<AnnotateAttr>() && "no annotate attribute");
2288
4
  llvm::Value *V = Addr.getPointer();
2289
4
  llvm::Type *VTy = V->getType();
2290
4
  llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::ptr_annotation,
2291
4
                                    CGM.Int8PtrTy);
2292
2293
8
  for (const auto *I : D->specific_attrs<AnnotateAttr>()) {
2294
    // FIXME Always emit the cast inst so we can differentiate between
2295
    // annotation on the first field of a struct and annotation on the struct
2296
    // itself.
2297
8
    if (VTy != CGM.Int8PtrTy)
2298
8
      V = Builder.CreateBitCast(V, CGM.Int8PtrTy);
2299
8
    V = EmitAnnotationCall(F, V, I->getAnnotation(), D->getLocation(), I);
2300
8
    V = Builder.CreateBitCast(V, VTy);
2301
8
  }
2302
2303
4
  return Address(V, Addr.getAlignment());
2304
4
}
2305
2306
66.7k
CodeGenFunction::CGCapturedStmtInfo::~CGCapturedStmtInfo() { }
2307
2308
CodeGenFunction::SanitizerScope::SanitizerScope(CodeGenFunction *CGF)
2309
54.7k
    : CGF(CGF) {
2310
54.7k
  assert(!CGF->IsSanitizerScope);
2311
54.7k
  CGF->IsSanitizerScope = true;
2312
54.7k
}
2313
2314
54.7k
CodeGenFunction::SanitizerScope::~SanitizerScope() {
2315
54.7k
  CGF->IsSanitizerScope = false;
2316
54.7k
}
2317
2318
void CodeGenFunction::InsertHelper(llvm::Instruction *I,
2319
                                   const llvm::Twine &Name,
2320
                                   llvm::BasicBlock *BB,
2321
4.99M
                                   llvm::BasicBlock::iterator InsertPt) const {
2322
4.99M
  LoopStack.InsertHelper(I);
2323
4.99M
  if (IsSanitizerScope)
2324
15.3k
    CGM.getSanitizerMetadata()->disableSanitizerForInstruction(I);
2325
4.99M
}
2326
2327
void CGBuilderInserter::InsertHelper(
2328
    llvm::Instruction *I, const llvm::Twine &Name, llvm::BasicBlock *BB,
2329
4.99M
    llvm::BasicBlock::iterator InsertPt) const {
2330
4.99M
  llvm::IRBuilderDefaultInserter::InsertHelper(I, Name, BB, InsertPt);
2331
4.99M
  if (CGF)
2332
4.99M
    CGF->InsertHelper(I, Name, BB, InsertPt);
2333
4.99M
}
2334
2335
// Emits an error if we don't have a valid set of target features for the
2336
// called function.
2337
void CodeGenFunction::checkTargetFeatures(const CallExpr *E,
2338
52.3k
                                          const FunctionDecl *TargetDecl) {
2339
52.3k
  return checkTargetFeatures(E->getBeginLoc(), TargetDecl);
2340
52.3k
}
2341
2342
// Emits an error if we don't have a valid set of target features for the
2343
// called function.
2344
void CodeGenFunction::checkTargetFeatures(SourceLocation Loc,
2345
64.2k
                                          const FunctionDecl *TargetDecl) {
2346
  // Early exit if this is an indirect call.
2347
64.2k
  if (!TargetDecl)
2348
0
    return;
2349
2350
  // Get the current enclosing function if it exists. If it doesn't
2351
  // we can't check the target features anyhow.
2352
64.2k
  const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(CurCodeDecl);
2353
64.2k
  if (!FD)
2354
0
    return;
2355
2356
  // Grab the required features for the call. For a builtin this is listed in
2357
  // the td file with the default cpu, for an always_inline function this is any
2358
  // listed cpu and any listed features.
2359
64.2k
  unsigned BuiltinID = TargetDecl->getBuiltinID();
2360
64.2k
  std::string MissingFeature;
2361
64.2k
  llvm::StringMap<bool> CallerFeatureMap;
2362
64.2k
  CGM.getContext().getFunctionFeatureMap(CallerFeatureMap, FD);
2363
64.2k
  if (BuiltinID) {
2364
52.3k
    StringRef FeatureList(
2365
52.3k
        CGM.getContext().BuiltinInfo.getRequiredFeatures(BuiltinID));
2366
    // Return if the builtin doesn't have any required features.
2367
52.3k
    if (FeatureList.empty())
2368
34.2k
      return;
2369
18.1k
    assert(FeatureList.find(' ') == StringRef::npos &&
2370
18.1k
           "Space in feature list");
2371
18.1k
    TargetFeatures TF(CallerFeatureMap);
2372
18.1k
    if (!TF.hasRequiredFeatures(FeatureList))
2373
543
      CGM.getDiags().Report(Loc, diag::err_builtin_needs_feature)
2374
543
          << TargetDecl->getDeclName() << FeatureList;
2375
11.9k
  } else if (!TargetDecl->isMultiVersion() &&
2376
11.9k
             TargetDecl->hasAttr<TargetAttr>()) {
2377
    // Get the required features for the callee.
2378
2379
11.9k
    const TargetAttr *TD = TargetDecl->getAttr<TargetAttr>();
2380
11.9k
    ParsedTargetAttr ParsedAttr =
2381
11.9k
        CGM.getContext().filterFunctionTargetAttrs(TD);
2382
2383
11.9k
    SmallVector<StringRef, 1> ReqFeatures;
2384
11.9k
    llvm::StringMap<bool> CalleeFeatureMap;
2385
11.9k
    CGM.getContext().getFunctionFeatureMap(CalleeFeatureMap, TargetDecl);
2386
2387
13.2k
    for (const auto &F : ParsedAttr.Features) {
2388
13.2k
      if (F[0] == '+' && CalleeFeatureMap.lookup(F.substr(1)))
2389
13.2k
        ReqFeatures.push_back(StringRef(F).substr(1));
2390
13.2k
    }
2391
2392
180k
    for (const auto &F : CalleeFeatureMap) {
2393
      // Only positive features are "required".
2394
180k
      if (F.getValue())
2395
180k
        ReqFeatures.push_back(F.getKey());
2396
180k
    }
2397
193k
    if (
!llvm::all_of(ReqFeatures, [&](StringRef Feature) 11.9k
{
2398
193k
      if (!CallerFeatureMap.lookup(Feature)) {
2399
19
        MissingFeature = Feature.str();
2400
19
        return false;
2401
19
      }
2402
193k
      return true;
2403
193k
    }))
2404
19
      CGM.getDiags().Report(Loc, diag::err_function_needs_feature)
2405
19
          << FD->getDeclName() << TargetDecl->getDeclName() << MissingFeature;
2406
11.9k
  }
2407
64.2k
}
2408
2409
87
void CodeGenFunction::EmitSanitizerStatReport(llvm::SanitizerStatKind SSK) {
2410
87
  if (!CGM.getCodeGenOpts().SanitizeStats)
2411
77
    return;
2412
2413
10
  llvm::IRBuilder<> IRB(Builder.GetInsertBlock(), Builder.GetInsertPoint());
2414
10
  IRB.SetCurrentDebugLocation(Builder.getCurrentDebugLocation());
2415
10
  CGM.getSanStats().create(IRB, SSK);
2416
10
}
2417
2418
llvm::Value *
2419
240
CodeGenFunction::FormResolverCondition(const MultiVersionResolverOption &RO) {
2420
240
  llvm::Value *Condition = nullptr;
2421
2422
240
  if (!RO.Conditions.Architecture.empty())
2423
86
    Condition = EmitX86CpuIs(RO.Conditions.Architecture);
2424
2425
240
  if (!RO.Conditions.Features.empty()) {
2426
96
    llvm::Value *FeatureCond = EmitX86CpuSupports(RO.Conditions.Features);
2427
96
    Condition =
2428
94
        Condition ? 
Builder.CreateAnd(Condition, FeatureCond)2
: FeatureCond;
2429
96
  }
2430
240
  return Condition;
2431
240
}
2432
2433
static void CreateMultiVersionResolverReturn(CodeGenModule &CGM,
2434
                                             llvm::Function *Resolver,
2435
                                             CGBuilderTy &Builder,
2436
                                             llvm::Function *FuncToReturn,
2437
240
                                             bool SupportsIFunc) {
2438
240
  if (SupportsIFunc) {
2439
123
    Builder.CreateRet(FuncToReturn);
2440
123
    return;
2441
123
  }
2442
2443
117
  llvm::SmallVector<llvm::Value *, 10> Args;
2444
117
  llvm::for_each(Resolver->args(),
2445
116
                 [&](llvm::Argument &Arg) { Args.push_back(&Arg); });
2446
2447
117
  llvm::CallInst *Result = Builder.CreateCall(FuncToReturn, Args);
2448
117
  Result->setTailCallKind(llvm::CallInst::TCK_MustTail);
2449
2450
117
  if (Resolver->getReturnType()->isVoidTy())
2451
29
    Builder.CreateRetVoid();
2452
88
  else
2453
88
    Builder.CreateRet(Result);
2454
117
}
2455
2456
void CodeGenFunction::EmitMultiVersionResolver(
2457
70
    llvm::Function *Resolver, ArrayRef<MultiVersionResolverOption> Options) {
2458
70
  assert(getContext().getTargetInfo().getTriple().isX86() &&
2459
70
         "Only implemented for x86 targets");
2460
2461
70
  bool SupportsIFunc = getContext().getTargetInfo().supportsIFunc();
2462
2463
  // Main function's basic block.
2464
70
  llvm::BasicBlock *CurBlock = createBasicBlock("resolver_entry", Resolver);
2465
70
  Builder.SetInsertPoint(CurBlock);
2466
70
  EmitX86CpuInit();
2467
2468
240
  for (const MultiVersionResolverOption &RO : Options) {
2469
240
    Builder.SetInsertPoint(CurBlock);
2470
240
    llvm::Value *Condition = FormResolverCondition(RO);
2471
2472
    // The 'default' or 'generic' case.
2473
240
    if (!Condition) {
2474
60
      assert(&RO == Options.end() - 1 &&
2475
60
             "Default or Generic case must be last");
2476
60
      CreateMultiVersionResolverReturn(CGM, Resolver, Builder, RO.Function,
2477
60
                                       SupportsIFunc);
2478
60
      return;
2479
60
    }
2480
2481
180
    llvm::BasicBlock *RetBlock = createBasicBlock("resolver_return", Resolver);
2482
180
    CGBuilderTy RetBuilder(*this, RetBlock);
2483
180
    CreateMultiVersionResolverReturn(CGM, Resolver, RetBuilder, RO.Function,
2484
180
                                     SupportsIFunc);
2485
180
    CurBlock = createBasicBlock("resolver_else", Resolver);
2486
180
    Builder.CreateCondBr(Condition, RetBlock, CurBlock);
2487
180
  }
2488
2489
  // If no generic/default, emit an unreachable.
2490
10
  Builder.SetInsertPoint(CurBlock);
2491
10
  llvm::CallInst *TrapCall = EmitTrapCall(llvm::Intrinsic::trap);
2492
10
  TrapCall->setDoesNotReturn();
2493
10
  TrapCall->setDoesNotThrow();
2494
10
  Builder.CreateUnreachable();
2495
10
  Builder.ClearInsertionPoint();
2496
10
}
2497
2498
// Loc - where the diagnostic will point, where in the source code this
2499
//  alignment has failed.
2500
// SecondaryLoc - if present (will be present if sufficiently different from
2501
//  Loc), the diagnostic will additionally point a "Note:" to this location.
2502
//  It should be the location where the __attribute__((assume_aligned))
2503
//  was written e.g.
2504
void CodeGenFunction::emitAlignmentAssumptionCheck(
2505
    llvm::Value *Ptr, QualType Ty, SourceLocation Loc,
2506
    SourceLocation SecondaryLoc, llvm::Value *Alignment,
2507
    llvm::Value *OffsetValue, llvm::Value *TheCheck,
2508
33
    llvm::Instruction *Assumption) {
2509
33
  assert(Assumption && isa<llvm::CallInst>(Assumption) &&
2510
33
         cast<llvm::CallInst>(Assumption)->getCalledOperand() ==
2511
33
             llvm::Intrinsic::getDeclaration(
2512
33
                 Builder.GetInsertBlock()->getParent()->getParent(),
2513
33
                 llvm::Intrinsic::assume) &&
2514
33
         "Assumption should be a call to llvm.assume().");
2515
33
  assert(&(Builder.GetInsertBlock()->back()) == Assumption &&
2516
33
         "Assumption should be the last instruction of the basic block, "
2517
33
         "since the basic block is still being generated.");
2518
2519
33
  if (!SanOpts.has(SanitizerKind::Alignment))
2520
0
    return;
2521
2522
  // Don't check pointers to volatile data. The behavior here is implementation-
2523
  // defined.
2524
33
  if (Ty->getPointeeType().isVolatileQualified())
2525
1
    return;
2526
2527
  // We need to temorairly remove the assumption so we can insert the
2528
  // sanitizer check before it, else the check will be dropped by optimizations.
2529
32
  Assumption->removeFromParent();
2530
2531
32
  {
2532
32
    SanitizerScope SanScope(this);
2533
2534
32
    if (!OffsetValue)
2535
23
      OffsetValue = Builder.getInt1(0); // no offset.
2536
2537
32
    llvm::Constant *StaticData[] = {EmitCheckSourceLocation(Loc),
2538
32
                                    EmitCheckSourceLocation(SecondaryLoc),
2539
32
                                    EmitCheckTypeDescriptor(Ty)};
2540
32
    llvm::Value *DynamicData[] = {EmitCheckValue(Ptr),
2541
32
                                  EmitCheckValue(Alignment),
2542
32
                                  EmitCheckValue(OffsetValue)};
2543
32
    EmitCheck({std::make_pair(TheCheck, SanitizerKind::Alignment)},
2544
32
              SanitizerHandler::AlignmentAssumption, StaticData, DynamicData);
2545
32
  }
2546
2547
  // We are now in the (new, empty) "cont" basic block.
2548
  // Reintroduce the assumption.
2549
32
  Builder.Insert(Assumption);
2550
  // FIXME: Assumption still has it's original basic block as it's Parent.
2551
32
}
2552
2553
53.8k
llvm::DebugLoc CodeGenFunction::SourceLocToDebugLoc(SourceLocation Location) {
2554
53.8k
  if (CGDebugInfo *DI = getDebugInfo())
2555
16.9k
    return DI->SourceLocToDebugLoc(Location);
2556
2557
36.8k
  return llvm::DebugLoc();
2558
36.8k
}
2559
2560
static Optional<std::pair<uint32_t, uint32_t>>
2561
100k
getLikelihoodWeights(Stmt::Likelihood LH) {
2562
100k
  switch (LH) {
2563
41
  case Stmt::LH_Unlikely:
2564
41
    return std::pair<uint32_t, uint32_t>(llvm::UnlikelyBranchWeight,
2565
41
                                         llvm::LikelyBranchWeight);
2566
100k
  case Stmt::LH_None:
2567
100k
    return None;
2568
17
  case Stmt::LH_Likely:
2569
17
    return std::pair<uint32_t, uint32_t>(llvm::LikelyBranchWeight,
2570
17
                                         llvm::UnlikelyBranchWeight);
2571
0
  }
2572
0
  llvm_unreachable("Unknown Likelihood");
2573
0
}
2574
2575
100k
llvm::MDNode *CodeGenFunction::createBranchWeights(Stmt::Likelihood LH) const {
2576
100k
  Optional<std::pair<uint32_t, uint32_t>> LHW = getLikelihoodWeights(LH);
2577
100k
  if (!LHW)
2578
100k
    return nullptr;
2579
2580
58
  llvm::MDBuilder MDHelper(CGM.getLLVMContext());
2581
58
  return MDHelper.createBranchWeights(LHW->first, LHW->second);
2582
58
}
2583
2584
llvm::MDNode *CodeGenFunction::createProfileOrBranchWeightsForLoop(
2585
11.0k
    const Stmt *Cond, uint64_t LoopCount, const Stmt *Body) const {
2586
11.0k
  llvm::MDNode *Weights = createProfileWeightsForLoop(Cond, LoopCount);
2587
11.0k
  if (!Weights && 
CGM.getCodeGenOpts().OptimizationLevel10.9k
)
2588
206
    Weights = createBranchWeights(Stmt::getLikelihood(Body));
2589
2590
11.0k
  return Weights;
2591
11.0k
}