Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
Line
Count
Source (jump to first uncovered line)
1
//===-- InstrProfiling.cpp - Frontend instrumentation based profiling -----===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This pass lowers instrprof_* intrinsics emitted by a frontend for profiling.
10
// It also builds the data structures and initialization code needed for
11
// updating execution counts and emitting the profile at runtime.
12
//
13
//===----------------------------------------------------------------------===//
14
15
#include "llvm/Transforms/Instrumentation/InstrProfiling.h"
16
#include "llvm/ADT/ArrayRef.h"
17
#include "llvm/ADT/SmallVector.h"
18
#include "llvm/ADT/StringRef.h"
19
#include "llvm/ADT/Triple.h"
20
#include "llvm/ADT/Twine.h"
21
#include "llvm/Analysis/BlockFrequencyInfo.h"
22
#include "llvm/Analysis/BranchProbabilityInfo.h"
23
#include "llvm/Analysis/LoopInfo.h"
24
#include "llvm/Analysis/TargetLibraryInfo.h"
25
#include "llvm/IR/Attributes.h"
26
#include "llvm/IR/BasicBlock.h"
27
#include "llvm/IR/Constant.h"
28
#include "llvm/IR/Constants.h"
29
#include "llvm/IR/DerivedTypes.h"
30
#include "llvm/IR/Dominators.h"
31
#include "llvm/IR/Function.h"
32
#include "llvm/IR/GlobalValue.h"
33
#include "llvm/IR/GlobalVariable.h"
34
#include "llvm/IR/IRBuilder.h"
35
#include "llvm/IR/Instruction.h"
36
#include "llvm/IR/Instructions.h"
37
#include "llvm/IR/IntrinsicInst.h"
38
#include "llvm/IR/Module.h"
39
#include "llvm/IR/Type.h"
40
#include "llvm/Pass.h"
41
#include "llvm/ProfileData/InstrProf.h"
42
#include "llvm/Support/Casting.h"
43
#include "llvm/Support/CommandLine.h"
44
#include "llvm/Support/Error.h"
45
#include "llvm/Support/ErrorHandling.h"
46
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
47
#include "llvm/Transforms/Utils/ModuleUtils.h"
48
#include "llvm/Transforms/Utils/SSAUpdater.h"
49
#include <algorithm>
50
#include <cassert>
51
#include <cstddef>
52
#include <cstdint>
53
#include <string>
54
55
using namespace llvm;
56
57
#define DEBUG_TYPE "instrprof"
58
59
// The start and end values of precise value profile range for memory
60
// intrinsic sizes
61
cl::opt<std::string> MemOPSizeRange(
62
    "memop-size-range",
63
    cl::desc("Set the range of size in memory intrinsic calls to be profiled "
64
             "precisely, in a format of <start_val>:<end_val>"),
65
    cl::init(""));
66
67
// The value that considered to be large value in  memory intrinsic.
68
cl::opt<unsigned> MemOPSizeLarge(
69
    "memop-size-large",
70
    cl::desc("Set large value thresthold in memory intrinsic size profiling. "
71
             "Value of 0 disables the large value profiling."),
72
    cl::init(8192));
73
74
namespace {
75
76
cl::opt<bool> DoNameCompression("enable-name-compression",
77
                                cl::desc("Enable name string compression"),
78
                                cl::init(true));
79
80
cl::opt<bool> DoHashBasedCounterSplit(
81
    "hash-based-counter-split",
82
    cl::desc("Rename counter variable of a comdat function based on cfg hash"),
83
    cl::init(true));
84
85
cl::opt<bool> ValueProfileStaticAlloc(
86
    "vp-static-alloc",
87
    cl::desc("Do static counter allocation for value profiler"),
88
    cl::init(true));
89
90
cl::opt<double> NumCountersPerValueSite(
91
    "vp-counters-per-site",
92
    cl::desc("The average number of profile counters allocated "
93
             "per value profiling site."),
94
    // This is set to a very small value because in real programs, only
95
    // a very small percentage of value sites have non-zero targets, e.g, 1/30.
96
    // For those sites with non-zero profile, the average number of targets
97
    // is usually smaller than 2.
98
    cl::init(1.0));
99
100
cl::opt<bool> AtomicCounterUpdateAll(
101
    "instrprof-atomic-counter-update-all", cl::ZeroOrMore,
102
    cl::desc("Make all profile counter updates atomic (for testing only)"),
103
    cl::init(false));
104
105
cl::opt<bool> AtomicCounterUpdatePromoted(
106
    "atomic-counter-update-promoted", cl::ZeroOrMore,
107
    cl::desc("Do counter update using atomic fetch add "
108
             " for promoted counters only"),
109
    cl::init(false));
110
111
// If the option is not specified, the default behavior about whether
112
// counter promotion is done depends on how instrumentaiton lowering
113
// pipeline is setup, i.e., the default value of true of this option
114
// does not mean the promotion will be done by default. Explicitly
115
// setting this option can override the default behavior.
116
cl::opt<bool> DoCounterPromotion("do-counter-promotion", cl::ZeroOrMore,
117
                                 cl::desc("Do counter register promotion"),
118
                                 cl::init(false));
119
cl::opt<unsigned> MaxNumOfPromotionsPerLoop(
120
    cl::ZeroOrMore, "max-counter-promotions-per-loop", cl::init(20),
121
    cl::desc("Max number counter promotions per loop to avoid"
122
             " increasing register pressure too much"));
123
124
// A debug option
125
cl::opt<int>
126
    MaxNumOfPromotions(cl::ZeroOrMore, "max-counter-promotions", cl::init(-1),
127
                       cl::desc("Max number of allowed counter promotions"));
128
129
cl::opt<unsigned> SpeculativeCounterPromotionMaxExiting(
130
    cl::ZeroOrMore, "speculative-counter-promotion-max-exiting", cl::init(3),
131
    cl::desc("The max number of exiting blocks of a loop to allow "
132
             " speculative counter promotion"));
133
134
cl::opt<bool> SpeculativeCounterPromotionToLoop(
135
    cl::ZeroOrMore, "speculative-counter-promotion-to-loop", cl::init(false),
136
    cl::desc("When the option is false, if the target block is in a loop, "
137
             "the promotion will be disallowed unless the promoted counter "
138
             " update can be further/iteratively promoted into an acyclic "
139
             " region."));
140
141
cl::opt<bool> IterativeCounterPromotion(
142
    cl::ZeroOrMore, "iterative-counter-promotion", cl::init(true),
143
    cl::desc("Allow counter promotion across the whole loop nest."));
144
145
class InstrProfilingLegacyPass : public ModulePass {
146
  InstrProfiling InstrProf;
147
148
public:
149
  static char ID;
150
151
42
  InstrProfilingLegacyPass() : ModulePass(ID) {}
152
  InstrProfilingLegacyPass(const InstrProfOptions &Options, bool IsCS = false)
153
100
      : ModulePass(ID), InstrProf(Options, IsCS) {}
154
155
11
  StringRef getPassName() const override {
156
11
    return "Frontend instrumentation-based coverage lowering";
157
11
  }
158
159
142
  bool runOnModule(Module &M) override {
160
142
    return InstrProf.run(M, getAnalysis<TargetLibraryInfoWrapperPass>().getTLI());
161
142
  }
162
163
142
  void getAnalysisUsage(AnalysisUsage &AU) const override {
164
142
    AU.setPreservesCFG();
165
142
    AU.addRequired<TargetLibraryInfoWrapperPass>();
166
142
  }
167
};
168
169
///
170
/// A helper class to promote one counter RMW operation in the loop
171
/// into register update.
172
///
173
/// RWM update for the counter will be sinked out of the loop after
174
/// the transformation.
175
///
176
class PGOCounterPromoterHelper : public LoadAndStorePromoter {
177
public:
178
  PGOCounterPromoterHelper(
179
      Instruction *L, Instruction *S, SSAUpdater &SSA, Value *Init,
180
      BasicBlock *PH, ArrayRef<BasicBlock *> ExitBlocks,
181
      ArrayRef<Instruction *> InsertPts,
182
      DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCands,
183
      LoopInfo &LI)
184
      : LoadAndStorePromoter({L, S}, SSA), Store(S), ExitBlocks(ExitBlocks),
185
47
        InsertPts(InsertPts), LoopToCandidates(LoopToCands), LI(LI) {
186
47
    assert(isa<LoadInst>(L));
187
47
    assert(isa<StoreInst>(S));
188
47
    SSA.AddAvailableValue(PH, Init);
189
47
  }
190
191
47
  void doExtraRewritesBeforeFinalDeletion() override {
192
100
    for (unsigned i = 0, e = ExitBlocks.size(); i != e; 
++i53
) {
193
53
      BasicBlock *ExitBlock = ExitBlocks[i];
194
53
      Instruction *InsertPos = InsertPts[i];
195
53
      // Get LiveIn value into the ExitBlock. If there are multiple
196
53
      // predecessors, the value is defined by a PHI node in this
197
53
      // block.
198
53
      Value *LiveInValue = SSA.GetValueInMiddleOfBlock(ExitBlock);
199
53
      Value *Addr = cast<StoreInst>(Store)->getPointerOperand();
200
53
      Type *Ty = LiveInValue->getType();
201
53
      IRBuilder<> Builder(InsertPos);
202
53
      if (AtomicCounterUpdatePromoted)
203
6
        // automic update currently can only be promoted across the current
204
6
        // loop, not the whole loop nest.
205
6
        Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, LiveInValue,
206
6
                                AtomicOrdering::SequentiallyConsistent);
207
47
      else {
208
47
        LoadInst *OldVal = Builder.CreateLoad(Ty, Addr, "pgocount.promoted");
209
47
        auto *NewVal = Builder.CreateAdd(OldVal, LiveInValue);
210
47
        auto *NewStore = Builder.CreateStore(NewVal, Addr);
211
47
212
47
        // Now update the parent loop's candidate list:
213
47
        if (IterativeCounterPromotion) {
214
47
          auto *TargetLoop = LI.getLoopFor(ExitBlock);
215
47
          if (TargetLoop)
216
8
            LoopToCandidates[TargetLoop].emplace_back(OldVal, NewStore);
217
47
        }
218
47
      }
219
53
    }
220
47
  }
221
222
private:
223
  Instruction *Store;
224
  ArrayRef<BasicBlock *> ExitBlocks;
225
  ArrayRef<Instruction *> InsertPts;
226
  DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCandidates;
227
  LoopInfo &LI;
228
};
229
230
/// A helper class to do register promotion for all profile counter
231
/// updates in a loop.
232
///
233
class PGOCounterPromoter {
234
public:
235
  PGOCounterPromoter(
236
      DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCands,
237
      Loop &CurLoop, LoopInfo &LI, BlockFrequencyInfo *BFI)
238
      : LoopToCandidates(LoopToCands), ExitBlocks(), InsertPts(), L(CurLoop),
239
24
        LI(LI), BFI(BFI) {
240
24
241
24
    SmallVector<BasicBlock *, 8> LoopExitBlocks;
242
24
    SmallPtrSet<BasicBlock *, 8> BlockSet;
243
24
    L.getExitBlocks(LoopExitBlocks);
244
24
245
27
    for (BasicBlock *ExitBlock : LoopExitBlocks) {
246
27
      if (BlockSet.insert(ExitBlock).second) {
247
25
        ExitBlocks.push_back(ExitBlock);
248
25
        InsertPts.push_back(&*ExitBlock->getFirstInsertionPt());
249
25
      }
250
27
    }
251
24
  }
252
253
24
  bool run(int64_t *NumPromoted) {
254
24
    // Skip 'infinite' loops:
255
24
    if (ExitBlocks.size() == 0)
256
1
      return false;
257
23
    unsigned MaxProm = getMaxNumOfPromotionsInLoop(&L);
258
23
    if (MaxProm == 0)
259
6
      return false;
260
17
261
17
    unsigned Promoted = 0;
262
47
    for (auto &Cand : LoopToCandidates[&L]) {
263
47
264
47
      SmallVector<PHINode *, 4> NewPHIs;
265
47
      SSAUpdater SSA(&NewPHIs);
266
47
      Value *InitVal = ConstantInt::get(Cand.first->getType(), 0);
267
47
268
47
      // If BFI is set, we will use it to guide the promotions.
269
47
      if (BFI) {
270
3
        auto *BB = Cand.first->getParent();
271
3
        auto InstrCount = BFI->getBlockProfileCount(BB);
272
3
        if (!InstrCount)
273
0
          continue;
274
3
        auto PreheaderCount = BFI->getBlockProfileCount(L.getLoopPreheader());
275
3
        // If the average loop trip count is not greater than 1.5, we skip
276
3
        // promotion.
277
3
        if (PreheaderCount &&
278
3
            (PreheaderCount.getValue() * 3) >= (InstrCount.getValue() * 2))
279
0
          continue;
280
47
      }
281
47
282
47
      PGOCounterPromoterHelper Promoter(Cand.first, Cand.second, SSA, InitVal,
283
47
                                        L.getLoopPreheader(), ExitBlocks,
284
47
                                        InsertPts, LoopToCandidates, LI);
285
47
      Promoter.run(SmallVector<Instruction *, 2>({Cand.first, Cand.second}));
286
47
      Promoted++;
287
47
      if (Promoted >= MaxProm)
288
0
        break;
289
47
290
47
      (*NumPromoted)++;
291
47
      if (MaxNumOfPromotions != -1 && 
*NumPromoted >= MaxNumOfPromotions0
)
292
0
        break;
293
47
    }
294
17
295
17
    LLVM_DEBUG(dbgs() << Promoted << " counters promoted for loop (depth="
296
17
                      << L.getLoopDepth() << ")\n");
297
17
    return Promoted != 0;
298
17
  }
299
300
private:
301
0
  bool allowSpeculativeCounterPromotion(Loop *LP) {
302
0
    SmallVector<BasicBlock *, 8> ExitingBlocks;
303
0
    L.getExitingBlocks(ExitingBlocks);
304
0
    // Not considierered speculative.
305
0
    if (ExitingBlocks.size() == 1)
306
0
      return true;
307
0
    if (ExitingBlocks.size() > SpeculativeCounterPromotionMaxExiting)
308
0
      return false;
309
0
    return true;
310
0
  }
311
312
  // Returns the max number of Counter Promotions for LP.
313
23
  unsigned getMaxNumOfPromotionsInLoop(Loop *LP) {
314
23
    // We can't insert into a catchswitch.
315
23
    SmallVector<BasicBlock *, 8> LoopExitBlocks;
316
23
    LP->getExitBlocks(LoopExitBlocks);
317
27
    if (
llvm::any_of(LoopExitBlocks, [](BasicBlock *Exit) 23
{
318
27
          return isa<CatchSwitchInst>(Exit->getTerminator());
319
27
        }))
320
0
      return 0;
321
23
322
23
    if (!LP->hasDedicatedExits())
323
0
      return 0;
324
23
325
23
    BasicBlock *PH = LP->getLoopPreheader();
326
23
    if (!PH)
327
6
      return 0;
328
17
329
17
    SmallVector<BasicBlock *, 8> ExitingBlocks;
330
17
    LP->getExitingBlocks(ExitingBlocks);
331
17
332
17
    // If BFI is set, we do more aggressive promotions based on BFI.
333
17
    if (BFI)
334
1
      return (unsigned)-1;
335
16
336
16
    // Not considierered speculative.
337
16
    if (ExitingBlocks.size() == 1)
338
12
      return MaxNumOfPromotionsPerLoop;
339
4
340
4
    if (ExitingBlocks.size() > SpeculativeCounterPromotionMaxExiting)
341
0
      return 0;
342
4
343
4
    // Whether the target block is in a loop does not matter:
344
4
    if (SpeculativeCounterPromotionToLoop)
345
0
      return MaxNumOfPromotionsPerLoop;
346
4
347
4
    // Now check the target block:
348
4
    unsigned MaxProm = MaxNumOfPromotionsPerLoop;
349
8
    for (auto *TargetBlock : LoopExitBlocks) {
350
8
      auto *TargetLoop = LI.getLoopFor(TargetBlock);
351
8
      if (!TargetLoop)
352
8
        continue;
353
0
      unsigned MaxPromForTarget = getMaxNumOfPromotionsInLoop(TargetLoop);
354
0
      unsigned PendingCandsInTarget = LoopToCandidates[TargetLoop].size();
355
0
      MaxProm =
356
0
          std::min(MaxProm, std::max(MaxPromForTarget, PendingCandsInTarget) -
357
0
                                PendingCandsInTarget);
358
0
    }
359
4
    return MaxProm;
360
4
  }
361
362
  DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCandidates;
363
  SmallVector<BasicBlock *, 8> ExitBlocks;
364
  SmallVector<Instruction *, 8> InsertPts;
365
  Loop &L;
366
  LoopInfo &LI;
367
  BlockFrequencyInfo *BFI;
368
};
369
370
} // end anonymous namespace
371
372
38
PreservedAnalyses InstrProfiling::run(Module &M, ModuleAnalysisManager &AM) {
373
38
  auto &TLI = AM.getResult<TargetLibraryAnalysis>(M);
374
38
  if (!run(M, TLI))
375
1
    return PreservedAnalyses::all();
376
37
377
37
  return PreservedAnalyses::none();
378
37
}
379
380
char InstrProfilingLegacyPass::ID = 0;
381
11.0k
INITIALIZE_PASS_BEGIN(
382
11.0k
    InstrProfilingLegacyPass, "instrprof",
383
11.0k
    "Frontend instrumentation-based coverage lowering.", false, false)
384
11.0k
INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
385
11.0k
INITIALIZE_PASS_END(
386
    InstrProfilingLegacyPass, "instrprof",
387
    "Frontend instrumentation-based coverage lowering.", false, false)
388
389
ModulePass *
390
llvm::createInstrProfilingLegacyPass(const InstrProfOptions &Options,
391
100
                                     bool IsCS) {
392
100
  return new InstrProfilingLegacyPass(Options, IsCS);
393
100
}
394
395
6.82k
static InstrProfIncrementInst *castToIncrementInst(Instruction *Instr) {
396
6.82k
  InstrProfIncrementInst *Inc = dyn_cast<InstrProfIncrementInstStep>(Instr);
397
6.82k
  if (Inc)
398
2
    return Inc;
399
6.82k
  return dyn_cast<InstrProfIncrementInst>(Instr);
400
6.82k
}
401
402
839
bool InstrProfiling::lowerIntrinsics(Function *F) {
403
839
  bool MadeChange = false;
404
839
  PromotionCandidates.clear();
405
1.70k
  for (BasicBlock &BB : *F) {
406
8.52k
    for (auto I = BB.begin(), E = BB.end(); I != E;) {
407
6.82k
      auto Instr = I++;
408
6.82k
      InstrProfIncrementInst *Inc = castToIncrementInst(&*Instr);
409
6.82k
      if (Inc) {
410
864
        lowerIncrement(Inc);
411
864
        MadeChange = true;
412
5.95k
      } else if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(Instr)) {
413
18
        lowerValueProfileInst(Ind);
414
18
        MadeChange = true;
415
18
      }
416
6.82k
    }
417
1.70k
  }
418
839
419
839
  if (!MadeChange)
420
442
    return false;
421
397
422
397
  promoteCounterLoadStores(F);
423
397
  return true;
424
397
}
425
426
1.25k
bool InstrProfiling::isCounterPromotionEnabled() const {
427
1.25k
  if (DoCounterPromotion.getNumOccurrences() > 0)
428
76
    return DoCounterPromotion;
429
1.18k
430
1.18k
  return Options.DoCounterPromotion;
431
1.18k
}
432
433
397
void InstrProfiling::promoteCounterLoadStores(Function *F) {
434
397
  if (!isCounterPromotionEnabled())
435
375
    return;
436
22
437
22
  DominatorTree DT(*F);
438
22
  LoopInfo LI(DT);
439
22
  DenseMap<Loop *, SmallVector<LoadStorePair, 8>> LoopPromotionCandidates;
440
22
441
22
  std::unique_ptr<BlockFrequencyInfo> BFI;
442
22
  if (Options.UseBFIInPromotion) {
443
4
    std::unique_ptr<BranchProbabilityInfo> BPI;
444
4
    BPI.reset(new BranchProbabilityInfo(*F, LI, TLI));
445
4
    BFI.reset(new BlockFrequencyInfo(*F, *BPI, LI));
446
4
  }
447
22
448
78
  for (const auto &LoadStore : PromotionCandidates) {
449
78
    auto *CounterLoad = LoadStore.first;
450
78
    auto *CounterStore = LoadStore.second;
451
78
    BasicBlock *BB = CounterLoad->getParent();
452
78
    Loop *ParentLoop = LI.getLoopFor(BB);
453
78
    if (!ParentLoop)
454
30
      continue;
455
48
    LoopPromotionCandidates[ParentLoop].emplace_back(CounterLoad, CounterStore);
456
48
  }
457
22
458
22
  SmallVector<Loop *, 4> Loops = LI.getLoopsInPreorder();
459
22
460
22
  // Do a post-order traversal of the loops so that counter updates can be
461
22
  // iteratively hoisted outside the loop nest.
462
24
  for (auto *Loop : llvm::reverse(Loops)) {
463
24
    PGOCounterPromoter Promoter(LoopPromotionCandidates, *Loop, LI, BFI.get());
464
24
    Promoter.run(&TotalCountersPromoted);
465
24
  }
466
22
}
467
468
/// Check if the module contains uses of any profiling intrinsics.
469
180
static bool containsProfilingIntrinsics(Module &M) {
470
180
  if (auto *F = M.getFunction(
471
153
          Intrinsic::getName(llvm::Intrinsic::instrprof_increment)))
472
153
    if (!F->use_empty())
473
152
      return true;
474
28
  if (auto *F = M.getFunction(
475
1
          Intrinsic::getName(llvm::Intrinsic::instrprof_increment_step)))
476
1
    if (!F->use_empty())
477
1
      return true;
478
27
  if (auto *F = M.getFunction(
479
0
          Intrinsic::getName(llvm::Intrinsic::instrprof_value_profile)))
480
0
    if (!F->use_empty())
481
0
      return true;
482
27
  return false;
483
27
}
484
485
180
bool InstrProfiling::run(Module &M, const TargetLibraryInfo &TLI) {
486
180
  this->M = &M;
487
180
  this->TLI = &TLI;
488
180
  NamesVar = nullptr;
489
180
  NamesSize = 0;
490
180
  ProfileDataMap.clear();
491
180
  UsedVars.clear();
492
180
  getMemOPSizeRangeFromOption(MemOPSizeRange, MemOPSizeRangeStart,
493
180
                              MemOPSizeRangeLast);
494
180
  TT = Triple(M.getTargetTriple());
495
180
496
180
  // Emit the runtime hook even if no counters are present.
497
180
  bool MadeChange = emitRuntimeHook();
498
180
499
180
  // Improve compile time by avoiding linear scans when there is no work.
500
180
  GlobalVariable *CoverageNamesVar =
501
180
      M.getNamedGlobal(getCoverageUnusedNamesVarName());
502
180
  if (!containsProfilingIntrinsics(M) && 
!CoverageNamesVar27
)
503
26
    return MadeChange;
504
154
505
154
  // We did not know how many value sites there would be inside
506
154
  // the instrumented function. This is counting the number of instrumented
507
154
  // target value sites to enter it as field in the profile data variable.
508
822
  
for (Function &F : M)154
{
509
822
    InstrProfIncrementInst *FirstProfIncInst = nullptr;
510
822
    for (BasicBlock &BB : F)
511
8.52k
      
for (auto I = BB.begin(), E = BB.end(); 1.70k
I != E;
I++6.82k
)
512
6.82k
        if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(I))
513
18
          computeNumValueSiteCounts(Ind);
514
6.80k
        else if (FirstProfIncInst == nullptr)
515
1.97k
          FirstProfIncInst = dyn_cast<InstrProfIncrementInst>(I);
516
822
517
822
    // Value profiling intrinsic lowering requires per-function profile data
518
822
    // variable to be created first.
519
822
    if (FirstProfIncInst != nullptr)
520
396
      static_cast<void>(getOrCreateRegionCounters(FirstProfIncInst));
521
822
  }
522
154
523
154
  for (Function &F : M)
524
839
    MadeChange |= lowerIntrinsics(&F);
525
154
526
154
  if (CoverageNamesVar) {
527
6
    lowerCoverageData(CoverageNamesVar);
528
6
    MadeChange = true;
529
6
  }
530
154
531
154
  if (!MadeChange)
532
0
    return false;
533
154
534
154
  emitVNodes();
535
154
  emitNameData();
536
154
  emitRegistration();
537
154
  emitUses();
538
154
  emitInitialization();
539
154
  return true;
540
154
}
541
542
static FunctionCallee
543
getOrInsertValueProfilingCall(Module &M, const TargetLibraryInfo &TLI,
544
18
                              bool IsRange = false) {
545
18
  LLVMContext &Ctx = M.getContext();
546
18
  auto *ReturnTy = Type::getVoidTy(M.getContext());
547
18
548
18
  AttributeList AL;
549
18
  if (auto AK = TLI.getExtAttrForI32Param(false))
550
6
    AL = AL.addParamAttribute(M.getContext(), 2, AK);
551
18
552
18
  if (!IsRange) {
553
16
    Type *ParamTypes[] = {
554
48
#define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
555
16
#include "llvm/ProfileData/InstrProfData.inc"
556
16
    };
557
16
    auto *ValueProfilingCallTy =
558
16
        FunctionType::get(ReturnTy, makeArrayRef(ParamTypes), false);
559
16
    return M.getOrInsertFunction(getInstrProfValueProfFuncName(),
560
16
                                 ValueProfilingCallTy, AL);
561
16
  } else {
562
2
    Type *RangeParamTypes[] = {
563
2
#define VALUE_RANGE_PROF 1
564
12
#define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
565
2
#include "llvm/ProfileData/InstrProfData.inc"
566
2
#undef VALUE_RANGE_PROF
567
2
    };
568
2
    auto *ValueRangeProfilingCallTy =
569
2
        FunctionType::get(ReturnTy, makeArrayRef(RangeParamTypes), false);
570
2
    return M.getOrInsertFunction(getInstrProfValueRangeProfFuncName(),
571
2
                                 ValueRangeProfilingCallTy, AL);
572
2
  }
573
18
}
574
575
18
void InstrProfiling::computeNumValueSiteCounts(InstrProfValueProfileInst *Ind) {
576
18
  GlobalVariable *Name = Ind->getName();
577
18
  uint64_t ValueKind = Ind->getValueKind()->getZExtValue();
578
18
  uint64_t Index = Ind->getIndex()->getZExtValue();
579
18
  auto It = ProfileDataMap.find(Name);
580
18
  if (It == ProfileDataMap.end()) {
581
18
    PerFunctionProfileData PD;
582
18
    PD.NumValueSites[ValueKind] = Index + 1;
583
18
    ProfileDataMap[Name] = PD;
584
18
  } else 
if (0
It->second.NumValueSites[ValueKind] <= Index0
)
585
0
    It->second.NumValueSites[ValueKind] = Index + 1;
586
18
}
587
588
18
void InstrProfiling::lowerValueProfileInst(InstrProfValueProfileInst *Ind) {
589
18
  GlobalVariable *Name = Ind->getName();
590
18
  auto It = ProfileDataMap.find(Name);
591
18
  assert(It != ProfileDataMap.end() && It->second.DataVar &&
592
18
         "value profiling detected in function with no counter incerement");
593
18
594
18
  GlobalVariable *DataVar = It->second.DataVar;
595
18
  uint64_t ValueKind = Ind->getValueKind()->getZExtValue();
596
18
  uint64_t Index = Ind->getIndex()->getZExtValue();
597
20
  for (uint32_t Kind = IPVK_First; Kind < ValueKind; 
++Kind2
)
598
2
    Index += It->second.NumValueSites[Kind];
599
18
600
18
  IRBuilder<> Builder(Ind);
601
18
  bool IsRange = (Ind->getValueKind()->getZExtValue() ==
602
18
                  llvm::InstrProfValueKind::IPVK_MemOPSize);
603
18
  CallInst *Call = nullptr;
604
18
  if (!IsRange) {
605
16
    Value *Args[3] = {Ind->getTargetValue(),
606
16
                      Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
607
16
                      Builder.getInt32(Index)};
608
16
    Call = Builder.CreateCall(getOrInsertValueProfilingCall(*M, *TLI), Args);
609
16
  } else {
610
2
    Value *Args[6] = {
611
2
        Ind->getTargetValue(),
612
2
        Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
613
2
        Builder.getInt32(Index),
614
2
        Builder.getInt64(MemOPSizeRangeStart),
615
2
        Builder.getInt64(MemOPSizeRangeLast),
616
2
        Builder.getInt64(MemOPSizeLarge == 0 ? INT64_MIN : MemOPSizeLarge)};
617
2
    Call =
618
2
        Builder.CreateCall(getOrInsertValueProfilingCall(*M, *TLI, true), Args);
619
2
  }
620
18
  if (auto AK = TLI->getExtAttrForI32Param(false))
621
6
    Call->addParamAttr(2, AK);
622
18
  Ind->replaceAllUsesWith(Call);
623
18
  Ind->eraseFromParent();
624
18
}
625
626
864
void InstrProfiling::lowerIncrement(InstrProfIncrementInst *Inc) {
627
864
  GlobalVariable *Counters = getOrCreateRegionCounters(Inc);
628
864
629
864
  IRBuilder<> Builder(Inc);
630
864
  uint64_t Index = Inc->getIndex()->getZExtValue();
631
864
  Value *Addr = Builder.CreateConstInBoundsGEP2_64(Counters->getValueType(),
632
864
                                                   Counters, 0, Index);
633
864
634
864
  if (Options.Atomic || 
AtomicCounterUpdateAll863
) {
635
2
    Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, Inc->getStep(),
636
2
                            AtomicOrdering::Monotonic);
637
862
  } else {
638
862
    Value *IncStep = Inc->getStep();
639
862
    Value *Load = Builder.CreateLoad(IncStep->getType(), Addr, "pgocount");
640
862
    auto *Count = Builder.CreateAdd(Load, Inc->getStep());
641
862
    auto *Store = Builder.CreateStore(Count, Addr);
642
862
    if (isCounterPromotionEnabled())
643
78
      PromotionCandidates.emplace_back(cast<Instruction>(Load), Store);
644
862
  }
645
864
  Inc->eraseFromParent();
646
864
}
647
648
6
void InstrProfiling::lowerCoverageData(GlobalVariable *CoverageNamesVar) {
649
6
  ConstantArray *Names =
650
6
      cast<ConstantArray>(CoverageNamesVar->getInitializer());
651
23
  for (unsigned I = 0, E = Names->getNumOperands(); I < E; 
++I17
) {
652
17
    Constant *NC = Names->getOperand(I);
653
17
    Value *V = NC->stripPointerCasts();
654
17
    assert(isa<GlobalVariable>(V) && "Missing reference to function name");
655
17
    GlobalVariable *Name = cast<GlobalVariable>(V);
656
17
657
17
    Name->setLinkage(GlobalValue::PrivateLinkage);
658
17
    ReferencedNames.push_back(Name);
659
17
    NC->dropAllReferences();
660
17
  }
661
6
  CoverageNamesVar->eraseFromParent();
662
6
}
663
664
/// Get the name of a profiling variable for a particular function.
665
852
static std::string getVarName(InstrProfIncrementInst *Inc, StringRef Prefix) {
666
852
  StringRef NamePrefix = getInstrProfNameVarPrefix();
667
852
  StringRef Name = Inc->getName()->getName().substr(NamePrefix.size());
668
852
  Function *F = Inc->getParent()->getParent();
669
852
  Module *M = F->getParent();
670
852
  if (!DoHashBasedCounterSplit || !isIRPGOFlagSet(M) ||
671
852
      
!canRenameComdatFunc(*F)61
)
672
843
    return (Prefix + Name).str();
673
9
  uint64_t FuncHash = Inc->getHash()->getZExtValue();
674
9
  SmallVector<char, 24> HashPostfix;
675
9
  if (Name.endswith((Twine(".") + Twine(FuncHash)).toStringRef(HashPostfix)))
676
0
    return (Prefix + Name).str();
677
9
  return (Prefix + Name + "." + Twine(FuncHash)).str();
678
9
}
679
680
397
static inline bool shouldRecordFunctionAddr(Function *F) {
681
397
  // Check the linkage
682
397
  bool HasAvailableExternallyLinkage = F->hasAvailableExternallyLinkage();
683
397
  if (!F->hasLinkOnceLinkage() && 
!F->hasLocalLinkage()322
&&
684
397
      
!HasAvailableExternallyLinkage299
)
685
287
    return true;
686
110
687
110
  // A function marked 'alwaysinline' with available_externally linkage can't
688
110
  // have its address taken. Doing so would create an undefined external ref to
689
110
  // the function, which would fail to link.
690
110
  if (HasAvailableExternallyLinkage &&
691
110
      
F->hasFnAttribute(Attribute::AlwaysInline)12
)
692
1
    return false;
693
109
694
109
  // Prohibit function address recording if the function is both internal and
695
109
  // COMDAT. This avoids the profile data variable referencing internal symbols
696
109
  // in COMDAT.
697
109
  if (F->hasLocalLinkage() && 
F->hasComdat()23
)
698
2
    return false;
699
107
700
107
  // Check uses of this function for other than direct calls or invokes to it.
701
107
  // Inline virtual functions have linkeOnceODR linkage. When a key method
702
107
  // exists, the vtable will only be emitted in the TU where the key method
703
107
  // is defined. In a TU where vtable is not available, the function won't
704
107
  // be 'addresstaken'. If its address is not recorded here, the profile data
705
107
  // with missing address may be picked by the linker leading  to missing
706
107
  // indirect call target info.
707
107
  return F->hasAddressTaken() || 
F->hasLinkOnceLinkage()101
;
708
107
}
709
710
701
static bool needsRuntimeRegistrationOfSectionRange(const Triple &TT) {
711
701
  // Don't do this for Darwin.  compiler-rt uses linker magic.
712
701
  if (TT.isOSDarwin())
713
463
    return false;
714
238
  // Use linker script magic to get data/cnts/name start/end.
715
238
  if (TT.isOSLinux() || 
TT.isOSFreeBSD()107
||
TT.isOSNetBSD()101
||
716
238
      
TT.isOSSolaris()101
||
TT.isOSFuchsia()95
||
TT.isPS4CPU()95
||
717
238
      
TT.isOSWindows()89
)
718
194
    return false;
719
44
720
44
  return true;
721
44
}
722
723
GlobalVariable *
724
1.26k
InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
725
1.26k
  GlobalVariable *NamePtr = Inc->getName();
726
1.26k
  auto It = ProfileDataMap.find(NamePtr);
727
1.26k
  PerFunctionProfileData PD;
728
1.26k
  if (It != ProfileDataMap.end()) {
729
881
    if (It->second.RegionCounters)
730
863
      return It->second.RegionCounters;
731
18
    PD = It->second;
732
18
  }
733
1.26k
734
1.26k
  // Match the linkage and visibility of the name global, except on COFF, where
735
1.26k
  // the linkage must be local and consequentially the visibility must be
736
1.26k
  // default.
737
1.26k
  Function *Fn = Inc->getParent()->getParent();
738
397
  GlobalValue::LinkageTypes Linkage = NamePtr->getLinkage();
739
397
  GlobalValue::VisibilityTypes Visibility = NamePtr->getVisibility();
740
397
  if (TT.isOSBinFormatCOFF()) {
741
23
    Linkage = GlobalValue::InternalLinkage;
742
23
    Visibility = GlobalValue::DefaultVisibility;
743
23
  }
744
397
745
397
  // Move the name variable to the right section. Place them in a COMDAT group
746
397
  // if the associated function is a COMDAT. This will make sure that only one
747
397
  // copy of counters of the COMDAT function will be emitted after linking. Keep
748
397
  // in mind that this pass may run before the inliner, so we need to create a
749
397
  // new comdat group for the counters and profiling data. If we use the comdat
750
397
  // of the parent function, that will result in relocations against discarded
751
397
  // sections.
752
397
  Comdat *Cmdt = nullptr;
753
397
  GlobalValue::LinkageTypes CounterLinkage = Linkage;
754
397
  if (needsComdatForCounter(*Fn, *M)) {
755
42
    StringRef CmdtPrefix = getInstrProfComdatPrefix();
756
42
    if (TT.isOSBinFormatCOFF()) {
757
9
      // For COFF, the comdat group name must be the name of a symbol in the
758
9
      // group. Use the counter variable name, and upgrade its linkage to
759
9
      // something externally visible, like linkonce_odr.
760
9
      CmdtPrefix = getInstrProfCountersVarPrefix();
761
9
      CounterLinkage = GlobalValue::LinkOnceODRLinkage;
762
9
    }
763
42
    Cmdt = M->getOrInsertComdat(getVarName(Inc, CmdtPrefix));
764
42
  }
765
397
766
397
  uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
767
397
  LLVMContext &Ctx = M->getContext();
768
397
  ArrayType *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters);
769
397
770
397
  // Create the counters variable.
771
397
  auto *CounterPtr =
772
397
      new GlobalVariable(*M, CounterTy, false, Linkage,
773
397
                         Constant::getNullValue(CounterTy),
774
397
                         getVarName(Inc, getInstrProfCountersVarPrefix()));
775
397
  CounterPtr->setVisibility(Visibility);
776
397
  CounterPtr->setSection(
777
397
      getInstrProfSectionName(IPSK_cnts, TT.getObjectFormat()));
778
397
  CounterPtr->setAlignment(8);
779
397
  CounterPtr->setComdat(Cmdt);
780
397
  CounterPtr->setLinkage(CounterLinkage);
781
397
782
397
  auto *Int8PtrTy = Type::getInt8PtrTy(Ctx);
783
397
  // Allocate statically the array of pointers to value profile nodes for
784
397
  // the current function.
785
397
  Constant *ValuesPtrExpr = ConstantPointerNull::get(Int8PtrTy);
786
397
  if (ValueProfileStaticAlloc && 
!needsRuntimeRegistrationOfSectionRange(TT)395
) {
787
379
    uint64_t NS = 0;
788
1.13k
    for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; 
++Kind758
)
789
758
      NS += PD.NumValueSites[Kind];
790
379
    if (NS) {
791
16
      ArrayType *ValuesTy = ArrayType::get(Type::getInt64Ty(Ctx), NS);
792
16
793
16
      auto *ValuesVar =
794
16
          new GlobalVariable(*M, ValuesTy, false, Linkage,
795
16
                             Constant::getNullValue(ValuesTy),
796
16
                             getVarName(Inc, getInstrProfValuesVarPrefix()));
797
16
      ValuesVar->setVisibility(Visibility);
798
16
      ValuesVar->setSection(
799
16
          getInstrProfSectionName(IPSK_vals, TT.getObjectFormat()));
800
16
      ValuesVar->setAlignment(8);
801
16
      ValuesVar->setComdat(Cmdt);
802
16
      ValuesPtrExpr =
803
16
          ConstantExpr::getBitCast(ValuesVar, Type::getInt8PtrTy(Ctx));
804
16
    }
805
379
  }
806
397
807
397
  // Create data variable.
808
397
  auto *Int16Ty = Type::getInt16Ty(Ctx);
809
397
  auto *Int16ArrayTy = ArrayType::get(Int16Ty, IPVK_Last + 1);
810
397
  Type *DataTypes[] = {
811
2.77k
#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) LLVMType,
812
397
#include "llvm/ProfileData/InstrProfData.inc"
813
397
  };
814
397
  auto *DataTy = StructType::get(Ctx, makeArrayRef(DataTypes));
815
397
816
397
  Constant *FunctionAddr = shouldRecordFunctionAddr(Fn)
817
397
                               ? 
ConstantExpr::getBitCast(Fn, Int8PtrTy)368
818
397
                               : 
ConstantPointerNull::get(Int8PtrTy)29
;
819
397
820
397
  Constant *Int16ArrayVals[IPVK_Last + 1];
821
1.19k
  for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; 
++Kind794
)
822
794
    Int16ArrayVals[Kind] = ConstantInt::get(Int16Ty, PD.NumValueSites[Kind]);
823
397
824
397
  Constant *DataVals[] = {
825
2.77k
#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init,
826
397
#include "llvm/ProfileData/InstrProfData.inc"
827
397
  };
828
397
  auto *Data = new GlobalVariable(*M, DataTy, false, Linkage,
829
397
                                  ConstantStruct::get(DataTy, DataVals),
830
397
                                  getVarName(Inc, getInstrProfDataVarPrefix()));
831
397
  Data->setVisibility(Visibility);
832
397
  Data->setSection(getInstrProfSectionName(IPSK_data, TT.getObjectFormat()));
833
397
  Data->setAlignment(INSTR_PROF_DATA_ALIGNMENT);
834
397
  Data->setComdat(Cmdt);
835
397
836
397
  PD.RegionCounters = CounterPtr;
837
397
  PD.DataVar = Data;
838
397
  ProfileDataMap[NamePtr] = PD;
839
397
840
397
  // Mark the data variable as used so that it isn't stripped out.
841
397
  UsedVars.push_back(Data);
842
397
  // Now that the linkage set by the FE has been passed to the data and counter
843
397
  // variables, reset Name variable's linkage and visibility to private so that
844
397
  // it can be removed later by the compiler.
845
397
  NamePtr->setLinkage(GlobalValue::PrivateLinkage);
846
397
  // Collect the referenced names to be used by emitNameData.
847
397
  ReferencedNames.push_back(NamePtr);
848
397
849
397
  return CounterPtr;
850
1.26k
}
851
852
154
void InstrProfiling::emitVNodes() {
853
154
  if (!ValueProfileStaticAlloc)
854
2
    return;
855
152
856
152
  // For now only support this on platforms that do
857
152
  // not require runtime registration to discover
858
152
  // named section start/end.
859
152
  if (needsRuntimeRegistrationOfSectionRange(TT))
860
14
    return;
861
138
862
138
  size_t TotalNS = 0;
863
379
  for (auto &PD : ProfileDataMap) {
864
1.13k
    for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; 
++Kind758
)
865
758
      TotalNS += PD.second.NumValueSites[Kind];
866
379
  }
867
138
868
138
  if (!TotalNS)
869
123
    return;
870
15
871
15
  uint64_t NumCounters = TotalNS * NumCountersPerValueSite;
872
15
// Heuristic for small programs with very few total value sites.
873
15
// The default value of vp-counters-per-site is chosen based on
874
15
// the observation that large apps usually have a low percentage
875
15
// of value sites that actually have any profile data, and thus
876
15
// the average number of counters per site is low. For small
877
15
// apps with very few sites, this may not be true. Bump up the
878
15
// number of counters in this case.
879
30
#define INSTR_PROF_MIN_VAL_COUNTS 10
880
15
  if (NumCounters < INSTR_PROF_MIN_VAL_COUNTS)
881
15
    NumCounters = std::max(INSTR_PROF_MIN_VAL_COUNTS, (int)NumCounters * 2);
882
15
883
15
  auto &Ctx = M->getContext();
884
15
  Type *VNodeTypes[] = {
885
45
#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Init) LLVMType,
886
15
#include "llvm/ProfileData/InstrProfData.inc"
887
15
  };
888
15
  auto *VNodeTy = StructType::get(Ctx, makeArrayRef(VNodeTypes));
889
15
890
15
  ArrayType *VNodesTy = ArrayType::get(VNodeTy, NumCounters);
891
15
  auto *VNodesVar = new GlobalVariable(
892
15
      *M, VNodesTy, false, GlobalValue::PrivateLinkage,
893
15
      Constant::getNullValue(VNodesTy), getInstrProfVNodesVarName());
894
15
  VNodesVar->setSection(
895
15
      getInstrProfSectionName(IPSK_vnodes, TT.getObjectFormat()));
896
15
  UsedVars.push_back(VNodesVar);
897
15
}
898
899
154
void InstrProfiling::emitNameData() {
900
154
  std::string UncompressedData;
901
154
902
154
  if (ReferencedNames.empty())
903
0
    return;
904
154
905
154
  std::string CompressedNameStr;
906
154
  if (Error E = collectPGOFuncNameStrings(ReferencedNames, CompressedNameStr,
907
0
                                          DoNameCompression)) {
908
0
    report_fatal_error(toString(std::move(E)), false);
909
0
  }
910
154
911
154
  auto &Ctx = M->getContext();
912
154
  auto *NamesVal = ConstantDataArray::getString(
913
154
      Ctx, StringRef(CompressedNameStr), false);
914
154
  NamesVar = new GlobalVariable(*M, NamesVal->getType(), true,
915
154
                                GlobalValue::PrivateLinkage, NamesVal,
916
154
                                getInstrProfNamesVarName());
917
154
  NamesSize = CompressedNameStr.size();
918
154
  NamesVar->setSection(
919
154
      getInstrProfSectionName(IPSK_name, TT.getObjectFormat()));
920
154
  // On COFF, it's important to reduce the alignment down to 1 to prevent the
921
154
  // linker from inserting padding before the start of the names section or
922
154
  // between names entries.
923
154
  NamesVar->setAlignment(1);
924
154
  UsedVars.push_back(NamesVar);
925
154
926
154
  for (auto *NamePtr : ReferencedNames)
927
414
    NamePtr->eraseFromParent();
928
154
}
929
930
154
void InstrProfiling::emitRegistration() {
931
154
  if (!needsRuntimeRegistrationOfSectionRange(TT))
932
140
    return;
933
14
934
14
  // Construct the function.
935
14
  auto *VoidTy = Type::getVoidTy(M->getContext());
936
14
  auto *VoidPtrTy = Type::getInt8PtrTy(M->getContext());
937
14
  auto *Int64Ty = Type::getInt64Ty(M->getContext());
938
14
  auto *RegisterFTy = FunctionType::get(VoidTy, false);
939
14
  auto *RegisterF = Function::Create(RegisterFTy, GlobalValue::InternalLinkage,
940
14
                                     getInstrProfRegFuncsName(), M);
941
14
  RegisterF->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
942
14
  if (Options.NoRedZone)
943
0
    RegisterF->addFnAttr(Attribute::NoRedZone);
944
14
945
14
  auto *RuntimeRegisterTy = FunctionType::get(VoidTy, VoidPtrTy, false);
946
14
  auto *RuntimeRegisterF =
947
14
      Function::Create(RuntimeRegisterTy, GlobalVariable::ExternalLinkage,
948
14
                       getInstrProfRegFuncName(), M);
949
14
950
14
  IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", RegisterF));
951
14
  for (Value *Data : UsedVars)
952
42
    if (Data != NamesVar && 
!isa<Function>(Data)28
)
953
16
      IRB.CreateCall(RuntimeRegisterF, IRB.CreateBitCast(Data, VoidPtrTy));
954
14
955
14
  if (NamesVar) {
956
14
    Type *ParamTypes[] = {VoidPtrTy, Int64Ty};
957
14
    auto *NamesRegisterTy =
958
14
        FunctionType::get(VoidTy, makeArrayRef(ParamTypes), false);
959
14
    auto *NamesRegisterF =
960
14
        Function::Create(NamesRegisterTy, GlobalVariable::ExternalLinkage,
961
14
                         getInstrProfNamesRegFuncName(), M);
962
14
    IRB.CreateCall(NamesRegisterF, {IRB.CreateBitCast(NamesVar, VoidPtrTy),
963
14
                                    IRB.getInt64(NamesSize)});
964
14
  }
965
14
966
14
  IRB.CreateRetVoid();
967
14
}
968
969
180
bool InstrProfiling::emitRuntimeHook() {
970
180
  // We expect the linker to be invoked with -u<hook_var> flag for linux,
971
180
  // for which case there is no need to emit the user function.
972
180
  if (TT.isOSLinux())
973
35
    return false;
974
145
975
145
  // If the module's provided its own runtime, we don't need to do anything.
976
145
  if (M->getGlobalVariable(getInstrProfRuntimeHookVarName()))
977
3
    return false;
978
142
979
142
  // Declare an external variable that will pull in the runtime initialization.
980
142
  auto *Int32Ty = Type::getInt32Ty(M->getContext());
981
142
  auto *Var =
982
142
      new GlobalVariable(*M, Int32Ty, false, GlobalValue::ExternalLinkage,
983
142
                         nullptr, getInstrProfRuntimeHookVarName());
984
142
985
142
  // Make a function that uses it.
986
142
  auto *User = Function::Create(FunctionType::get(Int32Ty, false),
987
142
                                GlobalValue::LinkOnceODRLinkage,
988
142
                                getInstrProfRuntimeHookVarUseFuncName(), M);
989
142
  User->addFnAttr(Attribute::NoInline);
990
142
  if (Options.NoRedZone)
991
0
    User->addFnAttr(Attribute::NoRedZone);
992
142
  User->setVisibility(GlobalValue::HiddenVisibility);
993
142
  if (TT.supportsCOMDAT())
994
34
    User->setComdat(M->getOrInsertComdat(User->getName()));
995
142
996
142
  IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", User));
997
142
  auto *Load = IRB.CreateLoad(Int32Ty, Var);
998
142
  IRB.CreateRet(Load);
999
142
1000
142
  // Mark the user variable as used so that it isn't stripped out.
1001
142
  UsedVars.push_back(User);
1002
142
  return true;
1003
142
}
1004
1005
154
void InstrProfiling::emitUses() {
1006
154
  if (!UsedVars.empty())
1007
154
    appendToUsed(*M, UsedVars);
1008
154
}
1009
1010
154
void InstrProfiling::emitInitialization() {
1011
154
  // Create ProfileFileName variable. Don't don't this for the
1012
154
  // context-sensitive instrumentation lowering: This lowering is after
1013
154
  // LTO/ThinLTO linking. Pass PGOInstrumentationGenCreateVar should
1014
154
  // have already create the variable before LTO/ThinLTO linking.
1015
154
  if (!IsCS)
1016
152
    createProfileFileNameVar(*M, Options.InstrProfileOutput);
1017
154
  Function *RegisterF = M->getFunction(getInstrProfRegFuncsName());
1018
154
  if (!RegisterF)
1019
140
    return;
1020
14
1021
14
  // Create the initialization function.
1022
14
  auto *VoidTy = Type::getVoidTy(M->getContext());
1023
14
  auto *F = Function::Create(FunctionType::get(VoidTy, false),
1024
14
                             GlobalValue::InternalLinkage,
1025
14
                             getInstrProfInitFuncName(), M);
1026
14
  F->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
1027
14
  F->addFnAttr(Attribute::NoInline);
1028
14
  if (Options.NoRedZone)
1029
0
    F->addFnAttr(Attribute::NoRedZone);
1030
14
1031
14
  // Add the basic block and the necessary calls.
1032
14
  IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", F));
1033
14
  IRB.CreateCall(RegisterF, {});
1034
14
  IRB.CreateRetVoid();
1035
14
1036
14
  appendToGlobalCtors(*M, F, 0);
1037
14
}