Coverage Report

Created: 2017-10-03 07:32

/Users/buildslave/jenkins/sharedspace/clang-stage2-coverage-R@2/llvm/include/llvm/ProfileData/InstrProf.h
Line
Count
Source (jump to first uncovered line)
1
//===- InstrProf.h - Instrumented profiling format support ------*- C++ -*-===//
2
//
3
//                     The LLVM Compiler Infrastructure
4
//
5
// This file is distributed under the University of Illinois Open Source
6
// License. See LICENSE.TXT for details.
7
//
8
//===----------------------------------------------------------------------===//
9
//
10
// Instrumentation-based profiling data is generated by instrumented
11
// binaries through library functions in compiler-rt, and read by the clang
12
// frontend to feed PGO.
13
//
14
//===----------------------------------------------------------------------===//
15
16
#ifndef LLVM_PROFILEDATA_INSTRPROF_H
17
#define LLVM_PROFILEDATA_INSTRPROF_H
18
19
#include "llvm/ADT/ArrayRef.h"
20
#include "llvm/ADT/STLExtras.h"
21
#include "llvm/ADT/StringRef.h"
22
#include "llvm/ADT/StringSet.h"
23
#include "llvm/ADT/Triple.h"
24
#include "llvm/IR/GlobalValue.h"
25
#include "llvm/IR/ProfileSummary.h"
26
#include "llvm/ProfileData/InstrProfData.inc"
27
#include "llvm/Support/Compiler.h"
28
#include "llvm/Support/Endian.h"
29
#include "llvm/Support/Error.h"
30
#include "llvm/Support/ErrorHandling.h"
31
#include "llvm/Support/Host.h"
32
#include "llvm/Support/MD5.h"
33
#include "llvm/Support/MathExtras.h"
34
#include "llvm/Support/raw_ostream.h"
35
#include <algorithm>
36
#include <cassert>
37
#include <cstddef>
38
#include <cstdint>
39
#include <cstring>
40
#include <list>
41
#include <memory>
42
#include <string>
43
#include <system_error>
44
#include <utility>
45
#include <vector>
46
47
namespace llvm {
48
49
class Function;
50
class GlobalVariable;
51
struct InstrProfRecord;
52
class InstrProfSymtab;
53
class Instruction;
54
class MDNode;
55
class Module;
56
57
enum InstrProfSectKind {
58
#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) Kind,
59
#include "llvm/ProfileData/InstrProfData.inc"
60
};
61
62
/// Return the name of the profile section corresponding to \p IPSK.
63
///
64
/// The name of the section depends on the object format type \p OF. If
65
/// \p AddSegmentInfo is true, a segment prefix and additional linker hints may
66
/// be added to the section name (this is the default).
67
std::string getInstrProfSectionName(InstrProfSectKind IPSK,
68
                                    Triple::ObjectFormatType OF,
69
                                    bool AddSegmentInfo = true);
70
71
/// Return the name profile runtime entry point to do value profiling
72
/// for a given site.
73
2.44M
inline StringRef getInstrProfValueProfFuncName() {
74
2.44M
  return INSTR_PROF_VALUE_PROF_FUNC_STR;
75
2.44M
}
76
77
/// Return the name profile runtime entry point to do value range profiling.
78
2
inline StringRef getInstrProfValueRangeProfFuncName() {
79
2
  return INSTR_PROF_VALUE_RANGE_PROF_FUNC_STR;
80
2
}
81
82
/// Return the name prefix of variables containing instrumented function names.
83
1.08k
inline StringRef getInstrProfNameVarPrefix() { return "__profn_"; }
84
85
/// Return the name prefix of variables containing per-function control data.
86
339
inline StringRef getInstrProfDataVarPrefix() { return "__profd_"; }
87
88
/// Return the name prefix of profile counter variables.
89
344
inline StringRef getInstrProfCountersVarPrefix() { return "__profc_"; }
90
91
/// Return the name prefix of value profile variables.
92
16
inline StringRef getInstrProfValuesVarPrefix() { return "__profvp_"; }
93
94
/// Return the name of value profile node array variables:
95
15
inline StringRef getInstrProfVNodesVarName() { return "__llvm_prf_vnodes"; }
96
97
/// Return the name prefix of the COMDAT group for instrumentation variables
98
/// associated with a COMDAT function.
99
29
inline StringRef getInstrProfComdatPrefix() { return "__profv_"; }
100
101
/// Return the name of the variable holding the strings (possibly compressed)
102
/// of all function's PGO names.
103
139
inline StringRef getInstrProfNamesVarName() {
104
139
  return "__llvm_prf_nm";
105
139
}
106
107
/// Return the name of a covarage mapping variable (internal linkage)
108
/// for each instrumented source module. Such variables are allocated
109
/// in the __llvm_covmap section.
110
54
inline StringRef getCoverageMappingVarName() {
111
54
  return "__llvm_coverage_mapping";
112
54
}
113
114
/// Return the name of the internal variable recording the array
115
/// of PGO name vars referenced by the coverage mapping. The owning
116
/// functions of those names are not emitted by FE (e.g, unused inline
117
/// functions.)
118
149
inline StringRef getCoverageUnusedNamesVarName() {
119
149
  return "__llvm_coverage_names";
120
149
}
121
122
/// Return the name of function that registers all the per-function control
123
/// data at program startup time by calling __llvm_register_function. This
124
/// function has internal linkage and is called by  __llvm_profile_init
125
/// runtime method. This function is not generated for these platforms:
126
/// Darwin, Linux, and FreeBSD.
127
161
inline StringRef getInstrProfRegFuncsName() {
128
161
  return "__llvm_profile_register_functions";
129
161
}
130
131
/// Return the name of the runtime interface that registers per-function control
132
/// data for one instrumented function.
133
22
inline StringRef getInstrProfRegFuncName() {
134
22
  return "__llvm_profile_register_function";
135
22
}
136
137
/// Return the name of the runtime interface that registers the PGO name strings.
138
22
inline StringRef getInstrProfNamesRegFuncName() {
139
22
  return "__llvm_profile_register_names_function";
140
22
}
141
142
/// Return the name of the runtime initialization method that is generated by
143
/// the compiler. The function calls __llvm_profile_register_functions and
144
/// __llvm_profile_override_default_filename functions if needed. This function
145
/// has internal linkage and invoked at startup time via init_array.
146
22
inline StringRef getInstrProfInitFuncName() { return "__llvm_profile_init"; }
147
148
/// Return the name of the hook variable defined in profile runtime library.
149
/// A reference to the variable causes the linker to link in the runtime
150
/// initialization module (which defines the hook variable).
151
223
inline StringRef getInstrProfRuntimeHookVarName() {
152
223
  return INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_RUNTIME_VAR);
153
223
}
154
155
/// Return the name of the compiler generated function that references the
156
/// runtime hook variable. The function is a weak global.
157
108
inline StringRef getInstrProfRuntimeHookVarUseFuncName() {
158
108
  return "__llvm_profile_runtime_user";
159
108
}
160
161
/// Return the marker used to separate PGO names during serialization.
162
224
inline StringRef getInstrProfNameSeparator() { return "\01"; }
163
164
/// Return the modified name for function \c F suitable to be
165
/// used the key for profile lookup. Variable \c InLTO indicates if this
166
/// is called in LTO optimization passes.
167
std::string getPGOFuncName(const Function &F, bool InLTO = false,
168
                           uint64_t Version = INSTR_PROF_INDEX_VERSION);
169
170
/// Return the modified name for a function suitable to be
171
/// used the key for profile lookup. The function's original
172
/// name is \c RawFuncName and has linkage of type \c Linkage.
173
/// The function is defined in module \c FileName.
174
std::string getPGOFuncName(StringRef RawFuncName,
175
                           GlobalValue::LinkageTypes Linkage,
176
                           StringRef FileName,
177
                           uint64_t Version = INSTR_PROF_INDEX_VERSION);
178
179
/// Return the name of the global variable used to store a function
180
/// name in PGO instrumentation. \c FuncName is the name of the function
181
/// returned by the \c getPGOFuncName call.
182
std::string getPGOFuncNameVarName(StringRef FuncName,
183
                                  GlobalValue::LinkageTypes Linkage);
184
185
/// Create and return the global variable for function name used in PGO
186
/// instrumentation. \c FuncName is the name of the function returned
187
/// by \c getPGOFuncName call.
188
GlobalVariable *createPGOFuncNameVar(Function &F, StringRef PGOFuncName);
189
190
/// Create and return the global variable for function name used in PGO
191
/// instrumentation.  /// \c FuncName is the name of the function
192
/// returned by \c getPGOFuncName call, \c M is the owning module,
193
/// and \c Linkage is the linkage of the instrumented function.
194
GlobalVariable *createPGOFuncNameVar(Module &M,
195
                                     GlobalValue::LinkageTypes Linkage,
196
                                     StringRef PGOFuncName);
197
198
/// Return the initializer in string of the PGO name var \c NameVar.
199
StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar);
200
201
/// Given a PGO function name, remove the filename prefix and return
202
/// the original (static) function name.
203
StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName,
204
                                   StringRef FileName = "<unknown>");
205
206
/// Given a vector of strings (function PGO names) \c NameStrs, the
207
/// method generates a combined string \c Result thatis ready to be
208
/// serialized.  The \c Result string is comprised of three fields:
209
/// The first field is the legnth of the uncompressed strings, and the
210
/// the second field is the length of the zlib-compressed string.
211
/// Both fields are encoded in ULEB128.  If \c doCompress is false, the
212
///  third field is the uncompressed strings; otherwise it is the
213
/// compressed string. When the string compression is off, the
214
/// second field will have value zero.
215
Error collectPGOFuncNameStrings(ArrayRef<std::string> NameStrs,
216
                                bool doCompression, std::string &Result);
217
218
/// Produce \c Result string with the same format described above. The input
219
/// is vector of PGO function name variables that are referenced.
220
Error collectPGOFuncNameStrings(ArrayRef<GlobalVariable *> NameVars,
221
                                std::string &Result, bool doCompression = true);
222
223
/// \c NameStrings is a string composed of one of more sub-strings encoded in
224
/// the format described above. The substrings are separated by 0 or more zero
225
/// bytes. This method decodes the string and populates the \c Symtab.
226
Error readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab);
227
228
/// Check if INSTR_PROF_RAW_VERSION_VAR is defined. This global is only being
229
/// set in IR PGO compilation.
230
bool isIRPGOFlagSet(const Module *M);
231
232
/// Check if we can safely rename this Comdat function. Instances of the same
233
/// comdat function may have different control flows thus can not share the
234
/// same counter variable.
235
bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken = false);
236
237
enum InstrProfValueKind : uint32_t {
238
#define VALUE_PROF_KIND(Enumerator, Value) Enumerator = Value,
239
#include "llvm/ProfileData/InstrProfData.inc"
240
};
241
242
/// Get the value profile data for value site \p SiteIdx from \p InstrProfR
243
/// and annotate the instruction \p Inst with the value profile meta data.
244
/// Annotate up to \p MaxMDCount (default 3) number of records per value site.
245
void annotateValueSite(Module &M, Instruction &Inst,
246
                       const InstrProfRecord &InstrProfR,
247
                       InstrProfValueKind ValueKind, uint32_t SiteIndx,
248
                       uint32_t MaxMDCount = 3);
249
250
/// Same as the above interface but using an ArrayRef, as well as \p Sum.
251
void annotateValueSite(Module &M, Instruction &Inst,
252
                       ArrayRef<InstrProfValueData> VDs, uint64_t Sum,
253
                       InstrProfValueKind ValueKind, uint32_t MaxMDCount);
254
255
/// Extract the value profile data from \p Inst which is annotated with
256
/// value profile meta data. Return false if there is no value data annotated,
257
/// otherwise  return true.
258
bool getValueProfDataFromInst(const Instruction &Inst,
259
                              InstrProfValueKind ValueKind,
260
                              uint32_t MaxNumValueData,
261
                              InstrProfValueData ValueData[],
262
                              uint32_t &ActualNumValueData, uint64_t &TotalC);
263
264
856
inline StringRef getPGOFuncNameMetadataName() { return "PGOFuncName"; }
265
266
/// Return the PGOFuncName meta data associated with a function.
267
MDNode *getPGOFuncNameMetadata(const Function &F);
268
269
/// Create the PGOFuncName meta data if PGOFuncName is different from
270
/// function's raw name. This should only apply to internal linkage functions
271
/// declared by users only.
272
void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName);
273
274
/// Check if we can use Comdat for profile variables. This will eliminate
275
/// the duplicated profile variables for Comdat functions.
276
bool needsComdatForCounter(const Function &F, const Module &M);
277
278
const std::error_category &instrprof_category();
279
280
enum class instrprof_error {
281
  success = 0,
282
  eof,
283
  unrecognized_format,
284
  bad_magic,
285
  bad_header,
286
  unsupported_version,
287
  unsupported_hash_type,
288
  too_large,
289
  truncated,
290
  malformed,
291
  unknown_function,
292
  hash_mismatch,
293
  count_mismatch,
294
  counter_overflow,
295
  value_site_count_mismatch,
296
  compress_failed,
297
  uncompress_failed,
298
  empty_raw_profile,
299
  zlib_unavailable
300
};
301
302
0
inline std::error_code make_error_code(instrprof_error E) {
303
0
  return std::error_code(static_cast<int>(E), instrprof_category());
304
0
}
305
306
class InstrProfError : public ErrorInfo<InstrProfError> {
307
public:
308
367
  InstrProfError(instrprof_error Err) : Err(Err) {
309
367
    assert(Err != instrprof_error::success && "Not an error");
310
367
  }
311
312
  std::string message() const override;
313
314
0
  void log(raw_ostream &OS) const override { OS << message(); }
315
316
0
  std::error_code convertToErrorCode() const override {
317
0
    return make_error_code(Err);
318
0
  }
319
320
365
  instrprof_error get() const { return Err; }
321
322
  /// Consume an Error and return the raw enum value contained within it. The
323
  /// Error must either be a success value, or contain a single InstrProfError.
324
334
  static instrprof_error take(Error E) {
325
334
    auto Err = instrprof_error::success;
326
334
    handleAllErrors(std::move(E), [&Err](const InstrProfError &IPE) {
327
334
      assert(Err == instrprof_error::success && "Multiple errors encountered");
328
334
      Err = IPE.get();
329
334
    });
330
334
    return Err;
331
334
  }
332
333
  static char ID;
334
335
private:
336
  instrprof_error Err;
337
};
338
339
class SoftInstrProfErrors {
340
  /// Count the number of soft instrprof_errors encountered and keep track of
341
  /// the first such error for reporting purposes.
342
343
  /// The first soft error encountered.
344
  instrprof_error FirstError = instrprof_error::success;
345
346
  /// The number of hash mismatches.
347
  unsigned NumHashMismatches = 0;
348
349
  /// The number of count mismatches.
350
  unsigned NumCountMismatches = 0;
351
352
  /// The number of counter overflows.
353
  unsigned NumCounterOverflows = 0;
354
355
  /// The number of value site count mismatches.
356
  unsigned NumValueSiteCountMismatches = 0;
357
358
public:
359
  SoftInstrProfErrors() = default;
360
361
0
  ~SoftInstrProfErrors() {
362
0
    assert(FirstError == instrprof_error::success &&
363
0
           "Unchecked soft error encountered");
364
0
  }
365
366
  /// Track a soft error (\p IE) and increment its associated counter.
367
  void addError(instrprof_error IE);
368
369
  /// Get the number of hash mismatches.
370
0
  unsigned getNumHashMismatches() const { return NumHashMismatches; }
371
372
  /// Get the number of count mismatches.
373
0
  unsigned getNumCountMismatches() const { return NumCountMismatches; }
374
375
  /// Get the number of counter overflows.
376
0
  unsigned getNumCounterOverflows() const { return NumCounterOverflows; }
377
378
  /// Get the number of value site count mismatches.
379
0
  unsigned getNumValueSiteCountMismatches() const {
380
0
    return NumValueSiteCountMismatches;
381
0
  }
382
383
  /// Return the first encountered error and reset FirstError to a success
384
  /// value.
385
0
  Error takeError() {
386
0
    if (FirstError == instrprof_error::success)
387
0
      return Error::success();
388
0
    auto E = make_error<InstrProfError>(FirstError);
389
0
    FirstError = instrprof_error::success;
390
0
    return E;
391
0
  }
392
};
393
394
namespace object {
395
396
class SectionRef;
397
398
} // end namespace object
399
400
namespace IndexedInstrProf {
401
402
uint64_t ComputeHash(StringRef K);
403
404
} // end namespace IndexedInstrProf
405
406
/// A symbol table used for function PGO name look-up with keys
407
/// (such as pointers, md5hash values) to the function. A function's
408
/// PGO name or name's md5hash are used in retrieving the profile
409
/// data of the function. See \c getPGOFuncName() method for details
410
/// on how PGO name is formed.
411
class InstrProfSymtab {
412
public:
413
  using AddrHashMap = std::vector<std::pair<uint64_t, uint64_t>>;
414
415
private:
416
  StringRef Data;
417
  uint64_t Address = 0;
418
  // Unique name strings.
419
  StringSet<> NameTab;
420
  // A map from MD5 keys to function name strings.
421
  std::vector<std::pair<uint64_t, StringRef>> MD5NameMap;
422
  // A map from MD5 keys to function define. We only populate this map
423
  // when build the Symtab from a Module.
424
  std::vector<std::pair<uint64_t, Function *>> MD5FuncMap;
425
  // A map from function runtime address to function name MD5 hash.
426
  // This map is only populated and used by raw instr profile reader.
427
  AddrHashMap AddrToMD5Map;
428
429
public:
430
666
  InstrProfSymtab() = default; 
431
432
  /// Create InstrProfSymtab from an object file section which
433
  /// contains function PGO names. When section may contain raw
434
  /// string data or string data in compressed form. This method
435
  /// only initialize the symtab with reference to the data and
436
  /// the section base address. The decompression will be delayed
437
  /// until before it is used. See also \c create(StringRef) method.
438
  Error create(object::SectionRef &Section);
439
440
  /// This interface is used by reader of CoverageMapping test
441
  /// format.
442
  inline Error create(StringRef D, uint64_t BaseAddr);
443
444
  /// \c NameStrings is a string composed of one of more sub-strings
445
  ///  encoded in the format described in \c collectPGOFuncNameStrings.
446
  /// This method is a wrapper to \c readPGOFuncNameStrings method.
447
  inline Error create(StringRef NameStrings);
448
449
  /// A wrapper interface to populate the PGO symtab with functions
450
  /// decls from module \c M. This interface is used by transformation
451
  /// passes such as indirect function call promotion. Variable \c InLTO
452
  /// indicates if this is called from LTO optimization passes.
453
  Error create(Module &M, bool InLTO = false);
454
455
  /// Create InstrProfSymtab from a set of names iteratable from
456
  /// \p IterRange. This interface is used by IndexedProfReader.
457
  template <typename NameIterRange> Error create(const NameIterRange &IterRange);
458
459
  // If the symtab is created by a series of calls to \c addFuncName, \c
460
  // finalizeSymtab needs to be called before looking up function names.
461
  // This is required because the underlying map is a vector (for space
462
  // efficiency) which needs to be sorted.
463
  inline void finalizeSymtab();
464
465
  /// Update the symtab by adding \p FuncName to the table. This interface
466
  /// is used by the raw and text profile readers.
467
1.52k
  Error addFuncName(StringRef FuncName) {
468
1.52k
    if (FuncName.empty())
469
2
      return make_error<InstrProfError>(instrprof_error::malformed);
470
1.51k
    auto Ins = NameTab.insert(FuncName);
471
1.51k
    if (Ins.second)
472
1.46k
      MD5NameMap.push_back(std::make_pair(
473
1.46k
          IndexedInstrProf::ComputeHash(FuncName), Ins.first->getKey()));
474
1.51k
    return Error::success();
475
1.52k
  }
476
477
  /// Map a function address to its name's MD5 hash. This interface
478
  /// is only used by the raw profiler reader.
479
44
  void mapAddress(uint64_t Addr, uint64_t MD5Val) {
480
44
    AddrToMD5Map.push_back(std::make_pair(Addr, MD5Val));
481
44
  }
482
483
2
  AddrHashMap &getAddrHashMap() { return AddrToMD5Map; }
484
485
  /// Return function's PGO name from the function name's symbol
486
  /// address in the object file. If an error occurs, return
487
  /// an empty string.
488
  StringRef getFuncName(uint64_t FuncNameAddress, size_t NameSize);
489
490
  /// Return function's PGO name from the name's md5 hash value.
491
  /// If not found, return an empty string.
492
  inline StringRef getFuncName(uint64_t FuncMD5Hash);
493
494
  /// Return function from the name's md5 hash. Return nullptr if not found.
495
  inline Function *getFunction(uint64_t FuncMD5Hash);
496
497
  /// Return the function's original assembly name by stripping off
498
  /// the prefix attached (to symbols with priviate linkage). For
499
  /// global functions, it returns the same string as getFuncName.
500
  inline StringRef getOrigFuncName(uint64_t FuncMD5Hash);
501
502
  /// Return the name section data.
503
33
  inline StringRef getNameData() const { return Data; }
504
};
505
506
77
Error InstrProfSymtab::create(StringRef D, uint64_t BaseAddr) {
507
77
  Data = D;
508
77
  Address = BaseAddr;
509
77
  return Error::success();
510
77
}
511
512
50
Error InstrProfSymtab::create(StringRef NameStrings) {
513
50
  return readPGOFuncNameStrings(NameStrings, *this);
514
50
}
515
516
template <typename NameIterRange>
517
2
Error InstrProfSymtab::create(const NameIterRange &IterRange) {
518
2
  for (auto Name : IterRange)
519
7
    
if (Error 7
E7
= addFuncName(Name))
520
0
      return E;
521
2
522
2
  finalizeSymtab();
523
2
  return Error::success();
524
2
}
525
526
848
void InstrProfSymtab::finalizeSymtab() {
527
848
  std::sort(MD5NameMap.begin(), MD5NameMap.end(), less_first());
528
848
  std::sort(MD5FuncMap.begin(), MD5FuncMap.end(), less_first());
529
848
  std::sort(AddrToMD5Map.begin(), AddrToMD5Map.end(), less_first());
530
848
  AddrToMD5Map.erase(std::unique(AddrToMD5Map.begin(), AddrToMD5Map.end()),
531
848
                     AddrToMD5Map.end());
532
848
}
533
534
356
StringRef InstrProfSymtab::getFuncName(uint64_t FuncMD5Hash) {
535
356
  auto Result =
536
356
      std::lower_bound(MD5NameMap.begin(), MD5NameMap.end(), FuncMD5Hash,
537
356
                       [](const std::pair<uint64_t, std::string> &LHS,
538
1.14k
                          uint64_t RHS) { return LHS.first < RHS; });
539
356
  if (
Result != MD5NameMap.end() && 356
Result->first == FuncMD5Hash356
)
540
352
    return Result->second;
541
4
  return StringRef();
542
356
}
543
544
44
Function* InstrProfSymtab::getFunction(uint64_t FuncMD5Hash) {
545
44
  auto Result =
546
44
      std::lower_bound(MD5FuncMap.begin(), MD5FuncMap.end(), FuncMD5Hash,
547
44
                       [](const std::pair<uint64_t, Function*> &LHS,
548
112
                          uint64_t RHS) { return LHS.first < RHS; });
549
44
  if (
Result != MD5FuncMap.end() && 44
Result->first == FuncMD5Hash44
)
550
42
    return Result->second;
551
2
  return nullptr;
552
44
}
553
554
// See also getPGOFuncName implementation. These two need to be
555
// matched.
556
StringRef InstrProfSymtab::getOrigFuncName(uint64_t FuncMD5Hash) {
557
  StringRef PGOName = getFuncName(FuncMD5Hash);
558
  size_t S = PGOName.find_first_of(':');
559
  if (S == StringRef::npos)
560
    return PGOName;
561
  return PGOName.drop_front(S + 1);
562
}
563
564
struct InstrProfValueSiteRecord {
565
  /// Value profiling data pairs at a given value site.
566
  std::list<InstrProfValueData> ValueData;
567
568
47
  InstrProfValueSiteRecord() { ValueData.clear(); }
569
  template <class InputIterator>
570
  InstrProfValueSiteRecord(InputIterator F, InputIterator L)
571
140
      : ValueData(F, L) {}
572
573
  /// Sort ValueData ascending by Value
574
32
  void sortByTargetValues() {
575
32
    ValueData.sort(
576
4.10k
        [](const InstrProfValueData &left, const InstrProfValueData &right) {
577
4.10k
          return left.Value < right.Value;
578
4.10k
        });
579
32
  }
580
  /// Sort ValueData Descending by Count
581
  inline void sortByCount();
582
583
  /// Merge data from another InstrProfValueSiteRecord
584
  /// Optionally scale merged counts by \p Weight.
585
  void merge(InstrProfValueSiteRecord &Input, uint64_t Weight,
586
             function_ref<void(instrprof_error)> Warn);
587
  /// Scale up value profile data counts.
588
  void scale(uint64_t Weight, function_ref<void(instrprof_error)> Warn);
589
};
590
591
/// Profiling information for a single function.
592
struct InstrProfRecord {
593
  std::vector<uint64_t> Counts;
594
595
1.06k
  InstrProfRecord() = default;
596
800
  InstrProfRecord(std::vector<uint64_t> Counts) : Counts(std::move(Counts)) {}
597
913
  InstrProfRecord(InstrProfRecord &&) = default;
598
  InstrProfRecord(const InstrProfRecord &RHS)
599
      : Counts(RHS.Counts),
600
        ValueData(RHS.ValueData
601
                      ? llvm::make_unique<ValueProfData>(*RHS.ValueData)
602
449
                      : nullptr) {}
603
619
  InstrProfRecord &operator=(InstrProfRecord &&) = default;
604
134
  InstrProfRecord &operator=(const InstrProfRecord &RHS) {
605
134
    Counts = RHS.Counts;
606
134
    if (
!RHS.ValueData134
) {
607
132
      ValueData = nullptr;
608
132
      return *this;
609
132
    }
610
2
    
if (2
!ValueData2
)
611
2
      ValueData = llvm::make_unique<ValueProfData>(*RHS.ValueData);
612
2
    else
613
0
      *ValueData = *RHS.ValueData;
614
2
    return *this;
615
134
  }
616
617
  using ValueMapType = std::vector<std::pair<uint64_t, uint64_t>>;
618
619
  /// Return the number of value profile kinds with non-zero number
620
  /// of profile sites.
621
  inline uint32_t getNumValueKinds() const;
622
  /// Return the number of instrumented sites for ValueKind.
623
  inline uint32_t getNumValueSites(uint32_t ValueKind) const;
624
625
  /// Return the total number of ValueData for ValueKind.
626
  inline uint32_t getNumValueData(uint32_t ValueKind) const;
627
628
  /// Return the number of value data collected for ValueKind at profiling
629
  /// site: Site.
630
  inline uint32_t getNumValueDataForSite(uint32_t ValueKind,
631
                                         uint32_t Site) const;
632
633
  /// Return the array of profiled values at \p Site. If \p TotalC
634
  /// is not null, the total count of all target values at this site
635
  /// will be stored in \c *TotalC.
636
  inline std::unique_ptr<InstrProfValueData[]>
637
  getValueForSite(uint32_t ValueKind, uint32_t Site,
638
                  uint64_t *TotalC = nullptr) const;
639
640
  /// Get the target value/counts of kind \p ValueKind collected at site
641
  /// \p Site and store the result in array \p Dest. Return the total
642
  /// counts of all target values at this site.
643
  inline uint64_t getValueForSite(InstrProfValueData Dest[], uint32_t ValueKind,
644
                                  uint32_t Site) const;
645
646
  /// Reserve space for NumValueSites sites.
647
  inline void reserveSites(uint32_t ValueKind, uint32_t NumValueSites);
648
649
  /// Add ValueData for ValueKind at value Site.
650
  void addValueData(uint32_t ValueKind, uint32_t Site,
651
                    InstrProfValueData *VData, uint32_t N,
652
                    ValueMapType *ValueMap);
653
654
  /// Merge the counts in \p Other into this one.
655
  /// Optionally scale merged counts by \p Weight.
656
  void merge(InstrProfRecord &Other, uint64_t Weight,
657
             function_ref<void(instrprof_error)> Warn);
658
659
  /// Scale up profile counts (including value profile data) by
660
  /// \p Weight.
661
  void scale(uint64_t Weight, function_ref<void(instrprof_error)> Warn);
662
663
  /// Sort value profile data (per site) by count.
664
  void sortValueData() {
665
    for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
666
      for (auto &SR : getValueSitesForKind(Kind))
667
        SR.sortByCount();
668
  }
669
670
  /// Clear value data entries and edge counters.
671
243
  void Clear() {
672
243
    Counts.clear();
673
243
    clearValueData();
674
243
  }
675
676
  /// Clear value data entries
677
289
  void clearValueData() { ValueData = nullptr; }
678
679
private:
680
  struct ValueProfData {
681
    std::vector<InstrProfValueSiteRecord> IndirectCallSites;
682
    std::vector<InstrProfValueSiteRecord> MemOPSizes;
683
  };
684
  std::unique_ptr<ValueProfData> ValueData;
685
686
  MutableArrayRef<InstrProfValueSiteRecord>
687
893
  getValueSitesForKind(uint32_t ValueKind) {
688
893
    // Cast to /add/ const (should be an implicit_cast, ideally, if that's ever
689
893
    // implemented in LLVM) to call the const overload of this function, then
690
893
    // cast away the constness from the result.
691
893
    auto AR = const_cast<const InstrProfRecord *>(this)->getValueSitesForKind(
692
893
        ValueKind);
693
893
    return makeMutableArrayRef(
694
893
        const_cast<InstrProfValueSiteRecord *>(AR.data()), AR.size());
695
893
  }
696
  ArrayRef<InstrProfValueSiteRecord>
697
4.56k
  getValueSitesForKind(uint32_t ValueKind) const {
698
4.56k
    if (!ValueData)
699
3.85k
      return None;
700
712
    switch (ValueKind) {
701
532
    case IPVK_IndirectCallTarget:
702
532
      return ValueData->IndirectCallSites;
703
181
    case IPVK_MemOPSize:
704
181
      return ValueData->MemOPSizes;
705
0
    default:
706
0
      llvm_unreachable("Unknown value kind!");
707
4.56k
    }
708
4.56k
  }
709
710
  std::vector<InstrProfValueSiteRecord> &
711
262
  getOrCreateValueSitesForKind(uint32_t ValueKind) {
712
262
    if (!ValueData)
713
63
      ValueData = llvm::make_unique<ValueProfData>();
714
262
    switch (ValueKind) {
715
241
    case IPVK_IndirectCallTarget:
716
241
      return ValueData->IndirectCallSites;
717
21
    case IPVK_MemOPSize:
718
21
      return ValueData->MemOPSizes;
719
0
    default:
720
0
      llvm_unreachable("Unknown value kind!");
721
262
    }
722
262
  }
723
724
  // Map indirect call target name hash to name string.
725
  uint64_t remapValue(uint64_t Value, uint32_t ValueKind,
726
                      ValueMapType *HashKeys);
727
728
  // Merge Value Profile data from Src record to this record for ValueKind.
729
  // Scale merged value counts by \p Weight.
730
  void mergeValueProfData(uint32_t ValkeKind, InstrProfRecord &Src,
731
                          uint64_t Weight,
732
                          function_ref<void(instrprof_error)> Warn);
733
734
  // Scale up value profile data count.
735
  void scaleValueProfData(uint32_t ValueKind, uint64_t Weight,
736
                          function_ref<void(instrprof_error)> Warn);
737
};
738
739
struct NamedInstrProfRecord : InstrProfRecord {
740
  StringRef Name;
741
  uint64_t Hash;
742
743
572
  NamedInstrProfRecord() = default;
744
  NamedInstrProfRecord(StringRef Name, uint64_t Hash,
745
                       std::vector<uint64_t> Counts)
746
796
      : InstrProfRecord(std::move(Counts)), Name(Name), Hash(Hash) {}
747
};
748
749
375
uint32_t InstrProfRecord::getNumValueKinds() const {
750
375
  uint32_t NumValueKinds = 0;
751
1.12k
  for (uint32_t Kind = IPVK_First; 
Kind <= IPVK_Last1.12k
;
++Kind750
)
752
750
    NumValueKinds += !(getValueSitesForKind(Kind).empty());
753
375
  return NumValueKinds;
754
375
}
755
756
42
uint32_t InstrProfRecord::getNumValueData(uint32_t ValueKind) const {
757
42
  uint32_t N = 0;
758
42
  for (auto &SR : getValueSitesForKind(ValueKind))
759
122
    N += SR.ValueData.size();
760
42
  return N;
761
42
}
762
763
2.52k
uint32_t InstrProfRecord::getNumValueSites(uint32_t ValueKind) const {
764
2.52k
  return getValueSitesForKind(ValueKind).size();
765
2.52k
}
766
767
uint32_t InstrProfRecord::getNumValueDataForSite(uint32_t ValueKind,
768
225
                                                 uint32_t Site) const {
769
225
  return getValueSitesForKind(ValueKind)[Site].ValueData.size();
770
225
}
771
772
std::unique_ptr<InstrProfValueData[]>
773
InstrProfRecord::getValueForSite(uint32_t ValueKind, uint32_t Site,
774
67
                                 uint64_t *TotalC) const {
775
67
  uint64_t Dummy;
776
67
  uint64_t &TotalCount = (TotalC == nullptr ? 
Dummy53
:
*TotalC14
);
777
67
  uint32_t N = getNumValueDataForSite(ValueKind, Site);
778
67
  if (
N == 067
) {
779
8
    TotalCount = 0;
780
8
    return std::unique_ptr<InstrProfValueData[]>(nullptr);
781
8
  }
782
67
783
59
  auto VD = llvm::make_unique<InstrProfValueData[]>(N);
784
59
  TotalCount = getValueForSite(VD.get(), ValueKind, Site);
785
59
786
59
  return VD;
787
67
}
788
789
uint64_t InstrProfRecord::getValueForSite(InstrProfValueData Dest[],
790
                                          uint32_t ValueKind,
791
130
                                          uint32_t Site) const {
792
130
  uint32_t I = 0;
793
130
  uint64_t TotalCount = 0;
794
1.39k
  for (auto V : getValueSitesForKind(ValueKind)[Site].ValueData) {
795
1.39k
    Dest[I].Value = V.Value;
796
1.39k
    Dest[I].Count = V.Count;
797
1.39k
    TotalCount = SaturatingAdd(TotalCount, V.Count);
798
1.39k
    I++;
799
1.39k
  }
800
130
  return TotalCount;
801
130
}
802
803
69
void InstrProfRecord::reserveSites(uint32_t ValueKind, uint32_t NumValueSites) {
804
69
  if (!NumValueSites)
805
0
    return;
806
69
  getOrCreateValueSitesForKind(ValueKind).reserve(NumValueSites);
807
69
}
808
809
874
inline support::endianness getHostEndianness() {
810
874
  return sys::IsLittleEndianHost ? 
support::little874
:
support::big0
;
811
874
}
812
813
// Include definitions for value profile data
814
#define INSTR_PROF_VALUE_PROF_DATA
815
#include "llvm/ProfileData/InstrProfData.inc"
816
817
void InstrProfValueSiteRecord::sortByCount() {
818
  ValueData.sort(
819
      [](const InstrProfValueData &left, const InstrProfValueData &right) {
820
        return left.Count > right.Count;
821
      });
822
  // Now truncate
823
  size_t max_s = INSTR_PROF_MAX_NUM_VAL_PER_SITE;
824
  if (ValueData.size() > max_s)
825
    ValueData.resize(max_s);
826
}
827
828
namespace IndexedInstrProf {
829
830
enum class HashT : uint32_t {
831
  MD5,
832
  Last = MD5
833
};
834
835
3.00k
inline uint64_t ComputeHash(HashT Type, StringRef K) {
836
3.00k
  switch (Type) {
837
3.00k
  case HashT::MD5:
838
3.00k
    return MD5Hash(K);
839
3.00k
  }
840
0
  
llvm_unreachable0
("Unhandled hash type");
841
3.00k
}
842
843
const uint64_t Magic = 0x8169666f72706cff; // "\xfflprofi\x81"
844
845
enum ProfVersion {
846
  // Version 1 is the first version. In this version, the value of
847
  // a key/value pair can only include profile data of a single function.
848
  // Due to this restriction, the number of block counters for a given
849
  // function is not recorded but derived from the length of the value.
850
  Version1 = 1,
851
  // The version 2 format supports recording profile data of multiple
852
  // functions which share the same key in one value field. To support this,
853
  // the number block counters is recorded as an uint64_t field right after the
854
  // function structural hash.
855
  Version2 = 2,
856
  // Version 3 supports value profile data. The value profile data is expected
857
  // to follow the block counter profile data.
858
  Version3 = 3,
859
  // In this version, profile summary data \c IndexedInstrProf::Summary is
860
  // stored after the profile header.
861
  Version4 = 4,
862
  // The current version is 4.
863
  CurrentVersion = INSTR_PROF_INDEX_VERSION
864
};
865
const uint64_t Version = ProfVersion::CurrentVersion;
866
867
const HashT HashType = HashT::MD5;
868
869
2.49k
inline uint64_t ComputeHash(StringRef K) { return ComputeHash(HashType, K); }
870
871
// This structure defines the file header of the LLVM profile
872
// data file in indexed-format.
873
struct Header {
874
  uint64_t Magic;
875
  uint64_t Version;
876
  uint64_t Unused; // Becomes unused since version 4
877
  uint64_t HashType;
878
  uint64_t HashOffset;
879
};
880
881
// Profile summary data recorded in the profile data file in indexed
882
// format. It is introduced in version 4. The summary data follows
883
// right after the profile file header.
884
struct Summary {
885
  struct Entry {
886
    uint64_t Cutoff; ///< The required percentile of total execution count.
887
    uint64_t
888
        MinBlockCount;  ///< The minimum execution count for this percentile.
889
    uint64_t NumBlocks; ///< Number of blocks >= the minumum execution count.
890
  };
891
  // The field kind enumerator to assigned value mapping should remain
892
  // unchanged  when a new kind is added or an old kind gets deleted in
893
  // the future.
894
  enum SummaryFieldKind {
895
    /// The total number of functions instrumented.
896
    TotalNumFunctions = 0,
897
    /// Total number of instrumented blocks/edges.
898
    TotalNumBlocks = 1,
899
    /// The maximal execution count among all functions.
900
    /// This field does not exist for profile data from IR based
901
    /// instrumentation.
902
    MaxFunctionCount = 2,
903
    /// Max block count of the program.
904
    MaxBlockCount = 3,
905
    /// Max internal block count of the program (excluding entry blocks).
906
    MaxInternalBlockCount = 4,
907
    /// The sum of all instrumented block counts.
908
    TotalBlockCount = 5,
909
    NumKinds = TotalBlockCount + 1
910
  };
911
912
  // The number of summmary fields following the summary header.
913
  uint64_t NumSummaryFields;
914
  // The number of Cutoff Entries (Summary::Entry) following summary fields.
915
  uint64_t NumCutoffEntries;
916
917
  Summary() = delete;
918
527
  Summary(uint32_t Size) { memset(this, 0, Size); }
919
920
527
  void operator delete(void *ptr) { ::operator delete(ptr); }
921
922
527
  static uint32_t getSize(uint32_t NumSumFields, uint32_t NumCutoffEntries) {
923
527
    return sizeof(Summary) + NumCutoffEntries * sizeof(Entry) +
924
527
           NumSumFields * sizeof(uint64_t);
925
527
  }
926
927
7.70k
  const uint64_t *getSummaryDataBase() const {
928
7.70k
    return reinterpret_cast<const uint64_t *>(this + 1);
929
7.70k
  }
930
931
0
  uint64_t *getSummaryDataBase() {
932
0
    return reinterpret_cast<uint64_t *>(this + 1);
933
0
  }
934
935
5.77k
  const Entry *getCutoffEntryBase() const {
936
5.77k
    return reinterpret_cast<const Entry *>(
937
5.77k
        &getSummaryDataBase()[NumSummaryFields]);
938
5.77k
  }
939
940
0
  Entry *getCutoffEntryBase() {
941
0
    return reinterpret_cast<Entry *>(&getSummaryDataBase()[NumSummaryFields]);
942
0
  }
943
944
1.92k
  uint64_t get(SummaryFieldKind K) const {
945
1.92k
    return getSummaryDataBase()[K];
946
1.92k
  }
947
948
0
  void set(SummaryFieldKind K, uint64_t V) {
949
0
    getSummaryDataBase()[K] = V;
950
0
  }
951
952
5.77k
  const Entry &getEntry(uint32_t I) const { return getCutoffEntryBase()[I]; }
953
954
0
  void setEntry(uint32_t I, const ProfileSummaryEntry &E) {
955
0
    Entry &ER = getCutoffEntryBase()[I];
956
0
    ER.Cutoff = E.Cutoff;
957
0
    ER.MinBlockCount = E.MinCount;
958
0
    ER.NumBlocks = E.NumCounts;
959
0
  }
960
};
961
962
527
inline std::unique_ptr<Summary> allocSummary(uint32_t TotalSize) {
963
527
  return std::unique_ptr<Summary>(new (::operator new(TotalSize))
964
527
                                      Summary(TotalSize));
965
527
}
966
967
} // end namespace IndexedInstrProf
968
969
namespace RawInstrProf {
970
971
// Version 1: First version
972
// Version 2: Added value profile data section. Per-function control data
973
// struct has more fields to describe value profile information.
974
// Version 3: Compressed name section support. Function PGO name reference
975
// from control data struct is changed from raw pointer to Name's MD5 value.
976
// Version 4: ValueDataBegin and ValueDataSizes fields are removed from the
977
// raw header.
978
const uint64_t Version = INSTR_PROF_RAW_VERSION;
979
980
template <class IntPtrT> inline uint64_t getMagic();
981
281
template <> inline uint64_t getMagic<uint64_t>() {
982
281
  return INSTR_PROF_RAW_MAGIC_64;
983
281
}
984
985
258
template <> inline uint64_t getMagic<uint32_t>() {
986
258
  return INSTR_PROF_RAW_MAGIC_32;
987
258
}
988
989
// Per-function profile data header/control structure.
990
// The definition should match the structure defined in
991
// compiler-rt/lib/profile/InstrProfiling.h.
992
// It should also match the synthesized type in
993
// Transforms/Instrumentation/InstrProfiling.cpp:getOrCreateRegionCounters.
994
template <class IntPtrT> struct LLVM_ALIGNAS(8) ProfileData {
995
  #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Type Name;
996
  #include "llvm/ProfileData/InstrProfData.inc"
997
};
998
999
// File header structure of the LLVM profile data in raw format.
1000
// The definition should match the header referenced in
1001
// compiler-rt/lib/profile/InstrProfilingFile.c  and
1002
// InstrProfilingBuffer.c.
1003
struct Header {
1004
#define INSTR_PROF_RAW_HEADER(Type, Name, Init) const Type Name;
1005
#include "llvm/ProfileData/InstrProfData.inc"
1006
};
1007
1008
} // end namespace RawInstrProf
1009
1010
// Parse MemOP Size range option.
1011
void getMemOPSizeRangeFromOption(StringRef Str, int64_t &RangeStart,
1012
                                 int64_t &RangeLast);
1013
1014
} // end namespace llvm
1015
1016
#endif // LLVM_PROFILEDATA_INSTRPROF_H