Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/include/llvm/ProfileData/InstrProfData.inc
Line
Count
Source (jump to first uncovered line)
1
1.17k
/*===-- InstrProfData.inc - instr profiling runtime structures -*- C++ -*-=== *\
2
1.17k
|*
3
1.17k
|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
1.17k
|* See https://llvm.org/LICENSE.txt for license information.
5
1.17k
|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
1.17k
|*
7
1.17k
\*===----------------------------------------------------------------------===*/
8
1.17k
/*
9
1.17k
 * This is the master file that defines all the data structure, signature,
10
1.17k
 * constant literals that are shared across profiling runtime library,
11
1.17k
 * compiler (instrumentation), and host tools (reader/writer). The entities
12
1.17k
 * defined in this file affect the profile runtime ABI, the raw profile format,
13
1.17k
 * or both.
14
1.17k
 *
15
1.17k
 * The file has two identical copies. The master copy lives in LLVM and
16
1.17k
 * the other one  sits in compiler-rt/lib/profile directory. To make changes
17
1.17k
 * in this file, first modify the master copy and copy it over to compiler-rt.
18
1.17k
 * Testing of any change in this file can start only after the two copies are
19
1.17k
 * synced up.
20
1.17k
 *
21
1.17k
 * The first part of the file includes macros that defines types, names, and
22
1.17k
 * initializers for the member fields of the core data structures. The field
23
1.17k
 * declarations for one structure is enabled by defining the field activation
24
1.17k
 * macro associated with that structure. Only one field activation record
25
1.17k
 * can be defined at one time and the rest definitions will be filtered out by
26
1.17k
 * the preprocessor.
27
1.17k
 *
28
1.17k
 * Examples of how the template is used to instantiate structure definition:
29
1.17k
 * 1. To declare a structure:
30
1.17k
 *
31
1.17k
 * struct ProfData {
32
1.17k
 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \
33
1.17k
 *    Type Name;
34
1.17k
 * #include "llvm/ProfileData/InstrProfData.inc"
35
1.17k
 * };
36
1.17k
 *
37
1.17k
 * 2. To construct LLVM type arrays for the struct type:
38
1.17k
 *
39
1.17k
 * Type *DataTypes[] = {
40
1.17k
 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \
41
1.17k
 *   LLVMType,
42
1.17k
 * #include "llvm/ProfileData/InstrProfData.inc"
43
1.17k
 * };
44
1.17k
 *
45
1.17k
 * 4. To construct constant array for the initializers:
46
1.17k
 * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \
47
1.17k
 *   Initializer,
48
1.17k
 * Constant *ConstantVals[] = {
49
1.17k
 * #include "llvm/ProfileData/InstrProfData.inc"
50
1.17k
 * };
51
1.17k
 *
52
1.17k
 *
53
1.17k
 * The second part of the file includes definitions all other entities that
54
1.17k
 * are related to runtime ABI and format. When no field activation macro is
55
1.17k
 * defined, this file can be included to introduce the definitions.
56
1.17k
 *
57
1.17k
\*===----------------------------------------------------------------------===*/
58
1.17k
59
1.17k
/* Functions marked with INSTR_PROF_VISIBILITY must have hidden visibility in
60
1.17k
 * the compiler runtime. */
61
#ifndef INSTR_PROF_VISIBILITY
62
#define INSTR_PROF_VISIBILITY
63
#endif
64
65
1.17k
/* INSTR_PROF_DATA start. */
66
1.17k
/* Definition of member fields of the per-function control structure. */
67
#ifndef INSTR_PROF_DATA
68
#define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer)
69
#else
70
#define INSTR_PROF_DATA_DEFINED
71
#endif
72
794
INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \
73
1.17k
                ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \
74
1.17k
                IndexedInstrProf::ComputeHash(getPGOFuncNameVarInitializer(Inc->getName()))))
75
1.17k
INSTR_PROF_DATA794
(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \
76
1.17k
                ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \
77
1.17k
                Inc->getHash()->getZExtValue()))
78
1.17k
INSTR_PROF_DATA794
(const IntPtrT, llvm::Type::getInt64PtrTy(Ctx), CounterPtr, \
79
1.17k
                ConstantExpr::getBitCast(CounterPtr, \
80
1.17k
                llvm::Type::getInt64PtrTy(Ctx)))
81
1.17k
/* This is used to map function pointers for the indirect call targets to
82
1.17k
 * function name hashes during the conversion from raw to merged profile
83
1.17k
 * data.
84
1.17k
 */
85
1.17k
INSTR_PROF_DATA794
(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), FunctionPointer, \
86
1.17k
                FunctionAddr)
87
1.17k
INSTR_PROF_DATA794
(IntPtrT, llvm::Type::getInt8PtrTy(Ctx), Values, \
88
1.17k
                ValuesPtrExpr)
89
1.17k
INSTR_PROF_DATA794
(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumCounters, \
90
1.17k
                ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumCounters))
91
1.17k
INSTR_PROF_DATA794
(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \
92
1.17k
                ConstantArray::get(Int16ArrayTy, Int16ArrayVals))
93
1.17k
#undef INSTR_PROF_DATA
94
1.17k
/* INSTR_PROF_DATA end. */
95
1.17k
96
1.17k
97
1.17k
/* This is an internal data structure used by value profiler. It
98
1.17k
 * is defined here to allow serialization code sharing by LLVM
99
1.17k
 * to be used in unit test.
100
1.17k
 *
101
1.17k
 * typedef struct ValueProfNode {
102
1.17k
 *   // InstrProfValueData VData;
103
1.17k
 *   uint64_t Value;
104
1.17k
 *   uint64_t Count;
105
1.17k
 *   struct ValueProfNode *Next;
106
1.17k
 * } ValueProfNode;
107
1.17k
 */
108
1.17k
/* INSTR_PROF_VALUE_NODE start. */
109
#ifndef INSTR_PROF_VALUE_NODE
110
#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Initializer)
111
#else
112
#define INSTR_PROF_DATA_DEFINED
113
#endif
114
15
INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Value, \
115
1.17k
                      ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0))
116
1.17k
INSTR_PROF_VALUE_NODE15
(uint64_t, llvm::Type::getInt64Ty(Ctx), Count, \
117
1.17k
                      ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0))
118
1.17k
INSTR_PROF_VALUE_NODE15
(PtrToNodeT, llvm::Type::getInt8PtrTy(Ctx), Next, \
119
1.17k
                      ConstantInt::get(llvm::Type::GetInt8PtrTy(Ctx), 0))
120
1.17k
#undef INSTR_PROF_VALUE_NODE
121
1.17k
/* INSTR_PROF_VALUE_NODE end. */
122
1.17k
123
1.17k
/* INSTR_PROF_RAW_HEADER  start */
124
1.17k
/* Definition of member fields of the raw profile header data structure. */
125
1.17k
#ifndef INSTR_PROF_RAW_HEADER
126
1.17k
#define INSTR_PROF_RAW_HEADER(Type, Name, Initializer)
127
#else
128
#define INSTR_PROF_DATA_DEFINED
129
#endif
130
INSTR_PROF_RAW_HEADER(uint64_t, Magic, __llvm_profile_get_magic())
131
1.17k
INSTR_PROF_RAW_HEADER(uint64_t, Version, __llvm_profile_get_version())
132
1.17k
INSTR_PROF_RAW_HEADER(uint64_t, DataSize, DataSize)
133
1.17k
INSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize)
134
1.17k
INSTR_PROF_RAW_HEADER(uint64_t, NamesSize,  NamesSize)
135
1.17k
INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin)
136
1.17k
INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin)
137
1.17k
INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last)
138
1.17k
#undef INSTR_PROF_RAW_HEADER
139
1.17k
/* INSTR_PROF_RAW_HEADER  end */
140
1.17k
141
1.17k
/* VALUE_PROF_FUNC_PARAM start */
142
1.17k
/* Definition of parameter types of the runtime API used to do value profiling
143
1.17k
 * for a given value site.
144
1.17k
 */
145
#ifndef VALUE_PROF_FUNC_PARAM
146
#define VALUE_PROF_FUNC_PARAM(ArgType, ArgName, ArgLLVMType)
147
#define INSTR_PROF_COMMA
148
#else
149
#define INSTR_PROF_DATA_DEFINED
150
#define INSTR_PROF_COMMA ,
151
#endif
152
18
VALUE_PROF_FUNC_PARAM(uint64_t, TargetValue, Type::getInt64Ty(Ctx)) \
153
1.17k
                      INSTR_PROF_COMMA
154
1.17k
VALUE_PROF_FUNC_PARAM18
(void *, Data, Type::getInt8PtrTy(Ctx)) INSTR_PROF_COMMA
155
#ifndef VALUE_RANGE_PROF
156
16
VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx))
157
#else /* VALUE_RANGE_PROF */
158
2
VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx)) \
159
                      INSTR_PROF_COMMA
160
2
VALUE_PROF_FUNC_PARAM(uint64_t, PreciseRangeStart, Type::getInt64Ty(Ctx)) \
161
                      INSTR_PROF_COMMA
162
2
VALUE_PROF_FUNC_PARAM(uint64_t, PreciseRangeLast, Type::getInt64Ty(Ctx)) \
163
                      INSTR_PROF_COMMA
164
2
VALUE_PROF_FUNC_PARAM(uint64_t, LargeValue, Type::getInt64Ty(Ctx))
165
#endif /*VALUE_RANGE_PROF */
166
#undef VALUE_PROF_FUNC_PARAM
167
1.17k
#undef INSTR_PROF_COMMA
168
1.17k
/* VALUE_PROF_FUNC_PARAM end */
169
1.17k
170
1.17k
/* VALUE_PROF_KIND start */
171
1.17k
#ifndef VALUE_PROF_KIND
172
1.17k
#define VALUE_PROF_KIND(Enumerator, Value, Descr)
173
#else
174
#define INSTR_PROF_DATA_DEFINED
175
#endif
176
/* For indirect function call value profiling, the addresses of the target
177
1.17k
 * functions are profiled by the instrumented code. The target addresses are
178
1.17k
 * written in the raw profile data and converted to target function name's MD5
179
1.17k
 * hash by the profile reader during deserialization.  Typically, this happens
180
1.17k
 * when the raw profile data is read during profile merging.
181
1.17k
 *
182
1.17k
 * For this remapping the ProfData is used.  ProfData contains both the function
183
1.17k
 * name hash and the function address.
184
1.17k
 */
185
1.17k
VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0, "indirect call target")
186
1.17k
/* For memory intrinsic functions size profiling. */
187
1.17k
VALUE_PROF_KIND(IPVK_MemOPSize, 1, "memory intrinsic functions size")
188
1.17k
/* These two kinds must be the last to be
189
1.17k
 * declared. This is to make sure the string
190
1.17k
 * array created with the template can be
191
1.17k
 * indexed with the kind value.
192
1.17k
 */
193
1.17k
VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget, "first")
194
1.17k
VALUE_PROF_KIND(IPVK_Last, IPVK_MemOPSize, "last")
195
1.17k
196
1.17k
#undef VALUE_PROF_KIND
197
1.17k
/* VALUE_PROF_KIND end */
198
1.17k
199
1.17k
/* COVMAP_FUNC_RECORD start */
200
1.17k
/* Definition of member fields of the function record structure in coverage
201
1.17k
 * map.
202
1.17k
 */
203
#ifndef COVMAP_FUNC_RECORD
204
#define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Initializer)
205
#else
206
#define INSTR_PROF_DATA_DEFINED
207
#endif
208
#ifdef COVMAP_V1
209
COVMAP_FUNC_RECORD(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), \
210
                   NamePtr, llvm::ConstantExpr::getBitCast(NamePtr, \
211
                   llvm::Type::getInt8PtrTy(Ctx)))
212
COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), NameSize, \
213
                   llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), \
214
                   NameValue.size()))
215
#else
216
236
COVMAP_FUNC_RECORD(const int64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \
217
1.17k
                   llvm::ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \
218
1.17k
                   llvm::IndexedInstrProf::ComputeHash(NameValue)))
219
1.17k
#endif
220
1.17k
COVMAP_FUNC_RECORD236
(const uint32_t, llvm::Type::getInt32Ty(Ctx), DataSize, \
221
1.17k
                   llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx),\
222
1.17k
                   CoverageMapping.size()))
223
1.17k
COVMAP_FUNC_RECORD236
(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \
224
1.17k
                   llvm::ConstantInt::get(llvm::Type::getInt64Ty(Ctx), FuncHash))
225
1.17k
#undef COVMAP_FUNC_RECORD
226
1.17k
/* COVMAP_FUNC_RECORD end.  */
227
1.17k
228
1.17k
/* COVMAP_HEADER start */
229
1.17k
/* Definition of member fields of coverage map header.
230
1.17k
 */
231
#ifndef COVMAP_HEADER
232
#define COVMAP_HEADER(Type, LLVMType, Name, Initializer)
233
#else
234
#define INSTR_PROF_DATA_DEFINED
235
#endif
236
116
COVMAP_HEADER(uint32_t, Int32Ty, NRecords, \
237
1.17k
              llvm::ConstantInt::get(Int32Ty,  FunctionRecords.size()))
238
1.17k
COVMAP_HEADER116
(uint32_t, Int32Ty, FilenamesSize, \
239
1.17k
              llvm::ConstantInt::get(Int32Ty, FilenamesSize))
240
1.17k
COVMAP_HEADER116
(uint32_t, Int32Ty, CoverageSize, \
241
1.17k
              llvm::ConstantInt::get(Int32Ty, CoverageMappingSize))
242
1.17k
COVMAP_HEADER116
(uint32_t, Int32Ty, Version, \
243
1.17k
              llvm::ConstantInt::get(Int32Ty, CovMapVersion::CurrentVersion))
244
1.17k
#undef COVMAP_HEADER
245
1.17k
/* COVMAP_HEADER end.  */
246
1.17k
247
1.17k
248
#ifdef INSTR_PROF_SECT_ENTRY
249
#define INSTR_PROF_DATA_DEFINED
250
INSTR_PROF_SECT_ENTRY(IPSK_data, \
251
                      INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON), \
252
                      INSTR_PROF_DATA_COFF, "__DATA,")
253
INSTR_PROF_SECT_ENTRY(IPSK_cnts, \
254
                      INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON), \
255
                      INSTR_PROF_CNTS_COFF, "__DATA,")
256
INSTR_PROF_SECT_ENTRY(IPSK_name, \
257
                      INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \
258
                      INSTR_PROF_NAME_COFF, "__DATA,")
259
INSTR_PROF_SECT_ENTRY(IPSK_vals, \
260
                      INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON), \
261
                      INSTR_PROF_VALS_COFF, "__DATA,")
262
INSTR_PROF_SECT_ENTRY(IPSK_vnodes, \
263
                      INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON), \
264
                      INSTR_PROF_VNODES_COFF, "__DATA,")
265
INSTR_PROF_SECT_ENTRY(IPSK_covmap, \
266
                      INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON), \
267
                      INSTR_PROF_COVMAP_COFF, "__LLVM_COV,")
268
INSTR_PROF_SECT_ENTRY(IPSK_orderfile, \
269
                      INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON), \
270
                      INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COFF), "__DATA,")
271
272
#undef INSTR_PROF_SECT_ENTRY
273
#endif
274
275
1.17k
276
#ifdef INSTR_PROF_VALUE_PROF_DATA
277
#define INSTR_PROF_DATA_DEFINED
278
279
#define INSTR_PROF_MAX_NUM_VAL_PER_SITE 255
280
/*!
281
 * This is the header of the data structure that defines the on-disk
282
 * layout of the value profile data of a particular kind for one function.
283
 */
284
typedef struct ValueProfRecord {
285
  /* The kind of the value profile record. */
286
  uint32_t Kind;
287
  /*
288
   * The number of value profile sites. It is guaranteed to be non-zero;
289
   * otherwise the record for this kind won't be emitted.
290
   */
291
  uint32_t NumValueSites;
292
  /*
293
   * The first element of the array that stores the number of profiled
294
   * values for each value site. The size of the array is NumValueSites.
295
   * Since NumValueSites is greater than zero, there is at least one
296
   * element in the array.
297
   */
298
  uint8_t SiteCountArray[1];
299
300
  /*
301
   * The fake declaration is for documentation purpose only.
302
   * Align the start of next field to be on 8 byte boundaries.
303
  uint8_t Padding[X];
304
   */
305
306
  /* The array of value profile data. The size of the array is the sum
307
   * of all elements in SiteCountArray[].
308
  InstrProfValueData ValueData[];
309
   */
310
311
#ifdef __cplusplus
312
  /*!
313
   * Return the number of value sites.
314
   */
315
0
  uint32_t getNumValueSites() const { return NumValueSites; }
316
  /*!
317
   * Read data from this record and save it to Record.
318
   */
319
  void deserializeTo(InstrProfRecord &Record,
320
                     InstrProfSymtab *SymTab);
321
  /*
322
   * In-place byte swap:
323
   * Do byte swap for this instance. \c Old is the original order before
324
   * the swap, and \c New is the New byte order.
325
   */
326
  void swapBytes(support::endianness Old, support::endianness New);
327
#endif
328
} ValueProfRecord;
329
330
/*!
331
 * Per-function header/control data structure for value profiling
332
 * data in indexed format.
333
 */
334
typedef struct ValueProfData {
335
  /*
336
   * Total size in bytes including this field. It must be a multiple
337
   * of sizeof(uint64_t).
338
   */
339
  uint32_t TotalSize;
340
  /*
341
   *The number of value profile kinds that has value profile data.
342
   * In this implementation, a value profile kind is considered to
343
   * have profile data if the number of value profile sites for the
344
   * kind is not zero. More aggressively, the implementation can
345
   * choose to check the actual data value: if none of the value sites
346
   * has any profiled values, the kind can be skipped.
347
   */
348
  uint32_t NumValueKinds;
349
350
  /*
351
   * Following are a sequence of variable length records. The prefix/header
352
   * of each record is defined by ValueProfRecord type. The number of
353
   * records is NumValueKinds.
354
   * ValueProfRecord Record_1;
355
   * ValueProfRecord Record_N;
356
   */
357
358
#if __cplusplus
359
  /*!
360
   * Return the total size in bytes of the on-disk value profile data
361
   * given the data stored in Record.
362
   */
363
  static uint32_t getSize(const InstrProfRecord &Record);
364
  /*!
365
   * Return a pointer to \c ValueProfData instance ready to be streamed.
366
   */
367
  static std::unique_ptr<ValueProfData>
368
  serializeFrom(const InstrProfRecord &Record);
369
  /*!
370
   * Check the integrity of the record.
371
   */
372
  Error checkIntegrity();
373
  /*!
374
   * Return a pointer to \c ValueProfileData instance ready to be read.
375
   * All data in the instance are properly byte swapped. The input
376
   * data is assumed to be in little endian order.
377
   */
378
  static Expected<std::unique_ptr<ValueProfData>>
379
  getValueProfData(const unsigned char *SrcBuffer,
380
                   const unsigned char *const SrcBufferEnd,
381
                   support::endianness SrcDataEndianness);
382
  /*!
383
   * Swap byte order from \c Endianness order to host byte order.
384
   */
385
  void swapBytesToHost(support::endianness Endianness);
386
  /*!
387
   * Swap byte order from host byte order to \c Endianness order.
388
   */
389
  void swapBytesFromHost(support::endianness Endianness);
390
  /*!
391
   * Return the total size of \c ValueProfileData.
392
   */
393
470
  uint32_t getSize() const { return TotalSize; }
394
  /*!
395
   * Read data from this data and save it to \c Record.
396
   */
397
  void deserializeTo(InstrProfRecord &Record,
398
                     InstrProfSymtab *SymTab);
399
1.48k
  void operator delete(void *ptr) { ::operator delete(ptr); }
400
#endif
401
} ValueProfData;
402
403
/*
404
 * The closure is designed to abstact away two types of value profile data:
405
 * - InstrProfRecord which is the primary data structure used to
406
 *   represent profile data in host tools (reader, writer, and profile-use)
407
 * - value profile runtime data structure suitable to be used by C
408
 *   runtime library.
409
 *
410
 * Both sources of data need to serialize to disk/memory-buffer in common
411
 * format: ValueProfData. The abstraction allows compiler-rt's raw profiler
412
 * writer to share the same format and code with indexed profile writer.
413
 *
414
 * For documentation of the member methods below, refer to corresponding methods
415
 * in class InstrProfRecord.
416
 */
417
typedef struct ValueProfRecordClosure {
418
  const void *Record;
419
  uint32_t (*GetNumValueKinds)(const void *Record);
420
  uint32_t (*GetNumValueSites)(const void *Record, uint32_t VKind);
421
  uint32_t (*GetNumValueData)(const void *Record, uint32_t VKind);
422
  uint32_t (*GetNumValueDataForSite)(const void *R, uint32_t VK, uint32_t S);
423
424
  /*
425
   * After extracting the value profile data from the value profile record,
426
   * this method is used to map the in-memory value to on-disk value. If
427
   * the method is null, value will be written out untranslated.
428
   */
429
  uint64_t (*RemapValueData)(uint32_t, uint64_t Value);
430
  void (*GetValueForSite)(const void *R, InstrProfValueData *Dst, uint32_t K,
431
                          uint32_t S);
432
  ValueProfData *(*AllocValueProfData)(size_t TotalSizeInBytes);
433
} ValueProfRecordClosure;
434
435
INSTR_PROF_VISIBILITY ValueProfRecord *
436
getFirstValueProfRecord(ValueProfData *VPD);
437
INSTR_PROF_VISIBILITY ValueProfRecord *
438
getValueProfRecordNext(ValueProfRecord *VPR);
439
INSTR_PROF_VISIBILITY InstrProfValueData *
440
getValueProfRecordValueData(ValueProfRecord *VPR);
441
INSTR_PROF_VISIBILITY uint32_t
442
getValueProfRecordHeaderSize(uint32_t NumValueSites);
443
444
#undef INSTR_PROF_VALUE_PROF_DATA
445
#endif  /* INSTR_PROF_VALUE_PROF_DATA */
446
447
1.17k
448
#ifdef INSTR_PROF_COMMON_API_IMPL
449
#define INSTR_PROF_DATA_DEFINED
450
#ifdef __cplusplus
451
#define INSTR_PROF_INLINE inline
452
#define INSTR_PROF_NULLPTR nullptr
453
#else
454
#define INSTR_PROF_INLINE
455
#define INSTR_PROF_NULLPTR NULL
456
#endif
457
458
#ifndef offsetof
459
#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
460
#endif
461
462
/*!
463
 * Return the \c ValueProfRecord header size including the
464
 * padding bytes.
465
 */
466
INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
467
213
uint32_t getValueProfRecordHeaderSize(uint32_t NumValueSites) {
468
213
  uint32_t Size = offsetof(ValueProfRecord, SiteCountArray) +
469
213
                  sizeof(uint8_t) * NumValueSites;
470
213
  /* Round the size to multiple of 8 bytes. */
471
213
  Size = (Size + 7) & ~7;
472
213
  return Size;
473
213
}
474
475
/*!
476
 * Return the total size of the value profile record including the
477
 * header and the value data.
478
 */
479
INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
480
uint32_t getValueProfRecordSize(uint32_t NumValueSites,
481
147
                                uint32_t NumValueData) {
482
147
  return getValueProfRecordHeaderSize(NumValueSites) +
483
147
         sizeof(InstrProfValueData) * NumValueData;
484
147
}
485
486
/*!
487
 * Return the pointer to the start of value data array.
488
 */
489
INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
490
66
InstrProfValueData *getValueProfRecordValueData(ValueProfRecord *This) {
491
66
  return (InstrProfValueData *)((char *)This + getValueProfRecordHeaderSize(
492
66
                                                   This->NumValueSites));
493
66
}
494
495
/*!
496
 * Return the total number of value data for \c This record.
497
 */
498
INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
499
101
uint32_t getValueProfRecordNumValueData(ValueProfRecord *This) {
500
101
  uint32_t NumValueData = 0;
501
101
  uint32_t I;
502
354
  for (I = 0; I < This->NumValueSites; 
I++253
)
503
253
    NumValueData += This->SiteCountArray[I];
504
101
  return NumValueData;
505
101
}
506
507
/*!
508
 * Use this method to advance to the next \c This \c ValueProfRecord.
509
 */
510
INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
511
97
ValueProfRecord *getValueProfRecordNext(ValueProfRecord *This) {
512
97
  uint32_t NumValueData = getValueProfRecordNumValueData(This);
513
97
  return (ValueProfRecord *)((char *)This +
514
97
                             getValueProfRecordSize(This->NumValueSites,
515
97
                                                    NumValueData));
516
97
}
517
518
/*!
519
 * Return the first \c ValueProfRecord instance.
520
 */
521
INSTR_PROF_VISIBILITY INSTR_PROF_INLINE
522
1.52k
ValueProfRecord *getFirstValueProfRecord(ValueProfData *This) {
523
1.52k
  return (ValueProfRecord *)((char *)This + sizeof(ValueProfData));
524
1.52k
}
525
526
/* Closure based interfaces.  */
527
528
/*!
529
 * Return the total size in bytes of the on-disk value profile data
530
 * given the data stored in Record.
531
 */
532
INSTR_PROF_VISIBILITY uint32_t
533
944
getValueProfDataSize(ValueProfRecordClosure *Closure) {
534
944
  uint32_t Kind;
535
944
  uint32_t TotalSize = sizeof(ValueProfData);
536
944
  const void *Record = Closure->Record;
537
944
538
2.83k
  for (Kind = IPVK_First; Kind <= IPVK_Last; 
Kind++1.88k
) {
539
1.88k
    uint32_t NumValueSites = Closure->GetNumValueSites(Record, Kind);
540
1.88k
    if (!NumValueSites)
541
1.83k
      continue;
542
50
    TotalSize += getValueProfRecordSize(NumValueSites,
543
50
                                        Closure->GetNumValueData(Record, Kind));
544
50
  }
545
944
  return TotalSize;
546
944
}
547
548
/*!
549
 * Extract value profile data of a function for the profile kind \c ValueKind
550
 * from the \c Closure and serialize the data into \c This record instance.
551
 */
552
INSTR_PROF_VISIBILITY void
553
serializeValueProfRecordFrom(ValueProfRecord *This,
554
                             ValueProfRecordClosure *Closure,
555
27
                             uint32_t ValueKind, uint32_t NumValueSites) {
556
27
  uint32_t S;
557
27
  const void *Record = Closure->Record;
558
27
  This->Kind = ValueKind;
559
27
  This->NumValueSites = NumValueSites;
560
27
  InstrProfValueData *DstVD = getValueProfRecordValueData(This);
561
27
562
102
  for (S = 0; S < NumValueSites; 
S++75
) {
563
75
    uint32_t ND = Closure->GetNumValueDataForSite(Record, ValueKind, S);
564
75
    This->SiteCountArray[S] = ND;
565
75
    Closure->GetValueForSite(Record, DstVD, ValueKind, S);
566
75
    DstVD += ND;
567
75
  }
568
27
}
569
570
/*!
571
 * Extract value profile data of a function  from the \c Closure
572
 * and serialize the data into \c DstData if it is not NULL or heap
573
 * memory allocated by the \c Closure's allocator method. If \c
574
 * DstData is not null, the caller is expected to set the TotalSize
575
 * in DstData.
576
 */
577
INSTR_PROF_VISIBILITY ValueProfData *
578
serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
579
474
                           ValueProfData *DstData) {
580
474
  uint32_t Kind;
581
474
  uint32_t TotalSize =
582
474
      DstData ? 
DstData->TotalSize0
: getValueProfDataSize(Closure);
583
474
584
474
  ValueProfData *VPD =
585
474
      DstData ? 
DstData0
: Closure->AllocValueProfData(TotalSize);
586
474
587
474
  VPD->TotalSize = TotalSize;
588
474
  VPD->NumValueKinds = Closure->GetNumValueKinds(Closure->Record);
589
474
  ValueProfRecord *VR = getFirstValueProfRecord(VPD);
590
1.42k
  for (Kind = IPVK_First; Kind <= IPVK_Last; 
Kind++948
) {
591
948
    uint32_t NumValueSites = Closure->GetNumValueSites(Closure->Record, Kind);
592
948
    if (!NumValueSites)
593
921
      continue;
594
27
    serializeValueProfRecordFrom(VR, Closure, Kind, NumValueSites);
595
27
    VR = getValueProfRecordNext(VR);
596
27
  }
597
474
  return VPD;
598
474
}
599
600
#undef INSTR_PROF_COMMON_API_IMPL
601
#endif /* INSTR_PROF_COMMON_API_IMPL */
602
603
1.17k
/*============================================================================*/
604
1.17k
605
#ifndef INSTR_PROF_DATA_DEFINED
606
607
#ifndef INSTR_PROF_DATA_INC
608
#define INSTR_PROF_DATA_INC
609
610
/* Helper macros.  */
611
1.70M
#define INSTR_PROF_SIMPLE_QUOTE(x) #x
612
1.70M
#define INSTR_PROF_QUOTE(x) INSTR_PROF_SIMPLE_QUOTE(x)
613
#define INSTR_PROF_SIMPLE_CONCAT(x,y) x ## y
614
#define INSTR_PROF_CONCAT(x,y) INSTR_PROF_SIMPLE_CONCAT(x,y)
615
616
/* Magic number to detect file format and endianness.
617
 * Use 255 at one end, since no UTF-8 file can use that character.  Avoid 0,
618
 * so that utilities, like strings, don't grab it as a string.  129 is also
619
 * invalid UTF-8, and high enough to be interesting.
620
 * Use "lprofr" in the centre to stand for "LLVM Profile Raw", or "lprofR"
621
 * for 32-bit platforms.
622
 */
623
435
#define INSTR_PROF_RAW_MAGIC_64 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \
624
435
       (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 |  \
625
435
        (uint64_t)'f' << 16 | (uint64_t)'r' << 8 | (uint64_t)129
626
412
#define INSTR_PROF_RAW_MAGIC_32 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \
627
412
       (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 |  \
628
412
        (uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129
629
630
/* Raw profile format version (start from 1). */
631
78
#define INSTR_PROF_RAW_VERSION 4
632
/* Indexed profile format version (start from 1). */
633
#define INSTR_PROF_INDEX_VERSION 5
634
/* Coverage mapping format vresion (start from 0). */
635
#define INSTR_PROF_COVMAP_VERSION 2
636
637
/* Profile version is always of type uint64_t. Reserve the upper 8 bits in the
638
 * version for other variants of profile. We set the lowest bit of the upper 8
639
 * bits (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentaiton
640
 * generated profile, and 0 if this is a Clang FE generated profile.
641
 * 1 in bit 57 indicates there are context-sensitive records in the profile.
642
 */
643
3.09k
#define VARIANT_MASKS_ALL 0xff00000000000000ULL
644
3.09k
#define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL)
645
636
#define VARIANT_MASK_IR_PROF (0x1ULL << 56)
646
685
#define VARIANT_MASK_CSIR_PROF (0x1ULL << 57)
647
#define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version
648
#define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime
649
650
/* The variable that holds the name of the profile data
651
 * specified via command line. */
652
#define INSTR_PROF_PROFILE_NAME_VAR __llvm_profile_filename
653
654
/* section name strings common to all targets other
655
   than WIN32 */
656
#define INSTR_PROF_DATA_COMMON __llvm_prf_data
657
#define INSTR_PROF_NAME_COMMON __llvm_prf_names
658
#define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts
659
#define INSTR_PROF_VALS_COMMON __llvm_prf_vals
660
#define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds
661
#define INSTR_PROF_COVMAP_COMMON __llvm_covmap
662
#define INSTR_PROF_ORDERFILE_COMMON __llvm_orderfile
663
/* Windows section names. Because these section names contain dollar characters,
664
 * they must be quoted.
665
 */
666
#define INSTR_PROF_DATA_COFF ".lprfd$M"
667
#define INSTR_PROF_NAME_COFF ".lprfn$M"
668
#define INSTR_PROF_CNTS_COFF ".lprfc$M"
669
#define INSTR_PROF_VALS_COFF ".lprfv$M"
670
#define INSTR_PROF_VNODES_COFF ".lprfnd$M"
671
#define INSTR_PROF_COVMAP_COFF ".lcovmap$M"
672
#define INSTR_PROF_ORDERFILE_COFF ".lorderfile$M"
673
674
#ifdef _WIN32
675
/* Runtime section names and name strings.  */
676
#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_DATA_COFF
677
#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_NAME_COFF
678
#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_CNTS_COFF
679
/* Array of pointers. Each pointer points to a list
680
 * of value nodes associated with one value site.
681
 */
682
#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_VALS_COFF
683
/* Value profile nodes section. */
684
#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_VNODES_COFF
685
#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COFF
686
#define INSTR_PROF_ORDERFILE_SECT_NAME INSTR_PROF_ORDERFILE_COFF
687
#else
688
/* Runtime section names and name strings.  */
689
#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON)
690
#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON)
691
#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON)
692
/* Array of pointers. Each pointer points to a list
693
 * of value nodes associated with one value site.
694
 */
695
#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON)
696
/* Value profile nodes section. */
697
#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON)
698
#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON)
699
/* Order file instrumentation. */
700
#define INSTR_PROF_ORDERFILE_SECT_NAME                                         \
701
  INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON)
702
#endif
703
704
#define INSTR_PROF_ORDERFILE_BUFFER_NAME _llvm_order_file_buffer
705
#define INSTR_PROF_ORDERFILE_BUFFER_NAME_STR                                   \
706
2
  INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_NAME)
707
#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME _llvm_order_file_buffer_idx
708
#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME_STR                               \
709
2
  INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME)
710
711
/* Macros to define start/stop section symbol for a given
712
 * section on Linux. For instance
713
 * INSTR_PROF_SECT_START(INSTR_PROF_DATA_SECT_NAME) will
714
 * expand to __start___llvm_prof_data
715
 */
716
#define INSTR_PROF_SECT_START(Sect) \
717
        INSTR_PROF_CONCAT(__start_,Sect)
718
#define INSTR_PROF_SECT_STOP(Sect) \
719
        INSTR_PROF_CONCAT(__stop_,Sect)
720
721
/* Value Profiling API linkage name.  */
722
#define INSTR_PROF_VALUE_PROF_FUNC __llvm_profile_instrument_target
723
#define INSTR_PROF_VALUE_PROF_FUNC_STR \
724
1.70M
        INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_FUNC)
725
#define INSTR_PROF_VALUE_RANGE_PROF_FUNC __llvm_profile_instrument_range
726
#define INSTR_PROF_VALUE_RANGE_PROF_FUNC_STR \
727
2
        INSTR_PROF_QUOTE(INSTR_PROF_VALUE_RANGE_PROF_FUNC)
728
729
/* InstrProfile per-function control data alignment.  */
730
397
#define INSTR_PROF_DATA_ALIGNMENT 8
731
732
/* The data structure that represents a tracked value by the
733
 * value profiler.
734
 */
735
typedef struct InstrProfValueData {
736
  /* Profiled value. */
737
  uint64_t Value;
738
  /* Number of times the value appears in the training run. */
739
  uint64_t Count;
740
} InstrProfValueData;
741
742
#endif /* INSTR_PROF_DATA_INC */
743
744
#ifndef INSTR_ORDER_FILE_INC
745
// The maximal # of functions: 128*1024 (the buffer size will be 128*4 KB).
746
2
#define INSTR_ORDER_FILE_BUFFER_SIZE 131072
747
#define INSTR_ORDER_FILE_BUFFER_BITS 17
748
2
#define INSTR_ORDER_FILE_BUFFER_MASK 0x1ffff
749
#endif /* INSTR_ORDER_FILE_INC */
750
#else
751
#undef INSTR_PROF_DATA_DEFINED
752
1.17k
#endif