Coverage Report

Created: 2017-10-03 07:32

/Users/buildslave/jenkins/sharedspace/clang-stage2-coverage-R@2/llvm/lib/Transforms/Instrumentation/EfficiencySanitizer.cpp
Line
Count
Source (jump to first uncovered line)
1
//===-- EfficiencySanitizer.cpp - performance tuner -----------------------===//
2
//
3
//                     The LLVM Compiler Infrastructure
4
//
5
// This file is distributed under the University of Illinois Open Source
6
// License. See LICENSE.TXT for details.
7
//
8
//===----------------------------------------------------------------------===//
9
//
10
// This file is a part of EfficiencySanitizer, a family of performance tuners
11
// that detects multiple performance issues via separate sub-tools.
12
//
13
// The instrumentation phase is straightforward:
14
//   - Take action on every memory access: either inlined instrumentation,
15
//     or Inserted calls to our run-time library.
16
//   - Optimizations may apply to avoid instrumenting some of the accesses.
17
//   - Turn mem{set,cpy,move} instrinsics into library calls.
18
// The rest is handled by the run-time library.
19
//===----------------------------------------------------------------------===//
20
21
#include "llvm/ADT/SmallString.h"
22
#include "llvm/ADT/SmallVector.h"
23
#include "llvm/ADT/Statistic.h"
24
#include "llvm/ADT/StringExtras.h"
25
#include "llvm/Analysis/TargetLibraryInfo.h"
26
#include "llvm/IR/Function.h"
27
#include "llvm/IR/IRBuilder.h"
28
#include "llvm/IR/IntrinsicInst.h"
29
#include "llvm/IR/Module.h"
30
#include "llvm/IR/Type.h"
31
#include "llvm/Support/CommandLine.h"
32
#include "llvm/Support/Debug.h"
33
#include "llvm/Support/raw_ostream.h"
34
#include "llvm/Transforms/Instrumentation.h"
35
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
36
#include "llvm/Transforms/Utils/Local.h"
37
#include "llvm/Transforms/Utils/ModuleUtils.h"
38
39
using namespace llvm;
40
41
#define DEBUG_TYPE "esan"
42
43
// The tool type must be just one of these ClTool* options, as the tools
44
// cannot be combined due to shadow memory constraints.
45
static cl::opt<bool>
46
    ClToolCacheFrag("esan-cache-frag", cl::init(false),
47
                    cl::desc("Detect data cache fragmentation"), cl::Hidden);
48
static cl::opt<bool>
49
    ClToolWorkingSet("esan-working-set", cl::init(false),
50
                    cl::desc("Measure the working set size"), cl::Hidden);
51
// Each new tool will get its own opt flag here.
52
// These are converted to EfficiencySanitizerOptions for use
53
// in the code.
54
55
static cl::opt<bool> ClInstrumentLoadsAndStores(
56
    "esan-instrument-loads-and-stores", cl::init(true),
57
    cl::desc("Instrument loads and stores"), cl::Hidden);
58
static cl::opt<bool> ClInstrumentMemIntrinsics(
59
    "esan-instrument-memintrinsics", cl::init(true),
60
    cl::desc("Instrument memintrinsics (memset/memcpy/memmove)"), cl::Hidden);
61
static cl::opt<bool> ClInstrumentFastpath(
62
    "esan-instrument-fastpath", cl::init(true),
63
    cl::desc("Instrument fastpath"), cl::Hidden);
64
static cl::opt<bool> ClAuxFieldInfo(
65
    "esan-aux-field-info", cl::init(true),
66
    cl::desc("Generate binary with auxiliary struct field information"),
67
    cl::Hidden);
68
69
// Experiments show that the performance difference can be 2x or more,
70
// and accuracy loss is typically negligible, so we turn this on by default.
71
static cl::opt<bool> ClAssumeIntraCacheLine(
72
    "esan-assume-intra-cache-line", cl::init(true),
73
    cl::desc("Assume each memory access touches just one cache line, for "
74
             "better performance but with a potential loss of accuracy."),
75
    cl::Hidden);
76
77
STATISTIC(NumInstrumentedLoads, "Number of instrumented loads");
78
STATISTIC(NumInstrumentedStores, "Number of instrumented stores");
79
STATISTIC(NumFastpaths, "Number of instrumented fastpaths");
80
STATISTIC(NumAccessesWithIrregularSize,
81
          "Number of accesses with a size outside our targeted callout sizes");
82
STATISTIC(NumIgnoredStructs, "Number of ignored structs");
83
STATISTIC(NumIgnoredGEPs, "Number of ignored GEP instructions");
84
STATISTIC(NumInstrumentedGEPs, "Number of instrumented GEP instructions");
85
STATISTIC(NumAssumedIntraCacheLine,
86
          "Number of accesses assumed to be intra-cache-line");
87
88
static const uint64_t EsanCtorAndDtorPriority = 0;
89
static const char *const EsanModuleCtorName = "esan.module_ctor";
90
static const char *const EsanModuleDtorName = "esan.module_dtor";
91
static const char *const EsanInitName = "__esan_init";
92
static const char *const EsanExitName = "__esan_exit";
93
94
// We need to specify the tool to the runtime earlier than
95
// the ctor is called in some cases, so we set a global variable.
96
static const char *const EsanWhichToolName = "__esan_which_tool";
97
98
// We must keep these Shadow* constants consistent with the esan runtime.
99
// FIXME: Try to place these shadow constants, the names of the __esan_*
100
// interface functions, and the ToolType enum into a header shared between
101
// llvm and compiler-rt.
102
struct ShadowMemoryParams {
103
  uint64_t ShadowMask;
104
  uint64_t ShadowOffs[3];
105
};
106
107
static const ShadowMemoryParams ShadowParams47 = {
108
    0x00000fffffffffffull,
109
    {
110
        0x0000130000000000ull, 0x0000220000000000ull, 0x0000440000000000ull,
111
    }};
112
113
static const ShadowMemoryParams ShadowParams40 = {
114
    0x0fffffffffull,
115
    {
116
        0x1300000000ull, 0x2200000000ull, 0x4400000000ull,
117
    }};
118
119
// This array is indexed by the ToolType enum.
120
static const int ShadowScale[] = {
121
  0, // ESAN_None.
122
  2, // ESAN_CacheFrag: 4B:1B, so 4 to 1 == >>2.
123
  6, // ESAN_WorkingSet: 64B:1B, so 64 to 1 == >>6.
124
};
125
126
// MaxStructCounterNameSize is a soft size limit to avoid insanely long
127
// names for those extremely large structs.
128
static const unsigned MaxStructCounterNameSize = 512;
129
130
namespace {
131
132
static EfficiencySanitizerOptions
133
19
OverrideOptionsFromCL(EfficiencySanitizerOptions Options) {
134
19
  if (ClToolCacheFrag)
135
3
    Options.ToolType = EfficiencySanitizerOptions::ESAN_CacheFrag;
136
16
  else 
if (16
ClToolWorkingSet16
)
137
3
    Options.ToolType = EfficiencySanitizerOptions::ESAN_WorkingSet;
138
19
139
19
  // Direct opt invocation with no params will have the default ESAN_None.
140
19
  // We run the default tool in that case.
141
19
  if (Options.ToolType == EfficiencySanitizerOptions::ESAN_None)
142
1
    Options.ToolType = EfficiencySanitizerOptions::ESAN_CacheFrag;
143
19
144
19
  return Options;
145
19
}
146
147
// Create a constant for Str so that we can pass it to the run-time lib.
148
static GlobalVariable *createPrivateGlobalForString(Module &M, StringRef Str,
149
46
                                                    bool AllowMerging) {
150
46
  Constant *StrConst = ConstantDataArray::getString(M.getContext(), Str);
151
46
  // We use private linkage for module-local strings. If they can be merged
152
46
  // with another one, we set the unnamed_addr attribute.
153
46
  GlobalVariable *GV =
154
46
    new GlobalVariable(M, StrConst->getType(), true,
155
46
                       GlobalValue::PrivateLinkage, StrConst, "");
156
46
  if (AllowMerging)
157
46
    GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
158
46
  GV->setAlignment(1);  // Strings may not be merged w/o setting align 1.
159
46
  return GV;
160
46
}
161
162
/// EfficiencySanitizer: instrument each module to find performance issues.
163
class EfficiencySanitizer : public ModulePass {
164
public:
165
  EfficiencySanitizer(
166
      const EfficiencySanitizerOptions &Opts = EfficiencySanitizerOptions())
167
19
      : ModulePass(ID), Options(OverrideOptionsFromCL(Opts)) {}
168
  StringRef getPassName() const override;
169
  void getAnalysisUsage(AnalysisUsage &AU) const override;
170
  bool runOnModule(Module &M) override;
171
  static char ID;
172
173
private:
174
  bool initOnModule(Module &M);
175
  void initializeCallbacks(Module &M);
176
  bool shouldIgnoreStructType(StructType *StructTy);
177
  void createStructCounterName(
178
      StructType *StructTy, SmallString<MaxStructCounterNameSize> &NameStr);
179
  void createCacheFragAuxGV(
180
    Module &M, const DataLayout &DL, StructType *StructTy,
181
    GlobalVariable *&TypeNames, GlobalVariable *&Offsets, GlobalVariable *&Size);
182
  GlobalVariable *createCacheFragInfoGV(Module &M, const DataLayout &DL,
183
                                        Constant *UnitName);
184
  Constant *createEsanInitToolInfoArg(Module &M, const DataLayout &DL);
185
  void createDestructor(Module &M, Constant *ToolInfoArg);
186
  bool runOnFunction(Function &F, Module &M);
187
  bool instrumentLoadOrStore(Instruction *I, const DataLayout &DL);
188
  bool instrumentMemIntrinsic(MemIntrinsic *MI);
189
  bool instrumentGetElementPtr(Instruction *I, Module &M);
190
  bool insertCounterUpdate(Instruction *I, StructType *StructTy,
191
                           unsigned CounterIdx);
192
32
  unsigned getFieldCounterIdx(StructType *StructTy) {
193
32
    return 0;
194
32
  }
195
23
  unsigned getArrayCounterIdx(StructType *StructTy) {
196
23
    return StructTy->getNumElements();
197
23
  }
198
43
  unsigned getStructCounterSize(StructType *StructTy) {
199
43
    // The struct counter array includes:
200
43
    // - one counter for each struct field,
201
43
    // - one counter for the struct access within an array.
202
43
    return (StructTy->getNumElements()/*field*/ + 1/*array*/);
203
43
  }
204
  bool shouldIgnoreMemoryAccess(Instruction *I);
205
  int getMemoryAccessFuncIndex(Value *Addr, const DataLayout &DL);
206
  Value *appToShadow(Value *Shadow, IRBuilder<> &IRB);
207
  bool instrumentFastpath(Instruction *I, const DataLayout &DL, bool IsStore,
208
                          Value *Addr, unsigned Alignment);
209
  // Each tool has its own fastpath routine:
210
  bool instrumentFastpathCacheFrag(Instruction *I, const DataLayout &DL,
211
                                   Value *Addr, unsigned Alignment);
212
  bool instrumentFastpathWorkingSet(Instruction *I, const DataLayout &DL,
213
                                    Value *Addr, unsigned Alignment);
214
215
  EfficiencySanitizerOptions Options;
216
  LLVMContext *Ctx;
217
  Type *IntptrTy;
218
  // Our slowpath involves callouts to the runtime library.
219
  // Access sizes are powers of two: 1, 2, 4, 8, 16.
220
  static const size_t NumberOfAccessSizes = 5;
221
  Function *EsanAlignedLoad[NumberOfAccessSizes];
222
  Function *EsanAlignedStore[NumberOfAccessSizes];
223
  Function *EsanUnalignedLoad[NumberOfAccessSizes];
224
  Function *EsanUnalignedStore[NumberOfAccessSizes];
225
  // For irregular sizes of any alignment:
226
  Function *EsanUnalignedLoadN, *EsanUnalignedStoreN;
227
  Function *MemmoveFn, *MemcpyFn, *MemsetFn;
228
  Function *EsanCtorFunction;
229
  Function *EsanDtorFunction;
230
  // Remember the counter variable for each struct type to avoid
231
  // recomputing the variable name later during instrumentation.
232
  std::map<Type *, GlobalVariable *> StructTyMap;
233
  ShadowMemoryParams ShadowParams;
234
};
235
} // namespace
236
237
char EfficiencySanitizer::ID = 0;
238
7.91k
INITIALIZE_PASS_BEGIN7.91k
(
239
7.91k
    EfficiencySanitizer, "esan",
240
7.91k
    "EfficiencySanitizer: finds performance issues.", false, false)
241
7.91k
INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
242
7.91k
INITIALIZE_PASS_END(
243
    EfficiencySanitizer, "esan",
244
    "EfficiencySanitizer: finds performance issues.", false, false)
245
246
0
StringRef EfficiencySanitizer::getPassName() const {
247
0
  return "EfficiencySanitizer";
248
0
}
249
250
19
void EfficiencySanitizer::getAnalysisUsage(AnalysisUsage &AU) const {
251
19
  AU.addRequired<TargetLibraryInfoWrapperPass>();
252
19
}
253
254
ModulePass *
255
12
llvm::createEfficiencySanitizerPass(const EfficiencySanitizerOptions &Options) {
256
12
  return new EfficiencySanitizer(Options);
257
12
}
258
259
19
void EfficiencySanitizer::initializeCallbacks(Module &M) {
260
19
  IRBuilder<> IRB(M.getContext());
261
19
  // Initialize the callbacks.
262
114
  for (size_t Idx = 0; 
Idx < NumberOfAccessSizes114
;
++Idx95
) {
263
95
    const unsigned ByteSize = 1U << Idx;
264
95
    std::string ByteSizeStr = utostr(ByteSize);
265
95
    // We'll inline the most common (i.e., aligned and frequent sizes)
266
95
    // load + store instrumentation: these callouts are for the slowpath.
267
95
    SmallString<32> AlignedLoadName("__esan_aligned_load" + ByteSizeStr);
268
95
    EsanAlignedLoad[Idx] =
269
95
        checkSanitizerInterfaceFunction(M.getOrInsertFunction(
270
95
            AlignedLoadName, IRB.getVoidTy(), IRB.getInt8PtrTy()));
271
95
    SmallString<32> AlignedStoreName("__esan_aligned_store" + ByteSizeStr);
272
95
    EsanAlignedStore[Idx] =
273
95
        checkSanitizerInterfaceFunction(M.getOrInsertFunction(
274
95
            AlignedStoreName, IRB.getVoidTy(), IRB.getInt8PtrTy()));
275
95
    SmallString<32> UnalignedLoadName("__esan_unaligned_load" + ByteSizeStr);
276
95
    EsanUnalignedLoad[Idx] =
277
95
        checkSanitizerInterfaceFunction(M.getOrInsertFunction(
278
95
            UnalignedLoadName, IRB.getVoidTy(), IRB.getInt8PtrTy()));
279
95
    SmallString<32> UnalignedStoreName("__esan_unaligned_store" + ByteSizeStr);
280
95
    EsanUnalignedStore[Idx] =
281
95
        checkSanitizerInterfaceFunction(M.getOrInsertFunction(
282
95
            UnalignedStoreName, IRB.getVoidTy(), IRB.getInt8PtrTy()));
283
95
  }
284
19
  EsanUnalignedLoadN = checkSanitizerInterfaceFunction(
285
19
      M.getOrInsertFunction("__esan_unaligned_loadN", IRB.getVoidTy(),
286
19
                            IRB.getInt8PtrTy(), IntptrTy));
287
19
  EsanUnalignedStoreN = checkSanitizerInterfaceFunction(
288
19
      M.getOrInsertFunction("__esan_unaligned_storeN", IRB.getVoidTy(),
289
19
                            IRB.getInt8PtrTy(), IntptrTy));
290
19
  MemmoveFn = checkSanitizerInterfaceFunction(
291
19
      M.getOrInsertFunction("memmove", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
292
19
                            IRB.getInt8PtrTy(), IntptrTy));
293
19
  MemcpyFn = checkSanitizerInterfaceFunction(
294
19
      M.getOrInsertFunction("memcpy", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
295
19
                            IRB.getInt8PtrTy(), IntptrTy));
296
19
  MemsetFn = checkSanitizerInterfaceFunction(
297
19
      M.getOrInsertFunction("memset", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
298
19
                            IRB.getInt32Ty(), IntptrTy));
299
19
}
300
301
47
bool EfficiencySanitizer::shouldIgnoreStructType(StructType *StructTy) {
302
47
  if (
StructTy == nullptr || 47
StructTy->isOpaque()43
/* no struct body */)
303
4
    return true;
304
43
  return false;
305
43
}
306
307
void EfficiencySanitizer::createStructCounterName(
308
12
    StructType *StructTy, SmallString<MaxStructCounterNameSize> &NameStr) {
309
12
  // Append NumFields and field type ids to avoid struct conflicts
310
12
  // with the same name but different fields.
311
12
  if (StructTy->hasName())
312
12
    NameStr += StructTy->getName();
313
12
  else
314
0
    NameStr += "struct.anon";
315
12
  // We allow the actual size of the StructCounterName to be larger than
316
12
  // MaxStructCounterNameSize and append $NumFields and at least one
317
12
  // field type id.
318
12
  // Append $NumFields.
319
12
  NameStr += "$";
320
12
  Twine(StructTy->getNumElements()).toVector(NameStr);
321
12
  // Append struct field type ids in the reverse order.
322
36
  for (int i = StructTy->getNumElements() - 1; 
i >= 036
;
--i24
) {
323
24
    NameStr += "$";
324
24
    Twine(StructTy->getElementType(i)->getTypeID()).toVector(NameStr);
325
24
    if (NameStr.size() >= MaxStructCounterNameSize)
326
0
      break;
327
24
  }
328
12
  if (
StructTy->isLiteral()12
) {
329
0
    // End with $ for literal struct.
330
0
    NameStr += "$";
331
0
  }
332
12
}
333
334
// Create global variables with auxiliary information (e.g., struct field size,
335
// offset, and type name) for better user report.
336
void EfficiencySanitizer::createCacheFragAuxGV(
337
    Module &M, const DataLayout &DL, StructType *StructTy,
338
    GlobalVariable *&TypeName, GlobalVariable *&Offset,
339
7
    GlobalVariable *&Size) {
340
7
  auto *Int8PtrTy = Type::getInt8PtrTy(*Ctx);
341
7
  auto *Int32Ty = Type::getInt32Ty(*Ctx);
342
7
  // FieldTypeName.
343
7
  auto *TypeNameArrayTy = ArrayType::get(Int8PtrTy, StructTy->getNumElements());
344
7
  TypeName = new GlobalVariable(M, TypeNameArrayTy, true,
345
7
                                 GlobalVariable::InternalLinkage, nullptr);
346
7
  SmallVector<Constant *, 16> TypeNameVec;
347
7
  // FieldOffset.
348
7
  auto *OffsetArrayTy = ArrayType::get(Int32Ty, StructTy->getNumElements());
349
7
  Offset = new GlobalVariable(M, OffsetArrayTy, true,
350
7
                              GlobalVariable::InternalLinkage, nullptr);
351
7
  SmallVector<Constant *, 16> OffsetVec;
352
7
  // FieldSize
353
7
  auto *SizeArrayTy = ArrayType::get(Int32Ty, StructTy->getNumElements());
354
7
  Size = new GlobalVariable(M, SizeArrayTy, true,
355
7
                            GlobalVariable::InternalLinkage, nullptr);
356
7
  SmallVector<Constant *, 16> SizeVec;
357
22
  for (unsigned i = 0; 
i < StructTy->getNumElements()22
;
++i15
) {
358
15
    Type *Ty = StructTy->getElementType(i);
359
15
    std::string Str;
360
15
    raw_string_ostream StrOS(Str);
361
15
    Ty->print(StrOS);
362
15
    TypeNameVec.push_back(
363
15
        ConstantExpr::getPointerCast(
364
15
            createPrivateGlobalForString(M, StrOS.str(), true),
365
15
            Int8PtrTy));
366
15
    OffsetVec.push_back(
367
15
        ConstantInt::get(Int32Ty,
368
15
                         DL.getStructLayout(StructTy)->getElementOffset(i)));
369
15
    SizeVec.push_back(ConstantInt::get(Int32Ty,
370
15
                                       DL.getTypeAllocSize(Ty)));
371
15
    }
372
7
  TypeName->setInitializer(ConstantArray::get(TypeNameArrayTy, TypeNameVec));
373
7
  Offset->setInitializer(ConstantArray::get(OffsetArrayTy, OffsetVec));
374
7
  Size->setInitializer(ConstantArray::get(SizeArrayTy, SizeVec));
375
7
}
376
377
// Create the global variable for the cache-fragmentation tool.
378
GlobalVariable *EfficiencySanitizer::createCacheFragInfoGV(
379
10
    Module &M, const DataLayout &DL, Constant *UnitName) {
380
10
  assert(Options.ToolType == EfficiencySanitizerOptions::ESAN_CacheFrag);
381
10
382
10
  auto *Int8PtrTy = Type::getInt8PtrTy(*Ctx);
383
10
  auto *Int8PtrPtrTy = Int8PtrTy->getPointerTo();
384
10
  auto *Int32Ty = Type::getInt32Ty(*Ctx);
385
10
  auto *Int32PtrTy = Type::getInt32PtrTy(*Ctx);
386
10
  auto *Int64Ty = Type::getInt64Ty(*Ctx);
387
10
  auto *Int64PtrTy = Type::getInt64PtrTy(*Ctx);
388
10
  // This structure should be kept consistent with the StructInfo struct
389
10
  // in the runtime library.
390
10
  // struct StructInfo {
391
10
  //   const char *StructName;
392
10
  //   u32 Size;
393
10
  //   u32 NumFields;
394
10
  //   u32 *FieldOffset;           // auxiliary struct field info.
395
10
  //   u32 *FieldSize;             // auxiliary struct field info.
396
10
  //   const char **FieldTypeName; // auxiliary struct field info.
397
10
  //   u64 *FieldCounters;
398
10
  //   u64 *ArrayCounter;
399
10
  // };
400
10
  auto *StructInfoTy =
401
10
      StructType::get(Int8PtrTy, Int32Ty, Int32Ty, Int32PtrTy, Int32PtrTy,
402
10
                      Int8PtrPtrTy, Int64PtrTy, Int64PtrTy);
403
10
  auto *StructInfoPtrTy = StructInfoTy->getPointerTo();
404
10
  // This structure should be kept consistent with the CacheFragInfo struct
405
10
  // in the runtime library.
406
10
  // struct CacheFragInfo {
407
10
  //   const char *UnitName;
408
10
  //   u32 NumStructs;
409
10
  //   StructInfo *Structs;
410
10
  // };
411
10
  auto *CacheFragInfoTy = StructType::get(Int8PtrTy, Int32Ty, StructInfoPtrTy);
412
10
413
10
  std::vector<StructType *> Vec = M.getIdentifiedStructTypes();
414
10
  unsigned NumStructs = 0;
415
10
  SmallVector<Constant *, 16> Initializers;
416
10
417
12
  for (auto &StructTy : Vec) {
418
12
    if (
shouldIgnoreStructType(StructTy)12
) {
419
0
      ++NumIgnoredStructs;
420
0
      continue;
421
0
    }
422
12
    ++NumStructs;
423
12
424
12
    // StructName.
425
12
    SmallString<MaxStructCounterNameSize> CounterNameStr;
426
12
    createStructCounterName(StructTy, CounterNameStr);
427
12
    GlobalVariable *StructCounterName = createPrivateGlobalForString(
428
12
        M, CounterNameStr, /*AllowMerging*/true);
429
12
430
12
    // Counters.
431
12
    // We create the counter array with StructCounterName and weak linkage
432
12
    // so that the structs with the same name and layout from different
433
12
    // compilation units will be merged into one.
434
12
    auto *CounterArrayTy = ArrayType::get(Int64Ty,
435
12
                                          getStructCounterSize(StructTy));
436
12
    GlobalVariable *Counters =
437
12
      new GlobalVariable(M, CounterArrayTy, false,
438
12
                         GlobalVariable::WeakAnyLinkage,
439
12
                         ConstantAggregateZero::get(CounterArrayTy),
440
12
                         CounterNameStr);
441
12
442
12
    // Remember the counter variable for each struct type.
443
12
    StructTyMap.insert(std::pair<Type *, GlobalVariable *>(StructTy, Counters));
444
12
445
12
    // We pass the field type name array, offset array, and size array to
446
12
    // the runtime for better reporting.
447
12
    GlobalVariable *TypeName = nullptr, *Offset = nullptr, *Size = nullptr;
448
12
    if (ClAuxFieldInfo)
449
7
      createCacheFragAuxGV(M, DL, StructTy, TypeName, Offset, Size);
450
12
451
12
    Constant *FieldCounterIdx[2];
452
12
    FieldCounterIdx[0] = ConstantInt::get(Int32Ty, 0);
453
12
    FieldCounterIdx[1] = ConstantInt::get(Int32Ty,
454
12
                                          getFieldCounterIdx(StructTy));
455
12
    Constant *ArrayCounterIdx[2];
456
12
    ArrayCounterIdx[0] = ConstantInt::get(Int32Ty, 0);
457
12
    ArrayCounterIdx[1] = ConstantInt::get(Int32Ty,
458
12
                                          getArrayCounterIdx(StructTy));
459
12
    Initializers.push_back(ConstantStruct::get(
460
12
        StructInfoTy,
461
12
        ConstantExpr::getPointerCast(StructCounterName, Int8PtrTy),
462
12
        ConstantInt::get(Int32Ty,
463
12
                         DL.getStructLayout(StructTy)->getSizeInBytes()),
464
12
        ConstantInt::get(Int32Ty, StructTy->getNumElements()),
465
5
        Offset == nullptr ? ConstantPointerNull::get(Int32PtrTy)
466
7
                          : ConstantExpr::getPointerCast(Offset, Int32PtrTy),
467
5
        Size == nullptr ? ConstantPointerNull::get(Int32PtrTy)
468
7
                        : ConstantExpr::getPointerCast(Size, Int32PtrTy),
469
12
        TypeName == nullptr
470
5
            ? ConstantPointerNull::get(Int8PtrPtrTy)
471
7
            : ConstantExpr::getPointerCast(TypeName, Int8PtrPtrTy),
472
12
        ConstantExpr::getGetElementPtr(CounterArrayTy, Counters,
473
12
                                       FieldCounterIdx),
474
12
        ConstantExpr::getGetElementPtr(CounterArrayTy, Counters,
475
12
                                       ArrayCounterIdx)));
476
12
  }
477
10
  // Structs.
478
10
  Constant *StructInfo;
479
10
  if (
NumStructs == 010
) {
480
7
    StructInfo = ConstantPointerNull::get(StructInfoPtrTy);
481
10
  } else {
482
3
    auto *StructInfoArrayTy = ArrayType::get(StructInfoTy, NumStructs);
483
3
    StructInfo = ConstantExpr::getPointerCast(
484
3
        new GlobalVariable(M, StructInfoArrayTy, false,
485
3
                           GlobalVariable::InternalLinkage,
486
3
                           ConstantArray::get(StructInfoArrayTy, Initializers)),
487
3
        StructInfoPtrTy);
488
3
  }
489
10
490
10
  auto *CacheFragInfoGV = new GlobalVariable(
491
10
      M, CacheFragInfoTy, true, GlobalVariable::InternalLinkage,
492
10
      ConstantStruct::get(CacheFragInfoTy, UnitName,
493
10
                          ConstantInt::get(Int32Ty, NumStructs), StructInfo));
494
10
  return CacheFragInfoGV;
495
10
}
496
497
// Create the tool-specific argument passed to EsanInit and EsanExit.
498
Constant *EfficiencySanitizer::createEsanInitToolInfoArg(Module &M,
499
19
                                                         const DataLayout &DL) {
500
19
  // This structure contains tool-specific information about each compilation
501
19
  // unit (module) and is passed to the runtime library.
502
19
  GlobalVariable *ToolInfoGV = nullptr;
503
19
504
19
  auto *Int8PtrTy = Type::getInt8PtrTy(*Ctx);
505
19
  // Compilation unit name.
506
19
  auto *UnitName = ConstantExpr::getPointerCast(
507
19
      createPrivateGlobalForString(M, M.getModuleIdentifier(), true),
508
19
      Int8PtrTy);
509
19
510
19
  // Create the tool-specific variable.
511
19
  if (Options.ToolType == EfficiencySanitizerOptions::ESAN_CacheFrag)
512
10
    ToolInfoGV = createCacheFragInfoGV(M, DL, UnitName);
513
19
514
19
  if (ToolInfoGV != nullptr)
515
10
    return ConstantExpr::getPointerCast(ToolInfoGV, Int8PtrTy);
516
9
517
9
  // Create the null pointer if no tool-specific variable created.
518
9
  return ConstantPointerNull::get(Int8PtrTy);
519
9
}
520
521
19
void EfficiencySanitizer::createDestructor(Module &M, Constant *ToolInfoArg) {
522
19
  PointerType *Int8PtrTy = Type::getInt8PtrTy(*Ctx);
523
19
  EsanDtorFunction = Function::Create(FunctionType::get(Type::getVoidTy(*Ctx),
524
19
                                                        false),
525
19
                                      GlobalValue::InternalLinkage,
526
19
                                      EsanModuleDtorName, &M);
527
19
  ReturnInst::Create(*Ctx, BasicBlock::Create(*Ctx, "", EsanDtorFunction));
528
19
  IRBuilder<> IRB_Dtor(EsanDtorFunction->getEntryBlock().getTerminator());
529
19
  Function *EsanExit = checkSanitizerInterfaceFunction(
530
19
      M.getOrInsertFunction(EsanExitName, IRB_Dtor.getVoidTy(),
531
19
                            Int8PtrTy));
532
19
  EsanExit->setLinkage(Function::ExternalLinkage);
533
19
  IRB_Dtor.CreateCall(EsanExit, {ToolInfoArg});
534
19
  appendToGlobalDtors(M, EsanDtorFunction, EsanCtorAndDtorPriority);
535
19
}
536
537
19
bool EfficiencySanitizer::initOnModule(Module &M) {
538
19
539
19
  Triple TargetTriple(M.getTargetTriple());
540
19
  if (
TargetTriple.getArch() == Triple::mips64 || 19
TargetTriple.getArch() == Triple::mips64el17
)
541
4
    ShadowParams = ShadowParams40;
542
19
  else
543
15
    ShadowParams = ShadowParams47;
544
19
545
19
  Ctx = &M.getContext();
546
19
  const DataLayout &DL = M.getDataLayout();
547
19
  IRBuilder<> IRB(M.getContext());
548
19
  IntegerType *OrdTy = IRB.getInt32Ty();
549
19
  PointerType *Int8PtrTy = Type::getInt8PtrTy(*Ctx);
550
19
  IntptrTy = DL.getIntPtrType(M.getContext());
551
19
  // Create the variable passed to EsanInit and EsanExit.
552
19
  Constant *ToolInfoArg = createEsanInitToolInfoArg(M, DL);
553
19
  // Constructor
554
19
  // We specify the tool type both in the EsanWhichToolName global
555
19
  // and as an arg to the init routine as a sanity check.
556
19
  std::tie(EsanCtorFunction, std::ignore) = createSanitizerCtorAndInitFunctions(
557
19
      M, EsanModuleCtorName, EsanInitName, /*InitArgTypes=*/{OrdTy, Int8PtrTy},
558
19
      /*InitArgs=*/{
559
19
        ConstantInt::get(OrdTy, static_cast<int>(Options.ToolType)),
560
19
        ToolInfoArg});
561
19
  appendToGlobalCtors(M, EsanCtorFunction, EsanCtorAndDtorPriority);
562
19
563
19
  createDestructor(M, ToolInfoArg);
564
19
565
19
  new GlobalVariable(M, OrdTy, true,
566
19
                     GlobalValue::WeakAnyLinkage,
567
19
                     ConstantInt::get(OrdTy,
568
19
                                      static_cast<int>(Options.ToolType)),
569
19
                     EsanWhichToolName);
570
19
571
19
  return true;
572
19
}
573
574
26
Value *EfficiencySanitizer::appToShadow(Value *Shadow, IRBuilder<> &IRB) {
575
26
  // Shadow = ((App & Mask) + Offs) >> Scale
576
26
  Shadow = IRB.CreateAnd(Shadow, ConstantInt::get(IntptrTy, ShadowParams.ShadowMask));
577
26
  uint64_t Offs;
578
26
  int Scale = ShadowScale[Options.ToolType];
579
26
  if (Scale <= 2)
580
0
    Offs = ShadowParams.ShadowOffs[Scale];
581
26
  else
582
26
    Offs = ShadowParams.ShadowOffs[0] << Scale;
583
26
  Shadow = IRB.CreateAdd(Shadow, ConstantInt::get(IntptrTy, Offs));
584
26
  if (Scale > 0)
585
26
    Shadow = IRB.CreateLShr(Shadow, Scale);
586
26
  return Shadow;
587
26
}
588
589
64
bool EfficiencySanitizer::shouldIgnoreMemoryAccess(Instruction *I) {
590
64
  if (
Options.ToolType == EfficiencySanitizerOptions::ESAN_CacheFrag64
) {
591
14
    // We'd like to know about cache fragmentation in vtable accesses and
592
14
    // constant data references, so we do not currently ignore anything.
593
14
    return false;
594
50
  } else 
if (50
Options.ToolType == EfficiencySanitizerOptions::ESAN_WorkingSet50
) {
595
50
    // TODO: the instrumentation disturbs the data layout on the stack, so we
596
50
    // may want to add an option to ignore stack references (if we can
597
50
    // distinguish them) to reduce overhead.
598
50
  }
599
64
  // TODO(bruening): future tools will be returning true for some cases.
600
50
  return false;
601
64
}
602
603
19
bool EfficiencySanitizer::runOnModule(Module &M) {
604
19
  bool Res = initOnModule(M);
605
19
  initializeCallbacks(M);
606
636
  for (auto &F : M) {
607
636
    Res |= runOnFunction(F, M);
608
636
  }
609
19
  return Res;
610
19
}
611
612
636
bool EfficiencySanitizer::runOnFunction(Function &F, Module &M) {
613
636
  // This is required to prevent instrumenting the call to __esan_init from
614
636
  // within the module constructor.
615
636
  if (&F == EsanCtorFunction)
616
19
    return false;
617
617
  SmallVector<Instruction *, 8> LoadsAndStores;
618
617
  SmallVector<Instruction *, 8> MemIntrinCalls;
619
617
  SmallVector<Instruction *, 8> GetElementPtrs;
620
617
  bool Res = false;
621
617
  const DataLayout &DL = M.getDataLayout();
622
617
  const TargetLibraryInfo *TLI =
623
617
      &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI();
624
617
625
85
  for (auto &BB : F) {
626
242
    for (auto &Inst : BB) {
627
242
      if (
(isa<LoadInst>(Inst) || 242
isa<StoreInst>(Inst)194
||
628
242
           
isa<AtomicRMWInst>(Inst)178
||
isa<AtomicCmpXchgInst>(Inst)178
) &&
629
64
          !shouldIgnoreMemoryAccess(&Inst))
630
64
        LoadsAndStores.push_back(&Inst);
631
178
      else 
if (178
isa<MemIntrinsic>(Inst)178
)
632
6
        MemIntrinCalls.push_back(&Inst);
633
172
      else 
if (172
isa<GetElementPtrInst>(Inst)172
)
634
33
        GetElementPtrs.push_back(&Inst);
635
139
      else 
if (CallInst *139
CI139
= dyn_cast<CallInst>(&Inst))
636
32
        maybeMarkSanitizerLibraryCallNoBuiltin(CI, TLI);
637
242
    }
638
85
  }
639
617
640
617
  if (
ClInstrumentLoadsAndStores617
) {
641
64
    for (auto Inst : LoadsAndStores) {
642
64
      Res |= instrumentLoadOrStore(Inst, DL);
643
64
    }
644
617
  }
645
617
646
617
  if (
ClInstrumentMemIntrinsics617
) {
647
6
    for (auto Inst : MemIntrinCalls) {
648
6
      Res |= instrumentMemIntrinsic(cast<MemIntrinsic>(Inst));
649
6
    }
650
617
  }
651
617
652
617
  if (
Options.ToolType == EfficiencySanitizerOptions::ESAN_CacheFrag617
) {
653
33
    for (auto Inst : GetElementPtrs) {
654
33
      Res |= instrumentGetElementPtr(Inst, M);
655
33
    }
656
297
  }
657
636
658
636
  return Res;
659
636
}
660
661
bool EfficiencySanitizer::instrumentLoadOrStore(Instruction *I,
662
64
                                                const DataLayout &DL) {
663
64
  IRBuilder<> IRB(I);
664
64
  bool IsStore;
665
64
  Value *Addr;
666
64
  unsigned Alignment;
667
64
  if (LoadInst *
Load64
= dyn_cast<LoadInst>(I)) {
668
48
    IsStore = false;
669
48
    Alignment = Load->getAlignment();
670
48
    Addr = Load->getPointerOperand();
671
64
  } else 
if (StoreInst *16
Store16
= dyn_cast<StoreInst>(I)) {
672
16
    IsStore = true;
673
16
    Alignment = Store->getAlignment();
674
16
    Addr = Store->getPointerOperand();
675
16
  } else 
if (AtomicRMWInst *0
RMW0
= dyn_cast<AtomicRMWInst>(I)) {
676
0
    IsStore = true;
677
0
    Alignment = 0;
678
0
    Addr = RMW->getPointerOperand();
679
0
  } else 
if (AtomicCmpXchgInst *0
Xchg0
= dyn_cast<AtomicCmpXchgInst>(I)) {
680
0
    IsStore = true;
681
0
    Alignment = 0;
682
0
    Addr = Xchg->getPointerOperand();
683
0
  } else
684
0
    llvm_unreachable("Unsupported mem access type");
685
64
686
64
  Type *OrigTy = cast<PointerType>(Addr->getType())->getElementType();
687
64
  const uint32_t TypeSizeBytes = DL.getTypeStoreSizeInBits(OrigTy) / 8;
688
64
  Value *OnAccessFunc = nullptr;
689
64
690
64
  // Convert 0 to the default alignment.
691
64
  if (Alignment == 0)
692
0
    Alignment = DL.getPrefTypeAlignment(OrigTy);
693
64
694
64
  if (IsStore)
695
16
    NumInstrumentedStores++;
696
64
  else
697
48
    NumInstrumentedLoads++;
698
64
  int Idx = getMemoryAccessFuncIndex(Addr, DL);
699
64
  if (
Idx < 064
) {
700
2
    OnAccessFunc = IsStore ? 
EsanUnalignedStoreN1
:
EsanUnalignedLoadN1
;
701
2
    IRB.CreateCall(OnAccessFunc,
702
2
                   {IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy()),
703
2
                    ConstantInt::get(IntptrTy, TypeSizeBytes)});
704
64
  } else {
705
62
    if (ClInstrumentFastpath &&
706
62
        
instrumentFastpath(I, DL, IsStore, Addr, Alignment)44
) {
707
40
      NumFastpaths++;
708
40
      return true;
709
40
    }
710
22
    
if (22
Alignment == 0 || 22
(Alignment % TypeSizeBytes) == 022
)
711
10
      
OnAccessFunc = IsStore ? 10
EsanAlignedStore[Idx]5
:
EsanAlignedLoad[Idx]5
;
712
22
    else
713
12
      
OnAccessFunc = IsStore ? 12
EsanUnalignedStore[Idx]4
:
EsanUnalignedLoad[Idx]8
;
714
62
    IRB.CreateCall(OnAccessFunc,
715
62
                   IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy()));
716
62
  }
717
24
  return true;
718
64
}
719
720
// It's simplest to replace the memset/memmove/memcpy intrinsics with
721
// calls that the runtime library intercepts.
722
// Our pass is late enough that calls should not turn back into intrinsics.
723
6
bool EfficiencySanitizer::instrumentMemIntrinsic(MemIntrinsic *MI) {
724
6
  IRBuilder<> IRB(MI);
725
6
  bool Res = false;
726
6
  if (
isa<MemSetInst>(MI)6
) {
727
2
    IRB.CreateCall(
728
2
        MemsetFn,
729
2
        {IRB.CreatePointerCast(MI->getArgOperand(0), IRB.getInt8PtrTy()),
730
2
         IRB.CreateIntCast(MI->getArgOperand(1), IRB.getInt32Ty(), false),
731
2
         IRB.CreateIntCast(MI->getArgOperand(2), IntptrTy, false)});
732
2
    MI->eraseFromParent();
733
2
    Res = true;
734
6
  } else 
if (4
isa<MemTransferInst>(MI)4
) {
735
4
    IRB.CreateCall(
736
4
        isa<MemCpyInst>(MI) ? 
MemcpyFn2
:
MemmoveFn2
,
737
4
        {IRB.CreatePointerCast(MI->getArgOperand(0), IRB.getInt8PtrTy()),
738
4
         IRB.CreatePointerCast(MI->getArgOperand(1), IRB.getInt8PtrTy()),
739
4
         IRB.CreateIntCast(MI->getArgOperand(2), IntptrTy, false)});
740
4
    MI->eraseFromParent();
741
4
    Res = true;
742
4
  } else
743
0
    llvm_unreachable("Unsupported mem intrinsic type");
744
6
  return Res;
745
6
}
746
747
33
bool EfficiencySanitizer::instrumentGetElementPtr(Instruction *I, Module &M) {
748
33
  GetElementPtrInst *GepInst = dyn_cast<GetElementPtrInst>(I);
749
33
  bool Res = false;
750
33
  if (
GepInst == nullptr || 33
GepInst->getNumIndices() == 133
) {
751
2
    ++NumIgnoredGEPs;
752
2
    return false;
753
2
  }
754
31
  Type *SourceTy = GepInst->getSourceElementType();
755
31
  StructType *StructTy = nullptr;
756
31
  ConstantInt *Idx;
757
31
  // Check if GEP calculates address from a struct array.
758
31
  if (
isa<StructType>(SourceTy)31
) {
759
19
    StructTy = cast<StructType>(SourceTy);
760
19
    Idx = dyn_cast<ConstantInt>(GepInst->getOperand(1));
761
19
    if (
(Idx == nullptr || 19
Idx->getSExtValue() != 019
) &&
762
19
        
!shouldIgnoreStructType(StructTy)1
&&
StructTyMap.count(StructTy) != 01
)
763
1
      Res |= insertCounterUpdate(I, StructTy, getArrayCounterIdx(StructTy));
764
19
  }
765
31
  // Iterate all (except the first and the last) idx within each GEP instruction
766
31
  // for possible nested struct field address calculation.
767
65
  for (unsigned i = 1; 
i < GepInst->getNumIndices()65
;
++i34
) {
768
34
    SmallVector<Value *, 8> IdxVec(GepInst->idx_begin(),
769
34
                                   GepInst->idx_begin() + i);
770
34
    Type *Ty = GetElementPtrInst::getIndexedType(SourceTy, IdxVec);
771
34
    unsigned CounterIdx = 0;
772
34
    if (
isa<ArrayType>(Ty)34
) {
773
14
      ArrayType *ArrayTy = cast<ArrayType>(Ty);
774
14
      StructTy = dyn_cast<StructType>(ArrayTy->getElementType());
775
14
      if (
shouldIgnoreStructType(StructTy) || 14
StructTyMap.count(StructTy) == 010
)
776
4
        continue;
777
10
      // The last counter for struct array access.
778
10
      CounterIdx = getArrayCounterIdx(StructTy);
779
34
    } else 
if (20
isa<StructType>(Ty)20
) {
780
20
      StructTy = cast<StructType>(Ty);
781
20
      if (
shouldIgnoreStructType(StructTy) || 20
StructTyMap.count(StructTy) == 020
)
782
0
        continue;
783
20
      // Get the StructTy's subfield index.
784
20
      Idx = cast<ConstantInt>(GepInst->getOperand(i+1));
785
20
      assert(Idx->getSExtValue() >= 0 &&
786
20
             Idx->getSExtValue() < StructTy->getNumElements());
787
20
      CounterIdx = getFieldCounterIdx(StructTy) + Idx->getSExtValue();
788
20
    }
789
30
    Res |= insertCounterUpdate(I, StructTy, CounterIdx);
790
30
  }
791
31
  if (Res)
792
29
    ++NumInstrumentedGEPs;
793
31
  else
794
2
    ++NumIgnoredGEPs;
795
33
  return Res;
796
33
}
797
798
bool EfficiencySanitizer::insertCounterUpdate(Instruction *I,
799
                                              StructType *StructTy,
800
31
                                              unsigned CounterIdx) {
801
31
  GlobalVariable *CounterArray = StructTyMap[StructTy];
802
31
  if (CounterArray == nullptr)
803
0
    return false;
804
31
  IRBuilder<> IRB(I);
805
31
  Constant *Indices[2];
806
31
  // Xref http://llvm.org/docs/LangRef.html#i-getelementptr and
807
31
  // http://llvm.org/docs/GetElementPtr.html.
808
31
  // The first index of the GEP instruction steps through the first operand,
809
31
  // i.e., the array itself.
810
31
  Indices[0] = ConstantInt::get(IRB.getInt32Ty(), 0);
811
31
  // The second index is the index within the array.
812
31
  Indices[1] = ConstantInt::get(IRB.getInt32Ty(), CounterIdx);
813
31
  Constant *Counter =
814
31
    ConstantExpr::getGetElementPtr(
815
31
        ArrayType::get(IRB.getInt64Ty(), getStructCounterSize(StructTy)),
816
31
        CounterArray, Indices);
817
31
  Value *Load = IRB.CreateLoad(Counter);
818
31
  IRB.CreateStore(IRB.CreateAdd(Load, ConstantInt::get(IRB.getInt64Ty(), 1)),
819
31
                  Counter);
820
31
  return true;
821
31
}
822
823
int EfficiencySanitizer::getMemoryAccessFuncIndex(Value *Addr,
824
64
                                                  const DataLayout &DL) {
825
64
  Type *OrigPtrTy = Addr->getType();
826
64
  Type *OrigTy = cast<PointerType>(OrigPtrTy)->getElementType();
827
64
  assert(OrigTy->isSized());
828
64
  // The size is always a multiple of 8.
829
64
  uint32_t TypeSizeBytes = DL.getTypeStoreSizeInBits(OrigTy) / 8;
830
64
  if (
TypeSizeBytes != 1 && 64
TypeSizeBytes != 260
&&
TypeSizeBytes != 452
&&
831
64
      
TypeSizeBytes != 832
&&
TypeSizeBytes != 1610
) {
832
2
    // Irregular sizes do not have per-size call targets.
833
2
    NumAccessesWithIrregularSize++;
834
2
    return -1;
835
2
  }
836
62
  size_t Idx = countTrailingZeros(TypeSizeBytes);
837
62
  assert(Idx < NumberOfAccessSizes);
838
62
  return Idx;
839
62
}
840
841
bool EfficiencySanitizer::instrumentFastpath(Instruction *I,
842
                                             const DataLayout &DL, bool IsStore,
843
44
                                             Value *Addr, unsigned Alignment) {
844
44
  if (
Options.ToolType == EfficiencySanitizerOptions::ESAN_CacheFrag44
) {
845
14
    return instrumentFastpathCacheFrag(I, DL, Addr, Alignment);
846
30
  } else 
if (30
Options.ToolType == EfficiencySanitizerOptions::ESAN_WorkingSet30
) {
847
30
    return instrumentFastpathWorkingSet(I, DL, Addr, Alignment);
848
30
  }
849
0
  return false;
850
0
}
851
852
bool EfficiencySanitizer::instrumentFastpathCacheFrag(Instruction *I,
853
                                                      const DataLayout &DL,
854
                                                      Value *Addr,
855
14
                                                      unsigned Alignment) {
856
14
  // Do nothing.
857
14
  return true; // Return true to avoid slowpath instrumentation.
858
14
}
859
860
bool EfficiencySanitizer::instrumentFastpathWorkingSet(
861
30
    Instruction *I, const DataLayout &DL, Value *Addr, unsigned Alignment) {
862
30
  assert(ShadowScale[Options.ToolType] == 6); // The code below assumes this
863
30
  IRBuilder<> IRB(I);
864
30
  Type *OrigTy = cast<PointerType>(Addr->getType())->getElementType();
865
30
  const uint32_t TypeSize = DL.getTypeStoreSizeInBits(OrigTy);
866
30
  // Bail to the slowpath if the access might touch multiple cache lines.
867
30
  // An access aligned to its size is guaranteed to be intra-cache-line.
868
30
  // getMemoryAccessFuncIndex has already ruled out a size larger than 16
869
30
  // and thus larger than a cache line for platforms this tool targets
870
30
  // (and our shadow memory setup assumes 64-byte cache lines).
871
30
  assert(TypeSize <= 128);
872
30
  if (!(TypeSize == 8 ||
873
30
        
(Alignment % (TypeSize / 8)) == 028
)) {
874
8
    if (ClAssumeIntraCacheLine)
875
4
      ++NumAssumedIntraCacheLine;
876
8
    else
877
4
      return false;
878
26
  }
879
26
880
26
  // We inline instrumentation to set the corresponding shadow bits for
881
26
  // each cache line touched by the application.  Here we handle a single
882
26
  // load or store where we've already ruled out the possibility that it
883
26
  // might touch more than one cache line and thus we simply update the
884
26
  // shadow memory for a single cache line.
885
26
  // Our shadow memory model is fine with races when manipulating shadow values.
886
26
  // We generate the following code:
887
26
  //
888
26
  //   const char BitMask = 0x81;
889
26
  //   char *ShadowAddr = appToShadow(AppAddr);
890
26
  //   if ((*ShadowAddr & BitMask) != BitMask)
891
26
  //     *ShadowAddr |= Bitmask;
892
26
  //
893
26
  Value *AddrPtr = IRB.CreatePointerCast(Addr, IntptrTy);
894
26
  Value *ShadowPtr = appToShadow(AddrPtr, IRB);
895
26
  Type *ShadowTy = IntegerType::get(*Ctx, 8U);
896
26
  Type *ShadowPtrTy = PointerType::get(ShadowTy, 0);
897
26
  // The bottom bit is used for the current sampling period's working set.
898
26
  // The top bit is used for the total working set.  We set both on each
899
26
  // memory access, if they are not already set.
900
26
  Value *ValueMask = ConstantInt::get(ShadowTy, 0x81); // 10000001B
901
26
902
26
  Value *OldValue = IRB.CreateLoad(IRB.CreateIntToPtr(ShadowPtr, ShadowPtrTy));
903
26
  // The AND and CMP will be turned into a TEST instruction by the compiler.
904
26
  Value *Cmp = IRB.CreateICmpNE(IRB.CreateAnd(OldValue, ValueMask), ValueMask);
905
26
  TerminatorInst *CmpTerm = SplitBlockAndInsertIfThen(Cmp, I, false);
906
26
  // FIXME: do I need to call SetCurrentDebugLocation?
907
26
  IRB.SetInsertPoint(CmpTerm);
908
26
  // We use OR to set the shadow bits to avoid corrupting the middle 6 bits,
909
26
  // which are used by the runtime library.
910
26
  Value *NewVal = IRB.CreateOr(OldValue, ValueMask);
911
26
  IRB.CreateStore(NewVal, IRB.CreateIntToPtr(ShadowPtr, ShadowPtrTy));
912
26
  IRB.SetInsertPoint(I);
913
26
914
26
  return true;
915
26
}