/Users/buildslave/jenkins/sharedspace/clang-stage2-coverage-R@2/llvm/lib/Transforms/Instrumentation/EfficiencySanitizer.cpp
Line | Count | Source (jump to first uncovered line) |
1 | | //===-- EfficiencySanitizer.cpp - performance tuner -----------------------===// |
2 | | // |
3 | | // The LLVM Compiler Infrastructure |
4 | | // |
5 | | // This file is distributed under the University of Illinois Open Source |
6 | | // License. See LICENSE.TXT for details. |
7 | | // |
8 | | //===----------------------------------------------------------------------===// |
9 | | // |
10 | | // This file is a part of EfficiencySanitizer, a family of performance tuners |
11 | | // that detects multiple performance issues via separate sub-tools. |
12 | | // |
13 | | // The instrumentation phase is straightforward: |
14 | | // - Take action on every memory access: either inlined instrumentation, |
15 | | // or Inserted calls to our run-time library. |
16 | | // - Optimizations may apply to avoid instrumenting some of the accesses. |
17 | | // - Turn mem{set,cpy,move} instrinsics into library calls. |
18 | | // The rest is handled by the run-time library. |
19 | | //===----------------------------------------------------------------------===// |
20 | | |
21 | | #include "llvm/ADT/SmallString.h" |
22 | | #include "llvm/ADT/SmallVector.h" |
23 | | #include "llvm/ADT/Statistic.h" |
24 | | #include "llvm/ADT/StringExtras.h" |
25 | | #include "llvm/Analysis/TargetLibraryInfo.h" |
26 | | #include "llvm/IR/Function.h" |
27 | | #include "llvm/IR/IRBuilder.h" |
28 | | #include "llvm/IR/IntrinsicInst.h" |
29 | | #include "llvm/IR/Module.h" |
30 | | #include "llvm/IR/Type.h" |
31 | | #include "llvm/Support/CommandLine.h" |
32 | | #include "llvm/Support/Debug.h" |
33 | | #include "llvm/Support/raw_ostream.h" |
34 | | #include "llvm/Transforms/Instrumentation.h" |
35 | | #include "llvm/Transforms/Utils/BasicBlockUtils.h" |
36 | | #include "llvm/Transforms/Utils/Local.h" |
37 | | #include "llvm/Transforms/Utils/ModuleUtils.h" |
38 | | |
39 | | using namespace llvm; |
40 | | |
41 | | #define DEBUG_TYPE "esan" |
42 | | |
43 | | // The tool type must be just one of these ClTool* options, as the tools |
44 | | // cannot be combined due to shadow memory constraints. |
45 | | static cl::opt<bool> |
46 | | ClToolCacheFrag("esan-cache-frag", cl::init(false), |
47 | | cl::desc("Detect data cache fragmentation"), cl::Hidden); |
48 | | static cl::opt<bool> |
49 | | ClToolWorkingSet("esan-working-set", cl::init(false), |
50 | | cl::desc("Measure the working set size"), cl::Hidden); |
51 | | // Each new tool will get its own opt flag here. |
52 | | // These are converted to EfficiencySanitizerOptions for use |
53 | | // in the code. |
54 | | |
55 | | static cl::opt<bool> ClInstrumentLoadsAndStores( |
56 | | "esan-instrument-loads-and-stores", cl::init(true), |
57 | | cl::desc("Instrument loads and stores"), cl::Hidden); |
58 | | static cl::opt<bool> ClInstrumentMemIntrinsics( |
59 | | "esan-instrument-memintrinsics", cl::init(true), |
60 | | cl::desc("Instrument memintrinsics (memset/memcpy/memmove)"), cl::Hidden); |
61 | | static cl::opt<bool> ClInstrumentFastpath( |
62 | | "esan-instrument-fastpath", cl::init(true), |
63 | | cl::desc("Instrument fastpath"), cl::Hidden); |
64 | | static cl::opt<bool> ClAuxFieldInfo( |
65 | | "esan-aux-field-info", cl::init(true), |
66 | | cl::desc("Generate binary with auxiliary struct field information"), |
67 | | cl::Hidden); |
68 | | |
69 | | // Experiments show that the performance difference can be 2x or more, |
70 | | // and accuracy loss is typically negligible, so we turn this on by default. |
71 | | static cl::opt<bool> ClAssumeIntraCacheLine( |
72 | | "esan-assume-intra-cache-line", cl::init(true), |
73 | | cl::desc("Assume each memory access touches just one cache line, for " |
74 | | "better performance but with a potential loss of accuracy."), |
75 | | cl::Hidden); |
76 | | |
77 | | STATISTIC(NumInstrumentedLoads, "Number of instrumented loads"); |
78 | | STATISTIC(NumInstrumentedStores, "Number of instrumented stores"); |
79 | | STATISTIC(NumFastpaths, "Number of instrumented fastpaths"); |
80 | | STATISTIC(NumAccessesWithIrregularSize, |
81 | | "Number of accesses with a size outside our targeted callout sizes"); |
82 | | STATISTIC(NumIgnoredStructs, "Number of ignored structs"); |
83 | | STATISTIC(NumIgnoredGEPs, "Number of ignored GEP instructions"); |
84 | | STATISTIC(NumInstrumentedGEPs, "Number of instrumented GEP instructions"); |
85 | | STATISTIC(NumAssumedIntraCacheLine, |
86 | | "Number of accesses assumed to be intra-cache-line"); |
87 | | |
88 | | static const uint64_t EsanCtorAndDtorPriority = 0; |
89 | | static const char *const EsanModuleCtorName = "esan.module_ctor"; |
90 | | static const char *const EsanModuleDtorName = "esan.module_dtor"; |
91 | | static const char *const EsanInitName = "__esan_init"; |
92 | | static const char *const EsanExitName = "__esan_exit"; |
93 | | |
94 | | // We need to specify the tool to the runtime earlier than |
95 | | // the ctor is called in some cases, so we set a global variable. |
96 | | static const char *const EsanWhichToolName = "__esan_which_tool"; |
97 | | |
98 | | // We must keep these Shadow* constants consistent with the esan runtime. |
99 | | // FIXME: Try to place these shadow constants, the names of the __esan_* |
100 | | // interface functions, and the ToolType enum into a header shared between |
101 | | // llvm and compiler-rt. |
102 | | struct ShadowMemoryParams { |
103 | | uint64_t ShadowMask; |
104 | | uint64_t ShadowOffs[3]; |
105 | | }; |
106 | | |
107 | | static const ShadowMemoryParams ShadowParams47 = { |
108 | | 0x00000fffffffffffull, |
109 | | { |
110 | | 0x0000130000000000ull, 0x0000220000000000ull, 0x0000440000000000ull, |
111 | | }}; |
112 | | |
113 | | static const ShadowMemoryParams ShadowParams40 = { |
114 | | 0x0fffffffffull, |
115 | | { |
116 | | 0x1300000000ull, 0x2200000000ull, 0x4400000000ull, |
117 | | }}; |
118 | | |
119 | | // This array is indexed by the ToolType enum. |
120 | | static const int ShadowScale[] = { |
121 | | 0, // ESAN_None. |
122 | | 2, // ESAN_CacheFrag: 4B:1B, so 4 to 1 == >>2. |
123 | | 6, // ESAN_WorkingSet: 64B:1B, so 64 to 1 == >>6. |
124 | | }; |
125 | | |
126 | | // MaxStructCounterNameSize is a soft size limit to avoid insanely long |
127 | | // names for those extremely large structs. |
128 | | static const unsigned MaxStructCounterNameSize = 512; |
129 | | |
130 | | namespace { |
131 | | |
132 | | static EfficiencySanitizerOptions |
133 | 19 | OverrideOptionsFromCL(EfficiencySanitizerOptions Options) { |
134 | 19 | if (ClToolCacheFrag) |
135 | 3 | Options.ToolType = EfficiencySanitizerOptions::ESAN_CacheFrag; |
136 | 16 | else if (16 ClToolWorkingSet16 ) |
137 | 3 | Options.ToolType = EfficiencySanitizerOptions::ESAN_WorkingSet; |
138 | 19 | |
139 | 19 | // Direct opt invocation with no params will have the default ESAN_None. |
140 | 19 | // We run the default tool in that case. |
141 | 19 | if (Options.ToolType == EfficiencySanitizerOptions::ESAN_None) |
142 | 1 | Options.ToolType = EfficiencySanitizerOptions::ESAN_CacheFrag; |
143 | 19 | |
144 | 19 | return Options; |
145 | 19 | } |
146 | | |
147 | | // Create a constant for Str so that we can pass it to the run-time lib. |
148 | | static GlobalVariable *createPrivateGlobalForString(Module &M, StringRef Str, |
149 | 46 | bool AllowMerging) { |
150 | 46 | Constant *StrConst = ConstantDataArray::getString(M.getContext(), Str); |
151 | 46 | // We use private linkage for module-local strings. If they can be merged |
152 | 46 | // with another one, we set the unnamed_addr attribute. |
153 | 46 | GlobalVariable *GV = |
154 | 46 | new GlobalVariable(M, StrConst->getType(), true, |
155 | 46 | GlobalValue::PrivateLinkage, StrConst, ""); |
156 | 46 | if (AllowMerging) |
157 | 46 | GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global); |
158 | 46 | GV->setAlignment(1); // Strings may not be merged w/o setting align 1. |
159 | 46 | return GV; |
160 | 46 | } |
161 | | |
162 | | /// EfficiencySanitizer: instrument each module to find performance issues. |
163 | | class EfficiencySanitizer : public ModulePass { |
164 | | public: |
165 | | EfficiencySanitizer( |
166 | | const EfficiencySanitizerOptions &Opts = EfficiencySanitizerOptions()) |
167 | 19 | : ModulePass(ID), Options(OverrideOptionsFromCL(Opts)) {} |
168 | | StringRef getPassName() const override; |
169 | | void getAnalysisUsage(AnalysisUsage &AU) const override; |
170 | | bool runOnModule(Module &M) override; |
171 | | static char ID; |
172 | | |
173 | | private: |
174 | | bool initOnModule(Module &M); |
175 | | void initializeCallbacks(Module &M); |
176 | | bool shouldIgnoreStructType(StructType *StructTy); |
177 | | void createStructCounterName( |
178 | | StructType *StructTy, SmallString<MaxStructCounterNameSize> &NameStr); |
179 | | void createCacheFragAuxGV( |
180 | | Module &M, const DataLayout &DL, StructType *StructTy, |
181 | | GlobalVariable *&TypeNames, GlobalVariable *&Offsets, GlobalVariable *&Size); |
182 | | GlobalVariable *createCacheFragInfoGV(Module &M, const DataLayout &DL, |
183 | | Constant *UnitName); |
184 | | Constant *createEsanInitToolInfoArg(Module &M, const DataLayout &DL); |
185 | | void createDestructor(Module &M, Constant *ToolInfoArg); |
186 | | bool runOnFunction(Function &F, Module &M); |
187 | | bool instrumentLoadOrStore(Instruction *I, const DataLayout &DL); |
188 | | bool instrumentMemIntrinsic(MemIntrinsic *MI); |
189 | | bool instrumentGetElementPtr(Instruction *I, Module &M); |
190 | | bool insertCounterUpdate(Instruction *I, StructType *StructTy, |
191 | | unsigned CounterIdx); |
192 | 32 | unsigned getFieldCounterIdx(StructType *StructTy) { |
193 | 32 | return 0; |
194 | 32 | } |
195 | 23 | unsigned getArrayCounterIdx(StructType *StructTy) { |
196 | 23 | return StructTy->getNumElements(); |
197 | 23 | } |
198 | 43 | unsigned getStructCounterSize(StructType *StructTy) { |
199 | 43 | // The struct counter array includes: |
200 | 43 | // - one counter for each struct field, |
201 | 43 | // - one counter for the struct access within an array. |
202 | 43 | return (StructTy->getNumElements()/*field*/ + 1/*array*/); |
203 | 43 | } |
204 | | bool shouldIgnoreMemoryAccess(Instruction *I); |
205 | | int getMemoryAccessFuncIndex(Value *Addr, const DataLayout &DL); |
206 | | Value *appToShadow(Value *Shadow, IRBuilder<> &IRB); |
207 | | bool instrumentFastpath(Instruction *I, const DataLayout &DL, bool IsStore, |
208 | | Value *Addr, unsigned Alignment); |
209 | | // Each tool has its own fastpath routine: |
210 | | bool instrumentFastpathCacheFrag(Instruction *I, const DataLayout &DL, |
211 | | Value *Addr, unsigned Alignment); |
212 | | bool instrumentFastpathWorkingSet(Instruction *I, const DataLayout &DL, |
213 | | Value *Addr, unsigned Alignment); |
214 | | |
215 | | EfficiencySanitizerOptions Options; |
216 | | LLVMContext *Ctx; |
217 | | Type *IntptrTy; |
218 | | // Our slowpath involves callouts to the runtime library. |
219 | | // Access sizes are powers of two: 1, 2, 4, 8, 16. |
220 | | static const size_t NumberOfAccessSizes = 5; |
221 | | Function *EsanAlignedLoad[NumberOfAccessSizes]; |
222 | | Function *EsanAlignedStore[NumberOfAccessSizes]; |
223 | | Function *EsanUnalignedLoad[NumberOfAccessSizes]; |
224 | | Function *EsanUnalignedStore[NumberOfAccessSizes]; |
225 | | // For irregular sizes of any alignment: |
226 | | Function *EsanUnalignedLoadN, *EsanUnalignedStoreN; |
227 | | Function *MemmoveFn, *MemcpyFn, *MemsetFn; |
228 | | Function *EsanCtorFunction; |
229 | | Function *EsanDtorFunction; |
230 | | // Remember the counter variable for each struct type to avoid |
231 | | // recomputing the variable name later during instrumentation. |
232 | | std::map<Type *, GlobalVariable *> StructTyMap; |
233 | | ShadowMemoryParams ShadowParams; |
234 | | }; |
235 | | } // namespace |
236 | | |
237 | | char EfficiencySanitizer::ID = 0; |
238 | 7.91k | INITIALIZE_PASS_BEGIN7.91k (
|
239 | 7.91k | EfficiencySanitizer, "esan", |
240 | 7.91k | "EfficiencySanitizer: finds performance issues.", false, false) |
241 | 7.91k | INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass) |
242 | 7.91k | INITIALIZE_PASS_END( |
243 | | EfficiencySanitizer, "esan", |
244 | | "EfficiencySanitizer: finds performance issues.", false, false) |
245 | | |
246 | 0 | StringRef EfficiencySanitizer::getPassName() const { |
247 | 0 | return "EfficiencySanitizer"; |
248 | 0 | } |
249 | | |
250 | 19 | void EfficiencySanitizer::getAnalysisUsage(AnalysisUsage &AU) const { |
251 | 19 | AU.addRequired<TargetLibraryInfoWrapperPass>(); |
252 | 19 | } |
253 | | |
254 | | ModulePass * |
255 | 12 | llvm::createEfficiencySanitizerPass(const EfficiencySanitizerOptions &Options) { |
256 | 12 | return new EfficiencySanitizer(Options); |
257 | 12 | } |
258 | | |
259 | 19 | void EfficiencySanitizer::initializeCallbacks(Module &M) { |
260 | 19 | IRBuilder<> IRB(M.getContext()); |
261 | 19 | // Initialize the callbacks. |
262 | 114 | for (size_t Idx = 0; Idx < NumberOfAccessSizes114 ; ++Idx95 ) { |
263 | 95 | const unsigned ByteSize = 1U << Idx; |
264 | 95 | std::string ByteSizeStr = utostr(ByteSize); |
265 | 95 | // We'll inline the most common (i.e., aligned and frequent sizes) |
266 | 95 | // load + store instrumentation: these callouts are for the slowpath. |
267 | 95 | SmallString<32> AlignedLoadName("__esan_aligned_load" + ByteSizeStr); |
268 | 95 | EsanAlignedLoad[Idx] = |
269 | 95 | checkSanitizerInterfaceFunction(M.getOrInsertFunction( |
270 | 95 | AlignedLoadName, IRB.getVoidTy(), IRB.getInt8PtrTy())); |
271 | 95 | SmallString<32> AlignedStoreName("__esan_aligned_store" + ByteSizeStr); |
272 | 95 | EsanAlignedStore[Idx] = |
273 | 95 | checkSanitizerInterfaceFunction(M.getOrInsertFunction( |
274 | 95 | AlignedStoreName, IRB.getVoidTy(), IRB.getInt8PtrTy())); |
275 | 95 | SmallString<32> UnalignedLoadName("__esan_unaligned_load" + ByteSizeStr); |
276 | 95 | EsanUnalignedLoad[Idx] = |
277 | 95 | checkSanitizerInterfaceFunction(M.getOrInsertFunction( |
278 | 95 | UnalignedLoadName, IRB.getVoidTy(), IRB.getInt8PtrTy())); |
279 | 95 | SmallString<32> UnalignedStoreName("__esan_unaligned_store" + ByteSizeStr); |
280 | 95 | EsanUnalignedStore[Idx] = |
281 | 95 | checkSanitizerInterfaceFunction(M.getOrInsertFunction( |
282 | 95 | UnalignedStoreName, IRB.getVoidTy(), IRB.getInt8PtrTy())); |
283 | 95 | } |
284 | 19 | EsanUnalignedLoadN = checkSanitizerInterfaceFunction( |
285 | 19 | M.getOrInsertFunction("__esan_unaligned_loadN", IRB.getVoidTy(), |
286 | 19 | IRB.getInt8PtrTy(), IntptrTy)); |
287 | 19 | EsanUnalignedStoreN = checkSanitizerInterfaceFunction( |
288 | 19 | M.getOrInsertFunction("__esan_unaligned_storeN", IRB.getVoidTy(), |
289 | 19 | IRB.getInt8PtrTy(), IntptrTy)); |
290 | 19 | MemmoveFn = checkSanitizerInterfaceFunction( |
291 | 19 | M.getOrInsertFunction("memmove", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), |
292 | 19 | IRB.getInt8PtrTy(), IntptrTy)); |
293 | 19 | MemcpyFn = checkSanitizerInterfaceFunction( |
294 | 19 | M.getOrInsertFunction("memcpy", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), |
295 | 19 | IRB.getInt8PtrTy(), IntptrTy)); |
296 | 19 | MemsetFn = checkSanitizerInterfaceFunction( |
297 | 19 | M.getOrInsertFunction("memset", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), |
298 | 19 | IRB.getInt32Ty(), IntptrTy)); |
299 | 19 | } |
300 | | |
301 | 47 | bool EfficiencySanitizer::shouldIgnoreStructType(StructType *StructTy) { |
302 | 47 | if (StructTy == nullptr || 47 StructTy->isOpaque()43 /* no struct body */) |
303 | 4 | return true; |
304 | 43 | return false; |
305 | 43 | } |
306 | | |
307 | | void EfficiencySanitizer::createStructCounterName( |
308 | 12 | StructType *StructTy, SmallString<MaxStructCounterNameSize> &NameStr) { |
309 | 12 | // Append NumFields and field type ids to avoid struct conflicts |
310 | 12 | // with the same name but different fields. |
311 | 12 | if (StructTy->hasName()) |
312 | 12 | NameStr += StructTy->getName(); |
313 | 12 | else |
314 | 0 | NameStr += "struct.anon"; |
315 | 12 | // We allow the actual size of the StructCounterName to be larger than |
316 | 12 | // MaxStructCounterNameSize and append $NumFields and at least one |
317 | 12 | // field type id. |
318 | 12 | // Append $NumFields. |
319 | 12 | NameStr += "$"; |
320 | 12 | Twine(StructTy->getNumElements()).toVector(NameStr); |
321 | 12 | // Append struct field type ids in the reverse order. |
322 | 36 | for (int i = StructTy->getNumElements() - 1; i >= 036 ; --i24 ) { |
323 | 24 | NameStr += "$"; |
324 | 24 | Twine(StructTy->getElementType(i)->getTypeID()).toVector(NameStr); |
325 | 24 | if (NameStr.size() >= MaxStructCounterNameSize) |
326 | 0 | break; |
327 | 24 | } |
328 | 12 | if (StructTy->isLiteral()12 ) { |
329 | 0 | // End with $ for literal struct. |
330 | 0 | NameStr += "$"; |
331 | 0 | } |
332 | 12 | } |
333 | | |
334 | | // Create global variables with auxiliary information (e.g., struct field size, |
335 | | // offset, and type name) for better user report. |
336 | | void EfficiencySanitizer::createCacheFragAuxGV( |
337 | | Module &M, const DataLayout &DL, StructType *StructTy, |
338 | | GlobalVariable *&TypeName, GlobalVariable *&Offset, |
339 | 7 | GlobalVariable *&Size) { |
340 | 7 | auto *Int8PtrTy = Type::getInt8PtrTy(*Ctx); |
341 | 7 | auto *Int32Ty = Type::getInt32Ty(*Ctx); |
342 | 7 | // FieldTypeName. |
343 | 7 | auto *TypeNameArrayTy = ArrayType::get(Int8PtrTy, StructTy->getNumElements()); |
344 | 7 | TypeName = new GlobalVariable(M, TypeNameArrayTy, true, |
345 | 7 | GlobalVariable::InternalLinkage, nullptr); |
346 | 7 | SmallVector<Constant *, 16> TypeNameVec; |
347 | 7 | // FieldOffset. |
348 | 7 | auto *OffsetArrayTy = ArrayType::get(Int32Ty, StructTy->getNumElements()); |
349 | 7 | Offset = new GlobalVariable(M, OffsetArrayTy, true, |
350 | 7 | GlobalVariable::InternalLinkage, nullptr); |
351 | 7 | SmallVector<Constant *, 16> OffsetVec; |
352 | 7 | // FieldSize |
353 | 7 | auto *SizeArrayTy = ArrayType::get(Int32Ty, StructTy->getNumElements()); |
354 | 7 | Size = new GlobalVariable(M, SizeArrayTy, true, |
355 | 7 | GlobalVariable::InternalLinkage, nullptr); |
356 | 7 | SmallVector<Constant *, 16> SizeVec; |
357 | 22 | for (unsigned i = 0; i < StructTy->getNumElements()22 ; ++i15 ) { |
358 | 15 | Type *Ty = StructTy->getElementType(i); |
359 | 15 | std::string Str; |
360 | 15 | raw_string_ostream StrOS(Str); |
361 | 15 | Ty->print(StrOS); |
362 | 15 | TypeNameVec.push_back( |
363 | 15 | ConstantExpr::getPointerCast( |
364 | 15 | createPrivateGlobalForString(M, StrOS.str(), true), |
365 | 15 | Int8PtrTy)); |
366 | 15 | OffsetVec.push_back( |
367 | 15 | ConstantInt::get(Int32Ty, |
368 | 15 | DL.getStructLayout(StructTy)->getElementOffset(i))); |
369 | 15 | SizeVec.push_back(ConstantInt::get(Int32Ty, |
370 | 15 | DL.getTypeAllocSize(Ty))); |
371 | 15 | } |
372 | 7 | TypeName->setInitializer(ConstantArray::get(TypeNameArrayTy, TypeNameVec)); |
373 | 7 | Offset->setInitializer(ConstantArray::get(OffsetArrayTy, OffsetVec)); |
374 | 7 | Size->setInitializer(ConstantArray::get(SizeArrayTy, SizeVec)); |
375 | 7 | } |
376 | | |
377 | | // Create the global variable for the cache-fragmentation tool. |
378 | | GlobalVariable *EfficiencySanitizer::createCacheFragInfoGV( |
379 | 10 | Module &M, const DataLayout &DL, Constant *UnitName) { |
380 | 10 | assert(Options.ToolType == EfficiencySanitizerOptions::ESAN_CacheFrag); |
381 | 10 | |
382 | 10 | auto *Int8PtrTy = Type::getInt8PtrTy(*Ctx); |
383 | 10 | auto *Int8PtrPtrTy = Int8PtrTy->getPointerTo(); |
384 | 10 | auto *Int32Ty = Type::getInt32Ty(*Ctx); |
385 | 10 | auto *Int32PtrTy = Type::getInt32PtrTy(*Ctx); |
386 | 10 | auto *Int64Ty = Type::getInt64Ty(*Ctx); |
387 | 10 | auto *Int64PtrTy = Type::getInt64PtrTy(*Ctx); |
388 | 10 | // This structure should be kept consistent with the StructInfo struct |
389 | 10 | // in the runtime library. |
390 | 10 | // struct StructInfo { |
391 | 10 | // const char *StructName; |
392 | 10 | // u32 Size; |
393 | 10 | // u32 NumFields; |
394 | 10 | // u32 *FieldOffset; // auxiliary struct field info. |
395 | 10 | // u32 *FieldSize; // auxiliary struct field info. |
396 | 10 | // const char **FieldTypeName; // auxiliary struct field info. |
397 | 10 | // u64 *FieldCounters; |
398 | 10 | // u64 *ArrayCounter; |
399 | 10 | // }; |
400 | 10 | auto *StructInfoTy = |
401 | 10 | StructType::get(Int8PtrTy, Int32Ty, Int32Ty, Int32PtrTy, Int32PtrTy, |
402 | 10 | Int8PtrPtrTy, Int64PtrTy, Int64PtrTy); |
403 | 10 | auto *StructInfoPtrTy = StructInfoTy->getPointerTo(); |
404 | 10 | // This structure should be kept consistent with the CacheFragInfo struct |
405 | 10 | // in the runtime library. |
406 | 10 | // struct CacheFragInfo { |
407 | 10 | // const char *UnitName; |
408 | 10 | // u32 NumStructs; |
409 | 10 | // StructInfo *Structs; |
410 | 10 | // }; |
411 | 10 | auto *CacheFragInfoTy = StructType::get(Int8PtrTy, Int32Ty, StructInfoPtrTy); |
412 | 10 | |
413 | 10 | std::vector<StructType *> Vec = M.getIdentifiedStructTypes(); |
414 | 10 | unsigned NumStructs = 0; |
415 | 10 | SmallVector<Constant *, 16> Initializers; |
416 | 10 | |
417 | 12 | for (auto &StructTy : Vec) { |
418 | 12 | if (shouldIgnoreStructType(StructTy)12 ) { |
419 | 0 | ++NumIgnoredStructs; |
420 | 0 | continue; |
421 | 0 | } |
422 | 12 | ++NumStructs; |
423 | 12 | |
424 | 12 | // StructName. |
425 | 12 | SmallString<MaxStructCounterNameSize> CounterNameStr; |
426 | 12 | createStructCounterName(StructTy, CounterNameStr); |
427 | 12 | GlobalVariable *StructCounterName = createPrivateGlobalForString( |
428 | 12 | M, CounterNameStr, /*AllowMerging*/true); |
429 | 12 | |
430 | 12 | // Counters. |
431 | 12 | // We create the counter array with StructCounterName and weak linkage |
432 | 12 | // so that the structs with the same name and layout from different |
433 | 12 | // compilation units will be merged into one. |
434 | 12 | auto *CounterArrayTy = ArrayType::get(Int64Ty, |
435 | 12 | getStructCounterSize(StructTy)); |
436 | 12 | GlobalVariable *Counters = |
437 | 12 | new GlobalVariable(M, CounterArrayTy, false, |
438 | 12 | GlobalVariable::WeakAnyLinkage, |
439 | 12 | ConstantAggregateZero::get(CounterArrayTy), |
440 | 12 | CounterNameStr); |
441 | 12 | |
442 | 12 | // Remember the counter variable for each struct type. |
443 | 12 | StructTyMap.insert(std::pair<Type *, GlobalVariable *>(StructTy, Counters)); |
444 | 12 | |
445 | 12 | // We pass the field type name array, offset array, and size array to |
446 | 12 | // the runtime for better reporting. |
447 | 12 | GlobalVariable *TypeName = nullptr, *Offset = nullptr, *Size = nullptr; |
448 | 12 | if (ClAuxFieldInfo) |
449 | 7 | createCacheFragAuxGV(M, DL, StructTy, TypeName, Offset, Size); |
450 | 12 | |
451 | 12 | Constant *FieldCounterIdx[2]; |
452 | 12 | FieldCounterIdx[0] = ConstantInt::get(Int32Ty, 0); |
453 | 12 | FieldCounterIdx[1] = ConstantInt::get(Int32Ty, |
454 | 12 | getFieldCounterIdx(StructTy)); |
455 | 12 | Constant *ArrayCounterIdx[2]; |
456 | 12 | ArrayCounterIdx[0] = ConstantInt::get(Int32Ty, 0); |
457 | 12 | ArrayCounterIdx[1] = ConstantInt::get(Int32Ty, |
458 | 12 | getArrayCounterIdx(StructTy)); |
459 | 12 | Initializers.push_back(ConstantStruct::get( |
460 | 12 | StructInfoTy, |
461 | 12 | ConstantExpr::getPointerCast(StructCounterName, Int8PtrTy), |
462 | 12 | ConstantInt::get(Int32Ty, |
463 | 12 | DL.getStructLayout(StructTy)->getSizeInBytes()), |
464 | 12 | ConstantInt::get(Int32Ty, StructTy->getNumElements()), |
465 | 5 | Offset == nullptr ? ConstantPointerNull::get(Int32PtrTy) |
466 | 7 | : ConstantExpr::getPointerCast(Offset, Int32PtrTy), |
467 | 5 | Size == nullptr ? ConstantPointerNull::get(Int32PtrTy) |
468 | 7 | : ConstantExpr::getPointerCast(Size, Int32PtrTy), |
469 | 12 | TypeName == nullptr |
470 | 5 | ? ConstantPointerNull::get(Int8PtrPtrTy) |
471 | 7 | : ConstantExpr::getPointerCast(TypeName, Int8PtrPtrTy), |
472 | 12 | ConstantExpr::getGetElementPtr(CounterArrayTy, Counters, |
473 | 12 | FieldCounterIdx), |
474 | 12 | ConstantExpr::getGetElementPtr(CounterArrayTy, Counters, |
475 | 12 | ArrayCounterIdx))); |
476 | 12 | } |
477 | 10 | // Structs. |
478 | 10 | Constant *StructInfo; |
479 | 10 | if (NumStructs == 010 ) { |
480 | 7 | StructInfo = ConstantPointerNull::get(StructInfoPtrTy); |
481 | 10 | } else { |
482 | 3 | auto *StructInfoArrayTy = ArrayType::get(StructInfoTy, NumStructs); |
483 | 3 | StructInfo = ConstantExpr::getPointerCast( |
484 | 3 | new GlobalVariable(M, StructInfoArrayTy, false, |
485 | 3 | GlobalVariable::InternalLinkage, |
486 | 3 | ConstantArray::get(StructInfoArrayTy, Initializers)), |
487 | 3 | StructInfoPtrTy); |
488 | 3 | } |
489 | 10 | |
490 | 10 | auto *CacheFragInfoGV = new GlobalVariable( |
491 | 10 | M, CacheFragInfoTy, true, GlobalVariable::InternalLinkage, |
492 | 10 | ConstantStruct::get(CacheFragInfoTy, UnitName, |
493 | 10 | ConstantInt::get(Int32Ty, NumStructs), StructInfo)); |
494 | 10 | return CacheFragInfoGV; |
495 | 10 | } |
496 | | |
497 | | // Create the tool-specific argument passed to EsanInit and EsanExit. |
498 | | Constant *EfficiencySanitizer::createEsanInitToolInfoArg(Module &M, |
499 | 19 | const DataLayout &DL) { |
500 | 19 | // This structure contains tool-specific information about each compilation |
501 | 19 | // unit (module) and is passed to the runtime library. |
502 | 19 | GlobalVariable *ToolInfoGV = nullptr; |
503 | 19 | |
504 | 19 | auto *Int8PtrTy = Type::getInt8PtrTy(*Ctx); |
505 | 19 | // Compilation unit name. |
506 | 19 | auto *UnitName = ConstantExpr::getPointerCast( |
507 | 19 | createPrivateGlobalForString(M, M.getModuleIdentifier(), true), |
508 | 19 | Int8PtrTy); |
509 | 19 | |
510 | 19 | // Create the tool-specific variable. |
511 | 19 | if (Options.ToolType == EfficiencySanitizerOptions::ESAN_CacheFrag) |
512 | 10 | ToolInfoGV = createCacheFragInfoGV(M, DL, UnitName); |
513 | 19 | |
514 | 19 | if (ToolInfoGV != nullptr) |
515 | 10 | return ConstantExpr::getPointerCast(ToolInfoGV, Int8PtrTy); |
516 | 9 | |
517 | 9 | // Create the null pointer if no tool-specific variable created. |
518 | 9 | return ConstantPointerNull::get(Int8PtrTy); |
519 | 9 | } |
520 | | |
521 | 19 | void EfficiencySanitizer::createDestructor(Module &M, Constant *ToolInfoArg) { |
522 | 19 | PointerType *Int8PtrTy = Type::getInt8PtrTy(*Ctx); |
523 | 19 | EsanDtorFunction = Function::Create(FunctionType::get(Type::getVoidTy(*Ctx), |
524 | 19 | false), |
525 | 19 | GlobalValue::InternalLinkage, |
526 | 19 | EsanModuleDtorName, &M); |
527 | 19 | ReturnInst::Create(*Ctx, BasicBlock::Create(*Ctx, "", EsanDtorFunction)); |
528 | 19 | IRBuilder<> IRB_Dtor(EsanDtorFunction->getEntryBlock().getTerminator()); |
529 | 19 | Function *EsanExit = checkSanitizerInterfaceFunction( |
530 | 19 | M.getOrInsertFunction(EsanExitName, IRB_Dtor.getVoidTy(), |
531 | 19 | Int8PtrTy)); |
532 | 19 | EsanExit->setLinkage(Function::ExternalLinkage); |
533 | 19 | IRB_Dtor.CreateCall(EsanExit, {ToolInfoArg}); |
534 | 19 | appendToGlobalDtors(M, EsanDtorFunction, EsanCtorAndDtorPriority); |
535 | 19 | } |
536 | | |
537 | 19 | bool EfficiencySanitizer::initOnModule(Module &M) { |
538 | 19 | |
539 | 19 | Triple TargetTriple(M.getTargetTriple()); |
540 | 19 | if (TargetTriple.getArch() == Triple::mips64 || 19 TargetTriple.getArch() == Triple::mips64el17 ) |
541 | 4 | ShadowParams = ShadowParams40; |
542 | 19 | else |
543 | 15 | ShadowParams = ShadowParams47; |
544 | 19 | |
545 | 19 | Ctx = &M.getContext(); |
546 | 19 | const DataLayout &DL = M.getDataLayout(); |
547 | 19 | IRBuilder<> IRB(M.getContext()); |
548 | 19 | IntegerType *OrdTy = IRB.getInt32Ty(); |
549 | 19 | PointerType *Int8PtrTy = Type::getInt8PtrTy(*Ctx); |
550 | 19 | IntptrTy = DL.getIntPtrType(M.getContext()); |
551 | 19 | // Create the variable passed to EsanInit and EsanExit. |
552 | 19 | Constant *ToolInfoArg = createEsanInitToolInfoArg(M, DL); |
553 | 19 | // Constructor |
554 | 19 | // We specify the tool type both in the EsanWhichToolName global |
555 | 19 | // and as an arg to the init routine as a sanity check. |
556 | 19 | std::tie(EsanCtorFunction, std::ignore) = createSanitizerCtorAndInitFunctions( |
557 | 19 | M, EsanModuleCtorName, EsanInitName, /*InitArgTypes=*/{OrdTy, Int8PtrTy}, |
558 | 19 | /*InitArgs=*/{ |
559 | 19 | ConstantInt::get(OrdTy, static_cast<int>(Options.ToolType)), |
560 | 19 | ToolInfoArg}); |
561 | 19 | appendToGlobalCtors(M, EsanCtorFunction, EsanCtorAndDtorPriority); |
562 | 19 | |
563 | 19 | createDestructor(M, ToolInfoArg); |
564 | 19 | |
565 | 19 | new GlobalVariable(M, OrdTy, true, |
566 | 19 | GlobalValue::WeakAnyLinkage, |
567 | 19 | ConstantInt::get(OrdTy, |
568 | 19 | static_cast<int>(Options.ToolType)), |
569 | 19 | EsanWhichToolName); |
570 | 19 | |
571 | 19 | return true; |
572 | 19 | } |
573 | | |
574 | 26 | Value *EfficiencySanitizer::appToShadow(Value *Shadow, IRBuilder<> &IRB) { |
575 | 26 | // Shadow = ((App & Mask) + Offs) >> Scale |
576 | 26 | Shadow = IRB.CreateAnd(Shadow, ConstantInt::get(IntptrTy, ShadowParams.ShadowMask)); |
577 | 26 | uint64_t Offs; |
578 | 26 | int Scale = ShadowScale[Options.ToolType]; |
579 | 26 | if (Scale <= 2) |
580 | 0 | Offs = ShadowParams.ShadowOffs[Scale]; |
581 | 26 | else |
582 | 26 | Offs = ShadowParams.ShadowOffs[0] << Scale; |
583 | 26 | Shadow = IRB.CreateAdd(Shadow, ConstantInt::get(IntptrTy, Offs)); |
584 | 26 | if (Scale > 0) |
585 | 26 | Shadow = IRB.CreateLShr(Shadow, Scale); |
586 | 26 | return Shadow; |
587 | 26 | } |
588 | | |
589 | 64 | bool EfficiencySanitizer::shouldIgnoreMemoryAccess(Instruction *I) { |
590 | 64 | if (Options.ToolType == EfficiencySanitizerOptions::ESAN_CacheFrag64 ) { |
591 | 14 | // We'd like to know about cache fragmentation in vtable accesses and |
592 | 14 | // constant data references, so we do not currently ignore anything. |
593 | 14 | return false; |
594 | 50 | } else if (50 Options.ToolType == EfficiencySanitizerOptions::ESAN_WorkingSet50 ) { |
595 | 50 | // TODO: the instrumentation disturbs the data layout on the stack, so we |
596 | 50 | // may want to add an option to ignore stack references (if we can |
597 | 50 | // distinguish them) to reduce overhead. |
598 | 50 | } |
599 | 64 | // TODO(bruening): future tools will be returning true for some cases. |
600 | 50 | return false; |
601 | 64 | } |
602 | | |
603 | 19 | bool EfficiencySanitizer::runOnModule(Module &M) { |
604 | 19 | bool Res = initOnModule(M); |
605 | 19 | initializeCallbacks(M); |
606 | 636 | for (auto &F : M) { |
607 | 636 | Res |= runOnFunction(F, M); |
608 | 636 | } |
609 | 19 | return Res; |
610 | 19 | } |
611 | | |
612 | 636 | bool EfficiencySanitizer::runOnFunction(Function &F, Module &M) { |
613 | 636 | // This is required to prevent instrumenting the call to __esan_init from |
614 | 636 | // within the module constructor. |
615 | 636 | if (&F == EsanCtorFunction) |
616 | 19 | return false; |
617 | 617 | SmallVector<Instruction *, 8> LoadsAndStores; |
618 | 617 | SmallVector<Instruction *, 8> MemIntrinCalls; |
619 | 617 | SmallVector<Instruction *, 8> GetElementPtrs; |
620 | 617 | bool Res = false; |
621 | 617 | const DataLayout &DL = M.getDataLayout(); |
622 | 617 | const TargetLibraryInfo *TLI = |
623 | 617 | &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(); |
624 | 617 | |
625 | 85 | for (auto &BB : F) { |
626 | 242 | for (auto &Inst : BB) { |
627 | 242 | if ((isa<LoadInst>(Inst) || 242 isa<StoreInst>(Inst)194 || |
628 | 242 | isa<AtomicRMWInst>(Inst)178 || isa<AtomicCmpXchgInst>(Inst)178 ) && |
629 | 64 | !shouldIgnoreMemoryAccess(&Inst)) |
630 | 64 | LoadsAndStores.push_back(&Inst); |
631 | 178 | else if (178 isa<MemIntrinsic>(Inst)178 ) |
632 | 6 | MemIntrinCalls.push_back(&Inst); |
633 | 172 | else if (172 isa<GetElementPtrInst>(Inst)172 ) |
634 | 33 | GetElementPtrs.push_back(&Inst); |
635 | 139 | else if (CallInst *139 CI139 = dyn_cast<CallInst>(&Inst)) |
636 | 32 | maybeMarkSanitizerLibraryCallNoBuiltin(CI, TLI); |
637 | 242 | } |
638 | 85 | } |
639 | 617 | |
640 | 617 | if (ClInstrumentLoadsAndStores617 ) { |
641 | 64 | for (auto Inst : LoadsAndStores) { |
642 | 64 | Res |= instrumentLoadOrStore(Inst, DL); |
643 | 64 | } |
644 | 617 | } |
645 | 617 | |
646 | 617 | if (ClInstrumentMemIntrinsics617 ) { |
647 | 6 | for (auto Inst : MemIntrinCalls) { |
648 | 6 | Res |= instrumentMemIntrinsic(cast<MemIntrinsic>(Inst)); |
649 | 6 | } |
650 | 617 | } |
651 | 617 | |
652 | 617 | if (Options.ToolType == EfficiencySanitizerOptions::ESAN_CacheFrag617 ) { |
653 | 33 | for (auto Inst : GetElementPtrs) { |
654 | 33 | Res |= instrumentGetElementPtr(Inst, M); |
655 | 33 | } |
656 | 297 | } |
657 | 636 | |
658 | 636 | return Res; |
659 | 636 | } |
660 | | |
661 | | bool EfficiencySanitizer::instrumentLoadOrStore(Instruction *I, |
662 | 64 | const DataLayout &DL) { |
663 | 64 | IRBuilder<> IRB(I); |
664 | 64 | bool IsStore; |
665 | 64 | Value *Addr; |
666 | 64 | unsigned Alignment; |
667 | 64 | if (LoadInst *Load64 = dyn_cast<LoadInst>(I)) { |
668 | 48 | IsStore = false; |
669 | 48 | Alignment = Load->getAlignment(); |
670 | 48 | Addr = Load->getPointerOperand(); |
671 | 64 | } else if (StoreInst *16 Store16 = dyn_cast<StoreInst>(I)) { |
672 | 16 | IsStore = true; |
673 | 16 | Alignment = Store->getAlignment(); |
674 | 16 | Addr = Store->getPointerOperand(); |
675 | 16 | } else if (AtomicRMWInst *0 RMW0 = dyn_cast<AtomicRMWInst>(I)) { |
676 | 0 | IsStore = true; |
677 | 0 | Alignment = 0; |
678 | 0 | Addr = RMW->getPointerOperand(); |
679 | 0 | } else if (AtomicCmpXchgInst *0 Xchg0 = dyn_cast<AtomicCmpXchgInst>(I)) { |
680 | 0 | IsStore = true; |
681 | 0 | Alignment = 0; |
682 | 0 | Addr = Xchg->getPointerOperand(); |
683 | 0 | } else |
684 | 0 | llvm_unreachable("Unsupported mem access type"); |
685 | 64 | |
686 | 64 | Type *OrigTy = cast<PointerType>(Addr->getType())->getElementType(); |
687 | 64 | const uint32_t TypeSizeBytes = DL.getTypeStoreSizeInBits(OrigTy) / 8; |
688 | 64 | Value *OnAccessFunc = nullptr; |
689 | 64 | |
690 | 64 | // Convert 0 to the default alignment. |
691 | 64 | if (Alignment == 0) |
692 | 0 | Alignment = DL.getPrefTypeAlignment(OrigTy); |
693 | 64 | |
694 | 64 | if (IsStore) |
695 | 16 | NumInstrumentedStores++; |
696 | 64 | else |
697 | 48 | NumInstrumentedLoads++; |
698 | 64 | int Idx = getMemoryAccessFuncIndex(Addr, DL); |
699 | 64 | if (Idx < 064 ) { |
700 | 2 | OnAccessFunc = IsStore ? EsanUnalignedStoreN1 : EsanUnalignedLoadN1 ; |
701 | 2 | IRB.CreateCall(OnAccessFunc, |
702 | 2 | {IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy()), |
703 | 2 | ConstantInt::get(IntptrTy, TypeSizeBytes)}); |
704 | 64 | } else { |
705 | 62 | if (ClInstrumentFastpath && |
706 | 62 | instrumentFastpath(I, DL, IsStore, Addr, Alignment)44 ) { |
707 | 40 | NumFastpaths++; |
708 | 40 | return true; |
709 | 40 | } |
710 | 22 | if (22 Alignment == 0 || 22 (Alignment % TypeSizeBytes) == 022 ) |
711 | 10 | OnAccessFunc = IsStore ? 10 EsanAlignedStore[Idx]5 : EsanAlignedLoad[Idx]5 ; |
712 | 22 | else |
713 | 12 | OnAccessFunc = IsStore ? 12 EsanUnalignedStore[Idx]4 : EsanUnalignedLoad[Idx]8 ; |
714 | 62 | IRB.CreateCall(OnAccessFunc, |
715 | 62 | IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy())); |
716 | 62 | } |
717 | 24 | return true; |
718 | 64 | } |
719 | | |
720 | | // It's simplest to replace the memset/memmove/memcpy intrinsics with |
721 | | // calls that the runtime library intercepts. |
722 | | // Our pass is late enough that calls should not turn back into intrinsics. |
723 | 6 | bool EfficiencySanitizer::instrumentMemIntrinsic(MemIntrinsic *MI) { |
724 | 6 | IRBuilder<> IRB(MI); |
725 | 6 | bool Res = false; |
726 | 6 | if (isa<MemSetInst>(MI)6 ) { |
727 | 2 | IRB.CreateCall( |
728 | 2 | MemsetFn, |
729 | 2 | {IRB.CreatePointerCast(MI->getArgOperand(0), IRB.getInt8PtrTy()), |
730 | 2 | IRB.CreateIntCast(MI->getArgOperand(1), IRB.getInt32Ty(), false), |
731 | 2 | IRB.CreateIntCast(MI->getArgOperand(2), IntptrTy, false)}); |
732 | 2 | MI->eraseFromParent(); |
733 | 2 | Res = true; |
734 | 6 | } else if (4 isa<MemTransferInst>(MI)4 ) { |
735 | 4 | IRB.CreateCall( |
736 | 4 | isa<MemCpyInst>(MI) ? MemcpyFn2 : MemmoveFn2 , |
737 | 4 | {IRB.CreatePointerCast(MI->getArgOperand(0), IRB.getInt8PtrTy()), |
738 | 4 | IRB.CreatePointerCast(MI->getArgOperand(1), IRB.getInt8PtrTy()), |
739 | 4 | IRB.CreateIntCast(MI->getArgOperand(2), IntptrTy, false)}); |
740 | 4 | MI->eraseFromParent(); |
741 | 4 | Res = true; |
742 | 4 | } else |
743 | 0 | llvm_unreachable("Unsupported mem intrinsic type"); |
744 | 6 | return Res; |
745 | 6 | } |
746 | | |
747 | 33 | bool EfficiencySanitizer::instrumentGetElementPtr(Instruction *I, Module &M) { |
748 | 33 | GetElementPtrInst *GepInst = dyn_cast<GetElementPtrInst>(I); |
749 | 33 | bool Res = false; |
750 | 33 | if (GepInst == nullptr || 33 GepInst->getNumIndices() == 133 ) { |
751 | 2 | ++NumIgnoredGEPs; |
752 | 2 | return false; |
753 | 2 | } |
754 | 31 | Type *SourceTy = GepInst->getSourceElementType(); |
755 | 31 | StructType *StructTy = nullptr; |
756 | 31 | ConstantInt *Idx; |
757 | 31 | // Check if GEP calculates address from a struct array. |
758 | 31 | if (isa<StructType>(SourceTy)31 ) { |
759 | 19 | StructTy = cast<StructType>(SourceTy); |
760 | 19 | Idx = dyn_cast<ConstantInt>(GepInst->getOperand(1)); |
761 | 19 | if ((Idx == nullptr || 19 Idx->getSExtValue() != 019 ) && |
762 | 19 | !shouldIgnoreStructType(StructTy)1 && StructTyMap.count(StructTy) != 01 ) |
763 | 1 | Res |= insertCounterUpdate(I, StructTy, getArrayCounterIdx(StructTy)); |
764 | 19 | } |
765 | 31 | // Iterate all (except the first and the last) idx within each GEP instruction |
766 | 31 | // for possible nested struct field address calculation. |
767 | 65 | for (unsigned i = 1; i < GepInst->getNumIndices()65 ; ++i34 ) { |
768 | 34 | SmallVector<Value *, 8> IdxVec(GepInst->idx_begin(), |
769 | 34 | GepInst->idx_begin() + i); |
770 | 34 | Type *Ty = GetElementPtrInst::getIndexedType(SourceTy, IdxVec); |
771 | 34 | unsigned CounterIdx = 0; |
772 | 34 | if (isa<ArrayType>(Ty)34 ) { |
773 | 14 | ArrayType *ArrayTy = cast<ArrayType>(Ty); |
774 | 14 | StructTy = dyn_cast<StructType>(ArrayTy->getElementType()); |
775 | 14 | if (shouldIgnoreStructType(StructTy) || 14 StructTyMap.count(StructTy) == 010 ) |
776 | 4 | continue; |
777 | 10 | // The last counter for struct array access. |
778 | 10 | CounterIdx = getArrayCounterIdx(StructTy); |
779 | 34 | } else if (20 isa<StructType>(Ty)20 ) { |
780 | 20 | StructTy = cast<StructType>(Ty); |
781 | 20 | if (shouldIgnoreStructType(StructTy) || 20 StructTyMap.count(StructTy) == 020 ) |
782 | 0 | continue; |
783 | 20 | // Get the StructTy's subfield index. |
784 | 20 | Idx = cast<ConstantInt>(GepInst->getOperand(i+1)); |
785 | 20 | assert(Idx->getSExtValue() >= 0 && |
786 | 20 | Idx->getSExtValue() < StructTy->getNumElements()); |
787 | 20 | CounterIdx = getFieldCounterIdx(StructTy) + Idx->getSExtValue(); |
788 | 20 | } |
789 | 30 | Res |= insertCounterUpdate(I, StructTy, CounterIdx); |
790 | 30 | } |
791 | 31 | if (Res) |
792 | 29 | ++NumInstrumentedGEPs; |
793 | 31 | else |
794 | 2 | ++NumIgnoredGEPs; |
795 | 33 | return Res; |
796 | 33 | } |
797 | | |
798 | | bool EfficiencySanitizer::insertCounterUpdate(Instruction *I, |
799 | | StructType *StructTy, |
800 | 31 | unsigned CounterIdx) { |
801 | 31 | GlobalVariable *CounterArray = StructTyMap[StructTy]; |
802 | 31 | if (CounterArray == nullptr) |
803 | 0 | return false; |
804 | 31 | IRBuilder<> IRB(I); |
805 | 31 | Constant *Indices[2]; |
806 | 31 | // Xref http://llvm.org/docs/LangRef.html#i-getelementptr and |
807 | 31 | // http://llvm.org/docs/GetElementPtr.html. |
808 | 31 | // The first index of the GEP instruction steps through the first operand, |
809 | 31 | // i.e., the array itself. |
810 | 31 | Indices[0] = ConstantInt::get(IRB.getInt32Ty(), 0); |
811 | 31 | // The second index is the index within the array. |
812 | 31 | Indices[1] = ConstantInt::get(IRB.getInt32Ty(), CounterIdx); |
813 | 31 | Constant *Counter = |
814 | 31 | ConstantExpr::getGetElementPtr( |
815 | 31 | ArrayType::get(IRB.getInt64Ty(), getStructCounterSize(StructTy)), |
816 | 31 | CounterArray, Indices); |
817 | 31 | Value *Load = IRB.CreateLoad(Counter); |
818 | 31 | IRB.CreateStore(IRB.CreateAdd(Load, ConstantInt::get(IRB.getInt64Ty(), 1)), |
819 | 31 | Counter); |
820 | 31 | return true; |
821 | 31 | } |
822 | | |
823 | | int EfficiencySanitizer::getMemoryAccessFuncIndex(Value *Addr, |
824 | 64 | const DataLayout &DL) { |
825 | 64 | Type *OrigPtrTy = Addr->getType(); |
826 | 64 | Type *OrigTy = cast<PointerType>(OrigPtrTy)->getElementType(); |
827 | 64 | assert(OrigTy->isSized()); |
828 | 64 | // The size is always a multiple of 8. |
829 | 64 | uint32_t TypeSizeBytes = DL.getTypeStoreSizeInBits(OrigTy) / 8; |
830 | 64 | if (TypeSizeBytes != 1 && 64 TypeSizeBytes != 260 && TypeSizeBytes != 452 && |
831 | 64 | TypeSizeBytes != 832 && TypeSizeBytes != 1610 ) { |
832 | 2 | // Irregular sizes do not have per-size call targets. |
833 | 2 | NumAccessesWithIrregularSize++; |
834 | 2 | return -1; |
835 | 2 | } |
836 | 62 | size_t Idx = countTrailingZeros(TypeSizeBytes); |
837 | 62 | assert(Idx < NumberOfAccessSizes); |
838 | 62 | return Idx; |
839 | 62 | } |
840 | | |
841 | | bool EfficiencySanitizer::instrumentFastpath(Instruction *I, |
842 | | const DataLayout &DL, bool IsStore, |
843 | 44 | Value *Addr, unsigned Alignment) { |
844 | 44 | if (Options.ToolType == EfficiencySanitizerOptions::ESAN_CacheFrag44 ) { |
845 | 14 | return instrumentFastpathCacheFrag(I, DL, Addr, Alignment); |
846 | 30 | } else if (30 Options.ToolType == EfficiencySanitizerOptions::ESAN_WorkingSet30 ) { |
847 | 30 | return instrumentFastpathWorkingSet(I, DL, Addr, Alignment); |
848 | 30 | } |
849 | 0 | return false; |
850 | 0 | } |
851 | | |
852 | | bool EfficiencySanitizer::instrumentFastpathCacheFrag(Instruction *I, |
853 | | const DataLayout &DL, |
854 | | Value *Addr, |
855 | 14 | unsigned Alignment) { |
856 | 14 | // Do nothing. |
857 | 14 | return true; // Return true to avoid slowpath instrumentation. |
858 | 14 | } |
859 | | |
860 | | bool EfficiencySanitizer::instrumentFastpathWorkingSet( |
861 | 30 | Instruction *I, const DataLayout &DL, Value *Addr, unsigned Alignment) { |
862 | 30 | assert(ShadowScale[Options.ToolType] == 6); // The code below assumes this |
863 | 30 | IRBuilder<> IRB(I); |
864 | 30 | Type *OrigTy = cast<PointerType>(Addr->getType())->getElementType(); |
865 | 30 | const uint32_t TypeSize = DL.getTypeStoreSizeInBits(OrigTy); |
866 | 30 | // Bail to the slowpath if the access might touch multiple cache lines. |
867 | 30 | // An access aligned to its size is guaranteed to be intra-cache-line. |
868 | 30 | // getMemoryAccessFuncIndex has already ruled out a size larger than 16 |
869 | 30 | // and thus larger than a cache line for platforms this tool targets |
870 | 30 | // (and our shadow memory setup assumes 64-byte cache lines). |
871 | 30 | assert(TypeSize <= 128); |
872 | 30 | if (!(TypeSize == 8 || |
873 | 30 | (Alignment % (TypeSize / 8)) == 028 )) { |
874 | 8 | if (ClAssumeIntraCacheLine) |
875 | 4 | ++NumAssumedIntraCacheLine; |
876 | 8 | else |
877 | 4 | return false; |
878 | 26 | } |
879 | 26 | |
880 | 26 | // We inline instrumentation to set the corresponding shadow bits for |
881 | 26 | // each cache line touched by the application. Here we handle a single |
882 | 26 | // load or store where we've already ruled out the possibility that it |
883 | 26 | // might touch more than one cache line and thus we simply update the |
884 | 26 | // shadow memory for a single cache line. |
885 | 26 | // Our shadow memory model is fine with races when manipulating shadow values. |
886 | 26 | // We generate the following code: |
887 | 26 | // |
888 | 26 | // const char BitMask = 0x81; |
889 | 26 | // char *ShadowAddr = appToShadow(AppAddr); |
890 | 26 | // if ((*ShadowAddr & BitMask) != BitMask) |
891 | 26 | // *ShadowAddr |= Bitmask; |
892 | 26 | // |
893 | 26 | Value *AddrPtr = IRB.CreatePointerCast(Addr, IntptrTy); |
894 | 26 | Value *ShadowPtr = appToShadow(AddrPtr, IRB); |
895 | 26 | Type *ShadowTy = IntegerType::get(*Ctx, 8U); |
896 | 26 | Type *ShadowPtrTy = PointerType::get(ShadowTy, 0); |
897 | 26 | // The bottom bit is used for the current sampling period's working set. |
898 | 26 | // The top bit is used for the total working set. We set both on each |
899 | 26 | // memory access, if they are not already set. |
900 | 26 | Value *ValueMask = ConstantInt::get(ShadowTy, 0x81); // 10000001B |
901 | 26 | |
902 | 26 | Value *OldValue = IRB.CreateLoad(IRB.CreateIntToPtr(ShadowPtr, ShadowPtrTy)); |
903 | 26 | // The AND and CMP will be turned into a TEST instruction by the compiler. |
904 | 26 | Value *Cmp = IRB.CreateICmpNE(IRB.CreateAnd(OldValue, ValueMask), ValueMask); |
905 | 26 | TerminatorInst *CmpTerm = SplitBlockAndInsertIfThen(Cmp, I, false); |
906 | 26 | // FIXME: do I need to call SetCurrentDebugLocation? |
907 | 26 | IRB.SetInsertPoint(CmpTerm); |
908 | 26 | // We use OR to set the shadow bits to avoid corrupting the middle 6 bits, |
909 | 26 | // which are used by the runtime library. |
910 | 26 | Value *NewVal = IRB.CreateOr(OldValue, ValueMask); |
911 | 26 | IRB.CreateStore(NewVal, IRB.CreateIntToPtr(ShadowPtr, ShadowPtrTy)); |
912 | 26 | IRB.SetInsertPoint(I); |
913 | 26 | |
914 | 26 | return true; |
915 | 26 | } |