Coverage Report

Created: 2017-10-03 07:32

/Users/buildslave/jenkins/sharedspace/clang-stage2-coverage-R@2/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
Line
Count
Source (jump to first uncovered line)
1
//===-- MemorySanitizer.cpp - detector of uninitialized reads -------------===//
2
//
3
//                     The LLVM Compiler Infrastructure
4
//
5
// This file is distributed under the University of Illinois Open Source
6
// License. See LICENSE.TXT for details.
7
//
8
//===----------------------------------------------------------------------===//
9
/// \file
10
/// This file is a part of MemorySanitizer, a detector of uninitialized
11
/// reads.
12
///
13
/// The algorithm of the tool is similar to Memcheck
14
/// (http://goo.gl/QKbem). We associate a few shadow bits with every
15
/// byte of the application memory, poison the shadow of the malloc-ed
16
/// or alloca-ed memory, load the shadow bits on every memory read,
17
/// propagate the shadow bits through some of the arithmetic
18
/// instruction (including MOV), store the shadow bits on every memory
19
/// write, report a bug on some other instructions (e.g. JMP) if the
20
/// associated shadow is poisoned.
21
///
22
/// But there are differences too. The first and the major one:
23
/// compiler instrumentation instead of binary instrumentation. This
24
/// gives us much better register allocation, possible compiler
25
/// optimizations and a fast start-up. But this brings the major issue
26
/// as well: msan needs to see all program events, including system
27
/// calls and reads/writes in system libraries, so we either need to
28
/// compile *everything* with msan or use a binary translation
29
/// component (e.g. DynamoRIO) to instrument pre-built libraries.
30
/// Another difference from Memcheck is that we use 8 shadow bits per
31
/// byte of application memory and use a direct shadow mapping. This
32
/// greatly simplifies the instrumentation code and avoids races on
33
/// shadow updates (Memcheck is single-threaded so races are not a
34
/// concern there. Memcheck uses 2 shadow bits per byte with a slow
35
/// path storage that uses 8 bits per byte).
36
///
37
/// The default value of shadow is 0, which means "clean" (not poisoned).
38
///
39
/// Every module initializer should call __msan_init to ensure that the
40
/// shadow memory is ready. On error, __msan_warning is called. Since
41
/// parameters and return values may be passed via registers, we have a
42
/// specialized thread-local shadow for return values
43
/// (__msan_retval_tls) and parameters (__msan_param_tls).
44
///
45
///                           Origin tracking.
46
///
47
/// MemorySanitizer can track origins (allocation points) of all uninitialized
48
/// values. This behavior is controlled with a flag (msan-track-origins) and is
49
/// disabled by default.
50
///
51
/// Origins are 4-byte values created and interpreted by the runtime library.
52
/// They are stored in a second shadow mapping, one 4-byte value for 4 bytes
53
/// of application memory. Propagation of origins is basically a bunch of
54
/// "select" instructions that pick the origin of a dirty argument, if an
55
/// instruction has one.
56
///
57
/// Every 4 aligned, consecutive bytes of application memory have one origin
58
/// value associated with them. If these bytes contain uninitialized data
59
/// coming from 2 different allocations, the last store wins. Because of this,
60
/// MemorySanitizer reports can show unrelated origins, but this is unlikely in
61
/// practice.
62
///
63
/// Origins are meaningless for fully initialized values, so MemorySanitizer
64
/// avoids storing origin to memory when a fully initialized value is stored.
65
/// This way it avoids needless overwritting origin of the 4-byte region on
66
/// a short (i.e. 1 byte) clean store, and it is also good for performance.
67
///
68
///                            Atomic handling.
69
///
70
/// Ideally, every atomic store of application value should update the
71
/// corresponding shadow location in an atomic way. Unfortunately, atomic store
72
/// of two disjoint locations can not be done without severe slowdown.
73
///
74
/// Therefore, we implement an approximation that may err on the safe side.
75
/// In this implementation, every atomically accessed location in the program
76
/// may only change from (partially) uninitialized to fully initialized, but
77
/// not the other way around. We load the shadow _after_ the application load,
78
/// and we store the shadow _before_ the app store. Also, we always store clean
79
/// shadow (if the application store is atomic). This way, if the store-load
80
/// pair constitutes a happens-before arc, shadow store and load are correctly
81
/// ordered such that the load will get either the value that was stored, or
82
/// some later value (which is always clean).
83
///
84
/// This does not work very well with Compare-And-Swap (CAS) and
85
/// Read-Modify-Write (RMW) operations. To follow the above logic, CAS and RMW
86
/// must store the new shadow before the app operation, and load the shadow
87
/// after the app operation. Computers don't work this way. Current
88
/// implementation ignores the load aspect of CAS/RMW, always returning a clean
89
/// value. It implements the store part as a simple atomic store by storing a
90
/// clean shadow.
91
92
//===----------------------------------------------------------------------===//
93
94
#include "llvm/ADT/DepthFirstIterator.h"
95
#include "llvm/ADT/SmallString.h"
96
#include "llvm/ADT/SmallVector.h"
97
#include "llvm/ADT/StringExtras.h"
98
#include "llvm/ADT/Triple.h"
99
#include "llvm/IR/DataLayout.h"
100
#include "llvm/IR/Function.h"
101
#include "llvm/IR/IRBuilder.h"
102
#include "llvm/IR/InlineAsm.h"
103
#include "llvm/IR/InstVisitor.h"
104
#include "llvm/IR/IntrinsicInst.h"
105
#include "llvm/IR/LLVMContext.h"
106
#include "llvm/IR/MDBuilder.h"
107
#include "llvm/IR/Module.h"
108
#include "llvm/IR/Type.h"
109
#include "llvm/IR/ValueMap.h"
110
#include "llvm/Support/CommandLine.h"
111
#include "llvm/Support/Debug.h"
112
#include "llvm/Support/raw_ostream.h"
113
#include "llvm/Transforms/Instrumentation.h"
114
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
115
#include "llvm/Transforms/Utils/Local.h"
116
#include "llvm/Transforms/Utils/ModuleUtils.h"
117
118
using namespace llvm;
119
120
#define DEBUG_TYPE "msan"
121
122
static const unsigned kOriginSize = 4;
123
static const unsigned kMinOriginAlignment = 4;
124
static const unsigned kShadowTLSAlignment = 8;
125
126
// These constants must be kept in sync with the ones in msan.h.
127
static const unsigned kParamTLSSize = 800;
128
static const unsigned kRetvalTLSSize = 800;
129
130
// Accesses sizes are powers of two: 1, 2, 4, 8.
131
static const size_t kNumberOfAccessSizes = 4;
132
133
/// \brief Track origins of uninitialized values.
134
///
135
/// Adds a section to MemorySanitizer report that points to the allocation
136
/// (stack or heap) the uninitialized bits came from originally.
137
static cl::opt<int> ClTrackOrigins("msan-track-origins",
138
       cl::desc("Track origins (allocation sites) of poisoned memory"),
139
       cl::Hidden, cl::init(0));
140
static cl::opt<bool> ClKeepGoing("msan-keep-going",
141
       cl::desc("keep going after reporting a UMR"),
142
       cl::Hidden, cl::init(false));
143
static cl::opt<bool> ClPoisonStack("msan-poison-stack",
144
       cl::desc("poison uninitialized stack variables"),
145
       cl::Hidden, cl::init(true));
146
static cl::opt<bool> ClPoisonStackWithCall("msan-poison-stack-with-call",
147
       cl::desc("poison uninitialized stack variables with a call"),
148
       cl::Hidden, cl::init(false));
149
static cl::opt<int> ClPoisonStackPattern("msan-poison-stack-pattern",
150
       cl::desc("poison uninitialized stack variables with the given pattern"),
151
       cl::Hidden, cl::init(0xff));
152
static cl::opt<bool> ClPoisonUndef("msan-poison-undef",
153
       cl::desc("poison undef temps"),
154
       cl::Hidden, cl::init(true));
155
156
static cl::opt<bool> ClHandleICmp("msan-handle-icmp",
157
       cl::desc("propagate shadow through ICmpEQ and ICmpNE"),
158
       cl::Hidden, cl::init(true));
159
160
static cl::opt<bool> ClHandleICmpExact("msan-handle-icmp-exact",
161
       cl::desc("exact handling of relational integer ICmp"),
162
       cl::Hidden, cl::init(false));
163
164
// This flag controls whether we check the shadow of the address
165
// operand of load or store. Such bugs are very rare, since load from
166
// a garbage address typically results in SEGV, but still happen
167
// (e.g. only lower bits of address are garbage, or the access happens
168
// early at program startup where malloc-ed memory is more likely to
169
// be zeroed. As of 2012-08-28 this flag adds 20% slowdown.
170
static cl::opt<bool> ClCheckAccessAddress("msan-check-access-address",
171
       cl::desc("report accesses through a pointer which has poisoned shadow"),
172
       cl::Hidden, cl::init(true));
173
174
static cl::opt<bool> ClDumpStrictInstructions("msan-dump-strict-instructions",
175
       cl::desc("print out instructions with default strict semantics"),
176
       cl::Hidden, cl::init(false));
177
178
static cl::opt<int> ClInstrumentationWithCallThreshold(
179
    "msan-instrumentation-with-call-threshold",
180
    cl::desc(
181
        "If the function being instrumented requires more than "
182
        "this number of checks and origin stores, use callbacks instead of "
183
        "inline checks (-1 means never use callbacks)."),
184
    cl::Hidden, cl::init(3500));
185
186
// This is an experiment to enable handling of cases where shadow is a non-zero
187
// compile-time constant. For some unexplainable reason they were silently
188
// ignored in the instrumentation.
189
static cl::opt<bool> ClCheckConstantShadow("msan-check-constant-shadow",
190
       cl::desc("Insert checks for constant shadow values"),
191
       cl::Hidden, cl::init(false));
192
193
// This is off by default because of a bug in gold:
194
// https://sourceware.org/bugzilla/show_bug.cgi?id=19002
195
static cl::opt<bool> ClWithComdat("msan-with-comdat",
196
       cl::desc("Place MSan constructors in comdat sections"),
197
       cl::Hidden, cl::init(false));
198
199
static const char *const kMsanModuleCtorName = "msan.module_ctor";
200
static const char *const kMsanInitName = "__msan_init";
201
202
namespace {
203
204
// Memory map parameters used in application-to-shadow address calculation.
205
// Offset = (Addr & ~AndMask) ^ XorMask
206
// Shadow = ShadowBase + Offset
207
// Origin = OriginBase + Offset
208
struct MemoryMapParams {
209
  uint64_t AndMask;
210
  uint64_t XorMask;
211
  uint64_t ShadowBase;
212
  uint64_t OriginBase;
213
};
214
215
struct PlatformMemoryMapParams {
216
  const MemoryMapParams *bits32;
217
  const MemoryMapParams *bits64;
218
};
219
220
// i386 Linux
221
static const MemoryMapParams Linux_I386_MemoryMapParams = {
222
  0x000080000000,  // AndMask
223
  0,               // XorMask (not used)
224
  0,               // ShadowBase (not used)
225
  0x000040000000,  // OriginBase
226
};
227
228
// x86_64 Linux
229
static const MemoryMapParams Linux_X86_64_MemoryMapParams = {
230
#ifdef MSAN_LINUX_X86_64_OLD_MAPPING
231
  0x400000000000,  // AndMask
232
  0,               // XorMask (not used)
233
  0,               // ShadowBase (not used)
234
  0x200000000000,  // OriginBase
235
#else
236
  0,               // AndMask (not used)
237
  0x500000000000,  // XorMask
238
  0,               // ShadowBase (not used)
239
  0x100000000000,  // OriginBase
240
#endif
241
};
242
243
// mips64 Linux
244
static const MemoryMapParams Linux_MIPS64_MemoryMapParams = {
245
  0,               // AndMask (not used)
246
  0x008000000000,  // XorMask
247
  0,               // ShadowBase (not used)
248
  0x002000000000,  // OriginBase
249
};
250
251
// ppc64 Linux
252
static const MemoryMapParams Linux_PowerPC64_MemoryMapParams = {
253
  0x200000000000,  // AndMask
254
  0x100000000000,  // XorMask
255
  0x080000000000,  // ShadowBase
256
  0x1C0000000000,  // OriginBase
257
};
258
259
// aarch64 Linux
260
static const MemoryMapParams Linux_AArch64_MemoryMapParams = {
261
  0,               // AndMask (not used)
262
  0x06000000000,   // XorMask
263
  0,               // ShadowBase (not used)
264
  0x01000000000,   // OriginBase
265
};
266
267
// i386 FreeBSD
268
static const MemoryMapParams FreeBSD_I386_MemoryMapParams = {
269
  0x000180000000,  // AndMask
270
  0x000040000000,  // XorMask
271
  0x000020000000,  // ShadowBase
272
  0x000700000000,  // OriginBase
273
};
274
275
// x86_64 FreeBSD
276
static const MemoryMapParams FreeBSD_X86_64_MemoryMapParams = {
277
  0xc00000000000,  // AndMask
278
  0x200000000000,  // XorMask
279
  0x100000000000,  // ShadowBase
280
  0x380000000000,  // OriginBase
281
};
282
283
static const PlatformMemoryMapParams Linux_X86_MemoryMapParams = {
284
  &Linux_I386_MemoryMapParams,
285
  &Linux_X86_64_MemoryMapParams,
286
};
287
288
static const PlatformMemoryMapParams Linux_MIPS_MemoryMapParams = {
289
  nullptr,
290
  &Linux_MIPS64_MemoryMapParams,
291
};
292
293
static const PlatformMemoryMapParams Linux_PowerPC_MemoryMapParams = {
294
  nullptr,
295
  &Linux_PowerPC64_MemoryMapParams,
296
};
297
298
static const PlatformMemoryMapParams Linux_ARM_MemoryMapParams = {
299
  nullptr,
300
  &Linux_AArch64_MemoryMapParams,
301
};
302
303
static const PlatformMemoryMapParams FreeBSD_X86_MemoryMapParams = {
304
  &FreeBSD_I386_MemoryMapParams,
305
  &FreeBSD_X86_64_MemoryMapParams,
306
};
307
308
/// \brief An instrumentation pass implementing detection of uninitialized
309
/// reads.
310
///
311
/// MemorySanitizer: instrument the code in module to find
312
/// uninitialized reads.
313
class MemorySanitizer : public FunctionPass {
314
 public:
315
  MemorySanitizer(int TrackOrigins = 0, bool Recover = false)
316
      : FunctionPass(ID),
317
        TrackOrigins(std::max(TrackOrigins, (int)ClTrackOrigins)),
318
        Recover(Recover || ClKeepGoing),
319
64
        WarningFn(nullptr) {}
320
0
  StringRef getPassName() const override { return "MemorySanitizer"; }
321
64
  void getAnalysisUsage(AnalysisUsage &AU) const override {
322
64
    AU.addRequired<TargetLibraryInfoWrapperPass>();
323
64
  }
324
  bool runOnFunction(Function &F) override;
325
  bool doInitialization(Module &M) override;
326
  static char ID;  // Pass identification, replacement for typeid.
327
328
 private:
329
  void initializeCallbacks(Module &M);
330
331
  /// \brief Track origins (allocation points) of uninitialized values.
332
  int TrackOrigins;
333
  bool Recover;
334
335
  LLVMContext *C;
336
  Type *IntptrTy;
337
  Type *OriginTy;
338
  /// \brief Thread-local shadow storage for function parameters.
339
  GlobalVariable *ParamTLS;
340
  /// \brief Thread-local origin storage for function parameters.
341
  GlobalVariable *ParamOriginTLS;
342
  /// \brief Thread-local shadow storage for function return value.
343
  GlobalVariable *RetvalTLS;
344
  /// \brief Thread-local origin storage for function return value.
345
  GlobalVariable *RetvalOriginTLS;
346
  /// \brief Thread-local shadow storage for in-register va_arg function
347
  /// parameters (x86_64-specific).
348
  GlobalVariable *VAArgTLS;
349
  /// \brief Thread-local shadow storage for va_arg overflow area
350
  /// (x86_64-specific).
351
  GlobalVariable *VAArgOverflowSizeTLS;
352
  /// \brief Thread-local space used to pass origin value to the UMR reporting
353
  /// function.
354
  GlobalVariable *OriginTLS;
355
356
  /// \brief The run-time callback to print a warning.
357
  Value *WarningFn;
358
  // These arrays are indexed by log2(AccessSize).
359
  Value *MaybeWarningFn[kNumberOfAccessSizes];
360
  Value *MaybeStoreOriginFn[kNumberOfAccessSizes];
361
362
  /// \brief Run-time helper that generates a new origin value for a stack
363
  /// allocation.
364
  Value *MsanSetAllocaOrigin4Fn;
365
  /// \brief Run-time helper that poisons stack on function entry.
366
  Value *MsanPoisonStackFn;
367
  /// \brief Run-time helper that records a store (or any event) of an
368
  /// uninitialized value and returns an updated origin id encoding this info.
369
  Value *MsanChainOriginFn;
370
  /// \brief MSan runtime replacements for memmove, memcpy and memset.
371
  Value *MemmoveFn, *MemcpyFn, *MemsetFn;
372
373
  /// \brief Memory map parameters used in application-to-shadow calculation.
374
  const MemoryMapParams *MapParams;
375
376
  MDNode *ColdCallWeights;
377
  /// \brief Branch weights for origin store.
378
  MDNode *OriginStoreWeights;
379
  /// \brief An empty volatile inline asm that prevents callback merge.
380
  InlineAsm *EmptyAsm;
381
  Function *MsanCtorFunction;
382
383
  friend struct MemorySanitizerVisitor;
384
  friend struct VarArgAMD64Helper;
385
  friend struct VarArgMIPS64Helper;
386
  friend struct VarArgAArch64Helper;
387
  friend struct VarArgPowerPC64Helper;
388
};
389
} // anonymous namespace
390
391
char MemorySanitizer::ID = 0;
392
7.91k
INITIALIZE_PASS_BEGIN7.91k
(
393
7.91k
    MemorySanitizer, "msan",
394
7.91k
    "MemorySanitizer: detects uninitialized reads.", false, false)
395
7.91k
INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
396
7.91k
INITIALIZE_PASS_END(
397
    MemorySanitizer, "msan",
398
    "MemorySanitizer: detects uninitialized reads.", false, false)
399
400
16
FunctionPass *llvm::createMemorySanitizerPass(int TrackOrigins, bool Recover) {
401
16
  return new MemorySanitizer(TrackOrigins, Recover);
402
16
}
403
404
/// \brief Create a non-const global initialized with the given string.
405
///
406
/// Creates a writable global for Str so that we can pass it to the
407
/// run-time lib. Runtime uses first 4 bytes of the string to store the
408
/// frame ID, so the string needs to be mutable.
409
static GlobalVariable *createPrivateNonConstGlobalForString(Module &M,
410
13
                                                            StringRef Str) {
411
13
  Constant *StrConst = ConstantDataArray::getString(M.getContext(), Str);
412
13
  return new GlobalVariable(M, StrConst->getType(), /*isConstant=*/false,
413
13
                            GlobalValue::PrivateLinkage, StrConst, "");
414
13
}
415
416
/// \brief Insert extern declaration of runtime-provided functions and globals.
417
373
void MemorySanitizer::initializeCallbacks(Module &M) {
418
373
  // Only do this once.
419
373
  if (WarningFn)
420
309
    return;
421
64
422
64
  IRBuilder<> IRB(*C);
423
64
  // Create the callback.
424
64
  // FIXME: this function should have "Cold" calling conv,
425
64
  // which is not yet implemented.
426
0
  StringRef WarningFnName = Recover ? "__msan_warning"
427
64
                                    : "__msan_warning_noreturn";
428
64
  WarningFn = M.getOrInsertFunction(WarningFnName, IRB.getVoidTy());
429
64
430
320
  for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes;
431
256
       
AccessSizeIndex++256
) {
432
256
    unsigned AccessSize = 1 << AccessSizeIndex;
433
256
    std::string FunctionName = "__msan_maybe_warning_" + itostr(AccessSize);
434
256
    MaybeWarningFn[AccessSizeIndex] = M.getOrInsertFunction(
435
256
        FunctionName, IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8),
436
256
        IRB.getInt32Ty());
437
256
438
256
    FunctionName = "__msan_maybe_store_origin_" + itostr(AccessSize);
439
256
    MaybeStoreOriginFn[AccessSizeIndex] = M.getOrInsertFunction(
440
256
        FunctionName, IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8),
441
256
        IRB.getInt8PtrTy(), IRB.getInt32Ty());
442
256
  }
443
373
444
373
  MsanSetAllocaOrigin4Fn = M.getOrInsertFunction(
445
373
    "__msan_set_alloca_origin4", IRB.getVoidTy(), IRB.getInt8PtrTy(), IntptrTy,
446
373
    IRB.getInt8PtrTy(), IntptrTy);
447
373
  MsanPoisonStackFn =
448
373
      M.getOrInsertFunction("__msan_poison_stack", IRB.getVoidTy(),
449
373
                            IRB.getInt8PtrTy(), IntptrTy);
450
373
  MsanChainOriginFn = M.getOrInsertFunction(
451
373
    "__msan_chain_origin", IRB.getInt32Ty(), IRB.getInt32Ty());
452
373
  MemmoveFn = M.getOrInsertFunction(
453
373
    "__msan_memmove", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
454
373
    IRB.getInt8PtrTy(), IntptrTy);
455
373
  MemcpyFn = M.getOrInsertFunction(
456
373
    "__msan_memcpy", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
457
373
    IntptrTy);
458
373
  MemsetFn = M.getOrInsertFunction(
459
373
    "__msan_memset", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IRB.getInt32Ty(),
460
373
    IntptrTy);
461
373
462
373
  // Create globals.
463
373
  RetvalTLS = new GlobalVariable(
464
373
    M, ArrayType::get(IRB.getInt64Ty(), kRetvalTLSSize / 8), false,
465
373
    GlobalVariable::ExternalLinkage, nullptr, "__msan_retval_tls", nullptr,
466
373
    GlobalVariable::InitialExecTLSModel);
467
373
  RetvalOriginTLS = new GlobalVariable(
468
373
    M, OriginTy, false, GlobalVariable::ExternalLinkage, nullptr,
469
373
    "__msan_retval_origin_tls", nullptr, GlobalVariable::InitialExecTLSModel);
470
373
471
373
  ParamTLS = new GlobalVariable(
472
373
    M, ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8), false,
473
373
    GlobalVariable::ExternalLinkage, nullptr, "__msan_param_tls", nullptr,
474
373
    GlobalVariable::InitialExecTLSModel);
475
373
  ParamOriginTLS = new GlobalVariable(
476
373
    M, ArrayType::get(OriginTy, kParamTLSSize / 4), false,
477
373
    GlobalVariable::ExternalLinkage, nullptr, "__msan_param_origin_tls",
478
373
    nullptr, GlobalVariable::InitialExecTLSModel);
479
373
480
373
  VAArgTLS = new GlobalVariable(
481
373
    M, ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8), false,
482
373
    GlobalVariable::ExternalLinkage, nullptr, "__msan_va_arg_tls", nullptr,
483
373
    GlobalVariable::InitialExecTLSModel);
484
373
  VAArgOverflowSizeTLS = new GlobalVariable(
485
373
    M, IRB.getInt64Ty(), false, GlobalVariable::ExternalLinkage, nullptr,
486
373
    "__msan_va_arg_overflow_size_tls", nullptr,
487
373
    GlobalVariable::InitialExecTLSModel);
488
373
  OriginTLS = new GlobalVariable(
489
373
    M, IRB.getInt32Ty(), false, GlobalVariable::ExternalLinkage, nullptr,
490
373
    "__msan_origin_tls", nullptr, GlobalVariable::InitialExecTLSModel);
491
373
492
373
  // We insert an empty inline asm after __msan_report* to avoid callback merge.
493
373
  EmptyAsm = InlineAsm::get(FunctionType::get(IRB.getVoidTy(), false),
494
373
                            StringRef(""), StringRef(""),
495
373
                            /*hasSideEffects=*/true);
496
373
}
497
498
/// \brief Module-level initialization.
499
///
500
/// inserts a call to __msan_init to the module's constructor list.
501
64
bool MemorySanitizer::doInitialization(Module &M) {
502
64
  auto &DL = M.getDataLayout();
503
64
504
64
  Triple TargetTriple(M.getTargetTriple());
505
64
  switch (TargetTriple.getOS()) {
506
0
    case Triple::FreeBSD:
507
0
      switch (TargetTriple.getArch()) {
508
0
        case Triple::x86_64:
509
0
          MapParams = FreeBSD_X86_MemoryMapParams.bits64;
510
0
          break;
511
0
        case Triple::x86:
512
0
          MapParams = FreeBSD_X86_MemoryMapParams.bits32;
513
0
          break;
514
0
        default:
515
0
          report_fatal_error("unsupported architecture");
516
0
      }
517
0
      break;
518
64
    case Triple::Linux:
519
64
      switch (TargetTriple.getArch()) {
520
53
        case Triple::x86_64:
521
53
          MapParams = Linux_X86_MemoryMapParams.bits64;
522
53
          break;
523
1
        case Triple::x86:
524
1
          MapParams = Linux_X86_MemoryMapParams.bits32;
525
1
          break;
526
4
        case Triple::mips64:
527
4
        case Triple::mips64el:
528
4
          MapParams = Linux_MIPS_MemoryMapParams.bits64;
529
4
          break;
530
4
        case Triple::ppc64:
531
4
        case Triple::ppc64le:
532
4
          MapParams = Linux_PowerPC_MemoryMapParams.bits64;
533
4
          break;
534
2
        case Triple::aarch64:
535
2
        case Triple::aarch64_be:
536
2
          MapParams = Linux_ARM_MemoryMapParams.bits64;
537
2
          break;
538
0
        default:
539
0
          report_fatal_error("unsupported architecture");
540
64
      }
541
64
      break;
542
0
    default:
543
0
      report_fatal_error("unsupported operating system");
544
64
  }
545
64
546
64
  C = &(M.getContext());
547
64
  IRBuilder<> IRB(*C);
548
64
  IntptrTy = IRB.getIntPtrTy(DL);
549
64
  OriginTy = IRB.getInt32Ty();
550
64
551
64
  ColdCallWeights = MDBuilder(*C).createBranchWeights(1, 1000);
552
64
  OriginStoreWeights = MDBuilder(*C).createBranchWeights(1, 1000);
553
64
554
64
  std::tie(MsanCtorFunction, std::ignore) =
555
64
      createSanitizerCtorAndInitFunctions(M, kMsanModuleCtorName, kMsanInitName,
556
64
                                          /*InitArgTypes=*/{},
557
64
                                          /*InitArgs=*/{});
558
64
  if (
ClWithComdat64
) {
559
1
    Comdat *MsanCtorComdat = M.getOrInsertComdat(kMsanModuleCtorName);
560
1
    MsanCtorFunction->setComdat(MsanCtorComdat);
561
1
    appendToGlobalCtors(M, MsanCtorFunction, 0, MsanCtorFunction);
562
64
  } else {
563
63
    appendToGlobalCtors(M, MsanCtorFunction, 0);
564
63
  }
565
64
566
64
567
64
  if (TrackOrigins)
568
17
    new GlobalVariable(M, IRB.getInt32Ty(), true, GlobalValue::WeakODRLinkage,
569
17
                       IRB.getInt32(TrackOrigins), "__msan_track_origins");
570
64
571
64
  if (Recover)
572
0
    new GlobalVariable(M, IRB.getInt32Ty(), true, GlobalValue::WeakODRLinkage,
573
0
                       IRB.getInt32(Recover), "__msan_keep_going");
574
64
575
64
  return true;
576
64
}
577
578
namespace {
579
580
/// \brief A helper class that handles instrumentation of VarArg
581
/// functions on a particular platform.
582
///
583
/// Implementations are expected to insert the instrumentation
584
/// necessary to propagate argument shadow through VarArg function
585
/// calls. Visit* methods are called during an InstVisitor pass over
586
/// the function, and should avoid creating new basic blocks. A new
587
/// instance of this class is created for each instrumented function.
588
struct VarArgHelper {
589
  /// \brief Visit a CallSite.
590
  virtual void visitCallSite(CallSite &CS, IRBuilder<> &IRB) = 0;
591
592
  /// \brief Visit a va_start call.
593
  virtual void visitVAStartInst(VAStartInst &I) = 0;
594
595
  /// \brief Visit a va_copy call.
596
  virtual void visitVACopyInst(VACopyInst &I) = 0;
597
598
  /// \brief Finalize function instrumentation.
599
  ///
600
  /// This method is called after visiting all interesting (see above)
601
  /// instructions in a function.
602
  virtual void finalizeInstrumentation() = 0;
603
604
373
  virtual ~VarArgHelper() {}
605
};
606
607
struct MemorySanitizerVisitor;
608
609
VarArgHelper*
610
CreateVarArgHelper(Function &Func, MemorySanitizer &Msan,
611
                   MemorySanitizerVisitor &Visitor);
612
613
114
unsigned TypeSizeToSizeIndex(unsigned TypeSize) {
614
114
  if (
TypeSize <= 8114
)
return 018
;
615
96
  return Log2_32_Ceil((TypeSize + 7) / 8);
616
96
}
617
618
/// This class does all the work for a given function. Store and Load
619
/// instructions store and load corresponding shadow and origin
620
/// values. Most instructions propagate shadow from arguments to their
621
/// return values. Certain instructions (most importantly, BranchInst)
622
/// test their argument shadow and print reports (with a runtime call) if it's
623
/// non-zero.
624
struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
625
  Function &F;
626
  MemorySanitizer &MS;
627
  SmallVector<PHINode *, 16> ShadowPHINodes, OriginPHINodes;
628
  ValueMap<Value*, Value*> ShadowMap, OriginMap;
629
  std::unique_ptr<VarArgHelper> VAHelper;
630
  const TargetLibraryInfo *TLI;
631
632
  // The following flags disable parts of MSan instrumentation based on
633
  // blacklist contents and command-line options.
634
  bool InsertChecks;
635
  bool PropagateShadow;
636
  bool PoisonStack;
637
  bool PoisonUndef;
638
  bool CheckReturnValue;
639
640
  struct ShadowOriginAndInsertPoint {
641
    Value *Shadow;
642
    Value *Origin;
643
    Instruction *OrigIns;
644
    ShadowOriginAndInsertPoint(Value *S, Value *O, Instruction *I)
645
91
      : Shadow(S), Origin(O), OrigIns(I) { }
646
  };
647
  SmallVector<ShadowOriginAndInsertPoint, 16> InstrumentationList;
648
  SmallVector<StoreInst *, 16> StoreList;
649
650
  MemorySanitizerVisitor(Function &F, MemorySanitizer &MS)
651
373
      : F(F), MS(MS), VAHelper(CreateVarArgHelper(F, MS, *this)) {
652
373
    bool SanitizeFunction = F.hasFnAttribute(Attribute::SanitizeMemory);
653
373
    InsertChecks = SanitizeFunction;
654
373
    PropagateShadow = SanitizeFunction;
655
327
    PoisonStack = SanitizeFunction && ClPoisonStack;
656
327
    PoisonUndef = SanitizeFunction && ClPoisonUndef;
657
373
    // FIXME: Consider using SpecialCaseList to specify a list of functions that
658
373
    // must always return fully initialized values. For now, we hardcode "main".
659
327
    CheckReturnValue = SanitizeFunction && (F.getName() == "main");
660
373
    TLI = &MS.getAnalysis<TargetLibraryInfoWrapperPass>().getTLI();
661
373
662
373
    DEBUG(if (!InsertChecks)
663
373
          dbgs() << "MemorySanitizer is not inserting checks into '"
664
373
                 << F.getName() << "'\n");
665
373
  }
666
667
27
  Value *updateOrigin(Value *V, IRBuilder<> &IRB) {
668
27
    if (
MS.TrackOrigins <= 127
)
return V21
;
669
6
    return IRB.CreateCall(MS.MsanChainOriginFn, V);
670
6
  }
671
672
7
  Value *originToIntptr(IRBuilder<> &IRB, Value *Origin) {
673
7
    const DataLayout &DL = F.getParent()->getDataLayout();
674
7
    unsigned IntptrSize = DL.getTypeStoreSize(MS.IntptrTy);
675
7
    if (
IntptrSize == kOriginSize7
)
return Origin0
;
676
7
    assert(IntptrSize == kOriginSize * 2);
677
7
    Origin = IRB.CreateIntCast(Origin, MS.IntptrTy, /* isSigned */ false);
678
7
    return IRB.CreateOr(Origin, IRB.CreateShl(Origin, kOriginSize * 8));
679
7
  }
680
681
  /// \brief Fill memory range with the given origin value.
682
  void paintOrigin(IRBuilder<> &IRB, Value *Origin, Value *OriginPtr,
683
27
                   unsigned Size, unsigned Alignment) {
684
27
    const DataLayout &DL = F.getParent()->getDataLayout();
685
27
    unsigned IntptrAlignment = DL.getABITypeAlignment(MS.IntptrTy);
686
27
    unsigned IntptrSize = DL.getTypeStoreSize(MS.IntptrTy);
687
27
    assert(IntptrAlignment >= kMinOriginAlignment);
688
27
    assert(IntptrSize >= kOriginSize);
689
27
690
27
    unsigned Ofs = 0;
691
27
    unsigned CurrentAlignment = Alignment;
692
27
    if (
Alignment >= IntptrAlignment && 27
IntptrSize > kOriginSize7
) {
693
7
      Value *IntptrOrigin = originToIntptr(IRB, Origin);
694
7
      Value *IntptrOriginPtr =
695
7
          IRB.CreatePointerCast(OriginPtr, PointerType::get(MS.IntptrTy, 0));
696
14
      for (unsigned i = 0; 
i < Size / IntptrSize14
;
++i7
) {
697
3
        Value *Ptr = i ? IRB.CreateConstGEP1_32(MS.IntptrTy, IntptrOriginPtr, i)
698
4
                       : IntptrOriginPtr;
699
7
        IRB.CreateAlignedStore(IntptrOrigin, Ptr, CurrentAlignment);
700
7
        Ofs += IntptrSize / kOriginSize;
701
7
        CurrentAlignment = IntptrAlignment;
702
7
      }
703
7
    }
704
27
705
57
    for (unsigned i = Ofs; 
i < (Size + kOriginSize - 1) / kOriginSize57
;
++i30
) {
706
30
      Value *GEP =
707
30
          i ? 
IRB.CreateConstGEP1_32(nullptr, OriginPtr, i)7
:
OriginPtr23
;
708
30
      IRB.CreateAlignedStore(Origin, GEP, CurrentAlignment);
709
30
      CurrentAlignment = kMinOriginAlignment;
710
30
    }
711
27
  }
712
713
  void storeOrigin(IRBuilder<> &IRB, Value *Addr, Value *Shadow, Value *Origin,
714
30
                   unsigned Alignment, bool AsCall) {
715
30
    const DataLayout &DL = F.getParent()->getDataLayout();
716
30
    unsigned OriginAlignment = std::max(kMinOriginAlignment, Alignment);
717
30
    unsigned StoreSize = DL.getTypeStoreSize(Shadow->getType());
718
30
    if (
Shadow->getType()->isAggregateType()30
) {
719
4
      paintOrigin(IRB, updateOrigin(Origin, IRB),
720
4
                  getOriginPtr(Addr, IRB, Alignment), StoreSize,
721
4
                  OriginAlignment);
722
30
    } else {
723
26
      Value *ConvertedShadow = convertToShadowTyNoVec(Shadow, IRB);
724
26
      Constant *ConstantShadow = dyn_cast_or_null<Constant>(ConvertedShadow);
725
26
      if (
ConstantShadow26
) {
726
2
        if (
ClCheckConstantShadow && 2
!ConstantShadow->isZeroValue()2
)
727
1
          paintOrigin(IRB, updateOrigin(Origin, IRB),
728
1
                      getOriginPtr(Addr, IRB, Alignment), StoreSize,
729
1
                      OriginAlignment);
730
2
        return;
731
2
      }
732
24
733
24
      unsigned TypeSizeInBits =
734
24
          DL.getTypeSizeInBits(ConvertedShadow->getType());
735
24
      unsigned SizeIndex = TypeSizeToSizeIndex(TypeSizeInBits);
736
24
      if (
AsCall && 24
SizeIndex < kNumberOfAccessSizes2
) {
737
2
        Value *Fn = MS.MaybeStoreOriginFn[SizeIndex];
738
2
        Value *ConvertedShadow2 = IRB.CreateZExt(
739
2
            ConvertedShadow, IRB.getIntNTy(8 * (1 << SizeIndex)));
740
2
        IRB.CreateCall(Fn, {ConvertedShadow2,
741
2
                            IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy()),
742
2
                            Origin});
743
24
      } else {
744
22
        Value *Cmp = IRB.CreateICmpNE(
745
22
            ConvertedShadow, getCleanShadow(ConvertedShadow), "_mscmp");
746
22
        Instruction *CheckTerm = SplitBlockAndInsertIfThen(
747
22
            Cmp, &*IRB.GetInsertPoint(), false, MS.OriginStoreWeights);
748
22
        IRBuilder<> IRBNew(CheckTerm);
749
22
        paintOrigin(IRBNew, updateOrigin(Origin, IRBNew),
750
22
                    getOriginPtr(Addr, IRBNew, Alignment), StoreSize,
751
22
                    OriginAlignment);
752
22
      }
753
26
    }
754
30
  }
755
756
373
  void materializeStores(bool InstrumentWithCalls) {
757
126
    for (StoreInst *SI : StoreList) {
758
126
      IRBuilder<> IRB(SI);
759
126
      Value *Val = SI->getValueOperand();
760
126
      Value *Addr = SI->getPointerOperand();
761
126
      Value *Shadow = SI->isAtomic() ? 
getCleanShadow(Val)12
:
getShadow(Val)114
;
762
126
      Value *ShadowPtr = getShadowPtr(Addr, Shadow->getType(), IRB);
763
126
764
126
      StoreInst *NewSI =
765
126
          IRB.CreateAlignedStore(Shadow, ShadowPtr, SI->getAlignment());
766
126
      DEBUG(dbgs() << "  STORE: " << *NewSI << "\n");
767
126
      (void)NewSI;
768
126
769
126
      if (ClCheckAccessAddress)
770
73
        insertShadowCheck(Addr, SI);
771
126
772
126
      if (SI->isAtomic())
773
12
        SI->setOrdering(addReleaseOrdering(SI->getOrdering()));
774
126
775
126
      if (
MS.TrackOrigins && 126
!SI->isAtomic()38
)
776
30
        storeOrigin(IRB, Addr, Shadow, getOrigin(Val), SI->getAlignment(),
777
30
                    InstrumentWithCalls);
778
126
    }
779
373
  }
780
781
  void materializeOneCheck(Instruction *OrigIns, Value *Shadow, Value *Origin,
782
91
                           bool AsCall) {
783
91
    IRBuilder<> IRB(OrigIns);
784
91
    DEBUG(dbgs() << "  SHAD0 : " << *Shadow << "\n");
785
91
    Value *ConvertedShadow = convertToShadowTyNoVec(Shadow, IRB);
786
91
    DEBUG(dbgs() << "  SHAD1 : " << *ConvertedShadow << "\n");
787
91
788
91
    Constant *ConstantShadow = dyn_cast_or_null<Constant>(ConvertedShadow);
789
91
    if (
ConstantShadow91
) {
790
1
      if (
ClCheckConstantShadow && 1
!ConstantShadow->isZeroValue()1
) {
791
1
        if (
MS.TrackOrigins1
) {
792
1
          IRB.CreateStore(Origin ? 
(Value *)Origin1
:
(Value *)IRB.getInt32(0)0
,
793
1
                          MS.OriginTLS);
794
1
        }
795
1
        IRB.CreateCall(MS.WarningFn, {});
796
1
        IRB.CreateCall(MS.EmptyAsm, {});
797
1
        // FIXME: Insert UnreachableInst if !MS.Recover?
798
1
        // This may invalidate some of the following checks and needs to be done
799
1
        // at the very end.
800
1
      }
801
1
      return;
802
1
    }
803
90
804
90
    const DataLayout &DL = OrigIns->getModule()->getDataLayout();
805
90
806
90
    unsigned TypeSizeInBits = DL.getTypeSizeInBits(ConvertedShadow->getType());
807
90
    unsigned SizeIndex = TypeSizeToSizeIndex(TypeSizeInBits);
808
90
    if (
AsCall && 90
SizeIndex < kNumberOfAccessSizes12
) {
809
11
      Value *Fn = MS.MaybeWarningFn[SizeIndex];
810
11
      Value *ConvertedShadow2 =
811
11
          IRB.CreateZExt(ConvertedShadow, IRB.getIntNTy(8 * (1 << SizeIndex)));
812
2
      IRB.CreateCall(Fn, {ConvertedShadow2, MS.TrackOrigins && Origin
813
2
                                                ? Origin
814
9
                                                : (Value *)IRB.getInt32(0)});
815
90
    } else {
816
79
      Value *Cmp = IRB.CreateICmpNE(ConvertedShadow,
817
79
                                    getCleanShadow(ConvertedShadow), "_mscmp");
818
79
      Instruction *CheckTerm = SplitBlockAndInsertIfThen(
819
79
          Cmp, OrigIns,
820
79
          /* Unreachable */ !MS.Recover, MS.ColdCallWeights);
821
79
822
79
      IRB.SetInsertPoint(CheckTerm);
823
79
      if (
MS.TrackOrigins79
) {
824
9
        IRB.CreateStore(Origin ? 
(Value *)Origin9
:
(Value *)IRB.getInt32(0)0
,
825
9
                        MS.OriginTLS);
826
9
      }
827
79
      IRB.CreateCall(MS.WarningFn, {});
828
79
      IRB.CreateCall(MS.EmptyAsm, {});
829
79
      DEBUG(dbgs() << "  CHECK: " << *Cmp << "\n");
830
79
    }
831
91
  }
832
833
373
  void materializeChecks(bool InstrumentWithCalls) {
834
91
    for (const auto &ShadowData : InstrumentationList) {
835
91
      Instruction *OrigIns = ShadowData.OrigIns;
836
91
      Value *Shadow = ShadowData.Shadow;
837
91
      Value *Origin = ShadowData.Origin;
838
91
      materializeOneCheck(OrigIns, Shadow, Origin, InstrumentWithCalls);
839
91
    }
840
373
    DEBUG(dbgs() << "DONE:\n" << F);
841
373
  }
842
843
  /// \brief Add MemorySanitizer instrumentation to a function.
844
373
  bool runOnFunction() {
845
373
    MS.initializeCallbacks(*F.getParent());
846
373
847
373
    // In the presence of unreachable blocks, we may see Phi nodes with
848
373
    // incoming nodes from such blocks. Since InstVisitor skips unreachable
849
373
    // blocks, such nodes will not have any shadow value associated with them.
850
373
    // It's easier to remove unreachable blocks than deal with missing shadow.
851
373
    removeUnreachableBlocks(F);
852
373
853
373
    // Iterate all BBs in depth-first order and create shadow instructions
854
373
    // for all instructions (where applicable).
855
373
    // For PHI nodes we create dummy shadow PHIs which will be finalized later.
856
373
    for (BasicBlock *BB : depth_first(&F.getEntryBlock()))
857
404
      visit(*BB);
858
373
859
373
860
373
    // Finalize PHI nodes.
861
2
    for (PHINode *PN : ShadowPHINodes) {
862
2
      PHINode *PNS = cast<PHINode>(getShadow(PN));
863
2
      PHINode *PNO = MS.TrackOrigins ? 
cast<PHINode>(getOrigin(PN))1
:
nullptr1
;
864
2
      size_t NumValues = PN->getNumIncomingValues();
865
6
      for (size_t v = 0; 
v < NumValues6
;
v++4
) {
866
4
        PNS->addIncoming(getShadow(PN, v), PN->getIncomingBlock(v));
867
4
        if (
PNO4
)
PNO->addIncoming(getOrigin(PN, v), PN->getIncomingBlock(v))2
;
868
4
      }
869
2
    }
870
373
871
373
    VAHelper->finalizeInstrumentation();
872
373
873
373
    bool InstrumentWithCalls = ClInstrumentationWithCallThreshold >= 0 &&
874
373
                               InstrumentationList.size() + StoreList.size() >
875
373
                                   (unsigned)ClInstrumentationWithCallThreshold;
876
373
877
373
    // Delayed instrumentation of StoreInst.
878
373
    // This may add new checks to be inserted later.
879
373
    materializeStores(InstrumentWithCalls);
880
373
881
373
    // Insert shadow value checks.
882
373
    materializeChecks(InstrumentWithCalls);
883
373
884
373
    return true;
885
373
  }
886
887
  /// \brief Compute the shadow type that corresponds to a given Value.
888
1.14k
  Type *getShadowTy(Value *V) {
889
1.14k
    return getShadowTy(V->getType());
890
1.14k
  }
891
892
  /// \brief Compute the shadow type that corresponds to a given Type.
893
2.25k
  Type *getShadowTy(Type *OrigTy) {
894
2.25k
    if (
!OrigTy->isSized()2.25k
) {
895
89
      return nullptr;
896
89
    }
897
2.17k
    // For integer type, shadow is the same as the original type.
898
2.17k
    // This may return weird-sized types like i1.
899
2.17k
    
if (IntegerType *2.17k
IT2.17k
= dyn_cast<IntegerType>(OrigTy))
900
1.15k
      return IT;
901
1.01k
    const DataLayout &DL = F.getParent()->getDataLayout();
902
1.01k
    if (VectorType *
VT1.01k
= dyn_cast<VectorType>(OrigTy)) {
903
187
      uint32_t EltSize = DL.getTypeSizeInBits(VT->getElementType());
904
187
      return VectorType::get(IntegerType::get(*MS.C, EltSize),
905
187
                             VT->getNumElements());
906
187
    }
907
829
    
if (ArrayType *829
AT829
= dyn_cast<ArrayType>(OrigTy)) {
908
39
      return ArrayType::get(getShadowTy(AT->getElementType()),
909
39
                            AT->getNumElements());
910
39
    }
911
790
    
if (StructType *790
ST790
= dyn_cast<StructType>(OrigTy)) {
912
42
      SmallVector<Type*, 4> Elements;
913
135
      for (unsigned i = 0, n = ST->getNumElements(); 
i < n135
;
i++93
)
914
93
        Elements.push_back(getShadowTy(ST->getElementType(i)));
915
42
      StructType *Res = StructType::get(*MS.C, Elements, ST->isPacked());
916
42
      DEBUG(dbgs() << "getShadowTy: " << *ST << " ===> " << *Res << "\n");
917
42
      return Res;
918
42
    }
919
748
    uint32_t TypeSize = DL.getTypeSizeInBits(OrigTy);
920
748
    return IntegerType::get(*MS.C, TypeSize);
921
748
  }
922
923
  /// \brief Flatten a vector type.
924
129
  Type *getShadowTyNoVec(Type *ty) {
925
129
    if (VectorType *vt = dyn_cast<VectorType>(ty))
926
4
      return IntegerType::get(*MS.C, vt->getBitWidth());
927
125
    return ty;
928
125
  }
929
930
  /// \brief Convert a shadow value to it's flattened variant.
931
128
  Value *convertToShadowTyNoVec(Value *V, IRBuilder<> &IRB) {
932
128
    Type *Ty = V->getType();
933
128
    Type *NoVecTy = getShadowTyNoVec(Ty);
934
128
    if (
Ty == NoVecTy128
)
return V125
;
935
3
    return IRB.CreateBitCast(V, NoVecTy);
936
3
  }
937
938
  /// \brief Compute the integer shadow offset that corresponds to a given
939
  /// application address.
940
  ///
941
  /// Offset = (Addr & ~AndMask) ^ XorMask
942
405
  Value *getShadowPtrOffset(Value *Addr, IRBuilder<> &IRB) {
943
405
    Value *OffsetLong = IRB.CreatePointerCast(Addr, MS.IntptrTy);
944
405
945
405
    uint64_t AndMask = MS.MapParams->AndMask;
946
405
    if (AndMask)
947
23
      OffsetLong =
948
23
          IRB.CreateAnd(OffsetLong, ConstantInt::get(MS.IntptrTy, ~AndMask));
949
405
950
405
    uint64_t XorMask = MS.MapParams->XorMask;
951
405
    if (XorMask)
952
404
      OffsetLong =
953
404
          IRB.CreateXor(OffsetLong, ConstantInt::get(MS.IntptrTy, XorMask));
954
405
    return OffsetLong;
955
405
  }
956
957
  /// \brief Compute the shadow address that corresponds to a given application
958
  /// address.
959
  ///
960
  /// Shadow = ShadowBase + Offset
961
  Value *getShadowPtr(Value *Addr, Type *ShadowTy,
962
356
                      IRBuilder<> &IRB) {
963
356
    Value *ShadowLong = getShadowPtrOffset(Addr, IRB);
964
356
    uint64_t ShadowBase = MS.MapParams->ShadowBase;
965
356
    if (ShadowBase != 0)
966
22
      ShadowLong =
967
22
        IRB.CreateAdd(ShadowLong,
968
22
                      ConstantInt::get(MS.IntptrTy, ShadowBase));
969
356
    return IRB.CreateIntToPtr(ShadowLong, PointerType::get(ShadowTy, 0));
970
356
  }
971
972
  /// \brief Compute the origin address that corresponds to a given application
973
  /// address.
974
  ///
975
  /// OriginAddr = (OriginBase + Offset) & ~3ULL
976
49
  Value *getOriginPtr(Value *Addr, IRBuilder<> &IRB, unsigned Alignment) {
977
49
    Value *OriginLong = getShadowPtrOffset(Addr, IRB);
978
49
    uint64_t OriginBase = MS.MapParams->OriginBase;
979
49
    if (OriginBase != 0)
980
49
      OriginLong =
981
49
        IRB.CreateAdd(OriginLong,
982
49
                      ConstantInt::get(MS.IntptrTy, OriginBase));
983
49
    if (
Alignment < kMinOriginAlignment49
) {
984
9
      uint64_t Mask = kMinOriginAlignment - 1;
985
9
      OriginLong = IRB.CreateAnd(OriginLong,
986
9
                                 ConstantInt::get(MS.IntptrTy, ~Mask));
987
9
    }
988
49
    return IRB.CreateIntToPtr(OriginLong,
989
49
                              PointerType::get(IRB.getInt32Ty(), 0));
990
49
  }
991
992
  /// \brief Compute the shadow address for a given function argument.
993
  ///
994
  /// Shadow = ParamTLS+ArgOffset.
995
  Value *getShadowPtrForArgument(Value *A, IRBuilder<> &IRB,
996
504
                                 int ArgOffset) {
997
504
    Value *Base = IRB.CreatePointerCast(MS.ParamTLS, MS.IntptrTy);
998
504
    Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
999
504
    return IRB.CreateIntToPtr(Base, PointerType::get(getShadowTy(A), 0),
1000
504
                              "_msarg");
1001
504
  }
1002
1003
  /// \brief Compute the origin address for a given function argument.
1004
  Value *getOriginPtrForArgument(Value *A, IRBuilder<> &IRB,
1005
97
                                 int ArgOffset) {
1006
97
    if (
!MS.TrackOrigins97
)
return nullptr0
;
1007
97
    Value *Base = IRB.CreatePointerCast(MS.ParamOriginTLS, MS.IntptrTy);
1008
97
    Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
1009
97
    return IRB.CreateIntToPtr(Base, PointerType::get(MS.OriginTy, 0),
1010
97
                              "_msarg_o");
1011
97
  }
1012
1013
  /// \brief Compute the shadow address for a retval.
1014
283
  Value *getShadowPtrForRetval(Value *A, IRBuilder<> &IRB) {
1015
283
    Value *Base = IRB.CreatePointerCast(MS.RetvalTLS, MS.IntptrTy);
1016
283
    return IRB.CreateIntToPtr(Base, PointerType::get(getShadowTy(A), 0),
1017
283
                              "_msret");
1018
283
  }
1019
1020
  /// \brief Compute the origin address for a retval.
1021
65
  Value *getOriginPtrForRetval(IRBuilder<> &IRB) {
1022
65
    // We keep a single origin for the entire retval. Might be too optimistic.
1023
65
    return MS.RetvalOriginTLS;
1024
65
  }
1025
1026
  /// \brief Set SV to be the shadow value for V.
1027
626
  void setShadow(Value *V, Value *SV) {
1028
626
    assert(!ShadowMap.count(V) && "Values may only have one shadow");
1029
626
    ShadowMap[V] = PropagateShadow ? 
SV543
:
getCleanShadow(V)83
;
1030
626
  }
1031
1032
  /// \brief Set Origin to be the origin value for V.
1033
679
  void setOrigin(Value *V, Value *Origin) {
1034
679
    if (
!MS.TrackOrigins679
)
return459
;
1035
679
    assert(!OriginMap.count(V) && "Values may only have one origin");
1036
220
    DEBUG(dbgs() << "ORIGIN: " << *V << "  ==> " << *Origin << "\n");
1037
679
    OriginMap[V] = Origin;
1038
679
  }
1039
1040
930
  Constant *getCleanShadow(Type *OrigTy) {
1041
930
    Type *ShadowTy = getShadowTy(OrigTy);
1042
930
    if (!ShadowTy)
1043
89
      return nullptr;
1044
841
    return Constant::getNullValue(ShadowTy);
1045
841
  }
1046
1047
  /// \brief Create a clean shadow value for a given value.
1048
  ///
1049
  /// Clean shadow (all zeroes) means all bits of the value are defined
1050
  /// (initialized).
1051
928
  Constant *getCleanShadow(Value *V) {
1052
928
    return getCleanShadow(V->getType());
1053
928
  }
1054
1055
  /// \brief Create a dirty shadow of a given shadow type.
1056
40
  Constant *getPoisonedShadow(Type *ShadowTy) {
1057
40
    assert(ShadowTy);
1058
40
    if (
isa<IntegerType>(ShadowTy) || 40
isa<VectorType>(ShadowTy)16
)
1059
27
      return Constant::getAllOnesValue(ShadowTy);
1060
13
    
if (ArrayType *13
AT13
= dyn_cast<ArrayType>(ShadowTy)) {
1061
6
      SmallVector<Constant *, 4> Vals(AT->getNumElements(),
1062
6
                                      getPoisonedShadow(AT->getElementType()));
1063
6
      return ConstantArray::get(AT, Vals);
1064
6
    }
1065
7
    
if (StructType *7
ST7
= dyn_cast<StructType>(ShadowTy)) {
1066
7
      SmallVector<Constant *, 4> Vals;
1067
21
      for (unsigned i = 0, n = ST->getNumElements(); 
i < n21
;
i++14
)
1068
14
        Vals.push_back(getPoisonedShadow(ST->getElementType(i)));
1069
7
      return ConstantStruct::get(ST, Vals);
1070
7
    }
1071
0
    
llvm_unreachable0
("Unexpected shadow type");
1072
0
  }
1073
1074
  /// \brief Create a dirty shadow for a given value.
1075
16
  Constant *getPoisonedShadow(Value *V) {
1076
16
    Type *ShadowTy = getShadowTy(V);
1077
16
    if (!ShadowTy)
1078
0
      return nullptr;
1079
16
    return getPoisonedShadow(ShadowTy);
1080
16
  }
1081
1082
  /// \brief Create a clean (zero) origin.
1083
410
  Value *getCleanOrigin() {
1084
410
    return Constant::getNullValue(MS.OriginTy);
1085
410
  }
1086
1087
  /// \brief Get the shadow value for a given Value.
1088
  ///
1089
  /// This function either returns the value set earlier with setShadow,
1090
  /// or extracts if from ParamTLS (for function arguments).
1091
1.38k
  Value *getShadow(Value *V) {
1092
1.38k
    if (
!PropagateShadow1.38k
)
return getCleanShadow(V)240
;
1093
1.14k
    
if (Instruction *1.14k
I1.14k
= dyn_cast<Instruction>(V)) {
1094
528
      // For instructions the shadow is already stored in the map.
1095
528
      Value *Shadow = ShadowMap[V];
1096
528
      if (
!Shadow528
) {
1097
0
        DEBUG(dbgs() << "No shadow: " << *V << "\n" << *(I->getParent()));
1098
0
        (void)I;
1099
0
        assert(Shadow && "No shadow for a value");
1100
0
      }
1101
528
      return Shadow;
1102
528
    }
1103
619
    
if (UndefValue *619
U619
= dyn_cast<UndefValue>(V)) {
1104
16
      Value *AllOnes = PoisonUndef ? 
getPoisonedShadow(V)16
:
getCleanShadow(V)0
;
1105
16
      DEBUG(dbgs() << "Undef: " << *U << " ==> " << *AllOnes << "\n");
1106
16
      (void)U;
1107
16
      return AllOnes;
1108
16
    }
1109
603
    
if (Argument *603
A603
= dyn_cast<Argument>(V)) {
1110
378
      // For arguments we compute the shadow on demand and store it in the map.
1111
378
      Value **ShadowPtr = &ShadowMap[V];
1112
378
      if (*ShadowPtr)
1113
66
        return *ShadowPtr;
1114
312
      Function *F = A->getParent();
1115
312
      IRBuilder<> EntryIRB(F->getEntryBlock().getFirstNonPHI());
1116
312
      unsigned ArgOffset = 0;
1117
312
      const DataLayout &DL = F->getParent()->getDataLayout();
1118
561
      for (auto &FArg : F->args()) {
1119
561
        if (
!FArg.getType()->isSized()561
) {
1120
0
          DEBUG(dbgs() << "Arg is not sized\n");
1121
0
          continue;
1122
0
        }
1123
561
        unsigned Size =
1124
561
            FArg.hasByValAttr()
1125
2
                ? DL.getTypeAllocSize(FArg.getType()->getPointerElementType())
1126
559
                : DL.getTypeAllocSize(FArg.getType());
1127
561
        if (
A == &FArg561
) {
1128
312
          bool Overflow = ArgOffset + Size > kParamTLSSize;
1129
312
          Value *Base = getShadowPtrForArgument(&FArg, EntryIRB, ArgOffset);
1130
312
          if (
FArg.hasByValAttr()312
) {
1131
2
            // ByVal pointer itself has clean shadow. We copy the actual
1132
2
            // argument shadow to the underlying memory.
1133
2
            // Figure out maximal valid memcpy alignment.
1134
2
            unsigned ArgAlign = FArg.getParamAlignment();
1135
2
            if (
ArgAlign == 02
) {
1136
2
              Type *EltType = A->getType()->getPointerElementType();
1137
2
              ArgAlign = DL.getABITypeAlignment(EltType);
1138
2
            }
1139
2
            if (
Overflow2
) {
1140
0
              // ParamTLS overflow.
1141
0
              EntryIRB.CreateMemSet(
1142
0
                  getShadowPtr(V, EntryIRB.getInt8Ty(), EntryIRB),
1143
0
                  Constant::getNullValue(EntryIRB.getInt8Ty()), Size, ArgAlign);
1144
2
            } else {
1145
2
              unsigned CopyAlign = std::min(ArgAlign, kShadowTLSAlignment);
1146
2
              Value *Cpy = EntryIRB.CreateMemCpy(
1147
2
                  getShadowPtr(V, EntryIRB.getInt8Ty(), EntryIRB), Base, Size,
1148
2
                  CopyAlign);
1149
2
              DEBUG(dbgs() << "  ByValCpy: " << *Cpy << "\n");
1150
2
              (void)Cpy;
1151
2
            }
1152
2
            *ShadowPtr = getCleanShadow(V);
1153
312
          } else {
1154
310
            if (
Overflow310
) {
1155
0
              // ParamTLS overflow.
1156
0
              *ShadowPtr = getCleanShadow(V);
1157
310
            } else {
1158
310
              *ShadowPtr =
1159
310
                  EntryIRB.CreateAlignedLoad(Base, kShadowTLSAlignment);
1160
310
            }
1161
310
          }
1162
312
          DEBUG(dbgs() << "  ARG:    "  << FArg << " ==> " <<
1163
312
                **ShadowPtr << "\n");
1164
312
          if (
MS.TrackOrigins && 312
!Overflow88
) {
1165
88
            Value *OriginPtr =
1166
88
                getOriginPtrForArgument(&FArg, EntryIRB, ArgOffset);
1167
88
            setOrigin(A, EntryIRB.CreateLoad(OriginPtr));
1168
312
          } else {
1169
224
            setOrigin(A, getCleanOrigin());
1170
224
          }
1171
312
        }
1172
561
        ArgOffset += alignTo(Size, kShadowTLSAlignment);
1173
561
      }
1174
378
      assert(*ShadowPtr && "Could not find shadow for an argument");
1175
378
      return *ShadowPtr;
1176
378
    }
1177
225
    // For everything else the shadow is zero.
1178
225
    return getCleanShadow(V);
1179
225
  }
1180
1181
  /// \brief Get the shadow for i-th argument of the instruction I.
1182
201
  Value *getShadow(Instruction *I, int i) {
1183
201
    return getShadow(I->getOperand(i));
1184
201
  }
1185
1186
  /// \brief Get the origin for a value.
1187
415
  Value *getOrigin(Value *V) {
1188
415
    if (
!MS.TrackOrigins415
)
return nullptr206
;
1189
209
    
if (209
!PropagateShadow209
)
return getCleanOrigin()9
;
1190
200
    
if (200
isa<Constant>(V)200
)
return getCleanOrigin()22
;
1191
200
    assert((isa<Instruction>(V) || isa<Argument>(V)) &&
1192
178
           "Unexpected value type in getOrigin()");
1193
178
    Value *Origin = OriginMap[V];
1194
178
    assert(Origin && "Missing origin");
1195
178
    return Origin;
1196
178
  }
1197
1198
  /// \brief Get the origin for i-th argument of the instruction I.
1199
121
  Value *getOrigin(Instruction *I, int i) {
1200
121
    return getOrigin(I->getOperand(i));
1201
121
  }
1202
1203
  /// \brief Remember the place where a shadow check should be inserted.
1204
  ///
1205
  /// This location will be later instrumented with a check that will print a
1206
  /// UMR warning in runtime if the shadow value is not 0.
1207
91
  void insertShadowCheck(Value *Shadow, Value *Origin, Instruction *OrigIns) {
1208
91
    assert(Shadow);
1209
91
    if (
!InsertChecks91
)
return0
;
1210
#ifndef NDEBUG
1211
    Type *ShadowTy = Shadow->getType();
1212
    assert((isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy)) &&
1213
           "Can only insert checks for integer and vector shadow types");
1214
#endif
1215
0
    InstrumentationList.push_back(
1216
91
        ShadowOriginAndInsertPoint(Shadow, Origin, OrigIns));
1217
91
  }
1218
1219
  /// \brief Remember the place where a shadow check should be inserted.
1220
  ///
1221
  /// This location will be later instrumented with a check that will print a
1222
  /// UMR warning in runtime if the value is not fully defined.
1223
263
  void insertShadowCheck(Value *Val, Instruction *OrigIns) {
1224
263
    assert(Val);
1225
263
    Value *Shadow, *Origin;
1226
263
    if (
ClCheckConstantShadow263
) {
1227
1
      Shadow = getShadow(Val);
1228
1
      if (
!Shadow1
)
return0
;
1229
1
      Origin = getOrigin(Val);
1230
263
    } else {
1231
262
      Shadow = dyn_cast_or_null<Instruction>(getShadow(Val));
1232
262
      if (
!Shadow262
)
return177
;
1233
85
      Origin = dyn_cast_or_null<Instruction>(getOrigin(Val));
1234
85
    }
1235
86
    insertShadowCheck(Shadow, Origin, OrigIns);
1236
86
  }
1237
1238
24
  AtomicOrdering addReleaseOrdering(AtomicOrdering a) {
1239
24
    switch (a) {
1240
0
      case AtomicOrdering::NotAtomic:
1241
0
        return AtomicOrdering::NotAtomic;
1242
12
      case AtomicOrdering::Unordered:
1243
12
      case AtomicOrdering::Monotonic:
1244
12
      case AtomicOrdering::Release:
1245
12
        return AtomicOrdering::Release;
1246
0
      case AtomicOrdering::Acquire:
1247
0
      case AtomicOrdering::AcquireRelease:
1248
0
        return AtomicOrdering::AcquireRelease;
1249
12
      case AtomicOrdering::SequentiallyConsistent:
1250
12
        return AtomicOrdering::SequentiallyConsistent;
1251
0
    }
1252
0
    
llvm_unreachable0
("Unknown ordering");
1253
0
  }
1254
1255
12
  AtomicOrdering addAcquireOrdering(AtomicOrdering a) {
1256
12
    switch (a) {
1257
0
      case AtomicOrdering::NotAtomic:
1258
0
        return AtomicOrdering::NotAtomic;
1259
9
      case AtomicOrdering::Unordered:
1260
9
      case AtomicOrdering::Monotonic:
1261
9
      case AtomicOrdering::Acquire:
1262
9
        return AtomicOrdering::Acquire;
1263
0
      case AtomicOrdering::Release:
1264
0
      case AtomicOrdering::AcquireRelease:
1265
0
        return AtomicOrdering::AcquireRelease;
1266
3
      case AtomicOrdering::SequentiallyConsistent:
1267
3
        return AtomicOrdering::SequentiallyConsistent;
1268
0
    }
1269
0
    
llvm_unreachable0
("Unknown ordering");
1270
0
  }
1271
1272
  // ------------------- Visitors.
1273
1274
  /// \brief Instrument LoadInst
1275
  ///
1276
  /// Loads the corresponding shadow and (optionally) origin.
1277
  /// Optionally, checks that the load address is fully defined.
1278
107
  void visitLoadInst(LoadInst &I) {
1279
107
    assert(I.getType()->isSized() && "Load type must have size");
1280
107
    IRBuilder<> IRB(I.getNextNode());
1281
107
    Type *ShadowTy = getShadowTy(&I);
1282
107
    Value *Addr = I.getPointerOperand();
1283
107
    if (
PropagateShadow && 107
!I.getMetadata("nosanitize")103
) {
1284
103
      Value *ShadowPtr = getShadowPtr(Addr, ShadowTy, IRB);
1285
103
      setShadow(&I,
1286
103
                IRB.CreateAlignedLoad(ShadowPtr, I.getAlignment(), "_msld"));
1287
107
    } else {
1288
4
      setShadow(&I, getCleanShadow(&I));
1289
4
    }
1290
107
1291
107
    if (ClCheckAccessAddress)
1292
68
      insertShadowCheck(I.getPointerOperand(), &I);
1293
107
1294
107
    if (I.isAtomic())
1295
12
      I.setOrdering(addAcquireOrdering(I.getOrdering()));
1296
107
1297
107
    if (
MS.TrackOrigins107
) {
1298
22
      if (
PropagateShadow22
) {
1299
21
        unsigned Alignment = I.getAlignment();
1300
21
        unsigned OriginAlignment = std::max(kMinOriginAlignment, Alignment);
1301
21
        setOrigin(&I, IRB.CreateAlignedLoad(getOriginPtr(Addr, IRB, Alignment),
1302
21
                                            OriginAlignment));
1303
22
      } else {
1304
1
        setOrigin(&I, getCleanOrigin());
1305
1
      }
1306
22
    }
1307
107
  }
1308
1309
  /// \brief Instrument StoreInst
1310
  ///
1311
  /// Stores the corresponding shadow and (optionally) origin.
1312
  /// Optionally, checks that the store address is fully defined.
1313
126
  void visitStoreInst(StoreInst &I) {
1314
126
    StoreList.push_back(&I);
1315
126
  }
1316
1317
12
  void handleCASOrRMW(Instruction &I) {
1318
12
    assert(isa<AtomicRMWInst>(I) || isa<AtomicCmpXchgInst>(I));
1319
12
1320
12
    IRBuilder<> IRB(&I);
1321
12
    Value *Addr = I.getOperand(0);
1322
12
    Value *ShadowPtr = getShadowPtr(Addr, I.getType(), IRB);
1323
12
1324
12
    if (ClCheckAccessAddress)
1325
0
      insertShadowCheck(Addr, &I);
1326
12
1327
12
    // Only test the conditional argument of cmpxchg instruction.
1328
12
    // The other argument can potentially be uninitialized, but we can not
1329
12
    // detect this situation reliably without possible false positives.
1330
12
    if (isa<AtomicCmpXchgInst>(I))
1331
6
      insertShadowCheck(I.getOperand(1), &I);
1332
12
1333
12
    IRB.CreateStore(getCleanShadow(&I), ShadowPtr);
1334
12
1335
12
    setShadow(&I, getCleanShadow(&I));
1336
12
    setOrigin(&I, getCleanOrigin());
1337
12
  }
1338
1339
6
  void visitAtomicRMWInst(AtomicRMWInst &I) {
1340
6
    handleCASOrRMW(I);
1341
6
    I.setOrdering(addReleaseOrdering(I.getOrdering()));
1342
6
  }
1343
1344
6
  void visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) {
1345
6
    handleCASOrRMW(I);
1346
6
    I.setSuccessOrdering(addReleaseOrdering(I.getSuccessOrdering()));
1347
6
  }
1348
1349
  // Vector manipulation.
1350
3
  void visitExtractElementInst(ExtractElementInst &I) {
1351
3
    insertShadowCheck(I.getOperand(1), &I);
1352
3
    IRBuilder<> IRB(&I);
1353
3
    setShadow(&I, IRB.CreateExtractElement(getShadow(&I, 0), I.getOperand(1),
1354
3
              "_msprop"));
1355
3
    setOrigin(&I, getOrigin(&I, 0));
1356
3
  }
1357
1358
12
  void visitInsertElementInst(InsertElementInst &I) {
1359
12
    insertShadowCheck(I.getOperand(2), &I);
1360
12
    IRBuilder<> IRB(&I);
1361
12
    setShadow(&I, IRB.CreateInsertElement(getShadow(&I, 0), getShadow(&I, 1),
1362
12
              I.getOperand(2), "_msprop"));
1363
12
    setOriginForNaryOp(I);
1364
12
  }
1365
1366
3
  void visitShuffleVectorInst(ShuffleVectorInst &I) {
1367
3
    insertShadowCheck(I.getOperand(2), &I);
1368
3
    IRBuilder<> IRB(&I);
1369
3
    setShadow(&I, IRB.CreateShuffleVector(getShadow(&I, 0), getShadow(&I, 1),
1370
3
              I.getOperand(2), "_msprop"));
1371
3
    setOriginForNaryOp(I);
1372
3
  }
1373
1374
  // Casts.
1375
2
  void visitSExtInst(SExtInst &I) {
1376
2
    IRBuilder<> IRB(&I);
1377
2
    setShadow(&I, IRB.CreateSExt(getShadow(&I, 0), I.getType(), "_msprop"));
1378
2
    setOrigin(&I, getOrigin(&I, 0));
1379
2
  }
1380
1381
0
  void visitZExtInst(ZExtInst &I) {
1382
0
    IRBuilder<> IRB(&I);
1383
0
    setShadow(&I, IRB.CreateZExt(getShadow(&I, 0), I.getType(), "_msprop"));
1384
0
    setOrigin(&I, getOrigin(&I, 0));
1385
0
  }
1386
1387
0
  void visitTruncInst(TruncInst &I) {
1388
0
    IRBuilder<> IRB(&I);
1389
0
    setShadow(&I, IRB.CreateTrunc(getShadow(&I, 0), I.getType(), "_msprop"));
1390
0
    setOrigin(&I, getOrigin(&I, 0));
1391
0
  }
1392
1393
109
  void visitBitCastInst(BitCastInst &I) {
1394
109
    // Special case: if this is the bitcast (there is exactly 1 allowed) between
1395
109
    // a musttail call and a ret, don't instrument. New instructions are not
1396
109
    // allowed after a musttail call.
1397
109
    if (auto *CI = dyn_cast<CallInst>(I.getOperand(0)))
1398
4
      
if (4
CI->isMustTailCall()4
)
1399
2
        return;
1400
107
    IRBuilder<> IRB(&I);
1401
107
    setShadow(&I, IRB.CreateBitCast(getShadow(&I, 0), getShadowTy(&I)));
1402
107
    setOrigin(&I, getOrigin(&I, 0));
1403
107
  }
1404
1405
0
  void visitPtrToIntInst(PtrToIntInst &I) {
1406
0
    IRBuilder<> IRB(&I);
1407
0
    setShadow(&I, IRB.CreateIntCast(getShadow(&I, 0), getShadowTy(&I), false,
1408
0
             "_msprop_ptrtoint"));
1409
0
    setOrigin(&I, getOrigin(&I, 0));
1410
0
  }
1411
1412
5
  void visitIntToPtrInst(IntToPtrInst &I) {
1413
5
    IRBuilder<> IRB(&I);
1414
5
    setShadow(&I, IRB.CreateIntCast(getShadow(&I, 0), getShadowTy(&I), false,
1415
5
             "_msprop_inttoptr"));
1416
5
    setOrigin(&I, getOrigin(&I, 0));
1417
5
  }
1418
1419
0
  void visitFPToSIInst(CastInst& I) { handleShadowOr(I); }
1420
0
  void visitFPToUIInst(CastInst& I) { handleShadowOr(I); }
1421
0
  void visitSIToFPInst(CastInst& I) { handleShadowOr(I); }
1422
0
  void visitUIToFPInst(CastInst& I) { handleShadowOr(I); }
1423
0
  void visitFPExtInst(CastInst& I) { handleShadowOr(I); }
1424
0
  void visitFPTruncInst(CastInst& I) { handleShadowOr(I); }
1425
1426
  /// \brief Propagate shadow for bitwise AND.
1427
  ///
1428
  /// This code is exact, i.e. if, for example, a bit in the left argument
1429
  /// is defined and 0, then neither the value not definedness of the
1430
  /// corresponding bit in B don't affect the resulting shadow.
1431
0
  void visitAnd(BinaryOperator &I) {
1432
0
    IRBuilder<> IRB(&I);
1433
0
    //  "And" of 0 and a poisoned value results in unpoisoned value.
1434
0
    //  1&1 => 1;     0&1 => 0;     p&1 => p;
1435
0
    //  1&0 => 0;     0&0 => 0;     p&0 => 0;
1436
0
    //  1&p => p;     0&p => 0;     p&p => p;
1437
0
    //  S = (S1 & S2) | (V1 & S2) | (S1 & V2)
1438
0
    Value *S1 = getShadow(&I, 0);
1439
0
    Value *S2 = getShadow(&I, 1);
1440
0
    Value *V1 = I.getOperand(0);
1441
0
    Value *V2 = I.getOperand(1);
1442
0
    if (
V1->getType() != S1->getType()0
) {
1443
0
      V1 = IRB.CreateIntCast(V1, S1->getType(), false);
1444
0
      V2 = IRB.CreateIntCast(V2, S2->getType(), false);
1445
0
    }
1446
0
    Value *S1S2 = IRB.CreateAnd(S1, S2);
1447
0
    Value *V1S2 = IRB.CreateAnd(V1, S2);
1448
0
    Value *S1V2 = IRB.CreateAnd(S1, V2);
1449
0
    setShadow(&I, IRB.CreateOr(S1S2, IRB.CreateOr(V1S2, S1V2)));
1450
0
    setOriginForNaryOp(I);
1451
0
  }
1452
1453
0
  void visitOr(BinaryOperator &I) {
1454
0
    IRBuilder<> IRB(&I);
1455
0
    //  "Or" of 1 and a poisoned value results in unpoisoned value.
1456
0
    //  1|1 => 1;     0|1 => 1;     p|1 => 1;
1457
0
    //  1|0 => 1;     0|0 => 0;     p|0 => p;
1458
0
    //  1|p => 1;     0|p => p;     p|p => p;
1459
0
    //  S = (S1 & S2) | (~V1 & S2) | (S1 & ~V2)
1460
0
    Value *S1 = getShadow(&I, 0);
1461
0
    Value *S2 = getShadow(&I, 1);
1462
0
    Value *V1 = IRB.CreateNot(I.getOperand(0));
1463
0
    Value *V2 = IRB.CreateNot(I.getOperand(1));
1464
0
    if (
V1->getType() != S1->getType()0
) {
1465
0
      V1 = IRB.CreateIntCast(V1, S1->getType(), false);
1466
0
      V2 = IRB.CreateIntCast(V2, S2->getType(), false);
1467
0
    }
1468
0
    Value *S1S2 = IRB.CreateAnd(S1, S2);
1469
0
    Value *V1S2 = IRB.CreateAnd(V1, S2);
1470
0
    Value *S1V2 = IRB.CreateAnd(S1, V2);
1471
0
    setShadow(&I, IRB.CreateOr(S1S2, IRB.CreateOr(V1S2, S1V2)));
1472
0
    setOriginForNaryOp(I);
1473
0
  }
1474
1475
  /// \brief Default propagation of shadow and/or origin.
1476
  ///
1477
  /// This class implements the general case of shadow propagation, used in all
1478
  /// cases where we don't know and/or don't care about what the operation
1479
  /// actually does. It converts all input shadow values to a common type
1480
  /// (extending or truncating as necessary), and bitwise OR's them.
1481
  ///
1482
  /// This is much cheaper than inserting checks (i.e. requiring inputs to be
1483
  /// fully initialized), and less prone to false positives.
1484
  ///
1485
  /// This class also implements the general case of origin propagation. For a
1486
  /// Nary operation, result origin is set to the origin of an argument that is
1487
  /// not entirely initialized. If there is more than one such arguments, the
1488
  /// rightmost of them is picked. It does not matter which one is picked if all
1489
  /// arguments are initialized.
1490
  template <bool CombineShadow>
1491
  class Combiner {
1492
    Value *Shadow;
1493
    Value *Origin;
1494
    IRBuilder<> &IRB;
1495
    MemorySanitizerVisitor *MSV;
1496
1497
  public:
1498
    Combiner(MemorySanitizerVisitor *MSV, IRBuilder<> &IRB) :
1499
103
      Shadow(nullptr), Origin(nullptr), IRB(IRB), MSV(MSV) {}
MemorySanitizer.cpp:(anonymous namespace)::MemorySanitizerVisitor::Combiner<true>::Combiner((anonymous namespace)::MemorySanitizerVisitor*, llvm::IRBuilder<llvm::ConstantFolder, llvm::IRBuilderDefaultInserter>&)
Line
Count
Source
1499
77
      Shadow(nullptr), Origin(nullptr), IRB(IRB), MSV(MSV) {}
MemorySanitizer.cpp:(anonymous namespace)::MemorySanitizerVisitor::Combiner<false>::Combiner((anonymous namespace)::MemorySanitizerVisitor*, llvm::IRBuilder<llvm::ConstantFolder, llvm::IRBuilderDefaultInserter>&)
Line
Count
Source
1499
26
      Shadow(nullptr), Origin(nullptr), IRB(IRB), MSV(MSV) {}
1500
1501
    /// \brief Add a pair of shadow and origin values to the mix.
1502
251
    Combiner &Add(Value *OpShadow, Value *OpOrigin) {
1503
251
      if (
CombineShadow251
) {
1504
204
        assert(OpShadow);
1505
204
        if (!Shadow)
1506
77
          Shadow = OpShadow;
1507
127
        else {
1508
127
          OpShadow = MSV->CreateShadowCast(IRB, OpShadow, Shadow->getType());
1509
127
          Shadow = IRB.CreateOr(Shadow, OpShadow, "_msprop");
1510
127
        }
1511
204
      }
1512
251
1513
251
      if (
MSV->MS.TrackOrigins251
) {
1514
55
        assert(OpOrigin);
1515
55
        if (
!Origin55
) {
1516
29
          Origin = OpOrigin;
1517
55
        } else {
1518
26
          Constant *ConstOrigin = dyn_cast<Constant>(OpOrigin);
1519
26
          // No point in adding something that might result in 0 origin value.
1520
26
          if (
!ConstOrigin || 26
!ConstOrigin->isNullValue()15
) {
1521
11
            Value *FlatShadow = MSV->convertToShadowTyNoVec(OpShadow, IRB);
1522
11
            Value *Cond =
1523
11
                IRB.CreateICmpNE(FlatShadow, MSV->getCleanShadow(FlatShadow));
1524
11
            Origin = IRB.CreateSelect(Cond, OpOrigin, Origin);
1525
11
          }
1526
26
        }
1527
55
      }
1528
251
      return *this;
1529
251
    }
MemorySanitizer.cpp:(anonymous namespace)::MemorySanitizerVisitor::Combiner<false>::Add(llvm::Value*, llvm::Value*)
Line
Count
Source
1502
47
    Combiner &Add(Value *OpShadow, Value *OpOrigin) {
1503
47
      if (
CombineShadow47
) {
1504
0
        assert(OpShadow);
1505
0
        if (!Shadow)
1506
0
          Shadow = OpShadow;
1507
0
        else {
1508
0
          OpShadow = MSV->CreateShadowCast(IRB, OpShadow, Shadow->getType());
1509
0
          Shadow = IRB.CreateOr(Shadow, OpShadow, "_msprop");
1510
0
        }
1511
0
      }
1512
47
1513
47
      if (
MSV->MS.TrackOrigins47
) {
1514
47
        assert(OpOrigin);
1515
47
        if (
!Origin47
) {
1516
26
          Origin = OpOrigin;
1517
47
        } else {
1518
21
          Constant *ConstOrigin = dyn_cast<Constant>(OpOrigin);
1519
21
          // No point in adding something that might result in 0 origin value.
1520
21
          if (
!ConstOrigin || 21
!ConstOrigin->isNullValue()11
) {
1521
10
            Value *FlatShadow = MSV->convertToShadowTyNoVec(OpShadow, IRB);
1522
10
            Value *Cond =
1523
10
                IRB.CreateICmpNE(FlatShadow, MSV->getCleanShadow(FlatShadow));
1524
10
            Origin = IRB.CreateSelect(Cond, OpOrigin, Origin);
1525
10
          }
1526
21
        }
1527
47
      }
1528
47
      return *this;
1529
47
    }
MemorySanitizer.cpp:(anonymous namespace)::MemorySanitizerVisitor::Combiner<true>::Add(llvm::Value*, llvm::Value*)
Line
Count
Source
1502
204
    Combiner &Add(Value *OpShadow, Value *OpOrigin) {
1503
204
      if (
CombineShadow204
) {
1504
204
        assert(OpShadow);
1505
204
        if (!Shadow)
1506
77
          Shadow = OpShadow;
1507
127
        else {
1508
127
          OpShadow = MSV->CreateShadowCast(IRB, OpShadow, Shadow->getType());
1509
127
          Shadow = IRB.CreateOr(Shadow, OpShadow, "_msprop");
1510
127
        }
1511
204
      }
1512
204
1513
204
      if (
MSV->MS.TrackOrigins204
) {
1514
8
        assert(OpOrigin);
1515
8
        if (
!Origin8
) {
1516
3
          Origin = OpOrigin;
1517
8
        } else {
1518
5
          Constant *ConstOrigin = dyn_cast<Constant>(OpOrigin);
1519
5
          // No point in adding something that might result in 0 origin value.
1520
5
          if (
!ConstOrigin || 5
!ConstOrigin->isNullValue()4
) {
1521
1
            Value *FlatShadow = MSV->convertToShadowTyNoVec(OpShadow, IRB);
1522
1
            Value *Cond =
1523
1
                IRB.CreateICmpNE(FlatShadow, MSV->getCleanShadow(FlatShadow));
1524
1
            Origin = IRB.CreateSelect(Cond, OpOrigin, Origin);
1525
1
          }
1526
5
        }
1527
8
      }
1528
204
      return *this;
1529
204
    }
1530
1531
    /// \brief Add an application value to the mix.
1532
251
    Combiner &Add(Value *V) {
1533
251
      Value *OpShadow = MSV->getShadow(V);
1534
251
      Value *OpOrigin = MSV->MS.TrackOrigins ? 
MSV->getOrigin(V)55
:
nullptr196
;
1535
251
      return Add(OpShadow, OpOrigin);
1536
251
    }
MemorySanitizer.cpp:(anonymous namespace)::MemorySanitizerVisitor::Combiner<false>::Add(llvm::Value*)
Line
Count
Source
1532
47
    Combiner &Add(Value *V) {
1533
47
      Value *OpShadow = MSV->getShadow(V);
1534
47
      Value *OpOrigin = MSV->MS.TrackOrigins ? 
MSV->getOrigin(V)47
:
nullptr0
;
1535
47
      return Add(OpShadow, OpOrigin);
1536
47
    }
MemorySanitizer.cpp:(anonymous namespace)::MemorySanitizerVisitor::Combiner<true>::Add(llvm::Value*)
Line
Count
Source
1532
204
    Combiner &Add(Value *V) {
1533
204
      Value *OpShadow = MSV->getShadow(V);
1534
204
      Value *OpOrigin = MSV->MS.TrackOrigins ? 
MSV->getOrigin(V)8
:
nullptr196
;
1535
204
      return Add(OpShadow, OpOrigin);
1536
204
    }
1537
1538
    /// \brief Set the current combined values as the given instruction's shadow
1539
    /// and origin.
1540
103
    void Done(Instruction *I) {
1541
103
      if (
CombineShadow103
) {
1542
77
        assert(Shadow);
1543
77
        Shadow = MSV->CreateShadowCast(IRB, Shadow, MSV->getShadowTy(I));
1544
77
        MSV->setShadow(I, Shadow);
1545
77
      }
1546
103
      if (
MSV->MS.TrackOrigins103
) {
1547
29
        assert(Origin);
1548
29
        MSV->setOrigin(I, Origin);
1549
29
      }
1550
103
    }
MemorySanitizer.cpp:(anonymous namespace)::MemorySanitizerVisitor::Combiner<false>::Done(llvm::Instruction*)
Line
Count
Source
1540
26
    void Done(Instruction *I) {
1541
26
      if (
CombineShadow26
) {
1542
0
        assert(Shadow);
1543
0
        Shadow = MSV->CreateShadowCast(IRB, Shadow, MSV->getShadowTy(I));
1544
0
        MSV->setShadow(I, Shadow);
1545
0
      }
1546
26
      if (
MSV->MS.TrackOrigins26
) {
1547
26
        assert(Origin);
1548
26
        MSV->setOrigin(I, Origin);
1549
26
      }
1550
26
    }
MemorySanitizer.cpp:(anonymous namespace)::MemorySanitizerVisitor::Combiner<true>::Done(llvm::Instruction*)
Line
Count
Source
1540
77
    void Done(Instruction *I) {
1541
77
      if (
CombineShadow77
) {
1542
77
        assert(Shadow);
1543
77
        Shadow = MSV->CreateShadowCast(IRB, Shadow, MSV->getShadowTy(I));
1544
77
        MSV->setShadow(I, Shadow);
1545
77
      }
1546
77
      if (
MSV->MS.TrackOrigins77
) {
1547
3
        assert(Origin);
1548
3
        MSV->setOrigin(I, Origin);
1549
3
      }
1550
77
    }
1551
  };
1552
1553
  typedef Combiner<true> ShadowAndOriginCombiner;
1554
  typedef Combiner<false> OriginCombiner;
1555
1556
  /// \brief Propagate origin for arbitrary operation.
1557
78
  void setOriginForNaryOp(Instruction &I) {
1558
78
    if (
!MS.TrackOrigins78
)
return52
;
1559
26
    IRBuilder<> IRB(&I);
1560
26
    OriginCombiner OC(this, IRB);
1561
73
    for (Instruction::op_iterator OI = I.op_begin(); 
OI != I.op_end()73
;
++OI47
)
1562
47
      OC.Add(OI->get());
1563
78
    OC.Done(&I);
1564
78
  }
1565
1566
430
  size_t VectorOrPrimitiveTypeSizeInBits(Type *Ty) {
1567
430
    assert(!(Ty->isVectorTy() && Ty->getScalarType()->isPointerTy()) &&
1568
430
           "Vector of pointers is not a valid shadow type");
1569
430
    return Ty->isVectorTy() ?
1570
16
      Ty->getVectorNumElements() * Ty->getScalarSizeInBits() :
1571
414
      Ty->getPrimitiveSizeInBits();
1572
430
  }
1573
1574
  /// \brief Cast between two shadow types, extending or truncating as
1575
  /// necessary.
1576
  Value *CreateShadowCast(IRBuilder<> &IRB, Value *V, Type *dstTy,
1577
215
                          bool Signed = false) {
1578
215
    Type *srcTy = V->getType();
1579
215
    size_t srcSizeInBits = VectorOrPrimitiveTypeSizeInBits(srcTy);
1580
215
    size_t dstSizeInBits = VectorOrPrimitiveTypeSizeInBits(dstTy);
1581
215
    if (
srcSizeInBits > 1 && 215
dstSizeInBits == 1206
)
1582
1
      return IRB.CreateICmpNE(V, getCleanShadow(V));
1583
214
1584
214
    
if (214
dstTy->isIntegerTy() && 214
srcTy->isIntegerTy()204
)
1585
202
      return IRB.CreateIntCast(V, dstTy, Signed);
1586
12
    
if (12
dstTy->isVectorTy() && 12
srcTy->isVectorTy()10
&&
1587
4
        dstTy->getVectorNumElements() == srcTy->getVectorNumElements())
1588
4
      return IRB.CreateIntCast(V, dstTy, Signed);
1589
8
    Value *V1 = IRB.CreateBitCast(V, Type::getIntNTy(*MS.C, srcSizeInBits));
1590
8
    Value *V2 =
1591
8
      IRB.CreateIntCast(V1, Type::getIntNTy(*MS.C, dstSizeInBits), Signed);
1592
8
    return IRB.CreateBitCast(V2, dstTy);
1593
8
    // TODO: handle struct types.
1594
8
  }
1595
1596
  /// \brief Cast an application value to the type of its own shadow.
1597
14
  Value *CreateAppToShadowCast(IRBuilder<> &IRB, Value *V) {
1598
14
    Type *ShadowTy = getShadowTy(V);
1599
14
    if (V->getType() == ShadowTy)
1600
12
      return V;
1601
2
    
if (2
V->getType()->isPtrOrPtrVectorTy()2
)
1602
0
      return IRB.CreatePtrToInt(V, ShadowTy);
1603
2
    else
1604
2
      return IRB.CreateBitCast(V, ShadowTy);
1605
0
  }
1606
1607
  /// \brief Propagate shadow for arbitrary operation.
1608
75
  void handleShadowOr(Instruction &I) {
1609
75
    IRBuilder<> IRB(&I);
1610
75
    ShadowAndOriginCombiner SC(this, IRB);
1611
275
    for (Instruction::op_iterator OI = I.op_begin(); 
OI != I.op_end()275
;
++OI200
)
1612
200
      SC.Add(OI->get());
1613
75
    SC.Done(&I);
1614
75
  }
1615
1616
  // \brief Handle multiplication by constant.
1617
  //
1618
  // Handle a special case of multiplication by constant that may have one or
1619
  // more zeros in the lower bits. This makes corresponding number of lower bits
1620
  // of the result zero as well. We model it by shifting the other operand
1621
  // shadow left by the required number of bits. Effectively, we transform
1622
  // (X * (A * 2**B)) to ((X << B) * A) and instrument (X << B) as (Sx << B).
1623
  // We use multiplication by 2**N instead of shift to cover the case of
1624
  // multiplication by 0, which may occur in some elements of a vector operand.
1625
  void handleMulByConstant(BinaryOperator &I, Constant *ConstArg,
1626
9
                           Value *OtherArg) {
1627
9
    Constant *ShadowMul;
1628
9
    Type *Ty = ConstArg->getType();
1629
9
    if (
Ty->isVectorTy()9
) {
1630
2
      unsigned NumElements = Ty->getVectorNumElements();
1631
2
      Type *EltTy = Ty->getSequentialElementType();
1632
2
      SmallVector<Constant *, 16> Elements;
1633
8
      for (unsigned Idx = 0; 
Idx < NumElements8
;
++Idx6
) {
1634
6
        if (ConstantInt *Elt =
1635
5
                dyn_cast<ConstantInt>(ConstArg->getAggregateElement(Idx))) {
1636
5
          const APInt &V = Elt->getValue();
1637
5
          APInt V2 = APInt(V.getBitWidth(), 1) << V.countTrailingZeros();
1638
5
          Elements.push_back(ConstantInt::get(EltTy, V2));
1639
6
        } else {
1640
1
          Elements.push_back(ConstantInt::get(EltTy, 1));
1641
1
        }
1642
6
      }
1643
2
      ShadowMul = ConstantVector::get(Elements);
1644
9
    } else {
1645
7
      if (ConstantInt *
Elt7
= dyn_cast<ConstantInt>(ConstArg)) {
1646
6
        const APInt &V = Elt->getValue();
1647
6
        APInt V2 = APInt(V.getBitWidth(), 1) << V.countTrailingZeros();
1648
6
        ShadowMul = ConstantInt::get(Ty, V2);
1649
7
      } else {
1650
1
        ShadowMul = ConstantInt::get(Ty, 1);
1651
1
      }
1652
7
    }
1653
9
1654
9
    IRBuilder<> IRB(&I);
1655
9
    setShadow(&I,
1656
9
              IRB.CreateMul(getShadow(OtherArg), ShadowMul, "msprop_mul_cst"));
1657
9
    setOrigin(&I, getOrigin(OtherArg));
1658
9
  }
1659
1660
9
  void visitMul(BinaryOperator &I) {
1661
9
    Constant *constOp0 = dyn_cast<Constant>(I.getOperand(0));
1662
9
    Constant *constOp1 = dyn_cast<Constant>(I.getOperand(1));
1663
9
    if (
constOp0 && 9
!constOp10
)
1664
0
      handleMulByConstant(I, constOp0, I.getOperand(1));
1665
9
    else 
if (9
constOp1 && 9
!constOp09
)
1666
9
      handleMulByConstant(I, constOp1, I.getOperand(0));
1667
9
    else
1668
0
      handleShadowOr(I);
1669
9
  }
1670
1671
0
  void visitFAdd(BinaryOperator &I) { handleShadowOr(I); }
1672
0
  void visitFSub(BinaryOperator &I) { handleShadowOr(I); }
1673
0
  void visitFMul(BinaryOperator &I) { handleShadowOr(I); }
1674
2
  void visitAdd(BinaryOperator &I) { handleShadowOr(I); }
1675
0
  void visitSub(BinaryOperator &I) { handleShadowOr(I); }
1676
0
  void visitXor(BinaryOperator &I) { handleShadowOr(I); }
1677
1678
2
  void handleDiv(Instruction &I) {
1679
2
    IRBuilder<> IRB(&I);
1680
2
    // Strict on the second argument.
1681
2
    insertShadowCheck(I.getOperand(1), &I);
1682
2
    setShadow(&I, getShadow(&I, 0));
1683
2
    setOrigin(&I, getOrigin(&I, 0));
1684
2
  }
1685
1686
2
  void visitUDiv(BinaryOperator &I) { handleDiv(I); }
1687
0
  void visitSDiv(BinaryOperator &I) { handleDiv(I); }
1688
0
  void visitFDiv(BinaryOperator &I) { handleDiv(I); }
1689
0
  void visitURem(BinaryOperator &I) { handleDiv(I); }
1690
0
  void visitSRem(BinaryOperator &I) { handleDiv(I); }
1691
0
  void visitFRem(BinaryOperator &I) { handleDiv(I); }
1692
1693
  /// \brief Instrument == and != comparisons.
1694
  ///
1695
  /// Sometimes the comparison result is known even if some of the bits of the
1696
  /// arguments are not.
1697
11
  void handleEqualityComparison(ICmpInst &I) {
1698
11
    IRBuilder<> IRB(&I);
1699
11
    Value *A = I.getOperand(0);
1700
11
    Value *B = I.getOperand(1);
1701
11
    Value *Sa = getShadow(A);
1702
11
    Value *Sb = getShadow(B);
1703
11
1704
11
    // Get rid of pointers and vectors of pointers.
1705
11
    // For ints (and vectors of ints), types of A and Sa match,
1706
11
    // and this is a no-op.
1707
11
    A = IRB.CreatePointerCast(A, Sa->getType());
1708
11
    B = IRB.CreatePointerCast(B, Sb->getType());
1709
11
1710
11
    // A == B  <==>  (C = A^B) == 0
1711
11
    // A != B  <==>  (C = A^B) != 0
1712
11
    // Sc = Sa | Sb
1713
11
    Value *C = IRB.CreateXor(A, B);
1714
11
    Value *Sc = IRB.CreateOr(Sa, Sb);
1715
11
    // Now dealing with i = (C == 0) comparison (or C != 0, does not matter now)
1716
11
    // Result is defined if one of the following is true
1717
11
    // * there is a defined 1 bit in C
1718
11
    // * C is fully defined
1719
11
    // Si = !(C & ~Sc) && Sc
1720
11
    Value *Zero = Constant::getNullValue(Sc->getType());
1721
11
    Value *MinusOne = Constant::getAllOnesValue(Sc->getType());
1722
11
    Value *Si =
1723
11
      IRB.CreateAnd(IRB.CreateICmpNE(Sc, Zero),
1724
11
                    IRB.CreateICmpEQ(
1725
11
                      IRB.CreateAnd(IRB.CreateXor(Sc, MinusOne), C), Zero));
1726
11
    Si->setName("_msprop_icmp");
1727
11
    setShadow(&I, Si);
1728
11
    setOriginForNaryOp(I);
1729
11
  }
1730
1731
  /// \brief Build the lowest possible value of V, taking into account V's
1732
  ///        uninitialized bits.
1733
  Value *getLowestPossibleValue(IRBuilder<> &IRB, Value *A, Value *Sa,
1734
4
                                bool isSigned) {
1735
4
    if (
isSigned4
) {
1736
0
      // Split shadow into sign bit and other bits.
1737
0
      Value *SaOtherBits = IRB.CreateLShr(IRB.CreateShl(Sa, 1), 1);
1738
0
      Value *SaSignBit = IRB.CreateXor(Sa, SaOtherBits);
1739
0
      // Maximise the undefined shadow bit, minimize other undefined bits.
1740
0
      return
1741
0
        IRB.CreateOr(IRB.CreateAnd(A, IRB.CreateNot(SaOtherBits)), SaSignBit);
1742
0
    } else {
1743
4
      // Minimize undefined bits.
1744
4
      return IRB.CreateAnd(A, IRB.CreateNot(Sa));
1745
4
    }
1746
0
  }
1747
1748
  /// \brief Build the highest possible value of V, taking into account V's
1749
  ///        uninitialized bits.
1750
  Value *getHighestPossibleValue(IRBuilder<> &IRB, Value *A, Value *Sa,
1751
4
                                bool isSigned) {
1752
4
    if (
isSigned4
) {
1753
0
      // Split shadow into sign bit and other bits.
1754
0
      Value *SaOtherBits = IRB.CreateLShr(IRB.CreateShl(Sa, 1), 1);
1755
0
      Value *SaSignBit = IRB.CreateXor(Sa, SaOtherBits);
1756
0
      // Minimise the undefined shadow bit, maximise other undefined bits.
1757
0
      return
1758
0
        IRB.CreateOr(IRB.CreateAnd(A, IRB.CreateNot(SaSignBit)), SaOtherBits);
1759
0
    } else {
1760
4
      // Maximize undefined bits.
1761
4
      return IRB.CreateOr(A, Sa);
1762
4
    }
1763
0
  }
1764
1765
  /// \brief Instrument relational comparisons.
1766
  ///
1767
  /// This function does exact shadow propagation for all relational
1768
  /// comparisons of integers, pointers and vectors of those.
1769
  /// FIXME: output seems suboptimal when one of the operands is a constant
1770
2
  void handleRelationalComparisonExact(ICmpInst &I) {
1771
2
    IRBuilder<> IRB(&I);
1772
2
    Value *A = I.getOperand(0);
1773
2
    Value *B = I.getOperand(1);
1774
2
    Value *Sa = getShadow(A);
1775
2
    Value *Sb = getShadow(B);
1776
2
1777
2
    // Get rid of pointers and vectors of pointers.
1778
2
    // For ints (and vectors of ints), types of A and Sa match,
1779
2
    // and this is a no-op.
1780
2
    A = IRB.CreatePointerCast(A, Sa->getType());
1781
2
    B = IRB.CreatePointerCast(B, Sb->getType());
1782
2
1783
2
    // Let [a0, a1] be the interval of possible values of A, taking into account
1784
2
    // its undefined bits. Let [b0, b1] be the interval of possible values of B.
1785
2
    // Then (A cmp B) is defined iff (a0 cmp b1) == (a1 cmp b0).
1786
2
    bool IsSigned = I.isSigned();
1787
2
    Value *S1 = IRB.CreateICmp(I.getPredicate(),
1788
2
                               getLowestPossibleValue(IRB, A, Sa, IsSigned),
1789
2
                               getHighestPossibleValue(IRB, B, Sb, IsSigned));
1790
2
    Value *S2 = IRB.CreateICmp(I.getPredicate(),
1791
2
                               getHighestPossibleValue(IRB, A, Sa, IsSigned),
1792
2
                               getLowestPossibleValue(IRB, B, Sb, IsSigned));
1793
2
    Value *Si = IRB.CreateXor(S1, S2);
1794
2
    setShadow(&I, Si);
1795
2
    setOriginForNaryOp(I);
1796
2
  }
1797
1798
  /// \brief Instrument signed relational comparisons.
1799
  ///
1800
  /// Handle sign bit tests: x<0, x>=0, x<=-1, x>-1 by propagating the highest
1801
  /// bit of the shadow. Everything else is delegated to handleShadowOr().
1802
21
  void handleSignedRelationalComparison(ICmpInst &I) {
1803
21
    Constant *constOp;
1804
21
    Value *op = nullptr;
1805
21
    CmpInst::Predicate pre;
1806
21
    if (
(constOp = dyn_cast<Constant>(I.getOperand(1)))21
) {
1807
10
      op = I.getOperand(0);
1808
10
      pre = I.getPredicate();
1809
21
    } else 
if (11
(constOp = dyn_cast<Constant>(I.getOperand(0)))11
) {
1810
10
      op = I.getOperand(1);
1811
10
      pre = I.getSwappedPredicate();
1812
11
    } else {
1813
1
      handleShadowOr(I);
1814
1
      return;
1815
1
    }
1816
20
1817
20
    
if (20
(constOp->isNullValue() &&
1818
10
         
(pre == CmpInst::ICMP_SLT || 10
pre == CmpInst::ICMP_SGE4
)) ||
1819
10
        (constOp->isAllOnesValue() &&
1820
20
         
(pre == CmpInst::ICMP_SGT || 10
pre == CmpInst::ICMP_SLE4
))) {
1821
20
      IRBuilder<> IRB(&I);
1822
20
      Value *Shadow = IRB.CreateICmpSLT(getShadow(op), getCleanShadow(op),
1823
20
                                        "_msprop_icmp_s");
1824
20
      setShadow(&I, Shadow);
1825
20
      setOrigin(&I, getOrigin(op));
1826
20
    } else {
1827
0
      handleShadowOr(I);
1828
0
    }
1829
21
  }
1830
1831
34
  void visitICmpInst(ICmpInst &I) {
1832
34
    if (
!ClHandleICmp34
) {
1833
0
      handleShadowOr(I);
1834
0
      return;
1835
0
    }
1836
34
    
if (34
I.isEquality()34
) {
1837
11
      handleEqualityComparison(I);
1838
11
      return;
1839
11
    }
1840
23
1841
34
    assert(I.isRelational());
1842
23
    if (
ClHandleICmpExact23
) {
1843
0
      handleRelationalComparisonExact(I);
1844
0
      return;
1845
0
    }
1846
23
    
if (23
I.isSigned()23
) {
1847
21
      handleSignedRelationalComparison(I);
1848
21
      return;
1849
21
    }
1850
2
1851
23
    assert(I.isUnsigned());
1852
2
    if (
(isa<Constant>(I.getOperand(0)) || 2
isa<Constant>(I.getOperand(1))2
)) {
1853
2
      handleRelationalComparisonExact(I);
1854
2
      return;
1855
2
    }
1856
0
1857
0
    handleShadowOr(I);
1858
0
  }
1859
1860
0
  void visitFCmpInst(FCmpInst &I) {
1861
0
    handleShadowOr(I);
1862
0
  }
1863
1864
4
  void handleShift(BinaryOperator &I) {
1865
4
    IRBuilder<> IRB(&I);
1866
4
    // If any of the S2 bits are poisoned, the whole thing is poisoned.
1867
4
    // Otherwise perform the same shift on S1.
1868
4
    Value *S1 = getShadow(&I, 0);
1869
4
    Value *S2 = getShadow(&I, 1);
1870
4
    Value *S2Conv = IRB.CreateSExt(IRB.CreateICmpNE(S2, getCleanShadow(S2)),
1871
4
                                   S2->getType());
1872
4
    Value *V2 = I.getOperand(1);
1873
4
    Value *Shift = IRB.CreateBinOp(I.getOpcode(), S1, V2);
1874
4
    setShadow(&I, IRB.CreateOr(Shift, S2Conv));
1875
4
    setOriginForNaryOp(I);
1876
4
  }
1877
1878
4
  void visitShl(BinaryOperator &I) { handleShift(I); }
1879
0
  void visitAShr(BinaryOperator &I) { handleShift(I); }
1880
0
  void visitLShr(BinaryOperator &I) { handleShift(I); }
1881
1882
  /// \brief Instrument llvm.memmove
1883
  ///
1884
  /// At this point we don't know if llvm.memmove will be inlined or not.
1885
  /// If we don't instrument it and it gets inlined,
1886
  /// our interceptor will not kick in and we will lose the memmove.
1887
  /// If we instrument the call here, but it does not get inlined,
1888
  /// we will memove the shadow twice: which is bad in case
1889
  /// of overlapping regions. So, we simply lower the intrinsic to a call.
1890
  ///
1891
  /// Similar situation exists for memcpy and memset.
1892
2
  void visitMemMoveInst(MemMoveInst &I) {
1893
2
    IRBuilder<> IRB(&I);
1894
2
    IRB.CreateCall(
1895
2
        MS.MemmoveFn,
1896
2
        {IRB.CreatePointerCast(I.getArgOperand(0), IRB.getInt8PtrTy()),
1897
2
         IRB.CreatePointerCast(I.getArgOperand(1), IRB.getInt8PtrTy()),
1898
2
         IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
1899
2
    I.eraseFromParent();
1900
2
  }
1901
1902
  // Similar to memmove: avoid copying shadow twice.
1903
  // This is somewhat unfortunate as it may slowdown small constant memcpys.
1904
  // FIXME: consider doing manual inline for small constant sizes and proper
1905
  // alignment.
1906
4
  void visitMemCpyInst(MemCpyInst &I) {
1907
4
    IRBuilder<> IRB(&I);
1908
4
    IRB.CreateCall(
1909
4
        MS.MemcpyFn,
1910
4
        {IRB.CreatePointerCast(I.getArgOperand(0), IRB.getInt8PtrTy()),
1911
4
         IRB.CreatePointerCast(I.getArgOperand(1), IRB.getInt8PtrTy()),
1912
4
         IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
1913
4
    I.eraseFromParent();
1914
4
  }
1915
1916
  // Same as memcpy.
1917
2
  void visitMemSetInst(MemSetInst &I) {
1918
2
    IRBuilder<> IRB(&I);
1919
2
    IRB.CreateCall(
1920
2
        MS.MemsetFn,
1921
2
        {IRB.CreatePointerCast(I.getArgOperand(0), IRB.getInt8PtrTy()),
1922
2
         IRB.CreateIntCast(I.getArgOperand(1), IRB.getInt32Ty(), false),
1923
2
         IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
1924
2
    I.eraseFromParent();
1925
2
  }
1926
1927
8
  void visitVAStartInst(VAStartInst &I) {
1928
8
    VAHelper->visitVAStartInst(I);
1929
8
  }
1930
1931
2
  void visitVACopyInst(VACopyInst &I) {
1932
2
    VAHelper->visitVACopyInst(I);
1933
2
  }
1934
1935
  /// \brief Handle vector store-like intrinsics.
1936
  ///
1937
  /// Instrument intrinsics that look like a simple SIMD store: writes memory,
1938
  /// has 1 pointer argument and 1 vector argument, returns void.
1939
0
  bool handleVectorStoreIntrinsic(IntrinsicInst &I) {
1940
0
    IRBuilder<> IRB(&I);
1941
0
    Value* Addr = I.getArgOperand(0);
1942
0
    Value *Shadow = getShadow(&I, 1);
1943
0
    Value *ShadowPtr = getShadowPtr(Addr, Shadow->getType(), IRB);
1944
0
1945
0
    // We don't know the pointer alignment (could be unaligned SSE store!).
1946
0
    // Have to assume to worst case.
1947
0
    IRB.CreateAlignedStore(Shadow, ShadowPtr, 1);
1948
0
1949
0
    if (ClCheckAccessAddress)
1950
0
      insertShadowCheck(Addr, &I);
1951
0
1952
0
    // FIXME: factor out common code from materializeStores
1953
0
    if (MS.TrackOrigins)
1954
0
      IRB.CreateStore(getOrigin(&I, 1), getOriginPtr(Addr, IRB, 1));
1955
0
    return true;
1956
0
  }
1957
1958
  /// \brief Handle vector load-like intrinsics.
1959
  ///
1960
  /// Instrument intrinsics that look like a simple SIMD load: reads memory,
1961
  /// has 1 pointer argument, returns a vector.
1962
2
  bool handleVectorLoadIntrinsic(IntrinsicInst &I) {
1963
2
    IRBuilder<> IRB(&I);
1964
2
    Value *Addr = I.getArgOperand(0);
1965
2
1966
2
    Type *ShadowTy = getShadowTy(&I);
1967
2
    if (
PropagateShadow2
) {
1968
2
      Value *ShadowPtr = getShadowPtr(Addr, ShadowTy, IRB);
1969
2
      // We don't know the pointer alignment (could be unaligned SSE load!).
1970
2
      // Have to assume to worst case.
1971
2
      setShadow(&I, IRB.CreateAlignedLoad(ShadowPtr, 1, "_msld"));
1972
2
    } else {
1973
0
      setShadow(&I, getCleanShadow(&I));
1974
0
    }
1975
2
1976
2
    if (ClCheckAccessAddress)
1977
0
      insertShadowCheck(Addr, &I);
1978
2
1979
2
    if (
MS.TrackOrigins2
) {
1980
1
      if (PropagateShadow)
1981
1
        setOrigin(&I, IRB.CreateLoad(getOriginPtr(Addr, IRB, 1)));
1982
1
      else
1983
0
        setOrigin(&I, getCleanOrigin());
1984
1
    }
1985
2
    return true;
1986
2
  }
1987
1988
  /// \brief Handle (SIMD arithmetic)-like intrinsics.
1989
  ///
1990
  /// Instrument intrinsics with any number of arguments of the same type,
1991
  /// equal to the return type. The type should be simple (no aggregates or
1992
  /// pointers; vectors are fine).
1993
  /// Caller guarantees that this intrinsic does not access memory.
1994
6
  bool maybeHandleSimpleNomemIntrinsic(IntrinsicInst &I) {
1995
6
    Type *RetTy = I.getType();
1996
6
    if (!(RetTy->isIntOrIntVectorTy() ||
1997
4
          RetTy->isFPOrFPVectorTy() ||
1998
4
          RetTy->isX86_MMXTy()))
1999
4
      return false;
2000
2
2001
2
    unsigned NumArgOperands = I.getNumArgOperands();
2002
2
2003
6
    for (unsigned i = 0; 
i < NumArgOperands6
;
++i4
) {
2004
4
      Type *Ty = I.getArgOperand(i)->getType();
2005
4
      if (Ty != RetTy)
2006
0
        return false;
2007
4
    }
2008
2
2009
2
    IRBuilder<> IRB(&I);
2010
2
    ShadowAndOriginCombiner SC(this, IRB);
2011
6
    for (unsigned i = 0; 
i < NumArgOperands6
;
++i4
)
2012
4
      SC.Add(I.getArgOperand(i));
2013
2
    SC.Done(&I);
2014
2
2015
2
    return true;
2016
6
  }
2017
2018
  /// \brief Heuristically instrument unknown intrinsics.
2019
  ///
2020
  /// The main purpose of this code is to do something reasonable with all
2021
  /// random intrinsics we might encounter, most importantly - SIMD intrinsics.
2022
  /// We recognize several classes of intrinsics by their argument types and
2023
  /// ModRefBehaviour and apply special intrumentation when we are reasonably
2024
  /// sure that we know what the intrinsic does.
2025
  ///
2026
  /// We special-case intrinsics where this approach fails. See llvm.bswap
2027
  /// handling as an example of that.
2028
31
  bool handleUnknownIntrinsic(IntrinsicInst &I) {
2029
31
    unsigned NumArgOperands = I.getNumArgOperands();
2030
31
    if (NumArgOperands == 0)
2031
0
      return false;
2032
31
2033
31
    
if (31
NumArgOperands == 2 &&
2034
12
        I.getArgOperand(0)->getType()->isPointerTy() &&
2035
0
        I.getArgOperand(1)->getType()->isVectorTy() &&
2036
0
        I.getType()->isVoidTy() &&
2037
31
        
!I.onlyReadsMemory()0
) {
2038
0
      // This looks like a vector store.
2039
0
      return handleVectorStoreIntrinsic(I);
2040
0
    }
2041
31
2042
31
    
if (31
NumArgOperands == 1 &&
2043
8
        I.getArgOperand(0)->getType()->isPointerTy() &&
2044
7
        I.getType()->isVectorTy() &&
2045
31
        
I.onlyReadsMemory()2
) {
2046
2
      // This looks like a vector load.
2047
2
      return handleVectorLoadIntrinsic(I);
2048
2
    }
2049
29
2050
29
    
if (29
I.doesNotAccessMemory()29
)
2051
6
      
if (6
maybeHandleSimpleNomemIntrinsic(I)6
)
2052
2
        return true;
2053
27
2054
27
    // FIXME: detect and handle SSE maskstore/maskload
2055
27
    return false;
2056
27
  }
2057
2058
2
  void handleBswap(IntrinsicInst &I) {
2059
2
    IRBuilder<> IRB(&I);
2060
2
    Value *Op = I.getArgOperand(0);
2061
2
    Type *OpType = Op->getType();
2062
2
    Function *BswapFunc = Intrinsic::getDeclaration(
2063
2
      F.getParent(), Intrinsic::bswap, makeArrayRef(&OpType, 1));
2064
2
    setShadow(&I, IRB.CreateCall(BswapFunc, getShadow(Op)));
2065
2
    setOrigin(&I, getOrigin(Op));
2066
2
  }
2067
2068
  // \brief Instrument vector convert instrinsic.
2069
  //
2070
  // This function instruments intrinsics like cvtsi2ss:
2071
  // %Out = int_xxx_cvtyyy(%ConvertOp)
2072
  // or
2073
  // %Out = int_xxx_cvtyyy(%CopyOp, %ConvertOp)
2074
  // Intrinsic converts \p NumUsedElements elements of \p ConvertOp to the same
2075
  // number \p Out elements, and (if has 2 arguments) copies the rest of the
2076
  // elements from \p CopyOp.
2077
  // In most cases conversion involves floating-point value which may trigger a
2078
  // hardware exception when not fully initialized. For this reason we require
2079
  // \p ConvertOp[0:NumUsedElements] to be fully initialized and trap otherwise.
2080
  // We copy the shadow of \p CopyOp[NumUsedElements:] to \p
2081
  // Out[NumUsedElements:]. This means that intrinsics without \p CopyOp always
2082
  // return a fully initialized value.
2083
3
  void handleVectorConvertIntrinsic(IntrinsicInst &I, int NumUsedElements) {
2084
3
    IRBuilder<> IRB(&I);
2085
3
    Value *CopyOp, *ConvertOp;
2086
3
2087
3
    switch (I.getNumArgOperands()) {
2088
0
    case 3:
2089
0
      assert(isa<ConstantInt>(I.getArgOperand(2)) && "Invalid rounding mode");
2090
0
      LLVM_FALLTHROUGH;
2091
1
    case 2:
2092
1
      CopyOp = I.getArgOperand(0);
2093
1
      ConvertOp = I.getArgOperand(1);
2094
1
      break;
2095
2
    case 1:
2096
2
      ConvertOp = I.getArgOperand(0);
2097
2
      CopyOp = nullptr;
2098
2
      break;
2099
0
    default:
2100
0
      llvm_unreachable("Cvt intrinsic with unsupported number of arguments.");
2101
3
    }
2102
3
2103
3
    // The first *NumUsedElements* elements of ConvertOp are converted to the
2104
3
    // same number of output elements. The rest of the output is copied from
2105
3
    // CopyOp, or (if not available) filled with zeroes.
2106
3
    // Combine shadow for elements of ConvertOp that are used in this operation,
2107
3
    // and insert a check.
2108
3
    // FIXME: consider propagating shadow of ConvertOp, at least in the case of
2109
3
    // int->any conversion.
2110
3
    Value *ConvertShadow = getShadow(ConvertOp);
2111
3
    Value *AggShadow = nullptr;
2112
3
    if (
ConvertOp->getType()->isVectorTy()3
) {
2113
2
      AggShadow = IRB.CreateExtractElement(
2114
2
          ConvertShadow, ConstantInt::get(IRB.getInt32Ty(), 0));
2115
3
      for (int i = 1; 
i < NumUsedElements3
;
++i1
) {
2116
1
        Value *MoreShadow = IRB.CreateExtractElement(
2117
1
            ConvertShadow, ConstantInt::get(IRB.getInt32Ty(), i));
2118
1
        AggShadow = IRB.CreateOr(AggShadow, MoreShadow);
2119
1
      }
2120
3
    } else {
2121
1
      AggShadow = ConvertShadow;
2122
1
    }
2123
3
    assert(AggShadow->getType()->isIntegerTy());
2124
3
    insertShadowCheck(AggShadow, getOrigin(ConvertOp), &I);
2125
3
2126
3
    // Build result shadow by zero-filling parts of CopyOp shadow that come from
2127
3
    // ConvertOp.
2128
3
    if (
CopyOp3
) {
2129
1
      assert(CopyOp->getType() == I.getType());
2130
1
      assert(CopyOp->getType()->isVectorTy());
2131
1
      Value *ResultShadow = getShadow(CopyOp);
2132
1
      Type *EltTy = ResultShadow->getType()->getVectorElementType();
2133
2
      for (int i = 0; 
i < NumUsedElements2
;
++i1
) {
2134
1
        ResultShadow = IRB.CreateInsertElement(
2135
1
            ResultShadow, ConstantInt::getNullValue(EltTy),
2136
1
            ConstantInt::get(IRB.getInt32Ty(), i));
2137
1
      }
2138
1
      setShadow(&I, ResultShadow);
2139
1
      setOrigin(&I, getOrigin(CopyOp));
2140
3
    } else {
2141
2
      setShadow(&I, getCleanShadow(&I));
2142
2
      setOrigin(&I, getCleanOrigin());
2143
2
    }
2144
3
  }
2145
2146
  // Given a scalar or vector, extract lower 64 bits (or less), and return all
2147
  // zeroes if it is zero, and all ones otherwise.
2148
5
  Value *Lower64ShadowExtend(IRBuilder<> &IRB, Value *S, Type *T) {
2149
5
    if (S->getType()->isVectorTy())
2150
2
      S = CreateShadowCast(IRB, S, IRB.getInt64Ty(), /* Signed */ true);
2151
5
    assert(S->getType()->getPrimitiveSizeInBits() <= 64);
2152
5
    Value *S2 = IRB.CreateICmpNE(S, getCleanShadow(S));
2153
5
    return CreateShadowCast(IRB, S2, T, /* Signed */ true);
2154
5
  }
2155
2156
  // Given a vector, extract its first element, and return all
2157
  // zeroes if it is zero, and all ones otherwise.
2158
4
  Value *LowerElementShadowExtend(IRBuilder<> &IRB, Value *S, Type *T) {
2159
4
    Value *S1 = IRB.CreateExtractElement(S, (uint64_t)0);
2160
4
    Value *S2 = IRB.CreateICmpNE(S1, getCleanShadow(S1));
2161
4
    return CreateShadowCast(IRB, S2, T, /* Signed */ true);
2162
4
  }
2163
2164
3
  Value *VariableShadowExtend(IRBuilder<> &IRB, Value *S) {
2165
3
    Type *T = S->getType();
2166
3
    assert(T->isVectorTy());
2167
3
    Value *S2 = IRB.CreateICmpNE(S, getCleanShadow(S));
2168
3
    return IRB.CreateSExt(S2, T);
2169
3
  }
2170
2171
  // \brief Instrument vector shift instrinsic.
2172
  //
2173
  // This function instruments intrinsics like int_x86_avx2_psll_w.
2174
  // Intrinsic shifts %In by %ShiftSize bits.
2175
  // %ShiftSize may be a vector. In that case the lower 64 bits determine shift
2176
  // size, and the rest is ignored. Behavior is defined even if shift size is
2177
  // greater than register (or field) width.
2178
8
  void handleVectorShiftIntrinsic(IntrinsicInst &I, bool Variable) {
2179
8
    assert(I.getNumArgOperands() == 2);
2180
8
    IRBuilder<> IRB(&I);
2181
8
    // If any of the S2 bits are poisoned, the whole thing is poisoned.
2182
8
    // Otherwise perform the same shift on S1.
2183
8
    Value *S1 = getShadow(&I, 0);
2184
8
    Value *S2 = getShadow(&I, 1);
2185
3
    Value *S2Conv = Variable ? VariableShadowExtend(IRB, S2)
2186
5
                             : Lower64ShadowExtend(IRB, S2, getShadowTy(&I));
2187
8
    Value *V1 = I.getOperand(0);
2188
8
    Value *V2 = I.getOperand(1);
2189
8
    Value *Shift = IRB.CreateCall(I.getCalledValue(),
2190
8
                                  {IRB.CreateBitCast(S1, V1->getType()), V2});
2191
8
    Shift = IRB.CreateBitCast(Shift, getShadowTy(&I));
2192
8
    setShadow(&I, IRB.CreateOr(Shift, S2Conv));
2193
8
    setOriginForNaryOp(I);
2194
8
  }
2195
2196
  // \brief Get an X86_MMX-sized vector type.
2197
2
  Type *getMMXVectorTy(unsigned EltSizeInBits) {
2198
2
    const unsigned X86_MMXSizeInBits = 64;
2199
2
    return VectorType::get(IntegerType::get(*MS.C, EltSizeInBits),
2200
2
                           X86_MMXSizeInBits / EltSizeInBits);
2201
2
  }
2202
2203
  // \brief Returns a signed counterpart for an (un)signed-saturate-and-pack
2204
  // intrinsic.
2205
3
  Intrinsic::ID getSignedPackIntrinsic(Intrinsic::ID id) {
2206
3
    switch (id) {
2207
0
      case llvm::Intrinsic::x86_sse2_packsswb_128:
2208
0
      case llvm::Intrinsic::x86_sse2_packuswb_128:
2209
0
        return llvm::Intrinsic::x86_sse2_packsswb_128;
2210
0
2211
1
      case llvm::Intrinsic::x86_sse2_packssdw_128:
2212
1
      case llvm::Intrinsic::x86_sse41_packusdw:
2213
1
        return llvm::Intrinsic::x86_sse2_packssdw_128;
2214
1
2215
1
      case llvm::Intrinsic::x86_avx2_packsswb:
2216
1
      case llvm::Intrinsic::x86_avx2_packuswb:
2217
1
        return llvm::Intrinsic::x86_avx2_packsswb;
2218
1
2219
0
      case llvm::Intrinsic::x86_avx2_packssdw:
2220
0
      case llvm::Intrinsic::x86_avx2_packusdw:
2221
0
        return llvm::Intrinsic::x86_avx2_packssdw;
2222
0
2223
1
      case llvm::Intrinsic::x86_mmx_packsswb:
2224
1
      case llvm::Intrinsic::x86_mmx_packuswb:
2225
1
        return llvm::Intrinsic::x86_mmx_packsswb;
2226
1
2227
0
      case llvm::Intrinsic::x86_mmx_packssdw:
2228
0
        return llvm::Intrinsic::x86_mmx_packssdw;
2229
0
      default:
2230
0
        llvm_unreachable("unexpected intrinsic id");
2231
0
    }
2232
0
  }
2233
2234
  // \brief Instrument vector pack instrinsic.
2235
  //
2236
  // This function instruments intrinsics like x86_mmx_packsswb, that
2237
  // packs elements of 2 input vectors into half as many bits with saturation.
2238
  // Shadow is propagated with the signed variant of the same intrinsic applied
2239
  // to sext(Sa != zeroinitializer), sext(Sb != zeroinitializer).
2240
  // EltSizeInBits is used only for x86mmx arguments.
2241
3
  void handleVectorPackIntrinsic(IntrinsicInst &I, unsigned EltSizeInBits = 0) {
2242
3
    assert(I.getNumArgOperands() == 2);
2243
3
    bool isX86_MMX = I.getOperand(0)->getType()->isX86_MMXTy();
2244
3
    IRBuilder<> IRB(&I);
2245
3
    Value *S1 = getShadow(&I, 0);
2246
3
    Value *S2 = getShadow(&I, 1);
2247
3
    assert(isX86_MMX || S1->getType()->isVectorTy());
2248
3
2249
3
    // SExt and ICmpNE below must apply to individual elements of input vectors.
2250
3
    // In case of x86mmx arguments, cast them to appropriate vector types and
2251
3
    // back.
2252
3
    Type *T = isX86_MMX ? 
getMMXVectorTy(EltSizeInBits)1
:
S1->getType()2
;
2253
3
    if (
isX86_MMX3
) {
2254
1
      S1 = IRB.CreateBitCast(S1, T);
2255
1
      S2 = IRB.CreateBitCast(S2, T);
2256
1
    }
2257
3
    Value *S1_ext = IRB.CreateSExt(
2258
3
        IRB.CreateICmpNE(S1, llvm::Constant::getNullValue(T)), T);
2259
3
    Value *S2_ext = IRB.CreateSExt(
2260
3
        IRB.CreateICmpNE(S2, llvm::Constant::getNullValue(T)), T);
2261
3
    if (
isX86_MMX3
) {
2262
1
      Type *X86_MMXTy = Type::getX86_MMXTy(*MS.C);
2263
1
      S1_ext = IRB.CreateBitCast(S1_ext, X86_MMXTy);
2264
1
      S2_ext = IRB.CreateBitCast(S2_ext, X86_MMXTy);
2265
1
    }
2266
3
2267
3
    Function *ShadowFn = Intrinsic::getDeclaration(
2268
3
        F.getParent(), getSignedPackIntrinsic(I.getIntrinsicID()));
2269
3
2270
3
    Value *S =
2271
3
        IRB.CreateCall(ShadowFn, {S1_ext, S2_ext}, "_msprop_vector_pack");
2272
3
    if (
isX86_MMX3
)
S = IRB.CreateBitCast(S, getShadowTy(&I))1
;
2273
3
    setShadow(&I, S);
2274
3
    setOriginForNaryOp(I);
2275
3
  }
2276
2277
  // \brief Instrument sum-of-absolute-differencies intrinsic.
2278
2
  void handleVectorSadIntrinsic(IntrinsicInst &I) {
2279
2
    const unsigned SignificantBitsPerResultElement = 16;
2280
2
    bool isX86_MMX = I.getOperand(0)->getType()->isX86_MMXTy();
2281
2
    Type *ResTy = isX86_MMX ? 
IntegerType::get(*MS.C, 64)1
:
I.getType()1
;
2282
2
    unsigned ZeroBitsPerResultElement =
2283
2
        ResTy->getScalarSizeInBits() - SignificantBitsPerResultElement;
2284
2
2285
2
    IRBuilder<> IRB(&I);
2286
2
    Value *S = IRB.CreateOr(getShadow(&I, 0), getShadow(&I, 1));
2287
2
    S = IRB.CreateBitCast(S, ResTy);
2288
2
    S = IRB.CreateSExt(IRB.CreateICmpNE(S, Constant::getNullValue(ResTy)),
2289
2
                       ResTy);
2290
2
    S = IRB.CreateLShr(S, ZeroBitsPerResultElement);
2291
2
    S = IRB.CreateBitCast(S, getShadowTy(&I));
2292
2
    setShadow(&I, S);
2293
2
    setOriginForNaryOp(I);
2294
2
  }
2295
2296
  // \brief Instrument multiply-add intrinsic.
2297
  void handleVectorPmaddIntrinsic(IntrinsicInst &I,
2298
2
                                  unsigned EltSizeInBits = 0) {
2299
2
    bool isX86_MMX = I.getOperand(0)->getType()->isX86_MMXTy();
2300
2
    Type *ResTy = isX86_MMX ? 
getMMXVectorTy(EltSizeInBits * 2)1
:
I.getType()1
;
2301
2
    IRBuilder<> IRB(&I);
2302
2
    Value *S = IRB.CreateOr(getShadow(&I, 0), getShadow(&I, 1));
2303
2
    S = IRB.CreateBitCast(S, ResTy);
2304
2
    S = IRB.CreateSExt(IRB.CreateICmpNE(S, Constant::getNullValue(ResTy)),
2305
2
                       ResTy);
2306
2
    S = IRB.CreateBitCast(S, getShadowTy(&I));
2307
2
    setShadow(&I, S);
2308
2
    setOriginForNaryOp(I);
2309
2
  }
2310
2311
  // \brief Instrument compare-packed intrinsic.
2312
  // Basically, an or followed by sext(icmp ne 0) to end up with all-zeros or
2313
  // all-ones shadow.
2314
1
  void handleVectorComparePackedIntrinsic(IntrinsicInst &I) {
2315
1
    IRBuilder<> IRB(&I);
2316
1
    Type *ResTy = getShadowTy(&I);
2317
1
    Value *S0 = IRB.CreateOr(getShadow(&I, 0), getShadow(&I, 1));
2318
1
    Value *S = IRB.CreateSExt(
2319
1
        IRB.CreateICmpNE(S0, Constant::getNullValue(ResTy)), ResTy);
2320
1
    setShadow(&I, S);
2321
1
    setOriginForNaryOp(I);
2322
1
  }
2323
2324
  // \brief Instrument compare-scalar intrinsic.
2325
  // This handles both cmp* intrinsics which return the result in the first
2326
  // element of a vector, and comi* which return the result as i32.
2327
4
  void handleVectorCompareScalarIntrinsic(IntrinsicInst &I) {
2328
4
    IRBuilder<> IRB(&I);
2329
4
    Value *S0 = IRB.CreateOr(getShadow(&I, 0), getShadow(&I, 1));
2330
4
    Value *S = LowerElementShadowExtend(IRB, S0, getShadowTy(&I));
2331
4
    setShadow(&I, S);
2332
4
    setOriginForNaryOp(I);
2333
4
  }
2334
2335
2
  void handleStmxcsr(IntrinsicInst &I) {
2336
2
    IRBuilder<> IRB(&I);
2337
2
    Value* Addr = I.getArgOperand(0);
2338
2
    Type *Ty = IRB.getInt32Ty();
2339
2
    Value *ShadowPtr = getShadowPtr(Addr, Ty, IRB);
2340
2
2341
2
    IRB.CreateStore(getCleanShadow(Ty),
2342
2
                    IRB.CreatePointerCast(ShadowPtr, Ty->getPointerTo()));
2343
2
2344
2
    if (ClCheckAccessAddress)
2345
1
      insertShadowCheck(Addr, &I);
2346
2
  }
2347
2348
2
  void handleLdmxcsr(IntrinsicInst &I) {
2349
2
    if (
!InsertChecks2
)
return0
;
2350
2
2351
2
    IRBuilder<> IRB(&I);
2352
2
    Value *Addr = I.getArgOperand(0);
2353
2
    Type *Ty = IRB.getInt32Ty();
2354
2
    unsigned Alignment = 1;
2355
2
2356
2
    if (ClCheckAccessAddress)
2357
1
      insertShadowCheck(Addr, &I);
2358
2
2359
2
    Value *Shadow = IRB.CreateAlignedLoad(getShadowPtr(Addr, Ty, IRB),
2360
2
                                          Alignment, "_ldmxcsr");
2361
2
    Value *Origin = MS.TrackOrigins
2362
0
                        ? IRB.CreateLoad(getOriginPtr(Addr, IRB, Alignment))
2363
2
                        : getCleanOrigin();
2364
2
    insertShadowCheck(Shadow, Origin, &I);
2365
2
  }
2366
2367
60
  void visitIntrinsicInst(IntrinsicInst &I) {
2368
60
    switch (I.getIntrinsicID()) {
2369
2
    case llvm::Intrinsic::bswap:
2370
2
      handleBswap(I);
2371
2
      break;
2372
2
    case llvm::Intrinsic::x86_sse_stmxcsr:
2373
2
      handleStmxcsr(I);
2374
2
      break;
2375
2
    case llvm::Intrinsic::x86_sse_ldmxcsr:
2376
2
      handleLdmxcsr(I);
2377
2
      break;
2378
2
    case llvm::Intrinsic::x86_avx512_vcvtsd2usi64:
2379
2
    case llvm::Intrinsic::x86_avx512_vcvtsd2usi32:
2380
2
    case llvm::Intrinsic::x86_avx512_vcvtss2usi64:
2381
2
    case llvm::Intrinsic::x86_avx512_vcvtss2usi32:
2382
2
    case llvm::Intrinsic::x86_avx512_cvttss2usi64:
2383
2
    case llvm::Intrinsic::x86_avx512_cvttss2usi:
2384
2
    case llvm::Intrinsic::x86_avx512_cvttsd2usi64:
2385
2
    case llvm::Intrinsic::x86_avx512_cvttsd2usi:
2386
2
    case llvm::Intrinsic::x86_avx512_cvtusi2sd:
2387
2
    case llvm::Intrinsic::x86_avx512_cvtusi2ss:
2388
2
    case llvm::Intrinsic::x86_avx512_cvtusi642sd:
2389
2
    case llvm::Intrinsic::x86_avx512_cvtusi642ss:
2390
2
    case llvm::Intrinsic::x86_sse2_cvtsd2si64:
2391
2
    case llvm::Intrinsic::x86_sse2_cvtsd2si:
2392
2
    case llvm::Intrinsic::x86_sse2_cvtsd2ss:
2393
2
    case llvm::Intrinsic::x86_sse2_cvtsi2sd:
2394
2
    case llvm::Intrinsic::x86_sse2_cvtsi642sd:
2395
2
    case llvm::Intrinsic::x86_sse2_cvtss2sd:
2396
2
    case llvm::Intrinsic::x86_sse2_cvttsd2si64:
2397
2
    case llvm::Intrinsic::x86_sse2_cvttsd2si:
2398
2
    case llvm::Intrinsic::x86_sse_cvtsi2ss:
2399
2
    case llvm::Intrinsic::x86_sse_cvtsi642ss:
2400
2
    case llvm::Intrinsic::x86_sse_cvtss2si64:
2401
2
    case llvm::Intrinsic::x86_sse_cvtss2si:
2402
2
    case llvm::Intrinsic::x86_sse_cvttss2si64:
2403
2
    case llvm::Intrinsic::x86_sse_cvttss2si:
2404
2
      handleVectorConvertIntrinsic(I, 1);
2405
2
      break;
2406
1
    case llvm::Intrinsic::x86_sse_cvtps2pi:
2407
1
    case llvm::Intrinsic::x86_sse_cvttps2pi:
2408
1
      handleVectorConvertIntrinsic(I, 2);
2409
1
      break;
2410
1
2411
5
    case llvm::Intrinsic::x86_avx512_psll_w_512:
2412
5
    case llvm::Intrinsic::x86_avx512_psll_d_512:
2413
5
    case llvm::Intrinsic::x86_avx512_psll_q_512:
2414
5
    case llvm::Intrinsic::x86_avx512_pslli_w_512:
2415
5
    case llvm::Intrinsic::x86_avx512_pslli_d_512:
2416
5
    case llvm::Intrinsic::x86_avx512_pslli_q_512:
2417
5
    case llvm::Intrinsic::x86_avx512_psrl_w_512:
2418
5
    case llvm::Intrinsic::x86_avx512_psrl_d_512:
2419
5
    case llvm::Intrinsic::x86_avx512_psrl_q_512:
2420
5
    case llvm::Intrinsic::x86_avx512_psra_w_512:
2421
5
    case llvm::Intrinsic::x86_avx512_psra_d_512:
2422
5
    case llvm::Intrinsic::x86_avx512_psra_q_512:
2423
5
    case llvm::Intrinsic::x86_avx512_psrli_w_512:
2424
5
    case llvm::Intrinsic::x86_avx512_psrli_d_512:
2425
5
    case llvm::Intrinsic::x86_avx512_psrli_q_512:
2426
5
    case llvm::Intrinsic::x86_avx512_psrai_w_512:
2427
5
    case llvm::Intrinsic::x86_avx512_psrai_d_512:
2428
5
    case llvm::Intrinsic::x86_avx512_psrai_q_512:
2429
5
    case llvm::Intrinsic::x86_avx512_psra_q_256:
2430
5
    case llvm::Intrinsic::x86_avx512_psra_q_128:
2431
5
    case llvm::Intrinsic::x86_avx512_psrai_q_256:
2432
5
    case llvm::Intrinsic::x86_avx512_psrai_q_128:
2433
5
    case llvm::Intrinsic::x86_avx2_psll_w:
2434
5
    case llvm::Intrinsic::x86_avx2_psll_d:
2435
5
    case llvm::Intrinsic::x86_avx2_psll_q:
2436
5
    case llvm::Intrinsic::x86_avx2_pslli_w:
2437
5
    case llvm::Intrinsic::x86_avx2_pslli_d:
2438
5
    case llvm::Intrinsic::x86_avx2_pslli_q:
2439
5
    case llvm::Intrinsic::x86_avx2_psrl_w:
2440
5
    case llvm::Intrinsic::x86_avx2_psrl_d:
2441
5
    case llvm::Intrinsic::x86_avx2_psrl_q:
2442
5
    case llvm::Intrinsic::x86_avx2_psra_w:
2443
5
    case llvm::Intrinsic::x86_avx2_psra_d:
2444
5
    case llvm::Intrinsic::x86_avx2_psrli_w:
2445
5
    case llvm::Intrinsic::x86_avx2_psrli_d:
2446
5
    case llvm::Intrinsic::x86_avx2_psrli_q:
2447
5
    case llvm::Intrinsic::x86_avx2_psrai_w:
2448
5
    case llvm::Intrinsic::x86_avx2_psrai_d:
2449
5
    case llvm::Intrinsic::x86_sse2_psll_w:
2450
5
    case llvm::Intrinsic::x86_sse2_psll_d:
2451
5
    case llvm::Intrinsic::x86_sse2_psll_q:
2452
5
    case llvm::Intrinsic::x86_sse2_pslli_w:
2453
5
    case llvm::Intrinsic::x86_sse2_pslli_d:
2454
5
    case llvm::Intrinsic::x86_sse2_pslli_q:
2455
5
    case llvm::Intrinsic::x86_sse2_psrl_w:
2456
5
    case llvm::Intrinsic::x86_sse2_psrl_d:
2457
5
    case llvm::Intrinsic::x86_sse2_psrl_q:
2458
5
    case llvm::Intrinsic::x86_sse2_psra_w:
2459
5
    case llvm::Intrinsic::x86_sse2_psra_d:
2460
5
    case llvm::Intrinsic::x86_sse2_psrli_w:
2461
5
    case llvm::Intrinsic::x86_sse2_psrli_d:
2462
5
    case llvm::Intrinsic::x86_sse2_psrli_q:
2463
5
    case llvm::Intrinsic::x86_sse2_psrai_w:
2464
5
    case llvm::Intrinsic::x86_sse2_psrai_d:
2465
5
    case llvm::Intrinsic::x86_mmx_psll_w:
2466
5
    case llvm::Intrinsic::x86_mmx_psll_d:
2467
5
    case llvm::Intrinsic::x86_mmx_psll_q:
2468
5
    case llvm::Intrinsic::x86_mmx_pslli_w:
2469
5
    case llvm::Intrinsic::x86_mmx_pslli_d:
2470
5
    case llvm::Intrinsic::x86_mmx_pslli_q:
2471
5
    case llvm::Intrinsic::x86_mmx_psrl_w:
2472
5
    case llvm::Intrinsic::x86_mmx_psrl_d:
2473
5
    case llvm::Intrinsic::x86_mmx_psrl_q:
2474
5
    case llvm::Intrinsic::x86_mmx_psra_w:
2475
5
    case llvm::Intrinsic::x86_mmx_psra_d:
2476
5
    case llvm::Intrinsic::x86_mmx_psrli_w:
2477
5
    case llvm::Intrinsic::x86_mmx_psrli_d:
2478
5
    case llvm::Intrinsic::x86_mmx_psrli_q:
2479
5
    case llvm::Intrinsic::x86_mmx_psrai_w:
2480
5
    case llvm::Intrinsic::x86_mmx_psrai_d:
2481
5
      handleVectorShiftIntrinsic(I, /* Variable */ false);
2482
5
      break;
2483
3
    case llvm::Intrinsic::x86_avx2_psllv_d:
2484
3
    case llvm::Intrinsic::x86_avx2_psllv_d_256:
2485
3
    case llvm::Intrinsic::x86_avx512_psllv_d_512:
2486
3
    case llvm::Intrinsic::x86_avx2_psllv_q:
2487
3
    case llvm::Intrinsic::x86_avx2_psllv_q_256:
2488
3
    case llvm::Intrinsic::x86_avx512_psllv_q_512:
2489
3
    case llvm::Intrinsic::x86_avx2_psrlv_d:
2490
3
    case llvm::Intrinsic::x86_avx2_psrlv_d_256:
2491
3
    case llvm::Intrinsic::x86_avx512_psrlv_d_512:
2492
3
    case llvm::Intrinsic::x86_avx2_psrlv_q:
2493
3
    case llvm::Intrinsic::x86_avx2_psrlv_q_256:
2494
3
    case llvm::Intrinsic::x86_avx512_psrlv_q_512:
2495
3
    case llvm::Intrinsic::x86_avx2_psrav_d:
2496
3
    case llvm::Intrinsic::x86_avx2_psrav_d_256:
2497
3
    case llvm::Intrinsic::x86_avx512_psrav_d_512:
2498
3
    case llvm::Intrinsic::x86_avx512_psrav_q_128:
2499
3
    case llvm::Intrinsic::x86_avx512_psrav_q_256:
2500
3
    case llvm::Intrinsic::x86_avx512_psrav_q_512:
2501
3
      handleVectorShiftIntrinsic(I, /* Variable */ true);
2502
3
      break;
2503
3
2504
2
    case llvm::Intrinsic::x86_sse2_packsswb_128:
2505
2
    case llvm::Intrinsic::x86_sse2_packssdw_128:
2506
2
    case llvm::Intrinsic::x86_sse2_packuswb_128:
2507
2
    case llvm::Intrinsic::x86_sse41_packusdw:
2508
2
    case llvm::Intrinsic::x86_avx2_packsswb:
2509
2
    case llvm::Intrinsic::x86_avx2_packssdw:
2510
2
    case llvm::Intrinsic::x86_avx2_packuswb:
2511
2
    case llvm::Intrinsic::x86_avx2_packusdw:
2512
2
      handleVectorPackIntrinsic(I);
2513
2
      break;
2514
2
2515
1
    case llvm::Intrinsic::x86_mmx_packsswb:
2516
1
    case llvm::Intrinsic::x86_mmx_packuswb:
2517
1
      handleVectorPackIntrinsic(I, 16);
2518
1
      break;
2519
1
2520
0
    case llvm::Intrinsic::x86_mmx_packssdw:
2521
0
      handleVectorPackIntrinsic(I, 32);
2522
0
      break;
2523
1
2524
2
    case llvm::Intrinsic::x86_mmx_psad_bw:
2525
2
    case llvm::Intrinsic::x86_sse2_psad_bw:
2526
2
    case llvm::Intrinsic::x86_avx2_psad_bw:
2527
2
      handleVectorSadIntrinsic(I);
2528
2
      break;
2529
2
2530
1
    case llvm::Intrinsic::x86_sse2_pmadd_wd:
2531
1
    case llvm::Intrinsic::x86_avx2_pmadd_wd:
2532
1
    case llvm::Intrinsic::x86_ssse3_pmadd_ub_sw_128:
2533
1
    case llvm::Intrinsic::x86_avx2_pmadd_ub_sw:
2534
1
      handleVectorPmaddIntrinsic(I);
2535
1
      break;
2536
1
2537
1
    case llvm::Intrinsic::x86_ssse3_pmadd_ub_sw:
2538
1
      handleVectorPmaddIntrinsic(I, 8);
2539
1
      break;
2540
1
2541
0
    case llvm::Intrinsic::x86_mmx_pmadd_wd:
2542
0
      handleVectorPmaddIntrinsic(I, 16);
2543
0
      break;
2544
1
2545
4
    case llvm::Intrinsic::x86_sse_cmp_ss:
2546
4
    case llvm::Intrinsic::x86_sse2_cmp_sd:
2547
4
    case llvm::Intrinsic::x86_sse_comieq_ss:
2548
4
    case llvm::Intrinsic::x86_sse_comilt_ss:
2549
4
    case llvm::Intrinsic::x86_sse_comile_ss:
2550
4
    case llvm::Intrinsic::x86_sse_comigt_ss:
2551
4
    case llvm::Intrinsic::x86_sse_comige_ss:
2552
4
    case llvm::Intrinsic::x86_sse_comineq_ss:
2553
4
    case llvm::Intrinsic::x86_sse_ucomieq_ss:
2554
4
    case llvm::Intrinsic::x86_sse_ucomilt_ss:
2555
4
    case llvm::Intrinsic::x86_sse_ucomile_ss:
2556
4
    case llvm::Intrinsic::x86_sse_ucomigt_ss:
2557
4
    case llvm::Intrinsic::x86_sse_ucomige_ss:
2558
4
    case llvm::Intrinsic::x86_sse_ucomineq_ss:
2559
4
    case llvm::Intrinsic::x86_sse2_comieq_sd:
2560
4
    case llvm::Intrinsic::x86_sse2_comilt_sd:
2561
4
    case llvm::Intrinsic::x86_sse2_comile_sd:
2562
4
    case llvm::Intrinsic::x86_sse2_comigt_sd:
2563
4
    case llvm::Intrinsic::x86_sse2_comige_sd:
2564
4
    case llvm::Intrinsic::x86_sse2_comineq_sd:
2565
4
    case llvm::Intrinsic::x86_sse2_ucomieq_sd:
2566
4
    case llvm::Intrinsic::x86_sse2_ucomilt_sd:
2567
4
    case llvm::Intrinsic::x86_sse2_ucomile_sd:
2568
4
    case llvm::Intrinsic::x86_sse2_ucomigt_sd:
2569
4
    case llvm::Intrinsic::x86_sse2_ucomige_sd:
2570
4
    case llvm::Intrinsic::x86_sse2_ucomineq_sd:
2571
4
      handleVectorCompareScalarIntrinsic(I);
2572
4
      break;
2573
4
2574
1
    case llvm::Intrinsic::x86_sse_cmp_ps:
2575
1
    case llvm::Intrinsic::x86_sse2_cmp_pd:
2576
1
      // FIXME: For x86_avx_cmp_pd_256 and x86_avx_cmp_ps_256 this function
2577
1
      // generates reasonably looking IR that fails in the backend with "Do not
2578
1
      // know how to split the result of this operator!".
2579
1
      handleVectorComparePackedIntrinsic(I);
2580
1
      break;
2581
1
2582
31
    default:
2583
31
      if (!handleUnknownIntrinsic(I))
2584
27
        visitInstruction(I);
2585
2
      break;
2586
60
    }
2587
60
  }
2588
2589
165
  void visitCallSite(CallSite CS) {
2590
165
    Instruction &I = *CS.getInstruction();
2591
165
    if (
I.getMetadata("nosanitize")165
)
return36
;
2592
165
    assert((CS.isCall() || CS.isInvoke()) && "Unknown type of CallSite");
2593
129
    if (
CS.isCall()129
) {
2594
129
      CallInst *Call = cast<CallInst>(&I);
2595
129
2596
129
      // For inline asm, do the usual thing: check argument shadow and mark all
2597
129
      // outputs as clean. Note that any side effects of the inline asm that are
2598
129
      // not immediately visible in its constraints are not handled.
2599
129
      if (
Call->isInlineAsm()129
) {
2600
0
        visitInstruction(I);
2601
0
        return;
2602
0
      }
2603
129
2604
129
      assert(!isa<IntrinsicInst>(&I) && "intrinsics are handled elsewhere");
2605
129
2606
129
      // We are going to insert code that relies on the fact that the callee
2607
129
      // will become a non-readonly function after it is instrumented by us. To
2608
129
      // prevent this code from being optimized out, mark that function
2609
129
      // non-readonly in advance.
2610
129
      if (Function *
Func129
= Call->getCalledFunction()) {
2611
129
        // Clear out readonly/readnone attributes.
2612
129
        AttrBuilder B;
2613
129
        B.addAttribute(Attribute::ReadOnly)
2614
129
          .addAttribute(Attribute::ReadNone);
2615
129
        Func->removeAttributes(AttributeList::FunctionIndex, B);
2616
129
      }
2617
129
2618
129
      maybeMarkSanitizerLibraryCallNoBuiltin(Call, TLI);
2619
129
    }
2620
129
    IRBuilder<> IRB(&I);
2621
129
2622
129
    unsigned ArgOffset = 0;
2623
129
    DEBUG(dbgs() << "  CallSite: " << I << "\n");
2624
129
    for (CallSite::arg_iterator ArgIt = CS.arg_begin(), End = CS.arg_end();
2625
321
         
ArgIt != End321
;
++ArgIt192
) {
2626
192
      Value *A = *ArgIt;
2627
192
      unsigned i = ArgIt - CS.arg_begin();
2628
192
      if (
!A->getType()->isSized()192
) {
2629
0
        DEBUG(dbgs() << "Arg " << i << " is not sized: " << I << "\n");
2630
0
        continue;
2631
0
      }
2632
192
      unsigned Size = 0;
2633
192
      Value *Store = nullptr;
2634
192
      // Compute the Shadow for arg even if it is ByVal, because
2635
192
      // in that case getShadow() will copy the actual arg shadow to
2636
192
      // __msan_param_tls.
2637
192
      Value *ArgShadow = getShadow(A);
2638
192
      Value *ArgShadowBase = getShadowPtrForArgument(A, IRB, ArgOffset);
2639
192
      DEBUG(dbgs() << "  Arg#" << i << ": " << *A <<
2640
192
            " Shadow: " << *ArgShadow << "\n");
2641
192
      bool ArgIsInitialized = false;
2642
192
      const DataLayout &DL = F.getParent()->getDataLayout();
2643
192
      if (
CS.paramHasAttr(i, Attribute::ByVal)192
) {
2644
7
        assert(A->getType()->isPointerTy() &&
2645
7
               "ByVal argument is not a pointer!");
2646
7
        Size = DL.getTypeAllocSize(A->getType()->getPointerElementType());
2647
7
        if (
ArgOffset + Size > kParamTLSSize7
)
break0
;
2648
7
        unsigned ParamAlignment = CS.getParamAlignment(i);
2649
7
        unsigned Alignment = std::min(ParamAlignment, kShadowTLSAlignment);
2650
7
        Store = IRB.CreateMemCpy(ArgShadowBase,
2651
7
                                 getShadowPtr(A, Type::getInt8Ty(*MS.C), IRB),
2652
7
                                 Size, Alignment);
2653
192
      } else {
2654
185
        Size = DL.getTypeAllocSize(A->getType());
2655
185
        if (
ArgOffset + Size > kParamTLSSize185
)
break0
;
2656
185
        Store = IRB.CreateAlignedStore(ArgShadow, ArgShadowBase,
2657
185
                                       kShadowTLSAlignment);
2658
185
        Constant *Cst = dyn_cast<Constant>(ArgShadow);
2659
185
        if (
Cst && 185
Cst->isNullValue()104
)
ArgIsInitialized = true102
;
2660
185
      }
2661
192
      
if (192
MS.TrackOrigins && 192
!ArgIsInitialized12
)
2662
9
        IRB.CreateStore(getOrigin(A),
2663
9
                        getOriginPtrForArgument(A, IRB, ArgOffset));
2664
192
      (void)Store;
2665
192
      assert(Size != 0 && Store != nullptr);
2666
192
      DEBUG(dbgs() << "  Param:" << *Store << "\n");
2667
192
      ArgOffset += alignTo(Size, 8);
2668
192
    }
2669
129
    DEBUG(dbgs() << "  done with call args\n");
2670
129
2671
129
    FunctionType *FT =
2672
129
      cast<FunctionType>(CS.getCalledValue()->getType()->getContainedType(0));
2673
129
    if (
FT->isVarArg()129
) {
2674
25
      VAHelper->visitCallSite(CS, IRB);
2675
25
    }
2676
129
2677
129
    // Now, get the shadow for the RetVal.
2678
129
    if (
!I.getType()->isSized()129
)
return79
;
2679
50
    // Don't emit the epilogue for musttail call returns.
2680
50
    
if (50
CS.isCall() && 50
cast<CallInst>(&I)->isMustTailCall()50
)
return4
;
2681
46
    IRBuilder<> IRBBefore(&I);
2682
46
    // Until we have full dynamic coverage, make sure the retval shadow is 0.
2683
46
    Value *Base = getShadowPtrForRetval(&I, IRBBefore);
2684
46
    IRBBefore.CreateAlignedStore(getCleanShadow(&I), Base, kShadowTLSAlignment);
2685
46
    BasicBlock::iterator NextInsn;
2686
46
    if (
CS.isCall()46
) {
2687
46
      NextInsn = ++I.getIterator();
2688
46
      assert(NextInsn != I.getParent()->end());
2689
46
    } else {
2690
0
      BasicBlock *NormalDest = cast<InvokeInst>(&I)->getNormalDest();
2691
0
      if (
!NormalDest->getSinglePredecessor()0
) {
2692
0
        // FIXME: this case is tricky, so we are just conservative here.
2693
0
        // Perhaps we need to split the edge between this BB and NormalDest,
2694
0
        // but a naive attempt to use SplitEdge leads to a crash.
2695
0
        setShadow(&I, getCleanShadow(&I));
2696
0
        setOrigin(&I, getCleanOrigin());
2697
0
        return;
2698
0
      }
2699
0
      NextInsn = NormalDest->getFirstInsertionPt();
2700
0
      assert(NextInsn != NormalDest->end() &&
2701
0
             "Could not find insertion point for retval shadow load");
2702
0
    }
2703
46
    IRBuilder<> IRBAfter(&*NextInsn);
2704
46
    Value *RetvalShadow =
2705
46
      IRBAfter.CreateAlignedLoad(getShadowPtrForRetval(&I, IRBAfter),
2706
46
                                 kShadowTLSAlignment, "_msret");
2707
46
    setShadow(&I, RetvalShadow);
2708
46
    if (MS.TrackOrigins)
2709
5
      setOrigin(&I, IRBAfter.CreateLoad(getOriginPtrForRetval(IRBAfter)));
2710
165
  }
2711
2712
195
  bool isAMustTailRetVal(Value *RetVal) {
2713
195
    if (auto *
I195
= dyn_cast<BitCastInst>(RetVal)) {
2714
2
      RetVal = I->getOperand(0);
2715
2
    }
2716
195
    if (auto *
I195
= dyn_cast<CallInst>(RetVal)) {
2717
57
      return I->isMustTailCall();
2718
57
    }
2719
138
    return false;
2720
138
  }
2721
2722
373
  void visitReturnInst(ReturnInst &I) {
2723
373
    IRBuilder<> IRB(&I);
2724
373
    Value *RetVal = I.getReturnValue();
2725
373
    if (
!RetVal373
)
return178
;
2726
195
    // Don't emit the epilogue for musttail call returns.
2727
195
    
if (195
isAMustTailRetVal(RetVal)195
)
return4
;
2728
191
    Value *ShadowPtr = getShadowPtrForRetval(RetVal, IRB);
2729
191
    if (
CheckReturnValue191
) {
2730
3
      insertShadowCheck(RetVal, &I);
2731
3
      Value *Shadow = getCleanShadow(RetVal);
2732
3
      IRB.CreateAlignedStore(Shadow, ShadowPtr, kShadowTLSAlignment);
2733
191
    } else {
2734
188
      Value *Shadow = getShadow(RetVal);
2735
188
      IRB.CreateAlignedStore(Shadow, ShadowPtr, kShadowTLSAlignment);
2736
188
      if (MS.TrackOrigins)
2737
60
        IRB.CreateStore(getOrigin(RetVal), getOriginPtrForRetval(IRB));
2738
188
    }
2739
373
  }
2740
2741
4
  void visitPHINode(PHINode &I) {
2742
4
    IRBuilder<> IRB(&I);
2743
4
    if (
!PropagateShadow4
) {
2744
2
      setShadow(&I, getCleanShadow(&I));
2745
2
      setOrigin(&I, getCleanOrigin());
2746
2
      return;
2747
2
    }
2748
2
2749
2
    ShadowPHINodes.push_back(&I);
2750
2
    setShadow(&I, IRB.CreatePHI(getShadowTy(&I), I.getNumIncomingValues(),
2751
2
                                "_msphi_s"));
2752
2
    if (MS.TrackOrigins)
2753
1
      setOrigin(&I, IRB.CreatePHI(MS.OriginTy, I.getNumIncomingValues(),
2754
1
                                  "_msphi_o"));
2755
4
  }
2756
2757
78
  void visitAllocaInst(AllocaInst &I) {
2758
78
    setShadow(&I, getCleanShadow(&I));
2759
78
    setOrigin(&I, getCleanOrigin());
2760
78
    IRBuilder<> IRB(I.getNextNode());
2761
78
    const DataLayout &DL = F.getParent()->getDataLayout();
2762
78
    uint64_t TypeSize = DL.getTypeAllocSize(I.getAllocatedType());
2763
78
    Value *Len = ConstantInt::get(MS.IntptrTy, TypeSize);
2764
78
    if (I.isArrayAllocation())
2765
8
      Len = IRB.CreateMul(Len, I.getArraySize());
2766
78
    if (
PoisonStack && 78
ClPoisonStackWithCall68
) {
2767
4
      IRB.CreateCall(MS.MsanPoisonStackFn,
2768
4
                     {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len});
2769
78
    } else {
2770
74
      Value *ShadowBase = getShadowPtr(&I, Type::getInt8PtrTy(*MS.C), IRB);
2771
74
      Value *PoisonValue = IRB.getInt8(PoisonStack ? 
ClPoisonStackPattern64
:
010
);
2772
74
      IRB.CreateMemSet(ShadowBase, PoisonValue, Len, I.getAlignment());
2773
74
    }
2774
78
2775
78
    if (
PoisonStack && 78
MS.TrackOrigins68
) {
2776
13
      SmallString<2048> StackDescriptionStorage;
2777
13
      raw_svector_ostream StackDescription(StackDescriptionStorage);
2778
13
      // We create a string with a description of the stack allocation and
2779
13
      // pass it into __msan_set_alloca_origin.
2780
13
      // It will be printed by the run-time if stack-originated UMR is found.
2781
13
      // The first 4 bytes of the string are set to '----' and will be replaced
2782
13
      // by __msan_va_arg_overflow_size_tls at the first call.
2783
13
      StackDescription << "----" << I.getName() << "@" << F.getName();
2784
13
      Value *Descr =
2785
13
          createPrivateNonConstGlobalForString(*F.getParent(),
2786
13
                                               StackDescription.str());
2787
13
2788
13
      IRB.CreateCall(MS.MsanSetAllocaOrigin4Fn,
2789
13
                     {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len,
2790
13
                      IRB.CreatePointerCast(Descr, IRB.getInt8PtrTy()),
2791
13
                      IRB.CreatePointerCast(&F, MS.IntptrTy)});
2792
13
    }
2793
78
  }
2794
2795
11
  void visitSelectInst(SelectInst& I) {
2796
11
    IRBuilder<> IRB(&I);
2797
11
    // a = select b, c, d
2798
11
    Value *B = I.getCondition();
2799
11
    Value *C = I.getTrueValue();
2800
11
    Value *D = I.getFalseValue();
2801
11
    Value *Sb = getShadow(B);
2802
11
    Value *Sc = getShadow(C);
2803
11
    Value *Sd = getShadow(D);
2804
11
2805
11
    // Result shadow if condition shadow is 0.
2806
11
    Value *Sa0 = IRB.CreateSelect(B, Sc, Sd);
2807
11
    Value *Sa1;
2808
11
    if (
I.getType()->isAggregateType()11
) {
2809
4
      // To avoid "sign extending" i1 to an arbitrary aggregate type, we just do
2810
4
      // an extra "select". This results in much more compact IR.
2811
4
      // Sa = select Sb, poisoned, (select b, Sc, Sd)
2812
4
      Sa1 = getPoisonedShadow(getShadowTy(I.getType()));
2813
11
    } else {
2814
7
      // Sa = select Sb, [ (c^d) | Sc | Sd ], [ b ? Sc : Sd ]
2815
7
      // If Sb (condition is poisoned), look for bits in c and d that are equal
2816
7
      // and both unpoisoned.
2817
7
      // If !Sb (condition is unpoisoned), simply pick one of Sc and Sd.
2818
7
2819
7
      // Cast arguments to shadow-compatible type.
2820
7
      C = CreateAppToShadowCast(IRB, C);
2821
7
      D = CreateAppToShadowCast(IRB, D);
2822
7
2823
7
      // Result shadow if condition shadow is 1.
2824
7
      Sa1 = IRB.CreateOr(IRB.CreateXor(C, D), IRB.CreateOr(Sc, Sd));
2825
7
    }
2826
11
    Value *Sa = IRB.CreateSelect(Sb, Sa1, Sa0, "_msprop_select");
2827
11
    setShadow(&I, Sa);
2828
11
    if (
MS.TrackOrigins11
) {
2829
6
      // Origins are always i32, so any vector conditions must be flattened.
2830
6
      // FIXME: consider tracking vector origins for app vectors?
2831
6
      if (
B->getType()->isVectorTy()6
) {
2832
1
        Type *FlatTy = getShadowTyNoVec(B->getType());
2833
1
        B = IRB.CreateICmpNE(IRB.CreateBitCast(B, FlatTy),
2834
1
                                ConstantInt::getNullValue(FlatTy));
2835
1
        Sb = IRB.CreateICmpNE(IRB.CreateBitCast(Sb, FlatTy),
2836
1
                                      ConstantInt::getNullValue(FlatTy));
2837
1
      }
2838
6
      // a = select b, c, d
2839
6
      // Oa = Sb ? Ob : (b ? Oc : Od)
2840
6
      setOrigin(
2841
6
          &I, IRB.CreateSelect(Sb, getOrigin(I.getCondition()),
2842
6
                               IRB.CreateSelect(B, getOrigin(I.getTrueValue()),
2843
6
                                                getOrigin(I.getFalseValue()))));
2844
6
    }
2845
11
  }
2846
2847
0
  void visitLandingPadInst(LandingPadInst &I) {
2848
0
    // Do nothing.
2849
0
    // See http://code.google.com/p/memory-sanitizer/issues/detail?id=1
2850
0
    setShadow(&I, getCleanShadow(&I));
2851
0
    setOrigin(&I, getCleanOrigin());
2852
0
  }
2853
2854
0
  void visitCatchSwitchInst(CatchSwitchInst &I) {
2855
0
    setShadow(&I, getCleanShadow(&I));
2856
0
    setOrigin(&I, getCleanOrigin());
2857
0
  }
2858
2859
0
  void visitFuncletPadInst(FuncletPadInst &I) {
2860
0
    setShadow(&I, getCleanShadow(&I));
2861
0
    setOrigin(&I, getCleanOrigin());
2862
0
  }
2863
2864
72
  void visitGetElementPtrInst(GetElementPtrInst &I) {
2865
72
    handleShadowOr(I);
2866
72
  }
2867
2868
14
  void visitExtractValueInst(ExtractValueInst &I) {
2869
14
    IRBuilder<> IRB(&I);
2870
14
    Value *Agg = I.getAggregateOperand();
2871
14
    DEBUG(dbgs() << "ExtractValue:  " << I << "\n");
2872
14
    Value *AggShadow = getShadow(Agg);
2873
14
    DEBUG(dbgs() << "   AggShadow:  " << *AggShadow << "\n");
2874
14
    Value *ResShadow = IRB.CreateExtractValue(AggShadow, I.getIndices());
2875
14
    DEBUG(dbgs() << "   ResShadow:  " << *ResShadow << "\n");
2876
14
    setShadow(&I, ResShadow);
2877
14
    setOriginForNaryOp(I);
2878
14
  }
2879
2880
12
  void visitInsertValueInst(InsertValueInst &I) {
2881
12
    IRBuilder<> IRB(&I);
2882
12
    DEBUG(dbgs() << "InsertValue:  " << I << "\n");
2883
12
    Value *AggShadow = getShadow(I.getAggregateOperand());
2884
12
    Value *InsShadow = getShadow(I.getInsertedValueOperand());
2885
12
    DEBUG(dbgs() << "   AggShadow:  " << *AggShadow << "\n");
2886
12
    DEBUG(dbgs() << "   InsShadow:  " << *InsShadow << "\n");
2887
12
    Value *Res = IRB.CreateInsertValue(AggShadow, InsShadow, I.getIndices());
2888
12
    DEBUG(dbgs() << "   Res:        " << *Res << "\n");
2889
12
    setShadow(&I, Res);
2890
12
    setOriginForNaryOp(I);
2891
12
  }
2892
2893
0
  void dumpInst(Instruction &I) {
2894
0
    if (CallInst *
CI0
= dyn_cast<CallInst>(&I)) {
2895
0
      errs() << "ZZZ call " << CI->getCalledFunction()->getName() << "\n";
2896
0
    } else {
2897
0
      errs() << "ZZZ " << I.getOpcodeName() << "\n";
2898
0
    }
2899
0
    errs() << "QQQ " << I << "\n";
2900
0
  }
2901
2902
0
  void visitResumeInst(ResumeInst &I) {
2903
0
    DEBUG(dbgs() << "Resume: " << I << "\n");
2904
0
    // Nothing to do here.
2905
0
  }
2906
2907
0
  void visitCleanupReturnInst(CleanupReturnInst &CRI) {
2908
0
    DEBUG(dbgs() << "CleanupReturn: " << CRI << "\n");
2909
0
    // Nothing to do here.
2910
0
  }
2911
2912
0
  void visitCatchReturnInst(CatchReturnInst &CRI) {
2913
0
    DEBUG(dbgs() << "CatchReturn: " << CRI << "\n");
2914
0
    // Nothing to do here.
2915
0
  }
2916
2917
58
  void visitInstruction(Instruction &I) {
2918
58
    // Everything else: stop propagating and check for poisoned shadow.
2919
58
    if (ClDumpStrictInstructions)
2920
0
      dumpInst(I);
2921
58
    DEBUG(dbgs() << "DEFAULT: " << I << "\n");
2922
204
    for (size_t i = 0, n = I.getNumOperands(); 
i < n204
;
i++146
) {
2923
146
      Value *Operand = I.getOperand(i);
2924
146
      if (Operand->getType()->isSized())
2925
91
        insertShadowCheck(Operand, &I);
2926
146
    }
2927
58
    setShadow(&I, getCleanShadow(&I));
2928
58
    setOrigin(&I, getCleanOrigin());
2929
58
  }
2930
};
2931
2932
/// \brief AMD64-specific implementation of VarArgHelper.
2933
struct VarArgAMD64Helper : public VarArgHelper {
2934
  // An unfortunate workaround for asymmetric lowering of va_arg stuff.
2935
  // See a comment in visitCallSite for more details.
2936
  static const unsigned AMD64GpEndOffset = 48;  // AMD64 ABI Draft 0.99.6 p3.5.7
2937
  static const unsigned AMD64FpEndOffset = 176;
2938
2939
  Function &F;
2940
  MemorySanitizer &MS;
2941
  MemorySanitizerVisitor &MSV;
2942
  Value *VAArgTLSCopy;
2943
  Value *VAArgOverflowSize;
2944
2945
  SmallVector<CallInst*, 16> VAStartInstrumentationList;
2946
2947
  VarArgAMD64Helper(Function &F, MemorySanitizer &MS,
2948
                    MemorySanitizerVisitor &MSV)
2949
    : F(F), MS(MS), MSV(MSV), VAArgTLSCopy(nullptr),
2950
344
      VAArgOverflowSize(nullptr) {}
2951
2952
  enum ArgKind { AK_GeneralPurpose, AK_FloatingPoint, AK_Memory };
2953
2954
10
  ArgKind classifyArgument(Value* arg) {
2955
10
    // A very rough approximation of X86_64 argument classification rules.
2956
10
    Type *T = arg->getType();
2957
10
    if (
T->isFPOrFPVectorTy() || 10
T->isX86_MMXTy()10
)
2958
0
      return AK_FloatingPoint;
2959
10
    
if (10
T->isIntegerTy() && 10
T->getPrimitiveSizeInBits() <= 6410
)
2960
10
      return AK_GeneralPurpose;
2961
0
    
if (0
T->isPointerTy()0
)
2962
0
      return AK_GeneralPurpose;
2963
0
    return AK_Memory;
2964
0
  }
2965
2966
  // For VarArg functions, store the argument shadow in an ABI-specific format
2967
  // that corresponds to va_list layout.
2968
  // We do this because Clang lowers va_arg in the frontend, and this pass
2969
  // only sees the low level code that deals with va_list internals.
2970
  // A much easier alternative (provided that Clang emits va_arg instructions)
2971
  // would have been to associate each live instance of va_list with a copy of
2972
  // MSanParamTLS, and extract shadow on va_arg() call in the argument list
2973
  // order.
2974
7
  void visitCallSite(CallSite &CS, IRBuilder<> &IRB) override {
2975
7
    unsigned GpOffset = 0;
2976
7
    unsigned FpOffset = AMD64GpEndOffset;
2977
7
    unsigned OverflowOffset = AMD64FpEndOffset;
2978
7
    const DataLayout &DL = F.getParent()->getDataLayout();
2979
7
    for (CallSite::arg_iterator ArgIt = CS.arg_begin(), End = CS.arg_end();
2980
19
         
ArgIt != End19
;
++ArgIt12
) {
2981
12
      Value *A = *ArgIt;
2982
12
      unsigned ArgNo = CS.getArgumentNo(ArgIt);
2983
12
      bool IsFixed = ArgNo < CS.getFunctionType()->getNumParams();
2984
12
      bool IsByVal = CS.paramHasAttr(ArgNo, Attribute::ByVal);
2985
12
      if (
IsByVal12
) {
2986
2
        // ByVal arguments always go to the overflow area.
2987
2
        // Fixed arguments passed through the overflow area will be stepped
2988
2
        // over by va_start, so don't count them towards the offset.
2989
2
        if (IsFixed)
2990
0
          continue;
2991
2
        assert(A->getType()->isPointerTy());
2992
2
        Type *RealTy = A->getType()->getPointerElementType();
2993
2
        uint64_t ArgSize = DL.getTypeAllocSize(RealTy);
2994
2
        Value *Base = getShadowPtrForVAArgument(RealTy, IRB, OverflowOffset);
2995
2
        OverflowOffset += alignTo(ArgSize, 8);
2996
2
        IRB.CreateMemCpy(Base, MSV.getShadowPtr(A, IRB.getInt8Ty(), IRB),
2997
2
                         ArgSize, kShadowTLSAlignment);
2998
12
      } else {
2999
10
        ArgKind AK = classifyArgument(A);
3000
10
        if (
AK == AK_GeneralPurpose && 10
GpOffset >= AMD64GpEndOffset10
)
3001
0
          AK = AK_Memory;
3002
10
        if (
AK == AK_FloatingPoint && 10
FpOffset >= AMD64FpEndOffset0
)
3003
0
          AK = AK_Memory;
3004
10
        Value *Base;
3005
10
        switch (AK) {
3006
10
          case AK_GeneralPurpose:
3007
10
            Base = getShadowPtrForVAArgument(A->getType(), IRB, GpOffset);
3008
10
            GpOffset += 8;
3009
10
            break;
3010
0
          case AK_FloatingPoint:
3011
0
            Base = getShadowPtrForVAArgument(A->getType(), IRB, FpOffset);
3012
0
            FpOffset += 16;
3013
0
            break;
3014
0
          case AK_Memory:
3015
0
            if (IsFixed)
3016
0
              continue;
3017
0
            uint64_t ArgSize = DL.getTypeAllocSize(A->getType());
3018
0
            Base = getShadowPtrForVAArgument(A->getType(), IRB, OverflowOffset);
3019
0
            OverflowOffset += alignTo(ArgSize, 8);
3020
10
        }
3021
10
        // Take fixed arguments into account for GpOffset and FpOffset,
3022
10
        // but don't actually store shadows for them.
3023
10
        
if (10
IsFixed10
)
3024
2
          continue;
3025
8
        IRB.CreateAlignedStore(MSV.getShadow(A), Base, kShadowTLSAlignment);
3026
8
      }
3027
12
    }
3028
7
    Constant *OverflowSize =
3029
7
      ConstantInt::get(IRB.getInt64Ty(), OverflowOffset - AMD64FpEndOffset);
3030
7
    IRB.CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS);
3031
7
  }
3032
3033
  /// \brief Compute the shadow address for a given va_arg.
3034
  Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB,
3035
12
                                   int ArgOffset) {
3036
12
    Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
3037
12
    Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
3038
12
    return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0),
3039
12
                              "_msarg");
3040
12
  }
3041
3042
2
  void visitVAStartInst(VAStartInst &I) override {
3043
2
    if (F.getCallingConv() == CallingConv::Win64)
3044
0
      return;
3045
2
    IRBuilder<> IRB(&I);
3046
2
    VAStartInstrumentationList.push_back(&I);
3047
2
    Value *VAListTag = I.getArgOperand(0);
3048
2
    Value *ShadowPtr = MSV.getShadowPtr(VAListTag, IRB.getInt8Ty(), IRB);
3049
2
3050
2
    // Unpoison the whole __va_list_tag.
3051
2
    // FIXME: magic ABI constants.
3052
2
    IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
3053
2
                     /* size */24, /* alignment */8, false);
3054
2
  }
3055
3056
2
  void visitVACopyInst(VACopyInst &I) override {
3057
2
    if (F.getCallingConv() == CallingConv::Win64)
3058
0
      return;
3059
2
    IRBuilder<> IRB(&I);
3060
2
    Value *VAListTag = I.getArgOperand(0);
3061
2
    Value *ShadowPtr = MSV.getShadowPtr(VAListTag, IRB.getInt8Ty(), IRB);
3062
2
3063
2
    // Unpoison the whole __va_list_tag.
3064
2
    // FIXME: magic ABI constants.
3065
2
    IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
3066
2
                     /* size */24, /* alignment */8, false);
3067
2
  }
3068
3069
344
  void finalizeInstrumentation() override {
3070
344
    assert(!VAArgOverflowSize && !VAArgTLSCopy &&
3071
344
           "finalizeInstrumentation called twice");
3072
344
    if (
!VAStartInstrumentationList.empty()344
) {
3073
2
      // If there is a va_start in this function, make a backup copy of
3074
2
      // va_arg_tls somewhere in the function entry block.
3075
2
      IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI());
3076
2
      VAArgOverflowSize = IRB.CreateLoad(MS.VAArgOverflowSizeTLS);
3077
2
      Value *CopySize =
3078
2
        IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, AMD64FpEndOffset),
3079
2
                      VAArgOverflowSize);
3080
2
      VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
3081
2
      IRB.CreateMemCpy(VAArgTLSCopy, MS.VAArgTLS, CopySize, 8);
3082
2
    }
3083
344
3084
344
    // Instrument va_start.
3085
344
    // Copy va_list shadow from the backup copy of the TLS contents.
3086
346
    for (size_t i = 0, n = VAStartInstrumentationList.size(); 
i < n346
;
i++2
) {
3087
2
      CallInst *OrigInst = VAStartInstrumentationList[i];
3088
2
      IRBuilder<> IRB(OrigInst->getNextNode());
3089
2
      Value *VAListTag = OrigInst->getArgOperand(0);
3090
2
3091
2
      Value *RegSaveAreaPtrPtr =
3092
2
        IRB.CreateIntToPtr(
3093
2
          IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
3094
2
                        ConstantInt::get(MS.IntptrTy, 16)),
3095
2
          Type::getInt64PtrTy(*MS.C));
3096
2
      Value *RegSaveAreaPtr = IRB.CreateLoad(RegSaveAreaPtrPtr);
3097
2
      Value *RegSaveAreaShadowPtr =
3098
2
        MSV.getShadowPtr(RegSaveAreaPtr, IRB.getInt8Ty(), IRB);
3099
2
      IRB.CreateMemCpy(RegSaveAreaShadowPtr, VAArgTLSCopy,
3100
2
                       AMD64FpEndOffset, 16);
3101
2
3102
2
      Value *OverflowArgAreaPtrPtr =
3103
2
        IRB.CreateIntToPtr(
3104
2
          IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
3105
2
                        ConstantInt::get(MS.IntptrTy, 8)),
3106
2
          Type::getInt64PtrTy(*MS.C));
3107
2
      Value *OverflowArgAreaPtr = IRB.CreateLoad(OverflowArgAreaPtrPtr);
3108
2
      Value *OverflowArgAreaShadowPtr =
3109
2
        MSV.getShadowPtr(OverflowArgAreaPtr, IRB.getInt8Ty(), IRB);
3110
2
      Value *SrcPtr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), VAArgTLSCopy,
3111
2
                                             AMD64FpEndOffset);
3112
2
      IRB.CreateMemCpy(OverflowArgAreaShadowPtr, SrcPtr, VAArgOverflowSize, 16);
3113
2
    }
3114
344
  }
3115
};
3116
3117
/// \brief MIPS64-specific implementation of VarArgHelper.
3118
struct VarArgMIPS64Helper : public VarArgHelper {
3119
  Function &F;
3120
  MemorySanitizer &MS;
3121
  MemorySanitizerVisitor &MSV;
3122
  Value *VAArgTLSCopy;
3123
  Value *VAArgSize;
3124
3125
  SmallVector<CallInst*, 16> VAStartInstrumentationList;
3126
3127
  VarArgMIPS64Helper(Function &F, MemorySanitizer &MS,
3128
                    MemorySanitizerVisitor &MSV)
3129
    : F(F), MS(MS), MSV(MSV), VAArgTLSCopy(nullptr),
3130
8
      VAArgSize(nullptr) {}
3131
3132
4
  void visitCallSite(CallSite &CS, IRBuilder<> &IRB) override {
3133
4
    unsigned VAArgOffset = 0;
3134
4
    const DataLayout &DL = F.getParent()->getDataLayout();
3135
4
    for (CallSite::arg_iterator ArgIt = CS.arg_begin() +
3136
4
         CS.getFunctionType()->getNumParams(), End = CS.arg_end();
3137
14
         
ArgIt != End14
;
++ArgIt10
) {
3138
10
      llvm::Triple TargetTriple(F.getParent()->getTargetTriple());
3139
10
      Value *A = *ArgIt;
3140
10
      Value *Base;
3141
10
      uint64_t ArgSize = DL.getTypeAllocSize(A->getType());
3142
10
      if (
TargetTriple.getArch() == llvm::Triple::mips6410
) {
3143
5
        // Adjusting the shadow for argument with size < 8 to match the placement
3144
5
        // of bits in big endian system
3145
5
        if (ArgSize < 8)
3146
1
          VAArgOffset += (8 - ArgSize);
3147
5
      }
3148
10
      Base = getShadowPtrForVAArgument(A->getType(), IRB, VAArgOffset);
3149
10
      VAArgOffset += ArgSize;
3150
10
      VAArgOffset = alignTo(VAArgOffset, 8);
3151
10
      IRB.CreateAlignedStore(MSV.getShadow(A), Base, kShadowTLSAlignment);
3152
10
    }
3153
4
3154
4
    Constant *TotalVAArgSize = ConstantInt::get(IRB.getInt64Ty(), VAArgOffset);
3155
4
    // Here using VAArgOverflowSizeTLS as VAArgSizeTLS to avoid creation of
3156
4
    // a new class member i.e. it is the total size of all VarArgs.
3157
4
    IRB.CreateStore(TotalVAArgSize, MS.VAArgOverflowSizeTLS);
3158
4
  }
3159
3160
  /// \brief Compute the shadow address for a given va_arg.
3161
  Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB,
3162
10
                                   int ArgOffset) {
3163
10
    Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
3164
10
    Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
3165
10
    return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0),
3166
10
                              "_msarg");
3167
10
  }
3168
3169
2
  void visitVAStartInst(VAStartInst &I) override {
3170
2
    IRBuilder<> IRB(&I);
3171
2
    VAStartInstrumentationList.push_back(&I);
3172
2
    Value *VAListTag = I.getArgOperand(0);
3173
2
    Value *ShadowPtr = MSV.getShadowPtr(VAListTag, IRB.getInt8Ty(), IRB);
3174
2
    IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
3175
2
                     /* size */8, /* alignment */8, false);
3176
2
  }
3177
3178
0
  void visitVACopyInst(VACopyInst &I) override {
3179
0
    IRBuilder<> IRB(&I);
3180
0
    Value *VAListTag = I.getArgOperand(0);
3181
0
    Value *ShadowPtr = MSV.getShadowPtr(VAListTag, IRB.getInt8Ty(), IRB);
3182
0
    // Unpoison the whole __va_list_tag.
3183
0
    // FIXME: magic ABI constants.
3184
0
    IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
3185
0
                     /* size */8, /* alignment */8, false);
3186
0
  }
3187
3188
8
  void finalizeInstrumentation() override {
3189
8
    assert(!VAArgSize && !VAArgTLSCopy &&
3190
8
           "finalizeInstrumentation called twice");
3191
8
    IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI());
3192
8
    VAArgSize = IRB.CreateLoad(MS.VAArgOverflowSizeTLS);
3193
8
    Value *CopySize = IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, 0),
3194
8
                                    VAArgSize);
3195
8
3196
8
    if (
!VAStartInstrumentationList.empty()8
) {
3197
2
      // If there is a va_start in this function, make a backup copy of
3198
2
      // va_arg_tls somewhere in the function entry block.
3199
2
      VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
3200
2
      IRB.CreateMemCpy(VAArgTLSCopy, MS.VAArgTLS, CopySize, 8);
3201
2
    }
3202
8
3203
8
    // Instrument va_start.
3204
8
    // Copy va_list shadow from the backup copy of the TLS contents.
3205
10
    for (size_t i = 0, n = VAStartInstrumentationList.size(); 
i < n10
;
i++2
) {
3206
2
      CallInst *OrigInst = VAStartInstrumentationList[i];
3207
2
      IRBuilder<> IRB(OrigInst->getNextNode());
3208
2
      Value *VAListTag = OrigInst->getArgOperand(0);
3209
2
      Value *RegSaveAreaPtrPtr =
3210
2
        IRB.CreateIntToPtr(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
3211
2
                        Type::getInt64PtrTy(*MS.C));
3212
2
      Value *RegSaveAreaPtr = IRB.CreateLoad(RegSaveAreaPtrPtr);
3213
2
      Value *RegSaveAreaShadowPtr =
3214
2
      MSV.getShadowPtr(RegSaveAreaPtr, IRB.getInt8Ty(), IRB);
3215
2
      IRB.CreateMemCpy(RegSaveAreaShadowPtr, VAArgTLSCopy, CopySize, 8);
3216
2
    }
3217
8
  }
3218
};
3219
3220
3221
/// \brief AArch64-specific implementation of VarArgHelper.
3222
struct VarArgAArch64Helper : public VarArgHelper {
3223
  static const unsigned kAArch64GrArgSize = 64;
3224
  static const unsigned kAArch64VrArgSize = 128;
3225
3226
  static const unsigned AArch64GrBegOffset = 0;
3227
  static const unsigned AArch64GrEndOffset = kAArch64GrArgSize;
3228
  // Make VR space aligned to 16 bytes.
3229
  static const unsigned AArch64VrBegOffset = AArch64GrEndOffset;
3230
  static const unsigned AArch64VrEndOffset = AArch64VrBegOffset
3231
                                             + kAArch64VrArgSize;
3232
  static const unsigned AArch64VAEndOffset = AArch64VrEndOffset;
3233
3234
  Function &F;
3235
  MemorySanitizer &MS;
3236
  MemorySanitizerVisitor &MSV;
3237
  Value *VAArgTLSCopy;
3238
  Value *VAArgOverflowSize;
3239
3240
  SmallVector<CallInst*, 16> VAStartInstrumentationList;
3241
3242
  VarArgAArch64Helper(Function &F, MemorySanitizer &MS,
3243
                    MemorySanitizerVisitor &MSV)
3244
    : F(F), MS(MS), MSV(MSV), VAArgTLSCopy(nullptr),
3245
3
      VAArgOverflowSize(nullptr) {}
3246
3247
  enum ArgKind { AK_GeneralPurpose, AK_FloatingPoint, AK_Memory };
3248
3249
12
  ArgKind classifyArgument(Value* arg) {
3250
12
    Type *T = arg->getType();
3251
12
    if (T->isFPOrFPVectorTy())
3252
3
      return AK_FloatingPoint;
3253
9
    
if (9
(T->isIntegerTy() && 9
T->getPrimitiveSizeInBits() <= 649
)
3254
0
        || (T->isPointerTy()))
3255
9
      return AK_GeneralPurpose;
3256
0
    return AK_Memory;
3257
0
  }
3258
3259
  // The instrumentation stores the argument shadow in a non ABI-specific
3260
  // format because it does not know which argument is named (since Clang,
3261
  // like x86_64 case, lowers the va_args in the frontend and this pass only
3262
  // sees the low level code that deals with va_list internals).
3263
  // The first seven GR registers are saved in the first 56 bytes of the
3264
  // va_arg tls arra, followers by the first 8 FP/SIMD registers, and then
3265
  // the remaining arguments.
3266
  // Using constant offset within the va_arg TLS array allows fast copy
3267
  // in the finalize instrumentation.
3268
1
  void visitCallSite(CallSite &CS, IRBuilder<> &IRB) override {
3269
1
    unsigned GrOffset = AArch64GrBegOffset;
3270
1
    unsigned VrOffset = AArch64VrBegOffset;
3271
1
    unsigned OverflowOffset = AArch64VAEndOffset;
3272
1
3273
1
    const DataLayout &DL = F.getParent()->getDataLayout();
3274
1
    for (CallSite::arg_iterator ArgIt = CS.arg_begin(), End = CS.arg_end();
3275
13
         
ArgIt != End13
;
++ArgIt12
) {
3276
12
      Value *A = *ArgIt;
3277
12
      unsigned ArgNo = CS.getArgumentNo(ArgIt);
3278
12
      bool IsFixed = ArgNo < CS.getFunctionType()->getNumParams();
3279
12
      ArgKind AK = classifyArgument(A);
3280
12
      if (
AK == AK_GeneralPurpose && 12
GrOffset >= AArch64GrEndOffset9
)
3281
1
        AK = AK_Memory;
3282
12
      if (
AK == AK_FloatingPoint && 12
VrOffset >= AArch64VrEndOffset3
)
3283
0
        AK = AK_Memory;
3284
12
      Value *Base;
3285
12
      switch (AK) {
3286
8
        case AK_GeneralPurpose:
3287
8
          Base = getShadowPtrForVAArgument(A->getType(), IRB, GrOffset);
3288
8
          GrOffset += 8;
3289
8
          break;
3290
3
        case AK_FloatingPoint:
3291
3
          Base = getShadowPtrForVAArgument(A->getType(), IRB, VrOffset);
3292
3
          VrOffset += 16;
3293
3
          break;
3294
1
        case AK_Memory:
3295
1
          // Don't count fixed arguments in the overflow area - va_start will
3296
1
          // skip right over them.
3297
1
          if (IsFixed)
3298
0
            continue;
3299
1
          uint64_t ArgSize = DL.getTypeAllocSize(A->getType());
3300
1
          Base = getShadowPtrForVAArgument(A->getType(), IRB, OverflowOffset);
3301
1
          OverflowOffset += alignTo(ArgSize, 8);
3302
1
          break;
3303
12
      }
3304
12
      // Count Gp/Vr fixed arguments to their respective offsets, but don't
3305
12
      // bother to actually store a shadow.
3306
12
      
if (12
IsFixed12
)
3307
1
        continue;
3308
11
      IRB.CreateAlignedStore(MSV.getShadow(A), Base, kShadowTLSAlignment);
3309
11
    }
3310
1
    Constant *OverflowSize =
3311
1
      ConstantInt::get(IRB.getInt64Ty(), OverflowOffset - AArch64VAEndOffset);
3312
1
    IRB.CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS);
3313
1
  }
3314
3315
  /// Compute the shadow address for a given va_arg.
3316
  Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB,
3317
12
                                   int ArgOffset) {
3318
12
    Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
3319
12
    Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
3320
12
    return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0),
3321
12
                              "_msarg");
3322
12
  }
3323
3324
1
  void visitVAStartInst(VAStartInst &I) override {
3325
1
    IRBuilder<> IRB(&I);
3326
1
    VAStartInstrumentationList.push_back(&I);
3327
1
    Value *VAListTag = I.getArgOperand(0);
3328
1
    Value *ShadowPtr = MSV.getShadowPtr(VAListTag, IRB.getInt8Ty(), IRB);
3329
1
    // Unpoison the whole __va_list_tag.
3330
1
    // FIXME: magic ABI constants (size of va_list).
3331
1
    IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
3332
1
                     /* size */32, /* alignment */8, false);
3333
1
  }
3334
3335
0
  void visitVACopyInst(VACopyInst &I) override {
3336
0
    IRBuilder<> IRB(&I);
3337
0
    Value *VAListTag = I.getArgOperand(0);
3338
0
    Value *ShadowPtr = MSV.getShadowPtr(VAListTag, IRB.getInt8Ty(), IRB);
3339
0
    // Unpoison the whole __va_list_tag.
3340
0
    // FIXME: magic ABI constants (size of va_list).
3341
0
    IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
3342
0
                     /* size */32, /* alignment */8, false);
3343
0
  }
3344
3345
  // Retrieve a va_list field of 'void*' size.
3346
3
  Value* getVAField64(IRBuilder<> &IRB, Value *VAListTag, int offset) {
3347
3
    Value *SaveAreaPtrPtr =
3348
3
      IRB.CreateIntToPtr(
3349
3
        IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
3350
3
                      ConstantInt::get(MS.IntptrTy, offset)),
3351
3
        Type::getInt64PtrTy(*MS.C));
3352
3
    return IRB.CreateLoad(SaveAreaPtrPtr);
3353
3
  }
3354
3355
  // Retrieve a va_list field of 'int' size.
3356
2
  Value* getVAField32(IRBuilder<> &IRB, Value *VAListTag, int offset) {
3357
2
    Value *SaveAreaPtr =
3358
2
      IRB.CreateIntToPtr(
3359
2
        IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
3360
2
                      ConstantInt::get(MS.IntptrTy, offset)),
3361
2
        Type::getInt32PtrTy(*MS.C));
3362
2
    Value *SaveArea32 = IRB.CreateLoad(SaveAreaPtr);
3363
2
    return IRB.CreateSExt(SaveArea32, MS.IntptrTy);
3364
2
  }
3365
3366
3
  void finalizeInstrumentation() override {
3367
3
    assert(!VAArgOverflowSize && !VAArgTLSCopy &&
3368
3
           "finalizeInstrumentation called twice");
3369
3
    if (
!VAStartInstrumentationList.empty()3
) {
3370
1
      // If there is a va_start in this function, make a backup copy of
3371
1
      // va_arg_tls somewhere in the function entry block.
3372
1
      IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI());
3373
1
      VAArgOverflowSize = IRB.CreateLoad(MS.VAArgOverflowSizeTLS);
3374
1
      Value *CopySize =
3375
1
        IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, AArch64VAEndOffset),
3376
1
                      VAArgOverflowSize);
3377
1
      VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
3378
1
      IRB.CreateMemCpy(VAArgTLSCopy, MS.VAArgTLS, CopySize, 8);
3379
1
    }
3380
3
3381
3
    Value *GrArgSize = ConstantInt::get(MS.IntptrTy, kAArch64GrArgSize);
3382
3
    Value *VrArgSize = ConstantInt::get(MS.IntptrTy, kAArch64VrArgSize);
3383
3
3384
3
    // Instrument va_start, copy va_list shadow from the backup copy of
3385
3
    // the TLS contents.
3386
4
    for (size_t i = 0, n = VAStartInstrumentationList.size(); 
i < n4
;
i++1
) {
3387
1
      CallInst *OrigInst = VAStartInstrumentationList[i];
3388
1
      IRBuilder<> IRB(OrigInst->getNextNode());
3389
1
3390
1
      Value *VAListTag = OrigInst->getArgOperand(0);
3391
1
3392
1
      // The variadic ABI for AArch64 creates two areas to save the incoming
3393
1
      // argument registers (one for 64-bit general register xn-x7 and another
3394
1
      // for 128-bit FP/SIMD vn-v7).
3395
1
      // We need then to propagate the shadow arguments on both regions
3396
1
      // 'va::__gr_top + va::__gr_offs' and 'va::__vr_top + va::__vr_offs'.
3397
1
      // The remaning arguments are saved on shadow for 'va::stack'.
3398
1
      // One caveat is it requires only to propagate the non-named arguments,
3399
1
      // however on the call site instrumentation 'all' the arguments are
3400
1
      // saved. So to copy the shadow values from the va_arg TLS array
3401
1
      // we need to adjust the offset for both GR and VR fields based on
3402
1
      // the __{gr,vr}_offs value (since they are stores based on incoming
3403
1
      // named arguments).
3404
1
3405
1
      // Read the stack pointer from the va_list.
3406
1
      Value *StackSaveAreaPtr = getVAField64(IRB, VAListTag, 0);
3407
1
3408
1
      // Read both the __gr_top and __gr_off and add them up.
3409
1
      Value *GrTopSaveAreaPtr = getVAField64(IRB, VAListTag, 8);
3410
1
      Value *GrOffSaveArea = getVAField32(IRB, VAListTag, 24);
3411
1
3412
1
      Value *GrRegSaveAreaPtr = IRB.CreateAdd(GrTopSaveAreaPtr, GrOffSaveArea);
3413
1
3414
1
      // Read both the __vr_top and __vr_off and add them up.
3415
1
      Value *VrTopSaveAreaPtr = getVAField64(IRB, VAListTag, 16);
3416
1
      Value *VrOffSaveArea = getVAField32(IRB, VAListTag, 28);
3417
1
3418
1
      Value *VrRegSaveAreaPtr = IRB.CreateAdd(VrTopSaveAreaPtr, VrOffSaveArea);
3419
1
3420
1
      // It does not know how many named arguments is being used and, on the
3421
1
      // callsite all the arguments were saved.  Since __gr_off is defined as
3422
1
      // '0 - ((8 - named_gr) * 8)', the idea is to just propagate the variadic
3423
1
      // argument by ignoring the bytes of shadow from named arguments.
3424
1
      Value *GrRegSaveAreaShadowPtrOff =
3425
1
        IRB.CreateAdd(GrArgSize, GrOffSaveArea);
3426
1
3427
1
      Value *GrRegSaveAreaShadowPtr =
3428
1
        MSV.getShadowPtr(GrRegSaveAreaPtr, IRB.getInt8Ty(), IRB);
3429
1
3430
1
      Value *GrSrcPtr = IRB.CreateInBoundsGEP(IRB.getInt8Ty(), VAArgTLSCopy,
3431
1
                                              GrRegSaveAreaShadowPtrOff);
3432
1
      Value *GrCopySize = IRB.CreateSub(GrArgSize, GrRegSaveAreaShadowPtrOff);
3433
1
3434
1
      IRB.CreateMemCpy(GrRegSaveAreaShadowPtr, GrSrcPtr, GrCopySize, 8);
3435
1
3436
1
      // Again, but for FP/SIMD values.
3437
1
      Value *VrRegSaveAreaShadowPtrOff =
3438
1
          IRB.CreateAdd(VrArgSize, VrOffSaveArea);
3439
1
3440
1
      Value *VrRegSaveAreaShadowPtr =
3441
1
        MSV.getShadowPtr(VrRegSaveAreaPtr, IRB.getInt8Ty(), IRB);
3442
1
3443
1
      Value *VrSrcPtr = IRB.CreateInBoundsGEP(
3444
1
        IRB.getInt8Ty(),
3445
1
        IRB.CreateInBoundsGEP(IRB.getInt8Ty(), VAArgTLSCopy,
3446
1
                              IRB.getInt32(AArch64VrBegOffset)),
3447
1
        VrRegSaveAreaShadowPtrOff);
3448
1
      Value *VrCopySize = IRB.CreateSub(VrArgSize, VrRegSaveAreaShadowPtrOff);
3449
1
3450
1
      IRB.CreateMemCpy(VrRegSaveAreaShadowPtr, VrSrcPtr, VrCopySize, 8);
3451
1
3452
1
      // And finally for remaining arguments.
3453
1
      Value *StackSaveAreaShadowPtr =
3454
1
        MSV.getShadowPtr(StackSaveAreaPtr, IRB.getInt8Ty(), IRB);
3455
1
3456
1
      Value *StackSrcPtr =
3457
1
        IRB.CreateInBoundsGEP(IRB.getInt8Ty(), VAArgTLSCopy,
3458
1
                              IRB.getInt32(AArch64VAEndOffset));
3459
1
3460
1
      IRB.CreateMemCpy(StackSaveAreaShadowPtr, StackSrcPtr,
3461
1
                       VAArgOverflowSize, 16);
3462
1
    }
3463
3
  }
3464
};
3465
3466
/// \brief PowerPC64-specific implementation of VarArgHelper.
3467
struct VarArgPowerPC64Helper : public VarArgHelper {
3468
  Function &F;
3469
  MemorySanitizer &MS;
3470
  MemorySanitizerVisitor &MSV;
3471
  Value *VAArgTLSCopy;
3472
  Value *VAArgSize;
3473
3474
  SmallVector<CallInst*, 16> VAStartInstrumentationList;
3475
3476
  VarArgPowerPC64Helper(Function &F, MemorySanitizer &MS,
3477
                    MemorySanitizerVisitor &MSV)
3478
    : F(F), MS(MS), MSV(MSV), VAArgTLSCopy(nullptr),
3479
17
      VAArgSize(nullptr) {}
3480
3481
13
  void visitCallSite(CallSite &CS, IRBuilder<> &IRB) override {
3482
13
    // For PowerPC, we need to deal with alignment of stack arguments -
3483
13
    // they are mostly aligned to 8 bytes, but vectors and i128 arrays
3484
13
    // are aligned to 16 bytes, byvals can be aligned to 8 or 16 bytes,
3485
13
    // and QPX vectors are aligned to 32 bytes.  For that reason, we
3486
13
    // compute current offset from stack pointer (which is always properly
3487
13
    // aligned), and offset for the first vararg, then subtract them.
3488
13
    unsigned VAArgBase;
3489
13
    llvm::Triple TargetTriple(F.getParent()->getTargetTriple());
3490
13
    // Parameter save area starts at 48 bytes from frame pointer for ABIv1,
3491
13
    // and 32 bytes for ABIv2.  This is usually determined by target
3492
13
    // endianness, but in theory could be overriden by function attribute.
3493
13
    // For simplicity, we ignore it here (it'd only matter for QPX vectors).
3494
13
    if (TargetTriple.getArch() == llvm::Triple::ppc64)
3495
7
      VAArgBase = 48;
3496
13
    else
3497
6
      VAArgBase = 32;
3498
13
    unsigned VAArgOffset = VAArgBase;
3499
13
    const DataLayout &DL = F.getParent()->getDataLayout();
3500
13
    for (CallSite::arg_iterator ArgIt = CS.arg_begin(), End = CS.arg_end();
3501
45
         
ArgIt != End45
;
++ArgIt32
) {
3502
32
      Value *A = *ArgIt;
3503
32
      unsigned ArgNo = CS.getArgumentNo(ArgIt);
3504
32
      bool IsFixed = ArgNo < CS.getFunctionType()->getNumParams();
3505
32
      bool IsByVal = CS.paramHasAttr(ArgNo, Attribute::ByVal);
3506
32
      if (
IsByVal32
) {
3507
4
        assert(A->getType()->isPointerTy());
3508
4
        Type *RealTy = A->getType()->getPointerElementType();
3509
4
        uint64_t ArgSize = DL.getTypeAllocSize(RealTy);
3510
4
        uint64_t ArgAlign = CS.getParamAlignment(ArgNo);
3511
4
        if (ArgAlign < 8)
3512
0
          ArgAlign = 8;
3513
4
        VAArgOffset = alignTo(VAArgOffset, ArgAlign);
3514
4
        if (
!IsFixed4
) {
3515
4
          Value *Base = getShadowPtrForVAArgument(RealTy, IRB,
3516
4
                                                  VAArgOffset - VAArgBase);
3517
4
          IRB.CreateMemCpy(Base, MSV.getShadowPtr(A, IRB.getInt8Ty(), IRB),
3518
4
                           ArgSize, kShadowTLSAlignment);
3519
4
        }
3520
4
        VAArgOffset += alignTo(ArgSize, 8);
3521
32
      } else {
3522
28
        Value *Base;
3523
28
        uint64_t ArgSize = DL.getTypeAllocSize(A->getType());
3524
28
        uint64_t ArgAlign = 8;
3525
28
        if (
A->getType()->isArrayTy()28
) {
3526
4
          // Arrays are aligned to element size, except for long double
3527
4
          // arrays, which are aligned to 8 bytes.
3528
4
          Type *ElementTy = A->getType()->getArrayElementType();
3529
4
          if (!ElementTy->isPPC_FP128Ty())
3530
4
            ArgAlign = DL.getTypeAllocSize(ElementTy);
3531
28
        } else 
if (24
A->getType()->isVectorTy()24
) {
3532
3
          // Vectors are naturally aligned.
3533
3
          ArgAlign = DL.getTypeAllocSize(A->getType());
3534
3
        }
3535
28
        if (ArgAlign < 8)
3536
0
          ArgAlign = 8;
3537
28
        VAArgOffset = alignTo(VAArgOffset, ArgAlign);
3538
28
        if (
DL.isBigEndian()28
) {
3539
16
          // Adjusting the shadow for argument with size < 8 to match the placement
3540
16
          // of bits in big endian system
3541
16
          if (ArgSize < 8)
3542
10
            VAArgOffset += (8 - ArgSize);
3543
16
        }
3544
28
        if (
!IsFixed28
) {
3545
15
          Base = getShadowPtrForVAArgument(A->getType(), IRB,
3546
15
                                           VAArgOffset - VAArgBase);
3547
15
          IRB.CreateAlignedStore(MSV.getShadow(A), Base, kShadowTLSAlignment);
3548
15
        }
3549
28
        VAArgOffset += ArgSize;
3550
28
        VAArgOffset = alignTo(VAArgOffset, 8);
3551
28
      }
3552
32
      if (IsFixed)
3553
13
        VAArgBase = VAArgOffset;
3554
32
    }
3555
13
3556
13
    Constant *TotalVAArgSize = ConstantInt::get(IRB.getInt64Ty(),
3557
13
                                                VAArgOffset - VAArgBase);
3558
13
    // Here using VAArgOverflowSizeTLS as VAArgSizeTLS to avoid creation of
3559
13
    // a new class member i.e. it is the total size of all VarArgs.
3560
13
    IRB.CreateStore(TotalVAArgSize, MS.VAArgOverflowSizeTLS);
3561
13
  }
3562
3563
  /// \brief Compute the shadow address for a given va_arg.
3564
  Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB,
3565
19
                                   int ArgOffset) {
3566
19
    Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
3567
19
    Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
3568
19
    return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0),
3569
19
                              "_msarg");
3570
19
  }
3571
3572
2
  void visitVAStartInst(VAStartInst &I) override {
3573
2
    IRBuilder<> IRB(&I);
3574
2
    VAStartInstrumentationList.push_back(&I);
3575
2
    Value *VAListTag = I.getArgOperand(0);
3576
2
    Value *ShadowPtr = MSV.getShadowPtr(VAListTag, IRB.getInt8Ty(), IRB);
3577
2
    IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
3578
2
                     /* size */8, /* alignment */8, false);
3579
2
  }
3580
3581
0
  void visitVACopyInst(VACopyInst &I) override {
3582
0
    IRBuilder<> IRB(&I);
3583
0
    Value *VAListTag = I.getArgOperand(0);
3584
0
    Value *ShadowPtr = MSV.getShadowPtr(VAListTag, IRB.getInt8Ty(), IRB);
3585
0
    // Unpoison the whole __va_list_tag.
3586
0
    // FIXME: magic ABI constants.
3587
0
    IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
3588
0
                     /* size */8, /* alignment */8, false);
3589
0
  }
3590
3591
17
  void finalizeInstrumentation() override {
3592
17
    assert(!VAArgSize && !VAArgTLSCopy &&
3593
17
           "finalizeInstrumentation called twice");
3594
17
    IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI());
3595
17
    VAArgSize = IRB.CreateLoad(MS.VAArgOverflowSizeTLS);
3596
17
    Value *CopySize = IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, 0),
3597
17
                                    VAArgSize);
3598
17
3599
17
    if (
!VAStartInstrumentationList.empty()17
) {
3600
2
      // If there is a va_start in this function, make a backup copy of
3601
2
      // va_arg_tls somewhere in the function entry block.
3602
2
      VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
3603
2
      IRB.CreateMemCpy(VAArgTLSCopy, MS.VAArgTLS, CopySize, 8);
3604
2
    }
3605
17
3606
17
    // Instrument va_start.
3607
17
    // Copy va_list shadow from the backup copy of the TLS contents.
3608
19
    for (size_t i = 0, n = VAStartInstrumentationList.size(); 
i < n19
;
i++2
) {
3609
2
      CallInst *OrigInst = VAStartInstrumentationList[i];
3610
2
      IRBuilder<> IRB(OrigInst->getNextNode());
3611
2
      Value *VAListTag = OrigInst->getArgOperand(0);
3612
2
      Value *RegSaveAreaPtrPtr =
3613
2
        IRB.CreateIntToPtr(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
3614
2
                        Type::getInt64PtrTy(*MS.C));
3615
2
      Value *RegSaveAreaPtr = IRB.CreateLoad(RegSaveAreaPtrPtr);
3616
2
      Value *RegSaveAreaShadowPtr =
3617
2
      MSV.getShadowPtr(RegSaveAreaPtr, IRB.getInt8Ty(), IRB);
3618
2
      IRB.CreateMemCpy(RegSaveAreaShadowPtr, VAArgTLSCopy, CopySize, 8);
3619
2
    }
3620
17
  }
3621
};
3622
3623
/// \brief A no-op implementation of VarArgHelper.
3624
struct VarArgNoOpHelper : public VarArgHelper {
3625
  VarArgNoOpHelper(Function &F, MemorySanitizer &MS,
3626
1
                   MemorySanitizerVisitor &MSV) {}
3627
3628
0
  void visitCallSite(CallSite &CS, IRBuilder<> &IRB) override {}
3629
3630
1
  void visitVAStartInst(VAStartInst &I) override {}
3631
3632
0
  void visitVACopyInst(VACopyInst &I) override {}
3633
3634
1
  void finalizeInstrumentation() override {}
3635
};
3636
3637
VarArgHelper *CreateVarArgHelper(Function &Func, MemorySanitizer &Msan,
3638
373
                                 MemorySanitizerVisitor &Visitor) {
3639
373
  // VarArg handling is only implemented on AMD64. False positives are possible
3640
373
  // on other platforms.
3641
373
  llvm::Triple TargetTriple(Func.getParent()->getTargetTriple());
3642
373
  if (TargetTriple.getArch() == llvm::Triple::x86_64)
3643
344
    return new VarArgAMD64Helper(Func, Msan, Visitor);
3644
29
  else 
if (29
TargetTriple.getArch() == llvm::Triple::mips64 ||
3645
25
           TargetTriple.getArch() == llvm::Triple::mips64el)
3646
8
    return new VarArgMIPS64Helper(Func, Msan, Visitor);
3647
21
  else 
if (21
TargetTriple.getArch() == llvm::Triple::aarch6421
)
3648
3
    return new VarArgAArch64Helper(Func, Msan, Visitor);
3649
18
  else 
if (18
TargetTriple.getArch() == llvm::Triple::ppc64 ||
3650
9
           TargetTriple.getArch() == llvm::Triple::ppc64le)
3651
17
    return new VarArgPowerPC64Helper(Func, Msan, Visitor);
3652
18
  else
3653
1
    return new VarArgNoOpHelper(Func, Msan, Visitor);
3654
0
}
3655
3656
} // anonymous namespace
3657
3658
437
bool MemorySanitizer::runOnFunction(Function &F) {
3659
437
  if (&F == MsanCtorFunction)
3660
64
    return false;
3661
373
  MemorySanitizerVisitor Visitor(F, *this);
3662
373
3663
373
  // Clear out readonly/readnone attributes.
3664
373
  AttrBuilder B;
3665
373
  B.addAttribute(Attribute::ReadOnly)
3666
373
    .addAttribute(Attribute::ReadNone);
3667
373
  F.removeAttributes(AttributeList::FunctionIndex, B);
3668
373
3669
373
  return Visitor.runOnFunction();
3670
373
}