Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
Line
Count
Source (jump to first uncovered line)
1
//===- DataFlowSanitizer.cpp - dynamic data flow analysis -----------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
/// \file
10
/// This file is a part of DataFlowSanitizer, a generalised dynamic data flow
11
/// analysis.
12
///
13
/// Unlike other Sanitizer tools, this tool is not designed to detect a specific
14
/// class of bugs on its own.  Instead, it provides a generic dynamic data flow
15
/// analysis framework to be used by clients to help detect application-specific
16
/// issues within their own code.
17
///
18
/// The analysis is based on automatic propagation of data flow labels (also
19
/// known as taint labels) through a program as it performs computation.  Each
20
/// byte of application memory is backed by two bytes of shadow memory which
21
/// hold the label.  On Linux/x86_64, memory is laid out as follows:
22
///
23
/// +--------------------+ 0x800000000000 (top of memory)
24
/// | application memory |
25
/// +--------------------+ 0x700000008000 (kAppAddr)
26
/// |                    |
27
/// |       unused       |
28
/// |                    |
29
/// +--------------------+ 0x200200000000 (kUnusedAddr)
30
/// |    union table     |
31
/// +--------------------+ 0x200000000000 (kUnionTableAddr)
32
/// |   shadow memory    |
33
/// +--------------------+ 0x000000010000 (kShadowAddr)
34
/// | reserved by kernel |
35
/// +--------------------+ 0x000000000000
36
///
37
/// To derive a shadow memory address from an application memory address,
38
/// bits 44-46 are cleared to bring the address into the range
39
/// [0x000000008000,0x100000000000).  Then the address is shifted left by 1 to
40
/// account for the double byte representation of shadow labels and move the
41
/// address into the shadow memory range.  See the function
42
/// DataFlowSanitizer::getShadowAddress below.
43
///
44
/// For more information, please refer to the design document:
45
/// http://clang.llvm.org/docs/DataFlowSanitizerDesign.html
46
//
47
//===----------------------------------------------------------------------===//
48
49
#include "llvm/ADT/DenseMap.h"
50
#include "llvm/ADT/DenseSet.h"
51
#include "llvm/ADT/DepthFirstIterator.h"
52
#include "llvm/ADT/None.h"
53
#include "llvm/ADT/SmallPtrSet.h"
54
#include "llvm/ADT/SmallVector.h"
55
#include "llvm/ADT/StringExtras.h"
56
#include "llvm/ADT/StringRef.h"
57
#include "llvm/ADT/Triple.h"
58
#include "llvm/Transforms/Utils/Local.h"
59
#include "llvm/Analysis/ValueTracking.h"
60
#include "llvm/IR/Argument.h"
61
#include "llvm/IR/Attributes.h"
62
#include "llvm/IR/BasicBlock.h"
63
#include "llvm/IR/CallSite.h"
64
#include "llvm/IR/Constant.h"
65
#include "llvm/IR/Constants.h"
66
#include "llvm/IR/DataLayout.h"
67
#include "llvm/IR/DerivedTypes.h"
68
#include "llvm/IR/Dominators.h"
69
#include "llvm/IR/Function.h"
70
#include "llvm/IR/GlobalAlias.h"
71
#include "llvm/IR/GlobalValue.h"
72
#include "llvm/IR/GlobalVariable.h"
73
#include "llvm/IR/IRBuilder.h"
74
#include "llvm/IR/InlineAsm.h"
75
#include "llvm/IR/InstVisitor.h"
76
#include "llvm/IR/InstrTypes.h"
77
#include "llvm/IR/Instruction.h"
78
#include "llvm/IR/Instructions.h"
79
#include "llvm/IR/IntrinsicInst.h"
80
#include "llvm/IR/LLVMContext.h"
81
#include "llvm/IR/MDBuilder.h"
82
#include "llvm/IR/Module.h"
83
#include "llvm/IR/Type.h"
84
#include "llvm/IR/User.h"
85
#include "llvm/IR/Value.h"
86
#include "llvm/Pass.h"
87
#include "llvm/Support/Casting.h"
88
#include "llvm/Support/CommandLine.h"
89
#include "llvm/Support/ErrorHandling.h"
90
#include "llvm/Support/SpecialCaseList.h"
91
#include "llvm/Transforms/Instrumentation.h"
92
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
93
#include <algorithm>
94
#include <cassert>
95
#include <cstddef>
96
#include <cstdint>
97
#include <iterator>
98
#include <memory>
99
#include <set>
100
#include <string>
101
#include <utility>
102
#include <vector>
103
104
using namespace llvm;
105
106
// External symbol to be used when generating the shadow address for
107
// architectures with multiple VMAs. Instead of using a constant integer
108
// the runtime will set the external mask based on the VMA range.
109
static const char *const kDFSanExternShadowPtrMask = "__dfsan_shadow_ptr_mask";
110
111
// The -dfsan-preserve-alignment flag controls whether this pass assumes that
112
// alignment requirements provided by the input IR are correct.  For example,
113
// if the input IR contains a load with alignment 8, this flag will cause
114
// the shadow load to have alignment 16.  This flag is disabled by default as
115
// we have unfortunately encountered too much code (including Clang itself;
116
// see PR14291) which performs misaligned access.
117
static cl::opt<bool> ClPreserveAlignment(
118
    "dfsan-preserve-alignment",
119
    cl::desc("respect alignment requirements provided by input IR"), cl::Hidden,
120
    cl::init(false));
121
122
// The ABI list files control how shadow parameters are passed. The pass treats
123
// every function labelled "uninstrumented" in the ABI list file as conforming
124
// to the "native" (i.e. unsanitized) ABI.  Unless the ABI list contains
125
// additional annotations for those functions, a call to one of those functions
126
// will produce a warning message, as the labelling behaviour of the function is
127
// unknown.  The other supported annotations are "functional" and "discard",
128
// which are described below under DataFlowSanitizer::WrapperKind.
129
static cl::list<std::string> ClABIListFiles(
130
    "dfsan-abilist",
131
    cl::desc("File listing native ABI functions and how the pass treats them"),
132
    cl::Hidden);
133
134
// Controls whether the pass uses IA_Args or IA_TLS as the ABI for instrumented
135
// functions (see DataFlowSanitizer::InstrumentedABI below).
136
static cl::opt<bool> ClArgsABI(
137
    "dfsan-args-abi",
138
    cl::desc("Use the argument ABI rather than the TLS ABI"),
139
    cl::Hidden);
140
141
// Controls whether the pass includes or ignores the labels of pointers in load
142
// instructions.
143
static cl::opt<bool> ClCombinePointerLabelsOnLoad(
144
    "dfsan-combine-pointer-labels-on-load",
145
    cl::desc("Combine the label of the pointer with the label of the data when "
146
             "loading from memory."),
147
    cl::Hidden, cl::init(true));
148
149
// Controls whether the pass includes or ignores the labels of pointers in
150
// stores instructions.
151
static cl::opt<bool> ClCombinePointerLabelsOnStore(
152
    "dfsan-combine-pointer-labels-on-store",
153
    cl::desc("Combine the label of the pointer with the label of the data when "
154
             "storing in memory."),
155
    cl::Hidden, cl::init(false));
156
157
static cl::opt<bool> ClDebugNonzeroLabels(
158
    "dfsan-debug-nonzero-labels",
159
    cl::desc("Insert calls to __dfsan_nonzero_label on observing a parameter, "
160
             "load or return with a nonzero label"),
161
    cl::Hidden);
162
163
0
static StringRef GetGlobalTypeString(const GlobalValue &G) {
164
0
  // Types of GlobalVariables are always pointer types.
165
0
  Type *GType = G.getValueType();
166
0
  // For now we support blacklisting struct types only.
167
0
  if (StructType *SGType = dyn_cast<StructType>(GType)) {
168
0
    if (!SGType->isLiteral())
169
0
      return SGType->getName();
170
0
  }
171
0
  return "<unknown type>";
172
0
}
173
174
namespace {
175
176
class DFSanABIList {
177
  std::unique_ptr<SpecialCaseList> SCL;
178
179
 public:
180
24
  DFSanABIList() = default;
181
182
24
  void set(std::unique_ptr<SpecialCaseList> List) { SCL = std::move(List); }
183
184
  /// Returns whether either this function or its source file are listed in the
185
  /// given category.
186
201
  bool isIn(const Function &F, StringRef Category) const {
187
201
    return isIn(*F.getParent(), Category) ||
188
201
           SCL->inSection("dataflow", "fun", F.getName(), Category);
189
201
  }
190
191
  /// Returns whether this global alias is listed in the given category.
192
  ///
193
  /// If GA aliases a function, the alias's name is matched as a function name
194
  /// would be.  Similarly, aliases of globals are matched like globals.
195
6
  bool isIn(const GlobalAlias &GA, StringRef Category) const {
196
6
    if (isIn(*GA.getParent(), Category))
197
0
      return true;
198
6
199
6
    if (isa<FunctionType>(GA.getValueType()))
200
6
      return SCL->inSection("dataflow", "fun", GA.getName(), Category);
201
0
202
0
    return SCL->inSection("dataflow", "global", GA.getName(), Category) ||
203
0
           SCL->inSection("dataflow", "type", GetGlobalTypeString(GA),
204
0
                          Category);
205
0
  }
206
207
  /// Returns whether this module is listed in the given category.
208
231
  bool isIn(const Module &M, StringRef Category) const {
209
231
    return SCL->inSection("dataflow", "src", M.getModuleIdentifier(), Category);
210
231
  }
211
};
212
213
/// TransformedFunction is used to express the result of transforming one
214
/// function type into another.  This struct is immutable.  It holds metadata
215
/// useful for updating calls of the old function to the new type.
216
struct TransformedFunction {
217
  TransformedFunction(FunctionType* OriginalType,
218
                      FunctionType* TransformedType,
219
                      std::vector<unsigned> ArgumentIndexMapping)
220
      : OriginalType(OriginalType),
221
        TransformedType(TransformedType),
222
24
        ArgumentIndexMapping(ArgumentIndexMapping) {}
223
224
  // Disallow copies.
225
  TransformedFunction(const TransformedFunction&) = delete;
226
  TransformedFunction& operator=(const TransformedFunction&) = delete;
227
228
  // Allow moves.
229
  TransformedFunction(TransformedFunction&&) = default;
230
  TransformedFunction& operator=(TransformedFunction&&) = default;
231
232
  /// Type of the function before the transformation.
233
  FunctionType *OriginalType;
234
235
  /// Type of the function after the transformation.
236
  FunctionType *TransformedType;
237
238
  /// Transforming a function may change the position of arguments.  This
239
  /// member records the mapping from each argument's old position to its new
240
  /// position.  Argument positions are zero-indexed.  If the transformation
241
  /// from F to F' made the first argument of F into the third argument of F',
242
  /// then ArgumentIndexMapping[0] will equal 2.
243
  std::vector<unsigned> ArgumentIndexMapping;
244
};
245
246
/// Given function attributes from a call site for the original function,
247
/// return function attributes appropriate for a call to the transformed
248
/// function.
249
AttributeList TransformFunctionAttributes(
250
    const TransformedFunction& TransformedFunction,
251
24
    LLVMContext& Ctx, AttributeList CallSiteAttrs) {
252
24
253
24
  // Construct a vector of AttributeSet for each function argument.
254
24
  std::vector<llvm::AttributeSet> ArgumentAttributes(
255
24
      TransformedFunction.TransformedType->getNumParams());
256
24
257
24
  // Copy attributes from the parameter of the original function to the
258
24
  // transformed version.  'ArgumentIndexMapping' holds the mapping from
259
24
  // old argument position to new.
260
24
  for (unsigned i=0, ie = TransformedFunction.ArgumentIndexMapping.size();
261
68
       i < ie; 
++i44
) {
262
44
    unsigned TransformedIndex = TransformedFunction.ArgumentIndexMapping[i];
263
44
    ArgumentAttributes[TransformedIndex] = CallSiteAttrs.getParamAttributes(i);
264
44
  }
265
24
266
24
  // Copy annotations on varargs arguments.
267
24
  for (unsigned i = TransformedFunction.OriginalType->getNumParams(),
268
48
       ie = CallSiteAttrs.getNumAttrSets(); i<ie; 
++i24
) {
269
24
    ArgumentAttributes.push_back(CallSiteAttrs.getParamAttributes(i));
270
24
  }
271
24
272
24
  return AttributeList::get(
273
24
      Ctx,
274
24
      CallSiteAttrs.getFnAttributes(),
275
24
      CallSiteAttrs.getRetAttributes(),
276
24
      llvm::makeArrayRef(ArgumentAttributes));
277
24
}
278
279
class DataFlowSanitizer : public ModulePass {
280
  friend struct DFSanFunction;
281
  friend class DFSanVisitor;
282
283
  enum {
284
    ShadowWidth = 16
285
  };
286
287
  /// Which ABI should be used for instrumented functions?
288
  enum InstrumentedABI {
289
    /// Argument and return value labels are passed through additional
290
    /// arguments and by modifying the return type.
291
    IA_Args,
292
293
    /// Argument and return value labels are passed through TLS variables
294
    /// __dfsan_arg_tls and __dfsan_retval_tls.
295
    IA_TLS
296
  };
297
298
  /// How should calls to uninstrumented functions be handled?
299
  enum WrapperKind {
300
    /// This function is present in an uninstrumented form but we don't know
301
    /// how it should be handled.  Print a warning and call the function anyway.
302
    /// Don't label the return value.
303
    WK_Warning,
304
305
    /// This function does not write to (user-accessible) memory, and its return
306
    /// value is unlabelled.
307
    WK_Discard,
308
309
    /// This function does not write to (user-accessible) memory, and the label
310
    /// of its return value is the union of the label of its arguments.
311
    WK_Functional,
312
313
    /// Instead of calling the function, a custom wrapper __dfsw_F is called,
314
    /// where F is the name of the function.  This function may wrap the
315
    /// original function or provide its own implementation.  This is similar to
316
    /// the IA_Args ABI, except that IA_Args uses a struct return type to
317
    /// pass the return value shadow in a register, while WK_Custom uses an
318
    /// extra pointer argument to return the shadow.  This allows the wrapped
319
    /// form of the function type to be expressed in C.
320
    WK_Custom
321
  };
322
323
  Module *Mod;
324
  LLVMContext *Ctx;
325
  IntegerType *ShadowTy;
326
  PointerType *ShadowPtrTy;
327
  IntegerType *IntptrTy;
328
  ConstantInt *ZeroShadow;
329
  ConstantInt *ShadowPtrMask;
330
  ConstantInt *ShadowPtrMul;
331
  Constant *ArgTLS;
332
  Constant *RetvalTLS;
333
  void *(*GetArgTLSPtr)();
334
  void *(*GetRetvalTLSPtr)();
335
  FunctionType *GetArgTLSTy;
336
  FunctionType *GetRetvalTLSTy;
337
  Constant *GetArgTLS;
338
  Constant *GetRetvalTLS;
339
  Constant *ExternalShadowMask;
340
  FunctionType *DFSanUnionFnTy;
341
  FunctionType *DFSanUnionLoadFnTy;
342
  FunctionType *DFSanUnimplementedFnTy;
343
  FunctionType *DFSanSetLabelFnTy;
344
  FunctionType *DFSanNonzeroLabelFnTy;
345
  FunctionType *DFSanVarargWrapperFnTy;
346
  FunctionCallee DFSanUnionFn;
347
  FunctionCallee DFSanCheckedUnionFn;
348
  FunctionCallee DFSanUnionLoadFn;
349
  FunctionCallee DFSanUnimplementedFn;
350
  FunctionCallee DFSanSetLabelFn;
351
  FunctionCallee DFSanNonzeroLabelFn;
352
  FunctionCallee DFSanVarargWrapperFn;
353
  MDNode *ColdCallWeights;
354
  DFSanABIList ABIList;
355
  DenseMap<Value *, Function *> UnwrappedFnMap;
356
  AttrBuilder ReadOnlyNoneAttrs;
357
  bool DFSanRuntimeShadowMask = false;
358
359
  Value *getShadowAddress(Value *Addr, Instruction *Pos);
360
  bool isInstrumented(const Function *F);
361
  bool isInstrumented(const GlobalAlias *GA);
362
  FunctionType *getArgsFunctionType(FunctionType *T);
363
  FunctionType *getTrampolineFunctionType(FunctionType *T);
364
  TransformedFunction getCustomFunctionType(FunctionType *T);
365
  InstrumentedABI getInstrumentedABI();
366
  WrapperKind getWrapperKind(Function *F);
367
  void addGlobalNamePrefix(GlobalValue *GV);
368
  Function *buildWrapperFunction(Function *F, StringRef NewFName,
369
                                 GlobalValue::LinkageTypes NewFLink,
370
                                 FunctionType *NewFT);
371
  Constant *getOrBuildTrampolineFunction(FunctionType *FT, StringRef FName);
372
373
public:
374
  static char ID;
375
376
  DataFlowSanitizer(
377
      const std::vector<std::string> &ABIListFiles = std::vector<std::string>(),
378
      void *(*getArgTLS)() = nullptr, void *(*getRetValTLS)() = nullptr);
379
380
  bool doInitialization(Module &M) override;
381
  bool runOnModule(Module &M) override;
382
};
383
384
struct DFSanFunction {
385
  DataFlowSanitizer &DFS;
386
  Function *F;
387
  DominatorTree DT;
388
  DataFlowSanitizer::InstrumentedABI IA;
389
  bool IsNativeABI;
390
  Value *ArgTLSPtr = nullptr;
391
  Value *RetvalTLSPtr = nullptr;
392
  AllocaInst *LabelReturnAlloca = nullptr;
393
  DenseMap<Value *, Value *> ValShadowMap;
394
  DenseMap<AllocaInst *, AllocaInst *> AllocaShadowMap;
395
  std::vector<std::pair<PHINode *, PHINode *>> PHIFixups;
396
  DenseSet<Instruction *> SkipInsts;
397
  std::vector<Value *> NonZeroChecks;
398
  bool AvoidNewBlocks;
399
400
  struct CachedCombinedShadow {
401
    BasicBlock *Block;
402
    Value *Shadow;
403
  };
404
  DenseMap<std::pair<Value *, Value *>, CachedCombinedShadow>
405
      CachedCombinedShadows;
406
  DenseMap<Value *, std::set<Value *>> ShadowElements;
407
408
  DFSanFunction(DataFlowSanitizer &DFS, Function *F, bool IsNativeABI)
409
95
      : DFS(DFS), F(F), IA(DFS.getInstrumentedABI()), IsNativeABI(IsNativeABI) {
410
95
    DT.recalculate(*F);
411
95
    // FIXME: Need to track down the register allocator issue which causes poor
412
95
    // performance in pathological cases with large numbers of basic blocks.
413
95
    AvoidNewBlocks = F->size() > 1000;
414
95
  }
415
416
  Value *getArgTLSPtr();
417
  Value *getArgTLS(unsigned Index, Instruction *Pos);
418
  Value *getRetvalTLS();
419
  Value *getShadow(Value *V);
420
  void setShadow(Instruction *I, Value *Shadow);
421
  Value *combineShadows(Value *V1, Value *V2, Instruction *Pos);
422
  Value *combineOperandShadows(Instruction *Inst);
423
  Value *loadShadow(Value *ShadowAddr, uint64_t Size, uint64_t Align,
424
                    Instruction *Pos);
425
  void storeShadow(Value *Addr, uint64_t Size, uint64_t Align, Value *Shadow,
426
                   Instruction *Pos);
427
};
428
429
class DFSanVisitor : public InstVisitor<DFSanVisitor> {
430
public:
431
  DFSanFunction &DFSF;
432
433
1.20k
  DFSanVisitor(DFSanFunction &DFSF) : DFSF(DFSF) {}
434
435
20
  const DataLayout &getDataLayout() const {
436
20
    return DFSF.F->getParent()->getDataLayout();
437
20
  }
438
439
  void visitOperandShadowInst(Instruction &I);
440
  void visitUnaryOperator(UnaryOperator &UO);
441
  void visitBinaryOperator(BinaryOperator &BO);
442
  void visitCastInst(CastInst &CI);
443
  void visitCmpInst(CmpInst &CI);
444
  void visitGetElementPtrInst(GetElementPtrInst &GEPI);
445
  void visitLoadInst(LoadInst &LI);
446
  void visitStoreInst(StoreInst &SI);
447
  void visitReturnInst(ReturnInst &RI);
448
  void visitCallSite(CallSite CS);
449
  void visitPHINode(PHINode &PN);
450
  void visitExtractElementInst(ExtractElementInst &I);
451
  void visitInsertElementInst(InsertElementInst &I);
452
  void visitShuffleVectorInst(ShuffleVectorInst &I);
453
  void visitExtractValueInst(ExtractValueInst &I);
454
  void visitInsertValueInst(InsertValueInst &I);
455
  void visitAllocaInst(AllocaInst &I);
456
  void visitSelectInst(SelectInst &I);
457
  void visitMemSetInst(MemSetInst &I);
458
  void visitMemTransferInst(MemTransferInst &I);
459
};
460
461
} // end anonymous namespace
462
463
char DataFlowSanitizer::ID;
464
465
INITIALIZE_PASS(DataFlowSanitizer, "dfsan",
466
                "DataFlowSanitizer: dynamic data flow analysis.", false, false)
467
468
ModulePass *
469
llvm::createDataFlowSanitizerPass(const std::vector<std::string> &ABIListFiles,
470
                                  void *(*getArgTLS)(),
471
0
                                  void *(*getRetValTLS)()) {
472
0
  return new DataFlowSanitizer(ABIListFiles, getArgTLS, getRetValTLS);
473
0
}
474
475
DataFlowSanitizer::DataFlowSanitizer(
476
    const std::vector<std::string> &ABIListFiles, void *(*getArgTLS)(),
477
    void *(*getRetValTLS)())
478
24
    : ModulePass(ID), GetArgTLSPtr(getArgTLS), GetRetvalTLSPtr(getRetValTLS) {
479
24
  std::vector<std::string> AllABIListFiles(std::move(ABIListFiles));
480
24
  AllABIListFiles.insert(AllABIListFiles.end(), ClABIListFiles.begin(),
481
24
                         ClABIListFiles.end());
482
24
  ABIList.set(SpecialCaseList::createOrDie(AllABIListFiles));
483
24
}
484
485
38
FunctionType *DataFlowSanitizer::getArgsFunctionType(FunctionType *T) {
486
38
  SmallVector<Type *, 4> ArgTypes(T->param_begin(), T->param_end());
487
38
  ArgTypes.append(T->getNumParams(), ShadowTy);
488
38
  if (T->isVarArg())
489
3
    ArgTypes.push_back(ShadowPtrTy);
490
38
  Type *RetType = T->getReturnType();
491
38
  if (!RetType->isVoidTy())
492
26
    RetType = StructType::get(RetType, ShadowTy);
493
38
  return FunctionType::get(RetType, ArgTypes, T->isVarArg());
494
38
}
495
496
28
FunctionType *DataFlowSanitizer::getTrampolineFunctionType(FunctionType *T) {
497
28
  assert(!T->isVarArg());
498
28
  SmallVector<Type *, 4> ArgTypes;
499
28
  ArgTypes.push_back(T->getPointerTo());
500
28
  ArgTypes.append(T->param_begin(), T->param_end());
501
28
  ArgTypes.append(T->getNumParams(), ShadowTy);
502
28
  Type *RetType = T->getReturnType();
503
28
  if (!RetType->isVoidTy())
504
28
    ArgTypes.push_back(ShadowPtrTy);
505
28
  return FunctionType::get(T->getReturnType(), ArgTypes, false);
506
28
}
507
508
24
TransformedFunction DataFlowSanitizer::getCustomFunctionType(FunctionType *T) {
509
24
  SmallVector<Type *, 4> ArgTypes;
510
24
511
24
  // Some parameters of the custom function being constructed are
512
24
  // parameters of T.  Record the mapping from parameters of T to
513
24
  // parameters of the custom function, so that parameter attributes
514
24
  // at call sites can be updated.
515
24
  std::vector<unsigned> ArgumentIndexMapping;
516
68
  for (unsigned i = 0, ie = T->getNumParams(); i != ie; 
++i44
) {
517
44
    Type* param_type = T->getParamType(i);
518
44
    FunctionType *FT;
519
44
    if (isa<PointerType>(param_type) && (FT = dyn_cast<FunctionType>(
520
14
            cast<PointerType>(param_type)->getElementType()))) {
521
14
      ArgumentIndexMapping.push_back(ArgTypes.size());
522
14
      ArgTypes.push_back(getTrampolineFunctionType(FT)->getPointerTo());
523
14
      ArgTypes.push_back(Type::getInt8PtrTy(*Ctx));
524
30
    } else {
525
30
      ArgumentIndexMapping.push_back(ArgTypes.size());
526
30
      ArgTypes.push_back(param_type);
527
30
    }
528
44
  }
529
68
  for (unsigned i = 0, e = T->getNumParams(); i != e; 
++i44
)
530
44
    ArgTypes.push_back(ShadowTy);
531
24
  if (T->isVarArg())
532
4
    ArgTypes.push_back(ShadowPtrTy);
533
24
  Type *RetType = T->getReturnType();
534
24
  if (!RetType->isVoidTy())
535
19
    ArgTypes.push_back(ShadowPtrTy);
536
24
  return TransformedFunction(
537
24
      T, FunctionType::get(T->getReturnType(), ArgTypes, T->isVarArg()),
538
24
      ArgumentIndexMapping);
539
24
}
540
541
24
bool DataFlowSanitizer::doInitialization(Module &M) {
542
24
  Triple TargetTriple(M.getTargetTriple());
543
24
  bool IsX86_64 = TargetTriple.getArch() == Triple::x86_64;
544
24
  bool IsMIPS64 = TargetTriple.isMIPS64();
545
24
  bool IsAArch64 = TargetTriple.getArch() == Triple::aarch64 ||
546
24
                   
TargetTriple.getArch() == Triple::aarch64_be23
;
547
24
548
24
  const DataLayout &DL = M.getDataLayout();
549
24
550
24
  Mod = &M;
551
24
  Ctx = &M.getContext();
552
24
  ShadowTy = IntegerType::get(*Ctx, ShadowWidth);
553
24
  ShadowPtrTy = PointerType::getUnqual(ShadowTy);
554
24
  IntptrTy = DL.getIntPtrType(*Ctx);
555
24
  ZeroShadow = ConstantInt::getSigned(ShadowTy, 0);
556
24
  ShadowPtrMul = ConstantInt::getSigned(IntptrTy, ShadowWidth / 8);
557
24
  if (IsX86_64)
558
23
    ShadowPtrMask = ConstantInt::getSigned(IntptrTy, ~0x700000000000LL);
559
1
  else if (IsMIPS64)
560
0
    ShadowPtrMask = ConstantInt::getSigned(IntptrTy, ~0xF000000000LL);
561
1
  // AArch64 supports multiple VMAs and the shadow mask is set at runtime.
562
1
  else if (IsAArch64)
563
1
    DFSanRuntimeShadowMask = true;
564
0
  else
565
0
    report_fatal_error("unsupported triple");
566
24
567
24
  Type *DFSanUnionArgs[2] = { ShadowTy, ShadowTy };
568
24
  DFSanUnionFnTy =
569
24
      FunctionType::get(ShadowTy, DFSanUnionArgs, /*isVarArg=*/ false);
570
24
  Type *DFSanUnionLoadArgs[2] = { ShadowPtrTy, IntptrTy };
571
24
  DFSanUnionLoadFnTy =
572
24
      FunctionType::get(ShadowTy, DFSanUnionLoadArgs, /*isVarArg=*/ false);
573
24
  DFSanUnimplementedFnTy = FunctionType::get(
574
24
      Type::getVoidTy(*Ctx), Type::getInt8PtrTy(*Ctx), /*isVarArg=*/false);
575
24
  Type *DFSanSetLabelArgs[3] = { ShadowTy, Type::getInt8PtrTy(*Ctx), IntptrTy };
576
24
  DFSanSetLabelFnTy = FunctionType::get(Type::getVoidTy(*Ctx),
577
24
                                        DFSanSetLabelArgs, /*isVarArg=*/false);
578
24
  DFSanNonzeroLabelFnTy = FunctionType::get(
579
24
      Type::getVoidTy(*Ctx), None, /*isVarArg=*/false);
580
24
  DFSanVarargWrapperFnTy = FunctionType::get(
581
24
      Type::getVoidTy(*Ctx), Type::getInt8PtrTy(*Ctx), /*isVarArg=*/false);
582
24
583
24
  if (GetArgTLSPtr) {
584
0
    Type *ArgTLSTy = ArrayType::get(ShadowTy, 64);
585
0
    ArgTLS = nullptr;
586
0
    GetArgTLSTy = FunctionType::get(PointerType::getUnqual(ArgTLSTy), false);
587
0
    GetArgTLS = ConstantExpr::getIntToPtr(
588
0
        ConstantInt::get(IntptrTy, uintptr_t(GetArgTLSPtr)),
589
0
        PointerType::getUnqual(GetArgTLSTy));
590
0
  }
591
24
  if (GetRetvalTLSPtr) {
592
0
    RetvalTLS = nullptr;
593
0
    GetRetvalTLSTy = FunctionType::get(PointerType::getUnqual(ShadowTy), false);
594
0
    GetRetvalTLS = ConstantExpr::getIntToPtr(
595
0
        ConstantInt::get(IntptrTy, uintptr_t(GetRetvalTLSPtr)),
596
0
        PointerType::getUnqual(GetRetvalTLSTy));
597
0
  }
598
24
599
24
  ColdCallWeights = MDBuilder(*Ctx).createBranchWeights(1, 1000);
600
24
  return true;
601
24
}
602
603
96
bool DataFlowSanitizer::isInstrumented(const Function *F) {
604
96
  return !ABIList.isIn(*F, "uninstrumented");
605
96
}
606
607
6
bool DataFlowSanitizer::isInstrumented(const GlobalAlias *GA) {
608
6
  return !ABIList.isIn(*GA, "uninstrumented");
609
6
}
610
611
246
DataFlowSanitizer::InstrumentedABI DataFlowSanitizer::getInstrumentedABI() {
612
246
  return ClArgsABI ? 
IA_Args104
:
IA_TLS142
;
613
246
}
614
615
37
DataFlowSanitizer::WrapperKind DataFlowSanitizer::getWrapperKind(Function *F) {
616
37
  if (ABIList.isIn(*F, "functional"))
617
1
    return WK_Functional;
618
36
  if (ABIList.isIn(*F, "discard"))
619
4
    return WK_Discard;
620
32
  if (ABIList.isIn(*F, "custom"))
621
24
    return WK_Custom;
622
8
623
8
  return WK_Warning;
624
8
}
625
626
72
void DataFlowSanitizer::addGlobalNamePrefix(GlobalValue *GV) {
627
72
  std::string GVName = GV->getName(), Prefix = "dfs$";
628
72
  GV->setName(Prefix + GVName);
629
72
630
72
  // Try to change the name of the function in module inline asm.  We only do
631
72
  // this for specific asm directives, currently only ".symver", to try to avoid
632
72
  // corrupting asm which happens to contain the symbol name as a substring.
633
72
  // Note that the substitution for .symver assumes that the versioned symbol
634
72
  // also has an instrumented name.
635
72
  std::string Asm = GV->getParent()->getModuleInlineAsm();
636
72
  std::string SearchStr = ".symver " + GVName + ",";
637
72
  size_t Pos = Asm.find(SearchStr);
638
72
  if (Pos != std::string::npos) {
639
2
    Asm.replace(Pos, SearchStr.size(),
640
2
                ".symver " + Prefix + GVName + "," + Prefix);
641
2
    GV->getParent()->setModuleInlineAsm(Asm);
642
2
  }
643
72
}
644
645
Function *
646
DataFlowSanitizer::buildWrapperFunction(Function *F, StringRef NewFName,
647
                                        GlobalValue::LinkageTypes NewFLink,
648
24
                                        FunctionType *NewFT) {
649
24
  FunctionType *FT = F->getFunctionType();
650
24
  Function *NewF = Function::Create(NewFT, NewFLink, F->getAddressSpace(),
651
24
                                    NewFName, F->getParent());
652
24
  NewF->copyAttributesFrom(F);
653
24
  NewF->removeAttributes(
654
24
      AttributeList::ReturnIndex,
655
24
      AttributeFuncs::typeIncompatible(NewFT->getReturnType()));
656
24
657
24
  BasicBlock *BB = BasicBlock::Create(*Ctx, "entry", NewF);
658
24
  if (F->isVarArg()) {
659
4
    NewF->removeAttributes(AttributeList::FunctionIndex,
660
4
                           AttrBuilder().addAttribute("split-stack"));
661
4
    CallInst::Create(DFSanVarargWrapperFn,
662
4
                     IRBuilder<>(BB).CreateGlobalStringPtr(F->getName()), "",
663
4
                     BB);
664
4
    new UnreachableInst(*Ctx, BB);
665
20
  } else {
666
20
    std::vector<Value *> Args;
667
20
    unsigned n = FT->getNumParams();
668
52
    for (Function::arg_iterator ai = NewF->arg_begin(); n != 0; 
++ai, --n32
)
669
32
      Args.push_back(&*ai);
670
20
    CallInst *CI = CallInst::Create(F, Args, "", BB);
671
20
    if (FT->getReturnType()->isVoidTy())
672
2
      ReturnInst::Create(*Ctx, BB);
673
18
    else
674
18
      ReturnInst::Create(*Ctx, CI, BB);
675
20
  }
676
24
677
24
  return NewF;
678
24
}
679
680
Constant *DataFlowSanitizer::getOrBuildTrampolineFunction(FunctionType *FT,
681
14
                                                          StringRef FName) {
682
14
  FunctionType *FTT = getTrampolineFunctionType(FT);
683
14
  FunctionCallee C = Mod->getOrInsertFunction(FName, FTT);
684
14
  Function *F = dyn_cast<Function>(C.getCallee());
685
14
  if (F && F->isDeclaration()) {
686
7
    F->setLinkage(GlobalValue::LinkOnceODRLinkage);
687
7
    BasicBlock *BB = BasicBlock::Create(*Ctx, "entry", F);
688
7
    std::vector<Value *> Args;
689
7
    Function::arg_iterator AI = F->arg_begin(); ++AI;
690
20
    for (unsigned N = FT->getNumParams(); N != 0; 
++AI, --N13
)
691
13
      Args.push_back(&*AI);
692
7
    CallInst *CI = CallInst::Create(FT, &*F->arg_begin(), Args, "", BB);
693
7
    ReturnInst *RI;
694
7
    if (FT->getReturnType()->isVoidTy())
695
0
      RI = ReturnInst::Create(*Ctx, BB);
696
7
    else
697
7
      RI = ReturnInst::Create(*Ctx, CI, BB);
698
7
699
7
    DFSanFunction DFSF(*this, F, /*IsNativeABI=*/true);
700
7
    Function::arg_iterator ValAI = F->arg_begin(), ShadowAI = AI; ++ValAI;
701
20
    for (unsigned N = FT->getNumParams(); N != 0; 
++ValAI, ++ShadowAI, --N13
)
702
13
      DFSF.ValShadowMap[&*ValAI] = &*ShadowAI;
703
7
    DFSanVisitor(DFSF).visitCallInst(*CI);
704
7
    if (!FT->getReturnType()->isVoidTy())
705
7
      new StoreInst(DFSF.getShadow(RI->getReturnValue()),
706
7
                    &*std::prev(F->arg_end()), RI);
707
7
  }
708
14
709
14
  return cast<Constant>(C.getCallee());
710
14
}
711
712
24
bool DataFlowSanitizer::runOnModule(Module &M) {
713
24
  if (ABIList.isIn(M, "skip"))
714
0
    return false;
715
24
716
24
  if (!GetArgTLSPtr) {
717
24
    Type *ArgTLSTy = ArrayType::get(ShadowTy, 64);
718
24
    ArgTLS = Mod->getOrInsertGlobal("__dfsan_arg_tls", ArgTLSTy);
719
24
    if (GlobalVariable *G = dyn_cast<GlobalVariable>(ArgTLS))
720
24
      G->setThreadLocalMode(GlobalVariable::InitialExecTLSModel);
721
24
  }
722
24
  if (!GetRetvalTLSPtr) {
723
24
    RetvalTLS = Mod->getOrInsertGlobal("__dfsan_retval_tls", ShadowTy);
724
24
    if (GlobalVariable *G = dyn_cast<GlobalVariable>(RetvalTLS))
725
24
      G->setThreadLocalMode(GlobalVariable::InitialExecTLSModel);
726
24
  }
727
24
728
24
  ExternalShadowMask =
729
24
      Mod->getOrInsertGlobal(kDFSanExternShadowPtrMask, IntptrTy);
730
24
731
24
  {
732
24
    AttributeList AL;
733
24
    AL = AL.addAttribute(M.getContext(), AttributeList::FunctionIndex,
734
24
                         Attribute::NoUnwind);
735
24
    AL = AL.addAttribute(M.getContext(), AttributeList::FunctionIndex,
736
24
                         Attribute::ReadNone);
737
24
    AL = AL.addAttribute(M.getContext(), AttributeList::ReturnIndex,
738
24
                         Attribute::ZExt);
739
24
    AL = AL.addParamAttribute(M.getContext(), 0, Attribute::ZExt);
740
24
    AL = AL.addParamAttribute(M.getContext(), 1, Attribute::ZExt);
741
24
    DFSanUnionFn =
742
24
        Mod->getOrInsertFunction("__dfsan_union", DFSanUnionFnTy, AL);
743
24
  }
744
24
745
24
  {
746
24
    AttributeList AL;
747
24
    AL = AL.addAttribute(M.getContext(), AttributeList::FunctionIndex,
748
24
                         Attribute::NoUnwind);
749
24
    AL = AL.addAttribute(M.getContext(), AttributeList::FunctionIndex,
750
24
                         Attribute::ReadNone);
751
24
    AL = AL.addAttribute(M.getContext(), AttributeList::ReturnIndex,
752
24
                         Attribute::ZExt);
753
24
    AL = AL.addParamAttribute(M.getContext(), 0, Attribute::ZExt);
754
24
    AL = AL.addParamAttribute(M.getContext(), 1, Attribute::ZExt);
755
24
    DFSanCheckedUnionFn =
756
24
        Mod->getOrInsertFunction("dfsan_union", DFSanUnionFnTy, AL);
757
24
  }
758
24
  {
759
24
    AttributeList AL;
760
24
    AL = AL.addAttribute(M.getContext(), AttributeList::FunctionIndex,
761
24
                         Attribute::NoUnwind);
762
24
    AL = AL.addAttribute(M.getContext(), AttributeList::FunctionIndex,
763
24
                         Attribute::ReadOnly);
764
24
    AL = AL.addAttribute(M.getContext(), AttributeList::ReturnIndex,
765
24
                         Attribute::ZExt);
766
24
    DFSanUnionLoadFn =
767
24
        Mod->getOrInsertFunction("__dfsan_union_load", DFSanUnionLoadFnTy, AL);
768
24
  }
769
24
  DFSanUnimplementedFn =
770
24
      Mod->getOrInsertFunction("__dfsan_unimplemented", DFSanUnimplementedFnTy);
771
24
  {
772
24
    AttributeList AL;
773
24
    AL = AL.addParamAttribute(M.getContext(), 0, Attribute::ZExt);
774
24
    DFSanSetLabelFn =
775
24
        Mod->getOrInsertFunction("__dfsan_set_label", DFSanSetLabelFnTy, AL);
776
24
  }
777
24
  DFSanNonzeroLabelFn =
778
24
      Mod->getOrInsertFunction("__dfsan_nonzero_label", DFSanNonzeroLabelFnTy);
779
24
  DFSanVarargWrapperFn = Mod->getOrInsertFunction("__dfsan_vararg_wrapper",
780
24
                                                  DFSanVarargWrapperFnTy);
781
24
782
24
  std::vector<Function *> FnsToInstrument;
783
24
  SmallPtrSet<Function *, 2> FnsWithNativeABI;
784
261
  for (Function &i : M) {
785
261
    if (!i.isIntrinsic() &&
786
261
        
&i != DFSanUnionFn.getCallee()->stripPointerCasts()256
&&
787
261
        
&i != DFSanCheckedUnionFn.getCallee()->stripPointerCasts()232
&&
788
261
        
&i != DFSanUnionLoadFn.getCallee()->stripPointerCasts()208
&&
789
261
        
&i != DFSanUnimplementedFn.getCallee()->stripPointerCasts()184
&&
790
261
        
&i != DFSanSetLabelFn.getCallee()->stripPointerCasts()160
&&
791
261
        
&i != DFSanNonzeroLabelFn.getCallee()->stripPointerCasts()136
&&
792
261
        
&i != DFSanVarargWrapperFn.getCallee()->stripPointerCasts()112
)
793
88
      FnsToInstrument.push_back(&i);
794
261
  }
795
24
796
24
  // Give function aliases prefixes when necessary, and build wrappers where the
797
24
  // instrumentedness is inconsistent.
798
30
  for (Module::alias_iterator i = M.alias_begin(), e = M.alias_end(); i != e;) {
799
6
    GlobalAlias *GA = &*i;
800
6
    ++i;
801
6
    // Don't stop on weak.  We assume people aren't playing games with the
802
6
    // instrumentedness of overridden weak aliases.
803
6
    if (auto F = dyn_cast<Function>(GA->getBaseObject())) {
804
6
      bool GAInst = isInstrumented(GA), FInst = isInstrumented(F);
805
6
      if (GAInst && 
FInst5
) {
806
4
        addGlobalNamePrefix(GA);
807
4
      } else 
if (2
GAInst != FInst2
) {
808
2
        // Non-instrumented alias of an instrumented function, or vice versa.
809
2
        // Replace the alias with a native-ABI wrapper of the aliasee.  The pass
810
2
        // below will take care of instrumenting it.
811
2
        Function *NewF =
812
2
            buildWrapperFunction(F, "", GA->getLinkage(), F->getFunctionType());
813
2
        GA->replaceAllUsesWith(ConstantExpr::getBitCast(NewF, GA->getType()));
814
2
        NewF->takeName(GA);
815
2
        GA->eraseFromParent();
816
2
        FnsToInstrument.push_back(NewF);
817
2
      }
818
6
    }
819
6
  }
820
24
821
24
  ReadOnlyNoneAttrs.addAttribute(Attribute::ReadOnly)
822
24
      .addAttribute(Attribute::ReadNone);
823
24
824
24
  // First, change the ABI of every function in the module.  ABI-listed
825
24
  // functions keep their original ABI and get a wrapper function.
826
24
  for (std::vector<Function *>::iterator i = FnsToInstrument.begin(),
827
24
                                         e = FnsToInstrument.end();
828
114
       i != e; 
++i90
) {
829
90
    Function &F = **i;
830
90
    FunctionType *FT = F.getFunctionType();
831
90
832
90
    bool IsZeroArgsVoidRet = (FT->getNumParams() == 0 && 
!FT->isVarArg()15
&&
833
90
                              
FT->getReturnType()->isVoidTy()13
);
834
90
835
90
    if (isInstrumented(&F)) {
836
68
      // Instrumented functions get a 'dfs$' prefix.  This allows us to more
837
68
      // easily identify cases of mismatching ABIs.
838
68
      if (getInstrumentedABI() == IA_Args && 
!IsZeroArgsVoidRet20
) {
839
18
        FunctionType *NewFT = getArgsFunctionType(FT);
840
18
        Function *NewF = Function::Create(NewFT, F.getLinkage(),
841
18
                                          F.getAddressSpace(), "", &M);
842
18
        NewF->copyAttributesFrom(&F);
843
18
        NewF->removeAttributes(
844
18
            AttributeList::ReturnIndex,
845
18
            AttributeFuncs::typeIncompatible(NewFT->getReturnType()));
846
18
        for (Function::arg_iterator FArg = F.arg_begin(),
847
18
                                    NewFArg = NewF->arg_begin(),
848
18
                                    FArgEnd = F.arg_end();
849
39
             FArg != FArgEnd; 
++FArg, ++NewFArg21
) {
850
21
          FArg->replaceAllUsesWith(&*NewFArg);
851
21
        }
852
18
        NewF->getBasicBlockList().splice(NewF->begin(), F.getBasicBlockList());
853
18
854
18
        for (Function::user_iterator UI = F.user_begin(), UE = F.user_end();
855
25
             UI != UE;) {
856
7
          BlockAddress *BA = dyn_cast<BlockAddress>(*UI);
857
7
          ++UI;
858
7
          if (BA) {
859
0
            BA->replaceAllUsesWith(
860
0
                BlockAddress::get(NewF, BA->getBasicBlock()));
861
0
            delete BA;
862
0
          }
863
7
        }
864
18
        F.replaceAllUsesWith(
865
18
            ConstantExpr::getBitCast(NewF, PointerType::getUnqual(FT)));
866
18
        NewF->takeName(&F);
867
18
        F.eraseFromParent();
868
18
        *i = NewF;
869
18
        addGlobalNamePrefix(NewF);
870
50
      } else {
871
50
        addGlobalNamePrefix(&F);
872
50
      }
873
68
    } else 
if (22
!IsZeroArgsVoidRet22
||
getWrapperKind(&F) == WK_Custom0
) {
874
22
      // Build a wrapper function for F.  The wrapper simply calls F, and is
875
22
      // added to FnsToInstrument so that any instrumentation according to its
876
22
      // WrapperKind is done in the second pass below.
877
22
      FunctionType *NewFT = getInstrumentedABI() == IA_Args
878
22
                                ? 
getArgsFunctionType(FT)13
879
22
                                : 
FT9
;
880
22
881
22
      // If the function being wrapped has local linkage, then preserve the
882
22
      // function's linkage in the wrapper function.
883
22
      GlobalValue::LinkageTypes wrapperLinkage =
884
22
          F.hasLocalLinkage()
885
22
              ? 
F.getLinkage()4
886
22
              : 
GlobalValue::LinkOnceODRLinkage18
;
887
22
888
22
      Function *NewF = buildWrapperFunction(
889
22
          &F, std::string("dfsw$") + std::string(F.getName()),
890
22
          wrapperLinkage, NewFT);
891
22
      if (getInstrumentedABI() == IA_TLS)
892
9
        NewF->removeAttributes(AttributeList::FunctionIndex, ReadOnlyNoneAttrs);
893
22
894
22
      Value *WrappedFnCst =
895
22
          ConstantExpr::getBitCast(NewF, PointerType::getUnqual(FT));
896
22
      F.replaceAllUsesWith(WrappedFnCst);
897
22
898
22
      UnwrappedFnMap[WrappedFnCst] = &F;
899
22
      *i = NewF;
900
22
901
22
      if (!F.isDeclaration()) {
902
8
        // This function is probably defining an interposition of an
903
8
        // uninstrumented function and hence needs to keep the original ABI.
904
8
        // But any functions it may call need to use the instrumented ABI, so
905
8
        // we instrument it in a mode which preserves the original ABI.
906
8
        FnsWithNativeABI.insert(&F);
907
8
908
8
        // This code needs to rebuild the iterators, as they may be invalidated
909
8
        // by the push_back, taking care that the new range does not include
910
8
        // any functions added by this code.
911
8
        size_t N = i - FnsToInstrument.begin(),
912
8
               Count = e - FnsToInstrument.begin();
913
8
        FnsToInstrument.push_back(&F);
914
8
        i = FnsToInstrument.begin() + N;
915
8
        e = FnsToInstrument.begin() + Count;
916
8
      }
917
22
               // Hopefully, nobody will try to indirectly call a vararg
918
22
               // function... yet.
919
22
    } else 
if (0
FT->isVarArg()0
) {
920
0
      UnwrappedFnMap[&F] = &F;
921
0
      *i = nullptr;
922
0
    }
923
90
  }
924
24
925
98
  for (Function *i : FnsToInstrument) {
926
98
    if (!i || i->isDeclaration())
927
10
      continue;
928
88
929
88
    removeUnreachableBlocks(*i);
930
88
931
88
    DFSanFunction DFSF(*this, i, FnsWithNativeABI.count(i));
932
88
933
88
    // DFSanVisitor may create new basic blocks, which confuses df_iterator.
934
88
    // Build a copy of the list before iterating over it.
935
88
    SmallVector<BasicBlock *, 4> BBList(depth_first(&i->getEntryBlock()));
936
88
937
1.09k
    for (BasicBlock *i : BBList) {
938
1.09k
      Instruction *Inst = &i->front();
939
1.19k
      while (true) {
940
1.19k
        // DFSanVisitor may split the current basic block, changing the current
941
1.19k
        // instruction's next pointer and moving the next instruction to the
942
1.19k
        // tail block from which we should continue.
943
1.19k
        Instruction *Next = Inst->getNextNode();
944
1.19k
        // DFSanVisitor may delete Inst, so keep track of whether it was a
945
1.19k
        // terminator.
946
1.19k
        bool IsTerminator = Inst->isTerminator();
947
1.19k
        if (!DFSF.SkipInsts.count(Inst))
948
1.19k
          DFSanVisitor(DFSF).visit(Inst);
949
1.19k
        if (IsTerminator)
950
1.09k
          break;
951
101
        Inst = Next;
952
101
      }
953
1.09k
    }
954
88
955
88
    // We will not necessarily be able to compute the shadow for every phi node
956
88
    // until we have visited every block.  Therefore, the code that handles phi
957
88
    // nodes adds them to the PHIFixups list so that they can be properly
958
88
    // handled here.
959
88
    for (std::vector<std::pair<PHINode *, PHINode *>>::iterator
960
88
             i = DFSF.PHIFixups.begin(),
961
88
             e = DFSF.PHIFixups.end();
962
88
         i != e; 
++i0
) {
963
0
      for (unsigned val = 0, n = i->first->getNumIncomingValues(); val != n;
964
0
           ++val) {
965
0
        i->second->setIncomingValue(
966
0
            val, DFSF.getShadow(i->first->getIncomingValue(val)));
967
0
      }
968
0
    }
969
88
970
88
    // -dfsan-debug-nonzero-labels will split the CFG in all kinds of crazy
971
88
    // places (i.e. instructions in basic blocks we haven't even begun visiting
972
88
    // yet).  To make our life easier, do this work in a pass after the main
973
88
    // instrumentation.
974
88
    if (ClDebugNonzeroLabels) {
975
4
      for (Value *V : DFSF.NonZeroChecks) {
976
4
        Instruction *Pos;
977
4
        if (Instruction *I = dyn_cast<Instruction>(V))
978
2
          Pos = I->getNextNode();
979
2
        else
980
2
          Pos = &DFSF.F->getEntryBlock().front();
981
8
        while (isa<PHINode>(Pos) || isa<AllocaInst>(Pos))
982
4
          Pos = Pos->getNextNode();
983
4
        IRBuilder<> IRB(Pos);
984
4
        Value *Ne = IRB.CreateICmpNE(V, DFSF.DFS.ZeroShadow);
985
4
        BranchInst *BI = cast<BranchInst>(SplitBlockAndInsertIfThen(
986
4
            Ne, Pos, /*Unreachable=*/false, ColdCallWeights));
987
4
        IRBuilder<> ThenIRB(BI);
988
4
        ThenIRB.CreateCall(DFSF.DFS.DFSanNonzeroLabelFn, {});
989
4
      }
990
1
    }
991
88
  }
992
24
993
24
  return false;
994
24
}
995
996
106
Value *DFSanFunction::getArgTLSPtr() {
997
106
  if (ArgTLSPtr)
998
72
    return ArgTLSPtr;
999
34
  if (DFS.ArgTLS)
1000
34
    return ArgTLSPtr = DFS.ArgTLS;
1001
0
1002
0
  IRBuilder<> IRB(&F->getEntryBlock().front());
1003
0
  return ArgTLSPtr = IRB.CreateCall(DFS.GetArgTLSTy, DFS.GetArgTLS, {});
1004
0
}
1005
1006
39
Value *DFSanFunction::getRetvalTLS() {
1007
39
  if (RetvalTLSPtr)
1008
2
    return RetvalTLSPtr;
1009
37
  if (DFS.RetvalTLS)
1010
37
    return RetvalTLSPtr = DFS.RetvalTLS;
1011
0
1012
0
  IRBuilder<> IRB(&F->getEntryBlock().front());
1013
0
  return RetvalTLSPtr =
1014
0
             IRB.CreateCall(DFS.GetRetvalTLSTy, DFS.GetRetvalTLS, {});
1015
0
}
1016
1017
57
Value *DFSanFunction::getArgTLS(unsigned Idx, Instruction *Pos) {
1018
57
  IRBuilder<> IRB(Pos);
1019
57
  return IRB.CreateConstGEP2_64(ArrayType::get(DFS.ShadowTy, 64),
1020
57
                                getArgTLSPtr(), 0, Idx);
1021
57
}
1022
1023
204
Value *DFSanFunction::getShadow(Value *V) {
1024
204
  if (!isa<Argument>(V) && 
!isa<Instruction>(V)113
)
1025
48
    return DFS.ZeroShadow;
1026
156
  Value *&Shadow = ValShadowMap[V];
1027
156
  if (!Shadow) {
1028
74
    if (Argument *A = dyn_cast<Argument>(V)) {
1029
73
      if (IsNativeABI)
1030
3
        return DFS.ZeroShadow;
1031
70
      switch (IA) {
1032
70
      case DataFlowSanitizer::IA_TLS: {
1033
49
        Value *ArgTLSPtr = getArgTLSPtr();
1034
49
        Instruction *ArgTLSPos =
1035
49
            DFS.ArgTLS ? &*F->getEntryBlock().begin()
1036
49
                       : 
cast<Instruction>(ArgTLSPtr)->getNextNode()0
;
1037
49
        IRBuilder<> IRB(ArgTLSPos);
1038
49
        Shadow =
1039
49
            IRB.CreateLoad(DFS.ShadowTy, getArgTLS(A->getArgNo(), ArgTLSPos));
1040
49
        break;
1041
70
      }
1042
70
      case DataFlowSanitizer::IA_Args: {
1043
21
        unsigned ArgIdx = A->getArgNo() + F->arg_size() / 2;
1044
21
        Function::arg_iterator i = F->arg_begin();
1045
71
        while (ArgIdx--)
1046
50
          ++i;
1047
21
        Shadow = &*i;
1048
21
        assert(Shadow->getType() == DFS.ShadowTy);
1049
21
        break;
1050
70
      }
1051
70
      }
1052
70
      NonZeroChecks.push_back(Shadow);
1053
70
    } else {
1054
1
      Shadow = DFS.ZeroShadow;
1055
1
    }
1056
74
  }
1057
156
  
return Shadow153
;
1058
156
}
1059
1060
80
void DFSanFunction::setShadow(Instruction *I, Value *Shadow) {
1061
80
  assert(!ValShadowMap.count(I));
1062
80
  assert(Shadow->getType() == DFS.ShadowTy);
1063
80
  ValShadowMap[I] = Shadow;
1064
80
}
1065
1066
21
Value *DataFlowSanitizer::getShadowAddress(Value *Addr, Instruction *Pos) {
1067
21
  assert(Addr != RetvalTLS && "Reinstrumenting?");
1068
21
  IRBuilder<> IRB(Pos);
1069
21
  Value *ShadowPtrMaskValue;
1070
21
  if (DFSanRuntimeShadowMask)
1071
1
    ShadowPtrMaskValue = IRB.CreateLoad(IntptrTy, ExternalShadowMask);
1072
20
  else
1073
20
    ShadowPtrMaskValue = ShadowPtrMask;
1074
21
  return IRB.CreateIntToPtr(
1075
21
      IRB.CreateMul(
1076
21
          IRB.CreateAnd(IRB.CreatePtrToInt(Addr, IntptrTy),
1077
21
                        IRB.CreatePtrToInt(ShadowPtrMaskValue, IntptrTy)),
1078
21
          ShadowPtrMul),
1079
21
      ShadowPtrTy);
1080
21
}
1081
1082
// Generates IR to compute the union of the two given shadows, inserting it
1083
// before Pos.  Returns the computed union Value.
1084
42
Value *DFSanFunction::combineShadows(Value *V1, Value *V2, Instruction *Pos) {
1085
42
  if (V1 == DFS.ZeroShadow)
1086
2
    return V2;
1087
40
  if (V2 == DFS.ZeroShadow)
1088
12
    return V1;
1089
28
  if (V1 == V2)
1090
0
    return V1;
1091
28
1092
28
  auto V1Elems = ShadowElements.find(V1);
1093
28
  auto V2Elems = ShadowElements.find(V2);
1094
28
  if (V1Elems != ShadowElements.end() && 
V2Elems != ShadowElements.end()3
) {
1095
0
    if (std::includes(V1Elems->second.begin(), V1Elems->second.end(),
1096
0
                      V2Elems->second.begin(), V2Elems->second.end())) {
1097
0
      return V1;
1098
0
    } else if (std::includes(V2Elems->second.begin(), V2Elems->second.end(),
1099
0
                             V1Elems->second.begin(), V1Elems->second.end())) {
1100
0
      return V2;
1101
0
    }
1102
28
  } else if (V1Elems != ShadowElements.end()) {
1103
3
    if (V1Elems->second.count(V2))
1104
1
      return V1;
1105
25
  } else if (V2Elems != ShadowElements.end()) {
1106
0
    if (V2Elems->second.count(V1))
1107
0
      return V2;
1108
27
  }
1109
27
1110
27
  auto Key = std::make_pair(V1, V2);
1111
27
  if (V1 > V2)
1112
5
    std::swap(Key.first, Key.second);
1113
27
  CachedCombinedShadow &CCS = CachedCombinedShadows[Key];
1114
27
  if (CCS.Block && 
DT.dominates(CCS.Block, Pos->getParent())2
)
1115
1
    return CCS.Shadow;
1116
26
1117
26
  IRBuilder<> IRB(Pos);
1118
26
  if (AvoidNewBlocks) {
1119
1
    CallInst *Call = IRB.CreateCall(DFS.DFSanCheckedUnionFn, {V1, V2});
1120
1
    Call->addAttribute(AttributeList::ReturnIndex, Attribute::ZExt);
1121
1
    Call->addParamAttr(0, Attribute::ZExt);
1122
1
    Call->addParamAttr(1, Attribute::ZExt);
1123
1
1124
1
    CCS.Block = Pos->getParent();
1125
1
    CCS.Shadow = Call;
1126
25
  } else {
1127
25
    BasicBlock *Head = Pos->getParent();
1128
25
    Value *Ne = IRB.CreateICmpNE(V1, V2);
1129
25
    BranchInst *BI = cast<BranchInst>(SplitBlockAndInsertIfThen(
1130
25
        Ne, Pos, /*Unreachable=*/false, DFS.ColdCallWeights, &DT));
1131
25
    IRBuilder<> ThenIRB(BI);
1132
25
    CallInst *Call = ThenIRB.CreateCall(DFS.DFSanUnionFn, {V1, V2});
1133
25
    Call->addAttribute(AttributeList::ReturnIndex, Attribute::ZExt);
1134
25
    Call->addParamAttr(0, Attribute::ZExt);
1135
25
    Call->addParamAttr(1, Attribute::ZExt);
1136
25
1137
25
    BasicBlock *Tail = BI->getSuccessor(0);
1138
25
    PHINode *Phi = PHINode::Create(DFS.ShadowTy, 2, "", &Tail->front());
1139
25
    Phi->addIncoming(Call, Call->getParent());
1140
25
    Phi->addIncoming(V1, Head);
1141
25
1142
25
    CCS.Block = Tail;
1143
25
    CCS.Shadow = Phi;
1144
25
  }
1145
26
1146
26
  std::set<Value *> UnionElems;
1147
26
  if (V1Elems != ShadowElements.end()) {
1148
2
    UnionElems = V1Elems->second;
1149
24
  } else {
1150
24
    UnionElems.insert(V1);
1151
24
  }
1152
26
  if (V2Elems != ShadowElements.end()) {
1153
0
    UnionElems.insert(V2Elems->second.begin(), V2Elems->second.end());
1154
26
  } else {
1155
26
    UnionElems.insert(V2);
1156
26
  }
1157
26
  ShadowElements[CCS.Shadow] = std::move(UnionElems);
1158
26
1159
26
  return CCS.Shadow;
1160
26
}
1161
1162
// A convenience function which folds the shadows of each of the operands
1163
// of the provided instruction Inst, inserting the IR before Inst.  Returns
1164
// the computed union Value.
1165
25
Value *DFSanFunction::combineOperandShadows(Instruction *Inst) {
1166
25
  if (Inst->getNumOperands() == 0)
1167
0
    return DFS.ZeroShadow;
1168
25
1169
25
  Value *Shadow = getShadow(Inst->getOperand(0));
1170
55
  for (unsigned i = 1, n = Inst->getNumOperands(); i != n; 
++i30
) {
1171
30
    Shadow = combineShadows(Shadow, getShadow(Inst->getOperand(i)), Inst);
1172
30
  }
1173
25
  return Shadow;
1174
25
}
1175
1176
25
void DFSanVisitor::visitOperandShadowInst(Instruction &I) {
1177
25
  Value *CombinedShadow = DFSF.combineOperandShadows(&I);
1178
25
  DFSF.setShadow(&I, CombinedShadow);
1179
25
}
1180
1181
// Generates IR to load shadow corresponding to bytes [Addr, Addr+Size), where
1182
// Addr has alignment Align, and take the union of each of those shadows.
1183
Value *DFSanFunction::loadShadow(Value *Addr, uint64_t Size, uint64_t Align,
1184
10
                                 Instruction *Pos) {
1185
10
  if (AllocaInst *AI = dyn_cast<AllocaInst>(Addr)) {
1186
1
    const auto i = AllocaShadowMap.find(AI);
1187
1
    if (i != AllocaShadowMap.end()) {
1188
1
      IRBuilder<> IRB(Pos);
1189
1
      return IRB.CreateLoad(DFS.ShadowTy, i->second);
1190
1
    }
1191
9
  }
1192
9
1193
9
  uint64_t ShadowAlign = Align * DFS.ShadowWidth / 8;
1194
9
  SmallVector<const Value *, 2> Objs;
1195
9
  GetUnderlyingObjects(Addr, Objs, Pos->getModule()->getDataLayout());
1196
9
  bool AllConstants = true;
1197
9
  for (const Value *Obj : Objs) {
1198
9
    if (isa<Function>(Obj) || isa<BlockAddress>(Obj))
1199
0
      continue;
1200
9
    if (isa<GlobalVariable>(Obj) && 
cast<GlobalVariable>(Obj)->isConstant()0
)
1201
0
      continue;
1202
9
1203
9
    AllConstants = false;
1204
9
    break;
1205
9
  }
1206
9
  if (AllConstants)
1207
0
    return DFS.ZeroShadow;
1208
9
1209
9
  Value *ShadowAddr = DFS.getShadowAddress(Addr, Pos);
1210
9
  switch (Size) {
1211
9
  case 0:
1212
0
    return DFS.ZeroShadow;
1213
9
  case 1: {
1214
2
    LoadInst *LI = new LoadInst(DFS.ShadowTy, ShadowAddr, "", Pos);
1215
2
    LI->setAlignment(ShadowAlign);
1216
2
    return LI;
1217
9
  }
1218
9
  case 2: {
1219
2
    IRBuilder<> IRB(Pos);
1220
2
    Value *ShadowAddr1 = IRB.CreateGEP(DFS.ShadowTy, ShadowAddr,
1221
2
                                       ConstantInt::get(DFS.IntptrTy, 1));
1222
2
    return combineShadows(
1223
2
        IRB.CreateAlignedLoad(DFS.ShadowTy, ShadowAddr, ShadowAlign),
1224
2
        IRB.CreateAlignedLoad(DFS.ShadowTy, ShadowAddr1, ShadowAlign), Pos);
1225
5
  }
1226
5
  }
1227
5
  if (!AvoidNewBlocks && Size % (64 / DFS.ShadowWidth) == 0) {
1228
5
    // Fast path for the common case where each byte has identical shadow: load
1229
5
    // shadow 64 bits at a time, fall out to a __dfsan_union_load call if any
1230
5
    // shadow is non-equal.
1231
5
    BasicBlock *FallbackBB = BasicBlock::Create(*DFS.Ctx, "", F);
1232
5
    IRBuilder<> FallbackIRB(FallbackBB);
1233
5
    CallInst *FallbackCall = FallbackIRB.CreateCall(
1234
5
        DFS.DFSanUnionLoadFn,
1235
5
        {ShadowAddr, ConstantInt::get(DFS.IntptrTy, Size)});
1236
5
    FallbackCall->addAttribute(AttributeList::ReturnIndex, Attribute::ZExt);
1237
5
1238
5
    // Compare each of the shadows stored in the loaded 64 bits to each other,
1239
5
    // by computing (WideShadow rotl ShadowWidth) == WideShadow.
1240
5
    IRBuilder<> IRB(Pos);
1241
5
    Value *WideAddr =
1242
5
        IRB.CreateBitCast(ShadowAddr, Type::getInt64PtrTy(*DFS.Ctx));
1243
5
    Value *WideShadow =
1244
5
        IRB.CreateAlignedLoad(IRB.getInt64Ty(), WideAddr, ShadowAlign);
1245
5
    Value *TruncShadow = IRB.CreateTrunc(WideShadow, DFS.ShadowTy);
1246
5
    Value *ShlShadow = IRB.CreateShl(WideShadow, DFS.ShadowWidth);
1247
5
    Value *ShrShadow = IRB.CreateLShr(WideShadow, 64 - DFS.ShadowWidth);
1248
5
    Value *RotShadow = IRB.CreateOr(ShlShadow, ShrShadow);
1249
5
    Value *ShadowsEq = IRB.CreateICmpEQ(WideShadow, RotShadow);
1250
5
1251
5
    BasicBlock *Head = Pos->getParent();
1252
5
    BasicBlock *Tail = Head->splitBasicBlock(Pos->getIterator());
1253
5
1254
5
    if (DomTreeNode *OldNode = DT.getNode(Head)) {
1255
5
      std::vector<DomTreeNode *> Children(OldNode->begin(), OldNode->end());
1256
5
1257
5
      DomTreeNode *NewNode = DT.addNewBlock(Tail, Head);
1258
5
      for (auto Child : Children)
1259
0
        DT.changeImmediateDominator(Child, NewNode);
1260
5
    }
1261
5
1262
5
    // In the following code LastBr will refer to the previous basic block's
1263
5
    // conditional branch instruction, whose true successor is fixed up to point
1264
5
    // to the next block during the loop below or to the tail after the final
1265
5
    // iteration.
1266
5
    BranchInst *LastBr = BranchInst::Create(FallbackBB, FallbackBB, ShadowsEq);
1267
5
    ReplaceInstWithInst(Head->getTerminator(), LastBr);
1268
5
    DT.addNewBlock(FallbackBB, Head);
1269
5
1270
7
    for (uint64_t Ofs = 64 / DFS.ShadowWidth; Ofs != Size;
1271
5
         
Ofs += 64 / DFS.ShadowWidth2
) {
1272
2
      BasicBlock *NextBB = BasicBlock::Create(*DFS.Ctx, "", F);
1273
2
      DT.addNewBlock(NextBB, LastBr->getParent());
1274
2
      IRBuilder<> NextIRB(NextBB);
1275
2
      WideAddr = NextIRB.CreateGEP(Type::getInt64Ty(*DFS.Ctx), WideAddr,
1276
2
                                   ConstantInt::get(DFS.IntptrTy, 1));
1277
2
      Value *NextWideShadow = NextIRB.CreateAlignedLoad(NextIRB.getInt64Ty(),
1278
2
                                                        WideAddr, ShadowAlign);
1279
2
      ShadowsEq = NextIRB.CreateICmpEQ(WideShadow, NextWideShadow);
1280
2
      LastBr->setSuccessor(0, NextBB);
1281
2
      LastBr = NextIRB.CreateCondBr(ShadowsEq, FallbackBB, FallbackBB);
1282
2
    }
1283
5
1284
5
    LastBr->setSuccessor(0, Tail);
1285
5
    FallbackIRB.CreateBr(Tail);
1286
5
    PHINode *Shadow = PHINode::Create(DFS.ShadowTy, 2, "", &Tail->front());
1287
5
    Shadow->addIncoming(FallbackCall, FallbackBB);
1288
5
    Shadow->addIncoming(TruncShadow, LastBr->getParent());
1289
5
    return Shadow;
1290
5
  }
1291
0
1292
0
  IRBuilder<> IRB(Pos);
1293
0
  CallInst *FallbackCall = IRB.CreateCall(
1294
0
      DFS.DFSanUnionLoadFn, {ShadowAddr, ConstantInt::get(DFS.IntptrTy, Size)});
1295
0
  FallbackCall->addAttribute(AttributeList::ReturnIndex, Attribute::ZExt);
1296
0
  return FallbackCall;
1297
0
}
1298
1299
12
void DFSanVisitor::visitLoadInst(LoadInst &LI) {
1300
12
  auto &DL = LI.getModule()->getDataLayout();
1301
12
  uint64_t Size = DL.getTypeStoreSize(LI.getType());
1302
12
  if (Size == 0) {
1303
2
    DFSF.setShadow(&LI, DFSF.DFS.ZeroShadow);
1304
2
    return;
1305
2
  }
1306
10
1307
10
  uint64_t Align;
1308
10
  if (ClPreserveAlignment) {
1309
0
    Align = LI.getAlignment();
1310
0
    if (Align == 0)
1311
0
      Align = DL.getABITypeAlignment(LI.getType());
1312
10
  } else {
1313
10
    Align = 1;
1314
10
  }
1315
10
  IRBuilder<> IRB(&LI);
1316
10
  Value *Shadow = DFSF.loadShadow(LI.getPointerOperand(), Size, Align, &LI);
1317
10
  if (ClCombinePointerLabelsOnLoad) {
1318
6
    Value *PtrShadow = DFSF.getShadow(LI.getPointerOperand());
1319
6
    Shadow = DFSF.combineShadows(Shadow, PtrShadow, &LI);
1320
6
  }
1321
10
  if (Shadow != DFSF.DFS.ZeroShadow)
1322
10
    DFSF.NonZeroChecks.push_back(Shadow);
1323
10
1324
10
  DFSF.setShadow(&LI, Shadow);
1325
10
}
1326
1327
void DFSanFunction::storeShadow(Value *Addr, uint64_t Size, uint64_t Align,
1328
13
                                Value *Shadow, Instruction *Pos) {
1329
13
  if (AllocaInst *AI = dyn_cast<AllocaInst>(Addr)) {
1330
1
    const auto i = AllocaShadowMap.find(AI);
1331
1
    if (i != AllocaShadowMap.end()) {
1332
1
      IRBuilder<> IRB(Pos);
1333
1
      IRB.CreateStore(Shadow, i->second);
1334
1
      return;
1335
1
    }
1336
12
  }
1337
12
1338
12
  uint64_t ShadowAlign = Align * DFS.ShadowWidth / 8;
1339
12
  IRBuilder<> IRB(Pos);
1340
12
  Value *ShadowAddr = DFS.getShadowAddress(Addr, Pos);
1341
12
  if (Shadow == DFS.ZeroShadow) {
1342
0
    IntegerType *ShadowTy = IntegerType::get(*DFS.Ctx, Size * DFS.ShadowWidth);
1343
0
    Value *ExtZeroShadow = ConstantInt::get(ShadowTy, 0);
1344
0
    Value *ExtShadowAddr =
1345
0
        IRB.CreateBitCast(ShadowAddr, PointerType::getUnqual(ShadowTy));
1346
0
    IRB.CreateAlignedStore(ExtZeroShadow, ExtShadowAddr, ShadowAlign);
1347
0
    return;
1348
0
  }
1349
12
1350
12
  const unsigned ShadowVecSize = 128 / DFS.ShadowWidth;
1351
12
  uint64_t Offset = 0;
1352
12
  if (Size >= ShadowVecSize) {
1353
2
    VectorType *ShadowVecTy = VectorType::get(DFS.ShadowTy, ShadowVecSize);
1354
2
    Value *ShadowVec = UndefValue::get(ShadowVecTy);
1355
18
    for (unsigned i = 0; i != ShadowVecSize; 
++i16
) {
1356
16
      ShadowVec = IRB.CreateInsertElement(
1357
16
          ShadowVec, Shadow, ConstantInt::get(Type::getInt32Ty(*DFS.Ctx), i));
1358
16
    }
1359
2
    Value *ShadowVecAddr =
1360
2
        IRB.CreateBitCast(ShadowAddr, PointerType::getUnqual(ShadowVecTy));
1361
2
    do {
1362
2
      Value *CurShadowVecAddr =
1363
2
          IRB.CreateConstGEP1_32(ShadowVecTy, ShadowVecAddr, Offset);
1364
2
      IRB.CreateAlignedStore(ShadowVec, CurShadowVecAddr, ShadowAlign);
1365
2
      Size -= ShadowVecSize;
1366
2
      ++Offset;
1367
2
    } while (Size >= ShadowVecSize);
1368
2
    Offset *= ShadowVecSize;
1369
2
  }
1370
42
  while (Size > 0) {
1371
30
    Value *CurShadowAddr =
1372
30
        IRB.CreateConstGEP1_32(DFS.ShadowTy, ShadowAddr, Offset);
1373
30
    IRB.CreateAlignedStore(Shadow, CurShadowAddr, ShadowAlign);
1374
30
    --Size;
1375
30
    ++Offset;
1376
30
  }
1377
12
}
1378
1379
15
void DFSanVisitor::visitStoreInst(StoreInst &SI) {
1380
15
  auto &DL = SI.getModule()->getDataLayout();
1381
15
  uint64_t Size = DL.getTypeStoreSize(SI.getValueOperand()->getType());
1382
15
  if (Size == 0)
1383
2
    return;
1384
13
1385
13
  uint64_t Align;
1386
13
  if (ClPreserveAlignment) {
1387
0
    Align = SI.getAlignment();
1388
0
    if (Align == 0)
1389
0
      Align = DL.getABITypeAlignment(SI.getValueOperand()->getType());
1390
13
  } else {
1391
13
    Align = 1;
1392
13
  }
1393
13
1394
13
  Value* Shadow = DFSF.getShadow(SI.getValueOperand());
1395
13
  if (ClCombinePointerLabelsOnStore) {
1396
4
    Value *PtrShadow = DFSF.getShadow(SI.getPointerOperand());
1397
4
    Shadow = DFSF.combineShadows(Shadow, PtrShadow, &SI);
1398
4
  }
1399
13
  DFSF.storeShadow(SI.getPointerOperand(), Size, Align, Shadow, &SI);
1400
13
}
1401
1402
1
void DFSanVisitor::visitUnaryOperator(UnaryOperator &UO) {
1403
1
  visitOperandShadowInst(UO);
1404
1
}
1405
1406
15
void DFSanVisitor::visitBinaryOperator(BinaryOperator &BO) {
1407
15
  visitOperandShadowInst(BO);
1408
15
}
1409
1410
3
void DFSanVisitor::visitCastInst(CastInst &CI) { visitOperandShadowInst(CI); }
1411
1412
0
void DFSanVisitor::visitCmpInst(CmpInst &CI) { visitOperandShadowInst(CI); }
1413
1414
0
void DFSanVisitor::visitGetElementPtrInst(GetElementPtrInst &GEPI) {
1415
0
  visitOperandShadowInst(GEPI);
1416
0
}
1417
1418
0
void DFSanVisitor::visitExtractElementInst(ExtractElementInst &I) {
1419
0
  visitOperandShadowInst(I);
1420
0
}
1421
1422
0
void DFSanVisitor::visitInsertElementInst(InsertElementInst &I) {
1423
0
  visitOperandShadowInst(I);
1424
0
}
1425
1426
0
void DFSanVisitor::visitShuffleVectorInst(ShuffleVectorInst &I) {
1427
0
  visitOperandShadowInst(I);
1428
0
}
1429
1430
1
void DFSanVisitor::visitExtractValueInst(ExtractValueInst &I) {
1431
1
  visitOperandShadowInst(I);
1432
1
}
1433
1434
0
void DFSanVisitor::visitInsertValueInst(InsertValueInst &I) {
1435
0
  visitOperandShadowInst(I);
1436
0
}
1437
1438
1
void DFSanVisitor::visitAllocaInst(AllocaInst &I) {
1439
1
  bool AllLoadsStores = true;
1440
2
  for (User *U : I.users()) {
1441
2
    if (isa<LoadInst>(U))
1442
1
      continue;
1443
1
1444
1
    if (StoreInst *SI = dyn_cast<StoreInst>(U)) {
1445
1
      if (SI->getPointerOperand() == &I)
1446
1
        continue;
1447
0
    }
1448
0
1449
0
    AllLoadsStores = false;
1450
0
    break;
1451
0
  }
1452
1
  if (AllLoadsStores) {
1453
1
    IRBuilder<> IRB(&I);
1454
1
    DFSF.AllocaShadowMap[&I] = IRB.CreateAlloca(DFSF.DFS.ShadowTy);
1455
1
  }
1456
1
  DFSF.setShadow(&I, DFSF.DFS.ZeroShadow);
1457
1
}
1458
1459
0
void DFSanVisitor::visitSelectInst(SelectInst &I) {
1460
0
  Value *CondShadow = DFSF.getShadow(I.getCondition());
1461
0
  Value *TrueShadow = DFSF.getShadow(I.getTrueValue());
1462
0
  Value *FalseShadow = DFSF.getShadow(I.getFalseValue());
1463
0
1464
0
  if (isa<VectorType>(I.getCondition()->getType())) {
1465
0
    DFSF.setShadow(
1466
0
        &I,
1467
0
        DFSF.combineShadows(
1468
0
            CondShadow, DFSF.combineShadows(TrueShadow, FalseShadow, &I), &I));
1469
0
  } else {
1470
0
    Value *ShadowSel;
1471
0
    if (TrueShadow == FalseShadow) {
1472
0
      ShadowSel = TrueShadow;
1473
0
    } else {
1474
0
      ShadowSel =
1475
0
          SelectInst::Create(I.getCondition(), TrueShadow, FalseShadow, "", &I);
1476
0
    }
1477
0
    DFSF.setShadow(&I, DFSF.combineShadows(CondShadow, ShadowSel, &I));
1478
0
  }
1479
0
}
1480
1481
1
void DFSanVisitor::visitMemSetInst(MemSetInst &I) {
1482
1
  IRBuilder<> IRB(&I);
1483
1
  Value *ValShadow = DFSF.getShadow(I.getValue());
1484
1
  IRB.CreateCall(DFSF.DFS.DFSanSetLabelFn,
1485
1
                 {ValShadow, IRB.CreateBitCast(I.getDest(), Type::getInt8PtrTy(
1486
1
                                                                *DFSF.DFS.Ctx)),
1487
1
                  IRB.CreateZExtOrTrunc(I.getLength(), DFSF.DFS.IntptrTy)});
1488
1
}
1489
1490
0
void DFSanVisitor::visitMemTransferInst(MemTransferInst &I) {
1491
0
  IRBuilder<> IRB(&I);
1492
0
  Value *DestShadow = DFSF.DFS.getShadowAddress(I.getDest(), &I);
1493
0
  Value *SrcShadow = DFSF.DFS.getShadowAddress(I.getSource(), &I);
1494
0
  Value *LenShadow = IRB.CreateMul(
1495
0
      I.getLength(),
1496
0
      ConstantInt::get(I.getLength()->getType(), DFSF.DFS.ShadowWidth / 8));
1497
0
  Type *Int8Ptr = Type::getInt8PtrTy(*DFSF.DFS.Ctx);
1498
0
  DestShadow = IRB.CreateBitCast(DestShadow, Int8Ptr);
1499
0
  SrcShadow = IRB.CreateBitCast(SrcShadow, Int8Ptr);
1500
0
  auto *MTI = cast<MemTransferInst>(
1501
0
      IRB.CreateCall(I.getFunctionType(), I.getCalledValue(),
1502
0
                     {DestShadow, SrcShadow, LenShadow, I.getVolatileCst()}));
1503
0
  if (ClPreserveAlignment) {
1504
0
    MTI->setDestAlignment(I.getDestAlignment() * (DFSF.DFS.ShadowWidth / 8));
1505
0
    MTI->setSourceAlignment(I.getSourceAlignment() * (DFSF.DFS.ShadowWidth / 8));
1506
0
  } else {
1507
0
    MTI->setDestAlignment(DFSF.DFS.ShadowWidth / 8);
1508
0
    MTI->setSourceAlignment(DFSF.DFS.ShadowWidth / 8);
1509
0
  }
1510
0
}
1511
1512
83
void DFSanVisitor::visitReturnInst(ReturnInst &RI) {
1513
83
  if (!DFSF.IsNativeABI && 
RI.getReturnValue()75
) {
1514
48
    switch (DFSF.IA) {
1515
48
    case DataFlowSanitizer::IA_TLS: {
1516
34
      Value *S = DFSF.getShadow(RI.getReturnValue());
1517
34
      IRBuilder<> IRB(&RI);
1518
34
      IRB.CreateStore(S, DFSF.getRetvalTLS());
1519
34
      break;
1520
48
    }
1521
48
    case DataFlowSanitizer::IA_Args: {
1522
14
      IRBuilder<> IRB(&RI);
1523
14
      Type *RT = DFSF.F->getFunctionType()->getReturnType();
1524
14
      Value *InsVal =
1525
14
          IRB.CreateInsertValue(UndefValue::get(RT), RI.getReturnValue(), 0);
1526
14
      Value *InsShadow =
1527
14
          IRB.CreateInsertValue(InsVal, DFSF.getShadow(RI.getReturnValue()), 1);
1528
14
      RI.setOperand(0, InsShadow);
1529
14
      break;
1530
48
    }
1531
48
    }
1532
48
  }
1533
83
}
1534
1535
59
void DFSanVisitor::visitCallSite(CallSite CS) {
1536
59
  Function *F = CS.getCalledFunction();
1537
59
  if ((F && 
F->isIntrinsic()29
) ||
isa<InlineAsm>(CS.getCalledValue())55
) {
1538
4
    visitOperandShadowInst(*CS.getInstruction());
1539
4
    return;
1540
4
  }
1541
55
1542
55
  // Calls to this function are synthesized in wrappers, and we shouldn't
1543
55
  // instrument them.
1544
55
  if (F == DFSF.DFS.DFSanVarargWrapperFn.getCallee()->stripPointerCasts())
1545
4
    return;
1546
51
1547
51
  IRBuilder<> IRB(CS.getInstruction());
1548
51
1549
51
  DenseMap<Value *, Function *>::iterator i =
1550
51
      DFSF.DFS.UnwrappedFnMap.find(CS.getCalledValue());
1551
51
  if (i != DFSF.DFS.UnwrappedFnMap.end()) {
1552
37
    Function *F = i->second;
1553
37
    switch (DFSF.DFS.getWrapperKind(F)) {
1554
37
    case DataFlowSanitizer::WK_Warning:
1555
8
      CS.setCalledFunction(F);
1556
8
      IRB.CreateCall(DFSF.DFS.DFSanUnimplementedFn,
1557
8
                     IRB.CreateGlobalStringPtr(F->getName()));
1558
8
      DFSF.setShadow(CS.getInstruction(), DFSF.DFS.ZeroShadow);
1559
8
      return;
1560
37
    case DataFlowSanitizer::WK_Discard:
1561
4
      CS.setCalledFunction(F);
1562
4
      DFSF.setShadow(CS.getInstruction(), DFSF.DFS.ZeroShadow);
1563
4
      return;
1564
37
    case DataFlowSanitizer::WK_Functional:
1565
1
      CS.setCalledFunction(F);
1566
1
      visitOperandShadowInst(*CS.getInstruction());
1567
1
      return;
1568
37
    case DataFlowSanitizer::WK_Custom:
1569
24
      // Don't try to handle invokes of custom functions, it's too complicated.
1570
24
      // Instead, invoke the dfsw$ wrapper, which will in turn call the __dfsw_
1571
24
      // wrapper.
1572
24
      if (CallInst *CI = dyn_cast<CallInst>(CS.getInstruction())) {
1573
24
        FunctionType *FT = F->getFunctionType();
1574
24
        TransformedFunction CustomFn = DFSF.DFS.getCustomFunctionType(FT);
1575
24
        std::string CustomFName = "__dfsw_";
1576
24
        CustomFName += F->getName();
1577
24
        FunctionCallee CustomF = DFSF.DFS.Mod->getOrInsertFunction(
1578
24
            CustomFName, CustomFn.TransformedType);
1579
24
        if (Function *CustomFn = dyn_cast<Function>(CustomF.getCallee())) {
1580
24
          CustomFn->copyAttributesFrom(F);
1581
24
1582
24
          // Custom functions returning non-void will write to the return label.
1583
24
          if (!FT->getReturnType()->isVoidTy()) {
1584
19
            CustomFn->removeAttributes(AttributeList::FunctionIndex,
1585
19
                                       DFSF.DFS.ReadOnlyNoneAttrs);
1586
19
          }
1587
24
        }
1588
24
1589
24
        std::vector<Value *> Args;
1590
24
1591
24
        CallSite::arg_iterator i = CS.arg_begin();
1592
68
        for (unsigned n = FT->getNumParams(); n != 0; 
++i, --n44
) {
1593
44
          Type *T = (*i)->getType();
1594
44
          FunctionType *ParamFT;
1595
44
          if (isa<PointerType>(T) &&
1596
44
              (ParamFT = dyn_cast<FunctionType>(
1597
14
                   cast<PointerType>(T)->getElementType()))) {
1598
14
            std::string TName = "dfst";
1599
14
            TName += utostr(FT->getNumParams() - n);
1600
14
            TName += "$";
1601
14
            TName += F->getName();
1602
14
            Constant *T = DFSF.DFS.getOrBuildTrampolineFunction(ParamFT, TName);
1603
14
            Args.push_back(T);
1604
14
            Args.push_back(
1605
14
                IRB.CreateBitCast(*i, Type::getInt8PtrTy(*DFSF.DFS.Ctx)));
1606
30
          } else {
1607
30
            Args.push_back(*i);
1608
30
          }
1609
44
        }
1610
24
1611
24
        i = CS.arg_begin();
1612
24
        const unsigned ShadowArgStart = Args.size();
1613
68
        for (unsigned n = FT->getNumParams(); n != 0; 
++i, --n44
)
1614
44
          Args.push_back(DFSF.getShadow(*i));
1615
24
1616
24
        if (FT->isVarArg()) {
1617
4
          auto *LabelVATy = ArrayType::get(DFSF.DFS.ShadowTy,
1618
4
                                           CS.arg_size() - FT->getNumParams());
1619
4
          auto *LabelVAAlloca = new AllocaInst(
1620
4
              LabelVATy, getDataLayout().getAllocaAddrSpace(),
1621
4
              "labelva", &DFSF.F->getEntryBlock().front());
1622
4
1623
14
          for (unsigned n = 0; i != CS.arg_end(); 
++i, ++n10
) {
1624
10
            auto LabelVAPtr = IRB.CreateStructGEP(LabelVATy, LabelVAAlloca, n);
1625
10
            IRB.CreateStore(DFSF.getShadow(*i), LabelVAPtr);
1626
10
          }
1627
4
1628
4
          Args.push_back(IRB.CreateStructGEP(LabelVATy, LabelVAAlloca, 0));
1629
4
        }
1630
24
1631
24
        if (!FT->getReturnType()->isVoidTy()) {
1632
19
          if (!DFSF.LabelReturnAlloca) {
1633
16
            DFSF.LabelReturnAlloca =
1634
16
              new AllocaInst(DFSF.DFS.ShadowTy,
1635
16
                             getDataLayout().getAllocaAddrSpace(),
1636
16
                             "labelreturn", &DFSF.F->getEntryBlock().front());
1637
16
          }
1638
19
          Args.push_back(DFSF.LabelReturnAlloca);
1639
19
        }
1640
24
1641
34
        for (i = CS.arg_begin() + FT->getNumParams(); i != CS.arg_end(); 
++i10
)
1642
10
          Args.push_back(*i);
1643
24
1644
24
        CallInst *CustomCI = IRB.CreateCall(CustomF, Args);
1645
24
        CustomCI->setCallingConv(CI->getCallingConv());
1646
24
        CustomCI->setAttributes(TransformFunctionAttributes(CustomFn,
1647
24
            CI->getContext(), CI->getAttributes()));
1648
24
1649
24
        // Update the parameter attributes of the custom call instruction to
1650
24
        // zero extend the shadow parameters. This is required for targets
1651
24
        // which consider ShadowTy an illegal type.
1652
68
        for (unsigned n = 0; n < FT->getNumParams(); 
n++44
) {
1653
44
          const unsigned ArgNo = ShadowArgStart + n;
1654
44
          if (CustomCI->getArgOperand(ArgNo)->getType() == DFSF.DFS.ShadowTy)
1655
44
            CustomCI->addParamAttr(ArgNo, Attribute::ZExt);
1656
44
        }
1657
24
1658
24
        if (!FT->getReturnType()->isVoidTy()) {
1659
19
          LoadInst *LabelLoad =
1660
19
              IRB.CreateLoad(DFSF.DFS.ShadowTy, DFSF.LabelReturnAlloca);
1661
19
          DFSF.setShadow(CustomCI, LabelLoad);
1662
19
        }
1663
24
1664
24
        CI->replaceAllUsesWith(CustomCI);
1665
24
        CI->eraseFromParent();
1666
24
        return;
1667
24
      }
1668
0
      break;
1669
37
    }
1670
37
  }
1671
14
1672
14
  FunctionType *FT = cast<FunctionType>(
1673
14
      CS.getCalledValue()->getType()->getPointerElementType());
1674
14
  if (DFSF.DFS.getInstrumentedABI() == DataFlowSanitizer::IA_TLS) {
1675
15
    for (unsigned i = 0, n = FT->getNumParams(); i != n; 
++i8
) {
1676
8
      IRB.CreateStore(DFSF.getShadow(CS.getArgument(i)),
1677
8
                      DFSF.getArgTLS(i, CS.getInstruction()));
1678
8
    }
1679
7
  }
1680
14
1681
14
  Instruction *Next = nullptr;
1682
14
  if (!CS.getType()->isVoidTy()) {
1683
11
    if (InvokeInst *II = dyn_cast<InvokeInst>(CS.getInstruction())) {
1684
0
      if (II->getNormalDest()->getSinglePredecessor()) {
1685
0
        Next = &II->getNormalDest()->front();
1686
0
      } else {
1687
0
        BasicBlock *NewBB =
1688
0
            SplitEdge(II->getParent(), II->getNormalDest(), &DFSF.DT);
1689
0
        Next = &NewBB->front();
1690
0
      }
1691
11
    } else {
1692
11
      assert(CS->getIterator() != CS->getParent()->end());
1693
11
      Next = CS->getNextNode();
1694
11
    }
1695
11
1696
11
    if (DFSF.DFS.getInstrumentedABI() == DataFlowSanitizer::IA_TLS) {
1697
5
      IRBuilder<> NextIRB(Next);
1698
5
      LoadInst *LI = NextIRB.CreateLoad(DFSF.DFS.ShadowTy, DFSF.getRetvalTLS());
1699
5
      DFSF.SkipInsts.insert(LI);
1700
5
      DFSF.setShadow(CS.getInstruction(), LI);
1701
5
      DFSF.NonZeroChecks.push_back(LI);
1702
5
    }
1703
11
  }
1704
14
1705
14
  // Do all instrumentation for IA_Args down here to defer tampering with the
1706
14
  // CFG in a way that SplitEdge may be able to detect.
1707
14
  if (DFSF.DFS.getInstrumentedABI() == DataFlowSanitizer::IA_Args) {
1708
7
    FunctionType *NewFT = DFSF.DFS.getArgsFunctionType(FT);
1709
7
    Value *Func =
1710
7
        IRB.CreateBitCast(CS.getCalledValue(), PointerType::getUnqual(NewFT));
1711
7
    std::vector<Value *> Args;
1712
7
1713
7
    CallSite::arg_iterator i = CS.arg_begin(), e = CS.arg_end();
1714
15
    for (unsigned n = FT->getNumParams(); n != 0; 
++i, --n8
)
1715
8
      Args.push_back(*i);
1716
7
1717
7
    i = CS.arg_begin();
1718
15
    for (unsigned n = FT->getNumParams(); n != 0; 
++i, --n8
)
1719
8
      Args.push_back(DFSF.getShadow(*i));
1720
7
1721
7
    if (FT->isVarArg()) {
1722
0
      unsigned VarArgSize = CS.arg_size() - FT->getNumParams();
1723
0
      ArrayType *VarArgArrayTy = ArrayType::get(DFSF.DFS.ShadowTy, VarArgSize);
1724
0
      AllocaInst *VarArgShadow =
1725
0
        new AllocaInst(VarArgArrayTy, getDataLayout().getAllocaAddrSpace(),
1726
0
                       "", &DFSF.F->getEntryBlock().front());
1727
0
      Args.push_back(IRB.CreateConstGEP2_32(VarArgArrayTy, VarArgShadow, 0, 0));
1728
0
      for (unsigned n = 0; i != e; ++i, ++n) {
1729
0
        IRB.CreateStore(
1730
0
            DFSF.getShadow(*i),
1731
0
            IRB.CreateConstGEP2_32(VarArgArrayTy, VarArgShadow, 0, n));
1732
0
        Args.push_back(*i);
1733
0
      }
1734
0
    }
1735
7
1736
7
    CallSite NewCS;
1737
7
    if (InvokeInst *II = dyn_cast<InvokeInst>(CS.getInstruction())) {
1738
0
      NewCS = IRB.CreateInvoke(NewFT, Func, II->getNormalDest(),
1739
0
                               II->getUnwindDest(), Args);
1740
7
    } else {
1741
7
      NewCS = IRB.CreateCall(NewFT, Func, Args);
1742
7
    }
1743
7
    NewCS.setCallingConv(CS.getCallingConv());
1744
7
    NewCS.setAttributes(CS.getAttributes().removeAttributes(
1745
7
        *DFSF.DFS.Ctx, AttributeList::ReturnIndex,
1746
7
        AttributeFuncs::typeIncompatible(NewCS.getInstruction()->getType())));
1747
7
1748
7
    if (Next) {
1749
6
      ExtractValueInst *ExVal =
1750
6
          ExtractValueInst::Create(NewCS.getInstruction(), 0, "", Next);
1751
6
      DFSF.SkipInsts.insert(ExVal);
1752
6
      ExtractValueInst *ExShadow =
1753
6
          ExtractValueInst::Create(NewCS.getInstruction(), 1, "", Next);
1754
6
      DFSF.SkipInsts.insert(ExShadow);
1755
6
      DFSF.setShadow(ExVal, ExShadow);
1756
6
      DFSF.NonZeroChecks.push_back(ExShadow);
1757
6
1758
6
      CS.getInstruction()->replaceAllUsesWith(ExVal);
1759
6
    }
1760
7
1761
7
    CS.getInstruction()->eraseFromParent();
1762
7
  }
1763
14
}
1764
1765
0
void DFSanVisitor::visitPHINode(PHINode &PN) {
1766
0
  PHINode *ShadowPN =
1767
0
      PHINode::Create(DFSF.DFS.ShadowTy, PN.getNumIncomingValues(), "", &PN);
1768
0
1769
0
  // Give the shadow phi node valid predecessors to fool SplitEdge into working.
1770
0
  Value *UndefShadow = UndefValue::get(DFSF.DFS.ShadowTy);
1771
0
  for (PHINode::block_iterator i = PN.block_begin(), e = PN.block_end(); i != e;
1772
0
       ++i) {
1773
0
    ShadowPN->addIncoming(UndefShadow, *i);
1774
0
  }
1775
0
1776
0
  DFSF.PHIFixups.push_back(std::make_pair(&PN, ShadowPN));
1777
0
  DFSF.setShadow(&PN, ShadowPN);
1778
0
}