Coverage Report

Created: 2019-07-24 05:18

/Users/buildslave/jenkins/workspace/clang-stage2-coverage-R/llvm/lib/Target/X86/X86SpeculativeLoadHardening.cpp
Line
Count
Source (jump to first uncovered line)
1
//====- X86SpeculativeLoadHardening.cpp - A Spectre v1 mitigation ---------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
/// \file
9
///
10
/// Provide a pass which mitigates speculative execution attacks which operate
11
/// by speculating incorrectly past some predicate (a type check, bounds check,
12
/// or other condition) to reach a load with invalid inputs and leak the data
13
/// accessed by that load using a side channel out of the speculative domain.
14
///
15
/// For details on the attacks, see the first variant in both the Project Zero
16
/// writeup and the Spectre paper:
17
/// https://googleprojectzero.blogspot.com/2018/01/reading-privileged-memory-with-side.html
18
/// https://spectreattack.com/spectre.pdf
19
///
20
//===----------------------------------------------------------------------===//
21
22
#include "X86.h"
23
#include "X86InstrBuilder.h"
24
#include "X86InstrInfo.h"
25
#include "X86Subtarget.h"
26
#include "llvm/ADT/ArrayRef.h"
27
#include "llvm/ADT/DenseMap.h"
28
#include "llvm/ADT/Optional.h"
29
#include "llvm/ADT/STLExtras.h"
30
#include "llvm/ADT/ScopeExit.h"
31
#include "llvm/ADT/SmallPtrSet.h"
32
#include "llvm/ADT/SmallSet.h"
33
#include "llvm/ADT/SmallVector.h"
34
#include "llvm/ADT/SparseBitVector.h"
35
#include "llvm/ADT/Statistic.h"
36
#include "llvm/CodeGen/MachineBasicBlock.h"
37
#include "llvm/CodeGen/MachineConstantPool.h"
38
#include "llvm/CodeGen/MachineFunction.h"
39
#include "llvm/CodeGen/MachineFunctionPass.h"
40
#include "llvm/CodeGen/MachineInstr.h"
41
#include "llvm/CodeGen/MachineInstrBuilder.h"
42
#include "llvm/CodeGen/MachineModuleInfo.h"
43
#include "llvm/CodeGen/MachineOperand.h"
44
#include "llvm/CodeGen/MachineRegisterInfo.h"
45
#include "llvm/CodeGen/MachineSSAUpdater.h"
46
#include "llvm/CodeGen/TargetInstrInfo.h"
47
#include "llvm/CodeGen/TargetRegisterInfo.h"
48
#include "llvm/CodeGen/TargetSchedule.h"
49
#include "llvm/CodeGen/TargetSubtargetInfo.h"
50
#include "llvm/IR/DebugLoc.h"
51
#include "llvm/MC/MCSchedule.h"
52
#include "llvm/Pass.h"
53
#include "llvm/Support/CommandLine.h"
54
#include "llvm/Support/Debug.h"
55
#include "llvm/Support/raw_ostream.h"
56
#include <algorithm>
57
#include <cassert>
58
#include <iterator>
59
#include <utility>
60
61
using namespace llvm;
62
63
#define PASS_KEY "x86-slh"
64
#define DEBUG_TYPE PASS_KEY
65
66
STATISTIC(NumCondBranchesTraced, "Number of conditional branches traced");
67
STATISTIC(NumBranchesUntraced, "Number of branches unable to trace");
68
STATISTIC(NumAddrRegsHardened,
69
          "Number of address mode used registers hardaned");
70
STATISTIC(NumPostLoadRegsHardened,
71
          "Number of post-load register values hardened");
72
STATISTIC(NumCallsOrJumpsHardened,
73
          "Number of calls or jumps requiring extra hardening");
74
STATISTIC(NumInstsInserted, "Number of instructions inserted");
75
STATISTIC(NumLFENCEsInserted, "Number of lfence instructions inserted");
76
77
static cl::opt<bool> EnableSpeculativeLoadHardening(
78
    "x86-speculative-load-hardening",
79
    cl::desc("Force enable speculative load hardening"), cl::init(false),
80
    cl::Hidden);
81
82
static cl::opt<bool> HardenEdgesWithLFENCE(
83
    PASS_KEY "-lfence",
84
    cl::desc(
85
        "Use LFENCE along each conditional edge to harden against speculative "
86
        "loads rather than conditional movs and poisoned pointers."),
87
    cl::init(false), cl::Hidden);
88
89
static cl::opt<bool> EnablePostLoadHardening(
90
    PASS_KEY "-post-load",
91
    cl::desc("Harden the value loaded *after* it is loaded by "
92
             "flushing the loaded bits to 1. This is hard to do "
93
             "in general but can be done easily for GPRs."),
94
    cl::init(true), cl::Hidden);
95
96
static cl::opt<bool> FenceCallAndRet(
97
    PASS_KEY "-fence-call-and-ret",
98
    cl::desc("Use a full speculation fence to harden both call and ret edges "
99
             "rather than a lighter weight mitigation."),
100
    cl::init(false), cl::Hidden);
101
102
static cl::opt<bool> HardenInterprocedurally(
103
    PASS_KEY "-ip",
104
    cl::desc("Harden interprocedurally by passing our state in and out of "
105
             "functions in the high bits of the stack pointer."),
106
    cl::init(true), cl::Hidden);
107
108
static cl::opt<bool>
109
    HardenLoads(PASS_KEY "-loads",
110
                cl::desc("Sanitize loads from memory. When disable, no "
111
                         "significant security is provided."),
112
                cl::init(true), cl::Hidden);
113
114
static cl::opt<bool> HardenIndirectCallsAndJumps(
115
    PASS_KEY "-indirect",
116
    cl::desc("Harden indirect calls and jumps against using speculatively "
117
             "stored attacker controlled addresses. This is designed to "
118
             "mitigate Spectre v1.2 style attacks."),
119
    cl::init(true), cl::Hidden);
120
121
namespace {
122
123
class X86SpeculativeLoadHardeningPass : public MachineFunctionPass {
124
public:
125
12.2k
  X86SpeculativeLoadHardeningPass() : MachineFunctionPass(ID) { }
126
127
150k
  StringRef getPassName() const override {
128
150k
    return "X86 speculative load hardening";
129
150k
  }
130
  bool runOnMachineFunction(MachineFunction &MF) override;
131
  void getAnalysisUsage(AnalysisUsage &AU) const override;
132
133
  /// Pass identification, replacement for typeid.
134
  static char ID;
135
136
private:
137
  /// The information about a block's conditional terminators needed to trace
138
  /// our predicate state through the exiting edges.
139
  struct BlockCondInfo {
140
    MachineBasicBlock *MBB;
141
142
    // We mostly have one conditional branch, and in extremely rare cases have
143
    // two. Three and more are so rare as to be unimportant for compile time.
144
    SmallVector<MachineInstr *, 2> CondBrs;
145
146
    MachineInstr *UncondBr;
147
  };
148
149
  /// Manages the predicate state traced through the program.
150
  struct PredState {
151
    unsigned InitialReg;
152
    unsigned PoisonReg;
153
154
    const TargetRegisterClass *RC;
155
    MachineSSAUpdater SSA;
156
157
    PredState(MachineFunction &MF, const TargetRegisterClass *RC)
158
93
        : RC(RC), SSA(MF) {}
159
  };
160
161
  const X86Subtarget *Subtarget;
162
  MachineRegisterInfo *MRI;
163
  const X86InstrInfo *TII;
164
  const TargetRegisterInfo *TRI;
165
166
  Optional<PredState> PS;
167
168
  void hardenEdgesWithLFENCE(MachineFunction &MF);
169
170
  SmallVector<BlockCondInfo, 16> collectBlockCondInfo(MachineFunction &MF);
171
172
  SmallVector<MachineInstr *, 16>
173
  tracePredStateThroughCFG(MachineFunction &MF, ArrayRef<BlockCondInfo> Infos);
174
175
  void unfoldCallAndJumpLoads(MachineFunction &MF);
176
177
  SmallVector<MachineInstr *, 16>
178
  tracePredStateThroughIndirectBranches(MachineFunction &MF);
179
180
  void tracePredStateThroughBlocksAndHarden(MachineFunction &MF);
181
182
  unsigned saveEFLAGS(MachineBasicBlock &MBB,
183
                      MachineBasicBlock::iterator InsertPt, DebugLoc Loc);
184
  void restoreEFLAGS(MachineBasicBlock &MBB,
185
                     MachineBasicBlock::iterator InsertPt, DebugLoc Loc,
186
                     unsigned OFReg);
187
188
  void mergePredStateIntoSP(MachineBasicBlock &MBB,
189
                            MachineBasicBlock::iterator InsertPt, DebugLoc Loc,
190
                            unsigned PredStateReg);
191
  unsigned extractPredStateFromSP(MachineBasicBlock &MBB,
192
                                  MachineBasicBlock::iterator InsertPt,
193
                                  DebugLoc Loc);
194
195
  void
196
  hardenLoadAddr(MachineInstr &MI, MachineOperand &BaseMO,
197
                 MachineOperand &IndexMO,
198
                 SmallDenseMap<unsigned, unsigned, 32> &AddrRegToHardenedReg);
199
  MachineInstr *
200
  sinkPostLoadHardenedInst(MachineInstr &MI,
201
                           SmallPtrSetImpl<MachineInstr *> &HardenedInstrs);
202
  bool canHardenRegister(unsigned Reg);
203
  unsigned hardenValueInRegister(unsigned Reg, MachineBasicBlock &MBB,
204
                                 MachineBasicBlock::iterator InsertPt,
205
                                 DebugLoc Loc);
206
  unsigned hardenPostLoad(MachineInstr &MI);
207
  void hardenReturnInstr(MachineInstr &MI);
208
  void tracePredStateThroughCall(MachineInstr &MI);
209
  void hardenIndirectCallOrJumpInstr(
210
      MachineInstr &MI,
211
      SmallDenseMap<unsigned, unsigned, 32> &AddrRegToHardenedReg);
212
};
213
214
} // end anonymous namespace
215
216
char X86SpeculativeLoadHardeningPass::ID = 0;
217
218
void X86SpeculativeLoadHardeningPass::getAnalysisUsage(
219
12.1k
    AnalysisUsage &AU) const {
220
12.1k
  MachineFunctionPass::getAnalysisUsage(AU);
221
12.1k
}
222
223
static MachineBasicBlock &splitEdge(MachineBasicBlock &MBB,
224
                                    MachineBasicBlock &Succ, int SuccCount,
225
                                    MachineInstr *Br, MachineInstr *&UncondBr,
226
20
                                    const X86InstrInfo &TII) {
227
20
  assert(!Succ.isEHPad() && "Shouldn't get edges to EH pads!");
228
20
229
20
  MachineFunction &MF = *MBB.getParent();
230
20
231
20
  MachineBasicBlock &NewMBB = *MF.CreateMachineBasicBlock();
232
20
233
20
  // We have to insert the new block immediately after the current one as we
234
20
  // don't know what layout-successor relationships the successor has and we
235
20
  // may not be able to (and generally don't want to) try to fix those up.
236
20
  MF.insert(std::next(MachineFunction::iterator(&MBB)), &NewMBB);
237
20
238
20
  // Update the branch instruction if necessary.
239
20
  if (Br) {
240
20
    assert(Br->getOperand(0).getMBB() == &Succ &&
241
20
           "Didn't start with the right target!");
242
20
    Br->getOperand(0).setMBB(&NewMBB);
243
20
244
20
    // If this successor was reached through a branch rather than fallthrough,
245
20
    // we might have *broken* fallthrough and so need to inject a new
246
20
    // unconditional branch.
247
20
    if (!UncondBr) {
248
0
      MachineBasicBlock &OldLayoutSucc =
249
0
          *std::next(MachineFunction::iterator(&NewMBB));
250
0
      assert(MBB.isSuccessor(&OldLayoutSucc) &&
251
0
             "Without an unconditional branch, the old layout successor should "
252
0
             "be an actual successor!");
253
0
      auto BrBuilder =
254
0
          BuildMI(&MBB, DebugLoc(), TII.get(X86::JMP_1)).addMBB(&OldLayoutSucc);
255
0
      // Update the unconditional branch now that we've added one.
256
0
      UncondBr = &*BrBuilder;
257
0
    }
258
20
259
20
    // Insert unconditional "jump Succ" instruction in the new block if
260
20
    // necessary.
261
20
    if (!NewMBB.isLayoutSuccessor(&Succ)) {
262
19
      SmallVector<MachineOperand, 4> Cond;
263
19
      TII.insertBranch(NewMBB, &Succ, nullptr, Cond, Br->getDebugLoc());
264
19
    }
265
20
  } else {
266
0
    assert(!UncondBr &&
267
0
           "Cannot have a branchless successor and an unconditional branch!");
268
0
    assert(NewMBB.isLayoutSuccessor(&Succ) &&
269
0
           "A non-branch successor must have been a layout successor before "
270
0
           "and now is a layout successor of the new block.");
271
0
  }
272
20
273
20
  // If this is the only edge to the successor, we can just replace it in the
274
20
  // CFG. Otherwise we need to add a new entry in the CFG for the new
275
20
  // successor.
276
20
  if (SuccCount == 1) {
277
20
    MBB.replaceSuccessor(&Succ, &NewMBB);
278
20
  } else {
279
0
    MBB.splitSuccessor(&Succ, &NewMBB);
280
0
  }
281
20
282
20
  // Hook up the edge from the new basic block to the old successor in the CFG.
283
20
  NewMBB.addSuccessor(&Succ);
284
20
285
20
  // Fix PHI nodes in Succ so they refer to NewMBB instead of MBB.
286
27
  for (MachineInstr &MI : Succ) {
287
27
    if (!MI.isPHI())
288
20
      break;
289
10
    
for (int OpIdx = 1, NumOps = MI.getNumOperands(); 7
OpIdx < NumOps;
290
10
         
OpIdx += 23
) {
291
10
      MachineOperand &OpV = MI.getOperand(OpIdx);
292
10
      MachineOperand &OpMBB = MI.getOperand(OpIdx + 1);
293
10
      assert(OpMBB.isMBB() && "Block operand to a PHI is not a block!");
294
10
      if (OpMBB.getMBB() != &MBB)
295
3
        continue;
296
7
297
7
      // If this is the last edge to the succesor, just replace MBB in the PHI
298
7
      if (SuccCount == 1) {
299
7
        OpMBB.setMBB(&NewMBB);
300
7
        break;
301
7
      }
302
0
303
0
      // Otherwise, append a new pair of operands for the new incoming edge.
304
0
      MI.addOperand(MF, OpV);
305
0
      MI.addOperand(MF, MachineOperand::CreateMBB(&NewMBB));
306
0
      break;
307
0
    }
308
7
  }
309
20
310
20
  // Inherit live-ins from the successor
311
20
  for (auto &LI : Succ.liveins())
312
0
    NewMBB.addLiveIn(LI);
313
20
314
20
  LLVM_DEBUG(dbgs() << "  Split edge from '" << MBB.getName() << "' to '"
315
20
                    << Succ.getName() << "'.\n");
316
20
  return NewMBB;
317
20
}
318
319
/// Removing duplicate PHI operands to leave the PHI in a canonical and
320
/// predictable form.
321
///
322
/// FIXME: It's really frustrating that we have to do this, but SSA-form in MIR
323
/// isn't what you might expect. We may have multiple entries in PHI nodes for
324
/// a single predecessor. This makes CFG-updating extremely complex, so here we
325
/// simplify all PHI nodes to a model even simpler than the IR's model: exactly
326
/// one entry per predecessor, regardless of how many edges there are.
327
81
static void canonicalizePHIOperands(MachineFunction &MF) {
328
81
  SmallPtrSet<MachineBasicBlock *, 4> Preds;
329
81
  SmallVector<int, 4> DupIndices;
330
81
  for (auto &MBB : MF)
331
179
    
for (auto &MI : MBB)163
{
332
179
      if (!MI.isPHI())
333
163
        break;
334
16
335
16
      // First we scan the operands of the PHI looking for duplicate entries
336
16
      // a particular predecessor. We retain the operand index of each duplicate
337
16
      // entry found.
338
48
      
for (int OpIdx = 1, NumOps = MI.getNumOperands(); 16
OpIdx < NumOps;
339
32
           OpIdx += 2)
340
32
        if (!Preds.insert(MI.getOperand(OpIdx + 1).getMBB()).second)
341
0
          DupIndices.push_back(OpIdx);
342
16
343
16
      // Now walk the duplicate indices, removing both the block and value. Note
344
16
      // that these are stored as a vector making this element-wise removal
345
16
      // :w
346
16
      // potentially quadratic.
347
16
      //
348
16
      // FIXME: It is really frustrating that we have to use a quadratic
349
16
      // removal algorithm here. There should be a better way, but the use-def
350
16
      // updates required make that impossible using the public API.
351
16
      //
352
16
      // Note that we have to process these backwards so that we don't
353
16
      // invalidate other indices with each removal.
354
16
      while (!DupIndices.empty()) {
355
0
        int OpIdx = DupIndices.pop_back_val();
356
0
        // Remove both the block and value operand, again in reverse order to
357
0
        // preserve indices.
358
0
        MI.RemoveOperand(OpIdx + 1);
359
0
        MI.RemoveOperand(OpIdx);
360
0
      }
361
16
362
16
      Preds.clear();
363
16
    }
364
81
}
365
366
/// Helper to scan a function for loads vulnerable to misspeculation that we
367
/// want to harden.
368
///
369
/// We use this to avoid making changes to functions where there is nothing we
370
/// need to do to harden against misspeculation.
371
84
static bool hasVulnerableLoad(MachineFunction &MF) {
372
117
  for (MachineBasicBlock &MBB : MF) {
373
472
    for (MachineInstr &MI : MBB) {
374
472
      // Loads within this basic block after an LFENCE are not at risk of
375
472
      // speculatively executing with invalid predicates from prior control
376
472
      // flow. So break out of this block but continue scanning the function.
377
472
      if (MI.getOpcode() == X86::LFENCE)
378
0
        break;
379
472
380
472
      // Looking for loads only.
381
472
      if (!MI.mayLoad())
382
392
        continue;
383
80
384
80
      // An MFENCE is modeled as a load but isn't vulnerable to misspeculation.
385
80
      if (MI.getOpcode() == X86::MFENCE)
386
0
        continue;
387
80
388
80
      // We found a load.
389
80
      return true;
390
80
    }
391
117
  }
392
84
393
84
  // No loads found.
394
84
  
return false4
;
395
84
}
396
397
bool X86SpeculativeLoadHardeningPass::runOnMachineFunction(
398
137k
    MachineFunction &MF) {
399
137k
  LLVM_DEBUG(dbgs() << "********** " << getPassName() << " : " << MF.getName()
400
137k
                    << " **********\n");
401
137k
402
137k
  // Only run if this pass is forced enabled or we detect the relevant function
403
137k
  // attribute requesting SLH.
404
137k
  if (!EnableSpeculativeLoadHardening &&
405
137k
      
!MF.getFunction().hasFnAttribute(Attribute::SpeculativeLoadHardening)137k
)
406
137k
    return false;
407
95
408
95
  Subtarget = &MF.getSubtarget<X86Subtarget>();
409
95
  MRI = &MF.getRegInfo();
410
95
  TII = Subtarget->getInstrInfo();
411
95
  TRI = Subtarget->getRegisterInfo();
412
95
413
95
  // FIXME: Support for 32-bit.
414
95
  PS.emplace(MF, &X86::GR64_NOSPRegClass);
415
95
416
95
  if (MF.begin() == MF.end())
417
0
    // Nothing to do for a degenerate empty function...
418
0
    return false;
419
95
420
95
  // We support an alternative hardening technique based on a debug flag.
421
95
  if (HardenEdgesWithLFENCE) {
422
9
    hardenEdgesWithLFENCE(MF);
423
9
    return true;
424
9
  }
425
86
426
86
  // Create a dummy debug loc to use for all the generated code here.
427
86
  DebugLoc Loc;
428
86
429
86
  MachineBasicBlock &Entry = *MF.begin();
430
86
  auto EntryInsertPt = Entry.SkipPHIsLabelsAndDebug(Entry.begin());
431
86
432
86
  // Do a quick scan to see if we have any checkable loads.
433
86
  bool HasVulnerableLoad = hasVulnerableLoad(MF);
434
86
435
86
  // See if we have any conditional branching blocks that we will need to trace
436
86
  // predicate state through.
437
86
  SmallVector<BlockCondInfo, 16> Infos = collectBlockCondInfo(MF);
438
86
439
86
  // If we have no interesting conditions or loads, nothing to do here.
440
86
  if (!HasVulnerableLoad && 
Infos.empty()4
)
441
3
    return true;
442
83
443
83
  // The poison value is required to be an all-ones value for many aspects of
444
83
  // this mitigation.
445
83
  const int PoisonVal = -1;
446
83
  PS->PoisonReg = MRI->createVirtualRegister(PS->RC);
447
83
  BuildMI(Entry, EntryInsertPt, Loc, TII->get(X86::MOV64ri32), PS->PoisonReg)
448
83
      .addImm(PoisonVal);
449
83
  ++NumInstsInserted;
450
83
451
83
  // If we have loads being hardened and we've asked for call and ret edges to
452
83
  // get a full fence-based mitigation, inject that fence.
453
83
  if (HasVulnerableLoad && 
FenceCallAndRet80
) {
454
0
    // We need to insert an LFENCE at the start of the function to suspend any
455
0
    // incoming misspeculation from the caller. This helps two-fold: the caller
456
0
    // may not have been protected as this code has been, and this code gets to
457
0
    // not take any specific action to protect across calls.
458
0
    // FIXME: We could skip this for functions which unconditionally return
459
0
    // a constant.
460
0
    BuildMI(Entry, EntryInsertPt, Loc, TII->get(X86::LFENCE));
461
0
    ++NumInstsInserted;
462
0
    ++NumLFENCEsInserted;
463
0
  }
464
83
465
83
  // If we guarded the entry with an LFENCE and have no conditionals to protect
466
83
  // in blocks, then we're done.
467
83
  if (FenceCallAndRet && 
Infos.empty()0
)
468
0
    // We may have changed the function's code at this point to insert fences.
469
0
    return true;
470
83
471
83
  // For every basic block in the function which can b
472
83
  if (HardenInterprocedurally && 
!FenceCallAndRet81
) {
473
81
    // Set up the predicate state by extracting it from the incoming stack
474
81
    // pointer so we pick up any misspeculation in our caller.
475
81
    PS->InitialReg = extractPredStateFromSP(Entry, EntryInsertPt, Loc);
476
81
  } else {
477
2
    // Otherwise, just build the predicate state itself by zeroing a register
478
2
    // as we don't need any initial state.
479
2
    PS->InitialReg = MRI->createVirtualRegister(PS->RC);
480
2
    unsigned PredStateSubReg = MRI->createVirtualRegister(&X86::GR32RegClass);
481
2
    auto ZeroI = BuildMI(Entry, EntryInsertPt, Loc, TII->get(X86::MOV32r0),
482
2
                         PredStateSubReg);
483
2
    ++NumInstsInserted;
484
2
    MachineOperand *ZeroEFLAGSDefOp =
485
2
        ZeroI->findRegisterDefOperand(X86::EFLAGS);
486
2
    assert(ZeroEFLAGSDefOp && ZeroEFLAGSDefOp->isImplicit() &&
487
2
           "Must have an implicit def of EFLAGS!");
488
2
    ZeroEFLAGSDefOp->setIsDead(true);
489
2
    BuildMI(Entry, EntryInsertPt, Loc, TII->get(X86::SUBREG_TO_REG),
490
2
            PS->InitialReg)
491
2
        .addImm(0)
492
2
        .addReg(PredStateSubReg)
493
2
        .addImm(X86::sub_32bit);
494
2
  }
495
83
496
83
  // We're going to need to trace predicate state throughout the function's
497
83
  // CFG. Prepare for this by setting up our initial state of PHIs with unique
498
83
  // predecessor entries and all the initial predicate state.
499
83
  canonicalizePHIOperands(MF);
500
83
501
83
  // Track the updated values in an SSA updater to rewrite into SSA form at the
502
83
  // end.
503
83
  PS->SSA.Initialize(PS->InitialReg);
504
83
  PS->SSA.AddAvailableValue(&Entry, PS->InitialReg);
505
83
506
83
  // Trace through the CFG.
507
83
  auto CMovs = tracePredStateThroughCFG(MF, Infos);
508
83
509
83
  // We may also enter basic blocks in this function via exception handling
510
83
  // control flow. Here, if we are hardening interprocedurally, we need to
511
83
  // re-capture the predicate state from the throwing code. In the Itanium ABI,
512
83
  // the throw will always look like a call to __cxa_throw and will have the
513
83
  // predicate state in the stack pointer, so extract fresh predicate state from
514
83
  // the stack pointer and make it available in SSA.
515
83
  // FIXME: Handle non-itanium ABI EH models.
516
83
  if (HardenInterprocedurally) {
517
183
    for (MachineBasicBlock &MBB : MF) {
518
183
      assert(!MBB.isEHScopeEntry() && "Only Itanium ABI EH supported!");
519
183
      assert(!MBB.isEHFuncletEntry() && "Only Itanium ABI EH supported!");
520
183
      assert(!MBB.isCleanupFuncletEntry() && "Only Itanium ABI EH supported!");
521
183
      if (!MBB.isEHPad())
522
182
        continue;
523
1
      PS->SSA.AddAvailableValue(
524
1
          &MBB,
525
1
          extractPredStateFromSP(MBB, MBB.SkipPHIsAndLabels(MBB.begin()), Loc));
526
1
    }
527
81
  }
528
83
529
83
  if (HardenIndirectCallsAndJumps) {
530
81
    // If we are going to harden calls and jumps we need to unfold their memory
531
81
    // operands.
532
81
    unfoldCallAndJumpLoads(MF);
533
81
534
81
    // Then we trace predicate state through the indirect branches.
535
81
    auto IndirectBrCMovs = tracePredStateThroughIndirectBranches(MF);
536
81
    CMovs.append(IndirectBrCMovs.begin(), IndirectBrCMovs.end());
537
81
  }
538
83
539
83
  // Now that we have the predicate state available at the start of each block
540
83
  // in the CFG, trace it through each block, hardening vulnerable instructions
541
83
  // as we go.
542
83
  tracePredStateThroughBlocksAndHarden(MF);
543
83
544
83
  // Now rewrite all the uses of the pred state using the SSA updater to insert
545
83
  // PHIs connecting the state between blocks along the CFG edges.
546
83
  for (MachineInstr *CMovI : CMovs)
547
430
    
for (MachineOperand &Op : CMovI->operands())86
{
548
430
      if (!Op.isReg() || 
Op.getReg() != PS->InitialReg344
)
549
344
        continue;
550
86
551
86
      PS->SSA.RewriteUse(Op);
552
86
    }
553
83
554
83
  LLVM_DEBUG(dbgs() << "Final speculative load hardened function:\n"; MF.dump();
555
83
             dbgs() << "\n"; MF.verify(this));
556
83
  return true;
557
83
}
558
559
/// Implements the naive hardening approach of putting an LFENCE after every
560
/// potentially mis-predicted control flow construct.
561
///
562
/// We include this as an alternative mostly for the purpose of comparison. The
563
/// performance impact of this is expected to be extremely severe and not
564
/// practical for any real-world users.
565
void X86SpeculativeLoadHardeningPass::hardenEdgesWithLFENCE(
566
9
    MachineFunction &MF) {
567
9
  // First, we scan the function looking for blocks that are reached along edges
568
9
  // that we might want to harden.
569
9
  SmallSetVector<MachineBasicBlock *, 8> Blocks;
570
27
  for (MachineBasicBlock &MBB : MF) {
571
27
    // If there are no or only one successor, nothing to do here.
572
27
    if (MBB.succ_size() <= 1)
573
16
      continue;
574
11
575
11
    // Skip blocks unless their terminators start with a branch. Other
576
11
    // terminators don't seem interesting for guarding against misspeculation.
577
11
    auto TermIt = MBB.getFirstTerminator();
578
11
    if (TermIt == MBB.end() || !TermIt->isBranch())
579
0
      continue;
580
11
581
11
    // Add all the non-EH-pad succossors to the blocks we want to harden. We
582
11
    // skip EH pads because there isn't really a condition of interest on
583
11
    // entering.
584
11
    for (MachineBasicBlock *SuccMBB : MBB.successors())
585
22
      if (!SuccMBB->isEHPad())
586
21
        Blocks.insert(SuccMBB);
587
11
  }
588
9
589
16
  for (MachineBasicBlock *MBB : Blocks) {
590
16
    auto InsertPt = MBB->SkipPHIsAndLabels(MBB->begin());
591
16
    BuildMI(*MBB, InsertPt, DebugLoc(), TII->get(X86::LFENCE));
592
16
    ++NumInstsInserted;
593
16
    ++NumLFENCEsInserted;
594
16
  }
595
9
}
596
597
SmallVector<X86SpeculativeLoadHardeningPass::BlockCondInfo, 16>
598
84
X86SpeculativeLoadHardeningPass::collectBlockCondInfo(MachineFunction &MF) {
599
84
  SmallVector<BlockCondInfo, 16> Infos;
600
84
601
84
  // Walk the function and build up a summary for each block's conditions that
602
84
  // we need to trace through.
603
171
  for (MachineBasicBlock &MBB : MF) {
604
171
    // If there are no or only one successor, nothing to do here.
605
171
    if (MBB.succ_size() <= 1)
606
135
      continue;
607
36
608
36
    // We want to reliably handle any conditional branch terminators in the
609
36
    // MBB, so we manually analyze the branch. We can handle all of the
610
36
    // permutations here, including ones that analyze branch cannot.
611
36
    //
612
36
    // The approach is to walk backwards across the terminators, resetting at
613
36
    // any unconditional non-indirect branch, and track all conditional edges
614
36
    // to basic blocks as well as the fallthrough or unconditional successor
615
36
    // edge. For each conditional edge, we track the target and the opposite
616
36
    // condition code in order to inject a "no-op" cmov into that successor
617
36
    // that will harden the predicate. For the fallthrough/unconditional
618
36
    // edge, we inject a separate cmov for each conditional branch with
619
36
    // matching condition codes. This effectively implements an "and" of the
620
36
    // condition flags, even if there isn't a single condition flag that would
621
36
    // directly implement that. We don't bother trying to optimize either of
622
36
    // these cases because if such an optimization is possible, LLVM should
623
36
    // have optimized the conditional *branches* in that way already to reduce
624
36
    // instruction count. This late, we simply assume the minimal number of
625
36
    // branch instructions is being emitted and use that to guide our cmov
626
36
    // insertion.
627
36
628
36
    BlockCondInfo Info = {&MBB, {}, nullptr};
629
36
630
36
    // Now walk backwards through the terminators and build up successors they
631
36
    // reach and the conditions.
632
95
    for (MachineInstr &MI : llvm::reverse(MBB)) {
633
95
      // Once we've handled all the terminators, we're done.
634
95
      if (!MI.isTerminator())
635
36
        break;
636
59
637
59
      // If we see a non-branch terminator, we can't handle anything so bail.
638
59
      if (!MI.isBranch()) {
639
0
        Info.CondBrs.clear();
640
0
        break;
641
0
      }
642
59
643
59
      // If we see an unconditional branch, reset our state, clear any
644
59
      // fallthrough, and set this is the "else" successor.
645
59
      if (MI.getOpcode() == X86::JMP_1) {
646
24
        Info.CondBrs.clear();
647
24
        Info.UncondBr = &MI;
648
24
        continue;
649
24
      }
650
35
651
35
      // If we get an invalid condition, we have an indirect branch or some
652
35
      // other unanalyzable "fallthrough" case. We model this as a nullptr for
653
35
      // the destination so we can still guard any conditional successors.
654
35
      // Consider code sequences like:
655
35
      // ```
656
35
      //   jCC L1
657
35
      //   jmpq *%rax
658
35
      // ```
659
35
      // We still want to harden the edge to `L1`.
660
35
      if (X86::getCondFromBranch(MI) == X86::COND_INVALID) {
661
8
        Info.CondBrs.clear();
662
8
        Info.UncondBr = &MI;
663
8
        continue;
664
8
      }
665
27
666
27
      // We have a vanilla conditional branch, add it to our list.
667
27
      Info.CondBrs.push_back(&MI);
668
27
    }
669
36
    if (Info.CondBrs.empty()) {
670
9
      ++NumBranchesUntraced;
671
9
      LLVM_DEBUG(dbgs() << "WARNING: unable to secure successors of block:\n";
672
9
                 MBB.dump());
673
9
      continue;
674
9
    }
675
27
676
27
    Infos.push_back(Info);
677
27
  }
678
84
679
84
  return Infos;
680
84
}
681
682
/// Trace the predicate state through the CFG, instrumenting each conditional
683
/// branch such that misspeculation through an edge will poison the predicate
684
/// state.
685
///
686
/// Returns the list of inserted CMov instructions so that they can have their
687
/// uses of the predicate state rewritten into proper SSA form once it is
688
/// complete.
689
SmallVector<MachineInstr *, 16>
690
X86SpeculativeLoadHardeningPass::tracePredStateThroughCFG(
691
81
    MachineFunction &MF, ArrayRef<BlockCondInfo> Infos) {
692
81
  // Collect the inserted cmov instructions so we can rewrite their uses of the
693
81
  // predicate state into SSA form.
694
81
  SmallVector<MachineInstr *, 16> CMovs;
695
81
696
81
  // Now walk all of the basic blocks looking for ones that end in conditional
697
81
  // jumps where we need to update this register along each edge.
698
81
  for (const BlockCondInfo &Info : Infos) {
699
27
    MachineBasicBlock &MBB = *Info.MBB;
700
27
    const SmallVectorImpl<MachineInstr *> &CondBrs = Info.CondBrs;
701
27
    MachineInstr *UncondBr = Info.UncondBr;
702
27
703
27
    LLVM_DEBUG(dbgs() << "Tracing predicate through block: " << MBB.getName()
704
27
                      << "\n");
705
27
    ++NumCondBranchesTraced;
706
27
707
27
    // Compute the non-conditional successor as either the target of any
708
27
    // unconditional branch or the layout successor.
709
27
    MachineBasicBlock *UncondSucc =
710
27
        UncondBr ? (UncondBr->getOpcode() == X86::JMP_1
711
23
                        ? UncondBr->getOperand(0).getMBB()
712
23
                        : 
nullptr0
)
713
27
                 : 
&*std::next(MachineFunction::iterator(&MBB))4
;
714
27
715
27
    // Count how many edges there are to any given successor.
716
27
    SmallDenseMap<MachineBasicBlock *, int> SuccCounts;
717
27
    if (UncondSucc)
718
27
      ++SuccCounts[UncondSucc];
719
27
    for (auto *CondBr : CondBrs)
720
27
      ++SuccCounts[CondBr->getOperand(0).getMBB()];
721
27
722
27
    // A lambda to insert cmov instructions into a block checking all of the
723
27
    // condition codes in a sequence.
724
27
    auto BuildCheckingBlockForSuccAndConds =
725
27
        [&](MachineBasicBlock &MBB, MachineBasicBlock &Succ, int SuccCount,
726
27
            MachineInstr *Br, MachineInstr *&UncondBr,
727
54
            ArrayRef<X86::CondCode> Conds) {
728
54
          // First, we split the edge to insert the checking block into a safe
729
54
          // location.
730
54
          auto &CheckingMBB =
731
54
              (SuccCount == 1 && Succ.pred_size() == 1)
732
54
                  ? 
Succ34
733
54
                  : 
splitEdge(MBB, Succ, SuccCount, Br, UncondBr, *TII)20
;
734
54
735
54
          bool LiveEFLAGS = Succ.isLiveIn(X86::EFLAGS);
736
54
          if (!LiveEFLAGS)
737
54
            CheckingMBB.addLiveIn(X86::EFLAGS);
738
54
739
54
          // Now insert the cmovs to implement the checks.
740
54
          auto InsertPt = CheckingMBB.begin();
741
54
          assert((InsertPt == CheckingMBB.end() || !InsertPt->isPHI()) &&
742
54
                 "Should never have a PHI in the initial checking block as it "
743
54
                 "always has a single predecessor!");
744
54
745
54
          // We will wire each cmov to each other, but need to start with the
746
54
          // incoming pred state.
747
54
          unsigned CurStateReg = PS->InitialReg;
748
54
749
54
          for (X86::CondCode Cond : Conds) {
750
54
            int PredStateSizeInBytes = TRI->getRegSizeInBits(*PS->RC) / 8;
751
54
            auto CMovOp = X86::getCMovOpcode(PredStateSizeInBytes);
752
54
753
54
            unsigned UpdatedStateReg = MRI->createVirtualRegister(PS->RC);
754
54
            // Note that we intentionally use an empty debug location so that
755
54
            // this picks up the preceding location.
756
54
            auto CMovI = BuildMI(CheckingMBB, InsertPt, DebugLoc(),
757
54
                                 TII->get(CMovOp), UpdatedStateReg)
758
54
                             .addReg(CurStateReg)
759
54
                             .addReg(PS->PoisonReg)
760
54
                             .addImm(Cond);
761
54
            // If this is the last cmov and the EFLAGS weren't originally
762
54
            // live-in, mark them as killed.
763
54
            if (!LiveEFLAGS && Cond == Conds.back())
764
54
              CMovI->findRegisterUseOperand(X86::EFLAGS)->setIsKill(true);
765
54
766
54
            ++NumInstsInserted;
767
54
            LLVM_DEBUG(dbgs() << "  Inserting cmov: "; CMovI->dump();
768
54
                       dbgs() << "\n");
769
54
770
54
            // The first one of the cmovs will be using the top level
771
54
            // `PredStateReg` and need to get rewritten into SSA form.
772
54
            if (CurStateReg == PS->InitialReg)
773
54
              CMovs.push_back(&*CMovI);
774
54
775
54
            // The next cmov should start from this one's def.
776
54
            CurStateReg = UpdatedStateReg;
777
54
          }
778
54
779
54
          // And put the last one into the available values for SSA form of our
780
54
          // predicate state.
781
54
          PS->SSA.AddAvailableValue(&CheckingMBB, CurStateReg);
782
54
        };
783
27
784
27
    std::vector<X86::CondCode> UncondCodeSeq;
785
27
    for (auto *CondBr : CondBrs) {
786
27
      MachineBasicBlock &Succ = *CondBr->getOperand(0).getMBB();
787
27
      int &SuccCount = SuccCounts[&Succ];
788
27
789
27
      X86::CondCode Cond = X86::getCondFromBranch(*CondBr);
790
27
      X86::CondCode InvCond = X86::GetOppositeBranchCondition(Cond);
791
27
      UncondCodeSeq.push_back(Cond);
792
27
793
27
      BuildCheckingBlockForSuccAndConds(MBB, Succ, SuccCount, CondBr, UncondBr,
794
27
                                        {InvCond});
795
27
796
27
      // Decrement the successor count now that we've split one of the edges.
797
27
      // We need to keep the count of edges to the successor accurate in order
798
27
      // to know above when to *replace* the successor in the CFG vs. just
799
27
      // adding the new successor.
800
27
      --SuccCount;
801
27
    }
802
27
803
27
    // Since we may have split edges and changed the number of successors,
804
27
    // normalize the probabilities. This avoids doing it each time we split an
805
27
    // edge.
806
27
    MBB.normalizeSuccProbs();
807
27
808
27
    // Finally, we need to insert cmovs into the "fallthrough" edge. Here, we
809
27
    // need to intersect the other condition codes. We can do this by just
810
27
    // doing a cmov for each one.
811
27
    if (!UncondSucc)
812
0
      // If we have no fallthrough to protect (perhaps it is an indirect jump?)
813
0
      // just skip this and continue.
814
0
      continue;
815
27
816
27
    assert(SuccCounts[UncondSucc] == 1 &&
817
27
           "We should never have more than one edge to the unconditional "
818
27
           "successor at this point because every other edge must have been "
819
27
           "split above!");
820
27
821
27
    // Sort and unique the codes to minimize them.
822
27
    llvm::sort(UncondCodeSeq);
823
27
    UncondCodeSeq.erase(std::unique(UncondCodeSeq.begin(), UncondCodeSeq.end()),
824
27
                        UncondCodeSeq.end());
825
27
826
27
    // Build a checking version of the successor.
827
27
    BuildCheckingBlockForSuccAndConds(MBB, *UncondSucc, /*SuccCount*/ 1,
828
27
                                      UncondBr, UncondBr, UncondCodeSeq);
829
27
  }
830
81
831
81
  return CMovs;
832
81
}
833
834
/// Compute the register class for the unfolded load.
835
///
836
/// FIXME: This should probably live in X86InstrInfo, potentially by adding
837
/// a way to unfold into a newly created vreg rather than requiring a register
838
/// input.
839
static const TargetRegisterClass *
840
getRegClassForUnfoldedLoad(MachineFunction &MF, const X86InstrInfo &TII,
841
14
                           unsigned Opcode) {
842
14
  unsigned Index;
843
14
  unsigned UnfoldedOpc = TII.getOpcodeAfterMemoryUnfold(
844
14
      Opcode, /*UnfoldLoad*/ true, /*UnfoldStore*/ false, &Index);
845
14
  const MCInstrDesc &MCID = TII.get(UnfoldedOpc);
846
14
  return TII.getRegClass(MCID, Index, &TII.getRegisterInfo(), MF);
847
14
}
848
849
void X86SpeculativeLoadHardeningPass::unfoldCallAndJumpLoads(
850
81
    MachineFunction &MF) {
851
81
  for (MachineBasicBlock &MBB : MF)
852
1.49k
    
for (auto MII = MBB.instr_begin(), MIE = MBB.instr_end(); 183
MII != MIE;) {
853
1.31k
      // Grab a reference and increment the iterator so we can remove this
854
1.31k
      // instruction if needed without disturbing the iteration.
855
1.31k
      MachineInstr &MI = *MII++;
856
1.31k
857
1.31k
      // Must either be a call or a branch.
858
1.31k
      if (!MI.isCall() && 
!MI.isBranch()1.25k
)
859
1.17k
        continue;
860
138
      // We only care about loading variants of these instructions.
861
138
      if (!MI.mayLoad())
862
124
        continue;
863
14
864
14
      switch (MI.getOpcode()) {
865
14
      default: {
866
0
        LLVM_DEBUG(
867
0
            dbgs() << "ERROR: Found an unexpected loading branch or call "
868
0
                      "instruction:\n";
869
0
            MI.dump(); dbgs() << "\n");
870
0
        report_fatal_error("Unexpected loading branch or call!");
871
14
      }
872
14
873
14
      case X86::FARCALL16m:
874
0
      case X86::FARCALL32m:
875
0
      case X86::FARCALL64:
876
0
      case X86::FARJMP16m:
877
0
      case X86::FARJMP32m:
878
0
      case X86::FARJMP64:
879
0
        // We cannot mitigate far jumps or calls, but we also don't expect them
880
0
        // to be vulnerable to Spectre v1.2 style attacks.
881
0
        continue;
882
0
883
14
      case X86::CALL16m:
884
14
      case X86::CALL16m_NT:
885
14
      case X86::CALL32m:
886
14
      case X86::CALL32m_NT:
887
14
      case X86::CALL64m:
888
14
      case X86::CALL64m_NT:
889
14
      case X86::JMP16m:
890
14
      case X86::JMP16m_NT:
891
14
      case X86::JMP32m:
892
14
      case X86::JMP32m_NT:
893
14
      case X86::JMP64m:
894
14
      case X86::JMP64m_NT:
895
14
      case X86::TAILJMPm64:
896
14
      case X86::TAILJMPm64_REX:
897
14
      case X86::TAILJMPm:
898
14
      case X86::TCRETURNmi64:
899
14
      case X86::TCRETURNmi: {
900
14
        // Use the generic unfold logic now that we know we're dealing with
901
14
        // expected instructions.
902
14
        // FIXME: We don't have test coverage for all of these!
903
14
        auto *UnfoldedRC = getRegClassForUnfoldedLoad(MF, *TII, MI.getOpcode());
904
14
        if (!UnfoldedRC) {
905
0
          LLVM_DEBUG(dbgs()
906
0
                         << "ERROR: Unable to unfold load from instruction:\n";
907
0
                     MI.dump(); dbgs() << "\n");
908
0
          report_fatal_error("Unable to unfold load!");
909
0
        }
910
14
        unsigned Reg = MRI->createVirtualRegister(UnfoldedRC);
911
14
        SmallVector<MachineInstr *, 2> NewMIs;
912
14
        // If we were able to compute an unfolded reg class, any failure here
913
14
        // is just a programming error so just assert.
914
14
        bool Unfolded =
915
14
            TII->unfoldMemoryOperand(MF, MI, Reg, /*UnfoldLoad*/ true,
916
14
                                     /*UnfoldStore*/ false, NewMIs);
917
14
        (void)Unfolded;
918
14
        assert(Unfolded &&
919
14
               "Computed unfolded register class but failed to unfold");
920
14
        // Now stitch the new instructions into place and erase the old one.
921
14
        for (auto *NewMI : NewMIs)
922
28
          MBB.insert(MI.getIterator(), NewMI);
923
14
        MI.eraseFromParent();
924
14
        LLVM_DEBUG({
925
14
          dbgs() << "Unfolded load successfully into:\n";
926
14
          for (auto *NewMI : NewMIs) {
927
14
            NewMI->dump();
928
14
            dbgs() << "\n";
929
14
          }
930
14
        });
931
14
        continue;
932
14
      }
933
0
      }
934
0
      llvm_unreachable("Escaped switch with default!");
935
0
    }
936
81
}
937
938
/// Trace the predicate state through indirect branches, instrumenting them to
939
/// poison the state if a target is reached that does not match the expected
940
/// target.
941
///
942
/// This is designed to mitigate Spectre variant 1 attacks where an indirect
943
/// branch is trained to predict a particular target and then mispredicts that
944
/// target in a way that can leak data. Despite using an indirect branch, this
945
/// is really a variant 1 style attack: it does not steer execution to an
946
/// arbitrary or attacker controlled address, and it does not require any
947
/// special code executing next to the victim. This attack can also be mitigated
948
/// through retpolines, but those require either replacing indirect branches
949
/// with conditional direct branches or lowering them through a device that
950
/// blocks speculation. This mitigation can replace these retpoline-style
951
/// mitigations for jump tables and other indirect branches within a function
952
/// when variant 2 isn't a risk while allowing limited speculation. Indirect
953
/// calls, however, cannot be mitigated through this technique without changing
954
/// the ABI in a fundamental way.
955
SmallVector<MachineInstr *, 16>
956
X86SpeculativeLoadHardeningPass::tracePredStateThroughIndirectBranches(
957
81
    MachineFunction &MF) {
958
81
  // We use the SSAUpdater to insert PHI nodes for the target addresses of
959
81
  // indirect branches. We don't actually need the full power of the SSA updater
960
81
  // in this particular case as we always have immediately available values, but
961
81
  // this avoids us having to re-implement the PHI construction logic.
962
81
  MachineSSAUpdater TargetAddrSSA(MF);
963
81
  TargetAddrSSA.Initialize(MRI->createVirtualRegister(&X86::GR64RegClass));
964
81
965
81
  // Track which blocks were terminated with an indirect branch.
966
81
  SmallPtrSet<MachineBasicBlock *, 4> IndirectTerminatedMBBs;
967
81
968
81
  // We need to know what blocks end up reached via indirect branches. We
969
81
  // expect this to be a subset of those whose address is taken and so track it
970
81
  // directly via the CFG.
971
81
  SmallPtrSet<MachineBasicBlock *, 4> IndirectTargetMBBs;
972
81
973
81
  // Walk all the blocks which end in an indirect branch and make the
974
81
  // target address available.
975
183
  for (MachineBasicBlock &MBB : MF) {
976
183
    // Find the last terminator.
977
183
    auto MII = MBB.instr_rbegin();
978
183
    while (MII != MBB.instr_rend() && MII->isDebugInstr())
979
0
      ++MII;
980
183
    if (MII == MBB.instr_rend())
981
0
      continue;
982
183
    MachineInstr &TI = *MII;
983
183
    if (!TI.isTerminator() || 
!TI.isBranch()164
)
984
127
      // No terminator or non-branch terminator.
985
127
      continue;
986
56
987
56
    unsigned TargetReg;
988
56
989
56
    switch (TI.getOpcode()) {
990
56
    default:
991
48
      // Direct branch or conditional branch (leading to fallthrough).
992
48
      continue;
993
56
994
56
    case X86::FARJMP16m:
995
0
    case X86::FARJMP32m:
996
0
    case X86::FARJMP64:
997
0
      // We cannot mitigate far jumps or calls, but we also don't expect them
998
0
      // to be vulnerable to Spectre v1.2 or v2 (self trained) style attacks.
999
0
      continue;
1000
0
1001
0
    case X86::JMP16m:
1002
0
    case X86::JMP16m_NT:
1003
0
    case X86::JMP32m:
1004
0
    case X86::JMP32m_NT:
1005
0
    case X86::JMP64m:
1006
0
    case X86::JMP64m_NT:
1007
0
      // Mostly as documentation.
1008
0
      report_fatal_error("Memory operand jumps should have been unfolded!");
1009
0
1010
0
    case X86::JMP16r:
1011
0
      report_fatal_error(
1012
0
          "Support for 16-bit indirect branches is not implemented.");
1013
0
    case X86::JMP32r:
1014
0
      report_fatal_error(
1015
0
          "Support for 32-bit indirect branches is not implemented.");
1016
0
1017
8
    case X86::JMP64r:
1018
8
      TargetReg = TI.getOperand(0).getReg();
1019
56
    }
1020
56
1021
56
    // We have definitely found an indirect  branch. Verify that there are no
1022
56
    // preceding conditional branches as we don't yet support that.
1023
56
    
if (8
llvm::any_of(MBB.terminators(), [&](MachineInstr &OtherTI) 8
{
1024
8
          return !OtherTI.isDebugInstr() && &OtherTI != &TI;
1025
8
        })) {
1026
0
      LLVM_DEBUG({
1027
0
        dbgs() << "ERROR: Found other terminators in a block with an indirect "
1028
0
                  "branch! This is not yet supported! Terminator sequence:\n";
1029
0
        for (MachineInstr &MI : MBB.terminators()) {
1030
0
          MI.dump();
1031
0
          dbgs() << '\n';
1032
0
        }
1033
0
      });
1034
0
      report_fatal_error("Unimplemented terminator sequence!");
1035
0
    }
1036
8
1037
8
    // Make the target register an available value for this block.
1038
8
    TargetAddrSSA.AddAvailableValue(&MBB, TargetReg);
1039
8
    IndirectTerminatedMBBs.insert(&MBB);
1040
8
1041
8
    // Add all the successors to our target candidates.
1042
8
    for (MachineBasicBlock *Succ : MBB.successors())
1043
32
      IndirectTargetMBBs.insert(Succ);
1044
8
  }
1045
81
1046
81
  // Keep track of the cmov instructions we insert so we can return them.
1047
81
  SmallVector<MachineInstr *, 16> CMovs;
1048
81
1049
81
  // If we didn't find any indirect branches with targets, nothing to do here.
1050
81
  if (IndirectTargetMBBs.empty())
1051
73
    return CMovs;
1052
8
1053
8
  // We found indirect branches and targets that need to be instrumented to
1054
8
  // harden loads within them. Walk the blocks of the function (to get a stable
1055
8
  // ordering) and instrument each target of an indirect branch.
1056
48
  
for (MachineBasicBlock &MBB : MF)8
{
1057
48
    // Skip the blocks that aren't candidate targets.
1058
48
    if (!IndirectTargetMBBs.count(&MBB))
1059
16
      continue;
1060
32
1061
32
    // We don't expect EH pads to ever be reached via an indirect branch. If
1062
32
    // this is desired for some reason, we could simply skip them here rather
1063
32
    // than asserting.
1064
32
    assert(!MBB.isEHPad() &&
1065
32
           "Unexpected EH pad as target of an indirect branch!");
1066
32
1067
32
    // We should never end up threading EFLAGS into a block to harden
1068
32
    // conditional jumps as there would be an additional successor via the
1069
32
    // indirect branch. As a consequence, all such edges would be split before
1070
32
    // reaching here, and the inserted block will handle the EFLAGS-based
1071
32
    // hardening.
1072
32
    assert(!MBB.isLiveIn(X86::EFLAGS) &&
1073
32
           "Cannot check within a block that already has live-in EFLAGS!");
1074
32
1075
32
    // We can't handle having non-indirect edges into this block unless this is
1076
32
    // the only successor and we can synthesize the necessary target address.
1077
40
    for (MachineBasicBlock *Pred : MBB.predecessors()) {
1078
40
      // If we've already handled this by extracting the target directly,
1079
40
      // nothing to do.
1080
40
      if (IndirectTerminatedMBBs.count(Pred))
1081
32
        continue;
1082
8
1083
8
      // Otherwise, we have to be the only successor. We generally expect this
1084
8
      // to be true as conditional branches should have had a critical edge
1085
8
      // split already. We don't however need to worry about EH pad successors
1086
8
      // as they'll happily ignore the target and their hardening strategy is
1087
8
      // resilient to all ways in which they could be reached speculatively.
1088
8
      if (!llvm::all_of(Pred->successors(), [&](MachineBasicBlock *Succ) {
1089
8
            return Succ->isEHPad() || Succ == &MBB;
1090
8
          })) {
1091
0
        LLVM_DEBUG({
1092
0
          dbgs() << "ERROR: Found conditional entry to target of indirect "
1093
0
                    "branch!\n";
1094
0
          Pred->dump();
1095
0
          MBB.dump();
1096
0
        });
1097
0
        report_fatal_error("Cannot harden a conditional entry to a target of "
1098
0
                           "an indirect branch!");
1099
0
      }
1100
8
1101
8
      // Now we need to compute the address of this block and install it as a
1102
8
      // synthetic target in the predecessor. We do this at the bottom of the
1103
8
      // predecessor.
1104
8
      auto InsertPt = Pred->getFirstTerminator();
1105
8
      unsigned TargetReg = MRI->createVirtualRegister(&X86::GR64RegClass);
1106
8
      if (MF.getTarget().getCodeModel() == CodeModel::Small &&
1107
8
          !Subtarget->isPositionIndependent()) {
1108
4
        // Directly materialize it into an immediate.
1109
4
        auto AddrI = BuildMI(*Pred, InsertPt, DebugLoc(),
1110
4
                             TII->get(X86::MOV64ri32), TargetReg)
1111
4
                         .addMBB(&MBB);
1112
4
        ++NumInstsInserted;
1113
4
        (void)AddrI;
1114
4
        LLVM_DEBUG(dbgs() << "  Inserting mov: "; AddrI->dump();
1115
4
                   dbgs() << "\n");
1116
4
      } else {
1117
4
        auto AddrI = BuildMI(*Pred, InsertPt, DebugLoc(), TII->get(X86::LEA64r),
1118
4
                             TargetReg)
1119
4
                         .addReg(/*Base*/ X86::RIP)
1120
4
                         .addImm(/*Scale*/ 1)
1121
4
                         .addReg(/*Index*/ 0)
1122
4
                         .addMBB(&MBB)
1123
4
                         .addReg(/*Segment*/ 0);
1124
4
        ++NumInstsInserted;
1125
4
        (void)AddrI;
1126
4
        LLVM_DEBUG(dbgs() << "  Inserting lea: "; AddrI->dump();
1127
4
                   dbgs() << "\n");
1128
4
      }
1129
8
      // And make this available.
1130
8
      TargetAddrSSA.AddAvailableValue(Pred, TargetReg);
1131
8
    }
1132
32
1133
32
    // Materialize the needed SSA value of the target. Note that we need the
1134
32
    // middle of the block as this block might at the bottom have an indirect
1135
32
    // branch back to itself. We can do this here because at this point, every
1136
32
    // predecessor of this block has an available value. This is basically just
1137
32
    // automating the construction of a PHI node for this target.
1138
32
    unsigned TargetReg = TargetAddrSSA.GetValueInMiddleOfBlock(&MBB);
1139
32
1140
32
    // Insert a comparison of the incoming target register with this block's
1141
32
    // address. This also requires us to mark the block as having its address
1142
32
    // taken explicitly.
1143
32
    MBB.setHasAddressTaken();
1144
32
    auto InsertPt = MBB.SkipPHIsLabelsAndDebug(MBB.begin());
1145
32
    if (MF.getTarget().getCodeModel() == CodeModel::Small &&
1146
32
        !Subtarget->isPositionIndependent()) {
1147
16
      // Check directly against a relocated immediate when we can.
1148
16
      auto CheckI = BuildMI(MBB, InsertPt, DebugLoc(), TII->get(X86::CMP64ri32))
1149
16
                        .addReg(TargetReg, RegState::Kill)
1150
16
                        .addMBB(&MBB);
1151
16
      ++NumInstsInserted;
1152
16
      (void)CheckI;
1153
16
      LLVM_DEBUG(dbgs() << "  Inserting cmp: "; CheckI->dump(); dbgs() << "\n");
1154
16
    } else {
1155
16
      // Otherwise compute the address into a register first.
1156
16
      unsigned AddrReg = MRI->createVirtualRegister(&X86::GR64RegClass);
1157
16
      auto AddrI =
1158
16
          BuildMI(MBB, InsertPt, DebugLoc(), TII->get(X86::LEA64r), AddrReg)
1159
16
              .addReg(/*Base*/ X86::RIP)
1160
16
              .addImm(/*Scale*/ 1)
1161
16
              .addReg(/*Index*/ 0)
1162
16
              .addMBB(&MBB)
1163
16
              .addReg(/*Segment*/ 0);
1164
16
      ++NumInstsInserted;
1165
16
      (void)AddrI;
1166
16
      LLVM_DEBUG(dbgs() << "  Inserting lea: "; AddrI->dump(); dbgs() << "\n");
1167
16
      auto CheckI = BuildMI(MBB, InsertPt, DebugLoc(), TII->get(X86::CMP64rr))
1168
16
                        .addReg(TargetReg, RegState::Kill)
1169
16
                        .addReg(AddrReg, RegState::Kill);
1170
16
      ++NumInstsInserted;
1171
16
      (void)CheckI;
1172
16
      LLVM_DEBUG(dbgs() << "  Inserting cmp: "; CheckI->dump(); dbgs() << "\n");
1173
16
    }
1174
32
1175
32
    // Now cmov over the predicate if the comparison wasn't equal.
1176
32
    int PredStateSizeInBytes = TRI->getRegSizeInBits(*PS->RC) / 8;
1177
32
    auto CMovOp = X86::getCMovOpcode(PredStateSizeInBytes);
1178
32
    unsigned UpdatedStateReg = MRI->createVirtualRegister(PS->RC);
1179
32
    auto CMovI =
1180
32
        BuildMI(MBB, InsertPt, DebugLoc(), TII->get(CMovOp), UpdatedStateReg)
1181
32
            .addReg(PS->InitialReg)
1182
32
            .addReg(PS->PoisonReg)
1183
32
            .addImm(X86::COND_NE);
1184
32
    CMovI->findRegisterUseOperand(X86::EFLAGS)->setIsKill(true);
1185
32
    ++NumInstsInserted;
1186
32
    LLVM_DEBUG(dbgs() << "  Inserting cmov: "; CMovI->dump(); dbgs() << "\n");
1187
32
    CMovs.push_back(&*CMovI);
1188
32
1189
32
    // And put the new value into the available values for SSA form of our
1190
32
    // predicate state.
1191
32
    PS->SSA.AddAvailableValue(&MBB, UpdatedStateReg);
1192
32
  }
1193
8
1194
8
  // Return all the newly inserted cmov instructions of the predicate state.
1195
8
  return CMovs;
1196
8
}
1197
1198
/// Returns true if the instruction has no behavior (specified or otherwise)
1199
/// that is based on the value of any of its register operands
1200
///
1201
/// A classical example of something that is inherently not data invariant is an
1202
/// indirect jump -- the destination is loaded into icache based on the bits set
1203
/// in the jump destination register.
1204
///
1205
/// FIXME: This should become part of our instruction tables.
1206
69
static bool isDataInvariant(MachineInstr &MI) {
1207
69
  switch (MI.getOpcode()) {
1208
69
  default:
1209
34
    // By default, assume that the instruction is not data invariant.
1210
34
    return false;
1211
69
1212
69
    // Some target-independent operations that trivially lower to data-invariant
1213
69
    // instructions.
1214
69
  case TargetOpcode::COPY:
1215
21
  case TargetOpcode::INSERT_SUBREG:
1216
21
  case TargetOpcode::SUBREG_TO_REG:
1217
21
    return true;
1218
21
1219
21
  // On x86 it is believed that imul is constant time w.r.t. the loaded data.
1220
21
  // However, they set flags and are perhaps the most surprisingly constant
1221
21
  // time operations so we call them out here separately.
1222
21
  case X86::IMUL16rr:
1223
9
  case X86::IMUL16rri8:
1224
9
  case X86::IMUL16rri:
1225
9
  case X86::IMUL32rr:
1226
9
  case X86::IMUL32rri8:
1227
9
  case X86::IMUL32rri:
1228
9
  case X86::IMUL64rr:
1229
9
  case X86::IMUL64rri32:
1230
9
  case X86::IMUL64rri8:
1231
9
1232
9
  // Bit scanning and counting instructions that are somewhat surprisingly
1233
9
  // constant time as they scan across bits and do other fairly complex
1234
9
  // operations like popcnt, but are believed to be constant time on x86.
1235
9
  // However, these set flags.
1236
9
  case X86::BSF16rr:
1237
9
  case X86::BSF32rr:
1238
9
  case X86::BSF64rr:
1239
9
  case X86::BSR16rr:
1240
9
  case X86::BSR32rr:
1241
9
  case X86::BSR64rr:
1242
9
  case X86::LZCNT16rr:
1243
9
  case X86::LZCNT32rr:
1244
9
  case X86::LZCNT64rr:
1245
9
  case X86::POPCNT16rr:
1246
9
  case X86::POPCNT32rr:
1247
9
  case X86::POPCNT64rr:
1248
9
  case X86::TZCNT16rr:
1249
9
  case X86::TZCNT32rr:
1250
9
  case X86::TZCNT64rr:
1251
9
1252
9
  // Bit manipulation instructions are effectively combinations of basic
1253
9
  // arithmetic ops, and should still execute in constant time. These also
1254
9
  // set flags.
1255
9
  case X86::BLCFILL32rr:
1256
9
  case X86::BLCFILL64rr:
1257
9
  case X86::BLCI32rr:
1258
9
  case X86::BLCI64rr:
1259
9
  case X86::BLCIC32rr:
1260
9
  case X86::BLCIC64rr:
1261
9
  case X86::BLCMSK32rr:
1262
9
  case X86::BLCMSK64rr:
1263
9
  case X86::BLCS32rr:
1264
9
  case X86::BLCS64rr:
1265
9
  case X86::BLSFILL32rr:
1266
9
  case X86::BLSFILL64rr:
1267
9
  case X86::BLSI32rr:
1268
9
  case X86::BLSI64rr:
1269
9
  case X86::BLSIC32rr:
1270
9
  case X86::BLSIC64rr:
1271
9
  case X86::BLSMSK32rr:
1272
9
  case X86::BLSMSK64rr:
1273
9
  case X86::BLSR32rr:
1274
9
  case X86::BLSR64rr:
1275
9
  case X86::TZMSK32rr:
1276
9
  case X86::TZMSK64rr:
1277
9
1278
9
  // Bit extracting and clearing instructions should execute in constant time,
1279
9
  // and set flags.
1280
9
  case X86::BEXTR32rr:
1281
9
  case X86::BEXTR64rr:
1282
9
  case X86::BEXTRI32ri:
1283
9
  case X86::BEXTRI64ri:
1284
9
  case X86::BZHI32rr:
1285
9
  case X86::BZHI64rr:
1286
9
1287
9
  // Shift and rotate.
1288
9
  case X86::ROL8r1:  case X86::ROL16r1:  case X86::ROL32r1:  case X86::ROL64r1:
1289
9
  case X86::ROL8rCL: case X86::ROL16rCL: case X86::ROL32rCL: case X86::ROL64rCL:
1290
9
  case X86::ROL8ri:  case X86::ROL16ri:  case X86::ROL32ri:  case X86::ROL64ri:
1291
9
  case X86::ROR8r1:  case X86::ROR16r1:  case X86::ROR32r1:  case X86::ROR64r1:
1292
9
  case X86::ROR8rCL: case X86::ROR16rCL: case X86::ROR32rCL: case X86::ROR64rCL:
1293
9
  case X86::ROR8ri:  case X86::ROR16ri:  case X86::ROR32ri:  case X86::ROR64ri:
1294
9
  case X86::SAR8r1:  case X86::SAR16r1:  case X86::SAR32r1:  case X86::SAR64r1:
1295
9
  case X86::SAR8rCL: case X86::SAR16rCL: case X86::SAR32rCL: case X86::SAR64rCL:
1296
9
  case X86::SAR8ri:  case X86::SAR16ri:  case X86::SAR32ri:  case X86::SAR64ri:
1297
9
  case X86::SHL8r1:  case X86::SHL16r1:  case X86::SHL32r1:  case X86::SHL64r1:
1298
9
  case X86::SHL8rCL: case X86::SHL16rCL: case X86::SHL32rCL: case X86::SHL64rCL:
1299
9
  case X86::SHL8ri:  case X86::SHL16ri:  case X86::SHL32ri:  case X86::SHL64ri:
1300
9
  case X86::SHR8r1:  case X86::SHR16r1:  case X86::SHR32r1:  case X86::SHR64r1:
1301
9
  case X86::SHR8rCL: case X86::SHR16rCL: case X86::SHR32rCL: case X86::SHR64rCL:
1302
9
  case X86::SHR8ri:  case X86::SHR16ri:  case X86::SHR32ri:  case X86::SHR64ri:
1303
9
  case X86::SHLD16rrCL: case X86::SHLD32rrCL: case X86::SHLD64rrCL:
1304
9
  case X86::SHLD16rri8: case X86::SHLD32rri8: case X86::SHLD64rri8:
1305
9
  case X86::SHRD16rrCL: case X86::SHRD32rrCL: case X86::SHRD64rrCL:
1306
9
  case X86::SHRD16rri8: case X86::SHRD32rri8: case X86::SHRD64rri8:
1307
9
1308
9
  // Basic arithmetic is constant time on the input but does set flags.
1309
9
  case X86::ADC8rr:   case X86::ADC8ri:
1310
9
  case X86::ADC16rr:  case X86::ADC16ri:   case X86::ADC16ri8:
1311
9
  case X86::ADC32rr:  case X86::ADC32ri:   case X86::ADC32ri8:
1312
9
  case X86::ADC64rr:  case X86::ADC64ri8:  case X86::ADC64ri32:
1313
9
  case X86::ADD8rr:   case X86::ADD8ri:
1314
9
  case X86::ADD16rr:  case X86::ADD16ri:   case X86::ADD16ri8:
1315
9
  case X86::ADD32rr:  case X86::ADD32ri:   case X86::ADD32ri8:
1316
9
  case X86::ADD64rr:  case X86::ADD64ri8:  case X86::ADD64ri32:
1317
9
  case X86::AND8rr:   case X86::AND8ri:
1318
9
  case X86::AND16rr:  case X86::AND16ri:   case X86::AND16ri8:
1319
9
  case X86::AND32rr:  case X86::AND32ri:   case X86::AND32ri8:
1320
9
  case X86::AND64rr:  case X86::AND64ri8:  case X86::AND64ri32:
1321
9
  case X86::OR8rr:    case X86::OR8ri:
1322
9
  case X86::OR16rr:   case X86::OR16ri:    case X86::OR16ri8:
1323
9
  case X86::OR32rr:   case X86::OR32ri:    case X86::OR32ri8:
1324
9
  case X86::OR64rr:   case X86::OR64ri8:   case X86::OR64ri32:
1325
9
  case X86::SBB8rr:   case X86::SBB8ri:
1326
9
  case X86::SBB16rr:  case X86::SBB16ri:   case X86::SBB16ri8:
1327
9
  case X86::SBB32rr:  case X86::SBB32ri:   case X86::SBB32ri8:
1328
9
  case X86::SBB64rr:  case X86::SBB64ri8:  case X86::SBB64ri32:
1329
9
  case X86::SUB8rr:   case X86::SUB8ri:
1330
9
  case X86::SUB16rr:  case X86::SUB16ri:   case X86::SUB16ri8:
1331
9
  case X86::SUB32rr:  case X86::SUB32ri:   case X86::SUB32ri8:
1332
9
  case X86::SUB64rr:  case X86::SUB64ri8:  case X86::SUB64ri32:
1333
9
  case X86::XOR8rr:   case X86::XOR8ri:
1334
9
  case X86::XOR16rr:  case X86::XOR16ri:   case X86::XOR16ri8:
1335
9
  case X86::XOR32rr:  case X86::XOR32ri:   case X86::XOR32ri8:
1336
9
  case X86::XOR64rr:  case X86::XOR64ri8:  case X86::XOR64ri32:
1337
9
  // Arithmetic with just 32-bit and 64-bit variants and no immediates.
1338
9
  case X86::ADCX32rr: case X86::ADCX64rr:
1339
9
  case X86::ADOX32rr: case X86::ADOX64rr:
1340
9
  case X86::ANDN32rr: case X86::ANDN64rr:
1341
9
  // Unary arithmetic operations.
1342
9
  case X86::DEC8r: case X86::DEC16r: case X86::DEC32r: case X86::DEC64r:
1343
9
  case X86::INC8r: case X86::INC16r: case X86::INC32r: case X86::INC64r:
1344
9
  case X86::NEG8r: case X86::NEG16r: case X86::NEG32r: case X86::NEG64r:
1345
9
    // Check whether the EFLAGS implicit-def is dead. We assume that this will
1346
9
    // always find the implicit-def because this code should only be reached
1347
9
    // for instructions that do in fact implicitly def this.
1348
9
    if (!MI.findRegisterDefOperand(X86::EFLAGS)->isDead()) {
1349
0
      // If we would clobber EFLAGS that are used, just bail for now.
1350
0
      LLVM_DEBUG(dbgs() << "    Unable to harden post-load due to EFLAGS: ";
1351
0
                 MI.dump(); dbgs() << "\n");
1352
0
      return false;
1353
0
    }
1354
9
1355
9
    // Otherwise, fallthrough to handle these the same as instructions that
1356
9
    // don't set EFLAGS.
1357
9
    LLVM_FALLTHROUGH;
1358
9
1359
9
  // Unlike other arithmetic, NOT doesn't set EFLAGS.
1360
14
  case X86::NOT8r: case X86::NOT16r: case X86::NOT32r: case X86::NOT64r:
1361
14
1362
14
  // Various move instructions used to zero or sign extend things. Note that we
1363
14
  // intentionally don't support the _NOREX variants as we can't handle that
1364
14
  // register constraint anyways.
1365
14
  case X86::MOVSX16rr8:
1366
14
  case X86::MOVSX32rr8: case X86::MOVSX32rr16:
1367
14
  case X86::MOVSX64rr8: case X86::MOVSX64rr16: case X86::MOVSX64rr32:
1368
14
  case X86::MOVZX16rr8:
1369
14
  case X86::MOVZX32rr8: case X86::MOVZX32rr16:
1370
14
  case X86::MOVZX64rr8: case X86::MOVZX64rr16:
1371
14
  case X86::MOV32rr:
1372
14
1373
14
  // Arithmetic instructions that are both constant time and don't set flags.
1374
14
  case X86::RORX32ri:
1375
14
  case X86::RORX64ri:
1376
14
  case X86::SARX32rr:
1377
14
  case X86::SARX64rr:
1378
14
  case X86::SHLX32rr:
1379
14
  case X86::SHLX64rr:
1380
14
  case X86::SHRX32rr:
1381
14
  case X86::SHRX64rr:
1382
14
1383
14
  // LEA doesn't actually access memory, and its arithmetic is constant time.
1384
14
  case X86::LEA16r:
1385
14
  case X86::LEA32r:
1386
14
  case X86::LEA64_32r:
1387
14
  case X86::LEA64r:
1388
14
    return true;
1389
69
  }
1390
69
}
1391
1392
/// Returns true if the instruction has no behavior (specified or otherwise)
1393
/// that is based on the value loaded from memory or the value of any
1394
/// non-address register operands.
1395
///
1396
/// For example, if the latency of the instruction is dependent on the
1397
/// particular bits set in any of the registers *or* any of the bits loaded from
1398
/// memory.
1399
///
1400
/// A classical example of something that is inherently not data invariant is an
1401
/// indirect jump -- the destination is loaded into icache based on the bits set
1402
/// in the jump destination register.
1403
///
1404
/// FIXME: This should become part of our instruction tables.
1405
225
static bool isDataInvariantLoad(MachineInstr &MI) {
1406
225
  switch (MI.getOpcode()) {
1407
225
  default:
1408
61
    // By default, assume that the load will immediately leak.
1409
61
    return false;
1410
225
1411
225
  // On x86 it is believed that imul is constant time w.r.t. the loaded data.
1412
225
  // However, they set flags and are perhaps the most surprisingly constant
1413
225
  // time operations so we call them out here separately.
1414
225
  case X86::IMUL16rm:
1415
61
  case X86::IMUL16rmi8:
1416
61
  case X86::IMUL16rmi:
1417
61
  case X86::IMUL32rm:
1418
61
  case X86::IMUL32rmi8:
1419
61
  case X86::IMUL32rmi:
1420
61
  case X86::IMUL64rm:
1421
61
  case X86::IMUL64rmi32:
1422
61
  case X86::IMUL64rmi8:
1423
61
1424
61
  // Bit scanning and counting instructions that are somewhat surprisingly
1425
61
  // constant time as they scan across bits and do other fairly complex
1426
61
  // operations like popcnt, but are believed to be constant time on x86.
1427
61
  // However, these set flags.
1428
61
  case X86::BSF16rm:
1429
61
  case X86::BSF32rm:
1430
61
  case X86::BSF64rm:
1431
61
  case X86::BSR16rm:
1432
61
  case X86::BSR32rm:
1433
61
  case X86::BSR64rm:
1434
61
  case X86::LZCNT16rm:
1435
61
  case X86::LZCNT32rm:
1436
61
  case X86::LZCNT64rm:
1437
61
  case X86::POPCNT16rm:
1438
61
  case X86::POPCNT32rm:
1439
61
  case X86::POPCNT64rm:
1440
61
  case X86::TZCNT16rm:
1441
61
  case X86::TZCNT32rm:
1442
61
  case X86::TZCNT64rm:
1443
61
1444
61
  // Bit manipulation instructions are effectively combinations of basic
1445
61
  // arithmetic ops, and should still execute in constant time. These also
1446
61
  // set flags.
1447
61
  case X86::BLCFILL32rm:
1448
61
  case X86::BLCFILL64rm:
1449
61
  case X86::BLCI32rm:
1450
61
  case X86::BLCI64rm:
1451
61
  case X86::BLCIC32rm:
1452
61
  case X86::BLCIC64rm:
1453
61
  case X86::BLCMSK32rm:
1454
61
  case X86::BLCMSK64rm:
1455
61
  case X86::BLCS32rm:
1456
61
  case X86::BLCS64rm:
1457
61
  case X86::BLSFILL32rm:
1458
61
  case X86::BLSFILL64rm:
1459
61
  case X86::BLSI32rm:
1460
61
  case X86::BLSI64rm:
1461
61
  case X86::BLSIC32rm:
1462
61
  case X86::BLSIC64rm:
1463
61
  case X86::BLSMSK32rm:
1464
61
  case X86::BLSMSK64rm:
1465
61
  case X86::BLSR32rm:
1466
61
  case X86::BLSR64rm:
1467
61
  case X86::TZMSK32rm:
1468
61
  case X86::TZMSK64rm:
1469
61
1470
61
  // Bit extracting and clearing instructions should execute in constant time,
1471
61
  // and set flags.
1472
61
  case X86::BEXTR32rm:
1473
61
  case X86::BEXTR64rm:
1474
61
  case X86::BEXTRI32mi:
1475
61
  case X86::BEXTRI64mi:
1476
61
  case X86::BZHI32rm:
1477
61
  case X86::BZHI64rm:
1478
61
1479
61
  // Basic arithmetic is constant time on the input but does set flags.
1480
61
  case X86::ADC8rm:
1481
61
  case X86::ADC16rm:
1482
61
  case X86::ADC32rm:
1483
61
  case X86::ADC64rm:
1484
61
  case X86::ADCX32rm:
1485
61
  case X86::ADCX64rm:
1486
61
  case X86::ADD8rm:
1487
61
  case X86::ADD16rm:
1488
61
  case X86::ADD32rm:
1489
61
  case X86::ADD64rm:
1490
61
  case X86::ADOX32rm:
1491
61
  case X86::ADOX64rm:
1492
61
  case X86::AND8rm:
1493
61
  case X86::AND16rm:
1494
61
  case X86::AND32rm:
1495
61
  case X86::AND64rm:
1496
61
  case X86::ANDN32rm:
1497
61
  case X86::ANDN64rm:
1498
61
  case X86::OR8rm:
1499
61
  case X86::OR16rm:
1500
61
  case X86::OR32rm:
1501
61
  case X86::OR64rm:
1502
61
  case X86::SBB8rm:
1503
61
  case X86::SBB16rm:
1504
61
  case X86::SBB32rm:
1505
61
  case X86::SBB64rm:
1506
61
  case X86::SUB8rm:
1507
61
  case X86::SUB16rm:
1508
61
  case X86::SUB32rm:
1509
61
  case X86::SUB64rm:
1510
61
  case X86::XOR8rm:
1511
61
  case X86::XOR16rm:
1512
61
  case X86::XOR32rm:
1513
61
  case X86::XOR64rm:
1514
61
    // Check whether the EFLAGS implicit-def is dead. We assume that this will
1515
61
    // always find the implicit-def because this code should only be reached
1516
61
    // for instructions that do in fact implicitly def this.
1517
61
    if (!MI.findRegisterDefOperand(X86::EFLAGS)->isDead()) {
1518
0
      // If we would clobber EFLAGS that are used, just bail for now.
1519
0
      LLVM_DEBUG(dbgs() << "    Unable to harden post-load due to EFLAGS: ";
1520
0
                 MI.dump(); dbgs() << "\n");
1521
0
      return false;
1522
0
    }
1523
61
1524
61
    // Otherwise, fallthrough to handle these the same as instructions that
1525
61
    // don't set EFLAGS.
1526
61
    LLVM_FALLTHROUGH;
1527
61
1528
61
  // Integer multiply w/o affecting flags is still believed to be constant
1529
61
  // time on x86. Called out separately as this is among the most surprising
1530
61
  // instructions to exhibit that behavior.
1531
164
  case X86::MULX32rm:
1532
164
  case X86::MULX64rm:
1533
164
1534
164
  // Arithmetic instructions that are both constant time and don't set flags.
1535
164
  case X86::RORX32mi:
1536
164
  case X86::RORX64mi:
1537
164
  case X86::SARX32rm:
1538
164
  case X86::SARX64rm:
1539
164
  case X86::SHLX32rm:
1540
164
  case X86::SHLX64rm:
1541
164
  case X86::SHRX32rm:
1542
164
  case X86::SHRX64rm:
1543
164
1544
164
  // Conversions are believed to be constant time and don't set flags.
1545
164
  case X86::CVTTSD2SI64rm: case X86::VCVTTSD2SI64rm: case X86::VCVTTSD2SI64Zrm:
1546
164
  case X86::CVTTSD2SIrm:   case X86::VCVTTSD2SIrm:   case X86::VCVTTSD2SIZrm:
1547
164
  case X86::CVTTSS2SI64rm: case X86::VCVTTSS2SI64rm: case X86::VCVTTSS2SI64Zrm:
1548
164
  case X86::CVTTSS2SIrm:   case X86::VCVTTSS2SIrm:   case X86::VCVTTSS2SIZrm:
1549
164
  case X86::CVTSI2SDrm:    case X86::VCVTSI2SDrm:    case X86::VCVTSI2SDZrm:
1550
164
  case X86::CVTSI2SSrm:    case X86::VCVTSI2SSrm:    case X86::VCVTSI2SSZrm:
1551
164
  case X86::CVTSI642SDrm:  case X86::VCVTSI642SDrm:  case X86::VCVTSI642SDZrm:
1552
164
  case X86::CVTSI642SSrm:  case X86::VCVTSI642SSrm:  case X86::VCVTSI642SSZrm:
1553
164
  case X86::CVTSS2SDrm:    case X86::VCVTSS2SDrm:    case X86::VCVTSS2SDZrm:
1554
164
  case X86::CVTSD2SSrm:    case X86::VCVTSD2SSrm:    case X86::VCVTSD2SSZrm:
1555
164
  // AVX512 added unsigned integer conversions.
1556
164
  case X86::VCVTTSD2USI64Zrm:
1557
164
  case X86::VCVTTSD2USIZrm:
1558
164
  case X86::VCVTTSS2USI64Zrm:
1559
164
  case X86::VCVTTSS2USIZrm:
1560
164
  case X86::VCVTUSI2SDZrm:
1561
164
  case X86::VCVTUSI642SDZrm:
1562
164
  case X86::VCVTUSI2SSZrm:
1563
164
  case X86::VCVTUSI642SSZrm:
1564
164
1565
164
  // Loads to register don't set flags.
1566
164
  case X86::MOV8rm:
1567
164
  case X86::MOV8rm_NOREX:
1568
164
  case X86::MOV16rm:
1569
164
  case X86::MOV32rm:
1570
164
  case X86::MOV64rm:
1571
164
  case X86::MOVSX16rm8:
1572
164
  case X86::MOVSX32rm16:
1573
164
  case X86::MOVSX32rm8:
1574
164
  case X86::MOVSX32rm8_NOREX:
1575
164
  case X86::MOVSX64rm16:
1576
164
  case X86::MOVSX64rm32:
1577
164
  case X86::MOVSX64rm8:
1578
164
  case X86::MOVZX16rm8:
1579
164
  case X86::MOVZX32rm16:
1580
164
  case X86::MOVZX32rm8:
1581
164
  case X86::MOVZX32rm8_NOREX:
1582
164
  case X86::MOVZX64rm16:
1583
164
  case X86::MOVZX64rm8:
1584
164
    return true;
1585
225
  }
1586
225
}
1587
1588
static bool isEFLAGSLive(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
1589
116
                         const TargetRegisterInfo &TRI) {
1590
116
  // Check if EFLAGS are alive by seeing if there is a def of them or they
1591
116
  // live-in, and then seeing if that def is in turn used.
1592
404
  for (MachineInstr &MI : llvm::reverse(llvm::make_range(MBB.begin(), I))) {
1593
404
    if (MachineOperand *DefOp = MI.findRegisterDefOperand(X86::EFLAGS)) {
1594
106
      // If the def is dead, then EFLAGS is not live.
1595
106
      if (DefOp->isDead())
1596
106
        return false;
1597
0
1598
0
      // Otherwise we've def'ed it, and it is live.
1599
0
      return true;
1600
0
    }
1601
298
    // While at this instruction, also check if we use and kill EFLAGS
1602
298
    // which means it isn't live.
1603
298
    if (MI.killsRegister(X86::EFLAGS, &TRI))
1604
5
      return false;
1605
298
  }
1606
116
1607
116
  // If we didn't find anything conclusive (neither definitely alive or
1608
116
  // definitely dead) return whether it lives into the block.
1609
116
  
return MBB.isLiveIn(X86::EFLAGS)5
;
1610
116
}
1611
1612
/// Trace the predicate state through each of the blocks in the function,
1613
/// hardening everything necessary along the way.
1614
///
1615
/// We call this routine once the initial predicate state has been established
1616
/// for each basic block in the function in the SSA updater. This routine traces
1617
/// it through the instructions within each basic block, and for non-returning
1618
/// blocks informs the SSA updater about the final state that lives out of the
1619
/// block. Along the way, it hardens any vulnerable instruction using the
1620
/// currently valid predicate state. We have to do these two things together
1621
/// because the SSA updater only works across blocks. Within a block, we track
1622
/// the current predicate state directly and update it as it changes.
1623
///
1624
/// This operates in two passes over each block. First, we analyze the loads in
1625
/// the block to determine which strategy will be used to harden them: hardening
1626
/// the address or hardening the loaded value when loaded into a register
1627
/// amenable to hardening. We have to process these first because the two
1628
/// strategies may interact -- later hardening may change what strategy we wish
1629
/// to use. We also will analyze data dependencies between loads and avoid
1630
/// hardening those loads that are data dependent on a load with a hardened
1631
/// address. We also skip hardening loads already behind an LFENCE as that is
1632
/// sufficient to harden them against misspeculation.
1633
///
1634
/// Second, we actively trace the predicate state through the block, applying
1635
/// the hardening steps we determined necessary in the first pass as we go.
1636
///
1637
/// These two passes are applied to each basic block. We operate one block at a
1638
/// time to simplify reasoning about reachability and sequencing.
1639
void X86SpeculativeLoadHardeningPass::tracePredStateThroughBlocksAndHarden(
1640
81
    MachineFunction &MF) {
1641
81
  SmallPtrSet<MachineInstr *, 16> HardenPostLoad;
1642
81
  SmallPtrSet<MachineInstr *, 16> HardenLoadAddr;
1643
81
1644
81
  SmallSet<unsigned, 16> HardenedAddrRegs;
1645
81
1646
81
  SmallDenseMap<unsigned, unsigned, 32> AddrRegToHardenedReg;
1647
81
1648
81
  // Track the set of load-dependent registers through the basic block. Because
1649
81
  // the values of these registers have an existing data dependency on a loaded
1650
81
  // value which we would have checked, we can omit any checks on them.
1651
81
  SparseBitVector<> LoadDepRegs;
1652
81
1653
183
  for (MachineBasicBlock &MBB : MF) {
1654
183
    // The first pass over the block: collect all the loads which can have their
1655
183
    // loaded value hardened and all the loads that instead need their address
1656
183
    // hardened. During this walk we propagate load dependence for address
1657
183
    // hardened loads and also look for LFENCE to stop hardening wherever
1658
183
    // possible. When deciding whether or not to harden the loaded value or not,
1659
183
    // we check to see if any registers used in the address will have been
1660
183
    // hardened at this point and if so, harden any remaining address registers
1661
183
    // as that often successfully re-uses hardened addresses and minimizes
1662
183
    // instructions.
1663
183
    //
1664
183
    // FIXME: We should consider an aggressive mode where we continue to keep as
1665
183
    // many loads value hardened even when some address register hardening would
1666
183
    // be free (due to reuse).
1667
183
    //
1668
183
    // Note that we only need this pass if we are actually hardening loads.
1669
183
    if (HardenLoads)
1670
1.42k
      
for (MachineInstr &MI : MBB)183
{
1671
1.42k
        // We naively assume that all def'ed registers of an instruction have
1672
1.42k
        // a data dependency on all of their operands.
1673
1.42k
        // FIXME: Do a more careful analysis of x86 to build a conservative
1674
1.42k
        // model here.
1675
3.95k
        if (
llvm::any_of(MI.uses(), [&](MachineOperand &Op) 1.42k
{
1676
3.95k
              return Op.isReg() && 
LoadDepRegs.test(Op.getReg())2.64k
;
1677
3.95k
            }))
1678
118
          for (MachineOperand &Def : MI.defs())
1679
60
            if (Def.isReg())
1680
60
              LoadDepRegs.set(Def.getReg());
1681
1.42k
1682
1.42k
        // Both Intel and AMD are guiding that they will change the semantics of
1683
1.42k
        // LFENCE to be a speculation barrier, so if we see an LFENCE, there is
1684
1.42k
        // no more need to guard things in this block.
1685
1.42k
        if (MI.getOpcode() == X86::LFENCE)
1686
0
          break;
1687
1.42k
1688
1.42k
        // If this instruction cannot load, nothing to do.
1689
1.42k
        if (!MI.mayLoad())
1690
1.28k
          continue;
1691
136
1692
136
        // Some instructions which "load" are trivially safe or unimportant.
1693
136
        if (MI.getOpcode() == X86::MFENCE)
1694
0
          continue;
1695
136
1696
136
        // Extract the memory operand information about this instruction.
1697
136
        // FIXME: This doesn't handle loading pseudo instructions which we often
1698
136
        // could handle with similarly generic logic. We probably need to add an
1699
136
        // MI-layer routine similar to the MC-layer one we use here which maps
1700
136
        // pseudos much like this maps real instructions.
1701
136
        const MCInstrDesc &Desc = MI.getDesc();
1702
136
        int MemRefBeginIdx = X86II::getMemoryOperandNo(Desc.TSFlags);
1703
136
        if (MemRefBeginIdx < 0) {
1704
0
          LLVM_DEBUG(dbgs()
1705
0
                         << "WARNING: unable to harden loading instruction: ";
1706
0
                     MI.dump());
1707
0
          continue;
1708
0
        }
1709
136
1710
136
        MemRefBeginIdx += X86II::getOperandBias(Desc);
1711
136
1712
136
        MachineOperand &BaseMO =
1713
136
            MI.getOperand(MemRefBeginIdx + X86::AddrBaseReg);
1714
136
        MachineOperand &IndexMO =
1715
136
            MI.getOperand(MemRefBeginIdx + X86::AddrIndexReg);
1716
136
1717
136
        // If we have at least one (non-frame-index, non-RIP) register operand,
1718
136
        // and neither operand is load-dependent, we need to check the load.
1719
136
        unsigned BaseReg = 0, IndexReg = 0;
1720
136
        if (!BaseMO.isFI() && BaseMO.getReg() != X86::RIP &&
1721
136
            
BaseMO.getReg() != X86::NoRegister129
)
1722
125
          BaseReg = BaseMO.getReg();
1723
136
        if (IndexMO.getReg() != X86::NoRegister)
1724
54
          IndexReg = IndexMO.getReg();
1725
136
1726
136
        if (!BaseReg && 
!IndexReg11
)
1727
7
          // No register operands!
1728
7
          continue;
1729
129
1730
129
        // If any register operand is dependent, this load is dependent and we
1731
129
        // needn't check it.
1732
129
        // FIXME: Is this true in the case where we are hardening loads after
1733
129
        // they complete? Unclear, need to investigate.
1734
129
        if ((BaseReg && 
LoadDepRegs.test(BaseReg)125
) ||
1735
129
            (IndexReg && 
LoadDepRegs.test(IndexReg)54
))
1736
0
          continue;
1737
129
1738
129
        // If post-load hardening is enabled, this load is compatible with
1739
129
        // post-load hardening, and we aren't already going to harden one of the
1740
129
        // address registers, queue it up to be hardened post-load. Notably,
1741
129
        // even once hardened this won't introduce a useful dependency that
1742
129
        // could prune out subsequent loads.
1743
129
        if (EnablePostLoadHardening && isDataInvariantLoad(MI) &&
1744
129
            
MI.getDesc().getNumDefs() == 178
&&
MI.getOperand(0).isReg()78
&&
1745
129
            
canHardenRegister(MI.getOperand(0).getReg())78
&&
1746
129
            
!HardenedAddrRegs.count(BaseReg)74
&&
1747
129
            
!HardenedAddrRegs.count(IndexReg)73
) {
1748
70
          HardenPostLoad.insert(&MI);
1749
70
          HardenedAddrRegs.insert(MI.getOperand(0).getReg());
1750
70
          continue;
1751
70
        }
1752
59
1753
59
        // Record this instruction for address hardening and record its register
1754
59
        // operands as being address-hardened.
1755
59
        HardenLoadAddr.insert(&MI);
1756
59
        if (BaseReg)
1757
59
          HardenedAddrRegs.insert(BaseReg);
1758
59
        if (IndexReg)
1759
44
          HardenedAddrRegs.insert(IndexReg);
1760
59
1761
59
        for (MachineOperand &Def : MI.defs())
1762
98
          if (Def.isReg())
1763
98
            LoadDepRegs.set(Def.getReg());
1764
59
      }
1765
183
1766
183
    // Now re-walk the instructions in the basic block, and apply whichever
1767
183
    // hardening strategy we have elected. Note that we do this in a second
1768
183
    // pass specifically so that we have the complete set of instructions for
1769
183
    // which we will do post-load hardening and can defer it in certain
1770
183
    // circumstances.
1771
1.77k
    for (MachineInstr &MI : MBB) {
1772
1.77k
      if (HardenLoads) {
1773
1.77k
        // We cannot both require hardening the def of a load and its address.
1774
1.77k
        assert(!(HardenLoadAddr.count(&MI) && HardenPostLoad.count(&MI)) &&
1775
1.77k
               "Requested to harden both the address and def of a load!");
1776
1.77k
1777
1.77k
        // Check if this is a load whose address needs to be hardened.
1778
1.77k
        if (HardenLoadAddr.erase(&MI)) {
1779
59
          const MCInstrDesc &Desc = MI.getDesc();
1780
59
          int MemRefBeginIdx = X86II::getMemoryOperandNo(Desc.TSFlags);
1781
59
          assert(MemRefBeginIdx >= 0 && "Cannot have an invalid index here!");
1782
59
1783
59
          MemRefBeginIdx += X86II::getOperandBias(Desc);
1784
59
1785
59
          MachineOperand &BaseMO =
1786
59
              MI.getOperand(MemRefBeginIdx + X86::AddrBaseReg);
1787
59
          MachineOperand &IndexMO =
1788
59
              MI.getOperand(MemRefBeginIdx + X86::AddrIndexReg);
1789
59
          hardenLoadAddr(MI, BaseMO, IndexMO, AddrRegToHardenedReg);
1790
59
          continue;
1791
59
        }
1792
1.71k
1793
1.71k
        // Test if this instruction is one of our post load instructions (and
1794
1.71k
        // remove it from the set if so).
1795
1.71k
        if (HardenPostLoad.erase(&MI)) {
1796
79
          assert(!MI.isCall() && "Must not try to post-load harden a call!");
1797
79
1798
79
          // If this is a data-invariant load, we want to try and sink any
1799
79
          // hardening as far as possible.
1800
79
          if (isDataInvariantLoad(MI)) {
1801
70
            // Sink the instruction we'll need to harden as far as we can down
1802
70
            // the graph.
1803
70
            MachineInstr *SunkMI = sinkPostLoadHardenedInst(MI, HardenPostLoad);
1804
70
1805
70
            // If we managed to sink this instruction, update everything so we
1806
70
            // harden that instruction when we reach it in the instruction
1807
70
            // sequence.
1808
70
            if (SunkMI != &MI) {
1809
24
              // If in sinking there was no instruction needing to be hardened,
1810
24
              // we're done.
1811
24
              if (!SunkMI)
1812
15
                continue;
1813
9
1814
9
              // Otherwise, add this to the set of defs we harden.
1815
9
              HardenPostLoad.insert(SunkMI);
1816
9
              continue;
1817
9
            }
1818
70
          }
1819
55
1820
55
          unsigned HardenedReg = hardenPostLoad(MI);
1821
55
1822
55
          // Mark the resulting hardened register as such so we don't re-harden.
1823
55
          AddrRegToHardenedReg[HardenedReg] = HardenedReg;
1824
55
1825
55
          continue;
1826
55
        }
1827
1.63k
1828
1.63k
        // Check for an indirect call or branch that may need its input hardened
1829
1.63k
        // even if we couldn't find the specific load used, or were able to
1830
1.63k
        // avoid hardening it for some reason. Note that here we cannot break
1831
1.63k
        // out afterward as we may still need to handle any call aspect of this
1832
1.63k
        // instruction.
1833
1.63k
        if ((MI.isCall() || 
MI.isBranch()1.57k
) &&
HardenIndirectCallsAndJumps138
)
1834
138
          hardenIndirectCallOrJumpInstr(MI, AddrRegToHardenedReg);
1835
1.63k
      }
1836
1.77k
1837
1.77k
      // After we finish hardening loads we handle interprocedural hardening if
1838
1.77k
      // enabled and relevant for this instruction.
1839
1.77k
      
if (1.63k
!HardenInterprocedurally1.63k
)
1840
0
        continue;
1841
1.63k
      if (!MI.isCall() && 
!MI.isReturn()1.57k
)
1842
1.47k
        continue;
1843
161
1844
161
      // If this is a direct return (IE, not a tail call) just directly harden
1845
161
      // it.
1846
161
      if (MI.isReturn() && 
!MI.isCall()108
) {
1847
102
        hardenReturnInstr(MI);
1848
102
        continue;
1849
102
      }
1850
59
1851
59
      // Otherwise we have a call. We need to handle transferring the predicate
1852
59
      // state into a call and recovering it after the call returns (unless this
1853
59
      // is a tail call).
1854
59
      assert(MI.isCall() && "Should only reach here for calls!");
1855
59
      tracePredStateThroughCall(MI);
1856
59
    }
1857
183
1858
183
    HardenPostLoad.clear();
1859
183
    HardenLoadAddr.clear();
1860
183
    HardenedAddrRegs.clear();
1861
183
    AddrRegToHardenedReg.clear();
1862
183
1863
183
    // Currently, we only track data-dependent loads within a basic block.
1864
183
    // FIXME: We should see if this is necessary or if we could be more
1865
183
    // aggressive here without opening up attack avenues.
1866
183
    LoadDepRegs.clear();
1867
183
  }
1868
81
}
1869
1870
/// Save EFLAGS into the returned GPR. This can in turn be restored with
1871
/// `restoreEFLAGS`.
1872
///
1873
/// Note that LLVM can only lower very simple patterns of saved and restored
1874
/// EFLAGS registers. The restore should always be within the same basic block
1875
/// as the save so that no PHI nodes are inserted.
1876
unsigned X86SpeculativeLoadHardeningPass::saveEFLAGS(
1877
    MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertPt,
1878
0
    DebugLoc Loc) {
1879
0
  // FIXME: Hard coding this to a 32-bit register class seems weird, but matches
1880
0
  // what instruction selection does.
1881
0
  unsigned Reg = MRI->createVirtualRegister(&X86::GR32RegClass);
1882
0
  // We directly copy the FLAGS register and rely on later lowering to clean
1883
0
  // this up into the appropriate setCC instructions.
1884
0
  BuildMI(MBB, InsertPt, Loc, TII->get(X86::COPY), Reg).addReg(X86::EFLAGS);
1885
0
  ++NumInstsInserted;
1886
0
  return Reg;
1887
0
}
1888
1889
/// Restore EFLAGS from the provided GPR. This should be produced by
1890
/// `saveEFLAGS`.
1891
///
1892
/// This must be done within the same basic block as the save in order to
1893
/// reliably lower.
1894
void X86SpeculativeLoadHardeningPass::restoreEFLAGS(
1895
    MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertPt, DebugLoc Loc,
1896
0
    unsigned Reg) {
1897
0
  BuildMI(MBB, InsertPt, Loc, TII->get(X86::COPY), X86::EFLAGS).addReg(Reg);
1898
0
  ++NumInstsInserted;
1899
0
}
1900
1901
/// Takes the current predicate state (in a register) and merges it into the
1902
/// stack pointer. The state is essentially a single bit, but we merge this in
1903
/// a way that won't form non-canonical pointers and also will be preserved
1904
/// across normal stack adjustments.
1905
void X86SpeculativeLoadHardeningPass::mergePredStateIntoSP(
1906
    MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertPt, DebugLoc Loc,
1907
161
    unsigned PredStateReg) {
1908
161
  unsigned TmpReg = MRI->createVirtualRegister(PS->RC);
1909
161
  // FIXME: This hard codes a shift distance based on the number of bits needed
1910
161
  // to stay canonical on 64-bit. We should compute this somehow and support
1911
161
  // 32-bit as part of that.
1912
161
  auto ShiftI = BuildMI(MBB, InsertPt, Loc, TII->get(X86::SHL64ri), TmpReg)
1913
161
                    .addReg(PredStateReg, RegState::Kill)
1914
161
                    .addImm(47);
1915
161
  ShiftI->addRegisterDead(X86::EFLAGS, TRI);
1916
161
  ++NumInstsInserted;
1917
161
  auto OrI = BuildMI(MBB, InsertPt, Loc, TII->get(X86::OR64rr), X86::RSP)
1918
161
                 .addReg(X86::RSP)
1919
161
                 .addReg(TmpReg, RegState::Kill);
1920
161
  OrI->addRegisterDead(X86::EFLAGS, TRI);
1921
161
  ++NumInstsInserted;
1922
161
}
1923
1924
/// Extracts the predicate state stored in the high bits of the stack pointer.
1925
unsigned X86SpeculativeLoadHardeningPass::extractPredStateFromSP(
1926
    MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertPt,
1927
135
    DebugLoc Loc) {
1928
135
  unsigned PredStateReg = MRI->createVirtualRegister(PS->RC);
1929
135
  unsigned TmpReg = MRI->createVirtualRegister(PS->RC);
1930
135
1931
135
  // We know that the stack pointer will have any preserved predicate state in
1932
135
  // its high bit. We just want to smear this across the other bits. Turns out,
1933
135
  // this is exactly what an arithmetic right shift does.
1934
135
  BuildMI(MBB, InsertPt, Loc, TII->get(TargetOpcode::COPY), TmpReg)
1935
135
      .addReg(X86::RSP);
1936
135
  auto ShiftI =
1937
135
      BuildMI(MBB, InsertPt, Loc, TII->get(X86::SAR64ri), PredStateReg)
1938
135
          .addReg(TmpReg, RegState::Kill)
1939
135
          .addImm(TRI->getRegSizeInBits(*PS->RC) - 1);
1940
135
  ShiftI->addRegisterDead(X86::EFLAGS, TRI);
1941
135
  ++NumInstsInserted;
1942
135
1943
135
  return PredStateReg;
1944
135
}
1945
1946
void X86SpeculativeLoadHardeningPass::hardenLoadAddr(
1947
    MachineInstr &MI, MachineOperand &BaseMO, MachineOperand &IndexMO,
1948
59
    SmallDenseMap<unsigned, unsigned, 32> &AddrRegToHardenedReg) {
1949
59
  MachineBasicBlock &MBB = *MI.getParent();
1950
59
  DebugLoc Loc = MI.getDebugLoc();
1951
59
1952
59
  // Check if EFLAGS are alive by seeing if there is a def of them or they
1953
59
  // live-in, and then seeing if that def is in turn used.
1954
59
  bool EFLAGSLive = isEFLAGSLive(MBB, MI.getIterator(), *TRI);
1955
59
1956
59
  SmallVector<MachineOperand *, 2> HardenOpRegs;
1957
59
1958
59
  if (BaseMO.isFI()) {
1959
0
    // A frame index is never a dynamically controllable load, so only
1960
0
    // harden it if we're covering fixed address loads as well.
1961
0
    LLVM_DEBUG(
1962
0
        dbgs() << "  Skipping hardening base of explicit stack frame load: ";
1963
0
        MI.dump(); dbgs() << "\n");
1964
59
  } else if (BaseMO.getReg() == X86::RSP) {
1965
0
    // Some idempotent atomic operations are lowered directly to a locked
1966
0
    // OR with 0 to the top of stack(or slightly offset from top) which uses an
1967
0
    // explicit RSP register as the base.
1968
0
    assert(IndexMO.getReg() == X86::NoRegister &&
1969
0
           "Explicit RSP access with dynamic index!");
1970
0
    LLVM_DEBUG(
1971
0
        dbgs() << "  Cannot harden base of explicit RSP offset in a load!");
1972
59
  } else if (BaseMO.getReg() == X86::RIP ||
1973
59
             BaseMO.getReg() == X86::NoRegister) {
1974
0
    // For both RIP-relative addressed loads or absolute loads, we cannot
1975
0
    // meaningfully harden them because the address being loaded has no
1976
0
    // dynamic component.
1977
0
    //
1978
0
    // FIXME: When using a segment base (like TLS does) we end up with the
1979
0
    // dynamic address being the base plus -1 because we can't mutate the
1980
0
    // segment register here. This allows the signed 32-bit offset to point at
1981
0
    // valid segment-relative addresses and load them successfully.
1982
0
    LLVM_DEBUG(
1983
0
        dbgs() << "  Cannot harden base of "
1984
0
               << (BaseMO.getReg() == X86::RIP ? "RIP-relative" : "no-base")
1985
0
               << " address in a load!");
1986
59
  } else {
1987
59
    assert(BaseMO.isReg() &&
1988
59
           "Only allowed to have a frame index or register base.");
1989
59
    HardenOpRegs.push_back(&BaseMO);
1990
59
  }
1991
59
1992
59
  if (IndexMO.getReg() != X86::NoRegister &&
1993
59
      
(44
HardenOpRegs.empty()44
||
1994
44
       HardenOpRegs.front()->getReg() != IndexMO.getReg()))
1995
44
    HardenOpRegs.push_back(&IndexMO);
1996
59
1997
59
  assert((HardenOpRegs.size() == 1 || HardenOpRegs.size() == 2) &&
1998
59
         "Should have exactly one or two registers to harden!");
1999
59
  assert((HardenOpRegs.size() == 1 ||
2000
59
          HardenOpRegs[0]->getReg() != HardenOpRegs[1]->getReg()) &&
2001
59
         "Should not have two of the same registers!");
2002
59
2003
59
  // Remove any registers that have alreaded been checked.
2004
103
  llvm::erase_if(HardenOpRegs, [&](MachineOperand *Op) {
2005
103
    // See if this operand's register has already been checked.
2006
103
    auto It = AddrRegToHardenedReg.find(Op->getReg());
2007
103
    if (It == AddrRegToHardenedReg.end())
2008
95
      // Not checked, so retain this one.
2009
95
      return false;
2010
8
2011
8
    // Otherwise, we can directly update this operand and remove it.
2012
8
    Op->setReg(It->second);
2013
8
    return true;
2014
8
  });
2015
59
  // If there are none left, we're done.
2016
59
  if (HardenOpRegs.empty())
2017
5
    return;
2018
54
2019
54
  // Compute the current predicate state.
2020
54
  unsigned StateReg = PS->SSA.GetValueAtEndOfBlock(&MBB);
2021
54
2022
54
  auto InsertPt = MI.getIterator();
2023
54
2024
54
  // If EFLAGS are live and we don't have access to instructions that avoid
2025
54
  // clobbering EFLAGS we need to save and restore them. This in turn makes
2026
54
  // the EFLAGS no longer live.
2027
54
  unsigned FlagsReg = 0;
2028
54
  if (EFLAGSLive && 
!Subtarget->hasBMI2()0
) {
2029
0
    EFLAGSLive = false;
2030
0
    FlagsReg = saveEFLAGS(MBB, InsertPt, Loc);
2031
0
  }
2032
54
2033
95
  for (MachineOperand *Op : HardenOpRegs) {
2034
95
    unsigned OpReg = Op->getReg();
2035
95
    auto *OpRC = MRI->getRegClass(OpReg);
2036
95
    unsigned TmpReg = MRI->createVirtualRegister(OpRC);
2037
95
2038
95
    // If this is a vector register, we'll need somewhat custom logic to handle
2039
95
    // hardening it.
2040
95
    if (!Subtarget->hasVLX() && 
(63
OpRC->hasSuperClassEq(&X86::VR128RegClass)63
||
2041
63
                                 
OpRC->hasSuperClassEq(&X86::VR256RegClass)53
)) {
2042
18
      assert(Subtarget->hasAVX2() && "AVX2-specific register classes!");
2043
18
      bool Is128Bit = OpRC->hasSuperClassEq(&X86::VR128RegClass);
2044
18
2045
18
      // Move our state into a vector register.
2046
18
      // FIXME: We could skip this at the cost of longer encodings with AVX-512
2047
18
      // but that doesn't seem likely worth it.
2048
18
      unsigned VStateReg = MRI->createVirtualRegister(&X86::VR128RegClass);
2049
18
      auto MovI =
2050
18
          BuildMI(MBB, InsertPt, Loc, TII->get(X86::VMOV64toPQIrr), VStateReg)
2051
18
              .addReg(StateReg);
2052
18
      (void)MovI;
2053
18
      ++NumInstsInserted;
2054
18
      LLVM_DEBUG(dbgs() << "  Inserting mov: "; MovI->dump(); dbgs() << "\n");
2055
18
2056
18
      // Broadcast it across the vector register.
2057
18
      unsigned VBStateReg = MRI->createVirtualRegister(OpRC);
2058
18
      auto BroadcastI = BuildMI(MBB, InsertPt, Loc,
2059
18
                                TII->get(Is128Bit ? 
X86::VPBROADCASTQrr10
2060
18
                                                  : 
X86::VPBROADCASTQYrr8
),
2061
18
                                VBStateReg)
2062
18
                            .addReg(VStateReg);
2063
18
      (void)BroadcastI;
2064
18
      ++NumInstsInserted;
2065
18
      LLVM_DEBUG(dbgs() << "  Inserting broadcast: "; BroadcastI->dump();
2066
18
                 dbgs() << "\n");
2067
18
2068
18
      // Merge our potential poison state into the value with a vector or.
2069
18
      auto OrI =
2070
18
          BuildMI(MBB, InsertPt, Loc,
2071
18
                  TII->get(Is128Bit ? 
X86::VPORrr10
:
X86::VPORYrr8
), TmpReg)
2072
18
              .addReg(VBStateReg)
2073
18
              .addReg(OpReg);
2074
18
      (void)OrI;
2075
18
      ++NumInstsInserted;
2076
18
      LLVM_DEBUG(dbgs() << "  Inserting or: "; OrI->dump(); dbgs() << "\n");
2077
77
    } else if (OpRC->hasSuperClassEq(&X86::VR128XRegClass) ||
2078
77
               
OpRC->hasSuperClassEq(&X86::VR256XRegClass)67
||
2079
77
               
OpRC->hasSuperClassEq(&X86::VR512RegClass)61
) {
2080
23
      assert(Subtarget->hasAVX512() && "AVX512-specific register classes!");
2081
23
      bool Is128Bit = OpRC->hasSuperClassEq(&X86::VR128XRegClass);
2082
23
      bool Is256Bit = OpRC->hasSuperClassEq(&X86::VR256XRegClass);
2083
23
      if (Is128Bit || 
Is256Bit13
)
2084
23
        assert(Subtarget->hasVLX() && "AVX512VL-specific register classes!");
2085
23
2086
23
      // Broadcast our state into a vector register.
2087
23
      unsigned VStateReg = MRI->createVirtualRegister(OpRC);
2088
23
      unsigned BroadcastOp =
2089
23
          Is128Bit ? 
X86::VPBROADCASTQrZ128r10
2090
23
                   : 
Is256Bit 13
?
X86::VPBROADCASTQrZ256r6
:
X86::VPBROADCASTQrZr7
;
2091
23
      auto BroadcastI =
2092
23
          BuildMI(MBB, InsertPt, Loc, TII->get(BroadcastOp), VStateReg)
2093
23
              .addReg(StateReg);
2094
23
      (void)BroadcastI;
2095
23
      ++NumInstsInserted;
2096
23
      LLVM_DEBUG(dbgs() << "  Inserting broadcast: "; BroadcastI->dump();
2097
23
                 dbgs() << "\n");
2098
23
2099
23
      // Merge our potential poison state into the value with a vector or.
2100
23
      unsigned OrOp = Is128Bit ? 
X86::VPORQZ128rr10
2101
23
                               : 
Is256Bit 13
?
X86::VPORQZ256rr6
:
X86::VPORQZrr7
;
2102
23
      auto OrI = BuildMI(MBB, InsertPt, Loc, TII->get(OrOp), TmpReg)
2103
23
                     .addReg(VStateReg)
2104
23
                     .addReg(OpReg);
2105
23
      (void)OrI;
2106
23
      ++NumInstsInserted;
2107
23
      LLVM_DEBUG(dbgs() << "  Inserting or: "; OrI->dump(); dbgs() << "\n");
2108
54
    } else {
2109
54
      // FIXME: Need to support GR32 here for 32-bit code.
2110
54
      assert(OpRC->hasSuperClassEq(&X86::GR64RegClass) &&
2111
54
             "Not a supported register class for address hardening!");
2112
54
2113
54
      if (!EFLAGSLive) {
2114
54
        // Merge our potential poison state into the value with an or.
2115
54
        auto OrI = BuildMI(MBB, InsertPt, Loc, TII->get(X86::OR64rr), TmpReg)
2116
54
                       .addReg(StateReg)
2117
54
                       .addReg(OpReg);
2118
54
        OrI->addRegisterDead(X86::EFLAGS, TRI);
2119
54
        ++NumInstsInserted;
2120
54
        LLVM_DEBUG(dbgs() << "  Inserting or: "; OrI->dump(); dbgs() << "\n");
2121
54
      } else {
2122
0
        // We need to avoid touching EFLAGS so shift out all but the least
2123
0
        // significant bit using the instruction that doesn't update flags.
2124
0
        auto ShiftI =
2125
0
            BuildMI(MBB, InsertPt, Loc, TII->get(X86::SHRX64rr), TmpReg)
2126
0
                .addReg(OpReg)
2127
0
                .addReg(StateReg);
2128
0
        (void)ShiftI;
2129
0
        ++NumInstsInserted;
2130
0
        LLVM_DEBUG(dbgs() << "  Inserting shrx: "; ShiftI->dump();
2131
0
                   dbgs() << "\n");
2132
0
      }
2133
54
    }
2134
95
2135
95
    // Record this register as checked and update the operand.
2136
95
    assert(!AddrRegToHardenedReg.count(Op->getReg()) &&
2137
95
           "Should not have checked this register yet!");
2138
95
    AddrRegToHardenedReg[Op->getReg()] = TmpReg;
2139
95
    Op->setReg(TmpReg);
2140
95
    ++NumAddrRegsHardened;
2141
95
  }
2142
54
2143
54
  // And restore the flags if needed.
2144
54
  if (FlagsReg)
2145
0
    restoreEFLAGS(MBB, InsertPt, Loc, FlagsReg);
2146
54
}
2147
2148
MachineInstr *X86SpeculativeLoadHardeningPass::sinkPostLoadHardenedInst(
2149
70
    MachineInstr &InitialMI, SmallPtrSetImpl<MachineInstr *> &HardenedInstrs) {
2150
70
  assert(isDataInvariantLoad(InitialMI) &&
2151
70
         "Cannot get here with a non-invariant load!");
2152
70
2153
70
  // See if we can sink hardening the loaded value.
2154
70
  auto SinkCheckToSingleUse =
2155
86
      [&](MachineInstr &MI) -> Optional<MachineInstr *> {
2156
86
    unsigned DefReg = MI.getOperand(0).getReg();
2157
86
2158
86
    // We need to find a single use which we can sink the check. We can
2159
86
    // primarily do this because many uses may already end up checked on their
2160
86
    // own.
2161
86
    MachineInstr *SingleUseMI = nullptr;
2162
87
    for (MachineInstr &UseMI : MRI->use_instructions(DefReg)) {
2163
87
      // If we're already going to harden this use, it is data invariant and
2164
87
      // within our block.
2165
87
      if (HardenedInstrs.count(&UseMI)) {
2166
17
        if (!isDataInvariantLoad(UseMI)) {
2167
1
          // If we've already decided to harden a non-load, we must have sunk
2168
1
          // some other post-load hardened instruction to it and it must itself
2169
1
          // be data-invariant.
2170
1
          assert(isDataInvariant(UseMI) &&
2171
1
                 "Data variant instruction being hardened!");
2172
1
          continue;
2173
1
        }
2174
16
2175
16
        // Otherwise, this is a load and the load component can't be data
2176
16
        // invariant so check how this register is being used.
2177
16
        const MCInstrDesc &Desc = UseMI.getDesc();
2178
16
        int MemRefBeginIdx = X86II::getMemoryOperandNo(Desc.TSFlags);
2179
16
        assert(MemRefBeginIdx >= 0 &&
2180
16
               "Should always have mem references here!");
2181
16
        MemRefBeginIdx += X86II::getOperandBias(Desc);
2182
16
2183
16
        MachineOperand &BaseMO =
2184
16
            UseMI.getOperand(MemRefBeginIdx + X86::AddrBaseReg);
2185
16
        MachineOperand &IndexMO =
2186
16
            UseMI.getOperand(MemRefBeginIdx + X86::AddrIndexReg);
2187
16
        if ((BaseMO.isReg() && BaseMO.getReg() == DefReg) ||
2188
16
            (IndexMO.isReg() && IndexMO.getReg() == DefReg))
2189
2
          // The load uses the register as part of its address making it not
2190
2
          // invariant.
2191
2
          return {};
2192
14
2193
14
        continue;
2194
14
      }
2195
70
2196
70
      if (SingleUseMI)
2197
1
        // We already have a single use, this would make two. Bail.
2198
1
        return {};
2199
69
2200
69
      // If this single use isn't data invariant, isn't in this block, or has
2201
69
      // interfering EFLAGS, we can't sink the hardening to it.
2202
69
      if (!isDataInvariant(UseMI) || 
UseMI.getParent() != MI.getParent()35
)
2203
34
        return {};
2204
35
2205
35
      // If this instruction defines multiple registers bail as we won't harden
2206
35
      // all of them.
2207
35
      if (UseMI.getDesc().getNumDefs() > 1)
2208
0
        return {};
2209
35
2210
35
      // If this register isn't a virtual register we can't walk uses of sanely,
2211
35
      // just bail. Also check that its register class is one of the ones we
2212
35
      // can harden.
2213
35
      unsigned UseDefReg = UseMI.getOperand(0).getReg();
2214
35
      if (!TRI->isVirtualRegister(UseDefReg) ||
2215
35
          
!canHardenRegister(UseDefReg)17
)
2216
18
        return {};
2217
17
2218
17
      SingleUseMI = &UseMI;
2219
17
    }
2220
86
2221
86
    // If SingleUseMI is still null, there is no use that needs its own
2222
86
    // checking. Otherwise, it is the single use that needs checking.
2223
86
    
return {SingleUseMI}31
;
2224
86
  };
2225
70
2226
70
  MachineInstr *MI = &InitialMI;
2227
86
  while (Optional<MachineInstr *> SingleUse = SinkCheckToSingleUse(*MI)) {
2228
31
    // Update which MI we're checking now.
2229
31
    MI = *SingleUse;
2230
31
    if (!MI)
2231
15
      break;
2232
31
  }
2233
70
2234
70
  return MI;
2235
70
}
2236
2237
95
bool X86SpeculativeLoadHardeningPass::canHardenRegister(unsigned Reg) {
2238
95
  auto *RC = MRI->getRegClass(Reg);
2239
95
  int RegBytes = TRI->getRegSizeInBits(*RC) / 8;
2240
95
  if (RegBytes > 8)
2241
0
    // We don't support post-load hardening of vectors.
2242
0
    return false;
2243
95
2244
95
  // If this register class is explicitly constrained to a class that doesn't
2245
95
  // require REX prefix, we may not be able to satisfy that constraint when
2246
95
  // emitting the hardening instructions, so bail out here.
2247
95
  // FIXME: This seems like a pretty lame hack. The way this comes up is when we
2248
95
  // end up both with a NOREX and REX-only register as operands to the hardening
2249
95
  // instructions. It would be better to fix that code to handle this situation
2250
95
  // rather than hack around it in this way.
2251
95
  const TargetRegisterClass *NOREXRegClasses[] = {
2252
95
      &X86::GR8_NOREXRegClass, &X86::GR16_NOREXRegClass,
2253
95
      &X86::GR32_NOREXRegClass, &X86::GR64_NOREXRegClass};
2254
95
  if (RC == NOREXRegClasses[Log2_32(RegBytes)])
2255
0
    return false;
2256
95
2257
95
  const TargetRegisterClass *GPRRegClasses[] = {
2258
95
      &X86::GR8RegClass, &X86::GR16RegClass, &X86::GR32RegClass,
2259
95
      &X86::GR64RegClass};
2260
95
  return RC->hasSuperClassEq(GPRRegClasses[Log2_32(RegBytes)]);
2261
95
}
2262
2263
/// Harden a value in a register.
2264
///
2265
/// This is the low-level logic to fully harden a value sitting in a register
2266
/// against leaking during speculative execution.
2267
///
2268
/// Unlike hardening an address that is used by a load, this routine is required
2269
/// to hide *all* incoming bits in the register.
2270
///
2271
/// `Reg` must be a virtual register. Currently, it is required to be a GPR no
2272
/// larger than the predicate state register. FIXME: We should support vector
2273
/// registers here by broadcasting the predicate state.
2274
///
2275
/// The new, hardened virtual register is returned. It will have the same
2276
/// register class as `Reg`.
2277
unsigned X86SpeculativeLoadHardeningPass::hardenValueInRegister(
2278
    unsigned Reg, MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertPt,
2279
57
    DebugLoc Loc) {
2280
57
  assert(canHardenRegister(Reg) && "Cannot harden this register!");
2281
57
  assert(TRI->isVirtualRegister(Reg) && "Cannot harden a physical register!");
2282
57
2283
57
  auto *RC = MRI->getRegClass(Reg);
2284
57
  int Bytes = TRI->getRegSizeInBits(*RC) / 8;
2285
57
2286
57
  unsigned StateReg = PS->SSA.GetValueAtEndOfBlock(&MBB);
2287
57
2288
57
  // FIXME: Need to teach this about 32-bit mode.
2289
57
  if (Bytes != 8) {
2290
30
    unsigned SubRegImms[] = {X86::sub_8bit, X86::sub_16bit, X86::sub_32bit};
2291
30
    unsigned SubRegImm = SubRegImms[Log2_32(Bytes)];
2292
30
    unsigned NarrowStateReg = MRI->createVirtualRegister(RC);
2293
30
    BuildMI(MBB, InsertPt, Loc, TII->get(TargetOpcode::COPY), NarrowStateReg)
2294
30
        .addReg(StateReg, 0, SubRegImm);
2295
30
    StateReg = NarrowStateReg;
2296
30
  }
2297
57
2298
57
  unsigned FlagsReg = 0;
2299
57
  if (isEFLAGSLive(MBB, InsertPt, *TRI))
2300
0
    FlagsReg = saveEFLAGS(MBB, InsertPt, Loc);
2301
57
2302
57
  unsigned NewReg = MRI->createVirtualRegister(RC);
2303
57
  unsigned OrOpCodes[] = {X86::OR8rr, X86::OR16rr, X86::OR32rr, X86::OR64rr};
2304
57
  unsigned OrOpCode = OrOpCodes[Log2_32(Bytes)];
2305
57
  auto OrI = BuildMI(MBB, InsertPt, Loc, TII->get(OrOpCode), NewReg)
2306
57
                 .addReg(StateReg)
2307
57
                 .addReg(Reg);
2308
57
  OrI->addRegisterDead(X86::EFLAGS, TRI);
2309
57
  ++NumInstsInserted;
2310
57
  LLVM_DEBUG(dbgs() << "  Inserting or: "; OrI->dump(); dbgs() << "\n");
2311
57
2312
57
  if (FlagsReg)
2313
0
    restoreEFLAGS(MBB, InsertPt, Loc, FlagsReg);
2314
57
2315
57
  return NewReg;
2316
57
}
2317
2318
/// Harden a load by hardening the loaded value in the defined register.
2319
///
2320
/// We can harden a non-leaking load into a register without touching the
2321
/// address by just hiding all of the loaded bits during misspeculation. We use
2322
/// an `or` instruction to do this because we set up our poison value as all
2323
/// ones. And the goal is just for the loaded bits to not be exposed to
2324
/// execution and coercing them to one is sufficient.
2325
///
2326
/// Returns the newly hardened register.
2327
55
unsigned X86SpeculativeLoadHardeningPass::hardenPostLoad(MachineInstr &MI) {
2328
55
  MachineBasicBlock &MBB = *MI.getParent();
2329
55
  DebugLoc Loc = MI.getDebugLoc();
2330
55
2331
55
  auto &DefOp = MI.getOperand(0);
2332
55
  unsigned OldDefReg = DefOp.getReg();
2333
55
  auto *DefRC = MRI->getRegClass(OldDefReg);
2334
55
2335
55
  // Because we want to completely replace the uses of this def'ed value with
2336
55
  // the hardened value, create a dedicated new register that will only be used
2337
55
  // to communicate the unhardened value to the hardening.
2338
55
  unsigned UnhardenedReg = MRI->createVirtualRegister(DefRC);
2339
55
  DefOp.setReg(UnhardenedReg);
2340
55
2341
55
  // Now harden this register's value, getting a hardened reg that is safe to
2342
55
  // use. Note that we insert the instructions to compute this *after* the
2343
55
  // defining instruction, not before it.
2344
55
  unsigned HardenedReg = hardenValueInRegister(
2345
55
      UnhardenedReg, MBB, std::next(MI.getIterator()), Loc);
2346
55
2347
55
  // Finally, replace the old register (which now only has the uses of the
2348
55
  // original def) with the hardened register.
2349
55
  MRI->replaceRegWith(/*FromReg*/ OldDefReg, /*ToReg*/ HardenedReg);
2350
55
2351
55
  ++NumPostLoadRegsHardened;
2352
55
  return HardenedReg;
2353
55
}
2354
2355
/// Harden a return instruction.
2356
///
2357
/// Returns implicitly perform a load which we need to harden. Without hardening
2358
/// this load, an attacker my speculatively write over the return address to
2359
/// steer speculation of the return to an attacker controlled address. This is
2360
/// called Spectre v1.1 or Bounds Check Bypass Store (BCBS) and is described in
2361
/// this paper:
2362
/// https://people.csail.mit.edu/vlk/spectre11.pdf
2363
///
2364
/// We can harden this by introducing an LFENCE that will delay any load of the
2365
/// return address until prior instructions have retired (and thus are not being
2366
/// speculated), or we can harden the address used by the implicit load: the
2367
/// stack pointer.
2368
///
2369
/// If we are not using an LFENCE, hardening the stack pointer has an additional
2370
/// benefit: it allows us to pass the predicate state accumulated in this
2371
/// function back to the caller. In the absence of a BCBS attack on the return,
2372
/// the caller will typically be resumed and speculatively executed due to the
2373
/// Return Stack Buffer (RSB) prediction which is very accurate and has a high
2374
/// priority. It is possible that some code from the caller will be executed
2375
/// speculatively even during a BCBS-attacked return until the steering takes
2376
/// effect. Whenever this happens, the caller can recover the (poisoned)
2377
/// predicate state from the stack pointer and continue to harden loads.
2378
102
void X86SpeculativeLoadHardeningPass::hardenReturnInstr(MachineInstr &MI) {
2379
102
  MachineBasicBlock &MBB = *MI.getParent();
2380
102
  DebugLoc Loc = MI.getDebugLoc();
2381
102
  auto InsertPt = MI.getIterator();
2382
102
2383
102
  if (FenceCallAndRet)
2384
0
    // No need to fence here as we'll fence at the return site itself. That
2385
0
    // handles more cases than we can handle here.
2386
0
    return;
2387
102
2388
102
  // Take our predicate state, shift it to the high 17 bits (so that we keep
2389
102
  // pointers canonical) and merge it into RSP. This will allow the caller to
2390
102
  // extract it when we return (speculatively).
2391
102
  mergePredStateIntoSP(MBB, InsertPt, Loc, PS->SSA.GetValueAtEndOfBlock(&MBB));
2392
102
}
2393
2394
/// Trace the predicate state through a call.
2395
///
2396
/// There are several layers of this needed to handle the full complexity of
2397
/// calls.
2398
///
2399
/// First, we need to send the predicate state into the called function. We do
2400
/// this by merging it into the high bits of the stack pointer.
2401
///
2402
/// For tail calls, this is all we need to do.
2403
///
2404
/// For calls where we might return and resume the control flow, we need to
2405
/// extract the predicate state from the high bits of the stack pointer after
2406
/// control returns from the called function.
2407
///
2408
/// We also need to verify that we intended to return to this location in the
2409
/// code. An attacker might arrange for the processor to mispredict the return
2410
/// to this valid but incorrect return address in the program rather than the
2411
/// correct one. See the paper on this attack, called "ret2spec" by the
2412
/// researchers, here:
2413
/// https://christian-rossow.de/publications/ret2spec-ccs2018.pdf
2414
///
2415
/// The way we verify that we returned to the correct location is by preserving
2416
/// the expected return address across the call. One technique involves taking
2417
/// advantage of the red-zone to load the return address from `8(%rsp)` where it
2418
/// was left by the RET instruction when it popped `%rsp`. Alternatively, we can
2419
/// directly save the address into a register that will be preserved across the
2420
/// call. We compare this intended return address against the address
2421
/// immediately following the call (the observed return address). If these
2422
/// mismatch, we have detected misspeculation and can poison our predicate
2423
/// state.
2424
void X86SpeculativeLoadHardeningPass::tracePredStateThroughCall(
2425
59
    MachineInstr &MI) {
2426
59
  MachineBasicBlock &MBB = *MI.getParent();
2427
59
  MachineFunction &MF = *MBB.getParent();
2428
59
  auto InsertPt = MI.getIterator();
2429
59
  DebugLoc Loc = MI.getDebugLoc();
2430
59
2431
59
  if (FenceCallAndRet) {
2432
0
    if (MI.isReturn())
2433
0
      // Tail call, we don't return to this function.
2434
0
      // FIXME: We should also handle noreturn calls.
2435
0
      return;
2436
0
2437
0
    // We don't need to fence before the call because the function should fence
2438
0
    // in its entry. However, we do need to fence after the call returns.
2439
0
    // Fencing before the return doesn't correctly handle cases where the return
2440
0
    // itself is mispredicted.
2441
0
    BuildMI(MBB, std::next(InsertPt), Loc, TII->get(X86::LFENCE));
2442
0
    ++NumInstsInserted;
2443
0
    ++NumLFENCEsInserted;
2444
0
    return;
2445
0
  }
2446
59
2447
59
  // First, we transfer the predicate state into the called function by merging
2448
59
  // it into the stack pointer. This will kill the current def of the state.
2449
59
  unsigned StateReg = PS->SSA.GetValueAtEndOfBlock(&MBB);
2450
59
  mergePredStateIntoSP(MBB, InsertPt, Loc, StateReg);
2451
59
2452
59
  // If this call is also a return, it is a tail call and we don't need anything
2453
59
  // else to handle it so just return. Also, if there are no further
2454
59
  // instructions and no successors, this call does not return so we can also
2455
59
  // bail.
2456
59
  if (MI.isReturn() || 
(53
std::next(InsertPt) == MBB.end()53
&&
MBB.succ_empty()0
))
2457
6
    return;
2458
53
2459
53
  // Create a symbol to track the return address and attach it to the call
2460
53
  // machine instruction. We will lower extra symbols attached to call
2461
53
  // instructions as label immediately following the call.
2462
53
  MCSymbol *RetSymbol =
2463
53
      MF.getContext().createTempSymbol("slh_ret_addr",
2464
53
                                       /*AlwaysAddSuffix*/ true);
2465
53
  MI.setPostInstrSymbol(MF, RetSymbol);
2466
53
2467
53
  const TargetRegisterClass *AddrRC = &X86::GR64RegClass;
2468
53
  unsigned ExpectedRetAddrReg = 0;
2469
53
2470
53
  // If we have no red zones or if the function returns twice (possibly without
2471
53
  // using the `ret` instruction) like setjmp, we need to save the expected
2472
53
  // return address prior to the call.
2473
53
  if (!Subtarget->getFrameLowering()->has128ByteRedZone(MF) ||
2474
53
      
MF.exposesReturnsTwice()47
) {
2475
15
    // If we don't have red zones, we need to compute the expected return
2476
15
    // address prior to the call and store it in a register that lives across
2477
15
    // the call.
2478
15
    //
2479
15
    // In some ways, this is doubly satisfying as a mitigation because it will
2480
15
    // also successfully detect stack smashing bugs in some cases (typically,
2481
15
    // when a callee-saved register is used and the callee doesn't push it onto
2482
15
    // the stack). But that isn't our primary goal, so we only use it as
2483
15
    // a fallback.
2484
15
    //
2485
15
    // FIXME: It isn't clear that this is reliable in the face of
2486
15
    // rematerialization in the register allocator. We somehow need to force
2487
15
    // that to not occur for this particular instruction, and instead to spill
2488
15
    // or otherwise preserve the value computed *prior* to the call.
2489
15
    //
2490
15
    // FIXME: It is even less clear why MachineCSE can't just fold this when we
2491
15
    // end up having to use identical instructions both before and after the
2492
15
    // call to feed the comparison.
2493
15
    ExpectedRetAddrReg = MRI->createVirtualRegister(AddrRC);
2494
15
    if (MF.getTarget().getCodeModel() == CodeModel::Small &&
2495
15
        
!Subtarget->isPositionIndependent()10
) {
2496
5
      BuildMI(MBB, InsertPt, Loc, TII->get(X86::MOV64ri32), ExpectedRetAddrReg)
2497
5
          .addSym(RetSymbol);
2498
10
    } else {
2499
10
      BuildMI(MBB, InsertPt, Loc, TII->get(X86::LEA64r), ExpectedRetAddrReg)
2500
10
          .addReg(/*Base*/ X86::RIP)
2501
10
          .addImm(/*Scale*/ 1)
2502
10
          .addReg(/*Index*/ 0)
2503
10
          .addSym(RetSymbol)
2504
10
          .addReg(/*Segment*/ 0);
2505
10
    }
2506
15
  }
2507
53
2508
53
  // Step past the call to handle when it returns.
2509
53
  ++InsertPt;
2510
53
2511
53
  // If we didn't pre-compute the expected return address into a register, then
2512
53
  // red zones are enabled and the return address is still available on the
2513
53
  // stack immediately after the call. As the very first instruction, we load it
2514
53
  // into a register.
2515
53
  if (!ExpectedRetAddrReg) {
2516
38
    ExpectedRetAddrReg = MRI->createVirtualRegister(AddrRC);
2517
38
    BuildMI(MBB, InsertPt, Loc, TII->get(X86::MOV64rm), ExpectedRetAddrReg)
2518
38
        .addReg(/*Base*/ X86::RSP)
2519
38
        .addImm(/*Scale*/ 1)
2520
38
        .addReg(/*Index*/ 0)
2521
38
        .addImm(/*Displacement*/ -8) // The stack pointer has been popped, so
2522
38
                                     // the return address is 8-bytes past it.
2523
38
        .addReg(/*Segment*/ 0);
2524
38
  }
2525
53
2526
53
  // Now we extract the callee's predicate state from the stack pointer.
2527
53
  unsigned NewStateReg = extractPredStateFromSP(MBB, InsertPt, Loc);
2528
53
2529
53
  // Test the expected return address against our actual address. If we can
2530
53
  // form this basic block's address as an immediate, this is easy. Otherwise
2531
53
  // we compute it.
2532
53
  if (MF.getTarget().getCodeModel() == CodeModel::Small &&
2533
53
      
!Subtarget->isPositionIndependent()46
) {
2534
37
    // FIXME: Could we fold this with the load? It would require careful EFLAGS
2535
37
    // management.
2536
37
    BuildMI(MBB, InsertPt, Loc, TII->get(X86::CMP64ri32))
2537
37
        .addReg(ExpectedRetAddrReg, RegState::Kill)
2538
37
        .addSym(RetSymbol);
2539
37
  } else {
2540
16
    unsigned ActualRetAddrReg = MRI->createVirtualRegister(AddrRC);
2541
16
    BuildMI(MBB, InsertPt, Loc, TII->get(X86::LEA64r), ActualRetAddrReg)
2542
16
        .addReg(/*Base*/ X86::RIP)
2543
16
        .addImm(/*Scale*/ 1)
2544
16
        .addReg(/*Index*/ 0)
2545
16
        .addSym(RetSymbol)
2546
16
        .addReg(/*Segment*/ 0);
2547
16
    BuildMI(MBB, InsertPt, Loc, TII->get(X86::CMP64rr))
2548
16
        .addReg(ExpectedRetAddrReg, RegState::Kill)
2549
16
        .addReg(ActualRetAddrReg, RegState::Kill);
2550
16
  }
2551
53
2552
53
  // Now conditionally update the predicate state we just extracted if we ended
2553
53
  // up at a different return address than expected.
2554
53
  int PredStateSizeInBytes = TRI->getRegSizeInBits(*PS->RC) / 8;
2555
53
  auto CMovOp = X86::getCMovOpcode(PredStateSizeInBytes);
2556
53
2557
53
  unsigned UpdatedStateReg = MRI->createVirtualRegister(PS->RC);
2558
53
  auto CMovI = BuildMI(MBB, InsertPt, Loc, TII->get(CMovOp), UpdatedStateReg)
2559
53
                   .addReg(NewStateReg, RegState::Kill)
2560
53
                   .addReg(PS->PoisonReg)
2561
53
                   .addImm(X86::COND_NE);
2562
53
  CMovI->findRegisterUseOperand(X86::EFLAGS)->setIsKill(true);
2563
53
  ++NumInstsInserted;
2564
53
  LLVM_DEBUG(dbgs() << "  Inserting cmov: "; CMovI->dump(); dbgs() << "\n");
2565
53
2566
53
  PS->SSA.AddAvailableValue(&MBB, UpdatedStateReg);
2567
53
}
2568
2569
/// An attacker may speculatively store over a value that is then speculatively
2570
/// loaded and used as the target of an indirect call or jump instruction. This
2571
/// is called Spectre v1.2 or Bounds Check Bypass Store (BCBS) and is described
2572
/// in this paper:
2573
/// https://people.csail.mit.edu/vlk/spectre11.pdf
2574
///
2575
/// When this happens, the speculative execution of the call or jump will end up
2576
/// being steered to this attacker controlled address. While most such loads
2577
/// will be adequately hardened already, we want to ensure that they are
2578
/// definitively treated as needing post-load hardening. While address hardening
2579
/// is sufficient to prevent secret data from leaking to the attacker, it may
2580
/// not be sufficient to prevent an attacker from steering speculative
2581
/// execution. We forcibly unfolded all relevant loads above and so will always
2582
/// have an opportunity to post-load harden here, we just need to scan for cases
2583
/// not already flagged and add them.
2584
void X86SpeculativeLoadHardeningPass::hardenIndirectCallOrJumpInstr(
2585
    MachineInstr &MI,
2586
138
    SmallDenseMap<unsigned, unsigned, 32> &AddrRegToHardenedReg) {
2587
138
  switch (MI.getOpcode()) {
2588
138
  case X86::FARCALL16m:
2589
0
  case X86::FARCALL32m:
2590
0
  case X86::FARCALL64:
2591
0
  case X86::FARJMP16m:
2592
0
  case X86::FARJMP32m:
2593
0
  case X86::FARJMP64:
2594
0
    // We don't need to harden either far calls or far jumps as they are
2595
0
    // safe from Spectre.
2596
0
    return;
2597
0
2598
138
  default:
2599
138
    break;
2600
138
  }
2601
138
2602
138
  // We should never see a loading instruction at this point, as those should
2603
138
  // have been unfolded.
2604
138
  assert(!MI.mayLoad() && "Found a lingering loading instruction!");
2605
138
2606
138
  // If the first operand isn't a register, this is a branch or call
2607
138
  // instruction with an immediate operand which doesn't need to be hardened.
2608
138
  if (!MI.getOperand(0).isReg())
2609
122
    return;
2610
16
2611
16
  // For all of these, the target register is the first operand of the
2612
16
  // instruction.
2613
16
  auto &TargetOp = MI.getOperand(0);
2614
16
  unsigned OldTargetReg = TargetOp.getReg();
2615
16
2616
16
  // Try to lookup a hardened version of this register. We retain a reference
2617
16
  // here as we want to update the map to track any newly computed hardened
2618
16
  // register.
2619
16
  unsigned &HardenedTargetReg = AddrRegToHardenedReg[OldTargetReg];
2620
16
2621
16
  // If we don't have a hardened register yet, compute one. Otherwise, just use
2622
16
  // the already hardened register.
2623
16
  //
2624
16
  // FIXME: It is a little suspect that we use partially hardened registers that
2625
16
  // only feed addresses. The complexity of partial hardening with SHRX
2626
16
  // continues to pile up. Should definitively measure its value and consider
2627
16
  // eliminating it.
2628
16
  if (!HardenedTargetReg)
2629
2
    HardenedTargetReg = hardenValueInRegister(
2630
2
        OldTargetReg, *MI.getParent(), MI.getIterator(), MI.getDebugLoc());
2631
16
2632
16
  // Set the target operand to the hardened register.
2633
16
  TargetOp.setReg(HardenedTargetReg);
2634
16
2635
16
  ++NumCallsOrJumpsHardened;
2636
16
}
2637
2638
102k
INITIALIZE_PASS_BEGIN(X86SpeculativeLoadHardeningPass, PASS_KEY,
2639
102k
                      "X86 speculative load hardener", false, false)
2640
102k
INITIALIZE_PASS_END(X86SpeculativeLoadHardeningPass, PASS_KEY,
2641
                    "X86 speculative load hardener", false, false)
2642
2643
12.2k
FunctionPass *llvm::createX86SpeculativeLoadHardeningPass() {
2644
12.2k
  return new X86SpeculativeLoadHardeningPass();
2645
12.2k
}