| //====- X86SpeculativeLoadHardening.cpp - A Spectre v1 mitigation ---------===// |
| // |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| // See https://llvm.org/LICENSE.txt for license information. |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| // |
| //===----------------------------------------------------------------------===// |
| /// \file |
| /// |
| /// Provide a pass which mitigates speculative execution attacks which operate |
| /// by speculating incorrectly past some predicate (a type check, bounds check, |
| /// or other condition) to reach a load with invalid inputs and leak the data |
| /// accessed by that load using a side channel out of the speculative domain. |
| /// |
| /// For details on the attacks, see the first variant in both the Project Zero |
| /// writeup and the Spectre paper: |
| /// https://googleprojectzero.blogspot.com/2018/01/reading-privileged-memory-with-side.html |
| /// https://spectreattack.com/spectre.pdf |
| /// |
| //===----------------------------------------------------------------------===// |
| |
| #include "X86.h" |
| #include "X86InstrBuilder.h" |
| #include "X86InstrInfo.h" |
| #include "X86Subtarget.h" |
| #include "llvm/ADT/ArrayRef.h" |
| #include "llvm/ADT/DenseMap.h" |
| #include "llvm/ADT/Optional.h" |
| #include "llvm/ADT/STLExtras.h" |
| #include "llvm/ADT/ScopeExit.h" |
| #include "llvm/ADT/SmallPtrSet.h" |
| #include "llvm/ADT/SmallSet.h" |
| #include "llvm/ADT/SmallVector.h" |
| #include "llvm/ADT/SparseBitVector.h" |
| #include "llvm/ADT/Statistic.h" |
| #include "llvm/CodeGen/MachineBasicBlock.h" |
| #include "llvm/CodeGen/MachineConstantPool.h" |
| #include "llvm/CodeGen/MachineFunction.h" |
| #include "llvm/CodeGen/MachineFunctionPass.h" |
| #include "llvm/CodeGen/MachineInstr.h" |
| #include "llvm/CodeGen/MachineInstrBuilder.h" |
| #include "llvm/CodeGen/MachineModuleInfo.h" |
| #include "llvm/CodeGen/MachineOperand.h" |
| #include "llvm/CodeGen/MachineRegisterInfo.h" |
| #include "llvm/CodeGen/MachineSSAUpdater.h" |
| #include "llvm/CodeGen/TargetInstrInfo.h" |
| #include "llvm/CodeGen/TargetRegisterInfo.h" |
| #include "llvm/CodeGen/TargetSchedule.h" |
| #include "llvm/CodeGen/TargetSubtargetInfo.h" |
| #include "llvm/IR/DebugLoc.h" |
| #include "llvm/MC/MCSchedule.h" |
| #include "llvm/Pass.h" |
| #include "llvm/Support/CommandLine.h" |
| #include "llvm/Support/Debug.h" |
| #include "llvm/Support/raw_ostream.h" |
| #include "llvm/Target/TargetMachine.h" |
| #include <algorithm> |
| #include <cassert> |
| #include <iterator> |
| #include <utility> |
| |
| using namespace llvm; |
| |
| #define PASS_KEY "x86-slh" |
| #define DEBUG_TYPE PASS_KEY |
| |
| STATISTIC(NumCondBranchesTraced, "Number of conditional branches traced"); |
| STATISTIC(NumBranchesUntraced, "Number of branches unable to trace"); |
| STATISTIC(NumAddrRegsHardened, |
| "Number of address mode used registers hardaned"); |
| STATISTIC(NumPostLoadRegsHardened, |
| "Number of post-load register values hardened"); |
| STATISTIC(NumCallsOrJumpsHardened, |
| "Number of calls or jumps requiring extra hardening"); |
| STATISTIC(NumInstsInserted, "Number of instructions inserted"); |
| STATISTIC(NumLFENCEsInserted, "Number of lfence instructions inserted"); |
| |
| static cl::opt<bool> EnableSpeculativeLoadHardening( |
| "x86-speculative-load-hardening", |
| cl::desc("Force enable speculative load hardening"), cl::init(false), |
| cl::Hidden); |
| |
| static cl::opt<bool> HardenEdgesWithLFENCE( |
| PASS_KEY "-lfence", |
| cl::desc( |
| "Use LFENCE along each conditional edge to harden against speculative " |
| "loads rather than conditional movs and poisoned pointers."), |
| cl::init(false), cl::Hidden); |
| |
| static cl::opt<bool> EnablePostLoadHardening( |
| PASS_KEY "-post-load", |
| cl::desc("Harden the value loaded *after* it is loaded by " |
| "flushing the loaded bits to 1. This is hard to do " |
| "in general but can be done easily for GPRs."), |
| cl::init(true), cl::Hidden); |
| |
| static cl::opt<bool> FenceCallAndRet( |
| PASS_KEY "-fence-call-and-ret", |
| cl::desc("Use a full speculation fence to harden both call and ret edges " |
| "rather than a lighter weight mitigation."), |
| cl::init(false), cl::Hidden); |
| |
| static cl::opt<bool> HardenInterprocedurally( |
| PASS_KEY "-ip", |
| cl::desc("Harden interprocedurally by passing our state in and out of " |
| "functions in the high bits of the stack pointer."), |
| cl::init(true), cl::Hidden); |
| |
| static cl::opt<bool> |
| HardenLoads(PASS_KEY "-loads", |
| cl::desc("Sanitize loads from memory. When disable, no " |
| "significant security is provided."), |
| cl::init(true), cl::Hidden); |
| |
| static cl::opt<bool> HardenIndirectCallsAndJumps( |
| PASS_KEY "-indirect", |
| cl::desc("Harden indirect calls and jumps against using speculatively " |
| "stored attacker controlled addresses. This is designed to " |
| "mitigate Spectre v1.2 style attacks."), |
| cl::init(true), cl::Hidden); |
| |
| namespace { |
| |
| class X86SpeculativeLoadHardeningPass : public MachineFunctionPass { |
| public: |
| X86SpeculativeLoadHardeningPass() : MachineFunctionPass(ID) { } |
| |
| StringRef getPassName() const override { |
| return "X86 speculative load hardening"; |
| } |
| bool runOnMachineFunction(MachineFunction &MF) override; |
| void getAnalysisUsage(AnalysisUsage &AU) const override; |
| |
| /// Pass identification, replacement for typeid. |
| static char ID; |
| |
| private: |
| /// The information about a block's conditional terminators needed to trace |
| /// our predicate state through the exiting edges. |
| struct BlockCondInfo { |
| MachineBasicBlock *MBB; |
| |
| // We mostly have one conditional branch, and in extremely rare cases have |
| // two. Three and more are so rare as to be unimportant for compile time. |
| SmallVector<MachineInstr *, 2> CondBrs; |
| |
| MachineInstr *UncondBr; |
| }; |
| |
| /// Manages the predicate state traced through the program. |
| struct PredState { |
| unsigned InitialReg = 0; |
| unsigned PoisonReg = 0; |
| |
| const TargetRegisterClass *RC; |
| MachineSSAUpdater SSA; |
| |
| PredState(MachineFunction &MF, const TargetRegisterClass *RC) |
| : RC(RC), SSA(MF) {} |
| }; |
| |
| const X86Subtarget *Subtarget = nullptr; |
| MachineRegisterInfo *MRI = nullptr; |
| const X86InstrInfo *TII = nullptr; |
| const TargetRegisterInfo *TRI = nullptr; |
| |
| Optional<PredState> PS; |
| |
| void hardenEdgesWithLFENCE(MachineFunction &MF); |
| |
| SmallVector<BlockCondInfo, 16> collectBlockCondInfo(MachineFunction &MF); |
| |
| SmallVector<MachineInstr *, 16> |
| tracePredStateThroughCFG(MachineFunction &MF, ArrayRef<BlockCondInfo> Infos); |
| |
| void unfoldCallAndJumpLoads(MachineFunction &MF); |
| |
| SmallVector<MachineInstr *, 16> |
| tracePredStateThroughIndirectBranches(MachineFunction &MF); |
| |
| void tracePredStateThroughBlocksAndHarden(MachineFunction &MF); |
| |
| unsigned saveEFLAGS(MachineBasicBlock &MBB, |
| MachineBasicBlock::iterator InsertPt, DebugLoc Loc); |
| void restoreEFLAGS(MachineBasicBlock &MBB, |
| MachineBasicBlock::iterator InsertPt, DebugLoc Loc, |
| Register Reg); |
| |
| void mergePredStateIntoSP(MachineBasicBlock &MBB, |
| MachineBasicBlock::iterator InsertPt, DebugLoc Loc, |
| unsigned PredStateReg); |
| unsigned extractPredStateFromSP(MachineBasicBlock &MBB, |
| MachineBasicBlock::iterator InsertPt, |
| DebugLoc Loc); |
| |
| void |
| hardenLoadAddr(MachineInstr &MI, MachineOperand &BaseMO, |
| MachineOperand &IndexMO, |
| SmallDenseMap<unsigned, unsigned, 32> &AddrRegToHardenedReg); |
| MachineInstr * |
| sinkPostLoadHardenedInst(MachineInstr &MI, |
| SmallPtrSetImpl<MachineInstr *> &HardenedInstrs); |
| bool canHardenRegister(Register Reg); |
| unsigned hardenValueInRegister(Register Reg, MachineBasicBlock &MBB, |
| MachineBasicBlock::iterator InsertPt, |
| DebugLoc Loc); |
| unsigned hardenPostLoad(MachineInstr &MI); |
| void hardenReturnInstr(MachineInstr &MI); |
| void tracePredStateThroughCall(MachineInstr &MI); |
| void hardenIndirectCallOrJumpInstr( |
| MachineInstr &MI, |
| SmallDenseMap<unsigned, unsigned, 32> &AddrRegToHardenedReg); |
| }; |
| |
| } // end anonymous namespace |
| |
| char X86SpeculativeLoadHardeningPass::ID = 0; |
| |
| void X86SpeculativeLoadHardeningPass::getAnalysisUsage( |
| AnalysisUsage &AU) const { |
| MachineFunctionPass::getAnalysisUsage(AU); |
| } |
| |
| static MachineBasicBlock &splitEdge(MachineBasicBlock &MBB, |
| MachineBasicBlock &Succ, int SuccCount, |
| MachineInstr *Br, MachineInstr *&UncondBr, |
| const X86InstrInfo &TII) { |
| assert(!Succ.isEHPad() && "Shouldn't get edges to EH pads!"); |
| |
| MachineFunction &MF = *MBB.getParent(); |
| |
| MachineBasicBlock &NewMBB = *MF.CreateMachineBasicBlock(); |
| |
| // We have to insert the new block immediately after the current one as we |
| // don't know what layout-successor relationships the successor has and we |
| // may not be able to (and generally don't want to) try to fix those up. |
| MF.insert(std::next(MachineFunction::iterator(&MBB)), &NewMBB); |
| |
| // Update the branch instruction if necessary. |
| if (Br) { |
| assert(Br->getOperand(0).getMBB() == &Succ && |
| "Didn't start with the right target!"); |
| Br->getOperand(0).setMBB(&NewMBB); |
| |
| // If this successor was reached through a branch rather than fallthrough, |
| // we might have *broken* fallthrough and so need to inject a new |
| // unconditional branch. |
| if (!UncondBr) { |
| MachineBasicBlock &OldLayoutSucc = |
| *std::next(MachineFunction::iterator(&NewMBB)); |
| assert(MBB.isSuccessor(&OldLayoutSucc) && |
| "Without an unconditional branch, the old layout successor should " |
| "be an actual successor!"); |
| auto BrBuilder = |
| BuildMI(&MBB, DebugLoc(), TII.get(X86::JMP_1)).addMBB(&OldLayoutSucc); |
| // Update the unconditional branch now that we've added one. |
| UncondBr = &*BrBuilder; |
| } |
| |
| // Insert unconditional "jump Succ" instruction in the new block if |
| // necessary. |
| if (!NewMBB.isLayoutSuccessor(&Succ)) { |
| SmallVector<MachineOperand, 4> Cond; |
| TII.insertBranch(NewMBB, &Succ, nullptr, Cond, Br->getDebugLoc()); |
| } |
| } else { |
| assert(!UncondBr && |
| "Cannot have a branchless successor and an unconditional branch!"); |
| assert(NewMBB.isLayoutSuccessor(&Succ) && |
| "A non-branch successor must have been a layout successor before " |
| "and now is a layout successor of the new block."); |
| } |
| |
| // If this is the only edge to the successor, we can just replace it in the |
| // CFG. Otherwise we need to add a new entry in the CFG for the new |
| // successor. |
| if (SuccCount == 1) { |
| MBB.replaceSuccessor(&Succ, &NewMBB); |
| } else { |
| MBB.splitSuccessor(&Succ, &NewMBB); |
| } |
| |
| // Hook up the edge from the new basic block to the old successor in the CFG. |
| NewMBB.addSuccessor(&Succ); |
| |
| // Fix PHI nodes in Succ so they refer to NewMBB instead of MBB. |
| for (MachineInstr &MI : Succ) { |
| if (!MI.isPHI()) |
| break; |
| for (int OpIdx = 1, NumOps = MI.getNumOperands(); OpIdx < NumOps; |
| OpIdx += 2) { |
| MachineOperand &OpV = MI.getOperand(OpIdx); |
| MachineOperand &OpMBB = MI.getOperand(OpIdx + 1); |
| assert(OpMBB.isMBB() && "Block operand to a PHI is not a block!"); |
| if (OpMBB.getMBB() != &MBB) |
| continue; |
| |
| // If this is the last edge to the succesor, just replace MBB in the PHI |
| if (SuccCount == 1) { |
| OpMBB.setMBB(&NewMBB); |
| break; |
| } |
| |
| // Otherwise, append a new pair of operands for the new incoming edge. |
| MI.addOperand(MF, OpV); |
| MI.addOperand(MF, MachineOperand::CreateMBB(&NewMBB)); |
| break; |
| } |
| } |
| |
| // Inherit live-ins from the successor |
| for (auto &LI : Succ.liveins()) |
| NewMBB.addLiveIn(LI); |
| |
| LLVM_DEBUG(dbgs() << " Split edge from '" << MBB.getName() << "' to '" |
| << Succ.getName() << "'.\n"); |
| return NewMBB; |
| } |
| |
| /// Removing duplicate PHI operands to leave the PHI in a canonical and |
| /// predictable form. |
| /// |
| /// FIXME: It's really frustrating that we have to do this, but SSA-form in MIR |
| /// isn't what you might expect. We may have multiple entries in PHI nodes for |
| /// a single predecessor. This makes CFG-updating extremely complex, so here we |
| /// simplify all PHI nodes to a model even simpler than the IR's model: exactly |
| /// one entry per predecessor, regardless of how many edges there are. |
| static void canonicalizePHIOperands(MachineFunction &MF) { |
| SmallPtrSet<MachineBasicBlock *, 4> Preds; |
| SmallVector<int, 4> DupIndices; |
| for (auto &MBB : MF) |
| for (auto &MI : MBB) { |
| if (!MI.isPHI()) |
| break; |
| |
| // First we scan the operands of the PHI looking for duplicate entries |
| // a particular predecessor. We retain the operand index of each duplicate |
| // entry found. |
| for (int OpIdx = 1, NumOps = MI.getNumOperands(); OpIdx < NumOps; |
| OpIdx += 2) |
| if (!Preds.insert(MI.getOperand(OpIdx + 1).getMBB()).second) |
| DupIndices.push_back(OpIdx); |
| |
| // Now walk the duplicate indices, removing both the block and value. Note |
| // that these are stored as a vector making this element-wise removal |
| // :w |
| // potentially quadratic. |
| // |
| // FIXME: It is really frustrating that we have to use a quadratic |
| // removal algorithm here. There should be a better way, but the use-def |
| // updates required make that impossible using the public API. |
| // |
| // Note that we have to process these backwards so that we don't |
| // invalidate other indices with each removal. |
| while (!DupIndices.empty()) { |
| int OpIdx = DupIndices.pop_back_val(); |
| // Remove both the block and value operand, again in reverse order to |
| // preserve indices. |
| MI.RemoveOperand(OpIdx + 1); |
| MI.RemoveOperand(OpIdx); |
| } |
| |
| Preds.clear(); |
| } |
| } |
| |
| /// Helper to scan a function for loads vulnerable to misspeculation that we |
| /// want to harden. |
| /// |
| /// We use this to avoid making changes to functions where there is nothing we |
| /// need to do to harden against misspeculation. |
| static bool hasVulnerableLoad(MachineFunction &MF) { |
| for (MachineBasicBlock &MBB : MF) { |
| for (MachineInstr &MI : MBB) { |
| // Loads within this basic block after an LFENCE are not at risk of |
| // speculatively executing with invalid predicates from prior control |
| // flow. So break out of this block but continue scanning the function. |
| if (MI.getOpcode() == X86::LFENCE) |
| break; |
| |
| // Looking for loads only. |
| if (!MI.mayLoad()) |
| continue; |
| |
| // An MFENCE is modeled as a load but isn't vulnerable to misspeculation. |
| if (MI.getOpcode() == X86::MFENCE) |
| continue; |
| |
| // We found a load. |
| return true; |
| } |
| } |
| |
| // No loads found. |
| return false; |
| } |
| |
| bool X86SpeculativeLoadHardeningPass::runOnMachineFunction( |
| MachineFunction &MF) { |
| LLVM_DEBUG(dbgs() << "********** " << getPassName() << " : " << MF.getName() |
| << " **********\n"); |
| |
| // Only run if this pass is forced enabled or we detect the relevant function |
| // attribute requesting SLH. |
| if (!EnableSpeculativeLoadHardening && |
| !MF.getFunction().hasFnAttribute(Attribute::SpeculativeLoadHardening)) |
| return false; |
| |
| Subtarget = &MF.getSubtarget<X86Subtarget>(); |
| MRI = &MF.getRegInfo(); |
| TII = Subtarget->getInstrInfo(); |
| TRI = Subtarget->getRegisterInfo(); |
| |
| // FIXME: Support for 32-bit. |
| PS.emplace(MF, &X86::GR64_NOSPRegClass); |
| |
| if (MF.begin() == MF.end()) |
| // Nothing to do for a degenerate empty function... |
| return false; |
| |
| // We support an alternative hardening technique based on a debug flag. |
| if (HardenEdgesWithLFENCE) { |
| hardenEdgesWithLFENCE(MF); |
| return true; |
| } |
| |
| // Create a dummy debug loc to use for all the generated code here. |
| DebugLoc Loc; |
| |
| MachineBasicBlock &Entry = *MF.begin(); |
| auto EntryInsertPt = Entry.SkipPHIsLabelsAndDebug(Entry.begin()); |
| |
| // Do a quick scan to see if we have any checkable loads. |
| bool HasVulnerableLoad = hasVulnerableLoad(MF); |
| |
| // See if we have any conditional branching blocks that we will need to trace |
| // predicate state through. |
| SmallVector<BlockCondInfo, 16> Infos = collectBlockCondInfo(MF); |
| |
| // If we have no interesting conditions or loads, nothing to do here. |
| if (!HasVulnerableLoad && Infos.empty()) |
| return true; |
| |
| // The poison value is required to be an all-ones value for many aspects of |
| // this mitigation. |
| const int PoisonVal = -1; |
| PS->PoisonReg = MRI->createVirtualRegister(PS->RC); |
| BuildMI(Entry, EntryInsertPt, Loc, TII->get(X86::MOV64ri32), PS->PoisonReg) |
| .addImm(PoisonVal); |
| ++NumInstsInserted; |
| |
| // If we have loads being hardened and we've asked for call and ret edges to |
| // get a full fence-based mitigation, inject that fence. |
| if (HasVulnerableLoad && FenceCallAndRet) { |
| // We need to insert an LFENCE at the start of the function to suspend any |
| // incoming misspeculation from the caller. This helps two-fold: the caller |
| // may not have been protected as this code has been, and this code gets to |
| // not take any specific action to protect across calls. |
| // FIXME: We could skip this for functions which unconditionally return |
| // a constant. |
| BuildMI(Entry, EntryInsertPt, Loc, TII->get(X86::LFENCE)); |
| ++NumInstsInserted; |
| ++NumLFENCEsInserted; |
| } |
| |
| // If we guarded the entry with an LFENCE and have no conditionals to protect |
| // in blocks, then we're done. |
| if (FenceCallAndRet && Infos.empty()) |
| // We may have changed the function's code at this point to insert fences. |
| return true; |
| |
| // For every basic block in the function which can b |
| if (HardenInterprocedurally && !FenceCallAndRet) { |
| // Set up the predicate state by extracting it from the incoming stack |
| // pointer so we pick up any misspeculation in our caller. |
| PS->InitialReg = extractPredStateFromSP(Entry, EntryInsertPt, Loc); |
| } else { |
| // Otherwise, just build the predicate state itself by zeroing a register |
| // as we don't need any initial state. |
| PS->InitialReg = MRI->createVirtualRegister(PS->RC); |
| Register PredStateSubReg = MRI->createVirtualRegister(&X86::GR32RegClass); |
| auto ZeroI = BuildMI(Entry, EntryInsertPt, Loc, TII->get(X86::MOV32r0), |
| PredStateSubReg); |
| ++NumInstsInserted; |
| MachineOperand *ZeroEFLAGSDefOp = |
| ZeroI->findRegisterDefOperand(X86::EFLAGS); |
| assert(ZeroEFLAGSDefOp && ZeroEFLAGSDefOp->isImplicit() && |
| "Must have an implicit def of EFLAGS!"); |
| ZeroEFLAGSDefOp->setIsDead(true); |
| BuildMI(Entry, EntryInsertPt, Loc, TII->get(X86::SUBREG_TO_REG), |
| PS->InitialReg) |
| .addImm(0) |
| .addReg(PredStateSubReg) |
| .addImm(X86::sub_32bit); |
| } |
| |
| // We're going to need to trace predicate state throughout the function's |
| // CFG. Prepare for this by setting up our initial state of PHIs with unique |
| // predecessor entries and all the initial predicate state. |
| canonicalizePHIOperands(MF); |
| |
| // Track the updated values in an SSA updater to rewrite into SSA form at the |
| // end. |
| PS->SSA.Initialize(PS->InitialReg); |
| PS->SSA.AddAvailableValue(&Entry, PS->InitialReg); |
| |
| // Trace through the CFG. |
| auto CMovs = tracePredStateThroughCFG(MF, Infos); |
| |
| // We may also enter basic blocks in this function via exception handling |
| // control flow. Here, if we are hardening interprocedurally, we need to |
| // re-capture the predicate state from the throwing code. In the Itanium ABI, |
| // the throw will always look like a call to __cxa_throw and will have the |
| // predicate state in the stack pointer, so extract fresh predicate state from |
| // the stack pointer and make it available in SSA. |
| // FIXME: Handle non-itanium ABI EH models. |
| if (HardenInterprocedurally) { |
| for (MachineBasicBlock &MBB : MF) { |
| assert(!MBB.isEHScopeEntry() && "Only Itanium ABI EH supported!"); |
| assert(!MBB.isEHFuncletEntry() && "Only Itanium ABI EH supported!"); |
| assert(!MBB.isCleanupFuncletEntry() && "Only Itanium ABI EH supported!"); |
| if (!MBB.isEHPad()) |
| continue; |
| PS->SSA.AddAvailableValue( |
| &MBB, |
| extractPredStateFromSP(MBB, MBB.SkipPHIsAndLabels(MBB.begin()), Loc)); |
| } |
| } |
| |
| if (HardenIndirectCallsAndJumps) { |
| // If we are going to harden calls and jumps we need to unfold their memory |
| // operands. |
| unfoldCallAndJumpLoads(MF); |
| |
| // Then we trace predicate state through the indirect branches. |
| auto IndirectBrCMovs = tracePredStateThroughIndirectBranches(MF); |
| CMovs.append(IndirectBrCMovs.begin(), IndirectBrCMovs.end()); |
| } |
| |
| // Now that we have the predicate state available at the start of each block |
| // in the CFG, trace it through each block, hardening vulnerable instructions |
| // as we go. |
| tracePredStateThroughBlocksAndHarden(MF); |
| |
| // Now rewrite all the uses of the pred state using the SSA updater to insert |
| // PHIs connecting the state between blocks along the CFG edges. |
| for (MachineInstr *CMovI : CMovs) |
| for (MachineOperand &Op : CMovI->operands()) { |
| if (!Op.isReg() || Op.getReg() != PS->InitialReg) |
| continue; |
| |
| PS->SSA.RewriteUse(Op); |
| } |
| |
| LLVM_DEBUG(dbgs() << "Final speculative load hardened function:\n"; MF.dump(); |
| dbgs() << "\n"; MF.verify(this)); |
| return true; |
| } |
| |
| /// Implements the naive hardening approach of putting an LFENCE after every |
| /// potentially mis-predicted control flow construct. |
| /// |
| /// We include this as an alternative mostly for the purpose of comparison. The |
| /// performance impact of this is expected to be extremely severe and not |
| /// practical for any real-world users. |
| void X86SpeculativeLoadHardeningPass::hardenEdgesWithLFENCE( |
| MachineFunction &MF) { |
| // First, we scan the function looking for blocks that are reached along edges |
| // that we might want to harden. |
| SmallSetVector<MachineBasicBlock *, 8> Blocks; |
| for (MachineBasicBlock &MBB : MF) { |
| // If there are no or only one successor, nothing to do here. |
| if (MBB.succ_size() <= 1) |
| continue; |
| |
| // Skip blocks unless their terminators start with a branch. Other |
| // terminators don't seem interesting for guarding against misspeculation. |
| auto TermIt = MBB.getFirstTerminator(); |
| if (TermIt == MBB.end() || !TermIt->isBranch()) |
| continue; |
| |
| // Add all the non-EH-pad succossors to the blocks we want to harden. We |
| // skip EH pads because there isn't really a condition of interest on |
| // entering. |
| for (MachineBasicBlock *SuccMBB : MBB.successors()) |
| if (!SuccMBB->isEHPad()) |
| Blocks.insert(SuccMBB); |
| } |
| |
| for (MachineBasicBlock *MBB : Blocks) { |
| auto InsertPt = MBB->SkipPHIsAndLabels(MBB->begin()); |
| BuildMI(*MBB, InsertPt, DebugLoc(), TII->get(X86::LFENCE)); |
| ++NumInstsInserted; |
| ++NumLFENCEsInserted; |
| } |
| } |
| |
| SmallVector<X86SpeculativeLoadHardeningPass::BlockCondInfo, 16> |
| X86SpeculativeLoadHardeningPass::collectBlockCondInfo(MachineFunction &MF) { |
| SmallVector<BlockCondInfo, 16> Infos; |
| |
| // Walk the function and build up a summary for each block's conditions that |
| // we need to trace through. |
| for (MachineBasicBlock &MBB : MF) { |
| // If there are no or only one successor, nothing to do here. |
| if (MBB.succ_size() <= 1) |
| continue; |
| |
| // We want to reliably handle any conditional branch terminators in the |
| // MBB, so we manually analyze the branch. We can handle all of the |
| // permutations here, including ones that analyze branch cannot. |
| // |
| // The approach is to walk backwards across the terminators, resetting at |
| // any unconditional non-indirect branch, and track all conditional edges |
| // to basic blocks as well as the fallthrough or unconditional successor |
| // edge. For each conditional edge, we track the target and the opposite |
| // condition code in order to inject a "no-op" cmov into that successor |
| // that will harden the predicate. For the fallthrough/unconditional |
| // edge, we inject a separate cmov for each conditional branch with |
| // matching condition codes. This effectively implements an "and" of the |
| // condition flags, even if there isn't a single condition flag that would |
| // directly implement that. We don't bother trying to optimize either of |
| // these cases because if such an optimization is possible, LLVM should |
| // have optimized the conditional *branches* in that way already to reduce |
| // instruction count. This late, we simply assume the minimal number of |
| // branch instructions is being emitted and use that to guide our cmov |
| // insertion. |
| |
| BlockCondInfo Info = {&MBB, {}, nullptr}; |
| |
| // Now walk backwards through the terminators and build up successors they |
| // reach and the conditions. |
| for (MachineInstr &MI : llvm::reverse(MBB)) { |
| // Once we've handled all the terminators, we're done. |
| if (!MI.isTerminator()) |
| break; |
| |
| // If we see a non-branch terminator, we can't handle anything so bail. |
| if (!MI.isBranch()) { |
| Info.CondBrs.clear(); |
| break; |
| } |
| |
| // If we see an unconditional branch, reset our state, clear any |
| // fallthrough, and set this is the "else" successor. |
| if (MI.getOpcode() == X86::JMP_1) { |
| Info.CondBrs.clear(); |
| Info.UncondBr = &MI; |
| continue; |
| } |
| |
| // If we get an invalid condition, we have an indirect branch or some |
| // other unanalyzable "fallthrough" case. We model this as a nullptr for |
| // the destination so we can still guard any conditional successors. |
| // Consider code sequences like: |
| // ``` |
| // jCC L1 |
| // jmpq *%rax |
| // ``` |
| // We still want to harden the edge to `L1`. |
| if (X86::getCondFromBranch(MI) == X86::COND_INVALID) { |
| Info.CondBrs.clear(); |
| Info.UncondBr = &MI; |
| continue; |
| } |
| |
| // We have a vanilla conditional branch, add it to our list. |
| Info.CondBrs.push_back(&MI); |
| } |
| if (Info.CondBrs.empty()) { |
| ++NumBranchesUntraced; |
| LLVM_DEBUG(dbgs() << "WARNING: unable to secure successors of block:\n"; |
| MBB.dump()); |
| continue; |
| } |
| |
| Infos.push_back(Info); |
| } |
| |
| return Infos; |
| } |
| |
| /// Trace the predicate state through the CFG, instrumenting each conditional |
| /// branch such that misspeculation through an edge will poison the predicate |
| /// state. |
| /// |
| /// Returns the list of inserted CMov instructions so that they can have their |
| /// uses of the predicate state rewritten into proper SSA form once it is |
| /// complete. |
| SmallVector<MachineInstr *, 16> |
| X86SpeculativeLoadHardeningPass::tracePredStateThroughCFG( |
| MachineFunction &MF, ArrayRef<BlockCondInfo> Infos) { |
| // Collect the inserted cmov instructions so we can rewrite their uses of the |
| // predicate state into SSA form. |
| SmallVector<MachineInstr *, 16> CMovs; |
| |
| // Now walk all of the basic blocks looking for ones that end in conditional |
| // jumps where we need to update this register along each edge. |
| for (const BlockCondInfo &Info : Infos) { |
| MachineBasicBlock &MBB = *Info.MBB; |
| const SmallVectorImpl<MachineInstr *> &CondBrs = Info.CondBrs; |
| MachineInstr *UncondBr = Info.UncondBr; |
| |
| LLVM_DEBUG(dbgs() << "Tracing predicate through block: " << MBB.getName() |
| << "\n"); |
| ++NumCondBranchesTraced; |
| |
| // Compute the non-conditional successor as either the target of any |
| // unconditional branch or the layout successor. |
| MachineBasicBlock *UncondSucc = |
| UncondBr ? (UncondBr->getOpcode() == X86::JMP_1 |
| ? UncondBr->getOperand(0).getMBB() |
| : nullptr) |
| : &*std::next(MachineFunction::iterator(&MBB)); |
| |
| // Count how many edges there are to any given successor. |
| SmallDenseMap<MachineBasicBlock *, int> SuccCounts; |
| if (UncondSucc) |
| ++SuccCounts[UncondSucc]; |
| for (auto *CondBr : CondBrs) |
| ++SuccCounts[CondBr->getOperand(0).getMBB()]; |
| |
| // A lambda to insert cmov instructions into a block checking all of the |
| // condition codes in a sequence. |
| auto BuildCheckingBlockForSuccAndConds = |
| [&](MachineBasicBlock &MBB, MachineBasicBlock &Succ, int SuccCount, |
| MachineInstr *Br, MachineInstr *&UncondBr, |
| ArrayRef<X86::CondCode> Conds) { |
| // First, we split the edge to insert the checking block into a safe |
| // location. |
| auto &CheckingMBB = |
| (SuccCount == 1 && Succ.pred_size() == 1) |
| ? Succ |
| : splitEdge(MBB, Succ, SuccCount, Br, UncondBr, *TII); |
| |
| bool LiveEFLAGS = Succ.isLiveIn(X86::EFLAGS); |
| if (!LiveEFLAGS) |
| CheckingMBB.addLiveIn(X86::EFLAGS); |
| |
| // Now insert the cmovs to implement the checks. |
| auto InsertPt = CheckingMBB.begin(); |
| assert((InsertPt == CheckingMBB.end() || !InsertPt->isPHI()) && |
| "Should never have a PHI in the initial checking block as it " |
| "always has a single predecessor!"); |
| |
| // We will wire each cmov to each other, but need to start with the |
| // incoming pred state. |
| unsigned CurStateReg = PS->InitialReg; |
| |
| for (X86::CondCode Cond : Conds) { |
| int PredStateSizeInBytes = TRI->getRegSizeInBits(*PS->RC) / 8; |
| auto CMovOp = X86::getCMovOpcode(PredStateSizeInBytes); |
| |
| Register UpdatedStateReg = MRI->createVirtualRegister(PS->RC); |
| // Note that we intentionally use an empty debug location so that |
| // this picks up the preceding location. |
| auto CMovI = BuildMI(CheckingMBB, InsertPt, DebugLoc(), |
| TII->get(CMovOp), UpdatedStateReg) |
| .addReg(CurStateReg) |
| .addReg(PS->PoisonReg) |
| .addImm(Cond); |
| // If this is the last cmov and the EFLAGS weren't originally |
| // live-in, mark them as killed. |
| if (!LiveEFLAGS && Cond == Conds.back()) |
| CMovI->findRegisterUseOperand(X86::EFLAGS)->setIsKill(true); |
| |
| ++NumInstsInserted; |
| LLVM_DEBUG(dbgs() << " Inserting cmov: "; CMovI->dump(); |
| dbgs() << "\n"); |
| |
| // The first one of the cmovs will be using the top level |
| // `PredStateReg` and need to get rewritten into SSA form. |
| if (CurStateReg == PS->InitialReg) |
| CMovs.push_back(&*CMovI); |
| |
| // The next cmov should start from this one's def. |
| CurStateReg = UpdatedStateReg; |
| } |
| |
| // And put the last one into the available values for SSA form of our |
| // predicate state. |
| PS->SSA.AddAvailableValue(&CheckingMBB, CurStateReg); |
| }; |
| |
| std::vector<X86::CondCode> UncondCodeSeq; |
| for (auto *CondBr : CondBrs) { |
| MachineBasicBlock &Succ = *CondBr->getOperand(0).getMBB(); |
| int &SuccCount = SuccCounts[&Succ]; |
| |
| X86::CondCode Cond = X86::getCondFromBranch(*CondBr); |
| X86::CondCode InvCond = X86::GetOppositeBranchCondition(Cond); |
| UncondCodeSeq.push_back(Cond); |
| |
| BuildCheckingBlockForSuccAndConds(MBB, Succ, SuccCount, CondBr, UncondBr, |
| {InvCond}); |
| |
| // Decrement the successor count now that we've split one of the edges. |
| // We need to keep the count of edges to the successor accurate in order |
| // to know above when to *replace* the successor in the CFG vs. just |
| // adding the new successor. |
| --SuccCount; |
| } |
| |
| // Since we may have split edges and changed the number of successors, |
| // normalize the probabilities. This avoids doing it each time we split an |
| // edge. |
| MBB.normalizeSuccProbs(); |
| |
| // Finally, we need to insert cmovs into the "fallthrough" edge. Here, we |
| // need to intersect the other condition codes. We can do this by just |
| // doing a cmov for each one. |
| if (!UncondSucc) |
| // If we have no fallthrough to protect (perhaps it is an indirect jump?) |
| // just skip this and continue. |
| continue; |
| |
| assert(SuccCounts[UncondSucc] == 1 && |
| "We should never have more than one edge to the unconditional " |
| "successor at this point because every other edge must have been " |
| "split above!"); |
| |
| // Sort and unique the codes to minimize them. |
| llvm::sort(UncondCodeSeq); |
| UncondCodeSeq.erase(std::unique(UncondCodeSeq.begin(), UncondCodeSeq.end()), |
| UncondCodeSeq.end()); |
| |
| // Build a checking version of the successor. |
| BuildCheckingBlockForSuccAndConds(MBB, *UncondSucc, /*SuccCount*/ 1, |
| UncondBr, UncondBr, UncondCodeSeq); |
| } |
| |
| return CMovs; |
| } |
| |
| /// Compute the register class for the unfolded load. |
| /// |
| /// FIXME: This should probably live in X86InstrInfo, potentially by adding |
| /// a way to unfold into a newly created vreg rather than requiring a register |
| /// input. |
| static const TargetRegisterClass * |
| getRegClassForUnfoldedLoad(MachineFunction &MF, const X86InstrInfo &TII, |
| unsigned Opcode) { |
| unsigned Index; |
| unsigned UnfoldedOpc = TII.getOpcodeAfterMemoryUnfold( |
| Opcode, /*UnfoldLoad*/ true, /*UnfoldStore*/ false, &Index); |
| const MCInstrDesc &MCID = TII.get(UnfoldedOpc); |
| return TII.getRegClass(MCID, Index, &TII.getRegisterInfo(), MF); |
| } |
| |
| void X86SpeculativeLoadHardeningPass::unfoldCallAndJumpLoads( |
| MachineFunction &MF) { |
| for (MachineBasicBlock &MBB : MF) |
| // We use make_early_inc_range here so we can remove instructions if needed |
| // without disturbing the iteration. |
| for (MachineInstr &MI : llvm::make_early_inc_range(MBB.instrs())) { |
| // Must either be a call or a branch. |
| if (!MI.isCall() && !MI.isBranch()) |
| continue; |
| // We only care about loading variants of these instructions. |
| if (!MI.mayLoad()) |
| continue; |
| |
| switch (MI.getOpcode()) { |
| default: { |
| LLVM_DEBUG( |
| dbgs() << "ERROR: Found an unexpected loading branch or call " |
| "instruction:\n"; |
| MI.dump(); dbgs() << "\n"); |
| report_fatal_error("Unexpected loading branch or call!"); |
| } |
| |
| case X86::FARCALL16m: |
| case X86::FARCALL32m: |
| case X86::FARCALL64m: |
| case X86::FARJMP16m: |
| case X86::FARJMP32m: |
| case X86::FARJMP64m: |
| // We cannot mitigate far jumps or calls, but we also don't expect them |
| // to be vulnerable to Spectre v1.2 style attacks. |
| continue; |
| |
| case X86::CALL16m: |
| case X86::CALL16m_NT: |
| case X86::CALL32m: |
| case X86::CALL32m_NT: |
| case X86::CALL64m: |
| case X86::CALL64m_NT: |
| case X86::JMP16m: |
| case X86::JMP16m_NT: |
| case X86::JMP32m: |
| case X86::JMP32m_NT: |
| case X86::JMP64m: |
| case X86::JMP64m_NT: |
| case X86::TAILJMPm64: |
| case X86::TAILJMPm64_REX: |
| case X86::TAILJMPm: |
| case X86::TCRETURNmi64: |
| case X86::TCRETURNmi: { |
| // Use the generic unfold logic now that we know we're dealing with |
| // expected instructions. |
| // FIXME: We don't have test coverage for all of these! |
| auto *UnfoldedRC = getRegClassForUnfoldedLoad(MF, *TII, MI.getOpcode()); |
| if (!UnfoldedRC) { |
| LLVM_DEBUG(dbgs() |
| << "ERROR: Unable to unfold load from instruction:\n"; |
| MI.dump(); dbgs() << "\n"); |
| report_fatal_error("Unable to unfold load!"); |
| } |
| Register Reg = MRI->createVirtualRegister(UnfoldedRC); |
| SmallVector<MachineInstr *, 2> NewMIs; |
| // If we were able to compute an unfolded reg class, any failure here |
| // is just a programming error so just assert. |
| bool Unfolded = |
| TII->unfoldMemoryOperand(MF, MI, Reg, /*UnfoldLoad*/ true, |
| /*UnfoldStore*/ false, NewMIs); |
| (void)Unfolded; |
| assert(Unfolded && |
| "Computed unfolded register class but failed to unfold"); |
| // Now stitch the new instructions into place and erase the old one. |
| for (auto *NewMI : NewMIs) |
| MBB.insert(MI.getIterator(), NewMI); |
| |
| // Update the call site info. |
| if (MI.isCandidateForCallSiteEntry()) |
| MF.eraseCallSiteInfo(&MI); |
| |
| MI.eraseFromParent(); |
| LLVM_DEBUG({ |
| dbgs() << "Unfolded load successfully into:\n"; |
| for (auto *NewMI : NewMIs) { |
| NewMI->dump(); |
| dbgs() << "\n"; |
| } |
| }); |
| continue; |
| } |
| } |
| llvm_unreachable("Escaped switch with default!"); |
| } |
| } |
| |
| /// Trace the predicate state through indirect branches, instrumenting them to |
| /// poison the state if a target is reached that does not match the expected |
| /// target. |
| /// |
| /// This is designed to mitigate Spectre variant 1 attacks where an indirect |
| /// branch is trained to predict a particular target and then mispredicts that |
| /// target in a way that can leak data. Despite using an indirect branch, this |
| /// is really a variant 1 style attack: it does not steer execution to an |
| /// arbitrary or attacker controlled address, and it does not require any |
| /// special code executing next to the victim. This attack can also be mitigated |
| /// through retpolines, but those require either replacing indirect branches |
| /// with conditional direct branches or lowering them through a device that |
| /// blocks speculation. This mitigation can replace these retpoline-style |
| /// mitigations for jump tables and other indirect branches within a function |
| /// when variant 2 isn't a risk while allowing limited speculation. Indirect |
| /// calls, however, cannot be mitigated through this technique without changing |
| /// the ABI in a fundamental way. |
| SmallVector<MachineInstr *, 16> |
| X86SpeculativeLoadHardeningPass::tracePredStateThroughIndirectBranches( |
| MachineFunction &MF) { |
| // We use the SSAUpdater to insert PHI nodes for the target addresses of |
| // indirect branches. We don't actually need the full power of the SSA updater |
| // in this particular case as we always have immediately available values, but |
| // this avoids us having to re-implement the PHI construction logic. |
| MachineSSAUpdater TargetAddrSSA(MF); |
| TargetAddrSSA.Initialize(MRI->createVirtualRegister(&X86::GR64RegClass)); |
| |
| // Track which blocks were terminated with an indirect branch. |
| SmallPtrSet<MachineBasicBlock *, 4> IndirectTerminatedMBBs; |
| |
| // We need to know what blocks end up reached via indirect branches. We |
| // expect this to be a subset of those whose address is taken and so track it |
| // directly via the CFG. |
| SmallPtrSet<MachineBasicBlock *, 4> IndirectTargetMBBs; |
| |
| // Walk all the blocks which end in an indirect branch and make the |
| // target address available. |
| for (MachineBasicBlock &MBB : MF) { |
| // Find the last terminator. |
| auto MII = MBB.instr_rbegin(); |
| while (MII != MBB.instr_rend() && MII->isDebugInstr()) |
| ++MII; |
| if (MII == MBB.instr_rend()) |
| continue; |
| MachineInstr &TI = *MII; |
| if (!TI.isTerminator() || !TI.isBranch()) |
| // No terminator or non-branch terminator. |
| continue; |
| |
| unsigned TargetReg; |
| |
| switch (TI.getOpcode()) { |
| default: |
| // Direct branch or conditional branch (leading to fallthrough). |
| continue; |
| |
| case X86::FARJMP16m: |
| case X86::FARJMP32m: |
| case X86::FARJMP64m: |
| // We cannot mitigate far jumps or calls, but we also don't expect them |
| // to be vulnerable to Spectre v1.2 or v2 (self trained) style attacks. |
| continue; |
| |
| case X86::JMP16m: |
| case X86::JMP16m_NT: |
| case X86::JMP32m: |
| case X86::JMP32m_NT: |
| case X86::JMP64m: |
| case X86::JMP64m_NT: |
| // Mostly as documentation. |
| report_fatal_error("Memory operand jumps should have been unfolded!"); |
| |
| case X86::JMP16r: |
| report_fatal_error( |
| "Support for 16-bit indirect branches is not implemented."); |
| case X86::JMP32r: |
| report_fatal_error( |
| "Support for 32-bit indirect branches is not implemented."); |
| |
| case X86::JMP64r: |
| TargetReg = TI.getOperand(0).getReg(); |
| } |
| |
| // We have definitely found an indirect branch. Verify that there are no |
| // preceding conditional branches as we don't yet support that. |
| if (llvm::any_of(MBB.terminators(), [&](MachineInstr &OtherTI) { |
| return !OtherTI.isDebugInstr() && &OtherTI != &TI; |
| })) { |
| LLVM_DEBUG({ |
| dbgs() << "ERROR: Found other terminators in a block with an indirect " |
| "branch! This is not yet supported! Terminator sequence:\n"; |
| for (MachineInstr &MI : MBB.terminators()) { |
| MI.dump(); |
| dbgs() << '\n'; |
| } |
| }); |
| report_fatal_error("Unimplemented terminator sequence!"); |
| } |
| |
| // Make the target register an available value for this block. |
| TargetAddrSSA.AddAvailableValue(&MBB, TargetReg); |
| IndirectTerminatedMBBs.insert(&MBB); |
| |
| // Add all the successors to our target candidates. |
| for (MachineBasicBlock *Succ : MBB.successors()) |
| IndirectTargetMBBs.insert(Succ); |
| } |
| |
| // Keep track of the cmov instructions we insert so we can return them. |
| SmallVector<MachineInstr *, 16> CMovs; |
| |
| // If we didn't find any indirect branches with targets, nothing to do here. |
| if (IndirectTargetMBBs.empty()) |
| return CMovs; |
| |
| // We found indirect branches and targets that need to be instrumented to |
| // harden loads within them. Walk the blocks of the function (to get a stable |
| // ordering) and instrument each target of an indirect branch. |
| for (MachineBasicBlock &MBB : MF) { |
| // Skip the blocks that aren't candidate targets. |
| if (!IndirectTargetMBBs.count(&MBB)) |
| continue; |
| |
| // We don't expect EH pads to ever be reached via an indirect branch. If |
| // this is desired for some reason, we could simply skip them here rather |
| // than asserting. |
| assert(!MBB.isEHPad() && |
| "Unexpected EH pad as target of an indirect branch!"); |
| |
| // We should never end up threading EFLAGS into a block to harden |
| // conditional jumps as there would be an additional successor via the |
| // indirect branch. As a consequence, all such edges would be split before |
| // reaching here, and the inserted block will handle the EFLAGS-based |
| // hardening. |
| assert(!MBB.isLiveIn(X86::EFLAGS) && |
| "Cannot check within a block that already has live-in EFLAGS!"); |
| |
| // We can't handle having non-indirect edges into this block unless this is |
| // the only successor and we can synthesize the necessary target address. |
| for (MachineBasicBlock *Pred : MBB.predecessors()) { |
| // If we've already handled this by extracting the target directly, |
| // nothing to do. |
| if (IndirectTerminatedMBBs.count(Pred)) |
| continue; |
| |
| // Otherwise, we have to be the only successor. We generally expect this |
| // to be true as conditional branches should have had a critical edge |
| // split already. We don't however need to worry about EH pad successors |
| // as they'll happily ignore the target and their hardening strategy is |
| // resilient to all ways in which they could be reached speculatively. |
| if (!llvm::all_of(Pred->successors(), [&](MachineBasicBlock *Succ) { |
| return Succ->isEHPad() || Succ == &MBB; |
| })) { |
| LLVM_DEBUG({ |
| dbgs() << "ERROR: Found conditional entry to target of indirect " |
| "branch!\n"; |
| Pred->dump(); |
| MBB.dump(); |
| }); |
| report_fatal_error("Cannot harden a conditional entry to a target of " |
| "an indirect branch!"); |
| } |
| |
| // Now we need to compute the address of this block and install it as a |
| // synthetic target in the predecessor. We do this at the bottom of the |
| // predecessor. |
| auto InsertPt = Pred->getFirstTerminator(); |
| Register TargetReg = MRI->createVirtualRegister(&X86::GR64RegClass); |
| if (MF.getTarget().getCodeModel() == CodeModel::Small && |
| !Subtarget->isPositionIndependent()) { |
| // Directly materialize it into an immediate. |
| auto AddrI = BuildMI(*Pred, InsertPt, DebugLoc(), |
| TII->get(X86::MOV64ri32), TargetReg) |
| .addMBB(&MBB); |
| ++NumInstsInserted; |
| (void)AddrI; |
| LLVM_DEBUG(dbgs() << " Inserting mov: "; AddrI->dump(); |
| dbgs() << "\n"); |
| } else { |
| auto AddrI = BuildMI(*Pred, InsertPt, DebugLoc(), TII->get(X86::LEA64r), |
| TargetReg) |
| .addReg(/*Base*/ X86::RIP) |
| .addImm(/*Scale*/ 1) |
| .addReg(/*Index*/ 0) |
| .addMBB(&MBB) |
| .addReg(/*Segment*/ 0); |
| ++NumInstsInserted; |
| (void)AddrI; |
| LLVM_DEBUG(dbgs() << " Inserting lea: "; AddrI->dump(); |
| dbgs() << "\n"); |
| } |
| // And make this available. |
| TargetAddrSSA.AddAvailableValue(Pred, TargetReg); |
| } |
| |
| // Materialize the needed SSA value of the target. Note that we need the |
| // middle of the block as this block might at the bottom have an indirect |
| // branch back to itself. We can do this here because at this point, every |
| // predecessor of this block has an available value. This is basically just |
| // automating the construction of a PHI node for this target. |
| unsigned TargetReg = TargetAddrSSA.GetValueInMiddleOfBlock(&MBB); |
| |
| // Insert a comparison of the incoming target register with this block's |
| // address. This also requires us to mark the block as having its address |
| // taken explicitly. |
| MBB.setHasAddressTaken(); |
| auto InsertPt = MBB.SkipPHIsLabelsAndDebug(MBB.begin()); |
| if (MF.getTarget().getCodeModel() == CodeModel::Small && |
| !Subtarget->isPositionIndependent()) { |
| // Check directly against a relocated immediate when we can. |
| auto CheckI = BuildMI(MBB, InsertPt, DebugLoc(), TII->get(X86::CMP64ri32)) |
| .addReg(TargetReg, RegState::Kill) |
| .addMBB(&MBB); |
| ++NumInstsInserted; |
| (void)CheckI; |
| LLVM_DEBUG(dbgs() << " Inserting cmp: "; CheckI->dump(); dbgs() << "\n"); |
| } else { |
| // Otherwise compute the address into a register first. |
| Register AddrReg = MRI->createVirtualRegister(&X86::GR64RegClass); |
| auto AddrI = |
| BuildMI(MBB, InsertPt, DebugLoc(), TII->get(X86::LEA64r), AddrReg) |
| .addReg(/*Base*/ X86::RIP) |
| .addImm(/*Scale*/ 1) |
| .addReg(/*Index*/ 0) |
| .addMBB(&MBB) |
| .addReg(/*Segment*/ 0); |
| ++NumInstsInserted; |
| (void)AddrI; |
| LLVM_DEBUG(dbgs() << " Inserting lea: "; AddrI->dump(); dbgs() << "\n"); |
| auto CheckI = BuildMI(MBB, InsertPt, DebugLoc(), TII->get(X86::CMP64rr)) |
| .addReg(TargetReg, RegState::Kill) |
| .addReg(AddrReg, RegState::Kill); |
| ++NumInstsInserted; |
| (void)CheckI; |
| LLVM_DEBUG(dbgs() << " Inserting cmp: "; CheckI->dump(); dbgs() << "\n"); |
| } |
| |
| // Now cmov over the predicate if the comparison wasn't equal. |
| int PredStateSizeInBytes = TRI->getRegSizeInBits(*PS->RC) / 8; |
| auto CMovOp = X86::getCMovOpcode(PredStateSizeInBytes); |
| Register UpdatedStateReg = MRI->createVirtualRegister(PS->RC); |
| auto CMovI = |
| BuildMI(MBB, InsertPt, DebugLoc(), TII->get(CMovOp), UpdatedStateReg) |
| .addReg(PS->InitialReg) |
| .addReg(PS->PoisonReg) |
| .addImm(X86::COND_NE); |
| CMovI->findRegisterUseOperand(X86::EFLAGS)->setIsKill(true); |
| ++NumInstsInserted; |
| LLVM_DEBUG(dbgs() << " Inserting cmov: "; CMovI->dump(); dbgs() << "\n"); |
| CMovs.push_back(&*CMovI); |
| |
| // And put the new value into the available values for SSA form of our |
| // predicate state. |
| PS->SSA.AddAvailableValue(&MBB, UpdatedStateReg); |
| } |
| |
| // Return all the newly inserted cmov instructions of the predicate state. |
| return CMovs; |
| } |
| |
| // Returns true if the MI has EFLAGS as a register def operand and it's live, |
| // otherwise it returns false |
| static bool isEFLAGSDefLive(const MachineInstr &MI) { |
| if (const MachineOperand *DefOp = MI.findRegisterDefOperand(X86::EFLAGS)) { |
| return !DefOp->isDead(); |
| } |
| return false; |
| } |
| |
| static bool isEFLAGSLive(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, |
| const TargetRegisterInfo &TRI) { |
| // Check if EFLAGS are alive by seeing if there is a def of them or they |
| // live-in, and then seeing if that def is in turn used. |
| for (MachineInstr &MI : llvm::reverse(llvm::make_range(MBB.begin(), I))) { |
| if (MachineOperand *DefOp = MI.findRegisterDefOperand(X86::EFLAGS)) { |
| // If the def is dead, then EFLAGS is not live. |
| if (DefOp->isDead()) |
| return false; |
| |
| // Otherwise we've def'ed it, and it is live. |
| return true; |
| } |
| // While at this instruction, also check if we use and kill EFLAGS |
| // which means it isn't live. |
| if (MI.killsRegister(X86::EFLAGS, &TRI)) |
| return false; |
| } |
| |
| // If we didn't find anything conclusive (neither definitely alive or |
| // definitely dead) return whether it lives into the block. |
| return MBB.isLiveIn(X86::EFLAGS); |
| } |
| |
| /// Trace the predicate state through each of the blocks in the function, |
| /// hardening everything necessary along the way. |
| /// |
| /// We call this routine once the initial predicate state has been established |
| /// for each basic block in the function in the SSA updater. This routine traces |
| /// it through the instructions within each basic block, and for non-returning |
| /// blocks informs the SSA updater about the final state that lives out of the |
| /// block. Along the way, it hardens any vulnerable instruction using the |
| /// currently valid predicate state. We have to do these two things together |
| /// because the SSA updater only works across blocks. Within a block, we track |
| /// the current predicate state directly and update it as it changes. |
| /// |
| /// This operates in two passes over each block. First, we analyze the loads in |
| /// the block to determine which strategy will be used to harden them: hardening |
| /// the address or hardening the loaded value when loaded into a register |
| /// amenable to hardening. We have to process these first because the two |
| /// strategies may interact -- later hardening may change what strategy we wish |
| /// to use. We also will analyze data dependencies between loads and avoid |
| /// hardening those loads that are data dependent on a load with a hardened |
| /// address. We also skip hardening loads already behind an LFENCE as that is |
| /// sufficient to harden them against misspeculation. |
| /// |
| /// Second, we actively trace the predicate state through the block, applying |
| /// the hardening steps we determined necessary in the first pass as we go. |
| /// |
| /// These two passes are applied to each basic block. We operate one block at a |
| /// time to simplify reasoning about reachability and sequencing. |
| void X86SpeculativeLoadHardeningPass::tracePredStateThroughBlocksAndHarden( |
| MachineFunction &MF) { |
| SmallPtrSet<MachineInstr *, 16> HardenPostLoad; |
| SmallPtrSet<MachineInstr *, 16> HardenLoadAddr; |
| |
| SmallSet<unsigned, 16> HardenedAddrRegs; |
| |
| SmallDenseMap<unsigned, unsigned, 32> AddrRegToHardenedReg; |
| |
| // Track the set of load-dependent registers through the basic block. Because |
| // the values of these registers have an existing data dependency on a loaded |
| // value which we would have checked, we can omit any checks on them. |
| SparseBitVector<> LoadDepRegs; |
| |
| for (MachineBasicBlock &MBB : MF) { |
| // The first pass over the block: collect all the loads which can have their |
| // loaded value hardened and all the loads that instead need their address |
| // hardened. During this walk we propagate load dependence for address |
| // hardened loads and also look for LFENCE to stop hardening wherever |
| // possible. When deciding whether or not to harden the loaded value or not, |
| // we check to see if any registers used in the address will have been |
| // hardened at this point and if so, harden any remaining address registers |
| // as that often successfully re-uses hardened addresses and minimizes |
| // instructions. |
| // |
| // FIXME: We should consider an aggressive mode where we continue to keep as |
| // many loads value hardened even when some address register hardening would |
| // be free (due to reuse). |
| // |
| // Note that we only need this pass if we are actually hardening loads. |
| if (HardenLoads) |
| for (MachineInstr &MI : MBB) { |
| // We naively assume that all def'ed registers of an instruction have |
| // a data dependency on all of their operands. |
| // FIXME: Do a more careful analysis of x86 to build a conservative |
| // model here. |
| if (llvm::any_of(MI.uses(), [&](MachineOperand &Op) { |
| return Op.isReg() && LoadDepRegs.test(Op.getReg()); |
| })) |
| for (MachineOperand &Def : MI.defs()) |
| if (Def.isReg()) |
| LoadDepRegs.set(Def.getReg()); |
| |
| // Both Intel and AMD are guiding that they will change the semantics of |
| // LFENCE to be a speculation barrier, so if we see an LFENCE, there is |
| // no more need to guard things in this block. |
| if (MI.getOpcode() == X86::LFENCE) |
| break; |
| |
| // If this instruction cannot load, nothing to do. |
| if (!MI.mayLoad()) |
| continue; |
| |
| // Some instructions which "load" are trivially safe or unimportant. |
| if (MI.getOpcode() == X86::MFENCE) |
| continue; |
| |
| // Extract the memory operand information about this instruction. |
| // FIXME: This doesn't handle loading pseudo instructions which we often |
| // could handle with similarly generic logic. We probably need to add an |
| // MI-layer routine similar to the MC-layer one we use here which maps |
| // pseudos much like this maps real instructions. |
| const MCInstrDesc &Desc = MI.getDesc(); |
| int MemRefBeginIdx = X86II::getMemoryOperandNo(Desc.TSFlags); |
| if (MemRefBeginIdx < 0) { |
| LLVM_DEBUG(dbgs() |
| << "WARNING: unable to harden loading instruction: "; |
| MI.dump()); |
| continue; |
| } |
| |
| MemRefBeginIdx += X86II::getOperandBias(Desc); |
| |
| MachineOperand &BaseMO = |
| MI.getOperand(MemRefBeginIdx + X86::AddrBaseReg); |
| MachineOperand &IndexMO = |
| MI.getOperand(MemRefBeginIdx + X86::AddrIndexReg); |
| |
| // If we have at least one (non-frame-index, non-RIP) register operand, |
| // and neither operand is load-dependent, we need to check the load. |
| unsigned BaseReg = 0, IndexReg = 0; |
| if (!BaseMO.isFI() && BaseMO.getReg() != X86::RIP && |
| BaseMO.getReg() != X86::NoRegister) |
| BaseReg = BaseMO.getReg(); |
| if (IndexMO.getReg() != X86::NoRegister) |
| IndexReg = IndexMO.getReg(); |
| |
| if (!BaseReg && !IndexReg) |
| // No register operands! |
| continue; |
| |
| // If any register operand is dependent, this load is dependent and we |
| // needn't check it. |
| // FIXME: Is this true in the case where we are hardening loads after |
| // they complete? Unclear, need to investigate. |
| if ((BaseReg && LoadDepRegs.test(BaseReg)) || |
| (IndexReg && LoadDepRegs.test(IndexReg))) |
| continue; |
| |
| // If post-load hardening is enabled, this load is compatible with |
| // post-load hardening, and we aren't already going to harden one of the |
| // address registers, queue it up to be hardened post-load. Notably, |
| // even once hardened this won't introduce a useful dependency that |
| // could prune out subsequent loads. |
| if (EnablePostLoadHardening && X86InstrInfo::isDataInvariantLoad(MI) && |
| !isEFLAGSDefLive(MI) && MI.getDesc().getNumDefs() == 1 && |
| MI.getOperand(0).isReg() && |
| canHardenRegister(MI.getOperand(0).getReg()) && |
| !HardenedAddrRegs.count(BaseReg) && |
| !HardenedAddrRegs.count(IndexReg)) { |
| HardenPostLoad.insert(&MI); |
| HardenedAddrRegs.insert(MI.getOperand(0).getReg()); |
| continue; |
| } |
| |
| // Record this instruction for address hardening and record its register |
| // operands as being address-hardened. |
| HardenLoadAddr.insert(&MI); |
| if (BaseReg) |
| HardenedAddrRegs.insert(BaseReg); |
| if (IndexReg) |
| HardenedAddrRegs.insert(IndexReg); |
| |
| for (MachineOperand &Def : MI.defs()) |
| if (Def.isReg()) |
| LoadDepRegs.set(Def.getReg()); |
| } |
| |
| // Now re-walk the instructions in the basic block, and apply whichever |
| // hardening strategy we have elected. Note that we do this in a second |
| // pass specifically so that we have the complete set of instructions for |
| // which we will do post-load hardening and can defer it in certain |
| // circumstances. |
| for (MachineInstr &MI : MBB) { |
| if (HardenLoads) { |
| // We cannot both require hardening the def of a load and its address. |
| assert(!(HardenLoadAddr.count(&MI) && HardenPostLoad.count(&MI)) && |
| "Requested to harden both the address and def of a load!"); |
| |
| // Check if this is a load whose address needs to be hardened. |
| if (HardenLoadAddr.erase(&MI)) { |
| const MCInstrDesc &Desc = MI.getDesc(); |
| int MemRefBeginIdx = X86II::getMemoryOperandNo(Desc.TSFlags); |
| assert(MemRefBeginIdx >= 0 && "Cannot have an invalid index here!"); |
| |
| MemRefBeginIdx += X86II::getOperandBias(Desc); |
| |
| MachineOperand &BaseMO = |
| MI.getOperand(MemRefBeginIdx + X86::AddrBaseReg); |
| MachineOperand &IndexMO = |
| MI.getOperand(MemRefBeginIdx + X86::AddrIndexReg); |
| hardenLoadAddr(MI, BaseMO, IndexMO, AddrRegToHardenedReg); |
| continue; |
| } |
| |
| // Test if this instruction is one of our post load instructions (and |
| // remove it from the set if so). |
| if (HardenPostLoad.erase(&MI)) { |
| assert(!MI.isCall() && "Must not try to post-load harden a call!"); |
| |
| // If this is a data-invariant load and there is no EFLAGS |
| // interference, we want to try and sink any hardening as far as |
| // possible. |
| if (X86InstrInfo::isDataInvariantLoad(MI) && !isEFLAGSDefLive(MI)) { |
| // Sink the instruction we'll need to harden as far as we can down |
| // the graph. |
| MachineInstr *SunkMI = sinkPostLoadHardenedInst(MI, HardenPostLoad); |
| |
| // If we managed to sink this instruction, update everything so we |
| // harden that instruction when we reach it in the instruction |
| // sequence. |
| if (SunkMI != &MI) { |
| // If in sinking there was no instruction needing to be hardened, |
| // we're done. |
| if (!SunkMI) |
| continue; |
| |
| // Otherwise, add this to the set of defs we harden. |
| HardenPostLoad.insert(SunkMI); |
| continue; |
| } |
| } |
| |
| unsigned HardenedReg = hardenPostLoad(MI); |
| |
| // Mark the resulting hardened register as such so we don't re-harden. |
| AddrRegToHardenedReg[HardenedReg] = HardenedReg; |
| |
| continue; |
| } |
| |
| // Check for an indirect call or branch that may need its input hardened |
| // even if we couldn't find the specific load used, or were able to |
| // avoid hardening it for some reason. Note that here we cannot break |
| // out afterward as we may still need to handle any call aspect of this |
| // instruction. |
| if ((MI.isCall() || MI.isBranch()) && HardenIndirectCallsAndJumps) |
| hardenIndirectCallOrJumpInstr(MI, AddrRegToHardenedReg); |
| } |
| |
| // After we finish hardening loads we handle interprocedural hardening if |
| // enabled and relevant for this instruction. |
| if (!HardenInterprocedurally) |
| continue; |
| if (!MI.isCall() && !MI.isReturn()) |
| continue; |
| |
| // If this is a direct return (IE, not a tail call) just directly harden |
| // it. |
| if (MI.isReturn() && !MI.isCall()) { |
| hardenReturnInstr(MI); |
| continue; |
| } |
| |
| // Otherwise we have a call. We need to handle transferring the predicate |
| // state into a call and recovering it after the call returns (unless this |
| // is a tail call). |
| assert(MI.isCall() && "Should only reach here for calls!"); |
| tracePredStateThroughCall(MI); |
| } |
| |
| HardenPostLoad.clear(); |
| HardenLoadAddr.clear(); |
| HardenedAddrRegs.clear(); |
| AddrRegToHardenedReg.clear(); |
| |
| // Currently, we only track data-dependent loads within a basic block. |
| // FIXME: We should see if this is necessary or if we could be more |
| // aggressive here without opening up attack avenues. |
| LoadDepRegs.clear(); |
| } |
| } |
| |
| /// Save EFLAGS into the returned GPR. This can in turn be restored with |
| /// `restoreEFLAGS`. |
| /// |
| /// Note that LLVM can only lower very simple patterns of saved and restored |
| /// EFLAGS registers. The restore should always be within the same basic block |
| /// as the save so that no PHI nodes are inserted. |
| unsigned X86SpeculativeLoadHardeningPass::saveEFLAGS( |
| MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertPt, |
| DebugLoc Loc) { |
| // FIXME: Hard coding this to a 32-bit register class seems weird, but matches |
| // what instruction selection does. |
| Register Reg = MRI->createVirtualRegister(&X86::GR32RegClass); |
| // We directly copy the FLAGS register and rely on later lowering to clean |
| // this up into the appropriate setCC instructions. |
| BuildMI(MBB, InsertPt, Loc, TII->get(X86::COPY), Reg).addReg(X86::EFLAGS); |
| ++NumInstsInserted; |
| return Reg; |
| } |
| |
| /// Restore EFLAGS from the provided GPR. This should be produced by |
| /// `saveEFLAGS`. |
| /// |
| /// This must be done within the same basic block as the save in order to |
| /// reliably lower. |
| void X86SpeculativeLoadHardeningPass::restoreEFLAGS( |
| MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertPt, DebugLoc Loc, |
| Register Reg) { |
| BuildMI(MBB, InsertPt, Loc, TII->get(X86::COPY), X86::EFLAGS).addReg(Reg); |
| ++NumInstsInserted; |
| } |
| |
| /// Takes the current predicate state (in a register) and merges it into the |
| /// stack pointer. The state is essentially a single bit, but we merge this in |
| /// a way that won't form non-canonical pointers and also will be preserved |
| /// across normal stack adjustments. |
| void X86SpeculativeLoadHardeningPass::mergePredStateIntoSP( |
| MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertPt, DebugLoc Loc, |
| unsigned PredStateReg) { |
| Register TmpReg = MRI->createVirtualRegister(PS->RC); |
| // FIXME: This hard codes a shift distance based on the number of bits needed |
| // to stay canonical on 64-bit. We should compute this somehow and support |
| // 32-bit as part of that. |
| auto ShiftI = BuildMI(MBB, InsertPt, Loc, TII->get(X86::SHL64ri), TmpReg) |
| .addReg(PredStateReg, RegState::Kill) |
| .addImm(47); |
| ShiftI->addRegisterDead(X86::EFLAGS, TRI); |
| ++NumInstsInserted; |
| auto OrI = BuildMI(MBB, InsertPt, Loc, TII->get(X86::OR64rr), X86::RSP) |
| .addReg(X86::RSP) |
| .addReg(TmpReg, RegState::Kill); |
| OrI->addRegisterDead(X86::EFLAGS, TRI); |
| ++NumInstsInserted; |
| } |
| |
| /// Extracts the predicate state stored in the high bits of the stack pointer. |
| unsigned X86SpeculativeLoadHardeningPass::extractPredStateFromSP( |
| MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertPt, |
| DebugLoc Loc) { |
| Register PredStateReg = MRI->createVirtualRegister(PS->RC); |
| Register TmpReg = MRI->createVirtualRegister(PS->RC); |
| |
| // We know that the stack pointer will have any preserved predicate state in |
| // its high bit. We just want to smear this across the other bits. Turns out, |
| // this is exactly what an arithmetic right shift does. |
| BuildMI(MBB, InsertPt, Loc, TII->get(TargetOpcode::COPY), TmpReg) |
| .addReg(X86::RSP); |
| auto ShiftI = |
| BuildMI(MBB, InsertPt, Loc, TII->get(X86::SAR64ri), PredStateReg) |
| .addReg(TmpReg, RegState::Kill) |
| .addImm(TRI->getRegSizeInBits(*PS->RC) - 1); |
| ShiftI->addRegisterDead(X86::EFLAGS, TRI); |
| ++NumInstsInserted; |
| |
| return PredStateReg; |
| } |
| |
| void X86SpeculativeLoadHardeningPass::hardenLoadAddr( |
| MachineInstr &MI, MachineOperand &BaseMO, MachineOperand &IndexMO, |
| SmallDenseMap<unsigned, unsigned, 32> &AddrRegToHardenedReg) { |
| MachineBasicBlock &MBB = *MI.getParent(); |
| const DebugLoc &Loc = MI.getDebugLoc(); |
| |
| // Check if EFLAGS are alive by seeing if there is a def of them or they |
| // live-in, and then seeing if that def is in turn used. |
| bool EFLAGSLive = isEFLAGSLive(MBB, MI.getIterator(), *TRI); |
| |
| SmallVector<MachineOperand *, 2> HardenOpRegs; |
| |
| if (BaseMO.isFI()) { |
| // A frame index is never a dynamically controllable load, so only |
| // harden it if we're covering fixed address loads as well. |
| LLVM_DEBUG( |
| dbgs() << " Skipping hardening base of explicit stack frame load: "; |
| MI.dump(); dbgs() << "\n"); |
| } else if (BaseMO.getReg() == X86::RSP) { |
| // Some idempotent atomic operations are lowered directly to a locked |
| // OR with 0 to the top of stack(or slightly offset from top) which uses an |
| // explicit RSP register as the base. |
| assert(IndexMO.getReg() == X86::NoRegister && |
| "Explicit RSP access with dynamic index!"); |
| LLVM_DEBUG( |
| dbgs() << " Cannot harden base of explicit RSP offset in a load!"); |
| } else if (BaseMO.getReg() == X86::RIP || |
| BaseMO.getReg() == X86::NoRegister) { |
| // For both RIP-relative addressed loads or absolute loads, we cannot |
| // meaningfully harden them because the address being loaded has no |
| // dynamic component. |
| // |
| // FIXME: When using a segment base (like TLS does) we end up with the |
| // dynamic address being the base plus -1 because we can't mutate the |
| // segment register here. This allows the signed 32-bit offset to point at |
| // valid segment-relative addresses and load them successfully. |
| LLVM_DEBUG( |
| dbgs() << " Cannot harden base of " |
| << (BaseMO.getReg() == X86::RIP ? "RIP-relative" : "no-base") |
| << " address in a load!"); |
| } else { |
| assert(BaseMO.isReg() && |
| "Only allowed to have a frame index or register base."); |
| HardenOpRegs.push_back(&BaseMO); |
| } |
| |
| if (IndexMO.getReg() != X86::NoRegister && |
| (HardenOpRegs.empty() || |
| HardenOpRegs.front()->getReg() != IndexMO.getReg())) |
| HardenOpRegs.push_back(&IndexMO); |
| |
| assert((HardenOpRegs.size() == 1 || HardenOpRegs.size() == 2) && |
| "Should have exactly one or two registers to harden!"); |
| assert((HardenOpRegs.size() == 1 || |
| HardenOpRegs[0]->getReg() != HardenOpRegs[1]->getReg()) && |
| "Should not have two of the same registers!"); |
| |
| // Remove any registers that have alreaded been checked. |
| llvm::erase_if(HardenOpRegs, [&](MachineOperand *Op) { |
| // See if this operand's register has already been checked. |
| auto It = AddrRegToHardenedReg.find(Op->getReg()); |
| if (It == AddrRegToHardenedReg.end()) |
| // Not checked, so retain this one. |
| return false; |
| |
| // Otherwise, we can directly update this operand and remove it. |
| Op->setReg(It->second); |
| return true; |
| }); |
| // If there are none left, we're done. |
| if (HardenOpRegs.empty()) |
| return; |
| |
| // Compute the current predicate state. |
| unsigned StateReg = PS->SSA.GetValueAtEndOfBlock(&MBB); |
| |
| auto InsertPt = MI.getIterator(); |
| |
| // If EFLAGS are live and we don't have access to instructions that avoid |
| // clobbering EFLAGS we need to save and restore them. This in turn makes |
| // the EFLAGS no longer live. |
| unsigned FlagsReg = 0; |
| if (EFLAGSLive && !Subtarget->hasBMI2()) { |
| EFLAGSLive = false; |
| FlagsReg = saveEFLAGS(MBB, InsertPt, Loc); |
| } |
| |
| for (MachineOperand *Op : HardenOpRegs) { |
| Register OpReg = Op->getReg(); |
| auto *OpRC = MRI->getRegClass(OpReg); |
| Register TmpReg = MRI->createVirtualRegister(OpRC); |
| |
| // If this is a vector register, we'll need somewhat custom logic to handle |
| // hardening it. |
| if (!Subtarget->hasVLX() && (OpRC->hasSuperClassEq(&X86::VR128RegClass) || |
| OpRC->hasSuperClassEq(&X86::VR256RegClass))) { |
| assert(Subtarget->hasAVX2() && "AVX2-specific register classes!"); |
| bool Is128Bit = OpRC->hasSuperClassEq(&X86::VR128RegClass); |
| |
| // Move our state into a vector register. |
| // FIXME: We could skip this at the cost of longer encodings with AVX-512 |
| // but that doesn't seem likely worth it. |
| Register VStateReg = MRI->createVirtualRegister(&X86::VR128RegClass); |
| auto MovI = |
| BuildMI(MBB, InsertPt, Loc, TII->get(X86::VMOV64toPQIrr), VStateReg) |
| .addReg(StateReg); |
| (void)MovI; |
| ++NumInstsInserted; |
| LLVM_DEBUG(dbgs() << " Inserting mov: "; MovI->dump(); dbgs() << "\n"); |
| |
| // Broadcast it across the vector register. |
| Register VBStateReg = MRI->createVirtualRegister(OpRC); |
| auto BroadcastI = BuildMI(MBB, InsertPt, Loc, |
| TII->get(Is128Bit ? X86::VPBROADCASTQrr |
| : X86::VPBROADCASTQYrr), |
| VBStateReg) |
| .addReg(VStateReg); |
| (void)BroadcastI; |
| ++NumInstsInserted; |
| LLVM_DEBUG(dbgs() << " Inserting broadcast: "; BroadcastI->dump(); |
| dbgs() << "\n"); |
| |
| // Merge our potential poison state into the value with a vector or. |
| auto OrI = |
| BuildMI(MBB, InsertPt, Loc, |
| TII->get(Is128Bit ? X86::VPORrr : X86::VPORYrr), TmpReg) |
| .addReg(VBStateReg) |
| .addReg(OpReg); |
| (void)OrI; |
| ++NumInstsInserted; |
| LLVM_DEBUG(dbgs() << " Inserting or: "; OrI->dump(); dbgs() << "\n"); |
| } else if (OpRC->hasSuperClassEq(&X86::VR128XRegClass) || |
| OpRC->hasSuperClassEq(&X86::VR256XRegClass) || |
| OpRC->hasSuperClassEq(&X86::VR512RegClass)) { |
| assert(Subtarget->hasAVX512() && "AVX512-specific register classes!"); |
| bool Is128Bit = OpRC->hasSuperClassEq(&X86::VR128XRegClass); |
| bool Is256Bit = OpRC->hasSuperClassEq(&X86::VR256XRegClass); |
| if (Is128Bit || Is256Bit) |
| assert(Subtarget->hasVLX() && "AVX512VL-specific register classes!"); |
| |
| // Broadcast our state into a vector register. |
| Register VStateReg = MRI->createVirtualRegister(OpRC); |
| unsigned BroadcastOp = Is128Bit ? X86::VPBROADCASTQrZ128rr |
| : Is256Bit ? X86::VPBROADCASTQrZ256rr |
| : X86::VPBROADCASTQrZrr; |
| auto BroadcastI = |
| BuildMI(MBB, InsertPt, Loc, TII->get(BroadcastOp), VStateReg) |
| .addReg(StateReg); |
| (void)BroadcastI; |
| ++NumInstsInserted; |
| LLVM_DEBUG(dbgs() << " Inserting broadcast: "; BroadcastI->dump(); |
| dbgs() << "\n"); |
| |
| // Merge our potential poison state into the value with a vector or. |
| unsigned OrOp = Is128Bit ? X86::VPORQZ128rr |
| : Is256Bit ? X86::VPORQZ256rr : X86::VPORQZrr; |
| auto OrI = BuildMI(MBB, InsertPt, Loc, TII->get(OrOp), TmpReg) |
| .addReg(VStateReg) |
| .addReg(OpReg); |
| (void)OrI; |
| ++NumInstsInserted; |
| LLVM_DEBUG(dbgs() << " Inserting or: "; OrI->dump(); dbgs() << "\n"); |
| } else { |
| // FIXME: Need to support GR32 here for 32-bit code. |
| assert(OpRC->hasSuperClassEq(&X86::GR64RegClass) && |
| "Not a supported register class for address hardening!"); |
| |
| if (!EFLAGSLive) { |
| // Merge our potential poison state into the value with an or. |
| auto OrI = BuildMI(MBB, InsertPt, Loc, TII->get(X86::OR64rr), TmpReg) |
| .addReg(StateReg) |
| .addReg(OpReg); |
| OrI->addRegisterDead(X86::EFLAGS, TRI); |
| ++NumInstsInserted; |
| LLVM_DEBUG(dbgs() << " Inserting or: "; OrI->dump(); dbgs() << "\n"); |
| } else { |
| // We need to avoid touching EFLAGS so shift out all but the least |
| // significant bit using the instruction that doesn't update flags. |
| auto ShiftI = |
| BuildMI(MBB, InsertPt, Loc, TII->get(X86::SHRX64rr), TmpReg) |
| .addReg(OpReg) |
| .addReg(StateReg); |
| (void)ShiftI; |
| ++NumInstsInserted; |
| LLVM_DEBUG(dbgs() << " Inserting shrx: "; ShiftI->dump(); |
| dbgs() << "\n"); |
| } |
| } |
| |
| // Record this register as checked and update the operand. |
| assert(!AddrRegToHardenedReg.count(Op->getReg()) && |
| "Should not have checked this register yet!"); |
| AddrRegToHardenedReg[Op->getReg()] = TmpReg; |
| Op->setReg(TmpReg); |
| ++NumAddrRegsHardened; |
| } |
| |
| // And restore the flags if needed. |
| if (FlagsReg) |
| restoreEFLAGS(MBB, InsertPt, Loc, FlagsReg); |
| } |
| |
| MachineInstr *X86SpeculativeLoadHardeningPass::sinkPostLoadHardenedInst( |
| MachineInstr &InitialMI, SmallPtrSetImpl<MachineInstr *> &HardenedInstrs) { |
| assert(X86InstrInfo::isDataInvariantLoad(InitialMI) && |
| "Cannot get here with a non-invariant load!"); |
| assert(!isEFLAGSDefLive(InitialMI) && |
| "Cannot get here with a data invariant load " |
| "that interferes with EFLAGS!"); |
| |
| // See if we can sink hardening the loaded value. |
| auto SinkCheckToSingleUse = |
| [&](MachineInstr &MI) -> Optional<MachineInstr *> { |
| Register DefReg = MI.getOperand(0).getReg(); |
| |
| // We need to find a single use which we can sink the check. We can |
| // primarily do this because many uses may already end up checked on their |
| // own. |
| MachineInstr *SingleUseMI = nullptr; |
| for (MachineInstr &UseMI : MRI->use_instructions(DefReg)) { |
| // If we're already going to harden this use, it is data invariant, it |
| // does not interfere with EFLAGS, and within our block. |
| if (HardenedInstrs.count(&UseMI)) { |
| if (!X86InstrInfo::isDataInvariantLoad(UseMI) || isEFLAGSDefLive(UseMI)) { |
| // If we've already decided to harden a non-load, we must have sunk |
| // some other post-load hardened instruction to it and it must itself |
| // be data-invariant. |
| assert(X86InstrInfo::isDataInvariant(UseMI) && |
| "Data variant instruction being hardened!"); |
| continue; |
| } |
| |
| // Otherwise, this is a load and the load component can't be data |
| // invariant so check how this register is being used. |
| const MCInstrDesc &Desc = UseMI.getDesc(); |
| int MemRefBeginIdx = X86II::getMemoryOperandNo(Desc.TSFlags); |
| assert(MemRefBeginIdx >= 0 && |
| "Should always have mem references here!"); |
| MemRefBeginIdx += X86II::getOperandBias(Desc); |
| |
| MachineOperand &BaseMO = |
| UseMI.getOperand(MemRefBeginIdx + X86::AddrBaseReg); |
| MachineOperand &IndexMO = |
| UseMI.getOperand(MemRefBeginIdx + X86::AddrIndexReg); |
| if ((BaseMO.isReg() && BaseMO.getReg() == DefReg) || |
| (IndexMO.isReg() && IndexMO.getReg() == DefReg)) |
| // The load uses the register as part of its address making it not |
| // invariant. |
| return {}; |
| |
| continue; |
| } |
| |
| if (SingleUseMI) |
| // We already have a single use, this would make two. Bail. |
| return {}; |
| |
| // If this single use isn't data invariant, isn't in this block, or has |
| // interfering EFLAGS, we can't sink the hardening to it. |
| if (!X86InstrInfo::isDataInvariant(UseMI) || UseMI.getParent() != MI.getParent() || |
| isEFLAGSDefLive(UseMI)) |
| return {}; |
| |
| // If this instruction defines multiple registers bail as we won't harden |
| // all of them. |
| if (UseMI.getDesc().getNumDefs() > 1) |
| return {}; |
| |
| // If this register isn't a virtual register we can't walk uses of sanely, |
| // just bail. Also check that its register class is one of the ones we |
| // can harden. |
| Register UseDefReg = UseMI.getOperand(0).getReg(); |
| if (!UseDefReg.isVirtual() || !canHardenRegister(UseDefReg)) |
| return {}; |
| |
| SingleUseMI = &UseMI; |
| } |
| |
| // If SingleUseMI is still null, there is no use that needs its own |
| // checking. Otherwise, it is the single use that needs checking. |
| return {SingleUseMI}; |
| }; |
| |
| MachineInstr *MI = &InitialMI; |
| while (Optional<MachineInstr *> SingleUse = SinkCheckToSingleUse(*MI)) { |
| // Update which MI we're checking now. |
| MI = *SingleUse; |
| if (!MI) |
| break; |
| } |
| |
| return MI; |
| } |
| |
| bool X86SpeculativeLoadHardeningPass::canHardenRegister(Register Reg) { |
| auto *RC = MRI->getRegClass(Reg); |
| int RegBytes = TRI->getRegSizeInBits(*RC) / 8; |
| if (RegBytes > 8) |
| // We don't support post-load hardening of vectors. |
| return false; |
| |
| unsigned RegIdx = Log2_32(RegBytes); |
| assert(RegIdx < 4 && "Unsupported register size"); |
| |
| // If this register class is explicitly constrained to a class that doesn't |
| // require REX prefix, we may not be able to satisfy that constraint when |
| // emitting the hardening instructions, so bail out here. |
| // FIXME: This seems like a pretty lame hack. The way this comes up is when we |
| // end up both with a NOREX and REX-only register as operands to the hardening |
| // instructions. It would be better to fix that code to handle this situation |
| // rather than hack around it in this way. |
| const TargetRegisterClass *NOREXRegClasses[] = { |
| &X86::GR8_NOREXRegClass, &X86::GR16_NOREXRegClass, |
| &X86::GR32_NOREXRegClass, &X86::GR64_NOREXRegClass}; |
| if (RC == NOREXRegClasses[RegIdx]) |
| return false; |
| |
| const TargetRegisterClass *GPRRegClasses[] = { |
| &X86::GR8RegClass, &X86::GR16RegClass, &X86::GR32RegClass, |
| &X86::GR64RegClass}; |
| return RC->hasSuperClassEq(GPRRegClasses[RegIdx]); |
| } |
| |
| /// Harden a value in a register. |
| /// |
| /// This is the low-level logic to fully harden a value sitting in a register |
| /// against leaking during speculative execution. |
| /// |
| /// Unlike hardening an address that is used by a load, this routine is required |
| /// to hide *all* incoming bits in the register. |
| /// |
| /// `Reg` must be a virtual register. Currently, it is required to be a GPR no |
| /// larger than the predicate state register. FIXME: We should support vector |
| /// registers here by broadcasting the predicate state. |
| /// |
| /// The new, hardened virtual register is returned. It will have the same |
| /// register class as `Reg`. |
| unsigned X86SpeculativeLoadHardeningPass::hardenValueInRegister( |
| Register Reg, MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertPt, |
| DebugLoc Loc) { |
| assert(canHardenRegister(Reg) && "Cannot harden this register!"); |
| assert(Reg.isVirtual() && "Cannot harden a physical register!"); |
| |
| auto *RC = MRI->getRegClass(Reg); |
| int Bytes = TRI->getRegSizeInBits(*RC) / 8; |
| unsigned StateReg = PS->SSA.GetValueAtEndOfBlock(&MBB); |
| assert((Bytes == 1 || Bytes == 2 || Bytes == 4 || Bytes == 8) && |
| "Unknown register size"); |
| |
| // FIXME: Need to teach this about 32-bit mode. |
| if (Bytes != 8) { |
| unsigned SubRegImms[] = {X86::sub_8bit, X86::sub_16bit, X86::sub_32bit}; |
| unsigned SubRegImm = SubRegImms[Log2_32(Bytes)]; |
| Register NarrowStateReg = MRI->createVirtualRegister(RC); |
| BuildMI(MBB, InsertPt, Loc, TII->get(TargetOpcode::COPY), NarrowStateReg) |
| .addReg(StateReg, 0, SubRegImm); |
| StateReg = NarrowStateReg; |
| } |
| |
| unsigned FlagsReg = 0; |
| if (isEFLAGSLive(MBB, InsertPt, *TRI)) |
| FlagsReg = saveEFLAGS(MBB, InsertPt, Loc); |
| |
| Register NewReg = MRI->createVirtualRegister(RC); |
| unsigned OrOpCodes[] = {X86::OR8rr, X86::OR16rr, X86::OR32rr, X86::OR64rr}; |
| unsigned OrOpCode = OrOpCodes[Log2_32(Bytes)]; |
| auto OrI = BuildMI(MBB, InsertPt, Loc, TII->get(OrOpCode), NewReg) |
| .addReg(StateReg) |
| .addReg(Reg); |
| OrI->addRegisterDead(X86::EFLAGS, TRI); |
| ++NumInstsInserted; |
| LLVM_DEBUG(dbgs() << " Inserting or: "; OrI->dump(); dbgs() << "\n"); |
| |
| if (FlagsReg) |
| restoreEFLAGS(MBB, InsertPt, Loc, FlagsReg); |
| |
| return NewReg; |
| } |
| |
| /// Harden a load by hardening the loaded value in the defined register. |
| /// |
| /// We can harden a non-leaking load into a register without touching the |
| /// address by just hiding all of the loaded bits during misspeculation. We use |
| /// an `or` instruction to do this because we set up our poison value as all |
| /// ones. And the goal is just for the loaded bits to not be exposed to |
| /// execution and coercing them to one is sufficient. |
| /// |
| /// Returns the newly hardened register. |
| unsigned X86SpeculativeLoadHardeningPass::hardenPostLoad(MachineInstr &MI) { |
| MachineBasicBlock &MBB = *MI.getParent(); |
| const DebugLoc &Loc = MI.getDebugLoc(); |
| |
| auto &DefOp = MI.getOperand(0); |
| Register OldDefReg = DefOp.getReg(); |
| auto *DefRC = MRI->getRegClass(OldDefReg); |
| |
| // Because we want to completely replace the uses of this def'ed value with |
| // the hardened value, create a dedicated new register that will only be used |
| // to communicate the unhardened value to the hardening. |
| Register UnhardenedReg = MRI->createVirtualRegister(DefRC); |
| DefOp.setReg(UnhardenedReg); |
| |
| // Now harden this register's value, getting a hardened reg that is safe to |
| // use. Note that we insert the instructions to compute this *after* the |
| // defining instruction, not before it. |
| unsigned HardenedReg = hardenValueInRegister( |
| UnhardenedReg, MBB, std::next(MI.getIterator()), Loc); |
| |
| // Finally, replace the old register (which now only has the uses of the |
| // original def) with the hardened register. |
| MRI->replaceRegWith(/*FromReg*/ OldDefReg, /*ToReg*/ HardenedReg); |
| |
| ++NumPostLoadRegsHardened; |
| return HardenedReg; |
| } |
| |
| /// Harden a return instruction. |
| /// |
| /// Returns implicitly perform a load which we need to harden. Without hardening |
| /// this load, an attacker my speculatively write over the return address to |
| /// steer speculation of the return to an attacker controlled address. This is |
| /// called Spectre v1.1 or Bounds Check Bypass Store (BCBS) and is described in |
| /// this paper: |
| /// https://people.csail.mit.edu/vlk/spectre11.pdf |
| /// |
| /// We can harden this by introducing an LFENCE that will delay any load of the |
| /// return address until prior instructions have retired (and thus are not being |
| /// speculated), or we can harden the address used by the implicit load: the |
| /// stack pointer. |
| /// |
| /// If we are not using an LFENCE, hardening the stack pointer has an additional |
| /// benefit: it allows us to pass the predicate state accumulated in this |
| /// function back to the caller. In the absence of a BCBS attack on the return, |
| /// the caller will typically be resumed and speculatively executed due to the |
| /// Return Stack Buffer (RSB) prediction which is very accurate and has a high |
| /// priority. It is possible that some code from the caller will be executed |
| /// speculatively even during a BCBS-attacked return until the steering takes |
| /// effect. Whenever this happens, the caller can recover the (poisoned) |
| /// predicate state from the stack pointer and continue to harden loads. |
| void X86SpeculativeLoadHardeningPass::hardenReturnInstr(MachineInstr &MI) { |
| MachineBasicBlock &MBB = *MI.getParent(); |
| const DebugLoc &Loc = MI.getDebugLoc(); |
| auto InsertPt = MI.getIterator(); |
| |
| if (FenceCallAndRet) |
| // No need to fence here as we'll fence at the return site itself. That |
| // handles more cases than we can handle here. |
| return; |
| |
| // Take our predicate state, shift it to the high 17 bits (so that we keep |
| // pointers canonical) and merge it into RSP. This will allow the caller to |
| // extract it when we return (speculatively). |
| mergePredStateIntoSP(MBB, InsertPt, Loc, PS->SSA.GetValueAtEndOfBlock(&MBB)); |
| } |
| |
| /// Trace the predicate state through a call. |
| /// |
| /// There are several layers of this needed to handle the full complexity of |
| /// calls. |
| /// |
| /// First, we need to send the predicate state into the called function. We do |
| /// this by merging it into the high bits of the stack pointer. |
| /// |
| /// For tail calls, this is all we need to do. |
| /// |
| /// For calls where we might return and resume the control flow, we need to |
| /// extract the predicate state from the high bits of the stack pointer after |
| /// control returns from the called function. |
| /// |
| /// We also need to verify that we intended to return to this location in the |
| /// code. An attacker might arrange for the processor to mispredict the return |
| /// to this valid but incorrect return address in the program rather than the |
| /// correct one. See the paper on this attack, called "ret2spec" by the |
| /// researchers, here: |
| /// https://christian-rossow.de/publications/ret2spec-ccs2018.pdf |
| /// |
| /// The way we verify that we returned to the correct location is by preserving |
| /// the expected return address across the call. One technique involves taking |
| /// advantage of the red-zone to load the return address from `8(%rsp)` where it |
| /// was left by the RET instruction when it popped `%rsp`. Alternatively, we can |
| /// directly save the address into a register that will be preserved across the |
| /// call. We compare this intended return address against the address |
| /// immediately following the call (the observed return address). If these |
| /// mismatch, we have detected misspeculation and can poison our predicate |
| /// state. |
| void X86SpeculativeLoadHardeningPass::tracePredStateThroughCall( |
| MachineInstr &MI) { |
| MachineBasicBlock &MBB = *MI.getParent(); |
| MachineFunction &MF = *MBB.getParent(); |
| auto InsertPt = MI.getIterator(); |
| const DebugLoc &Loc = MI.getDebugLoc(); |
| |
| if (FenceCallAndRet) { |
| if (MI.isReturn()) |
| // Tail call, we don't return to this function. |
| // FIXME: We should also handle noreturn calls. |
| return; |
| |
| // We don't need to fence before the call because the function should fence |
| // in its entry. However, we do need to fence after the call returns. |
| // Fencing before the return doesn't correctly handle cases where the return |
| // itself is mispredicted. |
| BuildMI(MBB, std::next(InsertPt), Loc, TII->get(X86::LFENCE)); |
| ++NumInstsInserted; |
| ++NumLFENCEsInserted; |
| return; |
| } |
| |
| // First, we transfer the predicate state into the called function by merging |
| // it into the stack pointer. This will kill the current def of the state. |
| unsigned StateReg = PS->SSA.GetValueAtEndOfBlock(&MBB); |
| mergePredStateIntoSP(MBB, InsertPt, Loc, StateReg); |
| |
| // If this call is also a return, it is a tail call and we don't need anything |
| // else to handle it so just return. Also, if there are no further |
| // instructions and no successors, this call does not return so we can also |
| // bail. |
| if (MI.isReturn() || (std::next(InsertPt) == MBB.end() && MBB.succ_empty())) |
| return; |
| |
| // Create a symbol to track the return address and attach it to the call |
| // machine instruction. We will lower extra symbols attached to call |
| // instructions as label immediately following the call. |
| MCSymbol *RetSymbol = |
| MF.getContext().createTempSymbol("slh_ret_addr", |
| /*AlwaysAddSuffix*/ true); |
| MI.setPostInstrSymbol(MF, RetSymbol); |
| |
| const TargetRegisterClass *AddrRC = &X86::GR64RegClass; |
| unsigned ExpectedRetAddrReg = 0; |
| |
| // If we have no red zones or if the function returns twice (possibly without |
| // using the `ret` instruction) like setjmp, we need to save the expected |
| // return address prior to the call. |
| if (!Subtarget->getFrameLowering()->has128ByteRedZone(MF) || |
| MF.exposesReturnsTwice()) { |
| // If we don't have red zones, we need to compute the expected return |
| // address prior to the call and store it in a register that lives across |
| // the call. |
| // |
| // In some ways, this is doubly satisfying as a mitigation because it will |
| // also successfully detect stack smashing bugs in some cases (typically, |
| // when a callee-saved register is used and the callee doesn't push it onto |
| // the stack). But that isn't our primary goal, so we only use it as |
| // a fallback. |
| // |
| // FIXME: It isn't clear that this is reliable in the face of |
| // rematerialization in the register allocator. We somehow need to force |
| // that to not occur for this particular instruction, and instead to spill |
| // or otherwise preserve the value computed *prior* to the call. |
| // |
| // FIXME: It is even less clear why MachineCSE can't just fold this when we |
| // end up having to use identical instructions both before and after the |
| // call to feed the comparison. |
| ExpectedRetAddrReg = MRI->createVirtualRegister(AddrRC); |
| if (MF.getTarget().getCodeModel() == CodeModel::Small && |
| !Subtarget->isPositionIndependent()) { |
| BuildMI(MBB, InsertPt, Loc, TII->get(X86::MOV64ri32), ExpectedRetAddrReg) |
| .addSym(RetSymbol); |
| } else { |
| BuildMI(MBB, InsertPt, Loc, TII->get(X86::LEA64r), ExpectedRetAddrReg) |
| .addReg(/*Base*/ X86::RIP) |
| .addImm(/*Scale*/ 1) |
| .addReg(/*Index*/ 0) |
| .addSym(RetSymbol) |
| .addReg(/*Segment*/ 0); |
| } |
| } |
| |
| // Step past the call to handle when it returns. |
| ++InsertPt; |
| |
| // If we didn't pre-compute the expected return address into a register, then |
| // red zones are enabled and the return address is still available on the |
| // stack immediately after the call. As the very first instruction, we load it |
| // into a register. |
| if (!ExpectedRetAddrReg) { |
| ExpectedRetAddrReg = MRI->createVirtualRegister(AddrRC); |
| BuildMI(MBB, InsertPt, Loc, TII->get(X86::MOV64rm), ExpectedRetAddrReg) |
| .addReg(/*Base*/ X86::RSP) |
| .addImm(/*Scale*/ 1) |
| .addReg(/*Index*/ 0) |
| .addImm(/*Displacement*/ -8) // The stack pointer has been popped, so |
| // the return address is 8-bytes past it. |
| .addReg(/*Segment*/ 0); |
| } |
| |
| // Now we extract the callee's predicate state from the stack pointer. |
| unsigned NewStateReg = extractPredStateFromSP(MBB, InsertPt, Loc); |
| |
| // Test the expected return address against our actual address. If we can |
| // form this basic block's address as an immediate, this is easy. Otherwise |
| // we compute it. |
| if (MF.getTarget().getCodeModel() == CodeModel::Small && |
| !Subtarget->isPositionIndependent()) { |
| // FIXME: Could we fold this with the load? It would require careful EFLAGS |
| // management. |
| BuildMI(MBB, InsertPt, Loc, TII->get(X86::CMP64ri32)) |
| .addReg(ExpectedRetAddrReg, RegState::Kill) |
| .addSym(RetSymbol); |
| } else { |
| Register ActualRetAddrReg = MRI->createVirtualRegister(AddrRC); |
| BuildMI(MBB, InsertPt, Loc, TII->get(X86::LEA64r), ActualRetAddrReg) |
| .addReg(/*Base*/ X86::RIP) |
| .addImm(/*Scale*/ 1) |
| .addReg(/*Index*/ 0) |
| .addSym(RetSymbol) |
| .addReg(/*Segment*/ 0); |
| BuildMI(MBB, InsertPt, Loc, TII->get(X86::CMP64rr)) |
| .addReg(ExpectedRetAddrReg, RegState::Kill) |
| .addReg(ActualRetAddrReg, RegState::Kill); |
| } |
| |
| // Now conditionally update the predicate state we just extracted if we ended |
| // up at a different return address than expected. |
| int PredStateSizeInBytes = TRI->getRegSizeInBits(*PS->RC) / 8; |
| auto CMovOp = X86::getCMovOpcode(PredStateSizeInBytes); |
| |
| Register UpdatedStateReg = MRI->createVirtualRegister(PS->RC); |
| auto CMovI = BuildMI(MBB, InsertPt, Loc, TII->get(CMovOp), UpdatedStateReg) |
| .addReg(NewStateReg, RegState::Kill) |
| .addReg(PS->PoisonReg) |
| .addImm(X86::COND_NE); |
| CMovI->findRegisterUseOperand(X86::EFLAGS)->setIsKill(true); |
| ++NumInstsInserted; |
| LLVM_DEBUG(dbgs() << " Inserting cmov: "; CMovI->dump(); dbgs() << "\n"); |
| |
| PS->SSA.AddAvailableValue(&MBB, UpdatedStateReg); |
| } |
| |
| /// An attacker may speculatively store over a value that is then speculatively |
| /// loaded and used as the target of an indirect call or jump instruction. This |
| /// is called Spectre v1.2 or Bounds Check Bypass Store (BCBS) and is described |
| /// in this paper: |
| /// https://people.csail.mit.edu/vlk/spectre11.pdf |
| /// |
| /// When this happens, the speculative execution of the call or jump will end up |
| /// being steered to this attacker controlled address. While most such loads |
| /// will be adequately hardened already, we want to ensure that they are |
| /// definitively treated as needing post-load hardening. While address hardening |
| /// is sufficient to prevent secret data from leaking to the attacker, it may |
| /// not be sufficient to prevent an attacker from steering speculative |
| /// execution. We forcibly unfolded all relevant loads above and so will always |
| /// have an opportunity to post-load harden here, we just need to scan for cases |
| /// not already flagged and add them. |
| void X86SpeculativeLoadHardeningPass::hardenIndirectCallOrJumpInstr( |
| MachineInstr &MI, |
| SmallDenseMap<unsigned, unsigned, 32> &AddrRegToHardenedReg) { |
| switch (MI.getOpcode()) { |
| case X86::FARCALL16m: |
| case X86::FARCALL32m: |
| case X86::FARCALL64m: |
| case X86::FARJMP16m: |
| case X86::FARJMP32m: |
| case X86::FARJMP64m: |
| // We don't need to harden either far calls or far jumps as they are |
| // safe from Spectre. |
| return; |
| |
| default: |
| break; |
| } |
| |
| // We should never see a loading instruction at this point, as those should |
| // have been unfolded. |
| assert(!MI.mayLoad() && "Found a lingering loading instruction!"); |
| |
| // If the first operand isn't a register, this is a branch or call |
| // instruction with an immediate operand which doesn't need to be hardened. |
| if (!MI.getOperand(0).isReg()) |
| return; |
| |
| // For all of these, the target register is the first operand of the |
| // instruction. |
| auto &TargetOp = MI.getOperand(0); |
| Register OldTargetReg = TargetOp.getReg(); |
| |
| // Try to lookup a hardened version of this register. We retain a reference |
| // here as we want to update the map to track any newly computed hardened |
| // register. |
| unsigned &HardenedTargetReg = AddrRegToHardenedReg[OldTargetReg]; |
| |
| // If we don't have a hardened register yet, compute one. Otherwise, just use |
| // the already hardened register. |
| // |
| // FIXME: It is a little suspect that we use partially hardened registers that |
| // only feed addresses. The complexity of partial hardening with SHRX |
| // continues to pile up. Should definitively measure its value and consider |
| // eliminating it. |
| if (!HardenedTargetReg) |
| HardenedTargetReg = hardenValueInRegister( |
| OldTargetReg, *MI.getParent(), MI.getIterator(), MI.getDebugLoc()); |
| |
| // Set the target operand to the hardened register. |
| TargetOp.setReg(HardenedTargetReg); |
| |
| ++NumCallsOrJumpsHardened; |
| } |
| |
| INITIALIZE_PASS_BEGIN(X86SpeculativeLoadHardeningPass, PASS_KEY, |
| "X86 speculative load hardener", false, false) |
| INITIALIZE_PASS_END(X86SpeculativeLoadHardeningPass, PASS_KEY, |
| "X86 speculative load hardener", false, false) |
| |
| FunctionPass *llvm::createX86SpeculativeLoadHardeningPass() { |
| return new X86SpeculativeLoadHardeningPass(); |
| } |