blob: 7e829ea43e89f1009d457e0362babe9627b4cb2e [file] [log] [blame]
//===----- BPFMISimplifyPatchable.cpp - MI Simplify Patchable Insts -------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This pass targets a subset of instructions like below
// ld_imm64 r1, @global
// ldd r2, r1, 0
// add r3, struct_base_reg, r2
//
// Here @global should represent an AMA (abstruct member access).
// Such an access is subject to bpf load time patching. After this pass, the
// code becomes
// ld_imm64 r1, @global
// add r3, struct_base_reg, r1
//
// Eventually, at BTF output stage, a relocation record will be generated
// for ld_imm64 which should be replaced later by bpf loader:
// r1 = <calculated field_info>
// add r3, struct_base_reg, r1
//
// This pass also removes the intermediate load generated in IR pass for
// __builtin_btf_type_id() intrinsic.
//
//===----------------------------------------------------------------------===//
#include "BPF.h"
#include "BPFCORE.h"
#include "BPFInstrInfo.h"
#include "BPFTargetMachine.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/Support/Debug.h"
using namespace llvm;
#define DEBUG_TYPE "bpf-mi-simplify-patchable"
namespace {
struct BPFMISimplifyPatchable : public MachineFunctionPass {
static char ID;
const BPFInstrInfo *TII;
MachineFunction *MF;
BPFMISimplifyPatchable() : MachineFunctionPass(ID) {
initializeBPFMISimplifyPatchablePass(*PassRegistry::getPassRegistry());
}
private:
// Initialize class variables.
void initialize(MachineFunction &MFParm);
bool removeLD(void);
void processCandidate(MachineRegisterInfo *MRI, MachineBasicBlock &MBB,
MachineInstr &MI, Register &SrcReg, Register &DstReg,
const GlobalValue *GVal, bool IsAma);
void processDstReg(MachineRegisterInfo *MRI, Register &DstReg,
Register &SrcReg, const GlobalValue *GVal,
bool doSrcRegProp, bool IsAma);
void processInst(MachineRegisterInfo *MRI, MachineInstr *Inst,
MachineOperand *RelocOp, const GlobalValue *GVal);
void checkADDrr(MachineRegisterInfo *MRI, MachineOperand *RelocOp,
const GlobalValue *GVal);
void checkShift(MachineRegisterInfo *MRI, MachineBasicBlock &MBB,
MachineOperand *RelocOp, const GlobalValue *GVal,
unsigned Opcode);
public:
// Main entry point for this pass.
bool runOnMachineFunction(MachineFunction &MF) override {
if (skipFunction(MF.getFunction()))
return false;
initialize(MF);
return removeLD();
}
};
// Initialize class variables.
void BPFMISimplifyPatchable::initialize(MachineFunction &MFParm) {
MF = &MFParm;
TII = MF->getSubtarget<BPFSubtarget>().getInstrInfo();
LLVM_DEBUG(dbgs() << "*** BPF simplify patchable insts pass ***\n\n");
}
void BPFMISimplifyPatchable::checkADDrr(MachineRegisterInfo *MRI,
MachineOperand *RelocOp, const GlobalValue *GVal) {
const MachineInstr *Inst = RelocOp->getParent();
const MachineOperand *Op1 = &Inst->getOperand(1);
const MachineOperand *Op2 = &Inst->getOperand(2);
const MachineOperand *BaseOp = (RelocOp == Op1) ? Op2 : Op1;
// Go through all uses of %1 as in %1 = ADD_rr %2, %3
const MachineOperand Op0 = Inst->getOperand(0);
for (MachineOperand &MO :
llvm::make_early_inc_range(MRI->use_operands(Op0.getReg()))) {
// The candidate needs to have a unique definition.
if (!MRI->getUniqueVRegDef(MO.getReg()))
continue;
MachineInstr *DefInst = MO.getParent();
unsigned Opcode = DefInst->getOpcode();
unsigned COREOp;
if (Opcode == BPF::LDB || Opcode == BPF::LDH || Opcode == BPF::LDW ||
Opcode == BPF::LDD || Opcode == BPF::STB || Opcode == BPF::STH ||
Opcode == BPF::STW || Opcode == BPF::STD)
COREOp = BPF::CORE_MEM;
else if (Opcode == BPF::LDB32 || Opcode == BPF::LDH32 ||
Opcode == BPF::LDW32 || Opcode == BPF::STB32 ||
Opcode == BPF::STH32 || Opcode == BPF::STW32)
COREOp = BPF::CORE_ALU32_MEM;
else
continue;
// It must be a form of %2 = *(type *)(%1 + 0) or *(type *)(%1 + 0) = %2.
const MachineOperand &ImmOp = DefInst->getOperand(2);
if (!ImmOp.isImm() || ImmOp.getImm() != 0)
continue;
// Reject the form:
// %1 = ADD_rr %2, %3
// *(type *)(%2 + 0) = %1
if (Opcode == BPF::STB || Opcode == BPF::STH || Opcode == BPF::STW ||
Opcode == BPF::STD || Opcode == BPF::STB32 || Opcode == BPF::STH32 ||
Opcode == BPF::STW32) {
const MachineOperand &Opnd = DefInst->getOperand(0);
if (Opnd.isReg() && Opnd.getReg() == MO.getReg())
continue;
}
BuildMI(*DefInst->getParent(), *DefInst, DefInst->getDebugLoc(), TII->get(COREOp))
.add(DefInst->getOperand(0)).addImm(Opcode).add(*BaseOp)
.addGlobalAddress(GVal);
DefInst->eraseFromParent();
}
}
void BPFMISimplifyPatchable::checkShift(MachineRegisterInfo *MRI,
MachineBasicBlock &MBB, MachineOperand *RelocOp, const GlobalValue *GVal,
unsigned Opcode) {
// Relocation operand should be the operand #2.
MachineInstr *Inst = RelocOp->getParent();
if (RelocOp != &Inst->getOperand(2))
return;
BuildMI(MBB, *Inst, Inst->getDebugLoc(), TII->get(BPF::CORE_SHIFT))
.add(Inst->getOperand(0)).addImm(Opcode)
.add(Inst->getOperand(1)).addGlobalAddress(GVal);
Inst->eraseFromParent();
}
void BPFMISimplifyPatchable::processCandidate(MachineRegisterInfo *MRI,
MachineBasicBlock &MBB, MachineInstr &MI, Register &SrcReg,
Register &DstReg, const GlobalValue *GVal, bool IsAma) {
if (MRI->getRegClass(DstReg) == &BPF::GPR32RegClass) {
if (IsAma) {
// We can optimize such a pattern:
// %1:gpr = LD_imm64 @"llvm.s:0:4$0:2"
// %2:gpr32 = LDW32 %1:gpr, 0
// %3:gpr = SUBREG_TO_REG 0, %2:gpr32, %subreg.sub_32
// %4:gpr = ADD_rr %0:gpr, %3:gpr
// or similar patterns below for non-alu32 case.
auto Begin = MRI->use_begin(DstReg), End = MRI->use_end();
decltype(End) NextI;
for (auto I = Begin; I != End; I = NextI) {
NextI = std::next(I);
if (!MRI->getUniqueVRegDef(I->getReg()))
continue;
unsigned Opcode = I->getParent()->getOpcode();
if (Opcode == BPF::SUBREG_TO_REG) {
Register TmpReg = I->getParent()->getOperand(0).getReg();
processDstReg(MRI, TmpReg, DstReg, GVal, false, IsAma);
}
}
}
BuildMI(MBB, MI, MI.getDebugLoc(), TII->get(BPF::COPY), DstReg)
.addReg(SrcReg, 0, BPF::sub_32);
return;
}
// All uses of DstReg replaced by SrcReg
processDstReg(MRI, DstReg, SrcReg, GVal, true, IsAma);
}
void BPFMISimplifyPatchable::processDstReg(MachineRegisterInfo *MRI,
Register &DstReg, Register &SrcReg, const GlobalValue *GVal,
bool doSrcRegProp, bool IsAma) {
auto Begin = MRI->use_begin(DstReg), End = MRI->use_end();
decltype(End) NextI;
for (auto I = Begin; I != End; I = NextI) {
NextI = std::next(I);
if (doSrcRegProp)
I->setReg(SrcReg);
// The candidate needs to have a unique definition.
if (IsAma && MRI->getUniqueVRegDef(I->getReg()))
processInst(MRI, I->getParent(), &*I, GVal);
}
}
// Check to see whether we could do some optimization
// to attach relocation to downstream dependent instructions.
// Two kinds of patterns are recognized below:
// Pattern 1:
// %1 = LD_imm64 @"llvm.b:0:4$0:1" <== patch_imm = 4
// %2 = LDD %1, 0 <== this insn will be removed
// %3 = ADD_rr %0, %2
// %4 = LDW[32] %3, 0 OR STW[32] %4, %3, 0
// The `%4 = ...` will be transformed to
// CORE_[ALU32_]MEM(%4, mem_opcode, %0, @"llvm.b:0:4$0:1")
// and later on, BTF emit phase will translate to
// %4 = LDW[32] %0, 4 STW[32] %4, %0, 4
// and attach a relocation to it.
// Pattern 2:
// %15 = LD_imm64 @"llvm.t:5:63$0:2" <== relocation type 5
// %16 = LDD %15, 0 <== this insn will be removed
// %17 = SRA_rr %14, %16
// The `%17 = ...` will be transformed to
// %17 = CORE_SHIFT(SRA_ri, %14, @"llvm.t:5:63$0:2")
// and later on, BTF emit phase will translate to
// %r4 = SRA_ri %r4, 63
void BPFMISimplifyPatchable::processInst(MachineRegisterInfo *MRI,
MachineInstr *Inst, MachineOperand *RelocOp, const GlobalValue *GVal) {
unsigned Opcode = Inst->getOpcode();
if (Opcode == BPF::ADD_rr)
checkADDrr(MRI, RelocOp, GVal);
else if (Opcode == BPF::SLL_rr)
checkShift(MRI, *Inst->getParent(), RelocOp, GVal, BPF::SLL_ri);
else if (Opcode == BPF::SRA_rr)
checkShift(MRI, *Inst->getParent(), RelocOp, GVal, BPF::SRA_ri);
else if (Opcode == BPF::SRL_rr)
checkShift(MRI, *Inst->getParent(), RelocOp, GVal, BPF::SRL_ri);
}
/// Remove unneeded Load instructions.
bool BPFMISimplifyPatchable::removeLD() {
MachineRegisterInfo *MRI = &MF->getRegInfo();
MachineInstr *ToErase = nullptr;
bool Changed = false;
for (MachineBasicBlock &MBB : *MF) {
for (MachineInstr &MI : MBB) {
if (ToErase) {
ToErase->eraseFromParent();
ToErase = nullptr;
}
// Ensure the register format is LOAD <reg>, <reg>, 0
if (MI.getOpcode() != BPF::LDD && MI.getOpcode() != BPF::LDW &&
MI.getOpcode() != BPF::LDH && MI.getOpcode() != BPF::LDB &&
MI.getOpcode() != BPF::LDW32 && MI.getOpcode() != BPF::LDH32 &&
MI.getOpcode() != BPF::LDB32)
continue;
if (!MI.getOperand(0).isReg() || !MI.getOperand(1).isReg())
continue;
if (!MI.getOperand(2).isImm() || MI.getOperand(2).getImm())
continue;
Register DstReg = MI.getOperand(0).getReg();
Register SrcReg = MI.getOperand(1).getReg();
MachineInstr *DefInst = MRI->getUniqueVRegDef(SrcReg);
if (!DefInst)
continue;
if (DefInst->getOpcode() != BPF::LD_imm64)
continue;
const MachineOperand &MO = DefInst->getOperand(1);
if (!MO.isGlobal())
continue;
const GlobalValue *GVal = MO.getGlobal();
auto *GVar = dyn_cast<GlobalVariable>(GVal);
if (!GVar)
continue;
// Global variables representing structure offset or type id.
bool IsAma = false;
if (GVar->hasAttribute(BPFCoreSharedInfo::AmaAttr))
IsAma = true;
else if (!GVar->hasAttribute(BPFCoreSharedInfo::TypeIdAttr))
continue;
processCandidate(MRI, MBB, MI, SrcReg, DstReg, GVal, IsAma);
ToErase = &MI;
Changed = true;
}
}
return Changed;
}
} // namespace
INITIALIZE_PASS(BPFMISimplifyPatchable, DEBUG_TYPE,
"BPF PreEmit SimplifyPatchable", false, false)
char BPFMISimplifyPatchable::ID = 0;
FunctionPass *llvm::createBPFMISimplifyPatchablePass() {
return new BPFMISimplifyPatchable();
}