blob: 11d1b309aa645e9bdcd2274393c4d965853c9bde [file] [log] [blame]
//===- XRayInstrumentation.cpp - Adds XRay instrumentation to functions. --===//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// This file implements a MachineFunctionPass that inserts the appropriate
// XRay instrumentation instructions. We look for XRay-specific attributes
// on the function to determine whether we should insert the replacement
// operations.
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Triple.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineDominators.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineLoopInfo.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/CodeGen/TargetSubtargetInfo.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/Function.h"
#include "llvm/InitializePasses.h"
#include "llvm/Pass.h"
#include "llvm/Target/TargetMachine.h"
using namespace llvm;
namespace {
struct InstrumentationOptions {
// Whether to emit PATCHABLE_TAIL_CALL.
bool HandleTailcall;
// Whether to emit PATCHABLE_RET/PATCHABLE_FUNCTION_EXIT for all forms of
// return, e.g. conditional return.
bool HandleAllReturns;
struct XRayInstrumentation : public MachineFunctionPass {
static char ID;
XRayInstrumentation() : MachineFunctionPass(ID) {
void getAnalysisUsage(AnalysisUsage &AU) const override {
bool runOnMachineFunction(MachineFunction &MF) override;
// Replace the original RET instruction with the exit sled code ("patchable
// ret" pseudo-instruction), so that at runtime XRay can replace the sled
// with a code jumping to XRay trampoline, which calls the tracing handler
// and, in the end, issues the RET instruction.
// This is the approach to go on CPUs which have a single RET instruction,
// like x86/x86_64.
void replaceRetWithPatchableRet(MachineFunction &MF,
const TargetInstrInfo *TII,
// Prepend the original return instruction with the exit sled code ("patchable
// function exit" pseudo-instruction), preserving the original return
// instruction just after the exit sled code.
// This is the approach to go on CPUs which have multiple options for the
// return instruction, like ARM. For such CPUs we can't just jump into the
// XRay trampoline and issue a single return instruction there. We rather
// have to call the trampoline and return from it to the original return
// instruction of the function being instrumented.
void prependRetWithPatchableExit(MachineFunction &MF,
const TargetInstrInfo *TII,
} // end anonymous namespace
void XRayInstrumentation::replaceRetWithPatchableRet(
MachineFunction &MF, const TargetInstrInfo *TII,
InstrumentationOptions op) {
// We look for *all* terminators and returns, then replace those with
// PATCHABLE_RET instructions.
SmallVector<MachineInstr *, 4> Terminators;
for (auto &MBB : MF) {
for (auto &T : MBB.terminators()) {
unsigned Opc = 0;
if (T.isReturn() &&
(op.HandleAllReturns || T.getOpcode() == TII->getReturnOpcode())) {
// Replace return instructions with:
// PATCHABLE_RET <Opcode>, <Operand>...
Opc = TargetOpcode::PATCHABLE_RET;
if (TII->isTailCall(T) && op.HandleTailcall) {
// Treat the tail call as a return instruction, which has a
// different-looking sled than the normal return case.
Opc = TargetOpcode::PATCHABLE_TAIL_CALL;
if (Opc != 0) {
auto MIB = BuildMI(MBB, T, T.getDebugLoc(), TII->get(Opc))
for (auto &MO : T.operands())
if (T.shouldUpdateCallSiteInfo())
for (auto &I : Terminators)
void XRayInstrumentation::prependRetWithPatchableExit(
MachineFunction &MF, const TargetInstrInfo *TII,
InstrumentationOptions op) {
for (auto &MBB : MF)
for (auto &T : MBB.terminators()) {
unsigned Opc = 0;
if (T.isReturn() &&
(op.HandleAllReturns || T.getOpcode() == TII->getReturnOpcode())) {
if (TII->isTailCall(T) && op.HandleTailcall) {
Opc = TargetOpcode::PATCHABLE_TAIL_CALL;
if (Opc != 0) {
// Prepend the return instruction with PATCHABLE_FUNCTION_EXIT or
BuildMI(MBB, T, T.getDebugLoc(), TII->get(Opc));
bool XRayInstrumentation::runOnMachineFunction(MachineFunction &MF) {
auto &F = MF.getFunction();
auto InstrAttr = F.getFnAttribute("function-instrument");
bool AlwaysInstrument = InstrAttr.isStringAttribute() &&
InstrAttr.getValueAsString() == "xray-always";
bool NeverInstrument = InstrAttr.isStringAttribute() &&
InstrAttr.getValueAsString() == "xray-never";
if (NeverInstrument && !AlwaysInstrument)
return false;
auto ThresholdAttr = F.getFnAttribute("xray-instruction-threshold");
auto IgnoreLoopsAttr = F.getFnAttribute("xray-ignore-loops");
unsigned int XRayThreshold = 0;
if (!AlwaysInstrument) {
if (!ThresholdAttr.isStringAttribute())
return false; // XRay threshold attribute not found.
if (ThresholdAttr.getValueAsString().getAsInteger(10, XRayThreshold))
return false; // Invalid value for threshold.
bool IgnoreLoops = IgnoreLoopsAttr.isValid();
// Count the number of MachineInstr`s in MachineFunction
int64_t MICount = 0;
for (const auto &MBB : MF)
MICount += MBB.size();
bool TooFewInstrs = MICount < XRayThreshold;
if (!IgnoreLoops) {
// Get MachineDominatorTree or compute it on the fly if it's unavailable
auto *MDT = getAnalysisIfAvailable<MachineDominatorTree>();
MachineDominatorTree ComputedMDT;
if (!MDT) {
MDT = &ComputedMDT;
// Get MachineLoopInfo or compute it on the fly if it's unavailable
auto *MLI = getAnalysisIfAvailable<MachineLoopInfo>();
MachineLoopInfo ComputedMLI;
if (!MLI) {
MLI = &ComputedMLI;
// Check if we have a loop.
// FIXME: Maybe make this smarter, and see whether the loops are dependent
// on inputs or side-effects?
if (MLI->empty() && TooFewInstrs)
return false; // Function is too small and has no loops.
} else if (TooFewInstrs) {
// Function is too small
return false;
// We look for the first non-empty MachineBasicBlock, so that we can insert
// the function instrumentation in the appropriate place.
auto MBI = llvm::find_if(
MF, [&](const MachineBasicBlock &MBB) { return !MBB.empty(); });
if (MBI == MF.end())
return false; // The function is empty.
auto *TII = MF.getSubtarget().getInstrInfo();
auto &FirstMBB = *MBI;
auto &FirstMI = *FirstMBB.begin();
if (!MF.getSubtarget().isXRaySupported()) {
FirstMI.emitError("An attempt to perform XRay instrumentation for an"
" unsupported target.");
return false;
if (!F.hasFnAttribute("xray-skip-entry")) {
// First, insert an PATCHABLE_FUNCTION_ENTER as the first instruction of the
// MachineFunction.
BuildMI(FirstMBB, FirstMI, FirstMI.getDebugLoc(),
if (!F.hasFnAttribute("xray-skip-exit")) {
switch (MF.getTarget().getTargetTriple().getArch()) {
case Triple::ArchType::arm:
case Triple::ArchType::thumb:
case Triple::ArchType::aarch64:
case Triple::ArchType::mips:
case Triple::ArchType::mipsel:
case Triple::ArchType::mips64:
case Triple::ArchType::mips64el: {
// For the architectures which don't have a single return instruction
InstrumentationOptions op;
op.HandleTailcall = false;
op.HandleAllReturns = true;
prependRetWithPatchableExit(MF, TII, op);
case Triple::ArchType::ppc64le: {
// PPC has conditional returns. Turn them into branch and plain returns.
InstrumentationOptions op;
op.HandleTailcall = false;
op.HandleAllReturns = true;
replaceRetWithPatchableRet(MF, TII, op);
default: {
// For the architectures that have a single return instruction (such as
// RETQ on x86_64).
InstrumentationOptions op;
op.HandleTailcall = true;
op.HandleAllReturns = false;
replaceRetWithPatchableRet(MF, TII, op);
return true;
char XRayInstrumentation::ID = 0;
char &llvm::XRayInstrumentationID = XRayInstrumentation::ID;
INITIALIZE_PASS_BEGIN(XRayInstrumentation, "xray-instrumentation",
"Insert XRay ops", false, false)
INITIALIZE_PASS_END(XRayInstrumentation, "xray-instrumentation",
"Insert XRay ops", false, false)