Google Git
Sign in
llvm / llvm-project.git / refs/heads/users/orodley/guid-2 / . / llvm / lib / Target / AArch64 / AArch64FrameLowering.cpp
blob: 885f2a94f85f537ca52dab61ca975561ecca7ce6 [file] [log] [blame]
//===- AArch64FrameLowering.cpp - AArch64 Frame Lowering -------*- C++ -*-====//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file contains the AArch64 implementation of TargetFrameLowering class.
//
// On AArch64, stack frames are structured as follows:
//
// The stack grows downward.
//
// All of the individual frame areas on the frame below are optional, i.e. it's
// possible to create a function so that the particular area isn't present
// in the frame.
//
// At function entry, the "frame" looks as follows:
//
// | | Higher address
// |-----------------------------------|
// | |
// | arguments passed on the stack |
// | |
// |-----------------------------------| <- sp
// | | Lower address
//
//
// After the prologue has run, the frame has the following general structure.
// Note that this doesn't depict the case where a red-zone is used. Also,
// technically the last frame area (VLAs) doesn't get created until in the
// main function body, after the prologue is run. However, it's depicted here
// for completeness.
//
// | | Higher address
// |-----------------------------------|
// | |
// | arguments passed on the stack |
// | |
// |-----------------------------------|
// | |
// | (Win64 only) varargs from reg |
// | |
// |-----------------------------------|
// | |
// | (Win64 only) callee-saved SVE reg |
// | |
// |-----------------------------------|
// | |
// | callee-saved gpr registers | <--.
// | | | On Darwin platforms these
// |- - - - - - - - - - - - - - - - - -| | callee saves are swapped,
// | prev_lr | | (frame record first)
// | prev_fp | <--'
// | async context if needed |
// | (a.k.a. "frame record") |
// |-----------------------------------| <- fp(=x29)
// | <hazard padding> |
// |-----------------------------------|
// | |
// | callee-saved fp/simd/SVE regs |
// | |
// |-----------------------------------|
// | |
// | SVE stack objects |
// | |
// |-----------------------------------|
// |.empty.space.to.make.part.below....|
// |.aligned.in.case.it.needs.more.than| (size of this area is unknown at
// |.the.standard.16-byte.alignment....| compile time; if present)
// |-----------------------------------|
// | local variables of fixed size |
// | including spill slots |
// | <FPR> |
// | <hazard padding> |
// | <GPR> |
// |-----------------------------------| <- bp(not defined by ABI,
// |.variable-sized.local.variables....| LLVM chooses X19)
// |.(VLAs)............................| (size of this area is unknown at
// |...................................| compile time)
// |-----------------------------------| <- sp
// | | Lower address
//
//
// To access the data in a frame, at-compile time, a constant offset must be
// computable from one of the pointers (fp, bp, sp) to access it. The size
// of the areas with a dotted background cannot be computed at compile-time
// if they are present, making it required to have all three of fp, bp and
// sp to be set up to be able to access all contents in the frame areas,
// assuming all of the frame areas are non-empty.
//
// For most functions, some of the frame areas are empty. For those functions,
// it may not be necessary to set up fp or bp:
// * A base pointer is definitely needed when there are both VLAs and local
// variables with more-than-default alignment requirements.
// * A frame pointer is definitely needed when there are local variables with
// more-than-default alignment requirements.
//
// For Darwin platforms the frame-record (fp, lr) is stored at the top of the
// callee-saved area, since the unwind encoding does not allow for encoding
// this dynamically and existing tools depend on this layout. For other
// platforms, the frame-record is stored at the bottom of the (gpr) callee-saved
// area to allow SVE stack objects (allocated directly below the callee-saves,
// if available) to be accessed directly from the framepointer.
// The SVE spill/fill instructions have VL-scaled addressing modes such
// as:
// ldr z8, [fp, #-7 mul vl]
// For SVE the size of the vector length (VL) is not known at compile-time, so
// '#-7 mul vl' is an offset that can only be evaluated at runtime. With this
// layout, we don't need to add an unscaled offset to the framepointer before
// accessing the SVE object in the frame.
//
// In some cases when a base pointer is not strictly needed, it is generated
// anyway when offsets from the frame pointer to access local variables become
// so large that the offset can't be encoded in the immediate fields of loads
// or stores.
//
// Outgoing function arguments must be at the bottom of the stack frame when
// calling another function. If we do not have variable-sized stack objects, we
// can allocate a "reserved call frame" area at the bottom of the local
// variable area, large enough for all outgoing calls. If we do have VLAs, then
// the stack pointer must be decremented and incremented around each call to
// make space for the arguments below the VLAs.
//
// FIXME: also explain the redzone concept.
//
// About stack hazards: Under some SME contexts, a coprocessor with its own
// separate cache can used for FP operations. This can create hazards if the CPU
// and the SME unit try to access the same area of memory, including if the
// access is to an area of the stack. To try to alleviate this we attempt to
// introduce extra padding into the stack frame between FP and GPR accesses,
// controlled by the aarch64-stack-hazard-size option. Without changing the
// layout of the stack frame in the diagram above, a stack object of size
// aarch64-stack-hazard-size is added between GPR and FPR CSRs. Another is added
// to the stack objects section, and stack objects are sorted so that FPR >
// Hazard padding slot > GPRs (where possible). Unfortunately some things are
// not handled well (VLA area, arguments on the stack, objects with both GPR and
// FPR accesses), but if those are controlled by the user then the entire stack
// frame becomes GPR at the start/end with FPR in the middle, surrounded by
// Hazard padding.
//
// An example of the prologue:
//
// .globl __foo
// .align 2
// __foo:
// Ltmp0:
// .cfi_startproc
// .cfi_personality 155, ___gxx_personality_v0
// Leh_func_begin:
// .cfi_lsda 16, Lexception33
//
// stp xa,bx, [sp, -#offset]!
// ...
// stp x28, x27, [sp, #offset-32]
// stp fp, lr, [sp, #offset-16]
// add fp, sp, #offset - 16
// sub sp, sp, #1360
//
// The Stack:
// +-------------------------------------------+
// 10000 | ........ | ........ | ........ | ........ |
// 10004 | ........ | ........ | ........ | ........ |
// +-------------------------------------------+
// 10008 | ........ | ........ | ........ | ........ |
// 1000c | ........ | ........ | ........ | ........ |
// +===========================================+
// 10010 | X28 Register |
// 10014 | X28 Register |
// +-------------------------------------------+
// 10018 | X27 Register |
// 1001c | X27 Register |
// +===========================================+
// 10020 | Frame Pointer |
// 10024 | Frame Pointer |
// +-------------------------------------------+
// 10028 | Link Register |
// 1002c | Link Register |
// +===========================================+
// 10030 | ........ | ........ | ........ | ........ |
// 10034 | ........ | ........ | ........ | ........ |
// +-------------------------------------------+
// 10038 | ........ | ........ | ........ | ........ |
// 1003c | ........ | ........ | ........ | ........ |
// +-------------------------------------------+
//
// [sp] = 10030 :: >>initial value<<
// sp = 10020 :: stp fp, lr, [sp, #-16]!
// fp = sp == 10020 :: mov fp, sp
// [sp] == 10020 :: stp x28, x27, [sp, #-16]!
// sp == 10010 :: >>final value<<
//
// The frame pointer (w29) points to address 10020. If we use an offset of
// '16' from 'w29', we get the CFI offsets of -8 for w30, -16 for w29, -24
// for w27, and -32 for w28:
//
// Ltmp1:
// .cfi_def_cfa w29, 16
// Ltmp2:
// .cfi_offset w30, -8
// Ltmp3:
// .cfi_offset w29, -16
// Ltmp4:
// .cfi_offset w27, -24
// Ltmp5:
// .cfi_offset w28, -32
//
//===----------------------------------------------------------------------===//
#include "AArch64FrameLowering.h"
#include "AArch64InstrInfo.h"
#include "AArch64MachineFunctionInfo.h"
#include "AArch64RegisterInfo.h"
#include "AArch64Subtarget.h"
#include "MCTargetDesc/AArch64AddressingModes.h"
#include "MCTargetDesc/AArch64MCTargetDesc.h"
#include "Utils/AArch64SMEAttributes.h"
#include "llvm/ADT/ScopeExit.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/CodeGen/CFIInstBuilder.h"
#include "llvm/CodeGen/LivePhysRegs.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineMemOperand.h"
#include "llvm/CodeGen/MachineModuleInfo.h"
#include "llvm/CodeGen/MachineOperand.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/RegisterScavenging.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/CodeGen/TargetRegisterInfo.h"
#include "llvm/CodeGen/TargetSubtargetInfo.h"
#include "llvm/CodeGen/WinEHFuncInfo.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/CallingConv.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DebugLoc.h"
#include "llvm/IR/Function.h"
#include "llvm/MC/MCAsmInfo.h"
#include "llvm/MC/MCDwarf.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/FormatVariadic.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Target/TargetOptions.h"
#include <cassert>
#include <cstdint>
#include <iterator>
#include <optional>
#include <vector>
using namespace llvm;
#define DEBUG_TYPE "frame-info"
static cl::opt<bool> EnableRedZone("aarch64-redzone",
cl::desc("enable use of redzone on AArch64"),
cl::init(false), cl::Hidden);
static cl::opt<bool> StackTaggingMergeSetTag(
"stack-tagging-merge-settag",
cl::desc("merge settag instruction in function epilog"), cl::init(true),
cl::Hidden);
static cl::opt<bool> OrderFrameObjects("aarch64-order-frame-objects",
cl::desc("sort stack allocations"),
cl::init(true), cl::Hidden);
cl::opt<bool> EnableHomogeneousPrologEpilog(
"homogeneous-prolog-epilog", cl::Hidden,
cl::desc("Emit homogeneous prologue and epilogue for the size "
"optimization (default = off)"));
// Stack hazard size for analysis remarks. StackHazardSize takes precedence.
static cl::opt<unsigned>
StackHazardRemarkSize("aarch64-stack-hazard-remark-size", cl::init(0),
cl::Hidden);
// Whether to insert padding into non-streaming functions (for testing).
static cl::opt<bool>
StackHazardInNonStreaming("aarch64-stack-hazard-in-non-streaming",
cl::init(false), cl::Hidden);
static cl::opt<bool> DisableMultiVectorSpillFill(
"aarch64-disable-multivector-spill-fill",
cl::desc("Disable use of LD/ST pairs for SME2 or SVE2p1"), cl::init(false),
cl::Hidden);
STATISTIC(NumRedZoneFunctions, "Number of functions using red zone");
/// Returns how much of the incoming argument stack area (in bytes) we should
/// clean up in an epilogue. For the C calling convention this will be 0, for
/// guaranteed tail call conventions it can be positive (a normal return or a
/// tail call to a function that uses less stack space for arguments) or
/// negative (for a tail call to a function that needs more stack space than us
/// for arguments).
static int64_t getArgumentStackToRestore(MachineFunction &MF,
MachineBasicBlock &MBB) {
MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr();
AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
bool IsTailCallReturn = (MBB.end() != MBBI)
? AArch64InstrInfo::isTailCallReturnInst(*MBBI)
: false;
int64_t ArgumentPopSize = 0;
if (IsTailCallReturn) {
MachineOperand &StackAdjust = MBBI->getOperand(1);
// For a tail-call in a callee-pops-arguments environment, some or all of
// the stack may actually be in use for the call's arguments, this is
// calculated during LowerCall and consumed here...
ArgumentPopSize = StackAdjust.getImm();
} else {
// ... otherwise the amount to pop is *all* of the argument space,
// conveniently stored in the MachineFunctionInfo by
// LowerFormalArguments. This will, of course, be zero for the C calling
// convention.
ArgumentPopSize = AFI->getArgumentStackToRestore();
}
return ArgumentPopSize;
}
static bool produceCompactUnwindFrame(MachineFunction &MF);
static bool needsWinCFI(const MachineFunction &MF);
static StackOffset getSVEStackSize(const MachineFunction &MF);
static Register findScratchNonCalleeSaveRegister(MachineBasicBlock *MBB,
bool HasCall = false);
static bool requiresSaveVG(const MachineFunction &MF);
// Conservatively, returns true if the function is likely to have an SVE vectors
// on the stack. This function is safe to be called before callee-saves or
// object offsets have been determined.
static bool isLikelyToHaveSVEStack(MachineFunction &MF) {
auto *AFI = MF.getInfo<AArch64FunctionInfo>();
if (AFI->isSVECC())
return true;
if (AFI->hasCalculatedStackSizeSVE())
return bool(getSVEStackSize(MF));
const MachineFrameInfo &MFI = MF.getFrameInfo();
for (int FI = MFI.getObjectIndexBegin(); FI < MFI.getObjectIndexEnd(); FI++) {
if (MFI.getStackID(FI) == TargetStackID::ScalableVector)
return true;
}
return false;
}
/// Returns true if a homogeneous prolog or epilog code can be emitted
/// for the size optimization. If possible, a frame helper call is injected.
/// When Exit block is given, this check is for epilog.
bool AArch64FrameLowering::homogeneousPrologEpilog(
MachineFunction &MF, MachineBasicBlock *Exit) const {
if (!MF.getFunction().hasMinSize())
return false;
if (!EnableHomogeneousPrologEpilog)
return false;
if (EnableRedZone)
return false;
// TODO: Window is supported yet.
if (needsWinCFI(MF))
return false;
// TODO: SVE is not supported yet.
if (isLikelyToHaveSVEStack(MF))
return false;
// Bail on stack adjustment needed on return for simplicity.
const MachineFrameInfo &MFI = MF.getFrameInfo();
const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
if (MFI.hasVarSizedObjects() || RegInfo->hasStackRealignment(MF))
return false;
if (Exit && getArgumentStackToRestore(MF, *Exit))
return false;
auto *AFI = MF.getInfo<AArch64FunctionInfo>();
if (AFI->hasSwiftAsyncContext() || AFI->hasStreamingModeChanges())
return false;
// If there are an odd number of GPRs before LR and FP in the CSRs list,
// they will not be paired into one RegPairInfo, which is incompatible with
// the assumption made by the homogeneous prolog epilog pass.
const MCPhysReg *CSRegs = MF.getRegInfo().getCalleeSavedRegs();
unsigned NumGPRs = 0;
for (unsigned I = 0; CSRegs[I]; ++I) {
Register Reg = CSRegs[I];
if (Reg == AArch64::LR) {
assert(CSRegs[I + 1] == AArch64::FP);
if (NumGPRs % 2 != 0)
return false;
break;
}
if (AArch64::GPR64RegClass.contains(Reg))
++NumGPRs;
}
return true;
}
/// Returns true if CSRs should be paired.
bool AArch64FrameLowering::producePairRegisters(MachineFunction &MF) const {
return produceCompactUnwindFrame(MF) || homogeneousPrologEpilog(MF);
}
/// This is the biggest offset to the stack pointer we can encode in aarch64
/// instructions (without using a separate calculation and a temp register).
/// Note that the exception here are vector stores/loads which cannot encode any
/// displacements (see estimateRSStackSizeLimit(), isAArch64FrameOffsetLegal()).
static const unsigned DefaultSafeSPDisplacement = 255;
/// Look at each instruction that references stack frames and return the stack
/// size limit beyond which some of these instructions will require a scratch
/// register during their expansion later.
static unsigned estimateRSStackSizeLimit(MachineFunction &MF) {
// FIXME: For now, just conservatively guesstimate based on unscaled indexing
// range. We'll end up allocating an unnecessary spill slot a lot, but
// realistically that's not a big deal at this stage of the game.
for (MachineBasicBlock &MBB : MF) {
for (MachineInstr &MI : MBB) {
if (MI.isDebugInstr() || MI.isPseudo() ||
MI.getOpcode() == AArch64::ADDXri ||
MI.getOpcode() == AArch64::ADDSXri)
continue;
for (const MachineOperand &MO : MI.operands()) {
if (!MO.isFI())
continue;
StackOffset Offset;
if (isAArch64FrameOffsetLegal(MI, Offset, nullptr, nullptr, nullptr) ==
AArch64FrameOffsetCannotUpdate)
return 0;
}
}
}
return DefaultSafeSPDisplacement;
}
TargetStackID::Value
AArch64FrameLowering::getStackIDForScalableVectors() const {
return TargetStackID::ScalableVector;
}
/// Returns the size of the fixed object area (allocated next to sp on entry)
/// On Win64 this may include a var args area and an UnwindHelp object for EH.
static unsigned getFixedObjectSize(const MachineFunction &MF,
const AArch64FunctionInfo *AFI, bool IsWin64,
bool IsFunclet) {
assert(AFI->getTailCallReservedStack() % 16 == 0 &&
"Tail call reserved stack must be aligned to 16 bytes");
if (!IsWin64 || IsFunclet) {
return AFI->getTailCallReservedStack();
} else {
if (AFI->getTailCallReservedStack() != 0 &&
!MF.getFunction().getAttributes().hasAttrSomewhere(
Attribute::SwiftAsync))
report_fatal_error("cannot generate ABI-changing tail call for Win64");
unsigned FixedObjectSize = AFI->getTailCallReservedStack();
// Var args are stored here in the primary function.
FixedObjectSize += AFI->getVarArgsGPRSize();
if (MF.hasEHFunclets()) {
// Catch objects are stored here in the primary function.
const MachineFrameInfo &MFI = MF.getFrameInfo();
const WinEHFuncInfo &EHInfo = *MF.getWinEHFuncInfo();
SmallSetVector<int, 8> CatchObjFrameIndices;
for (const WinEHTryBlockMapEntry &TBME : EHInfo.TryBlockMap) {
for (const WinEHHandlerType &H : TBME.HandlerArray) {
int FrameIndex = H.CatchObj.FrameIndex;
if ((FrameIndex != INT_MAX) &&
CatchObjFrameIndices.insert(FrameIndex)) {
FixedObjectSize = alignTo(FixedObjectSize,
MFI.getObjectAlign(FrameIndex).value()) +
MFI.getObjectSize(FrameIndex);
}
}
}
// To support EH funclets we allocate an UnwindHelp object
FixedObjectSize += 8;
}
return alignTo(FixedObjectSize, 16);
}
}
/// Returns the size of the entire SVE stackframe (calleesaves + spills).
static StackOffset getSVEStackSize(const MachineFunction &MF) {
const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
return StackOffset::getScalable((int64_t)AFI->getStackSizeSVE());
}
bool AArch64FrameLowering::canUseRedZone(const MachineFunction &MF) const {
if (!EnableRedZone)
return false;
// Don't use the red zone if the function explicitly asks us not to.
// This is typically used for kernel code.
const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
const unsigned RedZoneSize =
Subtarget.getTargetLowering()->getRedZoneSize(MF.getFunction());
if (!RedZoneSize)
return false;
const MachineFrameInfo &MFI = MF.getFrameInfo();
const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
uint64_t NumBytes = AFI->getLocalStackSize();
// If neither NEON or SVE are available, a COPY from one Q-reg to
// another requires a spill -> reload sequence. We can do that
// using a pre-decrementing store/post-decrementing load, but
// if we do so, we can't use the Red Zone.
bool LowerQRegCopyThroughMem = Subtarget.hasFPARMv8() &&
!Subtarget.isNeonAvailable() &&
!Subtarget.hasSVE();
return !(MFI.hasCalls() || hasFP(MF) || NumBytes > RedZoneSize ||
getSVEStackSize(MF) || LowerQRegCopyThroughMem);
}
/// hasFPImpl - Return true if the specified function should have a dedicated
/// frame pointer register.
bool AArch64FrameLowering::hasFPImpl(const MachineFunction &MF) const {
const MachineFrameInfo &MFI = MF.getFrameInfo();
const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
// Win64 EH requires a frame pointer if funclets are present, as the locals
// are accessed off the frame pointer in both the parent function and the
// funclets.
if (MF.hasEHFunclets())
return true;
// Retain behavior of always omitting the FP for leaf functions when possible.
if (MF.getTarget().Options.DisableFramePointerElim(MF))
return true;
if (MFI.hasVarSizedObjects() || MFI.isFrameAddressTaken() ||
MFI.hasStackMap() || MFI.hasPatchPoint() ||
RegInfo->hasStackRealignment(MF))
return true;
// With large callframes around we may need to use FP to access the scavenging
// emergency spillslot.
//
// Unfortunately some calls to hasFP() like machine verifier ->
// getReservedReg() -> hasFP in the middle of global isel are too early
// to know the max call frame size. Hopefully conservatively returning "true"
// in those cases is fine.
// DefaultSafeSPDisplacement is fine as we only emergency spill GP regs.
if (!MFI.isMaxCallFrameSizeComputed() ||
MFI.getMaxCallFrameSize() > DefaultSafeSPDisplacement)
return true;
return false;
}
/// Should the Frame Pointer be reserved for the current function?
bool AArch64FrameLowering::isFPReserved(const MachineFunction &MF) const {
const TargetMachine &TM = MF.getTarget();
const Triple &TT = TM.getTargetTriple();
// These OSes require the frame chain is valid, even if the current frame does
// not use a frame pointer.
if (TT.isOSDarwin() || TT.isOSWindows())
return true;
// If the function has a frame pointer, it is reserved.
if (hasFP(MF))
return true;
// Frontend has requested to preserve the frame pointer.
if (TM.Options.FramePointerIsReserved(MF))
return true;
return false;
}
/// hasReservedCallFrame - Under normal circumstances, when a frame pointer is
/// not required, we reserve argument space for call sites in the function
/// immediately on entry to the current function. This eliminates the need for
/// add/sub sp brackets around call sites. Returns true if the call frame is
/// included as part of the stack frame.
bool AArch64FrameLowering::hasReservedCallFrame(
const MachineFunction &MF) const {
// The stack probing code for the dynamically allocated outgoing arguments
// area assumes that the stack is probed at the top - either by the prologue
// code, which issues a probe if `hasVarSizedObjects` return true, or by the
// most recent variable-sized object allocation. Changing the condition here
// may need to be followed up by changes to the probe issuing logic.
return !MF.getFrameInfo().hasVarSizedObjects();
}
MachineBasicBlock::iterator AArch64FrameLowering::eliminateCallFramePseudoInstr(
MachineFunction &MF, MachineBasicBlock &MBB,
MachineBasicBlock::iterator I) const {
const AArch64InstrInfo *TII =
static_cast<const AArch64InstrInfo *>(MF.getSubtarget().getInstrInfo());
const AArch64TargetLowering *TLI =
MF.getSubtarget<AArch64Subtarget>().getTargetLowering();
[[maybe_unused]] MachineFrameInfo &MFI = MF.getFrameInfo();
DebugLoc DL = I->getDebugLoc();
unsigned Opc = I->getOpcode();
bool IsDestroy = Opc == TII->getCallFrameDestroyOpcode();
uint64_t CalleePopAmount = IsDestroy ? I->getOperand(1).getImm() : 0;
if (!hasReservedCallFrame(MF)) {
int64_t Amount = I->getOperand(0).getImm();
Amount = alignTo(Amount, getStackAlign());
if (!IsDestroy)
Amount = -Amount;
// N.b. if CalleePopAmount is valid but zero (i.e. callee would pop, but it
// doesn't have to pop anything), then the first operand will be zero too so
// this adjustment is a no-op.
if (CalleePopAmount == 0) {
// FIXME: in-function stack adjustment for calls is limited to 24-bits
// because there's no guaranteed temporary register available.
//
// ADD/SUB (immediate) has only LSL #0 and LSL #12 available.
// 1) For offset <= 12-bit, we use LSL #0
// 2) For 12-bit <= offset <= 24-bit, we use two instructions. One uses
// LSL #0, and the other uses LSL #12.
//
// Most call frames will be allocated at the start of a function so
// this is OK, but it is a limitation that needs dealing with.
assert(Amount > -0xffffff && Amount < 0xffffff && "call frame too large");
if (TLI->hasInlineStackProbe(MF) &&
-Amount >= AArch64::StackProbeMaxUnprobedStack) {
// When stack probing is enabled, the decrement of SP may need to be
// probed. We only need to do this if the call site needs 1024 bytes of
// space or more, because a region smaller than that is allowed to be
// unprobed at an ABI boundary. We rely on the fact that SP has been
// probed exactly at this point, either by the prologue or most recent
// dynamic allocation.
assert(MFI.hasVarSizedObjects() &&
"non-reserved call frame without var sized objects?");
Register ScratchReg =
MF.getRegInfo().createVirtualRegister(&AArch64::GPR64RegClass);
inlineStackProbeFixed(I, ScratchReg, -Amount, StackOffset::get(0, 0));
} else {
emitFrameOffset(MBB, I, DL, AArch64::SP, AArch64::SP,
StackOffset::getFixed(Amount), TII);
}
}
} else if (CalleePopAmount != 0) {
// If the calling convention demands that the callee pops arguments from the
// stack, we want to add it back if we have a reserved call frame.
assert(CalleePopAmount < 0xffffff && "call frame too large");
emitFrameOffset(MBB, I, DL, AArch64::SP, AArch64::SP,
StackOffset::getFixed(-(int64_t)CalleePopAmount), TII);
}
return MBB.erase(I);
}
void AArch64FrameLowering::emitCalleeSavedGPRLocations(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const {
MachineFunction &MF = *MBB.getParent();
MachineFrameInfo &MFI = MF.getFrameInfo();
AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
SMEAttrs Attrs = AFI->getSMEFnAttrs();
bool LocallyStreaming =
Attrs.hasStreamingBody() && !Attrs.hasStreamingInterface();
const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
if (CSI.empty())
return;
CFIInstBuilder CFIBuilder(MBB, MBBI, MachineInstr::FrameSetup);
for (const auto &Info : CSI) {
unsigned FrameIdx = Info.getFrameIdx();
if (MFI.getStackID(FrameIdx) == TargetStackID::ScalableVector)
continue;
assert(!Info.isSpilledToReg() && "Spilling to registers not implemented");
int64_t Offset = MFI.getObjectOffset(FrameIdx) - getOffsetOfLocalArea();
// The location of VG will be emitted before each streaming-mode change in
// the function. Only locally-streaming functions require emitting the
// non-streaming VG location here.
if ((LocallyStreaming && FrameIdx == AFI->getStreamingVGIdx()) ||
(!LocallyStreaming && Info.getReg() == AArch64::VG))
continue;
CFIBuilder.buildOffset(Info.getReg(), Offset);
}
}
void AArch64FrameLowering::emitCalleeSavedSVELocations(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const {
MachineFunction &MF = *MBB.getParent();
MachineFrameInfo &MFI = MF.getFrameInfo();
// Add callee saved registers to move list.
const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
if (CSI.empty())
return;
const TargetSubtargetInfo &STI = MF.getSubtarget();
const TargetRegisterInfo &TRI = *STI.getRegisterInfo();
AArch64FunctionInfo &AFI = *MF.getInfo<AArch64FunctionInfo>();
CFIInstBuilder CFIBuilder(MBB, MBBI, MachineInstr::FrameSetup);
for (const auto &Info : CSI) {
if (!(MFI.getStackID(Info.getFrameIdx()) == TargetStackID::ScalableVector))
continue;
// Not all unwinders may know about SVE registers, so assume the lowest
// common denominator.
assert(!Info.isSpilledToReg() && "Spilling to registers not implemented");
MCRegister Reg = Info.getReg();
if (!static_cast<const AArch64RegisterInfo &>(TRI).regNeedsCFI(Reg, Reg))
continue;
StackOffset Offset =
StackOffset::getScalable(MFI.getObjectOffset(Info.getFrameIdx())) -
StackOffset::getFixed(AFI.getCalleeSavedStackSize(MFI));
CFIBuilder.insertCFIInst(createCFAOffset(TRI, Reg, Offset));
}
}
void AArch64FrameLowering::resetCFIToInitialState(
MachineBasicBlock &MBB) const {
MachineFunction &MF = *MBB.getParent();
const auto &Subtarget = MF.getSubtarget<AArch64Subtarget>();
const auto &TRI = *Subtarget.getRegisterInfo();
const auto &MFI = *MF.getInfo<AArch64FunctionInfo>();
CFIInstBuilder CFIBuilder(MBB, MBB.begin(), MachineInstr::NoFlags);
// Reset the CFA to `SP + 0`.
CFIBuilder.buildDefCFA(AArch64::SP, 0);
// Flip the RA sign state.
if (MFI.shouldSignReturnAddress(MF))
MFI.branchProtectionPAuthLR() ? CFIBuilder.buildNegateRAStateWithPC()
: CFIBuilder.buildNegateRAState();
// Shadow call stack uses X18, reset it.
if (MFI.needsShadowCallStackPrologueEpilogue(MF))
CFIBuilder.buildSameValue(AArch64::X18);
// Emit .cfi_same_value for callee-saved registers.
const std::vector<CalleeSavedInfo> &CSI =
MF.getFrameInfo().getCalleeSavedInfo();
for (const auto &Info : CSI) {
MCRegister Reg = Info.getReg();
if (!TRI.regNeedsCFI(Reg, Reg))
continue;
CFIBuilder.buildSameValue(Reg);
}
}
static void emitCalleeSavedRestores(MachineBasicBlock &MBB,
MachineBasicBlock::iterator MBBI,
bool SVE) {
MachineFunction &MF = *MBB.getParent();
MachineFrameInfo &MFI = MF.getFrameInfo();
const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
if (CSI.empty())
return;
const TargetSubtargetInfo &STI = MF.getSubtarget();
const TargetRegisterInfo &TRI = *STI.getRegisterInfo();
CFIInstBuilder CFIBuilder(MBB, MBBI, MachineInstr::FrameDestroy);
for (const auto &Info : CSI) {
if (SVE !=
(MFI.getStackID(Info.getFrameIdx()) == TargetStackID::ScalableVector))
continue;
MCRegister Reg = Info.getReg();
if (SVE &&
!static_cast<const AArch64RegisterInfo &>(TRI).regNeedsCFI(Reg, Reg))
continue;
if (!Info.isRestored())
continue;
CFIBuilder.buildRestore(Info.getReg());
}
}
void AArch64FrameLowering::emitCalleeSavedGPRRestores(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const {
emitCalleeSavedRestores(MBB, MBBI, false);
}
void AArch64FrameLowering::emitCalleeSavedSVERestores(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const {
emitCalleeSavedRestores(MBB, MBBI, true);
}
// Return the maximum possible number of bytes for `Size` due to the
// architectural limit on the size of a SVE register.
static int64_t upperBound(StackOffset Size) {
static const int64_t MAX_BYTES_PER_SCALABLE_BYTE = 16;
return Size.getScalable() * MAX_BYTES_PER_SCALABLE_BYTE + Size.getFixed();
}
void AArch64FrameLowering::allocateStackSpace(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
int64_t RealignmentPadding, StackOffset AllocSize, bool NeedsWinCFI,
bool *HasWinCFI, bool EmitCFI, StackOffset InitialOffset,
bool FollowupAllocs) const {
if (!AllocSize)
return;
DebugLoc DL;
MachineFunction &MF = *MBB.getParent();
const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
const TargetInstrInfo &TII = *Subtarget.getInstrInfo();
AArch64FunctionInfo &AFI = *MF.getInfo<AArch64FunctionInfo>();
const MachineFrameInfo &MFI = MF.getFrameInfo();
const int64_t MaxAlign = MFI.getMaxAlign().value();
const uint64_t AndMask = ~(MaxAlign - 1);
if (!Subtarget.getTargetLowering()->hasInlineStackProbe(MF)) {
Register TargetReg = RealignmentPadding
? findScratchNonCalleeSaveRegister(&MBB)
: AArch64::SP;
// SUB Xd/SP, SP, AllocSize
emitFrameOffset(MBB, MBBI, DL, TargetReg, AArch64::SP, -AllocSize, &TII,
MachineInstr::FrameSetup, false, NeedsWinCFI, HasWinCFI,
EmitCFI, InitialOffset);
if (RealignmentPadding) {
// AND SP, X9, 0b11111...0000
BuildMI(MBB, MBBI, DL, TII.get(AArch64::ANDXri), AArch64::SP)
.addReg(TargetReg, RegState::Kill)
.addImm(AArch64_AM::encodeLogicalImmediate(AndMask, 64))
.setMIFlags(MachineInstr::FrameSetup);
AFI.setStackRealigned(true);
// No need for SEH instructions here; if we're realigning the stack,
// we've set a frame pointer and already finished the SEH prologue.
assert(!NeedsWinCFI);
}
return;
}
//
// Stack probing allocation.
//
// Fixed length allocation. If we don't need to re-align the stack and don't
// have SVE objects, we can use a more efficient sequence for stack probing.
if (AllocSize.getScalable() == 0 && RealignmentPadding == 0) {
Register ScratchReg = findScratchNonCalleeSaveRegister(&MBB);
assert(ScratchReg != AArch64::NoRegister);
BuildMI(MBB, MBBI, DL, TII.get(AArch64::PROBED_STACKALLOC))
.addDef(ScratchReg)
.addImm(AllocSize.getFixed())
.addImm(InitialOffset.getFixed())
.addImm(InitialOffset.getScalable());
// The fixed allocation may leave unprobed bytes at the top of the
// stack. If we have subsequent allocation (e.g. if we have variable-sized
// objects), we need to issue an extra probe, so these allocations start in
// a known state.
if (FollowupAllocs) {
// STR XZR, [SP]
BuildMI(MBB, MBBI, DL, TII.get(AArch64::STRXui))
.addReg(AArch64::XZR)
.addReg(AArch64::SP)
.addImm(0)
.setMIFlags(MachineInstr::FrameSetup);
}
return;
}
// Variable length allocation.
// If the (unknown) allocation size cannot exceed the probe size, decrement
// the stack pointer right away.
int64_t ProbeSize = AFI.getStackProbeSize();
if (upperBound(AllocSize) + RealignmentPadding <= ProbeSize) {
Register ScratchReg = RealignmentPadding
? findScratchNonCalleeSaveRegister(&MBB)
: AArch64::SP;
assert(ScratchReg != AArch64::NoRegister);
// SUB Xd, SP, AllocSize
emitFrameOffset(MBB, MBBI, DL, ScratchReg, AArch64::SP, -AllocSize, &TII,
MachineInstr::FrameSetup, false, NeedsWinCFI, HasWinCFI,
EmitCFI, InitialOffset);
if (RealignmentPadding) {
// AND SP, Xn, 0b11111...0000
BuildMI(MBB, MBBI, DL, TII.get(AArch64::ANDXri), AArch64::SP)
.addReg(ScratchReg, RegState::Kill)
.addImm(AArch64_AM::encodeLogicalImmediate(AndMask, 64))
.setMIFlags(MachineInstr::FrameSetup);
AFI.setStackRealigned(true);
}
if (FollowupAllocs || upperBound(AllocSize) + RealignmentPadding >
AArch64::StackProbeMaxUnprobedStack) {
// STR XZR, [SP]
BuildMI(MBB, MBBI, DL, TII.get(AArch64::STRXui))
.addReg(AArch64::XZR)
.addReg(AArch64::SP)
.addImm(0)
.setMIFlags(MachineInstr::FrameSetup);
}
return;
}
// Emit a variable-length allocation probing loop.
// TODO: As an optimisation, the loop can be "unrolled" into a few parts,
// each of them guaranteed to adjust the stack by less than the probe size.
Register TargetReg = findScratchNonCalleeSaveRegister(&MBB);
assert(TargetReg != AArch64::NoRegister);
// SUB Xd, SP, AllocSize
emitFrameOffset(MBB, MBBI, DL, TargetReg, AArch64::SP, -AllocSize, &TII,
MachineInstr::FrameSetup, false, NeedsWinCFI, HasWinCFI,
EmitCFI, InitialOffset);
if (RealignmentPadding) {
// AND Xn, Xn, 0b11111...0000
BuildMI(MBB, MBBI, DL, TII.get(AArch64::ANDXri), TargetReg)
.addReg(TargetReg, RegState::Kill)
.addImm(AArch64_AM::encodeLogicalImmediate(AndMask, 64))
.setMIFlags(MachineInstr::FrameSetup);
}
BuildMI(MBB, MBBI, DL, TII.get(AArch64::PROBED_STACKALLOC_VAR))
.addReg(TargetReg);
if (EmitCFI) {
// Set the CFA register back to SP.
CFIInstBuilder(MBB, MBBI, MachineInstr::FrameSetup)
.buildDefCFARegister(AArch64::SP);
}
if (RealignmentPadding)
AFI.setStackRealigned(true);
}
static MCRegister getRegisterOrZero(MCRegister Reg, bool HasSVE) {
switch (Reg.id()) {
default:
// The called routine is expected to preserve r19-r28
// r29 and r30 are used as frame pointer and link register resp.
return 0;
// GPRs
#define CASE(n) \
case AArch64::W##n: \
case AArch64::X##n: \
return AArch64::X##n
CASE(0);
CASE(1);
CASE(2);
CASE(3);
CASE(4);
CASE(5);
CASE(6);
CASE(7);
CASE(8);
CASE(9);
CASE(10);
CASE(11);
CASE(12);
CASE(13);
CASE(14);
CASE(15);
CASE(16);
CASE(17);
CASE(18);
#undef CASE
// FPRs
#define CASE(n) \
case AArch64::B##n: \
case AArch64::H##n: \
case AArch64::S##n: \
case AArch64::D##n: \
case AArch64::Q##n: \
return HasSVE ? AArch64::Z##n : AArch64::Q##n
CASE(0);
CASE(1);
CASE(2);
CASE(3);
CASE(4);
CASE(5);
CASE(6);
CASE(7);
CASE(8);
CASE(9);
CASE(10);
CASE(11);
CASE(12);
CASE(13);
CASE(14);
CASE(15);
CASE(16);
CASE(17);
CASE(18);
CASE(19);
CASE(20);
CASE(21);
CASE(22);
CASE(23);
CASE(24);
CASE(25);
CASE(26);
CASE(27);
CASE(28);
CASE(29);
CASE(30);
CASE(31);
#undef CASE
}
}
void AArch64FrameLowering::emitZeroCallUsedRegs(BitVector RegsToZero,
MachineBasicBlock &MBB) const {
// Insertion point.
MachineBasicBlock::iterator MBBI = MBB.getFirstTerminator();
// Fake a debug loc.
DebugLoc DL;
if (MBBI != MBB.end())
DL = MBBI->getDebugLoc();
const MachineFunction &MF = *MBB.getParent();
const AArch64Subtarget &STI = MF.getSubtarget<AArch64Subtarget>();
const AArch64RegisterInfo &TRI = *STI.getRegisterInfo();
BitVector GPRsToZero(TRI.getNumRegs());
BitVector FPRsToZero(TRI.getNumRegs());
bool HasSVE = STI.isSVEorStreamingSVEAvailable();
for (MCRegister Reg : RegsToZero.set_bits()) {
if (TRI.isGeneralPurposeRegister(MF, Reg)) {
// For GPRs, we only care to clear out the 64-bit register.
if (MCRegister XReg = getRegisterOrZero(Reg, HasSVE))
GPRsToZero.set(XReg);
} else if (AArch64InstrInfo::isFpOrNEON(Reg)) {
// For FPRs,
if (MCRegister XReg = getRegisterOrZero(Reg, HasSVE))
FPRsToZero.set(XReg);
}
}
const AArch64InstrInfo &TII = *STI.getInstrInfo();
// Zero out GPRs.
for (MCRegister Reg : GPRsToZero.set_bits())
TII.buildClearRegister(Reg, MBB, MBBI, DL);
// Zero out FP/vector registers.
for (MCRegister Reg : FPRsToZero.set_bits())
TII.buildClearRegister(Reg, MBB, MBBI, DL);
if (HasSVE) {
for (MCRegister PReg :
{AArch64::P0, AArch64::P1, AArch64::P2, AArch64::P3, AArch64::P4,
AArch64::P5, AArch64::P6, AArch64::P7, AArch64::P8, AArch64::P9,
AArch64::P10, AArch64::P11, AArch64::P12, AArch64::P13, AArch64::P14,
AArch64::P15}) {
if (RegsToZero[PReg])
BuildMI(MBB, MBBI, DL, TII.get(AArch64::PFALSE), PReg);
}
}
}
static bool windowsRequiresStackProbe(const MachineFunction &MF,
uint64_t StackSizeInBytes) {
const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
const AArch64FunctionInfo &MFI = *MF.getInfo<AArch64FunctionInfo>();
// TODO: When implementing stack protectors, take that into account
// for the probe threshold.
return Subtarget.isTargetWindows() && MFI.hasStackProbing() &&
StackSizeInBytes >= uint64_t(MFI.getStackProbeSize());
}
static void getLiveRegsForEntryMBB(LivePhysRegs &LiveRegs,
const MachineBasicBlock &MBB) {
const MachineFunction *MF = MBB.getParent();
LiveRegs.addLiveIns(MBB);
// Mark callee saved registers as used so we will not choose them.
const MCPhysReg *CSRegs = MF->getRegInfo().getCalleeSavedRegs();
for (unsigned i = 0; CSRegs[i]; ++i)
LiveRegs.addReg(CSRegs[i]);
}
// Find a scratch register that we can use at the start of the prologue to
// re-align the stack pointer. We avoid using callee-save registers since they
// may appear to be free when this is called from canUseAsPrologue (during
// shrink wrapping), but then no longer be free when this is called from
// emitPrologue.
//
// FIXME: This is a bit conservative, since in the above case we could use one
// of the callee-save registers as a scratch temp to re-align the stack pointer,
// but we would then have to make sure that we were in fact saving at least one
// callee-save register in the prologue, which is additional complexity that
// doesn't seem worth the benefit.
static Register findScratchNonCalleeSaveRegister(MachineBasicBlock *MBB,
bool HasCall) {
MachineFunction *MF = MBB->getParent();
// If MBB is an entry block, use X9 as the scratch register
// preserve_none functions may be using X9 to pass arguments,
// so prefer to pick an available register below.
if (&MF->front() == MBB &&
MF->getFunction().getCallingConv() != CallingConv::PreserveNone)
return AArch64::X9;
const AArch64Subtarget &Subtarget = MF->getSubtarget<AArch64Subtarget>();
const AArch64RegisterInfo &TRI = *Subtarget.getRegisterInfo();
LivePhysRegs LiveRegs(TRI);
getLiveRegsForEntryMBB(LiveRegs, *MBB);
if (HasCall) {
LiveRegs.addReg(AArch64::X16);
LiveRegs.addReg(AArch64::X17);
LiveRegs.addReg(AArch64::X18);
}
// Prefer X9 since it was historically used for the prologue scratch reg.
const MachineRegisterInfo &MRI = MF->getRegInfo();
if (LiveRegs.available(MRI, AArch64::X9))
return AArch64::X9;
for (unsigned Reg : AArch64::GPR64RegClass) {
if (LiveRegs.available(MRI, Reg))
return Reg;
}
return AArch64::NoRegister;
}
bool AArch64FrameLowering::canUseAsPrologue(
const MachineBasicBlock &MBB) const {
const MachineFunction *MF = MBB.getParent();
MachineBasicBlock *TmpMBB = const_cast<MachineBasicBlock *>(&MBB);
const AArch64Subtarget &Subtarget = MF->getSubtarget<AArch64Subtarget>();
const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
const AArch64TargetLowering *TLI = Subtarget.getTargetLowering();
const AArch64FunctionInfo *AFI = MF->getInfo<AArch64FunctionInfo>();
if (AFI->hasSwiftAsyncContext()) {
const AArch64RegisterInfo &TRI = *Subtarget.getRegisterInfo();
const MachineRegisterInfo &MRI = MF->getRegInfo();
LivePhysRegs LiveRegs(TRI);
getLiveRegsForEntryMBB(LiveRegs, MBB);
// The StoreSwiftAsyncContext clobbers X16 and X17. Make sure they are
// available.
if (!LiveRegs.available(MRI, AArch64::X16) ||
!LiveRegs.available(MRI, AArch64::X17))
return false;
}
// Certain stack probing sequences might clobber flags, then we can't use
// the block as a prologue if the flags register is a live-in.
if (MF->getInfo<AArch64FunctionInfo>()->hasStackProbing() &&
MBB.isLiveIn(AArch64::NZCV))
return false;
if (RegInfo->hasStackRealignment(*MF) || TLI->hasInlineStackProbe(*MF))
if (findScratchNonCalleeSaveRegister(TmpMBB) == AArch64::NoRegister)
return false;
// May need a scratch register (for return value) if require making a special
// call
if (requiresSaveVG(*MF) ||
windowsRequiresStackProbe(*MF, std::numeric_limits<uint64_t>::max()))
if (findScratchNonCalleeSaveRegister(TmpMBB, true) == AArch64::NoRegister)
return false;
return true;
}
static bool needsWinCFI(const MachineFunction &MF) {
const Function &F = MF.getFunction();
return MF.getTarget().getMCAsmInfo()->usesWindowsCFI() &&
F.needsUnwindTableEntry();
}
static bool shouldSignReturnAddressEverywhere(const MachineFunction &MF) {
// FIXME: With WinCFI, extra care should be taken to place SEH_PACSignLR
// and SEH_EpilogEnd instructions in the correct order.
if (MF.getTarget().getMCAsmInfo()->usesWindowsCFI())
return false;
const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
bool SignReturnAddressAll = AFI->shouldSignReturnAddress(/*SpillsLR=*/false);
return SignReturnAddressAll;
}
bool AArch64FrameLowering::shouldCombineCSRLocalStackBump(
MachineFunction &MF, uint64_t StackBumpBytes) const {
AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
const MachineFrameInfo &MFI = MF.getFrameInfo();
const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
if (homogeneousPrologEpilog(MF))
return false;
if (AFI->getLocalStackSize() == 0)
return false;
// For WinCFI, if optimizing for size, prefer to not combine the stack bump
// (to force a stp with predecrement) to match the packed unwind format,
// provided that there actually are any callee saved registers to merge the
// decrement with.
// This is potentially marginally slower, but allows using the packed
// unwind format for functions that both have a local area and callee saved
// registers. Using the packed unwind format notably reduces the size of
// the unwind info.
if (needsWinCFI(MF) && AFI->getCalleeSavedStackSize() > 0 &&
MF.getFunction().hasOptSize())
return false;
// 512 is the maximum immediate for stp/ldp that will be used for
// callee-save save/restores
if (StackBumpBytes >= 512 || windowsRequiresStackProbe(MF, StackBumpBytes))
return false;
if (MFI.hasVarSizedObjects())
return false;
if (RegInfo->hasStackRealignment(MF))
return false;
// This isn't strictly necessary, but it simplifies things a bit since the
// current RedZone handling code assumes the SP is adjusted by the
// callee-save save/restore code.
if (canUseRedZone(MF))
return false;
// When there is an SVE area on the stack, always allocate the
// callee-saves and spills/locals separately.
if (getSVEStackSize(MF))
return false;
return true;
}
bool AArch64FrameLowering::shouldCombineCSRLocalStackBumpInEpilogue(
MachineBasicBlock &MBB, uint64_t StackBumpBytes) const {
if (!shouldCombineCSRLocalStackBump(*MBB.getParent(), StackBumpBytes))
return false;
if (MBB.empty())
return true;
// Disable combined SP bump if the last instruction is an MTE tag store. It
// is almost always better to merge SP adjustment into those instructions.
MachineBasicBlock::iterator LastI = MBB.getFirstTerminator();
MachineBasicBlock::iterator Begin = MBB.begin();
while (LastI != Begin) {
--LastI;
if (LastI->isTransient())
continue;
if (!LastI->getFlag(MachineInstr::FrameDestroy))
break;
}
switch (LastI->getOpcode()) {
case AArch64::STGloop:
case AArch64::STZGloop:
case AArch64::STGi:
case AArch64::STZGi:
case AArch64::ST2Gi:
case AArch64::STZ2Gi:
return false;
default:
return true;
}
llvm_unreachable("unreachable");
}
// Given a load or a store instruction, generate an appropriate unwinding SEH
// code on Windows.
static MachineBasicBlock::iterator InsertSEH(MachineBasicBlock::iterator MBBI,
const TargetInstrInfo &TII,
MachineInstr::MIFlag Flag) {
unsigned Opc = MBBI->getOpcode();
MachineBasicBlock *MBB = MBBI->getParent();
MachineFunction &MF = *MBB->getParent();
DebugLoc DL = MBBI->getDebugLoc();
unsigned ImmIdx = MBBI->getNumOperands() - 1;
int Imm = MBBI->getOperand(ImmIdx).getImm();
MachineInstrBuilder MIB;
const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
switch (Opc) {
default:
report_fatal_error("No SEH Opcode for this instruction");
case AArch64::STR_ZXI:
case AArch64::LDR_ZXI: {
unsigned Reg0 = RegInfo->getSEHRegNum(MBBI->getOperand(0).getReg());
MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveZReg))
.addImm(Reg0)
.addImm(Imm)
.setMIFlag(Flag);
break;
}
case AArch64::STR_PXI:
case AArch64::LDR_PXI: {
unsigned Reg0 = RegInfo->getSEHRegNum(MBBI->getOperand(0).getReg());
MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SavePReg))
.addImm(Reg0)
.addImm(Imm)
.setMIFlag(Flag);
break;
}
case AArch64::LDPDpost:
Imm = -Imm;
[[fallthrough]];
case AArch64::STPDpre: {
unsigned Reg0 = RegInfo->getSEHRegNum(MBBI->getOperand(1).getReg());
unsigned Reg1 = RegInfo->getSEHRegNum(MBBI->getOperand(2).getReg());
MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFRegP_X))
.addImm(Reg0)
.addImm(Reg1)
.addImm(Imm * 8)
.setMIFlag(Flag);
break;
}
case AArch64::LDPXpost:
Imm = -Imm;
[[fallthrough]];
case AArch64::STPXpre: {
Register Reg0 = MBBI->getOperand(1).getReg();
Register Reg1 = MBBI->getOperand(2).getReg();
if (Reg0 == AArch64::FP && Reg1 == AArch64::LR)
MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFPLR_X))
.addImm(Imm * 8)
.setMIFlag(Flag);
else
MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveRegP_X))
.addImm(RegInfo->getSEHRegNum(Reg0))
.addImm(RegInfo->getSEHRegNum(Reg1))
.addImm(Imm * 8)
.setMIFlag(Flag);
break;
}
case AArch64::LDRDpost:
Imm = -Imm;
[[fallthrough]];
case AArch64::STRDpre: {
unsigned Reg = RegInfo->getSEHRegNum(MBBI->getOperand(1).getReg());
MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFReg_X))
.addImm(Reg)
.addImm(Imm)
.setMIFlag(Flag);
break;
}
case AArch64::LDRXpost:
Imm = -Imm;
[[fallthrough]];
case AArch64::STRXpre: {
unsigned Reg = RegInfo->getSEHRegNum(MBBI->getOperand(1).getReg());
MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveReg_X))
.addImm(Reg)
.addImm(Imm)
.setMIFlag(Flag);
break;
}
case AArch64::STPDi:
case AArch64::LDPDi: {
unsigned Reg0 = RegInfo->getSEHRegNum(MBBI->getOperand(0).getReg());
unsigned Reg1 = RegInfo->getSEHRegNum(MBBI->getOperand(1).getReg());
MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFRegP))
.addImm(Reg0)
.addImm(Reg1)
.addImm(Imm * 8)
.setMIFlag(Flag);
break;
}
case AArch64::STPXi:
case AArch64::LDPXi: {
Register Reg0 = MBBI->getOperand(0).getReg();
Register Reg1 = MBBI->getOperand(1).getReg();
if (Reg0 == AArch64::FP && Reg1 == AArch64::LR)
MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFPLR))
.addImm(Imm * 8)
.setMIFlag(Flag);
else
MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveRegP))
.addImm(RegInfo->getSEHRegNum(Reg0))
.addImm(RegInfo->getSEHRegNum(Reg1))
.addImm(Imm * 8)
.setMIFlag(Flag);
break;
}
case AArch64::STRXui:
case AArch64::LDRXui: {
int Reg = RegInfo->getSEHRegNum(MBBI->getOperand(0).getReg());
MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveReg))
.addImm(Reg)
.addImm(Imm * 8)
.setMIFlag(Flag);
break;
}
case AArch64::STRDui:
case AArch64::LDRDui: {
unsigned Reg = RegInfo->getSEHRegNum(MBBI->getOperand(0).getReg());
MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveFReg))
.addImm(Reg)
.addImm(Imm * 8)
.setMIFlag(Flag);
break;
}
case AArch64::STPQi:
case AArch64::LDPQi: {
unsigned Reg0 = RegInfo->getSEHRegNum(MBBI->getOperand(0).getReg());
unsigned Reg1 = RegInfo->getSEHRegNum(MBBI->getOperand(1).getReg());
MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveAnyRegQP))
.addImm(Reg0)
.addImm(Reg1)
.addImm(Imm * 16)
.setMIFlag(Flag);
break;
}
case AArch64::LDPQpost:
Imm = -Imm;
[[fallthrough]];
case AArch64::STPQpre: {
unsigned Reg0 = RegInfo->getSEHRegNum(MBBI->getOperand(1).getReg());
unsigned Reg1 = RegInfo->getSEHRegNum(MBBI->getOperand(2).getReg());
MIB = BuildMI(MF, DL, TII.get(AArch64::SEH_SaveAnyRegQPX))
.addImm(Reg0)
.addImm(Reg1)
.addImm(Imm * 16)
.setMIFlag(Flag);
break;
}
}
auto I = MBB->insertAfter(MBBI, MIB);
return I;
}
// Fix up the SEH opcode associated with the save/restore instruction.
static void fixupSEHOpcode(MachineBasicBlock::iterator MBBI,
unsigned LocalStackSize) {
MachineOperand *ImmOpnd = nullptr;
unsigned ImmIdx = MBBI->getNumOperands() - 1;
switch (MBBI->getOpcode()) {
default:
llvm_unreachable("Fix the offset in the SEH instruction");
case AArch64::SEH_SaveFPLR:
case AArch64::SEH_SaveRegP:
case AArch64::SEH_SaveReg:
case AArch64::SEH_SaveFRegP:
case AArch64::SEH_SaveFReg:
case AArch64::SEH_SaveAnyRegQP:
case AArch64::SEH_SaveAnyRegQPX:
ImmOpnd = &MBBI->getOperand(ImmIdx);
break;
}
if (ImmOpnd)
ImmOpnd->setImm(ImmOpnd->getImm() + LocalStackSize);
}
bool requiresGetVGCall(MachineFunction &MF) {
AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
return AFI->hasStreamingModeChanges() &&
!MF.getSubtarget<AArch64Subtarget>().hasSVE();
}
static bool requiresSaveVG(const MachineFunction &MF) {
const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
// For Darwin platforms we don't save VG for non-SVE functions, even if SME
// is enabled with streaming mode changes.
if (!AFI->hasStreamingModeChanges())
return false;
auto &ST = MF.getSubtarget<AArch64Subtarget>();
if (ST.isTargetDarwin())
return ST.hasSVE();
return true;
}
bool isVGInstruction(MachineBasicBlock::iterator MBBI) {
unsigned Opc = MBBI->getOpcode();
if (Opc == AArch64::CNTD_XPiI || Opc == AArch64::RDSVLI_XI ||
Opc == AArch64::UBFMXri)
return true;
if (requiresGetVGCall(*MBBI->getMF())) {
if (Opc == AArch64::ORRXrr)
return true;
if (Opc == AArch64::BL) {
auto Op1 = MBBI->getOperand(0);
return Op1.isSymbol() &&
(StringRef(Op1.getSymbolName()) == "__arm_get_current_vg");
}
}
return false;
}
// Convert callee-save register save/restore instruction to do stack pointer
// decrement/increment to allocate/deallocate the callee-save stack area by
// converting store/load to use pre/post increment version.
static MachineBasicBlock::iterator convertCalleeSaveRestoreToSPPrePostIncDec(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
const DebugLoc &DL, const TargetInstrInfo *TII, int CSStackSizeInc,
bool NeedsWinCFI, bool *HasWinCFI, bool EmitCFI,
MachineInstr::MIFlag FrameFlag = MachineInstr::FrameSetup,
int CFAOffset = 0) {
unsigned NewOpc;
// If the function contains streaming mode changes, we expect instructions
// to calculate the value of VG before spilling. For locally-streaming
// functions, we need to do this for both the streaming and non-streaming
// vector length. Move past these instructions if necessary.
MachineFunction &MF = *MBB.getParent();
if (requiresSaveVG(MF))
while (isVGInstruction(MBBI))
++MBBI;
switch (MBBI->getOpcode()) {
default:
llvm_unreachable("Unexpected callee-save save/restore opcode!");
case AArch64::STPXi:
NewOpc = AArch64::STPXpre;
break;
case AArch64::STPDi:
NewOpc = AArch64::STPDpre;
break;
case AArch64::STPQi:
NewOpc = AArch64::STPQpre;
break;
case AArch64::STRXui:
NewOpc = AArch64::STRXpre;
break;
case AArch64::STRDui:
NewOpc = AArch64::STRDpre;
break;
case AArch64::STRQui:
NewOpc = AArch64::STRQpre;
break;
case AArch64::LDPXi:
NewOpc = AArch64::LDPXpost;
break;
case AArch64::LDPDi:
NewOpc = AArch64::LDPDpost;
break;
case AArch64::LDPQi:
NewOpc = AArch64::LDPQpost;
break;
case AArch64::LDRXui:
NewOpc = AArch64::LDRXpost;
break;
case AArch64::LDRDui:
NewOpc = AArch64::LDRDpost;
break;
case AArch64::LDRQui:
NewOpc = AArch64::LDRQpost;
break;
}
TypeSize Scale = TypeSize::getFixed(1), Width = TypeSize::getFixed(0);
int64_t MinOffset, MaxOffset;
bool Success = static_cast<const AArch64InstrInfo *>(TII)->getMemOpInfo(
NewOpc, Scale, Width, MinOffset, MaxOffset);
(void)Success;
assert(Success && "unknown load/store opcode");
// If the first store isn't right where we want SP then we can't fold the
// update in so create a normal arithmetic instruction instead.
if (MBBI->getOperand(MBBI->getNumOperands() - 1).getImm() != 0 ||
CSStackSizeInc < MinOffset * (int64_t)Scale.getFixedValue() ||
CSStackSizeInc > MaxOffset * (int64_t)Scale.getFixedValue()) {
// If we are destroying the frame, make sure we add the increment after the
// last frame operation.
if (FrameFlag == MachineInstr::FrameDestroy) {
++MBBI;
// Also skip the SEH instruction, if needed
if (NeedsWinCFI && AArch64InstrInfo::isSEHInstruction(*MBBI))
++MBBI;
}
emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP,
StackOffset::getFixed(CSStackSizeInc), TII, FrameFlag,
false, NeedsWinCFI, HasWinCFI, EmitCFI,
StackOffset::getFixed(CFAOffset));
return std::prev(MBBI);
}
// Get rid of the SEH code associated with the old instruction.
if (NeedsWinCFI) {
auto SEH = std::next(MBBI);
if (AArch64InstrInfo::isSEHInstruction(*SEH))
SEH->eraseFromParent();
}
MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(NewOpc));
MIB.addReg(AArch64::SP, RegState::Define);
// Copy all operands other than the immediate offset.
unsigned OpndIdx = 0;
for (unsigned OpndEnd = MBBI->getNumOperands() - 1; OpndIdx < OpndEnd;
++OpndIdx)
MIB.add(MBBI->getOperand(OpndIdx));
assert(MBBI->getOperand(OpndIdx).getImm() == 0 &&
"Unexpected immediate offset in first/last callee-save save/restore "
"instruction!");
assert(MBBI->getOperand(OpndIdx - 1).getReg() == AArch64::SP &&
"Unexpected base register in callee-save save/restore instruction!");
assert(CSStackSizeInc % Scale == 0);
MIB.addImm(CSStackSizeInc / (int)Scale);
MIB.setMIFlags(MBBI->getFlags());
MIB.setMemRefs(MBBI->memoperands());
// Generate a new SEH code that corresponds to the new instruction.
if (NeedsWinCFI) {
*HasWinCFI = true;
InsertSEH(*MIB, *TII, FrameFlag);
}
if (EmitCFI)
CFIInstBuilder(MBB, MBBI, FrameFlag)
.buildDefCFAOffset(CFAOffset - CSStackSizeInc);
return std::prev(MBB.erase(MBBI));
}
// Fixup callee-save register save/restore instructions to take into account
// combined SP bump by adding the local stack size to the stack offsets.
static void fixupCalleeSaveRestoreStackOffset(MachineInstr &MI,
uint64_t LocalStackSize,
bool NeedsWinCFI,
bool *HasWinCFI) {
if (AArch64InstrInfo::isSEHInstruction(MI))
return;
unsigned Opc = MI.getOpcode();
unsigned Scale;
switch (Opc) {
case AArch64::STPXi:
case AArch64::STRXui:
case AArch64::STPDi:
case AArch64::STRDui:
case AArch64::LDPXi:
case AArch64::LDRXui:
case AArch64::LDPDi:
case AArch64::LDRDui:
Scale = 8;
break;
case AArch64::STPQi:
case AArch64::STRQui:
case AArch64::LDPQi:
case AArch64::LDRQui:
Scale = 16;
break;
default:
llvm_unreachable("Unexpected callee-save save/restore opcode!");
}
unsigned OffsetIdx = MI.getNumExplicitOperands() - 1;
assert(MI.getOperand(OffsetIdx - 1).getReg() == AArch64::SP &&
"Unexpected base register in callee-save save/restore instruction!");
// Last operand is immediate offset that needs fixing.
MachineOperand &OffsetOpnd = MI.getOperand(OffsetIdx);
// All generated opcodes have scaled offsets.
assert(LocalStackSize % Scale == 0);
OffsetOpnd.setImm(OffsetOpnd.getImm() + LocalStackSize / Scale);
if (NeedsWinCFI) {
*HasWinCFI = true;
auto MBBI = std::next(MachineBasicBlock::iterator(MI));
assert(MBBI != MI.getParent()->end() && "Expecting a valid instruction");
assert(AArch64InstrInfo::isSEHInstruction(*MBBI) &&
"Expecting a SEH instruction");
fixupSEHOpcode(MBBI, LocalStackSize);
}
}
static bool isTargetWindows(const MachineFunction &MF) {
return MF.getSubtarget<AArch64Subtarget>().isTargetWindows();
}
static unsigned getStackHazardSize(const MachineFunction &MF) {
return MF.getSubtarget<AArch64Subtarget>().getStreamingHazardSize();
}
// Convenience function to determine whether I is an SVE callee save.
static bool IsSVECalleeSave(MachineBasicBlock::iterator I) {
switch (I->getOpcode()) {
default:
return false;
case AArch64::PTRUE_C_B:
case AArch64::LD1B_2Z_IMM:
case AArch64::ST1B_2Z_IMM:
case AArch64::STR_ZXI:
case AArch64::STR_PXI:
case AArch64::LDR_ZXI:
case AArch64::LDR_PXI:
case AArch64::PTRUE_B:
case AArch64::CPY_ZPzI_B:
case AArch64::CMPNE_PPzZI_B:
return I->getFlag(MachineInstr::FrameSetup) ||
I->getFlag(MachineInstr::FrameDestroy);
case AArch64::SEH_SavePReg:
case AArch64::SEH_SaveZReg:
return true;
}
}
static void emitShadowCallStackPrologue(const TargetInstrInfo &TII,
MachineFunction &MF,
MachineBasicBlock &MBB,
MachineBasicBlock::iterator MBBI,
const DebugLoc &DL, bool NeedsWinCFI,
bool NeedsUnwindInfo) {
// Shadow call stack prolog: str x30, [x18], #8
BuildMI(MBB, MBBI, DL, TII.get(AArch64::STRXpost))
.addReg(AArch64::X18, RegState::Define)
.addReg(AArch64::LR)
.addReg(AArch64::X18)
.addImm(8)
.setMIFlag(MachineInstr::FrameSetup);
// This instruction also makes x18 live-in to the entry block.
MBB.addLiveIn(AArch64::X18);
if (NeedsWinCFI)
BuildMI(MBB, MBBI, DL, TII.get(AArch64::SEH_Nop))
.setMIFlag(MachineInstr::FrameSetup);
if (NeedsUnwindInfo) {
// Emit a CFI instruction that causes 8 to be subtracted from the value of
// x18 when unwinding past this frame.
static const char CFIInst[] = {
dwarf::DW_CFA_val_expression,
18, // register
2, // length
static_cast<char>(unsigned(dwarf::DW_OP_breg18)),
static_cast<char>(-8) & 0x7f, // addend (sleb128)
};
CFIInstBuilder(MBB, MBBI, MachineInstr::FrameSetup)
.buildEscape(StringRef(CFIInst, sizeof(CFIInst)));
}
}
static void emitShadowCallStackEpilogue(const TargetInstrInfo &TII,
MachineFunction &MF,
MachineBasicBlock &MBB,
MachineBasicBlock::iterator MBBI,
const DebugLoc &DL, bool NeedsWinCFI) {
// Shadow call stack epilog: ldr x30, [x18, #-8]!
BuildMI(MBB, MBBI, DL, TII.get(AArch64::LDRXpre))
.addReg(AArch64::X18, RegState::Define)
.addReg(AArch64::LR, RegState::Define)
.addReg(AArch64::X18)
.addImm(-8)
.setMIFlag(MachineInstr::FrameDestroy);
if (NeedsWinCFI)
BuildMI(MBB, MBBI, DL, TII.get(AArch64::SEH_Nop))
.setMIFlag(MachineInstr::FrameDestroy);
if (MF.getInfo<AArch64FunctionInfo>()->needsAsyncDwarfUnwindInfo(MF))
CFIInstBuilder(MBB, MBBI, MachineInstr::FrameDestroy)
.buildRestore(AArch64::X18);
}
// Define the current CFA rule to use the provided FP.
static void emitDefineCFAWithFP(MachineFunction &MF, MachineBasicBlock &MBB,
MachineBasicBlock::iterator MBBI,
unsigned FixedObject) {
const AArch64Subtarget &STI = MF.getSubtarget<AArch64Subtarget>();
const AArch64RegisterInfo *TRI = STI.getRegisterInfo();
AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
const int OffsetToFirstCalleeSaveFromFP =
AFI->getCalleeSaveBaseToFrameRecordOffset() -
AFI->getCalleeSavedStackSize();
Register FramePtr = TRI->getFrameRegister(MF);
CFIInstBuilder(MBB, MBBI, MachineInstr::FrameSetup)
.buildDefCFA(FramePtr, FixedObject - OffsetToFirstCalleeSaveFromFP);
}
#ifndef NDEBUG
/// Collect live registers from the end of \p MI's parent up to (including) \p
/// MI in \p LiveRegs.
static void getLivePhysRegsUpTo(MachineInstr &MI, const TargetRegisterInfo &TRI,
LivePhysRegs &LiveRegs) {
MachineBasicBlock &MBB = *MI.getParent();
LiveRegs.addLiveOuts(MBB);
for (const MachineInstr &MI :
reverse(make_range(MI.getIterator(), MBB.instr_end())))
LiveRegs.stepBackward(MI);
}
#endif
void AArch64FrameLowering::emitPacRetPlusLeafHardening(
MachineFunction &MF) const {
const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
const TargetInstrInfo *TII = Subtarget.getInstrInfo();
auto EmitSignRA = [&](MachineBasicBlock &MBB) {
DebugLoc DL; // Set debug location to unknown.
MachineBasicBlock::iterator MBBI = MBB.begin();
BuildMI(MBB, MBBI, DL, TII->get(AArch64::PAUTH_PROLOGUE))
.setMIFlag(MachineInstr::FrameSetup);
};
auto EmitAuthRA = [&](MachineBasicBlock &MBB) {
DebugLoc DL;
MachineBasicBlock::iterator MBBI = MBB.getFirstTerminator();
if (MBBI != MBB.end())
DL = MBBI->getDebugLoc();
BuildMI(MBB, MBBI, DL, TII->get(AArch64::PAUTH_EPILOGUE))
.setMIFlag(MachineInstr::FrameDestroy);
};
// This should be in sync with PEIImpl::calculateSaveRestoreBlocks.
EmitSignRA(MF.front());
for (MachineBasicBlock &MBB : MF) {
if (MBB.isEHFuncletEntry())
EmitSignRA(MBB);
if (MBB.isReturnBlock())
EmitAuthRA(MBB);
}
}
void AArch64FrameLowering::emitPrologue(MachineFunction &MF,
MachineBasicBlock &MBB) const {
MachineBasicBlock::iterator MBBI = MBB.begin();
const MachineFrameInfo &MFI = MF.getFrameInfo();
const Function &F = MF.getFunction();
const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
const TargetInstrInfo *TII = Subtarget.getInstrInfo();
AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
bool EmitCFI = AFI->needsDwarfUnwindInfo(MF);
bool EmitAsyncCFI = AFI->needsAsyncDwarfUnwindInfo(MF);
bool HasFP = hasFP(MF);
bool NeedsWinCFI = needsWinCFI(MF);
bool HasWinCFI = false;
auto Cleanup = make_scope_exit([&]() { MF.setHasWinCFI(HasWinCFI); });
MachineBasicBlock::iterator End = MBB.end();
#ifndef NDEBUG
const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
// Collect live register from the end of MBB up to the start of the existing
// frame setup instructions.
MachineBasicBlock::iterator NonFrameStart = MBB.begin();
while (NonFrameStart != End &&
NonFrameStart->getFlag(MachineInstr::FrameSetup))
++NonFrameStart;
LivePhysRegs LiveRegs(*TRI);
if (NonFrameStart != MBB.end()) {
getLivePhysRegsUpTo(*NonFrameStart, *TRI, LiveRegs);
// Ignore registers used for stack management for now.
LiveRegs.removeReg(AArch64::SP);
LiveRegs.removeReg(AArch64::X19);
LiveRegs.removeReg(AArch64::FP);
LiveRegs.removeReg(AArch64::LR);
// X0 will be clobbered by a call to __arm_get_current_vg in the prologue.
// This is necessary to spill VG if required where SVE is unavailable, but
// X0 is preserved around this call.
if (requiresGetVGCall(MF))
LiveRegs.removeReg(AArch64::X0);
}
auto VerifyClobberOnExit = make_scope_exit([&]() {
if (NonFrameStart == MBB.end())
return;
// Check if any of the newly instructions clobber any of the live registers.
for (MachineInstr &MI :
make_range(MBB.instr_begin(), NonFrameStart->getIterator())) {
for (auto &Op : MI.operands())
if (Op.isReg() && Op.isDef())
assert(!LiveRegs.contains(Op.getReg()) &&
"live register clobbered by inserted prologue instructions");
}
});
#endif
bool IsFunclet = MBB.isEHFuncletEntry();
// At this point, we're going to decide whether or not the function uses a
// redzone. In most cases, the function doesn't have a redzone so let's
// assume that's false and set it to true in the case that there's a redzone.
AFI->setHasRedZone(false);
// Debug location must be unknown since the first debug location is used
// to determine the end of the prologue.
DebugLoc DL;
const auto &MFnI = *MF.getInfo<AArch64FunctionInfo>();
if (MFnI.shouldSignReturnAddress(MF)) {
// If pac-ret+leaf is in effect, PAUTH_PROLOGUE pseudo instructions
// are inserted by emitPacRetPlusLeafHardening().
if (!shouldSignReturnAddressEverywhere(MF)) {
BuildMI(MBB, MBBI, DL, TII->get(AArch64::PAUTH_PROLOGUE))
.setMIFlag(MachineInstr::FrameSetup);
}
// AArch64PointerAuth pass will insert SEH_PACSignLR
HasWinCFI |= NeedsWinCFI;
}
if (MFnI.needsShadowCallStackPrologueEpilogue(MF)) {
emitShadowCallStackPrologue(*TII, MF, MBB, MBBI, DL, NeedsWinCFI,
MFnI.needsDwarfUnwindInfo(MF));
HasWinCFI |= NeedsWinCFI;
}
if (EmitCFI && MFnI.isMTETagged()) {
BuildMI(MBB, MBBI, DL, TII->get(AArch64::EMITMTETAGGED))
.setMIFlag(MachineInstr::FrameSetup);
}
// We signal the presence of a Swift extended frame to external tools by
// storing FP with 0b0001 in bits 63:60. In normal userland operation a simple
// ORR is sufficient, it is assumed a Swift kernel would initialize the TBI
// bits so that is still true.
if (HasFP && AFI->hasSwiftAsyncContext()) {
switch (MF.getTarget().Options.SwiftAsyncFramePointer) {
case SwiftAsyncFramePointerMode::DeploymentBased:
if (Subtarget.swiftAsyncContextIsDynamicallySet()) {
// The special symbol below is absolute and has a *value* that can be
// combined with the frame pointer to signal an extended frame.
BuildMI(MBB, MBBI, DL, TII->get(AArch64::LOADgot), AArch64::X16)
.addExternalSymbol("swift_async_extendedFramePointerFlags",
AArch64II::MO_GOT);
if (NeedsWinCFI) {
BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
.setMIFlags(MachineInstr::FrameSetup);
HasWinCFI = true;
}
BuildMI(MBB, MBBI, DL, TII->get(AArch64::ORRXrs), AArch64::FP)
.addUse(AArch64::FP)
.addUse(AArch64::X16)
.addImm(Subtarget.isTargetILP32() ? 32 : 0);
if (NeedsWinCFI) {
BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
.setMIFlags(MachineInstr::FrameSetup);
HasWinCFI = true;
}
break;
}
[[fallthrough]];
case SwiftAsyncFramePointerMode::Always:
// ORR x29, x29, #0x1000_0000_0000_0000
BuildMI(MBB, MBBI, DL, TII->get(AArch64::ORRXri), AArch64::FP)
.addUse(AArch64::FP)
.addImm(0x1100)
.setMIFlag(MachineInstr::FrameSetup);
if (NeedsWinCFI) {
BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
.setMIFlags(MachineInstr::FrameSetup);
HasWinCFI = true;
}
break;
case SwiftAsyncFramePointerMode::Never:
break;
}
}
// All calls are tail calls in GHC calling conv, and functions have no
// prologue/epilogue.
if (MF.getFunction().getCallingConv() == CallingConv::GHC)
return;
// Set tagged base pointer to the requested stack slot.
// Ideally it should match SP value after prologue.
std::optional<int> TBPI = AFI->getTaggedBasePointerIndex();
if (TBPI)
AFI->setTaggedBasePointerOffset(-MFI.getObjectOffset(*TBPI));
else
AFI->setTaggedBasePointerOffset(MFI.getStackSize());
const StackOffset &SVEStackSize = getSVEStackSize(MF);
// getStackSize() includes all the locals in its size calculation. We don't
// include these locals when computing the stack size of a funclet, as they
// are allocated in the parent's stack frame and accessed via the frame
// pointer from the funclet. We only save the callee saved registers in the
// funclet, which are really the callee saved registers of the parent
// function, including the funclet.
int64_t NumBytes =
IsFunclet ? getWinEHFuncletFrameSize(MF) : MFI.getStackSize();
if (!AFI->hasStackFrame() && !windowsRequiresStackProbe(MF, NumBytes)) {
assert(!HasFP && "unexpected function without stack frame but with FP");
assert(!SVEStackSize &&
"unexpected function without stack frame but with SVE objects");
// All of the stack allocation is for locals.
AFI->setLocalStackSize(NumBytes);
if (!NumBytes) {
if (NeedsWinCFI && HasWinCFI) {
BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PrologEnd))
.setMIFlag(MachineInstr::FrameSetup);
}
return;
}
// REDZONE: If the stack size is less than 128 bytes, we don't need
// to actually allocate.
if (canUseRedZone(MF)) {
AFI->setHasRedZone(true);
++NumRedZoneFunctions;
} else {
emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP,
StackOffset::getFixed(-NumBytes), TII,
MachineInstr::FrameSetup, false, NeedsWinCFI, &HasWinCFI);
if (EmitCFI) {
// Label used to tie together the PROLOG_LABEL and the MachineMoves.
MCSymbol *FrameLabel = MF.getContext().createTempSymbol();
// Encode the stack size of the leaf function.
CFIInstBuilder(MBB, MBBI, MachineInstr::FrameSetup)
.buildDefCFAOffset(NumBytes, FrameLabel);
}
}
if (NeedsWinCFI) {
HasWinCFI = true;
BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PrologEnd))
.setMIFlag(MachineInstr::FrameSetup);
}
return;
}
bool IsWin64 = Subtarget.isCallingConvWin64(F.getCallingConv(), F.isVarArg());
unsigned FixedObject = getFixedObjectSize(MF, AFI, IsWin64, IsFunclet);
// Windows unwind can't represent the required stack adjustments if we have
// both SVE callee-saves and dynamic stack allocations, and the frame
// pointer is before the SVE spills. The allocation of the frame pointer
// must be the last instruction in the prologue so the unwinder can restore
// the stack pointer correctly. (And there isn't any unwind opcode for
// `addvl sp, x29, -17`.)
//
// Because of this, we do spills in the opposite order on Windows: first SVE,
// then GPRs. The main side-effect of this is that it makes accessing
// parameters passed on the stack more expensive.
//
// We could consider rearranging the spills for simpler cases.
bool FPAfterSVECalleeSaves =
Subtarget.isTargetWindows() && AFI->getSVECalleeSavedStackSize();
if (FPAfterSVECalleeSaves && AFI->hasStackHazardSlotIndex())
reportFatalUsageError("SME hazard padding is not supported on Windows");
auto PrologueSaveSize = AFI->getCalleeSavedStackSize() + FixedObject;
// All of the remaining stack allocations are for locals.
AFI->setLocalStackSize(NumBytes - PrologueSaveSize);
bool CombineSPBump = shouldCombineCSRLocalStackBump(MF, NumBytes);
bool HomPrologEpilog = homogeneousPrologEpilog(MF);
if (FPAfterSVECalleeSaves) {
// If we're doing SVE saves first, we need to immediately allocate space
// for fixed objects, then space for the SVE callee saves.
//
// Windows unwind requires that the scalable size is a multiple of 16;
// that's handled when the callee-saved size is computed.
auto SaveSize =
StackOffset::getScalable(AFI->getSVECalleeSavedStackSize()) +
StackOffset::getFixed(FixedObject);
allocateStackSpace(MBB, MBBI, 0, SaveSize, NeedsWinCFI, &HasWinCFI,
/*EmitCFI=*/false, StackOffset{},
/*FollowupAllocs=*/true);
NumBytes -= FixedObject;
// Now allocate space for the GPR callee saves.
while (MBBI != End && IsSVECalleeSave(MBBI))
++MBBI;
MBBI = convertCalleeSaveRestoreToSPPrePostIncDec(
MBB, MBBI, DL, TII, -AFI->getCalleeSavedStackSize(), NeedsWinCFI,
&HasWinCFI, EmitAsyncCFI);
NumBytes -= AFI->getCalleeSavedStackSize();
} else if (CombineSPBump) {
assert(!SVEStackSize && "Cannot combine SP bump with SVE");
emitFrameOffset(MBB, MBBI, DL, AArch64::SP, AArch64::SP,
StackOffset::getFixed(-NumBytes), TII,
MachineInstr::FrameSetup, false, NeedsWinCFI, &HasWinCFI,
EmitAsyncCFI);
NumBytes = 0;
} else if (HomPrologEpilog) {
// Stack has been already adjusted.
NumBytes -= PrologueSaveSize;
} else if (PrologueSaveSize != 0) {
MBBI = convertCalleeSaveRestoreToSPPrePostIncDec(
MBB, MBBI, DL, TII, -PrologueSaveSize, NeedsWinCFI, &HasWinCFI,
EmitAsyncCFI);
NumBytes -= PrologueSaveSize;
}
assert(NumBytes >= 0 && "Negative stack allocation size!?");
// Move past the saves of the callee-saved registers, fixing up the offsets
// and pre-inc if we decided to combine the callee-save and local stack
// pointer bump above.
while (MBBI != End && MBBI->getFlag(MachineInstr::FrameSetup) &&
!IsSVECalleeSave(MBBI)) {
if (CombineSPBump &&
// Only fix-up frame-setup load/store instructions.
(!requiresSaveVG(MF) || !isVGInstruction(MBBI)))
fixupCalleeSaveRestoreStackOffset(*MBBI, AFI->getLocalStackSize(),
NeedsWinCFI, &HasWinCFI);
++MBBI;
}
// For funclets the FP belongs to the containing function.
if (!IsFunclet && HasFP) {
// Only set up FP if we actually need to.
int64_t FPOffset = AFI->getCalleeSaveBaseToFrameRecordOffset();
if (CombineSPBump)
FPOffset += AFI->getLocalStackSize();
if (AFI->hasSwiftAsyncContext()) {
// Before we update the live FP we have to ensure there's a valid (or
// null) asynchronous context in its slot just before FP in the frame
// record, so store it now.
const auto &Attrs = MF.getFunction().getAttributes();
bool HaveInitialContext = Attrs.hasAttrSomewhere(Attribute::SwiftAsync);
if (HaveInitialContext)
MBB.addLiveIn(AArch64::X22);
Register Reg = HaveInitialContext ? AArch64::X22 : AArch64::XZR;
BuildMI(MBB, MBBI, DL, TII->get(AArch64::StoreSwiftAsyncContext))
.addUse(Reg)
.addUse(AArch64::SP)
.addImm(FPOffset - 8)
.setMIFlags(MachineInstr::FrameSetup);
if (NeedsWinCFI) {
// WinCFI and arm64e, where StoreSwiftAsyncContext is expanded
// to multiple instructions, should be mutually-exclusive.
assert(Subtarget.getTargetTriple().getArchName() != "arm64e");
BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
.setMIFlags(MachineInstr::FrameSetup);
HasWinCFI = true;
}
}
if (HomPrologEpilog) {
auto Prolog = MBBI;
--Prolog;
assert(Prolog->getOpcode() == AArch64::HOM_Prolog);
Prolog->addOperand(MachineOperand::CreateImm(FPOffset));
} else {
// Issue sub fp, sp, FPOffset or
// mov fp,sp when FPOffset is zero.
// Note: All stores of callee-saved registers are marked as "FrameSetup".
// This code marks the instruction(s) that set the FP also.
emitFrameOffset(MBB, MBBI, DL, AArch64::FP, AArch64::SP,
StackOffset::getFixed(FPOffset), TII,
MachineInstr::FrameSetup, false, NeedsWinCFI, &HasWinCFI);
if (NeedsWinCFI && HasWinCFI) {
BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PrologEnd))
.setMIFlag(MachineInstr::FrameSetup);
// After setting up the FP, the rest of the prolog doesn't need to be
// included in the SEH unwind info.
NeedsWinCFI = false;
}
}
if (EmitAsyncCFI)
emitDefineCFAWithFP(MF, MBB, MBBI, FixedObject);
}
// Now emit the moves for whatever callee saved regs we have (including FP,
// LR if those are saved). Frame instructions for SVE register are emitted
// later, after the instruction which actually save SVE regs.
if (EmitAsyncCFI)
emitCalleeSavedGPRLocations(MBB, MBBI);
// Alignment is required for the parent frame, not the funclet
const bool NeedsRealignment =
NumBytes && !IsFunclet && RegInfo->hasStackRealignment(MF);
const int64_t RealignmentPadding =
(NeedsRealignment && MFI.getMaxAlign() > Align(16))
? MFI.getMaxAlign().value() - 16
: 0;
if (windowsRequiresStackProbe(MF, NumBytes + RealignmentPadding)) {
if (AFI->getSVECalleeSavedStackSize())
report_fatal_error(
"SVE callee saves not yet supported with stack probing");
// Find an available register to spill the value of X15 to, if X15 is being
// used already for nest.
unsigned X15Scratch = AArch64::NoRegister;
const AArch64Subtarget &STI = MF.getSubtarget<AArch64Subtarget>();
if (llvm::any_of(MBB.liveins(),
[&STI](const MachineBasicBlock::RegisterMaskPair &LiveIn) {
return STI.getRegisterInfo()->isSuperOrSubRegisterEq(
AArch64::X15, LiveIn.PhysReg);
})) {
X15Scratch = findScratchNonCalleeSaveRegister(&MBB, true);
assert(X15Scratch != AArch64::NoRegister &&
(X15Scratch < AArch64::X15 || X15Scratch > AArch64::X17));
#ifndef NDEBUG
LiveRegs.removeReg(AArch64::X15); // ignore X15 since we restore it
#endif
BuildMI(MBB, MBBI, DL, TII->get(AArch64::ORRXrr), X15Scratch)
.addReg(AArch64::XZR)
.addReg(AArch64::X15, RegState::Undef)
.addReg(AArch64::X15, RegState::Implicit)
.setMIFlag(MachineInstr::FrameSetup);
}
uint64_t NumWords = (NumBytes + RealignmentPadding) >> 4;
if (NeedsWinCFI) {
HasWinCFI = true;
// alloc_l can hold at most 256MB, so assume that NumBytes doesn't
// exceed this amount. We need to move at most 2^24 - 1 into x15.
// This is at most two instructions, MOVZ followed by MOVK.
// TODO: Fix to use multiple stack alloc unwind codes for stacks
// exceeding 256MB in size.
if (NumBytes >= (1 << 28))
report_fatal_error("Stack size cannot exceed 256MB for stack "
"unwinding purposes");
uint32_t LowNumWords = NumWords & 0xFFFF;
BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVZXi), AArch64::X15)
.addImm(LowNumWords)
.addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0))
.setMIFlag(MachineInstr::FrameSetup);
BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
.setMIFlag(MachineInstr::FrameSetup);
if ((NumWords & 0xFFFF0000) != 0) {
BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVKXi), AArch64::X15)
.addReg(AArch64::X15)
.addImm((NumWords & 0xFFFF0000) >> 16) // High half
.addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 16))
.setMIFlag(MachineInstr::FrameSetup);
BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
.setMIFlag(MachineInstr::FrameSetup);
}
} else {
BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVi64imm), AArch64::X15)
.addImm(NumWords)
.setMIFlags(MachineInstr::FrameSetup);
}
const char *ChkStk = Subtarget.getChkStkName();
switch (MF.getTarget().getCodeModel()) {
case CodeModel::Tiny:
case CodeModel::Small:
case CodeModel::Medium:
case CodeModel::Kernel:
BuildMI(MBB, MBBI, DL, TII->get(AArch64::BL))
.addExternalSymbol(ChkStk)
.addReg(AArch64::X15, RegState::Implicit)
.addReg(AArch64::X16, RegState::Implicit | RegState::Define | RegState::Dead)
.addReg(AArch64::X17, RegState::Implicit | RegState::Define | RegState::Dead)
.addReg(AArch64::NZCV, RegState::Implicit | RegState::Define | RegState::Dead)
.setMIFlags(MachineInstr::FrameSetup);
if (NeedsWinCFI) {
HasWinCFI = true;
BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
.setMIFlag(MachineInstr::FrameSetup);
}
break;
case CodeModel::Large:
BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVaddrEXT))
.addReg(AArch64::X16, RegState::Define)
.addExternalSymbol(ChkStk)
.addExternalSymbol(ChkStk)
.setMIFlags(MachineInstr::FrameSetup);
if (NeedsWinCFI) {
HasWinCFI = true;
BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
.setMIFlag(MachineInstr::FrameSetup);
}
BuildMI(MBB, MBBI, DL, TII->get(getBLRCallOpcode(MF)))
.addReg(AArch64::X16, RegState::Kill)
.addReg(AArch64::X15, RegState::Implicit | RegState::Define)
.addReg(AArch64::X16, RegState::Implicit | RegState::Define | RegState::Dead)
.addReg(AArch64::X17, RegState::Implicit | RegState::Define | RegState::Dead)
.addReg(AArch64::NZCV, RegState::Implicit | RegState::Define | RegState::Dead)
.setMIFlags(MachineInstr::FrameSetup);
if (NeedsWinCFI) {
HasWinCFI = true;
BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
.setMIFlag(MachineInstr::FrameSetup);
}
break;
}
BuildMI(MBB, MBBI, DL, TII->get(AArch64::SUBXrx64), AArch64::SP)
.addReg(AArch64::SP, RegState::Kill)
.addReg(AArch64::X15, RegState::Kill)
.addImm(AArch64_AM::getArithExtendImm(AArch64_AM::UXTX, 4))
.setMIFlags(MachineInstr::FrameSetup);
if (NeedsWinCFI) {
HasWinCFI = true;
BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_StackAlloc))
.addImm(NumBytes)
.setMIFlag(MachineInstr::FrameSetup);
}
NumBytes = 0;
if (RealignmentPadding > 0) {
if (RealignmentPadding >= 4096) {
BuildMI(MBB, MBBI, DL, TII->get(AArch64::MOVi64imm))
.addReg(AArch64::X16, RegState::Define)
.addImm(RealignmentPadding)
.setMIFlags(MachineInstr::FrameSetup);
BuildMI(MBB, MBBI, DL, TII->get(AArch64::ADDXrx64), AArch64::X15)
.addReg(AArch64::SP)
.addReg(AArch64::X16, RegState::Kill)
.addImm(AArch64_AM::getArithExtendImm(AArch64_AM::UXTX, 0))
.setMIFlag(MachineInstr::FrameSetup);
} else {
BuildMI(MBB, MBBI, DL, TII->get(AArch64::ADDXri), AArch64::X15)
.addReg(AArch64::SP)
.addImm(RealignmentPadding)
.addImm(0)
.setMIFlag(MachineInstr::FrameSetup);
}
uint64_t AndMask = ~(MFI.getMaxAlign().value() - 1);
BuildMI(MBB, MBBI, DL, TII->get(AArch64::ANDXri), AArch64::SP)
.addReg(AArch64::X15, RegState::Kill)
.addImm(AArch64_AM::encodeLogicalImmediate(AndMask, 64));
AFI->setStackRealigned(true);
// No need for SEH instructions here; if we're realigning the stack,
// we've set a frame pointer and already finished the SEH prologue.
assert(!NeedsWinCFI);
}
if (X15Scratch != AArch64::NoRegister) {
BuildMI(MBB, MBBI, DL, TII->get(AArch64::ORRXrr), AArch64::X15)
.addReg(AArch64::XZR)
.addReg(X15Scratch, RegState::Undef)
.addReg(X15Scratch, RegState::Implicit)
.setMIFlag(MachineInstr::FrameSetup);
}
}
StackOffset SVECalleeSavesSize = {}, SVELocalsSize = SVEStackSize;
MachineBasicBlock::iterator CalleeSavesEnd = MBBI;
StackOffset CFAOffset =
StackOffset::getFixed((int64_t)MFI.getStackSize() - NumBytes);
// Process the SVE callee-saves to determine what space needs to be
// allocated.
if (int64_t CalleeSavedSize = AFI->getSVECalleeSavedStackSize()) {
LLVM_DEBUG(dbgs() << "SVECalleeSavedStackSize = " << CalleeSavedSize
<< "\n");
SVECalleeSavesSize = StackOffset::getScalable(CalleeSavedSize);
SVELocalsSize = SVEStackSize - SVECalleeSavesSize;
// Find callee save instructions in frame.
// Note: With FPAfterSVECalleeSaves the callee saves have already been
// allocated.
if (!FPAfterSVECalleeSaves) {
MachineBasicBlock::iterator CalleeSavesBegin = MBBI;
assert(IsSVECalleeSave(CalleeSavesBegin) && "Unexpected instruction");
while (IsSVECalleeSave(MBBI) && MBBI != MBB.getFirstTerminator())
++MBBI;
CalleeSavesEnd = MBBI;
StackOffset LocalsSize = SVELocalsSize + StackOffset::getFixed(NumBytes);
// Allocate space for the callee saves (if any).
allocateStackSpace(MBB, CalleeSavesBegin, 0, SVECalleeSavesSize, false,
nullptr, EmitAsyncCFI && !HasFP, CFAOffset,
MFI.hasVarSizedObjects() || LocalsSize);
}
}
CFAOffset += SVECalleeSavesSize;
if (EmitAsyncCFI)
emitCalleeSavedSVELocations(MBB, CalleeSavesEnd);
// Allocate space for the rest of the frame including SVE locals. Align the
// stack as necessary.
assert(!(canUseRedZone(MF) && NeedsRealignment) &&
"Cannot use redzone with stack realignment");
if (!canUseRedZone(MF)) {
// FIXME: in the case of dynamic re-alignment, NumBytes doesn't have
// the correct value here, as NumBytes also includes padding bytes,
// which shouldn't be counted here.
allocateStackSpace(MBB, CalleeSavesEnd, RealignmentPadding,
SVELocalsSize + StackOffset::getFixed(NumBytes),
NeedsWinCFI, &HasWinCFI, EmitAsyncCFI && !HasFP,
CFAOffset, MFI.hasVarSizedObjects());
}
// If we need a base pointer, set it up here. It's whatever the value of the
// stack pointer is at this point. Any variable size objects will be allocated
// after this, so we can still use the base pointer to reference locals.
//
// FIXME: Clarify FrameSetup flags here.
// Note: Use emitFrameOffset() like above for FP if the FrameSetup flag is
// needed.
// For funclets the BP belongs to the containing function.
if (!IsFunclet && RegInfo->hasBasePointer(MF)) {
TII->copyPhysReg(MBB, MBBI, DL, RegInfo->getBaseRegister(), AArch64::SP,
false);
if (NeedsWinCFI) {
HasWinCFI = true;
BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
.setMIFlag(MachineInstr::FrameSetup);
}
}
// The very last FrameSetup instruction indicates the end of prologue. Emit a
// SEH opcode indicating the prologue end.
if (NeedsWinCFI && HasWinCFI) {
BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PrologEnd))
.setMIFlag(MachineInstr::FrameSetup);
}
// SEH funclets are passed the frame pointer in X1. If the parent
// function uses the base register, then the base register is used
// directly, and is not retrieved from X1.
if (IsFunclet && F.hasPersonalityFn()) {
EHPersonality Per = classifyEHPersonality(F.getPersonalityFn());
if (isAsynchronousEHPersonality(Per)) {
BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::COPY), AArch64::FP)
.addReg(AArch64::X1)
.setMIFlag(MachineInstr::FrameSetup);
MBB.addLiveIn(AArch64::X1);
}
}
if (EmitCFI && !EmitAsyncCFI) {
if (HasFP) {
emitDefineCFAWithFP(MF, MBB, MBBI, FixedObject);
} else {
StackOffset TotalSize =
SVEStackSize + StackOffset::getFixed((int64_t)MFI.getStackSize());
CFIInstBuilder CFIBuilder(MBB, MBBI, MachineInstr::FrameSetup);
CFIBuilder.insertCFIInst(
createDefCFA(*RegInfo, /*FrameReg=*/AArch64::SP, /*Reg=*/AArch64::SP,
TotalSize, /*LastAdjustmentWasScalable=*/false));
}
emitCalleeSavedGPRLocations(MBB, MBBI);
emitCalleeSavedSVELocations(MBB, MBBI);
}
}
static bool isFuncletReturnInstr(const MachineInstr &MI) {
switch (MI.getOpcode()) {
default:
return false;
case AArch64::CATCHRET:
case AArch64::CLEANUPRET:
return true;
}
}
void AArch64FrameLowering::emitEpilogue(MachineFunction &MF,
MachineBasicBlock &MBB) const {
MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr();
MachineFrameInfo &MFI = MF.getFrameInfo();
AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
const TargetInstrInfo *TII = Subtarget.getInstrInfo();
DebugLoc DL;
bool NeedsWinCFI = needsWinCFI(MF);
bool EmitCFI = AFI->needsAsyncDwarfUnwindInfo(MF);
bool HasWinCFI = false;
bool IsFunclet = false;
if (MBB.end() != MBBI) {
DL = MBBI->getDebugLoc();
IsFunclet = isFuncletReturnInstr(*MBBI);
}
MachineBasicBlock::iterator EpilogStartI = MBB.end();
auto FinishingTouches = make_scope_exit([&]() {
if (AFI->needsShadowCallStackPrologueEpilogue(MF)) {
emitShadowCallStackEpilogue(*TII, MF, MBB, MBB.getFirstTerminator(), DL,
NeedsWinCFI);
HasWinCFI |= NeedsWinCFI;
}
if (EmitCFI)
emitCalleeSavedGPRRestores(MBB, MBB.getFirstTerminator());
if (AFI->shouldSignReturnAddress(MF)) {
// If pac-ret+leaf is in effect, PAUTH_EPILOGUE pseudo instructions
// are inserted by emitPacRetPlusLeafHardening().
if (!shouldSignReturnAddressEverywhere(MF)) {
BuildMI(MBB, MBB.getFirstTerminator(), DL,
TII->get(AArch64::PAUTH_EPILOGUE))
.setMIFlag(MachineInstr::FrameDestroy);
}
// AArch64PointerAuth pass will insert SEH_PACSignLR
HasWinCFI |= NeedsWinCFI;
}
if (HasWinCFI) {
BuildMI(MBB, MBB.getFirstTerminator(), DL,
TII->get(AArch64::SEH_EpilogEnd))
.setMIFlag(MachineInstr::FrameDestroy);
if (!MF.hasWinCFI())
MF.setHasWinCFI(true);
}
if (NeedsWinCFI) {
assert(EpilogStartI != MBB.end());
if (!HasWinCFI)
MBB.erase(EpilogStartI);
}
});
int64_t NumBytes = IsFunclet ? getWinEHFuncletFrameSize(MF)
: MFI.getStackSize();
// All calls are tail calls in GHC calling conv, and functions have no
// prologue/epilogue.
if (MF.getFunction().getCallingConv() == CallingConv::GHC)
return;
// How much of the stack used by incoming arguments this function is expected
// to restore in this particular epilogue.
int64_t ArgumentStackToRestore = getArgumentStackToRestore(MF, MBB);
bool IsWin64 = Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv(),
MF.getFunction().isVarArg());
unsigned FixedObject = getFixedObjectSize(MF, AFI, IsWin64, IsFunclet);
int64_t AfterCSRPopSize = ArgumentStackToRestore;
auto PrologueSaveSize = AFI->getCalleeSavedStackSize() + FixedObject;
// We cannot rely on the local stack size set in emitPrologue if the function
// has funclets, as funclets have different local stack size requirements, and
// the current value set in emitPrologue may be that of the containing
// function.
if (MF.hasEHFunclets())
AFI->setLocalStackSize(NumBytes - PrologueSaveSize);
if (homogeneousPrologEpilog(MF, &MBB)) {
assert(!NeedsWinCFI);
auto LastPopI = MBB.getFirstTerminator();
if (LastPopI != MBB.begin()) {
auto HomogeneousEpilog = std::prev(LastPopI);
if (HomogeneousEpilog->getOpcode() == AArch64::HOM_Epilog)
LastPopI = HomogeneousEpilog;
}
// Adjust local stack
emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::SP,
StackOffset::getFixed(AFI->getLocalStackSize()), TII,
MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI);
// SP has been already adjusted while restoring callee save regs.
// We've bailed-out the case with adjusting SP for arguments.
assert(AfterCSRPopSize == 0);
return;
}
bool FPAfterSVECalleeSaves =
Subtarget.isTargetWindows() && AFI->getSVECalleeSavedStackSize();
bool CombineSPBump = shouldCombineCSRLocalStackBumpInEpilogue(MBB, NumBytes);
// Assume we can't combine the last pop with the sp restore.
bool CombineAfterCSRBump = false;
if (FPAfterSVECalleeSaves) {
AfterCSRPopSize += FixedObject;
} else if (!CombineSPBump && PrologueSaveSize != 0) {
MachineBasicBlock::iterator Pop = std::prev(MBB.getFirstTerminator());
while (Pop->getOpcode() == TargetOpcode::CFI_INSTRUCTION ||
AArch64InstrInfo::isSEHInstruction(*Pop))
Pop = std::prev(Pop);
// Converting the last ldp to a post-index ldp is valid only if the last
// ldp's offset is 0.
const MachineOperand &OffsetOp = Pop->getOperand(Pop->getNumOperands() - 1);
// If the offset is 0 and the AfterCSR pop is not actually trying to
// allocate more stack for arguments (in space that an untimely interrupt
// may clobber), convert it to a post-index ldp.
if (OffsetOp.getImm() == 0 && AfterCSRPopSize >= 0) {
convertCalleeSaveRestoreToSPPrePostIncDec(
MBB, Pop, DL, TII, PrologueSaveSize, NeedsWinCFI, &HasWinCFI, EmitCFI,
MachineInstr::FrameDestroy, PrologueSaveSize);
} else {
// If not, make sure to emit an add after the last ldp.
// We're doing this by transferring the size to be restored from the
// adjustment *before* the CSR pops to the adjustment *after* the CSR
// pops.
AfterCSRPopSize += PrologueSaveSize;
CombineAfterCSRBump = true;
}
}
// Move past the restores of the callee-saved registers.
// If we plan on combining the sp bump of the local stack size and the callee
// save stack size, we might need to adjust the CSR save and restore offsets.
MachineBasicBlock::iterator LastPopI = MBB.getFirstTerminator();
MachineBasicBlock::iterator Begin = MBB.begin();
while (LastPopI != Begin) {
--LastPopI;
if (!LastPopI->getFlag(MachineInstr::FrameDestroy) ||
(!FPAfterSVECalleeSaves && IsSVECalleeSave(LastPopI))) {
++LastPopI;
break;
} else if (CombineSPBump)
fixupCalleeSaveRestoreStackOffset(*LastPopI, AFI->getLocalStackSize(),
NeedsWinCFI, &HasWinCFI);
}
if (NeedsWinCFI) {
// Note that there are cases where we insert SEH opcodes in the
// epilogue when we had no SEH opcodes in the prologue. For
// example, when there is no stack frame but there are stack
// arguments. Insert the SEH_EpilogStart and remove it later if it
// we didn't emit any SEH opcodes to avoid generating WinCFI for
// functions that don't need it.
BuildMI(MBB, LastPopI, DL, TII->get(AArch64::SEH_EpilogStart))
.setMIFlag(MachineInstr::FrameDestroy);
EpilogStartI = LastPopI;
--EpilogStartI;
}
if (hasFP(MF) && AFI->hasSwiftAsyncContext()) {
switch (MF.getTarget().Options.SwiftAsyncFramePointer) {
case SwiftAsyncFramePointerMode::DeploymentBased:
// Avoid the reload as it is GOT relative, and instead fall back to the
// hardcoded value below. This allows a mismatch between the OS and
// application without immediately terminating on the difference.
[[fallthrough]];
case SwiftAsyncFramePointerMode::Always:
// We need to reset FP to its untagged state on return. Bit 60 is
// currently used to show the presence of an extended frame.
// BIC x29, x29, #0x1000_0000_0000_0000
BuildMI(MBB, MBB.getFirstTerminator(), DL, TII->get(AArch64::ANDXri),
AArch64::FP)
.addUse(AArch64::FP)
.addImm(0x10fe)
.setMIFlag(MachineInstr::FrameDestroy);
if (NeedsWinCFI) {
BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
.setMIFlags(MachineInstr::FrameDestroy);
HasWinCFI = true;
}
break;
case SwiftAsyncFramePointerMode::Never:
break;
}
}
const StackOffset &SVEStackSize = getSVEStackSize(MF);
// If there is a single SP update, insert it before the ret and we're done.
if (CombineSPBump) {
assert(!SVEStackSize && "Cannot combine SP bump with SVE");
// When we are about to restore the CSRs, the CFA register is SP again.
if (EmitCFI && hasFP(MF))
CFIInstBuilder(MBB, LastPopI, MachineInstr::FrameDestroy)
.buildDefCFA(AArch64::SP, NumBytes);
emitFrameOffset(MBB, MBB.getFirstTerminator(), DL, AArch64::SP, AArch64::SP,
StackOffset::getFixed(NumBytes + AfterCSRPopSize), TII,
MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI,
EmitCFI, StackOffset::getFixed(NumBytes));
return;
}
NumBytes -= PrologueSaveSize;
assert(NumBytes >= 0 && "Negative stack allocation size!?");
// Process the SVE callee-saves to determine what space needs to be
// deallocated.
StackOffset DeallocateBefore = {}, DeallocateAfter = SVEStackSize;
MachineBasicBlock::iterator RestoreBegin = LastPopI, RestoreEnd = LastPopI;
if (int64_t CalleeSavedSize = AFI->getSVECalleeSavedStackSize()) {
if (FPAfterSVECalleeSaves)
RestoreEnd = MBB.getFirstTerminator();
RestoreBegin = std::prev(RestoreEnd);
while (RestoreBegin != MBB.begin() &&
IsSVECalleeSave(std::prev(RestoreBegin)))
--RestoreBegin;
assert(IsSVECalleeSave(RestoreBegin) &&
IsSVECalleeSave(std::prev(RestoreEnd)) && "Unexpected instruction");
StackOffset CalleeSavedSizeAsOffset =
StackOffset::getScalable(CalleeSavedSize);
DeallocateBefore = SVEStackSize - CalleeSavedSizeAsOffset;
DeallocateAfter = CalleeSavedSizeAsOffset;
}
// Deallocate the SVE area.
if (FPAfterSVECalleeSaves) {
// If the callee-save area is before FP, restoring the FP implicitly
// deallocates non-callee-save SVE allocations. Otherwise, deallocate
// them explicitly.
if (!AFI->isStackRealigned() && !MFI.hasVarSizedObjects()) {
emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::SP,
DeallocateBefore, TII, MachineInstr::FrameDestroy, false,
NeedsWinCFI, &HasWinCFI);
}
// Deallocate callee-save non-SVE registers.
emitFrameOffset(MBB, RestoreBegin, DL, AArch64::SP, AArch64::SP,
StackOffset::getFixed(AFI->getCalleeSavedStackSize()), TII,
MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI);
// Deallocate fixed objects.
emitFrameOffset(MBB, RestoreEnd, DL, AArch64::SP, AArch64::SP,
StackOffset::getFixed(FixedObject), TII,
MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI);
// Deallocate callee-save SVE registers.
emitFrameOffset(MBB, RestoreEnd, DL, AArch64::SP, AArch64::SP,
DeallocateAfter, TII, MachineInstr::FrameDestroy, false,
NeedsWinCFI, &HasWinCFI);
} else if (SVEStackSize) {
int64_t SVECalleeSavedSize = AFI->getSVECalleeSavedStackSize();
// If we have stack realignment or variable-sized objects we must use the
// FP to restore SVE callee saves (as there is an unknown amount of
// data/padding between the SP and SVE CS area).
Register BaseForSVEDealloc =
(AFI->isStackRealigned() || MFI.hasVarSizedObjects()) ? AArch64::FP
: AArch64::SP;
if (SVECalleeSavedSize && BaseForSVEDealloc == AArch64::FP) {
Register CalleeSaveBase = AArch64::FP;
if (int64_t CalleeSaveBaseOffset =
AFI->getCalleeSaveBaseToFrameRecordOffset()) {
// If we have have an non-zero offset to the non-SVE CS base we need to
// compute the base address by subtracting the offest in a temporary
// register first (to avoid briefly deallocating the SVE CS).
CalleeSaveBase = MBB.getParent()->getRegInfo().createVirtualRegister(
&AArch64::GPR64RegClass);
emitFrameOffset(MBB, RestoreBegin, DL, CalleeSaveBase, AArch64::FP,
StackOffset::getFixed(-CalleeSaveBaseOffset), TII,
MachineInstr::FrameDestroy);
}
// The code below will deallocate the stack space space by moving the
// SP to the start of the SVE callee-save area.
emitFrameOffset(MBB, RestoreBegin, DL, AArch64::SP, CalleeSaveBase,
StackOffset::getScalable(-SVECalleeSavedSize), TII,
MachineInstr::FrameDestroy);
} else if (BaseForSVEDealloc == AArch64::SP) {
if (SVECalleeSavedSize) {
// Deallocate the non-SVE locals first before we can deallocate (and
// restore callee saves) from the SVE area.
emitFrameOffset(
MBB, RestoreBegin, DL, AArch64::SP, AArch64::SP,
StackOffset::getFixed(NumBytes), TII, MachineInstr::FrameDestroy,
false, NeedsWinCFI, &HasWinCFI, EmitCFI && !hasFP(MF),
SVEStackSize + StackOffset::getFixed(NumBytes + PrologueSaveSize));
NumBytes = 0;
}
emitFrameOffset(MBB, RestoreBegin, DL, AArch64::SP, AArch64::SP,
DeallocateBefore, TII, MachineInstr::FrameDestroy, false,
NeedsWinCFI, &HasWinCFI, EmitCFI && !hasFP(MF),
SVEStackSize +
StackOffset::getFixed(NumBytes + PrologueSaveSize));
emitFrameOffset(MBB, RestoreEnd, DL, AArch64::SP, AArch64::SP,
DeallocateAfter, TII, MachineInstr::FrameDestroy, false,
NeedsWinCFI, &HasWinCFI, EmitCFI && !hasFP(MF),
DeallocateAfter +
StackOffset::getFixed(NumBytes + PrologueSaveSize));
}
if (EmitCFI)
emitCalleeSavedSVERestores(MBB, RestoreEnd);
}
if (!hasFP(MF)) {
bool RedZone = canUseRedZone(MF);
// If this was a redzone leaf function, we don't need to restore the
// stack pointer (but we may need to pop stack args for fastcc).
if (RedZone && AfterCSRPopSize == 0)
return;
// Pop the local variables off the stack. If there are no callee-saved
// registers, it means we are actually positioned at the terminator and can
// combine stack increment for the locals and the stack increment for
// callee-popped arguments into (possibly) a single instruction and be done.
bool NoCalleeSaveRestore = PrologueSaveSize == 0;
int64_t StackRestoreBytes = RedZone ? 0 : NumBytes;
if (NoCalleeSaveRestore)
StackRestoreBytes += AfterCSRPopSize;
emitFrameOffset(
MBB, LastPopI, DL, AArch64::SP, AArch64::SP,
StackOffset::getFixed(StackRestoreBytes), TII,
MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI, EmitCFI,
StackOffset::getFixed((RedZone ? 0 : NumBytes) + PrologueSaveSize));
// If we were able to combine the local stack pop with the argument pop,
// then we're done.
if (NoCalleeSaveRestore || AfterCSRPopSize == 0) {
return;
}
NumBytes = 0;
}
// Restore the original stack pointer.
// FIXME: Rather than doing the math here, we should instead just use
// non-post-indexed loads for the restores if we aren't actually going to
// be able to save any instructions.
if (!IsFunclet && (MFI.hasVarSizedObjects() || AFI->isStackRealigned())) {
emitFrameOffset(
MBB, LastPopI, DL, AArch64::SP, AArch64::FP,
StackOffset::getFixed(-AFI->getCalleeSaveBaseToFrameRecordOffset()),
TII, MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI);
} else if (NumBytes)
emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::SP,
StackOffset::getFixed(NumBytes), TII,
MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI);
// When we are about to restore the CSRs, the CFA register is SP again.
if (EmitCFI && hasFP(MF))
CFIInstBuilder(MBB, LastPopI, MachineInstr::FrameDestroy)
.buildDefCFA(AArch64::SP, PrologueSaveSize);
// This must be placed after the callee-save restore code because that code
// assumes the SP is at the same location as it was after the callee-save save
// code in the prologue.
if (AfterCSRPopSize) {
assert(AfterCSRPopSize > 0 && "attempting to reallocate arg stack that an "
"interrupt may have clobbered");
emitFrameOffset(
MBB, MBB.getFirstTerminator(), DL, AArch64::SP, AArch64::SP,
StackOffset::getFixed(AfterCSRPopSize), TII, MachineInstr::FrameDestroy,
false, NeedsWinCFI, &HasWinCFI, EmitCFI,
StackOffset::getFixed(CombineAfterCSRBump ? PrologueSaveSize : 0));
}
}
bool AArch64FrameLowering::enableCFIFixup(const MachineFunction &MF) const {
return TargetFrameLowering::enableCFIFixup(MF) &&
MF.getInfo<AArch64FunctionInfo>()->needsDwarfUnwindInfo(MF);
}
bool AArch64FrameLowering::enableFullCFIFixup(const MachineFunction &MF) const {
return enableCFIFixup(MF) &&
MF.getInfo<AArch64FunctionInfo>()->needsAsyncDwarfUnwindInfo(MF);
}
/// getFrameIndexReference - Provide a base+offset reference to an FI slot for
/// debug info. It's the same as what we use for resolving the code-gen
/// references for now. FIXME: This can go wrong when references are
/// SP-relative and simple call frames aren't used.
StackOffset
AArch64FrameLowering::getFrameIndexReference(const MachineFunction &MF, int FI,
Register &FrameReg) const {
return resolveFrameIndexReference(
MF, FI, FrameReg,
/*PreferFP=*/
MF.getFunction().hasFnAttribute(Attribute::SanitizeHWAddress) ||
MF.getFunction().hasFnAttribute(Attribute::SanitizeMemTag),
/*ForSimm=*/false);
}
StackOffset
AArch64FrameLowering::getFrameIndexReferenceFromSP(const MachineFunction &MF,
int FI) const {
// This function serves to provide a comparable offset from a single reference
// point (the value of SP at function entry) that can be used for analysis,
// e.g. the stack-frame-layout analysis pass. It is not guaranteed to be
// correct for all objects in the presence of VLA-area objects or dynamic
// stack re-alignment.
const auto &MFI = MF.getFrameInfo();
int64_t ObjectOffset = MFI.getObjectOffset(FI);
StackOffset SVEStackSize = getSVEStackSize(MF);
// For VLA-area objects, just emit an offset at the end of the stack frame.
// Whilst not quite correct, these objects do live at the end of the frame and
// so it is more useful for analysis for the offset to reflect this.
if (MFI.isVariableSizedObjectIndex(FI)) {
return StackOffset::getFixed(-((int64_t)MFI.getStackSize())) - SVEStackSize;
}
// This is correct in the absence of any SVE stack objects.
if (!SVEStackSize)
return StackOffset::getFixed(ObjectOffset - getOffsetOfLocalArea());
const auto *AFI = MF.getInfo<AArch64FunctionInfo>();
bool FPAfterSVECalleeSaves =
isTargetWindows(MF) && AFI->getSVECalleeSavedStackSize();
if (MFI.getStackID(FI) == TargetStackID::ScalableVector) {
if (FPAfterSVECalleeSaves &&
-ObjectOffset <= (int64_t)AFI->getSVECalleeSavedStackSize())
return StackOffset::getScalable(ObjectOffset);
return StackOffset::get(-((int64_t)AFI->getCalleeSavedStackSize()),
ObjectOffset);
}
bool IsFixed = MFI.isFixedObjectIndex(FI);
bool IsCSR =
!IsFixed && ObjectOffset >= -((int)AFI->getCalleeSavedStackSize(MFI));
StackOffset ScalableOffset = {};
if (!IsFixed && !IsCSR) {
ScalableOffset = -SVEStackSize;
} else if (FPAfterSVECalleeSaves && IsCSR) {
ScalableOffset =
-StackOffset::getScalable(AFI->getSVECalleeSavedStackSize());
}
return StackOffset::getFixed(ObjectOffset) + ScalableOffset;
}
StackOffset
AArch64FrameLowering::getNonLocalFrameIndexReference(const MachineFunction &MF,
int FI) const {
return StackOffset::getFixed(getSEHFrameIndexOffset(MF, FI));
}
static StackOffset getFPOffset(const MachineFunction &MF,
int64_t ObjectOffset) {
const auto *AFI = MF.getInfo<AArch64FunctionInfo>();
const auto &Subtarget = MF.getSubtarget<AArch64Subtarget>();
const Function &F = MF.getFunction();
bool IsWin64 = Subtarget.isCallingConvWin64(F.getCallingConv(), F.isVarArg());
unsigned FixedObject =
getFixedObjectSize(MF, AFI, IsWin64, /*IsFunclet=*/false);
int64_t CalleeSaveSize = AFI->getCalleeSavedStackSize(MF.getFrameInfo());
int64_t FPAdjust =
CalleeSaveSize - AFI->getCalleeSaveBaseToFrameRecordOffset();
return StackOffset::getFixed(ObjectOffset + FixedObject + FPAdjust);
}
static StackOffset getStackOffset(const MachineFunction &MF,
int64_t ObjectOffset) {
const auto &MFI = MF.getFrameInfo();
return StackOffset::getFixed(ObjectOffset + (int64_t)MFI.getStackSize());
}
// TODO: This function currently does not work for scalable vectors.
int AArch64FrameLowering::getSEHFrameIndexOffset(const MachineFunction &MF,
int FI) const {
const auto *RegInfo = static_cast<const AArch64RegisterInfo *>(
MF.getSubtarget().getRegisterInfo());
int ObjectOffset = MF.getFrameInfo().getObjectOffset(FI);
return RegInfo->getLocalAddressRegister(MF) == AArch64::FP
? getFPOffset(MF, ObjectOffset).getFixed()
: getStackOffset(MF, ObjectOffset).getFixed();
}
StackOffset AArch64FrameLowering::resolveFrameIndexReference(
const MachineFunction &MF, int FI, Register &FrameReg, bool PreferFP,
bool ForSimm) const {
const auto &MFI = MF.getFrameInfo();
int64_t ObjectOffset = MFI.getObjectOffset(FI);
bool isFixed = MFI.isFixedObjectIndex(FI);
bool isSVE = MFI.getStackID(FI) == TargetStackID::ScalableVector;
return resolveFrameOffsetReference(MF, ObjectOffset, isFixed, isSVE, FrameReg,
PreferFP, ForSimm);
}
StackOffset AArch64FrameLowering::resolveFrameOffsetReference(
const MachineFunction &MF, int64_t ObjectOffset, bool isFixed, bool isSVE,
Register &FrameReg, bool PreferFP, bool ForSimm) const {
const auto &MFI = MF.getFrameInfo();
const auto *RegInfo = static_cast<const AArch64RegisterInfo *>(
MF.getSubtarget().getRegisterInfo());
const auto *AFI = MF.getInfo<AArch64FunctionInfo>();
const auto &Subtarget = MF.getSubtarget<AArch64Subtarget>();
int64_t FPOffset = getFPOffset(MF, ObjectOffset).getFixed();
int64_t Offset = getStackOffset(MF, ObjectOffset).getFixed();
bool isCSR =
!isFixed && ObjectOffset >= -((int)AFI->getCalleeSavedStackSize(MFI));
const StackOffset &SVEStackSize = getSVEStackSize(MF);
// Use frame pointer to reference fixed objects. Use it for locals if
// there are VLAs or a dynamically realigned SP (and thus the SP isn't
// reliable as a base). Make sure useFPForScavengingIndex() does the
// right thing for the emergency spill slot.
bool UseFP = false;
if (AFI->hasStackFrame() && !isSVE) {
// We shouldn't prefer using the FP to access fixed-sized stack objects when
// there are scalable (SVE) objects in between the FP and the fixed-sized
// objects.
PreferFP &= !SVEStackSize;
// Note: Keeping the following as multiple 'if' statements rather than
// merging to a single expression for readability.
//
// Argument access should always use the FP.
if (isFixed) {
UseFP = hasFP(MF);
} else if (isCSR && RegInfo->hasStackRealignment(MF)) {
// References to the CSR area must use FP if we're re-aligning the stack
// since the dynamically-sized alignment padding is between the SP/BP and
// the CSR area.
assert(hasFP(MF) && "Re-aligned stack must have frame pointer");
UseFP = true;
} else if (hasFP(MF) && !RegInfo->hasStackRealignment(MF)) {
// If the FPOffset is negative and we're producing a signed immediate, we
// have to keep in mind that the available offset range for negative
// offsets is smaller than for positive ones. If an offset is available
// via the FP and the SP, use whichever is closest.
bool FPOffsetFits = !ForSimm || FPOffset >= -256;
PreferFP |= Offset > -FPOffset && !SVEStackSize;
if (FPOffset >= 0) {
// If the FPOffset is positive, that'll always be best, as the SP/BP
// will be even further away.
UseFP = true;
} else if (MFI.hasVarSizedObjects()) {
// If we have variable sized objects, we can use either FP or BP, as the
// SP offset is unknown. We can use the base pointer if we have one and
// FP is not preferred. If not, we're stuck with using FP.
bool CanUseBP = RegInfo->hasBasePointer(MF);
if (FPOffsetFits && CanUseBP) // Both are ok. Pick the best.
UseFP = PreferFP;
else if (!CanUseBP) // Can't use BP. Forced to use FP.
UseFP = true;
// else we can use BP and FP, but the offset from FP won't fit.
// That will make us scavenge registers which we can probably avoid by
// using BP. If it won't fit for BP either, we'll scavenge anyway.
} else if (MF.hasEHFunclets() && !RegInfo->hasBasePointer(MF)) {
// Funclets access the locals contained in the parent's stack frame
// via the frame pointer, so we have to use the FP in the parent
// function.
(void) Subtarget;
assert(Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv(),
MF.getFunction().isVarArg()) &&
"Funclets should only be present on Win64");
UseFP = true;
} else {
// We have the choice between FP and (SP or BP).
if (FPOffsetFits && PreferFP) // If FP is the best fit, use it.
UseFP = true;
}
}
}
assert(
((isFixed || isCSR) || !RegInfo->hasStackRealignment(MF) || !UseFP) &&
"In the presence of dynamic stack pointer realignment, "
"non-argument/CSR objects cannot be accessed through the frame pointer");
bool FPAfterSVECalleeSaves =
isTargetWindows(MF) && AFI->getSVECalleeSavedStackSize();
if (isSVE) {
StackOffset FPOffset =
StackOffset::get(-AFI->getCalleeSaveBaseToFrameRecordOffset(), ObjectOffset);
StackOffset SPOffset =
SVEStackSize +
StackOffset::get(MFI.getStackSize() - AFI->getCalleeSavedStackSize(),
ObjectOffset);
if (FPAfterSVECalleeSaves) {
FPOffset += StackOffset::getScalable(AFI->getSVECalleeSavedStackSize());
if (-ObjectOffset <= (int64_t)AFI->getSVECalleeSavedStackSize()) {
FPOffset += StackOffset::getFixed(AFI->getCalleeSavedStackSize());
SPOffset += StackOffset::getFixed(AFI->getCalleeSavedStackSize());
}
}
// Always use the FP for SVE spills if available and beneficial.
if (hasFP(MF) && (SPOffset.getFixed() ||
FPOffset.getScalable() < SPOffset.getScalable() ||
RegInfo->hasStackRealignment(MF))) {
FrameReg = RegInfo->getFrameRegister(MF);
return FPOffset;
}
FrameReg = RegInfo->hasBasePointer(MF) ? RegInfo->getBaseRegister()
: (unsigned)AArch64::SP;
return SPOffset;
}
StackOffset ScalableOffset = {};
if (FPAfterSVECalleeSaves) {
// In this stack layout, the FP is in between the callee saves and other
// SVE allocations.
StackOffset SVECalleeSavedStack =
StackOffset::getScalable(AFI->getSVECalleeSavedStackSize());
if (UseFP) {
if (isFixed)
ScalableOffset = SVECalleeSavedStack;
else if (!isCSR)
ScalableOffset = SVECalleeSavedStack - SVEStackSize;
} else {
if (isFixed)
ScalableOffset = SVEStackSize;
else if (isCSR)
ScalableOffset = SVEStackSize - SVECalleeSavedStack;
}
} else {
if (UseFP && !(isFixed || isCSR))
ScalableOffset = -SVEStackSize;
if (!UseFP && (isFixed || isCSR))
ScalableOffset = SVEStackSize;
}
if (UseFP) {
FrameReg = RegInfo->getFrameRegister(MF);
return StackOffset::getFixed(FPOffset) + ScalableOffset;
}
// Use the base pointer if we have one.
if (RegInfo->hasBasePointer(MF))
FrameReg = RegInfo->getBaseRegister();
else {
assert(!MFI.hasVarSizedObjects() &&
"Can't use SP when we have var sized objects.");
FrameReg = AArch64::SP;
// If we're using the red zone for this function, the SP won't actually
// be adjusted, so the offsets will be negative. They're also all
// within range of the signed 9-bit immediate instructions.
if (canUseRedZone(MF))
Offset -= AFI->getLocalStackSize();
}
return StackOffset::getFixed(Offset) + ScalableOffset;
}
static unsigned getPrologueDeath(MachineFunction &MF, unsigned Reg) {
// Do not set a kill flag on values that are also marked as live-in. This
// happens with the @llvm-returnaddress intrinsic and with arguments passed in
// callee saved registers.
// Omitting the kill flags is conservatively correct even if the live-in
// is not used after all.
bool IsLiveIn = MF.getRegInfo().isLiveIn(Reg);
return getKillRegState(!IsLiveIn);
}
static bool produceCompactUnwindFrame(MachineFunction &MF) {
const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
AttributeList Attrs = MF.getFunction().getAttributes();
AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
return Subtarget.isTargetMachO() &&
!(Subtarget.getTargetLowering()->supportSwiftError() &&
Attrs.hasAttrSomewhere(Attribute::SwiftError)) &&
MF.getFunction().getCallingConv() != CallingConv::SwiftTail &&
!requiresSaveVG(MF) && !AFI->isSVECC();
}
static bool invalidateWindowsRegisterPairing(unsigned Reg1, unsigned Reg2,
bool NeedsWinCFI, bool IsFirst,
const TargetRegisterInfo *TRI) {
// If we are generating register pairs for a Windows function that requires
// EH support, then pair consecutive registers only. There are no unwind
// opcodes for saves/restores of non-consecutive register pairs.
// The unwind opcodes are save_regp, save_regp_x, save_fregp, save_frepg_x,
// save_lrpair.
// https://docs.microsoft.com/en-us/cpp/build/arm64-exception-handling
if (Reg2 == AArch64::FP)
return true;
if (!NeedsWinCFI)
return false;
if (TRI->getEncodingValue(Reg2) == TRI->getEncodingValue(Reg1) + 1)
return false;
// If pairing a GPR with LR, the pair can be described by the save_lrpair
// opcode. If this is the first register pair, it would end up with a
// predecrement, but there's no save_lrpair_x opcode, so we can only do this
// if LR is paired with something else than the first register.
// The save_lrpair opcode requires the first register to be an odd one.
if (Reg1 >= AArch64::X19 && Reg1 <= AArch64::X27 &&
(Reg1 - AArch64::X19) % 2 == 0 && Reg2 == AArch64::LR && !IsFirst)
return false;
return true;
}
/// Returns true if Reg1 and Reg2 cannot be paired using a ldp/stp instruction.
/// WindowsCFI requires that only consecutive registers can be paired.
/// LR and FP need to be allocated together when the frame needs to save
/// the frame-record. This means any other register pairing with LR is invalid.
static bool invalidateRegisterPairing(unsigned Reg1, unsigned Reg2,
bool UsesWinAAPCS, bool NeedsWinCFI,
bool NeedsFrameRecord, bool IsFirst,
const TargetRegisterInfo *TRI) {
if (UsesWinAAPCS)
return invalidateWindowsRegisterPairing(Reg1, Reg2, NeedsWinCFI, IsFirst,
TRI);
// If we need to store the frame record, don't pair any register
// with LR other than FP.
if (NeedsFrameRecord)
return Reg2 == AArch64::LR;
return false;
}
namespace {
struct RegPairInfo {
unsigned Reg1 = AArch64::NoRegister;
unsigned Reg2 = AArch64::NoRegister;
int FrameIdx;
int Offset;
enum RegType { GPR, FPR64, FPR128, PPR, ZPR, VG } Type;
const TargetRegisterClass *RC;
RegPairInfo() = default;
bool isPaired() const { return Reg2 != AArch64::NoRegister; }
bool isScalable() const { return Type == PPR || Type == ZPR; }
};
} // end anonymous namespace
unsigned findFreePredicateReg(BitVector &SavedRegs) {
for (unsigned PReg = AArch64::P8; PReg <= AArch64::P15; ++PReg) {
if (SavedRegs.test(PReg)) {
unsigned PNReg = PReg - AArch64::P0 + AArch64::PN0;
return PNReg;
}
}
return AArch64::NoRegister;
}
// The multivector LD/ST are available only for SME or SVE2p1 targets
bool enableMultiVectorSpillFill(const AArch64Subtarget &Subtarget,
MachineFunction &MF) {
if (DisableMultiVectorSpillFill)
return false;
SMEAttrs FuncAttrs = MF.getInfo<AArch64FunctionInfo>()->getSMEFnAttrs();
bool IsLocallyStreaming =
FuncAttrs.hasStreamingBody() && !FuncAttrs.hasStreamingInterface();
// Only when in streaming mode SME2 instructions can be safely used.
// It is not safe to use SME2 instructions when in streaming compatible or
// locally streaming mode.
return Subtarget.hasSVE2p1() ||
(Subtarget.hasSME2() &&
(!IsLocallyStreaming && Subtarget.isStreaming()));
}
static void computeCalleeSaveRegisterPairs(
MachineFunction &MF, ArrayRef<CalleeSavedInfo> CSI,
const TargetRegisterInfo *TRI, SmallVectorImpl<RegPairInfo> &RegPairs,
bool NeedsFrameRecord) {
if (CSI.empty())
return;
bool IsWindows = isTargetWindows(MF);
bool NeedsWinCFI = needsWinCFI(MF);
AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
unsigned StackHazardSize = getStackHazardSize(MF);
MachineFrameInfo &MFI = MF.getFrameInfo();
CallingConv::ID CC = MF.getFunction().getCallingConv();
unsigned Count = CSI.size();
(void)CC;
// MachO's compact unwind format relies on all registers being stored in
// pairs.
assert((!produceCompactUnwindFrame(MF) || CC == CallingConv::PreserveMost ||
CC == CallingConv::PreserveAll || CC == CallingConv::CXX_FAST_TLS ||
CC == CallingConv::Win64 || (Count & 1) == 0) &&
"Odd number of callee-saved regs to spill!");
int ByteOffset = AFI->getCalleeSavedStackSize();
int StackFillDir = -1;
int RegInc = 1;
unsigned FirstReg = 0;
if (NeedsWinCFI) {
// For WinCFI, fill the stack from the bottom up.
ByteOffset = 0;
StackFillDir = 1;
// As the CSI array is reversed to match PrologEpilogInserter, iterate
// backwards, to pair up registers starting from lower numbered registers.
RegInc = -1;
FirstReg = Count - 1;
}
bool FPAfterSVECalleeSaves = IsWindows && AFI->getSVECalleeSavedStackSize();
int ScalableByteOffset =
FPAfterSVECalleeSaves ? 0 : AFI->getSVECalleeSavedStackSize();
bool NeedGapToAlignStack = AFI->hasCalleeSaveStackFreeSpace();
Register LastReg = 0;
// When iterating backwards, the loop condition relies on unsigned wraparound.
for (unsigned i = FirstReg; i < Count; i += RegInc) {
RegPairInfo RPI;
RPI.Reg1 = CSI[i].getReg();
if (AArch64::GPR64RegClass.contains(RPI.Reg1)) {
RPI.Type = RegPairInfo::GPR;
RPI.RC = &AArch64::GPR64RegClass;
} else if (AArch64::FPR64RegClass.contains(RPI.Reg1)) {
RPI.Type = RegPairInfo::FPR64;
RPI.RC = &AArch64::FPR64RegClass;
} else if (AArch64::FPR128RegClass.contains(RPI.Reg1)) {
RPI.Type = RegPairInfo::FPR128;
RPI.RC = &AArch64::FPR128RegClass;
} else if (AArch64::ZPRRegClass.contains(RPI.Reg1)) {
RPI.Type = RegPairInfo::ZPR;
RPI.RC = &AArch64::ZPRRegClass;
} else if (AArch64::PPRRegClass.contains(RPI.Reg1)) {
RPI.Type = RegPairInfo::PPR;
RPI.RC = &AArch64::PPRRegClass;
} else if (RPI.Reg1 == AArch64::VG) {
RPI.Type = RegPairInfo::VG;
RPI.RC = &AArch64::FIXED_REGSRegClass;
} else {
llvm_unreachable("Unsupported register class.");
}
// Add the stack hazard size as we transition from GPR->FPR CSRs.
if (AFI->hasStackHazardSlotIndex() &&
(!LastReg || !AArch64InstrInfo::isFpOrNEON(LastReg)) &&
AArch64InstrInfo::isFpOrNEON(RPI.Reg1))
ByteOffset += StackFillDir * StackHazardSize;
LastReg = RPI.Reg1;
int Scale = TRI->getSpillSize(*RPI.RC);
// Add the next reg to the pair if it is in the same register class.
if (unsigned(i + RegInc) < Count && !AFI->hasStackHazardSlotIndex()) {
MCRegister NextReg = CSI[i + RegInc].getReg();
bool IsFirst = i == FirstReg;
switch (RPI.Type) {
case RegPairInfo::GPR:
if (AArch64::GPR64RegClass.contains(NextReg) &&
!invalidateRegisterPairing(RPI.Reg1, NextReg, IsWindows,
NeedsWinCFI, NeedsFrameRecord, IsFirst,
TRI))
RPI.Reg2 = NextReg;
break;
case RegPairInfo::FPR64:
if (AArch64::FPR64RegClass.contains(NextReg) &&
!invalidateWindowsRegisterPairing(RPI.Reg1, NextReg, NeedsWinCFI,
IsFirst, TRI))
RPI.Reg2 = NextReg;
break;
case RegPairInfo::FPR128:
if (AArch64::FPR128RegClass.contains(NextReg))
RPI.Reg2 = NextReg;
break;
case RegPairInfo::PPR:
break;
case RegPairInfo::ZPR:
if (AFI->getPredicateRegForFillSpill() != 0 &&
((RPI.Reg1 - AArch64::Z0) & 1) == 0 && (NextReg == RPI.Reg1 + 1)) {
// Calculate offset of register pair to see if pair instruction can be
// used.
int Offset = (ScalableByteOffset + StackFillDir * 2 * Scale) / Scale;
if ((-16 <= Offset && Offset <= 14) && (Offset % 2 == 0))
RPI.Reg2 = NextReg;
}
break;
case RegPairInfo::VG:
break;
}
}
// GPRs and FPRs are saved in pairs of 64-bit regs. We expect the CSI
// list to come in sorted by frame index so that we can issue the store
// pair instructions directly. Assert if we see anything otherwise.
//
// The order of the registers in the list is controlled by
// getCalleeSavedRegs(), so they will always be in-order, as well.
assert((!RPI.isPaired() ||
(CSI[i].getFrameIdx() + RegInc == CSI[i + RegInc].getFrameIdx())) &&
"Out of order callee saved regs!");
assert((!RPI.isPaired() || !NeedsFrameRecord || RPI.Reg2 != AArch64::FP ||
RPI.Reg1 == AArch64::LR) &&
"FrameRecord must be allocated together with LR");
// Windows AAPCS has FP and LR reversed.
assert((!RPI.isPaired() || !NeedsFrameRecord || RPI.Reg1 != AArch64::FP ||
RPI.Reg2 == AArch64::LR) &&
"FrameRecord must be allocated together with LR");
// MachO's compact unwind format relies on all registers being stored in
// adjacent register pairs.
assert((!produceCompactUnwindFrame(MF) || CC == CallingConv::PreserveMost ||
CC == CallingConv::PreserveAll || CC == CallingConv::CXX_FAST_TLS ||
CC == CallingConv::Win64 ||
(RPI.isPaired() &&
((RPI.Reg1 == AArch64::LR && RPI.Reg2 == AArch64::FP) ||
RPI.Reg1 + 1 == RPI.Reg2))) &&
"Callee-save registers not saved as adjacent register pair!");
RPI.FrameIdx = CSI[i].getFrameIdx();
if (NeedsWinCFI &&
RPI.isPaired()) // RPI.FrameIdx must be the lower index of the pair
RPI.FrameIdx = CSI[i + RegInc].getFrameIdx();
// Realign the scalable offset if necessary. This is relevant when
// spilling predicates on Windows.
if (RPI.isScalable() && ScalableByteOffset % Scale != 0) {
ScalableByteOffset = alignTo(ScalableByteOffset, Scale);
}
int OffsetPre = RPI.isScalable() ? ScalableByteOffset : ByteOffset;
assert(OffsetPre % Scale == 0);
if (RPI.isScalable())
ScalableByteOffset += StackFillDir * (RPI.isPaired() ? 2 * Scale : Scale);
else
ByteOffset += StackFillDir * (RPI.isPaired() ? 2 * Scale : Scale);
// Swift's async context is directly before FP, so allocate an extra
// 8 bytes for it.
if (NeedsFrameRecord && AFI->hasSwiftAsyncContext() &&
((!IsWindows && RPI.Reg2 == AArch64::FP) ||
(IsWindows && RPI.Reg2 == AArch64::LR)))
ByteOffset += StackFillDir * 8;
// Round up size of non-pair to pair size if we need to pad the
// callee-save area to ensure 16-byte alignment.
if (NeedGapToAlignStack && !NeedsWinCFI && !RPI.isScalable() &&
RPI.Type != RegPairInfo::FPR128 && !RPI.isPaired() &&
ByteOffset % 16 != 0) {
ByteOffset += 8 * StackFillDir;
assert(MFI.getObjectAlign(RPI.FrameIdx) <= Align(16));
// A stack frame with a gap looks like this, bottom up:
// d9, d8. x21, gap, x20, x19.
// Set extra alignment on the x21 object to create the gap above it.
MFI.setObjectAlignment(RPI.FrameIdx, Align(16));
NeedGapToAlignStack = false;
}
int OffsetPost = RPI.isScalable() ? ScalableByteOffset : ByteOffset;
assert(OffsetPost % Scale == 0);
// If filling top down (default), we want the offset after incrementing it.
// If filling bottom up (WinCFI) we need the original offset.
int Offset = NeedsWinCFI ? OffsetPre : OffsetPost;
// The FP, LR pair goes 8 bytes into our expanded 24-byte slot so that the
// Swift context can directly precede FP.
if (NeedsFrameRecord && AFI->hasSwiftAsyncContext() &&
((!IsWindows && RPI.Reg2 == AArch64::FP) ||
(IsWindows && RPI.Reg2 == AArch64::LR)))
Offset += 8;
RPI.Offset = Offset / Scale;
assert((!RPI.isPaired() ||
(!RPI.isScalable() && RPI.Offset >= -64 && RPI.Offset <= 63) ||
(RPI.isScalable() && RPI.Offset >= -256 && RPI.Offset <= 255)) &&
"Offset out of bounds for LDP/STP immediate");
auto isFrameRecord = [&] {
if (RPI.isPaired())
return IsWindows ? RPI.Reg1 == AArch64::FP && RPI.Reg2 == AArch64::LR
: RPI.Reg1 == AArch64::LR && RPI.Reg2 == AArch64::FP;
// Otherwise, look for the frame record as two unpaired registers. This is
// needed for -aarch64-stack-hazard-size=<val>, which disables register
// pairing (as the padding may be too large for the LDP/STP offset). Note:
// On Windows, this check works out as current reg == FP, next reg == LR,
// and on other platforms current reg == FP, previous reg == LR. This
// works out as the correct pre-increment or post-increment offsets
// respectively.
return i > 0 && RPI.Reg1 == AArch64::FP &&
CSI[i - 1].getReg() == AArch64::LR;
};
// Save the offset to frame record so that the FP register can point to the
// innermost frame record (spilled FP and LR registers).
if (NeedsFrameRecord && isFrameRecord())
AFI->setCalleeSaveBaseToFrameRecordOffset(Offset);
RegPairs.push_back(RPI);
if (RPI.isPaired())
i += RegInc;
}
if (NeedsWinCFI) {
// If we need an alignment gap in the stack, align the topmost stack
// object. A stack frame with a gap looks like this, bottom up:
// x19, d8. d9, gap.
// Set extra alignment on the topmost stack object (the first element in
// CSI, which goes top down), to create the gap above it.
if (AFI->hasCalleeSaveStackFreeSpace())
MFI.setObjectAlignment(CSI[0].getFrameIdx(), Align(16));
// We iterated bottom up over the registers; flip RegPairs back to top
// down order.
std::reverse(RegPairs.begin(), RegPairs.end());
}
}
bool AArch64FrameLowering::spillCalleeSavedRegisters(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
ArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const {
MachineFunction &MF = *MBB.getParent();
const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
bool NeedsWinCFI = needsWinCFI(MF);
DebugLoc DL;
SmallVector<RegPairInfo, 8> RegPairs;
computeCalleeSaveRegisterPairs(MF, CSI, TRI, RegPairs, hasFP(MF));
MachineRegisterInfo &MRI = MF.getRegInfo();
// Refresh the reserved regs in case there are any potential changes since the
// last freeze.
MRI.freezeReservedRegs();
if (homogeneousPrologEpilog(MF)) {
auto MIB = BuildMI(MBB, MI, DL, TII.get(AArch64::HOM_Prolog))
.setMIFlag(MachineInstr::FrameSetup);
for (auto &RPI : RegPairs) {
MIB.addReg(RPI.Reg1);
MIB.addReg(RPI.Reg2);
// Update register live in.
if (!MRI.isReserved(RPI.Reg1))
MBB.addLiveIn(RPI.Reg1);
if (RPI.isPaired() && !MRI.isReserved(RPI.Reg2))
MBB.addLiveIn(RPI.Reg2);
}
return true;
}
bool PTrueCreated = false;
for (const RegPairInfo &RPI : llvm::reverse(RegPairs)) {
unsigned Reg1 = RPI.Reg1;
unsigned Reg2 = RPI.Reg2;
unsigned StrOpc;
// Issue sequence of spills for cs regs. The first spill may be converted
// to a pre-decrement store later by emitPrologue if the callee-save stack
// area allocation can't be combined with the local stack area allocation.
// For example:
// stp x22, x21, [sp, #0] // addImm(+0)
// stp x20, x19, [sp, #16] // addImm(+2)
// stp fp, lr, [sp, #32] // addImm(+4)
// Rationale: This sequence saves uop updates compared to a sequence of
// pre-increment spills like stp xi,xj,[sp,#-16]!
// Note: Similar rationale and sequence for restores in epilog.
unsigned Size = TRI->getSpillSize(*RPI.RC);
Align Alignment = TRI->getSpillAlign(*RPI.RC);
switch (RPI.Type) {
case RegPairInfo::GPR:
StrOpc = RPI.isPaired() ? AArch64::STPXi : AArch64::STRXui;
break;
case RegPairInfo::FPR64:
StrOpc = RPI.isPaired() ? AArch64::STPDi : AArch64::STRDui;
break;
case RegPairInfo::FPR128:
StrOpc = RPI.isPaired() ? AArch64::STPQi : AArch64::STRQui;
break;
case RegPairInfo::ZPR:
StrOpc = RPI.isPaired() ? AArch64::ST1B_2Z_IMM : AArch64::STR_ZXI;
break;
case RegPairInfo::PPR:
StrOpc =
Size == 16 ? AArch64::SPILL_PPR_TO_ZPR_SLOT_PSEUDO : AArch64::STR_PXI;
break;
case RegPairInfo::VG:
StrOpc = AArch64::STRXui;
break;
}
unsigned X0Scratch = AArch64::NoRegister;
if (Reg1 == AArch64::VG) {
// Find an available register to store value of VG to.
Reg1 = findScratchNonCalleeSaveRegister(&MBB, true);
assert(Reg1 != AArch64::NoRegister);
SMEAttrs Attrs = AFI->getSMEFnAttrs();
if (Attrs.hasStreamingBody() && !Attrs.hasStreamingInterface() &&
AFI->getStreamingVGIdx() == std::numeric_limits<int>::max()) {
// For locally-streaming functions, we need to store both the streaming
// & non-streaming VG. Spill the streaming value first.
BuildMI(MBB, MI, DL, TII.get(AArch64::RDSVLI_XI), Reg1)
.addImm(1)
.setMIFlag(MachineInstr::FrameSetup);
BuildMI(MBB, MI, DL, TII.get(AArch64::UBFMXri), Reg1)
.addReg(Reg1)
.addImm(3)
.addImm(63)
.setMIFlag(MachineInstr::FrameSetup);
AFI->setStreamingVGIdx(RPI.FrameIdx);
} else if (MF.getSubtarget<AArch64Subtarget>().hasSVE()) {
BuildMI(MBB, MI, DL, TII.get(AArch64::CNTD_XPiI), Reg1)
.addImm(31)
.addImm(1)
.setMIFlag(MachineInstr::FrameSetup);
AFI->setVGIdx(RPI.FrameIdx);
} else {
const AArch64Subtarget &STI = MF.getSubtarget<AArch64Subtarget>();
if (llvm::any_of(
MBB.liveins(),
[&STI](const MachineBasicBlock::RegisterMaskPair &LiveIn) {
return STI.getRegisterInfo()->isSuperOrSubRegisterEq(
AArch64::X0, LiveIn.PhysReg);
}))
X0Scratch = Reg1;
if (X0Scratch != AArch64::NoRegister)
BuildMI(MBB, MI, DL, TII.get(AArch64::ORRXrr), Reg1)
.addReg(AArch64::XZR)
.addReg(AArch64::X0, RegState::Undef)
.addReg(AArch64::X0, RegState::Implicit)
.setMIFlag(MachineInstr::FrameSetup);
const uint32_t *RegMask = TRI->getCallPreservedMask(
MF,
CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1);
BuildMI(MBB, MI, DL, TII.get(AArch64::BL))
.addExternalSymbol("__arm_get_current_vg")
.addRegMask(RegMask)
.addReg(AArch64::X0, RegState::ImplicitDefine)
.setMIFlag(MachineInstr::FrameSetup);
Reg1 = AArch64::X0;
AFI->setVGIdx(RPI.FrameIdx);
}
}
LLVM_DEBUG(dbgs() << "CSR spill: (" << printReg(Reg1, TRI);
if (RPI.isPaired()) dbgs() << ", " << printReg(Reg2, TRI);
dbgs() << ") -> fi#(" << RPI.FrameIdx;
if (RPI.isPaired()) dbgs() << ", " << RPI.FrameIdx + 1;
dbgs() << ")\n");
assert((!NeedsWinCFI || !(Reg1 == AArch64::LR && Reg2 == AArch64::FP)) &&
"Windows unwdinding requires a consecutive (FP,LR) pair");
// Windows unwind codes require consecutive registers if registers are
// paired. Make the switch here, so that the code below will save (x,x+1)
// and not (x+1,x).
unsigned FrameIdxReg1 = RPI.FrameIdx;
unsigned FrameIdxReg2 = RPI.FrameIdx + 1;
if (NeedsWinCFI && RPI.isPaired()) {
std::swap(Reg1, Reg2);
std::swap(FrameIdxReg1, FrameIdxReg2);
}
if (RPI.isPaired() && RPI.isScalable()) {
[[maybe_unused]] const AArch64Subtarget &Subtarget =
MF.getSubtarget<AArch64Subtarget>();
AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
unsigned PnReg = AFI->getPredicateRegForFillSpill();
assert((PnReg != 0 && enableMultiVectorSpillFill(Subtarget, MF)) &&
"Expects SVE2.1 or SME2 target and a predicate register");
#ifdef EXPENSIVE_CHECKS
auto IsPPR = [](const RegPairInfo &c) {
return c.Reg1 == RegPairInfo::PPR;
};
auto PPRBegin = std::find_if(RegPairs.begin(), RegPairs.end(), IsPPR);
auto IsZPR = [](const RegPairInfo &c) {
return c.Type == RegPairInfo::ZPR;
};
auto ZPRBegin = std::find_if(RegPairs.begin(), RegPairs.end(), IsZPR);
assert(!(PPRBegin < ZPRBegin) &&
"Expected callee save predicate to be handled first");
#endif
if (!PTrueCreated) {
PTrueCreated = true;
BuildMI(MBB, MI, DL, TII.get(AArch64::PTRUE_C_B), PnReg)
.setMIFlags(MachineInstr::FrameSetup);
}
MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(StrOpc));
if (!MRI.isReserved(Reg1))
MBB.addLiveIn(Reg1);
if (!MRI.isReserved(Reg2))
MBB.addLiveIn(Reg2);
MIB.addReg(/*PairRegs*/ AArch64::Z0_Z1 + (RPI.Reg1 - AArch64::Z0));
MIB.addMemOperand(MF.getMachineMemOperand(
MachinePointerInfo::getFixedStack(MF, FrameIdxReg2),
MachineMemOperand::MOStore, Size, Alignment));
MIB.addReg(PnReg);
MIB.addReg(AArch64::SP)
.addImm(RPI.Offset / 2) // [sp, #imm*2*vscale],
// where 2*vscale is implicit
.setMIFlag(MachineInstr::FrameSetup);
MIB.addMemOperand(MF.getMachineMemOperand(
MachinePointerInfo::getFixedStack(MF, FrameIdxReg1),
MachineMemOperand::MOStore, Size, Alignment));
if (NeedsWinCFI)
InsertSEH(MIB, TII, MachineInstr::FrameSetup);
} else { // The code when the pair of ZReg is not present
MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(StrOpc));
if (!MRI.isReserved(Reg1))
MBB.addLiveIn(Reg1);
if (RPI.isPaired()) {
if (!MRI.isReserved(Reg2))
MBB.addLiveIn(Reg2);
MIB.addReg(Reg2, getPrologueDeath(MF, Reg2));
MIB.addMemOperand(MF.getMachineMemOperand(
MachinePointerInfo::getFixedStack(MF, FrameIdxReg2),
MachineMemOperand::MOStore, Size, Alignment));
}
MIB.addReg(Reg1, getPrologueDeath(MF, Reg1))
.addReg(AArch64::SP)
.addImm(RPI.Offset) // [sp, #offset*vscale],
// where factor*vscale is implicit
.setMIFlag(MachineInstr::FrameSetup);
MIB.addMemOperand(MF.getMachineMemOperand(
MachinePointerInfo::getFixedStack(MF, FrameIdxReg1),
MachineMemOperand::MOStore, Size, Alignment));
if (NeedsWinCFI)
InsertSEH(MIB, TII, MachineInstr::FrameSetup);
}
// Update the StackIDs of the SVE stack slots.
MachineFrameInfo &MFI = MF.getFrameInfo();
if (RPI.Type == RegPairInfo::ZPR || RPI.Type == RegPairInfo::PPR) {
MFI.setStackID(FrameIdxReg1, TargetStackID::ScalableVector);
if (RPI.isPaired())
MFI.setStackID(FrameIdxReg2, TargetStackID::ScalableVector);
}
if (X0Scratch != AArch64::NoRegister)
BuildMI(MBB, MI, DL, TII.get(AArch64::ORRXrr), AArch64::X0)
.addReg(AArch64::XZR)
.addReg(X0Scratch, RegState::Undef)
.addReg(X0Scratch, RegState::Implicit)
.setMIFlag(MachineInstr::FrameSetup);
}
return true;
}
bool AArch64FrameLowering::restoreCalleeSavedRegisters(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
MutableArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const {
MachineFunction &MF = *MBB.getParent();
const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
DebugLoc DL;
SmallVector<RegPairInfo, 8> RegPairs;
bool NeedsWinCFI = needsWinCFI(MF);
if (MBBI != MBB.end())
DL = MBBI->getDebugLoc();
computeCalleeSaveRegisterPairs(MF, CSI, TRI, RegPairs, hasFP(MF));
if (homogeneousPrologEpilog(MF, &MBB)) {
auto MIB = BuildMI(MBB, MBBI, DL, TII.get(AArch64::HOM_Epilog))
.setMIFlag(MachineInstr::FrameDestroy);
for (auto &RPI : RegPairs) {
MIB.addReg(RPI.Reg1, RegState::Define);
MIB.addReg(RPI.Reg2, RegState::Define);
}
return true;
}
// For performance reasons restore SVE register in increasing order
auto IsPPR = [](const RegPairInfo &c) { return c.Type == RegPairInfo::PPR; };
auto PPRBegin = llvm::find_if(RegPairs, IsPPR);
auto PPREnd = std::find_if_not(PPRBegin, RegPairs.end(), IsPPR);
std::reverse(PPRBegin, PPREnd);
auto IsZPR = [](const RegPairInfo &c) { return c.Type == RegPairInfo::ZPR; };
auto ZPRBegin = llvm::find_if(RegPairs, IsZPR);
auto ZPREnd = std::find_if_not(ZPRBegin, RegPairs.end(), IsZPR);
std::reverse(ZPRBegin, ZPREnd);
bool PTrueCreated = false;
for (const RegPairInfo &RPI : RegPairs) {
unsigned Reg1 = RPI.Reg1;
unsigned Reg2 = RPI.Reg2;
// Issue sequence of restores for cs regs. The last restore may be converted
// to a post-increment load later by emitEpilogue if the callee-save stack
// area allocation can't be combined with the local stack area allocation.
// For example:
// ldp fp, lr, [sp, #32] // addImm(+4)
// ldp x20, x19, [sp, #16] // addImm(+2)
// ldp x22, x21, [sp, #0] // addImm(+0)
// Note: see comment in spillCalleeSavedRegisters()
unsigned LdrOpc;
unsigned Size = TRI->getSpillSize(*RPI.RC);
Align Alignment = TRI->getSpillAlign(*RPI.RC);
switch (RPI.Type) {
case RegPairInfo::GPR:
LdrOpc = RPI.isPaired() ? AArch64::LDPXi : AArch64::LDRXui;
break;
case RegPairInfo::FPR64:
LdrOpc = RPI.isPaired() ? AArch64::LDPDi : AArch64::LDRDui;
break;
case RegPairInfo::FPR128:
LdrOpc = RPI.isPaired() ? AArch64::LDPQi : AArch64::LDRQui;
break;
case RegPairInfo::ZPR:
LdrOpc = RPI.isPaired() ? AArch64::LD1B_2Z_IMM : AArch64::LDR_ZXI;
break;
case RegPairInfo::PPR:
LdrOpc = Size == 16 ? AArch64::FILL_PPR_FROM_ZPR_SLOT_PSEUDO
: AArch64::LDR_PXI;
break;
case RegPairInfo::VG:
continue;
}
LLVM_DEBUG(dbgs() << "CSR restore: (" << printReg(Reg1, TRI);
if (RPI.isPaired()) dbgs() << ", " << printReg(Reg2, TRI);
dbgs() << ") -> fi#(" << RPI.FrameIdx;
if (RPI.isPaired()) dbgs() << ", " << RPI.FrameIdx + 1;
dbgs() << ")\n");
// Windows unwind codes require consecutive registers if registers are
// paired. Make the switch here, so that the code below will save (x,x+1)
// and not (x+1,x).
unsigned FrameIdxReg1 = RPI.FrameIdx;
unsigned FrameIdxReg2 = RPI.FrameIdx + 1;
if (NeedsWinCFI && RPI.isPaired()) {
std::swap(Reg1, Reg2);
std::swap(FrameIdxReg1, FrameIdxReg2);
}
AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
if (RPI.isPaired() && RPI.isScalable()) {
[[maybe_unused]] const AArch64Subtarget &Subtarget =
MF.getSubtarget<AArch64Subtarget>();
unsigned PnReg = AFI->getPredicateRegForFillSpill();
assert((PnReg != 0 && enableMultiVectorSpillFill(Subtarget, MF)) &&
"Expects SVE2.1 or SME2 target and a predicate register");
#ifdef EXPENSIVE_CHECKS
assert(!(PPRBegin < ZPRBegin) &&
"Expected callee save predicate to be handled first");
#endif
if (!PTrueCreated) {
PTrueCreated = true;
BuildMI(MBB, MBBI, DL, TII.get(AArch64::PTRUE_C_B), PnReg)
.setMIFlags(MachineInstr::FrameDestroy);
}
MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII.get(LdrOpc));
MIB.addReg(/*PairRegs*/ AArch64::Z0_Z1 + (RPI.Reg1 - AArch64::Z0),
getDefRegState(true));
MIB.addMemOperand(MF.getMachineMemOperand(
MachinePointerInfo::getFixedStack(MF, FrameIdxReg2),
MachineMemOperand::MOLoad, Size, Alignment));
MIB.addReg(PnReg);
MIB.addReg(AArch64::SP)
.addImm(RPI.Offset / 2) // [sp, #imm*2*vscale]
// where 2*vscale is implicit
.setMIFlag(MachineInstr::FrameDestroy);
MIB.addMemOperand(MF.getMachineMemOperand(
MachinePointerInfo::getFixedStack(MF, FrameIdxReg1),
MachineMemOperand::MOLoad, Size, Alignment));
if (NeedsWinCFI)
InsertSEH(MIB, TII, MachineInstr::FrameDestroy);
} else {
MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII.get(LdrOpc));
if (RPI.isPaired()) {
MIB.addReg(Reg2, getDefRegState(true));
MIB.addMemOperand(MF.getMachineMemOperand(
MachinePointerInfo::getFixedStack(MF, FrameIdxReg2),
MachineMemOperand::MOLoad, Size, Alignment));
}
MIB.addReg(Reg1, getDefRegState(true));
MIB.addReg(AArch64::SP)
.addImm(RPI.Offset) // [sp, #offset*vscale]
// where factor*vscale is implicit
.setMIFlag(MachineInstr::FrameDestroy);
MIB.addMemOperand(MF.getMachineMemOperand(
MachinePointerInfo::getFixedStack(MF, FrameIdxReg1),
MachineMemOperand::MOLoad, Size, Alignment));
if (NeedsWinCFI)
InsertSEH(MIB, TII, MachineInstr::FrameDestroy);
}
}
return true;
}
// Return the FrameID for a MMO.
static std::optional<int> getMMOFrameID(MachineMemOperand *MMO,
const MachineFrameInfo &MFI) {
auto *PSV =
dyn_cast_or_null<FixedStackPseudoSourceValue>(MMO->getPseudoValue());
if (PSV)
return std::optional<int>(PSV->getFrameIndex());
if (MMO->getValue()) {
if (auto *Al = dyn_cast<AllocaInst>(getUnderlyingObject(MMO->getValue()))) {
for (int FI = MFI.getObjectIndexBegin(); FI < MFI.getObjectIndexEnd();
FI++)
if (MFI.getObjectAllocation(FI) == Al)
return FI;
}
}
return std::nullopt;
}
// Return the FrameID for a Load/Store instruction by looking at the first MMO.
static std::optional<int> getLdStFrameID(const MachineInstr &MI,
const MachineFrameInfo &MFI) {
if (!MI.mayLoadOrStore() || MI.getNumMemOperands() < 1)
return std::nullopt;
return getMMOFrameID(*MI.memoperands_begin(), MFI);
}
// Check if a Hazard slot is needed for the current function, and if so create
// one for it. The index is stored in AArch64FunctionInfo->StackHazardSlotIndex,
// which can be used to determine if any hazard padding is needed.
void AArch64FrameLowering::determineStackHazardSlot(
MachineFunction &MF, BitVector &SavedRegs) const {
unsigned StackHazardSize = getStackHazardSize(MF);
auto *AFI = MF.getInfo<AArch64FunctionInfo>();
if (StackHazardSize == 0 || StackHazardSize % 16 != 0 ||
AFI->hasStackHazardSlotIndex())
return;
// Stack hazards are only needed in streaming functions.
SMEAttrs Attrs = AFI->getSMEFnAttrs();
if (!StackHazardInNonStreaming && Attrs.hasNonStreamingInterfaceAndBody())
return;
MachineFrameInfo &MFI = MF.getFrameInfo();
// Add a hazard slot if there are any CSR FPR registers, or are any fp-only
// stack objects.
bool HasFPRCSRs = any_of(SavedRegs.set_bits(), [](unsigned Reg) {
return AArch64::FPR64RegClass.contains(Reg) ||
AArch64::FPR128RegClass.contains(Reg) ||
AArch64::ZPRRegClass.contains(Reg) ||
AArch64::PPRRegClass.contains(Reg);
});
bool HasFPRStackObjects = false;
if (!HasFPRCSRs) {
std::vector<unsigned> FrameObjects(MFI.getObjectIndexEnd());
for (auto &MBB : MF) {
for (auto &MI : MBB) {
std::optional<int> FI = getLdStFrameID(MI, MFI);
if (FI && *FI >= 0 && *FI < (int)FrameObjects.size()) {
if (MFI.getStackID(*FI) == TargetStackID::ScalableVector ||
AArch64InstrInfo::isFpOrNEON(MI))
FrameObjects[*FI] |= 2;
else
FrameObjects[*FI] |= 1;
}
}
}
HasFPRStackObjects =
any_of(FrameObjects, [](unsigned B) { return (B & 3) == 2; });
}
if (HasFPRCSRs || HasFPRStackObjects) {
int ID = MFI.CreateStackObject(StackHazardSize, Align(16), false);
LLVM_DEBUG(dbgs() << "Created Hazard slot at " << ID << " size "
<< StackHazardSize << "\n");
AFI->setStackHazardSlotIndex(ID);
}
}
void AArch64FrameLowering::determineCalleeSaves(MachineFunction &MF,
BitVector &SavedRegs,
RegScavenger *RS) const {
// All calls are tail calls in GHC calling conv, and functions have no
// prologue/epilogue.
if (MF.getFunction().getCallingConv() == CallingConv::GHC)
return;
TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
const AArch64RegisterInfo *RegInfo = static_cast<const AArch64RegisterInfo *>(
MF.getSubtarget().getRegisterInfo());
const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
unsigned UnspilledCSGPR = AArch64::NoRegister;
unsigned UnspilledCSGPRPaired = AArch64::NoRegister;
MachineFrameInfo &MFI = MF.getFrameInfo();
const MCPhysReg *CSRegs = MF.getRegInfo().getCalleeSavedRegs();
unsigned BasePointerReg = RegInfo->hasBasePointer(MF)
? RegInfo->getBaseRegister()
: (unsigned)AArch64::NoRegister;
unsigned ExtraCSSpill = 0;
bool HasUnpairedGPR64 = false;
bool HasPairZReg = false;
BitVector UserReservedRegs = RegInfo->getUserReservedRegs(MF);
BitVector ReservedRegs = RegInfo->getReservedRegs(MF);
// Figure out which callee-saved registers to save/restore.
for (unsigned i = 0; CSRegs[i]; ++i) {
const unsigned Reg = CSRegs[i];
// Add the base pointer register to SavedRegs if it is callee-save.
if (Reg == BasePointerReg)
SavedRegs.set(Reg);
// Don't save manually reserved registers set through +reserve-x#i,
// even for callee-saved registers, as per GCC's behavior.
if (UserReservedRegs[Reg]) {
SavedRegs.reset(Reg);
continue;
}
bool RegUsed = SavedRegs.test(Reg);
unsigned PairedReg = AArch64::NoRegister;
const bool RegIsGPR64 = AArch64::GPR64RegClass.contains(Reg);
if (RegIsGPR64 || AArch64::FPR64RegClass.contains(Reg) ||
AArch64::FPR128RegClass.contains(Reg)) {
// Compensate for odd numbers of GP CSRs.
// For now, all the known cases of odd number of CSRs are of GPRs.
if (HasUnpairedGPR64)
PairedReg = CSRegs[i % 2 == 0 ? i - 1 : i + 1];
else
PairedReg = CSRegs[i ^ 1];
}
// If the function requires all the GP registers to save (SavedRegs),
// and there are an odd number of GP CSRs at the same time (CSRegs),
// PairedReg could be in a different register class from Reg, which would
// lead to a FPR (usually D8) accidentally being marked saved.
if (RegIsGPR64 && !AArch64::GPR64RegClass.contains(PairedReg)) {
PairedReg = AArch64::NoRegister;
HasUnpairedGPR64 = true;
}
assert(PairedReg == AArch64::NoRegister ||
AArch64::GPR64RegClass.contains(Reg, PairedReg) ||
AArch64::FPR64RegClass.contains(Reg, PairedReg) ||
AArch64::FPR128RegClass.contains(Reg, PairedReg));
if (!RegUsed) {
if (AArch64::GPR64RegClass.contains(Reg) && !ReservedRegs[Reg]) {
UnspilledCSGPR = Reg;
UnspilledCSGPRPaired = PairedReg;
}
continue;
}
// Always save P4 when PPR spills are ZPR-sized and a predicate above p8 is
// spilled. If all of p0-p3 are used as return values p4 is must be free
// to reload p8-p15.
if (RegInfo->getSpillSize(AArch64::PPRRegClass) == 16 &&
AArch64::PPR_p8to15RegClass.contains(Reg)) {
SavedRegs.set(AArch64::P4);
}
// MachO's compact unwind format relies on all registers being stored in
// pairs.
// FIXME: the usual format is actually better if unwinding isn't needed.
if (producePairRegisters(MF) && PairedReg != AArch64::NoRegister &&
!SavedRegs.test(PairedReg)) {
SavedRegs.set(PairedReg);
if (AArch64::GPR64RegClass.contains(PairedReg) &&
!ReservedRegs[PairedReg])
ExtraCSSpill = PairedReg;
}
// Check if there is a pair of ZRegs, so it can select PReg for spill/fill
HasPairZReg |= (AArch64::ZPRRegClass.contains(Reg, CSRegs[i ^ 1]) &&
SavedRegs.test(CSRegs[i ^ 1]));
}
if (HasPairZReg && enableMultiVectorSpillFill(Subtarget, MF)) {
AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
// Find a suitable predicate register for the multi-vector spill/fill
// instructions.
unsigned PnReg = findFreePredicateReg(SavedRegs);
if (PnReg != AArch64::NoRegister)
AFI->setPredicateRegForFillSpill(PnReg);
// If no free callee-save has been found assign one.
if (!AFI->getPredicateRegForFillSpill() &&
MF.getFunction().getCallingConv() ==
CallingConv::AArch64_SVE_VectorCall) {
SavedRegs.set(AArch64::P8);
AFI->setPredicateRegForFillSpill(AArch64::PN8);
}
assert(!ReservedRegs[AFI->getPredicateRegForFillSpill()] &&
"Predicate cannot be a reserved register");
}
if (MF.getFunction().getCallingConv() == CallingConv::Win64 &&
!Subtarget.isTargetWindows()) {
// For Windows calling convention on a non-windows OS, where X18 is treated
// as reserved, back up X18 when entering non-windows code (marked with the
// Windows calling convention) and restore when returning regardless of
// whether the individual function uses it - it might call other functions
// that clobber it.
SavedRegs.set(AArch64::X18);
}
// Calculates the callee saved stack size.
unsigned CSStackSize = 0;
unsigned SVECSStackSize = 0;
const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
for (unsigned Reg : SavedRegs.set_bits()) {
auto *RC = TRI->getMinimalPhysRegClass(Reg);
assert(RC && "expected register class!");
auto SpillSize = TRI->getSpillSize(*RC);
if (AArch64::PPRRegClass.contains(Reg) ||
AArch64::ZPRRegClass.contains(Reg))
SVECSStackSize += SpillSize;
else
CSStackSize += SpillSize;
}
// Save number of saved regs, so we can easily update CSStackSize later to
// account for any additional 64-bit GPR saves. Note: After this point
// only 64-bit GPRs can be added to SavedRegs.
unsigned NumSavedRegs = SavedRegs.count();
// Increase the callee-saved stack size if the function has streaming mode
// changes, as we will need to spill the value of the VG register.
// For locally streaming functions, we spill both the streaming and
// non-streaming VG value.
SMEAttrs Attrs = AFI->getSMEFnAttrs();
if (requiresSaveVG(MF)) {
if (Attrs.hasStreamingBody() && !Attrs.hasStreamingInterface())
CSStackSize += 16;
else
CSStackSize += 8;
}
// Determine if a Hazard slot should be used, and increase the CSStackSize by
// StackHazardSize if so.
determineStackHazardSlot(MF, SavedRegs);
if (AFI->hasStackHazardSlotIndex())
CSStackSize += getStackHazardSize(MF);
// If we must call __arm_get_current_vg in the prologue preserve the LR.
if (requiresSaveVG(MF) && !Subtarget.hasSVE())
SavedRegs.set(AArch64::LR);
// The frame record needs to be created by saving the appropriate registers
uint64_t EstimatedStackSize = MFI.estimateStackSize(MF);
if (hasFP(MF) ||
windowsRequiresStackProbe(MF, EstimatedStackSize + CSStackSize + 16)) {
SavedRegs.set(AArch64::FP);
SavedRegs.set(AArch64::LR);
}
LLVM_DEBUG({
dbgs() << "*** determineCalleeSaves\nSaved CSRs:";
for (unsigned Reg : SavedRegs.set_bits())
dbgs() << ' ' << printReg(Reg, RegInfo);
dbgs() << "\n";
});
// If any callee-saved registers are used, the frame cannot be eliminated.
int64_t SVEStackSize =
alignTo(SVECSStackSize + estimateSVEStackObjectOffsets(MFI), 16);
bool CanEliminateFrame = (SavedRegs.count() == 0) && !SVEStackSize;
// The CSR spill slots have not been allocated yet, so estimateStackSize
// won't include them.
unsigned EstimatedStackSizeLimit = estimateRSStackSizeLimit(MF);
// We may address some of the stack above the canonical frame address, either
// for our own arguments or during a call. Include that in calculating whether
// we have complicated addressing concerns.
int64_t CalleeStackUsed = 0;
for (int I = MFI.getObjectIndexBegin(); I != 0; ++I) {
int64_t FixedOff = MFI.getObjectOffset(I);
if (FixedOff > CalleeStackUsed)
CalleeStackUsed = FixedOff;
}
// Conservatively always assume BigStack when there are SVE spills.
bool BigStack = SVEStackSize || (EstimatedStackSize + CSStackSize +
CalleeStackUsed) > EstimatedStackSizeLimit;
if (BigStack || !CanEliminateFrame || RegInfo->cannotEliminateFrame(MF))
AFI->setHasStackFrame(true);
// Estimate if we might need to scavenge a register at some point in order
// to materialize a stack offset. If so, either spill one additional
// callee-saved register or reserve a special spill slot to facilitate
// register scavenging. If we already spilled an extra callee-saved register
// above to keep the number of spills even, we don't need to do anything else
// here.
if (BigStack) {
if (!ExtraCSSpill && UnspilledCSGPR != AArch64::NoRegister) {
LLVM_DEBUG(dbgs() << "Spilling " << printReg(UnspilledCSGPR, RegInfo)
<< " to get a scratch register.\n");
SavedRegs.set(UnspilledCSGPR);
ExtraCSSpill = UnspilledCSGPR;
// MachO's compact unwind format relies on all registers being stored in
// pairs, so if we need to spill one extra for BigStack, then we need to
// store the pair.
if (producePairRegisters(MF)) {
if (UnspilledCSGPRPaired == AArch64::NoRegister) {
// Failed to make a pair for compact unwind format, revert spilling.
if (produceCompactUnwindFrame(MF)) {
SavedRegs.reset(UnspilledCSGPR);
ExtraCSSpill = AArch64::NoRegister;
}
} else
SavedRegs.set(UnspilledCSGPRPaired);
}
}
// If we didn't find an extra callee-saved register to spill, create
// an emergency spill slot.
if (!ExtraCSSpill || MF.getRegInfo().isPhysRegUsed(ExtraCSSpill)) {
const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
const TargetRegisterClass &RC = AArch64::GPR64RegClass;
unsigned Size = TRI->getSpillSize(RC);
Align Alignment = TRI->getSpillAlign(RC);
int FI = MFI.CreateSpillStackObject(Size, Alignment);
RS->addScavengingFrameIndex(FI);
LLVM_DEBUG(dbgs() << "No available CS registers, allocated fi#" << FI
<< " as the emergency spill slot.\n");
}
}
// Adding the size of additional 64bit GPR saves.
CSStackSize += 8 * (SavedRegs.count() - NumSavedRegs);
// A Swift asynchronous context extends the frame record with a pointer
// directly before FP.
if (hasFP(MF) && AFI->hasSwiftAsyncContext())
CSStackSize += 8;
uint64_t AlignedCSStackSize = alignTo(CSStackSize, 16);
LLVM_DEBUG(dbgs() << "Estimated stack frame size: "
<< EstimatedStackSize + AlignedCSStackSize << " bytes.\n");
assert((!MFI.isCalleeSavedInfoValid() ||
AFI->getCalleeSavedStackSize() == AlignedCSStackSize) &&
"Should not invalidate callee saved info");
// Round up to register pair alignment to avoid additional SP adjustment
// instructions.
AFI->setCalleeSavedStackSize(AlignedCSStackSize);
AFI->setCalleeSaveStackHasFreeSpace(AlignedCSStackSize != CSStackSize);
AFI->setSVECalleeSavedStackSize(alignTo(SVECSStackSize, 16));
}
bool AArch64FrameLowering::assignCalleeSavedSpillSlots(
MachineFunction &MF, const TargetRegisterInfo *RegInfo,
std::vector<CalleeSavedInfo> &CSI, unsigned &MinCSFrameIndex,
unsigned &MaxCSFrameIndex) const {
bool NeedsWinCFI = needsWinCFI(MF);
unsigned StackHazardSize = getStackHazardSize(MF);
// To match the canonical windows frame layout, reverse the list of
// callee saved registers to get them laid out by PrologEpilogInserter
// in the right order. (PrologEpilogInserter allocates stack objects top
// down. Windows canonical prologs store higher numbered registers at
// the top, thus have the CSI array start from the highest registers.)
if (NeedsWinCFI)
std::reverse(CSI.begin(), CSI.end());
if (CSI.empty())
return true; // Early exit if no callee saved registers are modified!
// Now that we know which registers need to be saved and restored, allocate
// stack slots for them.
MachineFrameInfo &MFI = MF.getFrameInfo();
auto *AFI = MF.getInfo<AArch64FunctionInfo>();
bool UsesWinAAPCS = isTargetWindows(MF);
if (UsesWinAAPCS && hasFP(MF) && AFI->hasSwiftAsyncContext()) {
int FrameIdx = MFI.CreateStackObject(8, Align(16), true);
AFI->setSwiftAsyncContextFrameIdx(FrameIdx);
if ((unsigned)FrameIdx < MinCSFrameIndex)
MinCSFrameIndex = FrameIdx;
if ((unsigned)FrameIdx > MaxCSFrameIndex)
MaxCSFrameIndex = FrameIdx;
}
// Insert VG into the list of CSRs, immediately before LR if saved.
if (requiresSaveVG(MF)) {
std::vector<CalleeSavedInfo> VGSaves;
SMEAttrs Attrs = AFI->getSMEFnAttrs();
auto VGInfo = CalleeSavedInfo(AArch64::VG);
VGInfo.setRestored(false);
VGSaves.push_back(VGInfo);
// Add VG again if the function is locally-streaming, as we will spill two
// values.
if (Attrs.hasStreamingBody() && !Attrs.hasStreamingInterface())
VGSaves.push_back(VGInfo);
bool InsertBeforeLR = false;
for (unsigned I = 0; I < CSI.size(); I++)
if (CSI[I].getReg() == AArch64::LR) {
InsertBeforeLR = true;
CSI.insert(CSI.begin() + I, VGSaves.begin(), VGSaves.end());
break;
}
if (!InsertBeforeLR)
llvm::append_range(CSI, VGSaves);
}
Register LastReg = 0;
int HazardSlotIndex = std::numeric_limits<int>::max();
for (auto &CS : CSI) {
MCRegister Reg = CS.getReg();
const TargetRegisterClass *RC = RegInfo->getMinimalPhysRegClass(Reg);
// Create a hazard slot as we switch between GPR and FPR CSRs.
if (AFI->hasStackHazardSlotIndex() &&
(!LastReg || !AArch64InstrInfo::isFpOrNEON(LastReg)) &&
AArch64InstrInfo::isFpOrNEON(Reg)) {
assert(HazardSlotIndex == std::numeric_limits<int>::max() &&
"Unexpected register order for hazard slot");
HazardSlotIndex = MFI.CreateStackObject(StackHazardSize, Align(8), true);
LLVM_DEBUG(dbgs() << "Created CSR Hazard at slot " << HazardSlotIndex
<< "\n");
AFI->setStackHazardCSRSlotIndex(HazardSlotIndex);
if ((unsigned)HazardSlotIndex < MinCSFrameIndex)
MinCSFrameIndex = HazardSlotIndex;
if ((unsigned)HazardSlotIndex > MaxCSFrameIndex)
MaxCSFrameIndex = HazardSlotIndex;
}
unsigned Size = RegInfo->getSpillSize(*RC);
Align Alignment(RegInfo->getSpillAlign(*RC));
int FrameIdx = MFI.CreateStackObject(Size, Alignment, true);
CS.setFrameIdx(FrameIdx);
if ((unsigned)FrameIdx < MinCSFrameIndex)
MinCSFrameIndex = FrameIdx;
if ((unsigned)FrameIdx > MaxCSFrameIndex)
MaxCSFrameIndex = FrameIdx;
// Grab 8 bytes below FP for the extended asynchronous frame info.
if (hasFP(MF) && AFI->hasSwiftAsyncContext() && !UsesWinAAPCS &&
Reg == AArch64::FP) {
FrameIdx = MFI.CreateStackObject(8, Alignment, true);
AFI->setSwiftAsyncContextFrameIdx(FrameIdx);
if ((unsigned)FrameIdx < MinCSFrameIndex)
MinCSFrameIndex = FrameIdx;
if ((unsigned)FrameIdx > MaxCSFrameIndex)
MaxCSFrameIndex = FrameIdx;
}
LastReg = Reg;
}
// Add hazard slot in the case where no FPR CSRs are present.
if (AFI->hasStackHazardSlotIndex() &&
HazardSlotIndex == std::numeric_limits<int>::max()) {
HazardSlotIndex = MFI.CreateStackObject(StackHazardSize, Align(8), true);
LLVM_DEBUG(dbgs() << "Created CSR Hazard at slot " << HazardSlotIndex
<< "\n");
AFI->setStackHazardCSRSlotIndex(HazardSlotIndex);
if ((unsigned)HazardSlotIndex < MinCSFrameIndex)
MinCSFrameIndex = HazardSlotIndex;
if ((unsigned)HazardSlotIndex > MaxCSFrameIndex)
MaxCSFrameIndex = HazardSlotIndex;
}
return true;
}
bool AArch64FrameLowering::enableStackSlotScavenging(
const MachineFunction &MF) const {
const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
// If the function has streaming-mode changes, don't scavenge a
// spillslot in the callee-save area, as that might require an
// 'addvl' in the streaming-mode-changing call-sequence when the
// function doesn't use a FP.
if (AFI->hasStreamingModeChanges() && !hasFP(MF))
return false;
// Don't allow register salvaging with hazard slots, in case it moves objects
// into the wrong place.
if (AFI->hasStackHazardSlotIndex())
return false;
return AFI->hasCalleeSaveStackFreeSpace();
}
/// returns true if there are any SVE callee saves.
static bool getSVECalleeSaveSlotRange(const MachineFrameInfo &MFI,
int &Min, int &Max) {
Min = std::numeric_limits<int>::max();
Max = std::numeric_limits<int>::min();
if (!MFI.isCalleeSavedInfoValid())
return false;
const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
for (auto &CS : CSI) {
if (AArch64::ZPRRegClass.contains(CS.getReg()) ||
AArch64::PPRRegClass.contains(CS.getReg())) {
assert((Max == std::numeric_limits<int>::min() ||
Max + 1 == CS.getFrameIdx()) &&
"SVE CalleeSaves are not consecutive");
Min = std::min(Min, CS.getFrameIdx());
Max = std::max(Max, CS.getFrameIdx());
}
}
return Min != std::numeric_limits<int>::max();
}
// Process all the SVE stack objects and determine offsets for each
// object. If AssignOffsets is true, the offsets get assigned.
// Fills in the first and last callee-saved frame indices into
// Min/MaxCSFrameIndex, respectively.
// Returns the size of the stack.
static int64_t determineSVEStackObjectOffsets(MachineFrameInfo &MFI,
int &MinCSFrameIndex,
int &MaxCSFrameIndex,
bool AssignOffsets) {
#ifndef NDEBUG
// First process all fixed stack objects.
for (int I = MFI.getObjectIndexBegin(); I != 0; ++I)
assert(MFI.getStackID(I) != TargetStackID::ScalableVector &&
"SVE vectors should never be passed on the stack by value, only by "
"reference.");
#endif
auto Assign = [&MFI](int FI, int64_t Offset) {
LLVM_DEBUG(dbgs() << "alloc FI(" << FI << ") at SP[" << Offset << "]\n");
MFI.setObjectOffset(FI, Offset);
};
int64_t Offset = 0;
// Then process all callee saved slots.
if (getSVECalleeSaveSlotRange(MFI, MinCSFrameIndex, MaxCSFrameIndex)) {
// Assign offsets to the callee save slots.
for (int I = MinCSFrameIndex; I <= MaxCSFrameIndex; ++I) {
Offset += MFI.getObjectSize(I);
Offset = alignTo(Offset, MFI.getObjectAlign(I));
if (AssignOffsets)
Assign(I, -Offset);
}
}
// Ensure that the Callee-save area is aligned to 16bytes.
Offset = alignTo(Offset, Align(16U));
// Create a buffer of SVE objects to allocate and sort it.
SmallVector<int, 8> ObjectsToAllocate;
// If we have a stack protector, and we've previously decided that we have SVE
// objects on the stack and thus need it to go in the SVE stack area, then it
// needs to go first.
int StackProtectorFI = -1;
if (MFI.hasStackProtectorIndex()) {
StackProtectorFI = MFI.getStackProtectorIndex();
if (MFI.getStackID(StackProtectorFI) == TargetStackID::ScalableVector)
ObjectsToAllocate.push_back(StackProtectorFI);
}
for (int I = 0, E = MFI.getObjectIndexEnd(); I != E; ++I) {
unsigned StackID = MFI.getStackID(I);
if (StackID != TargetStackID::ScalableVector)
continue;
if (I == StackProtectorFI)
continue;
if (MaxCSFrameIndex >= I && I >= MinCSFrameIndex)
continue;
if (MFI.isDeadObjectIndex(I))
continue;
ObjectsToAllocate.push_back(I);
}
// Allocate all SVE locals and spills
for (unsigned FI : ObjectsToAllocate) {
Align Alignment = MFI.getObjectAlign(FI);
// FIXME: Given that the length of SVE vectors is not necessarily a power of
// two, we'd need to align every object dynamically at runtime if the
// alignment is larger than 16. This is not yet supported.
if (Alignment > Align(16))
report_fatal_error(
"Alignment of scalable vectors > 16 bytes is not yet supported");
Offset = alignTo(Offset + MFI.getObjectSize(FI), Alignment);
if (AssignOffsets)
Assign(FI, -Offset);
}
return Offset;
}
int64_t AArch64FrameLowering::estimateSVEStackObjectOffsets(
MachineFrameInfo &MFI) const {
int MinCSFrameIndex, MaxCSFrameIndex;
return determineSVEStackObjectOffsets(MFI, MinCSFrameIndex, MaxCSFrameIndex, false);
}
int64_t AArch64FrameLowering::assignSVEStackObjectOffsets(
MachineFrameInfo &MFI, int &MinCSFrameIndex, int &MaxCSFrameIndex) const {
return determineSVEStackObjectOffsets(MFI, MinCSFrameIndex, MaxCSFrameIndex,
true);
}
/// Attempts to scavenge a register from \p ScavengeableRegs given the used
/// registers in \p UsedRegs.
static Register tryScavengeRegister(LiveRegUnits const &UsedRegs,
BitVector const &ScavengeableRegs,
Register PreferredReg) {
if (PreferredReg != AArch64::NoRegister && UsedRegs.available(PreferredReg))
return PreferredReg;
for (auto Reg : ScavengeableRegs.set_bits()) {
if (UsedRegs.available(Reg))
return Reg;
}
return AArch64::NoRegister;
}
/// Propagates frame-setup/destroy flags from \p SourceMI to all instructions in
/// \p MachineInstrs.
static void propagateFrameFlags(MachineInstr &SourceMI,
ArrayRef<MachineInstr *> MachineInstrs) {
for (MachineInstr *MI : MachineInstrs) {
if (SourceMI.getFlag(MachineInstr::FrameSetup))
MI->setFlag(MachineInstr::FrameSetup);
if (SourceMI.getFlag(MachineInstr::FrameDestroy))
MI->setFlag(MachineInstr::FrameDestroy);
}
}
/// RAII helper class for scavenging or spilling a register. On construction
/// attempts to find a free register of class \p RC (given \p UsedRegs and \p
/// AllocatableRegs), if no register can be found spills \p SpillCandidate to \p
/// MaybeSpillFI to free a register. The free'd register is returned via the \p
/// FreeReg output parameter. On destruction, if there is a spill, its previous
/// value is reloaded. The spilling and scavenging is only valid at the
/// insertion point \p MBBI, this class should _not_ be used in places that
/// create or manipulate basic blocks, moving the expected insertion point.
struct ScopedScavengeOrSpill {
ScopedScavengeOrSpill(const ScopedScavengeOrSpill &) = delete;
ScopedScavengeOrSpill(ScopedScavengeOrSpill &&) = delete;
ScopedScavengeOrSpill(MachineFunction &MF, MachineBasicBlock &MBB,
MachineBasicBlock::iterator MBBI,
Register SpillCandidate, const TargetRegisterClass &RC,
LiveRegUnits const &UsedRegs,
BitVector const &AllocatableRegs,
std::optional<int> *MaybeSpillFI,
Register PreferredReg = AArch64::NoRegister)
: MBB(MBB), MBBI(MBBI), RC(RC), TII(static_cast<const AArch64InstrInfo &>(
*MF.getSubtarget().getInstrInfo())),
TRI(*MF.getSubtarget().getRegisterInfo()) {
FreeReg = tryScavengeRegister(UsedRegs, AllocatableRegs, PreferredReg);
if (FreeReg != AArch64::NoRegister)
return;
assert(MaybeSpillFI && "Expected emergency spill slot FI information "
"(attempted to spill in prologue/epilogue?)");
if (!MaybeSpillFI->has_value()) {
MachineFrameInfo &MFI = MF.getFrameInfo();
*MaybeSpillFI = MFI.CreateSpillStackObject(TRI.getSpillSize(RC),
TRI.getSpillAlign(RC));
}
FreeReg = SpillCandidate;
SpillFI = MaybeSpillFI->value();
TII.storeRegToStackSlot(MBB, MBBI, FreeReg, false, *SpillFI, &RC, &TRI,
Register());
}
bool hasSpilled() const { return SpillFI.has_value(); }
/// Returns the free register (found from scavenging or spilling a register).
Register freeRegister() const { return FreeReg; }
Register operator*() const { return freeRegister(); }
~ScopedScavengeOrSpill() {
if (hasSpilled())
TII.loadRegFromStackSlot(MBB, MBBI, FreeReg, *SpillFI, &RC, &TRI,
Register());
}
private:
MachineBasicBlock &MBB;
MachineBasicBlock::iterator MBBI;
const TargetRegisterClass &RC;
const AArch64InstrInfo &TII;
const TargetRegisterInfo &TRI;
Register FreeReg = AArch64::NoRegister;
std::optional<int> SpillFI;
};
/// Emergency stack slots for expanding SPILL_PPR_TO_ZPR_SLOT_PSEUDO and
/// FILL_PPR_FROM_ZPR_SLOT_PSEUDO.
struct EmergencyStackSlots {
std::optional<int> ZPRSpillFI;
std::optional<int> PPRSpillFI;
std::optional<int> GPRSpillFI;
};
/// Registers available for scavenging (ZPR, PPR3b, GPR).
struct ScavengeableRegs {
BitVector ZPRRegs;
BitVector PPR3bRegs;
BitVector GPRRegs;
};
static bool isInPrologueOrEpilogue(const MachineInstr &MI) {
return MI.getFlag(MachineInstr::FrameSetup) ||
MI.getFlag(MachineInstr::FrameDestroy);
}
/// Expands:
/// ```
/// SPILL_PPR_TO_ZPR_SLOT_PSEUDO $p0, %stack.0, 0
/// ```
/// To:
/// ```
/// $z0 = CPY_ZPzI_B $p0, 1, 0
/// STR_ZXI $z0, $stack.0, 0
/// ```
/// While ensuring a ZPR ($z0 in this example) is free for the predicate (
/// spilling if necessary).
static void expandSpillPPRToZPRSlotPseudo(MachineBasicBlock &MBB,
MachineInstr &MI,
const TargetRegisterInfo &TRI,
LiveRegUnits const &UsedRegs,
ScavengeableRegs const &SR,
EmergencyStackSlots &SpillSlots) {
MachineFunction &MF = *MBB.getParent();
auto *TII =
static_cast<const AArch64InstrInfo *>(MF.getSubtarget().getInstrInfo());
ScopedScavengeOrSpill ZPredReg(
MF, MBB, MI, AArch64::Z0, AArch64::ZPRRegClass, UsedRegs, SR.ZPRRegs,
isInPrologueOrEpilogue(MI) ? nullptr : &SpillSlots.ZPRSpillFI);
SmallVector<MachineInstr *, 2> MachineInstrs;
const DebugLoc &DL = MI.getDebugLoc();
MachineInstrs.push_back(BuildMI(MBB, MI, DL, TII->get(AArch64::CPY_ZPzI_B))
.addReg(*ZPredReg, RegState::Define)
.add(MI.getOperand(0))
.addImm(1)
.addImm(0)
.getInstr());
MachineInstrs.push_back(BuildMI(MBB, MI, DL, TII->get(AArch64::STR_ZXI))
.addReg(*ZPredReg)
.add(MI.getOperand(1))
.addImm(MI.getOperand(2).getImm())
.setMemRefs(MI.memoperands())
.getInstr());
propagateFrameFlags(MI, MachineInstrs);
}
/// Expands:
/// ```
/// $p0 = FILL_PPR_FROM_ZPR_SLOT_PSEUDO %stack.0, 0
/// ```
/// To:
/// ```
/// $z0 = LDR_ZXI %stack.0, 0
/// $p0 = PTRUE_B 31, implicit $vg
/// $p0 = CMPNE_PPzZI_B $p0, $z0, 0, implicit-def $nzcv, implicit-def $nzcv
/// ```
/// While ensuring a ZPR ($z0 in this example) is free for the predicate (
/// spilling if necessary). If the status flags are in use at the point of
/// expansion they are preserved (by moving them to/from a GPR). This may cause
/// an additional spill if no GPR is free at the expansion point.
static bool expandFillPPRFromZPRSlotPseudo(
MachineBasicBlock &MBB, MachineInstr &MI, const TargetRegisterInfo &TRI,
LiveRegUnits const &UsedRegs, ScavengeableRegs const &SR,
MachineInstr *&LastPTrue, EmergencyStackSlots &SpillSlots) {
MachineFunction &MF = *MBB.getParent();
auto *TII =
static_cast<const AArch64InstrInfo *>(MF.getSubtarget().getInstrInfo());
ScopedScavengeOrSpill ZPredReg(
MF, MBB, MI, AArch64::Z0, AArch64::ZPRRegClass, UsedRegs, SR.ZPRRegs,
isInPrologueOrEpilogue(MI) ? nullptr : &SpillSlots.ZPRSpillFI);
ScopedScavengeOrSpill PredReg(
MF, MBB, MI, AArch64::P0, AArch64::PPR_3bRegClass, UsedRegs, SR.PPR3bRegs,
isInPrologueOrEpilogue(MI) ? nullptr : &SpillSlots.PPRSpillFI,
/*PreferredReg=*/
LastPTrue ? LastPTrue->getOperand(0).getReg() : AArch64::NoRegister);
// Elide NZCV spills if we know it is not used.
bool IsNZCVUsed = !UsedRegs.available(AArch64::NZCV);
std::optional<ScopedScavengeOrSpill> NZCVSaveReg;
if (IsNZCVUsed)
NZCVSaveReg.emplace(
MF, MBB, MI, AArch64::X0, AArch64::GPR64RegClass, UsedRegs, SR.GPRRegs,
isInPrologueOrEpilogue(MI) ? nullptr : &SpillSlots.GPRSpillFI);
SmallVector<MachineInstr *, 4> MachineInstrs;
const DebugLoc &DL = MI.getDebugLoc();
MachineInstrs.push_back(BuildMI(MBB, MI, DL, TII->get(AArch64::LDR_ZXI))
.addReg(*ZPredReg, RegState::Define)
.add(MI.getOperand(1))
.addImm(MI.getOperand(2).getImm())
.setMemRefs(MI.memoperands())
.getInstr());
if (IsNZCVUsed)
MachineInstrs.push_back(
BuildMI(MBB, MI, DL, TII->get(AArch64::MRS))
.addReg(NZCVSaveReg->freeRegister(), RegState::Define)
.addImm(AArch64SysReg::NZCV)
.addReg(AArch64::NZCV, RegState::Implicit)
.getInstr());
// Reuse previous ptrue if we know it has not been clobbered.
if (LastPTrue) {
assert(*PredReg == LastPTrue->getOperand(0).getReg());
LastPTrue->moveBefore(&MI);
} else {
LastPTrue = BuildMI(MBB, MI, DL, TII->get(AArch64::PTRUE_B))
.addReg(*PredReg, RegState::Define)
.addImm(31);
}
MachineInstrs.push_back(LastPTrue);
MachineInstrs.push_back(
BuildMI(MBB, MI, DL, TII->get(AArch64::CMPNE_PPzZI_B))
.addReg(MI.getOperand(0).getReg(), RegState::Define)
.addReg(*PredReg)
.addReg(*ZPredReg)
.addImm(0)
.addReg(AArch64::NZCV, RegState::ImplicitDefine)
.getInstr());
if (IsNZCVUsed)
MachineInstrs.push_back(BuildMI(MBB, MI, DL, TII->get(AArch64::MSR))
.addImm(AArch64SysReg::NZCV)
.addReg(NZCVSaveReg->freeRegister())
.addReg(AArch64::NZCV, RegState::ImplicitDefine)
.getInstr());
propagateFrameFlags(MI, MachineInstrs);
return PredReg.hasSpilled();
}
/// Expands all FILL_PPR_FROM_ZPR_SLOT_PSEUDO and SPILL_PPR_TO_ZPR_SLOT_PSEUDO
/// operations within the MachineBasicBlock \p MBB.
static bool expandSMEPPRToZPRSpillPseudos(MachineBasicBlock &MBB,
const TargetRegisterInfo &TRI,
ScavengeableRegs const &SR,
EmergencyStackSlots &SpillSlots) {
LiveRegUnits UsedRegs(TRI);
UsedRegs.addLiveOuts(MBB);
bool HasPPRSpills = false;
MachineInstr *LastPTrue = nullptr;
for (MachineInstr &MI : make_early_inc_range(reverse(MBB))) {
UsedRegs.stepBackward(MI);
switch (MI.getOpcode()) {
case AArch64::FILL_PPR_FROM_ZPR_SLOT_PSEUDO:
if (LastPTrue &&
MI.definesRegister(LastPTrue->getOperand(0).getReg(), &TRI))
LastPTrue = nullptr;
HasPPRSpills |= expandFillPPRFromZPRSlotPseudo(MBB, MI, TRI, UsedRegs, SR,
LastPTrue, SpillSlots);
MI.eraseFromParent();
break;
case AArch64::SPILL_PPR_TO_ZPR_SLOT_PSEUDO:
expandSpillPPRToZPRSlotPseudo(MBB, MI, TRI, UsedRegs, SR, SpillSlots);
MI.eraseFromParent();
[[fallthrough]];
default:
LastPTrue = nullptr;
break;
}
}
return HasPPRSpills;
}
void AArch64FrameLowering::processFunctionBeforeFrameFinalized(
MachineFunction &MF, RegScavenger *RS) const {
AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
const TargetSubtargetInfo &TSI = MF.getSubtarget();
const TargetRegisterInfo &TRI = *TSI.getRegisterInfo();
// If predicates spills are 16-bytes we may need to expand
// SPILL_PPR_TO_ZPR_SLOT_PSEUDO/FILL_PPR_FROM_ZPR_SLOT_PSEUDO.
if (AFI->hasStackFrame() && TRI.getSpillSize(AArch64::PPRRegClass) == 16) {
auto ComputeScavengeableRegisters = [&](unsigned RegClassID) {
BitVector Regs = TRI.getAllocatableSet(MF, TRI.getRegClass(RegClassID));
assert(Regs.count() > 0 && "Expected scavengeable registers");
return Regs;
};
ScavengeableRegs SR{};
SR.ZPRRegs = ComputeScavengeableRegisters(AArch64::ZPRRegClassID);
// Only p0-7 are possible as the second operand of cmpne (needed for fills).
SR.PPR3bRegs = ComputeScavengeableRegisters(AArch64::PPR_3bRegClassID);
SR.GPRRegs = ComputeScavengeableRegisters(AArch64::GPR64RegClassID);
EmergencyStackSlots SpillSlots;
for (MachineBasicBlock &MBB : MF) {
// In the case we had to spill a predicate (in the range p0-p7) to reload
// a predicate (>= p8), additional spill/fill pseudos will be created.
// These need an additional expansion pass. Note: There will only be at
// most two expansion passes, as spilling/filling a predicate in the range
// p0-p7 never requires spilling another predicate.
for (int Pass = 0; Pass < 2; Pass++) {
bool HasPPRSpills =
expandSMEPPRToZPRSpillPseudos(MBB, TRI, SR, SpillSlots);
assert((Pass == 0 || !HasPPRSpills) && "Did not expect PPR spills");
if (!HasPPRSpills)
break;
}
}
}
MachineFrameInfo &MFI = MF.getFrameInfo();
assert(getStackGrowthDirection() == TargetFrameLowering::StackGrowsDown &&
"Upwards growing stack unsupported");
int MinCSFrameIndex, MaxCSFrameIndex;
int64_t SVEStackSize =
assignSVEStackObjectOffsets(MFI, MinCSFrameIndex, MaxCSFrameIndex);
AFI->setStackSizeSVE(alignTo(SVEStackSize, 16U));
AFI->setMinMaxSVECSFrameIndex(MinCSFrameIndex, MaxCSFrameIndex);
// If this function isn't doing Win64-style C++ EH, we don't need to do
// anything.
if (!MF.hasEHFunclets())
return;
// Win64 C++ EH needs to allocate space for the catch objects in the fixed
// object area right next to the UnwindHelp object.
WinEHFuncInfo &EHInfo = *MF.getWinEHFuncInfo();
int64_t CurrentOffset =
AFI->getVarArgsGPRSize() + AFI->getTailCallReservedStack();
for (WinEHTryBlockMapEntry &TBME : EHInfo.TryBlockMap) {
for (WinEHHandlerType &H : TBME.HandlerArray) {
int FrameIndex = H.CatchObj.FrameIndex;
if ((FrameIndex != INT_MAX) && MFI.getObjectOffset(FrameIndex) == 0) {
CurrentOffset =
alignTo(CurrentOffset, MFI.getObjectAlign(FrameIndex).value());
CurrentOffset += MFI.getObjectSize(FrameIndex);
MFI.setObjectOffset(FrameIndex, -CurrentOffset);
}
}
}
// Create an UnwindHelp object.
// The UnwindHelp object is allocated at the start of the fixed object area
int64_t UnwindHelpOffset = alignTo(CurrentOffset + 8, Align(16));
assert(UnwindHelpOffset == getFixedObjectSize(MF, AFI, /*IsWin64*/ true,
/*IsFunclet*/ false) &&
"UnwindHelpOffset must be at the start of the fixed object area");
int UnwindHelpFI = MFI.CreateFixedObject(/*Size*/ 8, -UnwindHelpOffset,
/*IsImmutable=*/false);
EHInfo.UnwindHelpFrameIdx = UnwindHelpFI;
MachineBasicBlock &MBB = MF.front();
auto MBBI = MBB.begin();
while (MBBI != MBB.end() && MBBI->getFlag(MachineInstr::FrameSetup))
++MBBI;
// We need to store -2 into the UnwindHelp object at the start of the
// function.
DebugLoc DL;
RS->enterBasicBlockEnd(MBB);
RS->backward(MBBI);
Register DstReg = RS->FindUnusedReg(&AArch64::GPR64commonRegClass);
assert(DstReg && "There must be a free register after frame setup");
const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
BuildMI(MBB, MBBI, DL, TII.get(AArch64::MOVi64imm), DstReg).addImm(-2);
BuildMI(MBB, MBBI, DL, TII.get(AArch64::STURXi))
.addReg(DstReg, getKillRegState(true))
.addFrameIndex(UnwindHelpFI)
.addImm(0);
}
namespace {
struct TagStoreInstr {
MachineInstr *MI;
int64_t Offset, Size;
explicit TagStoreInstr(MachineInstr *MI, int64_t Offset, int64_t Size)
: MI(MI), Offset(Offset), Size(Size) {}
};
class TagStoreEdit {
MachineFunction *MF;
MachineBasicBlock *MBB;
MachineRegisterInfo *MRI;
// Tag store instructions that are being replaced.
SmallVector<TagStoreInstr, 8> TagStores;
// Combined memref arguments of the above instructions.
SmallVector<MachineMemOperand *, 8> CombinedMemRefs;
// Replace allocation tags in [FrameReg + FrameRegOffset, FrameReg +
// FrameRegOffset + Size) with the address tag of SP.
Register FrameReg;
StackOffset FrameRegOffset;
int64_t Size;
// If not std::nullopt, move FrameReg to (FrameReg + FrameRegUpdate) at the
// end.
std::optional<int64_t> FrameRegUpdate;
// MIFlags for any FrameReg updating instructions.
unsigned FrameRegUpdateFlags;
// Use zeroing instruction variants.
bool ZeroData;
DebugLoc DL;
void emitUnrolled(MachineBasicBlock::iterator InsertI);
void emitLoop(MachineBasicBlock::iterator InsertI);
public:
TagStoreEdit(MachineBasicBlock *MBB, bool ZeroData)
: MBB(MBB), ZeroData(ZeroData) {
MF = MBB->getParent();
MRI = &MF->getRegInfo();
}
// Add an instruction to be replaced. Instructions must be added in the
// ascending order of Offset, and have to be adjacent.
void addInstruction(TagStoreInstr I) {
assert((TagStores.empty() ||
TagStores.back().Offset + TagStores.back().Size == I.Offset) &&
"Non-adjacent tag store instructions.");
TagStores.push_back(I);
}
void clear() { TagStores.clear(); }
// Emit equivalent code at the given location, and erase the current set of
// instructions. May skip if the replacement is not profitable. May invalidate
// the input iterator and replace it with a valid one.
void emitCode(MachineBasicBlock::iterator &InsertI,
const AArch64FrameLowering *TFI, bool TryMergeSPUpdate);
};
void TagStoreEdit::emitUnrolled(MachineBasicBlock::iterator InsertI) {
const AArch64InstrInfo *TII =
MF->getSubtarget<AArch64Subtarget>().getInstrInfo();
const int64_t kMinOffset = -256 * 16;
const int64_t kMaxOffset = 255 * 16;
Register BaseReg = FrameReg;
int64_t BaseRegOffsetBytes = FrameRegOffset.getFixed();
if (BaseRegOffsetBytes < kMinOffset ||
BaseRegOffsetBytes + (Size - Size % 32) > kMaxOffset ||
// BaseReg can be FP, which is not necessarily aligned to 16-bytes. In
// that case, BaseRegOffsetBytes will not be aligned to 16 bytes, which
// is required for the offset of ST2G.
BaseRegOffsetBytes % 16 != 0) {
Register ScratchReg = MRI->createVirtualRegister(&AArch64::GPR64RegClass);
emitFrameOffset(*MBB, InsertI, DL, ScratchReg, BaseReg,
StackOffset::getFixed(BaseRegOffsetBytes), TII);
BaseReg = ScratchReg;
BaseRegOffsetBytes = 0;
}
MachineInstr *LastI = nullptr;
while (Size) {
int64_t InstrSize = (Size > 16) ? 32 : 16;
unsigned Opcode =
InstrSize == 16
? (ZeroData ? AArch64::STZGi : AArch64::STGi)
: (ZeroData ? AArch64::STZ2Gi : AArch64::ST2Gi);
assert(BaseRegOffsetBytes % 16 == 0);
MachineInstr *I = BuildMI(*MBB, InsertI, DL, TII->get(Opcode))
.addReg(AArch64::SP)
.addReg(BaseReg)
.addImm(BaseRegOffsetBytes / 16)
.setMemRefs(CombinedMemRefs);
// A store to [BaseReg, #0] should go last for an opportunity to fold the
// final SP adjustment in the epilogue.
if (BaseRegOffsetBytes == 0)
LastI = I;
BaseRegOffsetBytes += InstrSize;
Size -= InstrSize;
}
if (LastI)
MBB->splice(InsertI, MBB, LastI);
}
void TagStoreEdit::emitLoop(MachineBasicBlock::iterator InsertI) {
const AArch64InstrInfo *TII =
MF->getSubtarget<AArch64Subtarget>().getInstrInfo();
Register BaseReg = FrameRegUpdate
? FrameReg
: MRI->createVirtualRegister(&AArch64::GPR64RegClass);
Register SizeReg = MRI->createVirtualRegister(&AArch64::GPR64RegClass);
emitFrameOffset(*MBB, InsertI, DL, BaseReg, FrameReg, FrameRegOffset, TII);
int64_t LoopSize = Size;
// If the loop size is not a multiple of 32, split off one 16-byte store at
// the end to fold BaseReg update into.
if (FrameRegUpdate && *FrameRegUpdate)
LoopSize -= LoopSize % 32;
MachineInstr *LoopI = BuildMI(*MBB, InsertI, DL,
TII->get(ZeroData ? AArch64::STZGloop_wback
: AArch64::STGloop_wback))
.addDef(SizeReg)
.addDef(BaseReg)
.addImm(LoopSize)
.addReg(BaseReg)
.setMemRefs(CombinedMemRefs);
if (FrameRegUpdate)
LoopI->setFlags(FrameRegUpdateFlags);
int64_t ExtraBaseRegUpdate =
FrameRegUpdate ? (*FrameRegUpdate - FrameRegOffset.getFixed() - Size) : 0;
LLVM_DEBUG(dbgs() << "TagStoreEdit::emitLoop: LoopSize=" << LoopSize
<< ", Size=" << Size
<< ", ExtraBaseRegUpdate=" << ExtraBaseRegUpdate
<< ", FrameRegUpdate=" << FrameRegUpdate
<< ", FrameRegOffset.getFixed()="
<< FrameRegOffset.getFixed() << "\n");
if (LoopSize < Size) {
assert(FrameRegUpdate);
assert(Size - LoopSize == 16);
// Tag 16 more bytes at BaseReg and update BaseReg.
int64_t STGOffset = ExtraBaseRegUpdate + 16;
assert(STGOffset % 16 == 0 && STGOffset >= -4096 && STGOffset <= 4080 &&
"STG immediate out of range");
BuildMI(*MBB, InsertI, DL,
TII->get(ZeroData ? AArch64::STZGPostIndex : AArch64::STGPostIndex))
.addDef(BaseReg)
.addReg(BaseReg)
.addReg(BaseReg)
.addImm(STGOffset / 16)
.setMemRefs(CombinedMemRefs)
.setMIFlags(FrameRegUpdateFlags);
} else if (ExtraBaseRegUpdate) {
// Update BaseReg.
int64_t AddSubOffset = std::abs(ExtraBaseRegUpdate);
assert(AddSubOffset <= 4095 && "ADD/SUB immediate out of range");
BuildMI(
*MBB, InsertI, DL,
TII->get(ExtraBaseRegUpdate > 0 ? AArch64::ADDXri : AArch64::SUBXri))
.addDef(BaseReg)
.addReg(BaseReg)
.addImm(AddSubOffset)
.addImm(0)
.setMIFlags(FrameRegUpdateFlags);
}
}
// Check if *II is a register update that can be merged into STGloop that ends
// at (Reg + Size). RemainingOffset is the required adjustment to Reg after the
// end of the loop.
bool canMergeRegUpdate(MachineBasicBlock::iterator II, unsigned Reg,
int64_t Size, int64_t *TotalOffset) {
MachineInstr &MI = *II;
if ((MI.getOpcode() == AArch64::ADDXri ||
MI.getOpcode() == AArch64::SUBXri) &&
MI.getOperand(0).getReg() == Reg && MI.getOperand(1).getReg() == Reg) {
unsigned Shift = AArch64_AM::getShiftValue(MI.getOperand(3).getImm());
int64_t Offset = MI.getOperand(2).getImm() << Shift;
if (MI.getOpcode() == AArch64::SUBXri)
Offset = -Offset;
int64_t PostOffset = Offset - Size;
// TagStoreEdit::emitLoop might emit either an ADD/SUB after the loop, or
// an STGPostIndex which does the last 16 bytes of tag write. Which one is
// chosen depends on the alignment of the loop size, but the difference
// between the valid ranges for the two instructions is small, so we
// conservatively assume that it could be either case here.
//
// Max offset of STGPostIndex, minus the 16 byte tag write folded into that
// instruction.
const int64_t kMaxOffset = 4080 - 16;
// Max offset of SUBXri.
const int64_t kMinOffset = -4095;
if (PostOffset <= kMaxOffset && PostOffset >= kMinOffset &&
PostOffset % 16 == 0) {
*TotalOffset = Offset;
return true;
}
}
return false;
}
void mergeMemRefs(const SmallVectorImpl<TagStoreInstr> &TSE,
SmallVectorImpl<MachineMemOperand *> &MemRefs) {
MemRefs.clear();
for (auto &TS : TSE) {
MachineInstr *MI = TS.MI;
// An instruction without memory operands may access anything. Be
// conservative and return an empty list.
if (MI->memoperands_empty()) {
MemRefs.clear();
return;
}
MemRefs.append(MI->memoperands_begin(), MI->memoperands_end());
}
}
void TagStoreEdit::emitCode(MachineBasicBlock::iterator &InsertI,
const AArch64FrameLowering *TFI,
bool TryMergeSPUpdate) {
if (TagStores.empty())
return;
TagStoreInstr &FirstTagStore = TagStores[0];
TagStoreInstr &LastTagStore = TagStores[TagStores.size() - 1];
Size = LastTagStore.Offset - FirstTagStore.Offset + LastTagStore.Size;
DL = TagStores[0].MI->getDebugLoc();
Register Reg;
FrameRegOffset = TFI->resolveFrameOffsetReference(
*MF, FirstTagStore.Offset, false /*isFixed*/, false /*isSVE*/, Reg,
/*PreferFP=*/false, /*ForSimm=*/true);
FrameReg = Reg;
FrameRegUpdate = std::nullopt;
mergeMemRefs(TagStores, CombinedMemRefs);
LLVM_DEBUG({
dbgs() << "Replacing adjacent STG instructions:\n";
for (const auto &Instr : TagStores) {
dbgs() << " " << *Instr.MI;
}
});
// Size threshold where a loop becomes shorter than a linear sequence of
// tagging instructions.
const int kSetTagLoopThreshold = 176;
if (Size < kSetTagLoopThreshold) {
if (TagStores.size() < 2)
return;
emitUnrolled(InsertI);
} else {
MachineInstr *UpdateInstr = nullptr;
int64_t TotalOffset = 0;
if (TryMergeSPUpdate) {
// See if we can merge base register update into the STGloop.
// This is done in AArch64LoadStoreOptimizer for "normal" stores,
// but STGloop is way too unusual for that, and also it only
// realistically happens in function epilogue. Also, STGloop is expanded
// before that pass.
if (InsertI != MBB->end() &&
canMergeRegUpdate(InsertI, FrameReg, FrameRegOffset.getFixed() + Size,
&TotalOffset)) {
UpdateInstr = &*InsertI++;
LLVM_DEBUG(dbgs() << "Folding SP update into loop:\n "
<< *UpdateInstr);
}
}
if (!UpdateInstr && TagStores.size() < 2)
return;
if (UpdateInstr) {
FrameRegUpdate = TotalOffset;
FrameRegUpdateFlags = UpdateInstr->getFlags();
}
emitLoop(InsertI);
if (UpdateInstr)
UpdateInstr->eraseFromParent();
}
for (auto &TS : TagStores)
TS.MI->eraseFromParent();
}
bool isMergeableStackTaggingInstruction(MachineInstr &MI, int64_t &Offset,
int64_t &Size, bool &ZeroData) {
MachineFunction &MF = *MI.getParent()->getParent();
const MachineFrameInfo &MFI = MF.getFrameInfo();
unsigned Opcode = MI.getOpcode();
ZeroData = (Opcode == AArch64::STZGloop || Opcode == AArch64::STZGi ||
Opcode == AArch64::STZ2Gi);
if (Opcode == AArch64::STGloop || Opcode == AArch64::STZGloop) {
if (!MI.getOperand(0).isDead() || !MI.getOperand(1).isDead())
return false;
if (!MI.getOperand(2).isImm() || !MI.getOperand(3).isFI())
return false;
Offset = MFI.getObjectOffset(MI.getOperand(3).getIndex());
Size = MI.getOperand(2).getImm();
return true;
}
if (Opcode == AArch64::STGi || Opcode == AArch64::STZGi)
Size = 16;
else if (Opcode == AArch64::ST2Gi || Opcode == AArch64::STZ2Gi)
Size = 32;
else
return false;
if (MI.getOperand(0).getReg() != AArch64::SP || !MI.getOperand(1).isFI())
return false;
Offset = MFI.getObjectOffset(MI.getOperand(1).getIndex()) +
16 * MI.getOperand(2).getImm();
return true;
}
// Detect a run of memory tagging instructions for adjacent stack frame slots,
// and replace them with a shorter instruction sequence:
// * replace STG + STG with ST2G
// * replace STGloop + STGloop with STGloop
// This code needs to run when stack slot offsets are already known, but before
// FrameIndex operands in STG instructions are eliminated.
MachineBasicBlock::iterator tryMergeAdjacentSTG(MachineBasicBlock::iterator II,
const AArch64FrameLowering *TFI,
RegScavenger *RS) {
bool FirstZeroData;
int64_t Size, Offset;
MachineInstr &MI = *II;
MachineBasicBlock *MBB = MI.getParent();
MachineBasicBlock::iterator NextI = ++II;
if (&MI == &MBB->instr_back())
return II;
if (!isMergeableStackTaggingInstruction(MI, Offset, Size, FirstZeroData))
return II;
SmallVector<TagStoreInstr, 4> Instrs;
Instrs.emplace_back(&MI, Offset, Size);
constexpr int kScanLimit = 10;
int Count = 0;
for (MachineBasicBlock::iterator E = MBB->end();
NextI != E && Count < kScanLimit; ++NextI) {
MachineInstr &MI = *NextI;
bool ZeroData;
int64_t Size, Offset;
// Collect instructions that update memory tags with a FrameIndex operand
// and (when applicable) constant size, and whose output registers are dead
// (the latter is almost always the case in practice). Since these
// instructions effectively have no inputs or outputs, we are free to skip
// any non-aliasing instructions in between without tracking used registers.
if (isMergeableStackTaggingInstruction(MI, Offset, Size, ZeroData)) {
if (ZeroData != FirstZeroData)
break;
Instrs.emplace_back(&MI, Offset, Size);
continue;
}
// Only count non-transient, non-tagging instructions toward the scan
// limit.
if (!MI.isTransient())
++Count;
// Just in case, stop before the epilogue code starts.
if (MI.getFlag(MachineInstr::FrameSetup) ||
MI.getFlag(MachineInstr::FrameDestroy))
break;
// Reject anything that may alias the collected instructions.
if (MI.mayLoadOrStore() || MI.hasUnmodeledSideEffects() || MI.isCall())
break;
}
// New code will be inserted after the last tagging instruction we've found.
MachineBasicBlock::iterator InsertI = Instrs.back().MI;
// All the gathered stack tag instructions are merged and placed after
// last tag store in the list. The check should be made if the nzcv
// flag is live at the point where we are trying to insert. Otherwise
// the nzcv flag might get clobbered if any stg loops are present.
// FIXME : This approach of bailing out from merge is conservative in
// some ways like even if stg loops are not present after merge the
// insert list, this liveness check is done (which is not needed).
LivePhysRegs LiveRegs(*(MBB->getParent()->getSubtarget().getRegisterInfo()));
LiveRegs.addLiveOuts(*MBB);
for (auto I = MBB->rbegin();; ++I) {
MachineInstr &MI = *I;
if (MI == InsertI)
break;
LiveRegs.stepBackward(*I);
}
InsertI++;
if (LiveRegs.contains(AArch64::NZCV))
return InsertI;
llvm::stable_sort(Instrs,
[](const TagStoreInstr &Left, const TagStoreInstr &Right) {
return Left.Offset < Right.Offset;
});
// Make sure that we don't have any overlapping stores.
int64_t CurOffset = Instrs[0].Offset;
for (auto &Instr : Instrs) {
if (CurOffset > Instr.Offset)
return NextI;
CurOffset = Instr.Offset + Instr.Size;
}
// Find contiguous runs of tagged memory and emit shorter instruction
// sequences for them when possible.
TagStoreEdit TSE(MBB, FirstZeroData);
std::optional<int64_t> EndOffset;
for (auto &Instr : Instrs) {
if (EndOffset && *EndOffset != Instr.Offset) {
// Found a gap.
TSE.emitCode(InsertI, TFI, /*TryMergeSPUpdate = */ false);
TSE.clear();
}
TSE.addInstruction(Instr);
EndOffset = Instr.Offset + Instr.Size;
}
const MachineFunction *MF = MBB->getParent();
// Multiple FP/SP updates in a loop cannot be described by CFI instructions.
TSE.emitCode(
InsertI, TFI, /*TryMergeSPUpdate = */
!MF->getInfo<AArch64FunctionInfo>()->needsAsyncDwarfUnwindInfo(*MF));
return InsertI;
}
} // namespace
static void emitVGSaveRestore(MachineBasicBlock::iterator II,
const AArch64FrameLowering *TFI) {
MachineInstr &MI = *II;
MachineBasicBlock *MBB = MI.getParent();
MachineFunction *MF = MBB->getParent();
if (MI.getOpcode() != AArch64::VGSavePseudo &&
MI.getOpcode() != AArch64::VGRestorePseudo)
return;
auto *AFI = MF->getInfo<AArch64FunctionInfo>();
SMEAttrs FuncAttrs = AFI->getSMEFnAttrs();
bool LocallyStreaming =
FuncAttrs.hasStreamingBody() && !FuncAttrs.hasStreamingInterface();
int64_t VGFrameIdx =
LocallyStreaming ? AFI->getStreamingVGIdx() : AFI->getVGIdx();
assert(VGFrameIdx != std::numeric_limits<int>::max() &&
"Expected FrameIdx for VG");
CFIInstBuilder CFIBuilder(*MBB, II, MachineInstr::NoFlags);
if (MI.getOpcode() == AArch64::VGSavePseudo) {
const MachineFrameInfo &MFI = MF->getFrameInfo();
int64_t Offset =
MFI.getObjectOffset(VGFrameIdx) - TFI->getOffsetOfLocalArea();
CFIBuilder.buildOffset(AArch64::VG, Offset);
} else {
CFIBuilder.buildRestore(AArch64::VG);
}
MI.eraseFromParent();
}
void AArch64FrameLowering::processFunctionBeforeFrameIndicesReplaced(
MachineFunction &MF, RegScavenger *RS = nullptr) const {
for (auto &BB : MF)
for (MachineBasicBlock::iterator II = BB.begin(); II != BB.end();) {
if (requiresSaveVG(MF))
emitVGSaveRestore(II++, this);
else if (StackTaggingMergeSetTag)
II = tryMergeAdjacentSTG(II, this, RS);
}
// By the time this method is called, most of the prologue/epilogue code is
// already emitted, whether its location was affected by the shrink-wrapping
// optimization or not.
if (!MF.getFunction().hasFnAttribute(Attribute::Naked) &&
shouldSignReturnAddressEverywhere(MF))
emitPacRetPlusLeafHardening(MF);
}
/// For Win64 AArch64 EH, the offset to the Unwind object is from the SP
/// before the update. This is easily retrieved as it is exactly the offset
/// that is set in processFunctionBeforeFrameFinalized.
StackOffset AArch64FrameLowering::getFrameIndexReferencePreferSP(
const MachineFunction &MF, int FI, Register &FrameReg,
bool IgnoreSPUpdates) const {
const MachineFrameInfo &MFI = MF.getFrameInfo();
if (IgnoreSPUpdates) {
LLVM_DEBUG(dbgs() << "Offset from the SP for " << FI << " is "
<< MFI.getObjectOffset(FI) << "\n");
FrameReg = AArch64::SP;
return StackOffset::getFixed(MFI.getObjectOffset(FI));
}
// Go to common code if we cannot provide sp + offset.
if (MFI.hasVarSizedObjects() ||
MF.getInfo<AArch64FunctionInfo>()->getStackSizeSVE() ||
MF.getSubtarget().getRegisterInfo()->hasStackRealignment(MF))
return getFrameIndexReference(MF, FI, FrameReg);
FrameReg = AArch64::SP;
return getStackOffset(MF, MFI.getObjectOffset(FI));
}
/// The parent frame offset (aka dispFrame) is only used on X86_64 to retrieve
/// the parent's frame pointer
unsigned AArch64FrameLowering::getWinEHParentFrameOffset(
const MachineFunction &MF) const {
return 0;
}
/// Funclets only need to account for space for the callee saved registers,
/// as the locals are accounted for in the parent's stack frame.
unsigned AArch64FrameLowering::getWinEHFuncletFrameSize(
const MachineFunction &MF) const {
// This is the size of the pushed CSRs.
unsigned CSSize =
MF.getInfo<AArch64FunctionInfo>()->getCalleeSavedStackSize();
// This is the amount of stack a funclet needs to allocate.
return alignTo(CSSize + MF.getFrameInfo().getMaxCallFrameSize(),
getStackAlign());
}
namespace {
struct FrameObject {
bool IsValid = false;
// Index of the object in MFI.
int ObjectIndex = 0;
// Group ID this object belongs to.
int GroupIndex = -1;
// This object should be placed first (closest to SP).
bool ObjectFirst = false;
// This object's group (which always contains the object with
// ObjectFirst==true) should be placed first.
bool GroupFirst = false;
// Used to distinguish between FP and GPR accesses. The values are decided so
// that they sort FPR < Hazard < GPR and they can be or'd together.
unsigned Accesses = 0;
enum { AccessFPR = 1, AccessHazard = 2, AccessGPR = 4 };
};
class GroupBuilder {
SmallVector<int, 8> CurrentMembers;
int NextGroupIndex = 0;
std::vector<FrameObject> &Objects;
public:
GroupBuilder(std::vector<FrameObject> &Objects) : Objects(Objects) {}
void AddMember(int Index) { CurrentMembers.push_back(Index); }
void EndCurrentGroup() {
if (CurrentMembers.size() > 1) {
// Create a new group with the current member list. This might remove them
// from their pre-existing groups. That's OK, dealing with overlapping
// groups is too hard and unlikely to make a difference.
LLVM_DEBUG(dbgs() << "group:");
for (int Index : CurrentMembers) {
Objects[Index].GroupIndex = NextGroupIndex;
LLVM_DEBUG(dbgs() << " " << Index);
}
LLVM_DEBUG(dbgs() << "\n");
NextGroupIndex++;
}
CurrentMembers.clear();
}
};
bool FrameObjectCompare(const FrameObject &A, const FrameObject &B) {
// Objects at a lower index are closer to FP; objects at a higher index are
// closer to SP.
//
// For consistency in our comparison, all invalid objects are placed
// at the end. This also allows us to stop walking when we hit the
// first invalid item after it's all sorted.
//
// If we want to include a stack hazard region, order FPR accesses < the
// hazard object < GPRs accesses in order to create a separation between the
// two. For the Accesses field 1 = FPR, 2 = Hazard Object, 4 = GPR.
//
// Otherwise the "first" object goes first (closest to SP), followed by the
// members of the "first" group.
//
// The rest are sorted by the group index to keep the groups together.
// Higher numbered groups are more likely to be around longer (i.e. untagged
// in the function epilogue and not at some earlier point). Place them closer
// to SP.
//
// If all else equal, sort by the object index to keep the objects in the
// original order.
return std::make_tuple(!A.IsValid, A.Accesses, A.ObjectFirst, A.GroupFirst,
A.GroupIndex, A.ObjectIndex) <
std::make_tuple(!B.IsValid, B.Accesses, B.ObjectFirst, B.GroupFirst,
B.GroupIndex, B.ObjectIndex);
}
} // namespace
void AArch64FrameLowering::orderFrameObjects(
const MachineFunction &MF, SmallVectorImpl<int> &ObjectsToAllocate) const {
if (!OrderFrameObjects || ObjectsToAllocate.empty())
return;
const AArch64FunctionInfo &AFI = *MF.getInfo<AArch64FunctionInfo>();
const MachineFrameInfo &MFI = MF.getFrameInfo();
std::vector<FrameObject> FrameObjects(MFI.getObjectIndexEnd());
for (auto &Obj : ObjectsToAllocate) {
FrameObjects[Obj].IsValid = true;
FrameObjects[Obj].ObjectIndex = Obj;
}
// Identify FPR vs GPR slots for hazards, and stack slots that are tagged at
// the same time.
GroupBuilder GB(FrameObjects);
for (auto &MBB : MF) {
for (auto &MI : MBB) {
if (MI.isDebugInstr())
continue;
if (AFI.hasStackHazardSlotIndex()) {
std::optional<int> FI = getLdStFrameID(MI, MFI);
if (FI && *FI >= 0 && *FI < (int)FrameObjects.size()) {
if (MFI.getStackID(*FI) == TargetStackID::ScalableVector ||
AArch64InstrInfo::isFpOrNEON(MI))
FrameObjects[*FI].Accesses |= FrameObject::AccessFPR;
else
FrameObjects[*FI].Accesses |= FrameObject::AccessGPR;
}
}
int OpIndex;
switch (MI.getOpcode()) {
case AArch64::STGloop:
case AArch64::STZGloop:
OpIndex = 3;
break;
case AArch64::STGi:
case AArch64::STZGi:
case AArch64::ST2Gi:
case AArch64::STZ2Gi:
OpIndex = 1;
break;
default:
OpIndex = -1;
}
int TaggedFI = -1;
if (OpIndex >= 0) {
const MachineOperand &MO = MI.getOperand(OpIndex);
if (MO.isFI()) {
int FI = MO.getIndex();
if (FI >= 0 && FI < MFI.getObjectIndexEnd() &&
FrameObjects[FI].IsValid)
TaggedFI = FI;
}
}
// If this is a stack tagging instruction for a slot that is not part of a
// group yet, either start a new group or add it to the current one.
if (TaggedFI >= 0)
GB.AddMember(TaggedFI);
else
GB.EndCurrentGroup();
}
// Groups should never span multiple basic blocks.
GB.EndCurrentGroup();
}
if (AFI.hasStackHazardSlotIndex()) {
FrameObjects[AFI.getStackHazardSlotIndex()].Accesses =
FrameObject::AccessHazard;
// If a stack object is unknown or both GPR and FPR, sort it into GPR.
for (auto &Obj : FrameObjects)
if (!Obj.Accesses ||
Obj.Accesses == (FrameObject::AccessGPR | FrameObject::AccessFPR))
Obj.Accesses = FrameObject::AccessGPR;
}
// If the function's tagged base pointer is pinned to a stack slot, we want to
// put that slot first when possible. This will likely place it at SP + 0,
// and save one instruction when generating the base pointer because IRG does
// not allow an immediate offset.
std::optional<int> TBPI = AFI.getTaggedBasePointerIndex();
if (TBPI) {
FrameObjects[*TBPI].ObjectFirst = true;
FrameObjects[*TBPI].GroupFirst = true;
int FirstGroupIndex = FrameObjects[*TBPI].GroupIndex;
if (FirstGroupIndex >= 0)
for (FrameObject &Object : FrameObjects)
if (Object.GroupIndex == FirstGroupIndex)
Object.GroupFirst = true;
}
llvm::stable_sort(FrameObjects, FrameObjectCompare);
int i = 0;
for (auto &Obj : FrameObjects) {
// All invalid items are sorted at the end, so it's safe to stop.
if (!Obj.IsValid)
break;
ObjectsToAllocate[i++] = Obj.ObjectIndex;
}
LLVM_DEBUG({
dbgs() << "Final frame order:\n";
for (auto &Obj : FrameObjects) {
if (!Obj.IsValid)
break;
dbgs() << " " << Obj.ObjectIndex << ": group " << Obj.GroupIndex;
if (Obj.ObjectFirst)
dbgs() << ", first";
if (Obj.GroupFirst)
dbgs() << ", group-first";
dbgs() << "\n";
}
});
}
/// Emit a loop to decrement SP until it is equal to TargetReg, with probes at
/// least every ProbeSize bytes. Returns an iterator of the first instruction
/// after the loop. The difference between SP and TargetReg must be an exact
/// multiple of ProbeSize.
MachineBasicBlock::iterator
AArch64FrameLowering::inlineStackProbeLoopExactMultiple(
MachineBasicBlock::iterator MBBI, int64_t ProbeSize,
Register TargetReg) const {
MachineBasicBlock &MBB = *MBBI->getParent();
MachineFunction &MF = *MBB.getParent();
const AArch64InstrInfo *TII =
MF.getSubtarget<AArch64Subtarget>().getInstrInfo();
DebugLoc DL = MBB.findDebugLoc(MBBI);
MachineFunction::iterator MBBInsertPoint = std::next(MBB.getIterator());
MachineBasicBlock *LoopMBB = MF.CreateMachineBasicBlock(MBB.getBasicBlock());
MF.insert(MBBInsertPoint, LoopMBB);
MachineBasicBlock *ExitMBB = MF.CreateMachineBasicBlock(MBB.getBasicBlock());
MF.insert(MBBInsertPoint, ExitMBB);
// SUB SP, SP, #ProbeSize (or equivalent if ProbeSize is not encodable
// in SUB).
emitFrameOffset(*LoopMBB, LoopMBB->end(), DL, AArch64::SP, AArch64::SP,
StackOffset::getFixed(-ProbeSize), TII,
MachineInstr::FrameSetup);
// STR XZR, [SP]
BuildMI(*LoopMBB, LoopMBB->end(), DL, TII->get(AArch64::STRXui))
.addReg(AArch64::XZR)
.addReg(AArch64::SP)
.addImm(0)
.setMIFlags(MachineInstr::FrameSetup);
// CMP SP, TargetReg
BuildMI(*LoopMBB, LoopMBB->end(), DL, TII->get(AArch64::SUBSXrx64),
AArch64::XZR)
.addReg(AArch64::SP)
.addReg(TargetReg)
.addImm(AArch64_AM::getArithExtendImm(AArch64_AM::UXTX, 0))
.setMIFlags(MachineInstr::FrameSetup);
// B.CC Loop
BuildMI(*LoopMBB, LoopMBB->end(), DL, TII->get(AArch64::Bcc))
.addImm(AArch64CC::NE)
.addMBB(LoopMBB)
.setMIFlags(MachineInstr::FrameSetup);
LoopMBB->addSuccessor(ExitMBB);
LoopMBB->addSuccessor(LoopMBB);
// Synthesize the exit MBB.
ExitMBB->splice(ExitMBB->end(), &MBB, MBBI, MBB.end());
ExitMBB->transferSuccessorsAndUpdatePHIs(&MBB);
MBB.addSuccessor(LoopMBB);
// Update liveins.
fullyRecomputeLiveIns({ExitMBB, LoopMBB});
return ExitMBB->begin();
}
void AArch64FrameLowering::inlineStackProbeFixed(
MachineBasicBlock::iterator MBBI, Register ScratchReg, int64_t FrameSize,
StackOffset CFAOffset) const {
MachineBasicBlock *MBB = MBBI->getParent();
MachineFunction &MF = *MBB->getParent();
const AArch64InstrInfo *TII =
MF.getSubtarget<AArch64Subtarget>().getInstrInfo();
AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
bool EmitAsyncCFI = AFI->needsAsyncDwarfUnwindInfo(MF);
bool HasFP = hasFP(MF);
DebugLoc DL;
int64_t ProbeSize = MF.getInfo<AArch64FunctionInfo>()->getStackProbeSize();
int64_t NumBlocks = FrameSize / ProbeSize;
int64_t ResidualSize = FrameSize % ProbeSize;
LLVM_DEBUG(dbgs() << "Stack probing: total " << FrameSize << " bytes, "
<< NumBlocks << " blocks of " << ProbeSize
<< " bytes, plus " << ResidualSize << " bytes\n");
// Decrement SP by NumBlock * ProbeSize bytes, with either unrolled or
// ordinary loop.
if (NumBlocks <= AArch64::StackProbeMaxLoopUnroll) {
for (int i = 0; i < NumBlocks; ++i) {
// SUB SP, SP, #ProbeSize (or equivalent if ProbeSize is not
// encodable in a SUB).
emitFrameOffset(*MBB, MBBI, DL, AArch64::SP, AArch64::SP,
StackOffset::getFixed(-ProbeSize), TII,
MachineInstr::FrameSetup, false, false, nullptr,
EmitAsyncCFI && !HasFP, CFAOffset);
CFAOffset += StackOffset::getFixed(ProbeSize);
// STR XZR, [SP]
BuildMI(*MBB, MBBI, DL, TII->get(AArch64::STRXui))
.addReg(AArch64::XZR)
.addReg(AArch64::SP)
.addImm(0)
.setMIFlags(MachineInstr::FrameSetup);
}
} else if (NumBlocks != 0) {
// SUB ScratchReg, SP, #FrameSize (or equivalent if FrameSize is not
// encodable in ADD). ScrathReg may temporarily become the CFA register.
emitFrameOffset(*MBB, MBBI, DL, ScratchReg, AArch64::SP,
StackOffset::getFixed(-ProbeSize * NumBlocks), TII,
MachineInstr::FrameSetup, false, false, nullptr,
EmitAsyncCFI && !HasFP, CFAOffset);
CFAOffset += StackOffset::getFixed(ProbeSize * NumBlocks);
MBBI = inlineStackProbeLoopExactMultiple(MBBI, ProbeSize, ScratchReg);
MBB = MBBI->getParent();
if (EmitAsyncCFI && !HasFP) {
// Set the CFA register back to SP.
CFIInstBuilder(*MBB, MBBI, MachineInstr::FrameSetup)
.buildDefCFARegister(AArch64::SP);
}
}
if (ResidualSize != 0) {
// SUB SP, SP, #ResidualSize (or equivalent if ResidualSize is not encodable
// in SUB).
emitFrameOffset(*MBB, MBBI, DL, AArch64::SP, AArch64::SP,
StackOffset::getFixed(-ResidualSize), TII,
MachineInstr::FrameSetup, false, false, nullptr,
EmitAsyncCFI && !HasFP, CFAOffset);
if (ResidualSize > AArch64::StackProbeMaxUnprobedStack) {
// STR XZR, [SP]
BuildMI(*MBB, MBBI, DL, TII->get(AArch64::STRXui))
.addReg(AArch64::XZR)
.addReg(AArch64::SP)
.addImm(0)
.setMIFlags(MachineInstr::FrameSetup);
}
}
}
void AArch64FrameLowering::inlineStackProbe(MachineFunction &MF,
MachineBasicBlock &MBB) const {
// Get the instructions that need to be replaced. We emit at most two of
// these. Remember them in order to avoid complications coming from the need
// to traverse the block while potentially creating more blocks.
SmallVector<MachineInstr *, 4> ToReplace;
for (MachineInstr &MI : MBB)
if (MI.getOpcode() == AArch64::PROBED_STACKALLOC ||
MI.getOpcode() == AArch64::PROBED_STACKALLOC_VAR)
ToReplace.push_back(&MI);
for (MachineInstr *MI : ToReplace) {
if (MI->getOpcode() == AArch64::PROBED_STACKALLOC) {
Register ScratchReg = MI->getOperand(0).getReg();
int64_t FrameSize = MI->getOperand(1).getImm();
StackOffset CFAOffset = StackOffset::get(MI->getOperand(2).getImm(),
MI->getOperand(3).getImm());
inlineStackProbeFixed(MI->getIterator(), ScratchReg, FrameSize,
CFAOffset);
} else {
assert(MI->getOpcode() == AArch64::PROBED_STACKALLOC_VAR &&
"Stack probe pseudo-instruction expected");
const AArch64InstrInfo *TII =
MI->getMF()->getSubtarget<AArch64Subtarget>().getInstrInfo();
Register TargetReg = MI->getOperand(0).getReg();
(void)TII->probedStackAlloc(MI->getIterator(), TargetReg, true);
}
MI->eraseFromParent();
}
}
struct StackAccess {
enum AccessType {
NotAccessed = 0, // Stack object not accessed by load/store instructions.
GPR = 1 << 0, // A general purpose register.
PPR = 1 << 1, // A predicate register.
FPR = 1 << 2, // A floating point/Neon/SVE register.
};
int Idx;
StackOffset Offset;
int64_t Size;
unsigned AccessTypes;
StackAccess() : Idx(0), Offset(), Size(0), AccessTypes(NotAccessed) {}
bool operator<(const StackAccess &Rhs) const {
return std::make_tuple(start(), Idx) <
std::make_tuple(Rhs.start(), Rhs.Idx);
}
bool isCPU() const {
// Predicate register load and store instructions execute on the CPU.
return AccessTypes & (AccessType::GPR | AccessType::PPR);
}
bool isSME() const { return AccessTypes & AccessType::FPR; }
bool isMixed() const { return isCPU() && isSME(); }
int64_t start() const { return Offset.getFixed() + Offset.getScalable(); }
int64_t end() const { return start() + Size; }
std::string getTypeString() const {
switch (AccessTypes) {
case AccessType::FPR:
return "FPR";
case AccessType::PPR:
return "PPR";
case AccessType::GPR:
return "GPR";
case AccessType::NotAccessed:
return "NA";
default:
return "Mixed";
}
}
void print(raw_ostream &OS) const {
OS << getTypeString() << " stack object at [SP"
<< (Offset.getFixed() < 0 ? "" : "+") << Offset.getFixed();
if (Offset.getScalable())
OS << (Offset.getScalable() < 0 ? "" : "+") << Offset.getScalable()
<< " * vscale";
OS << "]";
}
};
static inline raw_ostream &operator<<(raw_ostream &OS, const StackAccess &SA) {
SA.print(OS);
return OS;
}
void AArch64FrameLowering::emitRemarks(
const MachineFunction &MF, MachineOptimizationRemarkEmitter *ORE) const {
auto *AFI = MF.getInfo<AArch64FunctionInfo>();
if (AFI->getSMEFnAttrs().hasNonStreamingInterfaceAndBody())
return;
unsigned StackHazardSize = getStackHazardSize(MF);
const uint64_t HazardSize =
(StackHazardSize) ? StackHazardSize : StackHazardRemarkSize;
if (HazardSize == 0)
return;
const MachineFrameInfo &MFI = MF.getFrameInfo();
// Bail if function has no stack objects.
if (!MFI.hasStackObjects())
return;
std::vector<StackAccess> StackAccesses(MFI.getNumObjects());
size_t NumFPLdSt = 0;
size_t NumNonFPLdSt = 0;
// Collect stack accesses via Load/Store instructions.
for (const MachineBasicBlock &MBB : MF) {
for (const MachineInstr &MI : MBB) {
if (!MI.mayLoadOrStore() || MI.getNumMemOperands() < 1)
continue;
for (MachineMemOperand *MMO : MI.memoperands()) {
std::optional<int> FI = getMMOFrameID(MMO, MFI);
if (FI && !MFI.isDeadObjectIndex(*FI)) {
int FrameIdx = *FI;
size_t ArrIdx = FrameIdx + MFI.getNumFixedObjects();
if (StackAccesses[ArrIdx].AccessTypes == StackAccess::NotAccessed) {
StackAccesses[ArrIdx].Idx = FrameIdx;
StackAccesses[ArrIdx].Offset =
getFrameIndexReferenceFromSP(MF, FrameIdx);
StackAccesses[ArrIdx].Size = MFI.getObjectSize(FrameIdx);
}
unsigned RegTy = StackAccess::AccessType::GPR;
if (MFI.getStackID(FrameIdx) == TargetStackID::ScalableVector) {
// SPILL_PPR_TO_ZPR_SLOT_PSEUDO and FILL_PPR_FROM_ZPR_SLOT_PSEUDO
// spill/fill the predicate as a data vector (so are an FPR access).
if (MI.getOpcode() != AArch64::SPILL_PPR_TO_ZPR_SLOT_PSEUDO &&
MI.getOpcode() != AArch64::FILL_PPR_FROM_ZPR_SLOT_PSEUDO &&
AArch64::PPRRegClass.contains(MI.getOperand(0).getReg())) {
RegTy = StackAccess::PPR;
} else
RegTy = StackAccess::FPR;
} else if (AArch64InstrInfo::isFpOrNEON(MI)) {
RegTy = StackAccess::FPR;
}
StackAccesses[ArrIdx].AccessTypes |= RegTy;
if (RegTy == StackAccess::FPR)
++NumFPLdSt;
else
++NumNonFPLdSt;
}
}
}
}
if (NumFPLdSt == 0 || NumNonFPLdSt == 0)
return;
llvm::sort(StackAccesses);
llvm::erase_if(StackAccesses, [](const StackAccess &S) {
return S.AccessTypes == StackAccess::NotAccessed;
});
SmallVector<const StackAccess *> MixedObjects;
SmallVector<std::pair<const StackAccess *, const StackAccess *>> HazardPairs;
if (StackAccesses.front().isMixed())
MixedObjects.push_back(&StackAccesses.front());
for (auto It = StackAccesses.begin(), End = std::prev(StackAccesses.end());
It != End; ++It) {
const auto &First = *It;
const auto &Second = *(It + 1);
if (Second.isMixed())
MixedObjects.push_back(&Second);
if ((First.isSME() && Second.isCPU()) ||
(First.isCPU() && Second.isSME())) {
uint64_t Distance = static_cast<uint64_t>(Second.start() - First.end());
if (Distance < HazardSize)
HazardPairs.emplace_back(&First, &Second);
}
}
auto EmitRemark = [&](llvm::StringRef Str) {
ORE->emit([&]() {
auto R = MachineOptimizationRemarkAnalysis(
"sme", "StackHazard", MF.getFunction().getSubprogram(), &MF.front());
return R << formatv("stack hazard in '{0}': ", MF.getName()).str() << Str;
});
};
for (const auto &P : HazardPairs)
EmitRemark(formatv("{0} is too close to {1}", *P.first, *P.second).str());
for (const auto *Obj : MixedObjects)
EmitRemark(
formatv("{0} accessed by both GP and FP instructions", *Obj).str());
}