| //===- AArch64RegisterInfo.cpp - AArch64 Register Information -------------===// |
| // |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| // See https://llvm.org/LICENSE.txt for license information. |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| // |
| //===----------------------------------------------------------------------===// |
| // |
| // This file contains the AArch64 implementation of the TargetRegisterInfo |
| // class. |
| // |
| //===----------------------------------------------------------------------===// |
| |
| #include "AArch64RegisterInfo.h" |
| #include "AArch64FrameLowering.h" |
| #include "AArch64InstrInfo.h" |
| #include "AArch64MachineFunctionInfo.h" |
| #include "AArch64Subtarget.h" |
| #include "MCTargetDesc/AArch64AddressingModes.h" |
| #include "MCTargetDesc/AArch64InstPrinter.h" |
| #include "Utils/AArch64SMEAttributes.h" |
| #include "llvm/ADT/BitVector.h" |
| #include "llvm/BinaryFormat/Dwarf.h" |
| #include "llvm/CodeGen/LiveRegMatrix.h" |
| #include "llvm/CodeGen/MachineFrameInfo.h" |
| #include "llvm/CodeGen/MachineInstrBuilder.h" |
| #include "llvm/CodeGen/MachineRegisterInfo.h" |
| #include "llvm/CodeGen/RegisterScavenging.h" |
| #include "llvm/CodeGen/TargetFrameLowering.h" |
| #include "llvm/IR/DebugInfoMetadata.h" |
| #include "llvm/IR/DiagnosticInfo.h" |
| #include "llvm/IR/Function.h" |
| #include "llvm/Target/TargetOptions.h" |
| #include "llvm/TargetParser/Triple.h" |
| |
| using namespace llvm; |
| |
| #define GET_CC_REGISTER_LISTS |
| #include "AArch64GenCallingConv.inc" |
| #define GET_REGINFO_TARGET_DESC |
| #include "AArch64GenRegisterInfo.inc" |
| |
| AArch64RegisterInfo::AArch64RegisterInfo(const Triple &TT, unsigned HwMode) |
| : AArch64GenRegisterInfo(AArch64::LR, 0, 0, 0, HwMode), TT(TT) { |
| AArch64_MC::initLLVMToCVRegMapping(this); |
| } |
| |
| /// Return whether the register needs a CFI entry. Not all unwinders may know |
| /// about SVE registers, so we assume the lowest common denominator, i.e. the |
| /// callee-saves required by the base ABI. For the SVE registers z8-z15 only the |
| /// lower 64-bits (d8-d15) need to be saved. The lower 64-bits subreg is |
| /// returned in \p RegToUseForCFI. |
| bool AArch64RegisterInfo::regNeedsCFI(MCRegister Reg, |
| MCRegister &RegToUseForCFI) const { |
| if (AArch64::PPRRegClass.contains(Reg)) |
| return false; |
| |
| if (AArch64::ZPRRegClass.contains(Reg)) { |
| RegToUseForCFI = getSubReg(Reg, AArch64::dsub); |
| for (int I = 0; CSR_AArch64_AAPCS_SaveList[I]; ++I) { |
| if (CSR_AArch64_AAPCS_SaveList[I] == RegToUseForCFI) |
| return true; |
| } |
| return false; |
| } |
| |
| RegToUseForCFI = Reg; |
| return true; |
| } |
| |
| const MCPhysReg * |
| AArch64RegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const { |
| assert(MF && "Invalid MachineFunction pointer."); |
| |
| if (MF->getFunction().getCallingConv() == CallingConv::GHC) |
| // GHC set of callee saved regs is empty as all those regs are |
| // used for passing STG regs around |
| return CSR_AArch64_NoRegs_SaveList; |
| if (MF->getFunction().getCallingConv() == CallingConv::PreserveNone) |
| return CSR_AArch64_NoneRegs_SaveList; |
| if (MF->getFunction().getCallingConv() == CallingConv::AnyReg) |
| return CSR_AArch64_AllRegs_SaveList; |
| |
| if (MF->getFunction().getCallingConv() == CallingConv::ARM64EC_Thunk_X64) |
| return CSR_Win_AArch64_Arm64EC_Thunk_SaveList; |
| |
| // Darwin has its own CSR_AArch64_AAPCS_SaveList, which means most CSR save |
| // lists depending on that will need to have their Darwin variant as well. |
| if (MF->getSubtarget<AArch64Subtarget>().isTargetDarwin()) |
| return getDarwinCalleeSavedRegs(MF); |
| |
| if (MF->getFunction().getCallingConv() == CallingConv::CFGuard_Check) |
| return CSR_Win_AArch64_CFGuard_Check_SaveList; |
| if (MF->getSubtarget<AArch64Subtarget>().isTargetWindows()) { |
| if (MF->getSubtarget<AArch64Subtarget>().getTargetLowering() |
| ->supportSwiftError() && |
| MF->getFunction().getAttributes().hasAttrSomewhere( |
| Attribute::SwiftError)) |
| return CSR_Win_AArch64_AAPCS_SwiftError_SaveList; |
| if (MF->getFunction().getCallingConv() == CallingConv::SwiftTail) |
| return CSR_Win_AArch64_AAPCS_SwiftTail_SaveList; |
| if (MF->getFunction().getCallingConv() == CallingConv::AArch64_VectorCall) |
| return CSR_Win_AArch64_AAVPCS_SaveList; |
| if (MF->getFunction().getCallingConv() == |
| CallingConv::AArch64_SVE_VectorCall) |
| return CSR_Win_AArch64_SVE_AAPCS_SaveList; |
| if (MF->getInfo<AArch64FunctionInfo>()->isSVECC()) |
| return CSR_Win_AArch64_SVE_AAPCS_SaveList; |
| return CSR_Win_AArch64_AAPCS_SaveList; |
| } |
| if (MF->getFunction().getCallingConv() == CallingConv::AArch64_VectorCall) |
| return CSR_AArch64_AAVPCS_SaveList; |
| if (MF->getFunction().getCallingConv() == CallingConv::AArch64_SVE_VectorCall) |
| return CSR_AArch64_SVE_AAPCS_SaveList; |
| if (MF->getFunction().getCallingConv() == |
| CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0) |
| report_fatal_error( |
| "Calling convention " |
| "AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0 is only " |
| "supported to improve calls to SME ACLE save/restore/disable-za " |
| "functions, and is not intended to be used beyond that scope."); |
| if (MF->getFunction().getCallingConv() == |
| CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1) |
| report_fatal_error( |
| "Calling convention " |
| "AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1 is " |
| "only supported to improve calls to SME ACLE __arm_get_current_vg " |
| "function, and is not intended to be used beyond that scope."); |
| if (MF->getFunction().getCallingConv() == |
| CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2) |
| report_fatal_error( |
| "Calling convention " |
| "AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2 is " |
| "only supported to improve calls to SME ACLE __arm_sme_state " |
| "and is not intended to be used beyond that scope."); |
| if (MF->getSubtarget<AArch64Subtarget>().getTargetLowering() |
| ->supportSwiftError() && |
| MF->getFunction().getAttributes().hasAttrSomewhere( |
| Attribute::SwiftError)) |
| return CSR_AArch64_AAPCS_SwiftError_SaveList; |
| if (MF->getFunction().getCallingConv() == CallingConv::SwiftTail) |
| return CSR_AArch64_AAPCS_SwiftTail_SaveList; |
| if (MF->getFunction().getCallingConv() == CallingConv::PreserveMost) |
| return CSR_AArch64_RT_MostRegs_SaveList; |
| if (MF->getFunction().getCallingConv() == CallingConv::PreserveAll) |
| return CSR_AArch64_RT_AllRegs_SaveList; |
| if (MF->getFunction().getCallingConv() == CallingConv::Win64) |
| // This is for OSes other than Windows; Windows is a separate case further |
| // above. |
| return CSR_AArch64_AAPCS_X18_SaveList; |
| if (MF->getInfo<AArch64FunctionInfo>()->isSVECC()) |
| return CSR_AArch64_SVE_AAPCS_SaveList; |
| return CSR_AArch64_AAPCS_SaveList; |
| } |
| |
| const MCPhysReg * |
| AArch64RegisterInfo::getDarwinCalleeSavedRegs(const MachineFunction *MF) const { |
| assert(MF && "Invalid MachineFunction pointer."); |
| assert(MF->getSubtarget<AArch64Subtarget>().isTargetDarwin() && |
| "Invalid subtarget for getDarwinCalleeSavedRegs"); |
| |
| if (MF->getFunction().getCallingConv() == CallingConv::CFGuard_Check) |
| report_fatal_error( |
| "Calling convention CFGuard_Check is unsupported on Darwin."); |
| if (MF->getFunction().getCallingConv() == CallingConv::AArch64_VectorCall) |
| return CSR_Darwin_AArch64_AAVPCS_SaveList; |
| if (MF->getFunction().getCallingConv() == CallingConv::AArch64_SVE_VectorCall) |
| report_fatal_error( |
| "Calling convention SVE_VectorCall is unsupported on Darwin."); |
| if (MF->getFunction().getCallingConv() == |
| CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0) |
| report_fatal_error( |
| "Calling convention " |
| "AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0 is " |
| "only supported to improve calls to SME ACLE save/restore/disable-za " |
| "functions, and is not intended to be used beyond that scope."); |
| if (MF->getFunction().getCallingConv() == |
| CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1) |
| report_fatal_error( |
| "Calling convention " |
| "AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1 is " |
| "only supported to improve calls to SME ACLE __arm_get_current_vg " |
| "function, and is not intended to be used beyond that scope."); |
| if (MF->getFunction().getCallingConv() == |
| CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2) |
| report_fatal_error( |
| "Calling convention " |
| "AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2 is " |
| "only supported to improve calls to SME ACLE __arm_sme_state " |
| "and is not intended to be used beyond that scope."); |
| if (MF->getFunction().getCallingConv() == CallingConv::CXX_FAST_TLS) |
| return MF->getInfo<AArch64FunctionInfo>()->isSplitCSR() |
| ? CSR_Darwin_AArch64_CXX_TLS_PE_SaveList |
| : CSR_Darwin_AArch64_CXX_TLS_SaveList; |
| if (MF->getSubtarget<AArch64Subtarget>().getTargetLowering() |
| ->supportSwiftError() && |
| MF->getFunction().getAttributes().hasAttrSomewhere( |
| Attribute::SwiftError)) |
| return CSR_Darwin_AArch64_AAPCS_SwiftError_SaveList; |
| if (MF->getFunction().getCallingConv() == CallingConv::SwiftTail) |
| return CSR_Darwin_AArch64_AAPCS_SwiftTail_SaveList; |
| if (MF->getFunction().getCallingConv() == CallingConv::PreserveMost) |
| return CSR_Darwin_AArch64_RT_MostRegs_SaveList; |
| if (MF->getFunction().getCallingConv() == CallingConv::PreserveAll) |
| return CSR_Darwin_AArch64_RT_AllRegs_SaveList; |
| if (MF->getFunction().getCallingConv() == CallingConv::Win64) |
| return CSR_Darwin_AArch64_AAPCS_Win64_SaveList; |
| if (MF->getInfo<AArch64FunctionInfo>()->isSVECC()) |
| return CSR_Darwin_AArch64_SVE_AAPCS_SaveList; |
| return CSR_Darwin_AArch64_AAPCS_SaveList; |
| } |
| |
| const MCPhysReg *AArch64RegisterInfo::getCalleeSavedRegsViaCopy( |
| const MachineFunction *MF) const { |
| assert(MF && "Invalid MachineFunction pointer."); |
| if (MF->getFunction().getCallingConv() == CallingConv::CXX_FAST_TLS && |
| MF->getInfo<AArch64FunctionInfo>()->isSplitCSR()) |
| return CSR_Darwin_AArch64_CXX_TLS_ViaCopy_SaveList; |
| return nullptr; |
| } |
| |
| void AArch64RegisterInfo::UpdateCustomCalleeSavedRegs( |
| MachineFunction &MF) const { |
| const MCPhysReg *CSRs = getCalleeSavedRegs(&MF); |
| SmallVector<MCPhysReg, 32> UpdatedCSRs; |
| for (const MCPhysReg *I = CSRs; *I; ++I) |
| UpdatedCSRs.push_back(*I); |
| |
| for (size_t i = 0; i < AArch64::GPR64commonRegClass.getNumRegs(); ++i) { |
| if (MF.getSubtarget<AArch64Subtarget>().isXRegCustomCalleeSaved(i)) { |
| UpdatedCSRs.push_back(AArch64::GPR64commonRegClass.getRegister(i)); |
| } |
| } |
| // Register lists are zero-terminated. |
| UpdatedCSRs.push_back(0); |
| MF.getRegInfo().setCalleeSavedRegs(UpdatedCSRs); |
| } |
| |
| const TargetRegisterClass * |
| AArch64RegisterInfo::getSubClassWithSubReg(const TargetRegisterClass *RC, |
| unsigned Idx) const { |
| // edge case for GPR/FPR register classes |
| if (RC == &AArch64::GPR32allRegClass && Idx == AArch64::hsub) |
| return &AArch64::FPR32RegClass; |
| else if (RC == &AArch64::GPR64allRegClass && Idx == AArch64::hsub) |
| return &AArch64::FPR64RegClass; |
| |
| // Forward to TableGen's default version. |
| return AArch64GenRegisterInfo::getSubClassWithSubReg(RC, Idx); |
| } |
| |
| const uint32_t * |
| AArch64RegisterInfo::getDarwinCallPreservedMask(const MachineFunction &MF, |
| CallingConv::ID CC) const { |
| assert(MF.getSubtarget<AArch64Subtarget>().isTargetDarwin() && |
| "Invalid subtarget for getDarwinCallPreservedMask"); |
| |
| if (CC == CallingConv::CXX_FAST_TLS) |
| return CSR_Darwin_AArch64_CXX_TLS_RegMask; |
| if (CC == CallingConv::AArch64_VectorCall) |
| return CSR_Darwin_AArch64_AAVPCS_RegMask; |
| if (CC == CallingConv::AArch64_SVE_VectorCall) |
| return CSR_Darwin_AArch64_SVE_AAPCS_RegMask; |
| if (CC == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0) |
| return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0_RegMask; |
| if (CC == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1) |
| return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1_RegMask; |
| if (CC == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2) |
| return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2_RegMask; |
| if (CC == CallingConv::CFGuard_Check) |
| report_fatal_error( |
| "Calling convention CFGuard_Check is unsupported on Darwin."); |
| if (MF.getSubtarget<AArch64Subtarget>() |
| .getTargetLowering() |
| ->supportSwiftError() && |
| MF.getFunction().getAttributes().hasAttrSomewhere(Attribute::SwiftError)) |
| return CSR_Darwin_AArch64_AAPCS_SwiftError_RegMask; |
| if (CC == CallingConv::SwiftTail) |
| return CSR_Darwin_AArch64_AAPCS_SwiftTail_RegMask; |
| if (CC == CallingConv::PreserveMost) |
| return CSR_Darwin_AArch64_RT_MostRegs_RegMask; |
| if (CC == CallingConv::PreserveAll) |
| return CSR_Darwin_AArch64_RT_AllRegs_RegMask; |
| return CSR_Darwin_AArch64_AAPCS_RegMask; |
| } |
| |
| const uint32_t * |
| AArch64RegisterInfo::getCallPreservedMask(const MachineFunction &MF, |
| CallingConv::ID CC) const { |
| bool SCS = MF.getFunction().hasFnAttribute(Attribute::ShadowCallStack); |
| if (CC == CallingConv::GHC) |
| // This is academic because all GHC calls are (supposed to be) tail calls |
| return SCS ? CSR_AArch64_NoRegs_SCS_RegMask : CSR_AArch64_NoRegs_RegMask; |
| if (CC == CallingConv::PreserveNone) |
| return SCS ? CSR_AArch64_NoneRegs_SCS_RegMask |
| : CSR_AArch64_NoneRegs_RegMask; |
| if (CC == CallingConv::AnyReg) |
| return SCS ? CSR_AArch64_AllRegs_SCS_RegMask : CSR_AArch64_AllRegs_RegMask; |
| |
| // All the following calling conventions are handled differently on Darwin. |
| if (MF.getSubtarget<AArch64Subtarget>().isTargetDarwin()) { |
| if (SCS) |
| report_fatal_error("ShadowCallStack attribute not supported on Darwin."); |
| return getDarwinCallPreservedMask(MF, CC); |
| } |
| |
| if (CC == CallingConv::AArch64_VectorCall) |
| return SCS ? CSR_AArch64_AAVPCS_SCS_RegMask : CSR_AArch64_AAVPCS_RegMask; |
| if (CC == CallingConv::AArch64_SVE_VectorCall) |
| return SCS ? CSR_AArch64_SVE_AAPCS_SCS_RegMask |
| : CSR_AArch64_SVE_AAPCS_RegMask; |
| if (CC == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0) |
| return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0_RegMask; |
| if (CC == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1) |
| return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1_RegMask; |
| if (CC == CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2) |
| return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2_RegMask; |
| if (CC == CallingConv::CFGuard_Check) |
| return CSR_Win_AArch64_CFGuard_Check_RegMask; |
| if (MF.getSubtarget<AArch64Subtarget>().getTargetLowering() |
| ->supportSwiftError() && |
| MF.getFunction().getAttributes().hasAttrSomewhere(Attribute::SwiftError)) |
| return SCS ? CSR_AArch64_AAPCS_SwiftError_SCS_RegMask |
| : CSR_AArch64_AAPCS_SwiftError_RegMask; |
| if (CC == CallingConv::SwiftTail) { |
| if (SCS) |
| report_fatal_error("ShadowCallStack attribute not supported with swifttail"); |
| return CSR_AArch64_AAPCS_SwiftTail_RegMask; |
| } |
| if (CC == CallingConv::PreserveMost) |
| return SCS ? CSR_AArch64_RT_MostRegs_SCS_RegMask |
| : CSR_AArch64_RT_MostRegs_RegMask; |
| if (CC == CallingConv::PreserveAll) |
| return SCS ? CSR_AArch64_RT_AllRegs_SCS_RegMask |
| : CSR_AArch64_RT_AllRegs_RegMask; |
| |
| return SCS ? CSR_AArch64_AAPCS_SCS_RegMask : CSR_AArch64_AAPCS_RegMask; |
| } |
| |
| const uint32_t *AArch64RegisterInfo::getCustomEHPadPreservedMask( |
| const MachineFunction &MF) const { |
| if (MF.getSubtarget<AArch64Subtarget>().isTargetLinux()) |
| return CSR_AArch64_AAPCS_RegMask; |
| |
| return nullptr; |
| } |
| |
| const uint32_t *AArch64RegisterInfo::getTLSCallPreservedMask() const { |
| if (TT.isOSDarwin()) |
| return CSR_Darwin_AArch64_TLS_RegMask; |
| |
| assert(TT.isOSBinFormatELF() && "Invalid target"); |
| return CSR_AArch64_TLS_ELF_RegMask; |
| } |
| |
| void AArch64RegisterInfo::UpdateCustomCallPreservedMask(MachineFunction &MF, |
| const uint32_t **Mask) const { |
| uint32_t *UpdatedMask = MF.allocateRegMask(); |
| unsigned RegMaskSize = MachineOperand::getRegMaskSize(getNumRegs()); |
| memcpy(UpdatedMask, *Mask, sizeof(UpdatedMask[0]) * RegMaskSize); |
| |
| for (size_t i = 0; i < AArch64::GPR64commonRegClass.getNumRegs(); ++i) { |
| if (MF.getSubtarget<AArch64Subtarget>().isXRegCustomCalleeSaved(i)) { |
| for (MCPhysReg SubReg : |
| subregs_inclusive(AArch64::GPR64commonRegClass.getRegister(i))) { |
| // See TargetRegisterInfo::getCallPreservedMask for how to interpret the |
| // register mask. |
| UpdatedMask[SubReg / 32] |= 1u << (SubReg % 32); |
| } |
| } |
| } |
| *Mask = UpdatedMask; |
| } |
| |
| const uint32_t *AArch64RegisterInfo::getSMStartStopCallPreservedMask() const { |
| return CSR_AArch64_SMStartStop_RegMask; |
| } |
| |
| const uint32_t * |
| AArch64RegisterInfo::SMEABISupportRoutinesCallPreservedMaskFromX0() const { |
| return CSR_AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0_RegMask; |
| } |
| |
| const uint32_t *AArch64RegisterInfo::getNoPreservedMask() const { |
| return CSR_AArch64_NoRegs_RegMask; |
| } |
| |
| const uint32_t * |
| AArch64RegisterInfo::getThisReturnPreservedMask(const MachineFunction &MF, |
| CallingConv::ID CC) const { |
| // This should return a register mask that is the same as that returned by |
| // getCallPreservedMask but that additionally preserves the register used for |
| // the first i64 argument (which must also be the register used to return a |
| // single i64 return value) |
| // |
| // In case that the calling convention does not use the same register for |
| // both, the function should return NULL (does not currently apply) |
| assert(CC != CallingConv::GHC && "should not be GHC calling convention."); |
| if (MF.getSubtarget<AArch64Subtarget>().isTargetDarwin()) |
| return CSR_Darwin_AArch64_AAPCS_ThisReturn_RegMask; |
| return CSR_AArch64_AAPCS_ThisReturn_RegMask; |
| } |
| |
| const uint32_t *AArch64RegisterInfo::getWindowsStackProbePreservedMask() const { |
| return CSR_AArch64_StackProbe_Windows_RegMask; |
| } |
| |
| std::optional<std::string> |
| AArch64RegisterInfo::explainReservedReg(const MachineFunction &MF, |
| MCRegister PhysReg) const { |
| if (hasBasePointer(MF) && MCRegisterInfo::regsOverlap(PhysReg, AArch64::X19)) |
| return std::string("X19 is used as the frame base pointer register."); |
| |
| if (MF.getSubtarget<AArch64Subtarget>().isWindowsArm64EC()) { |
| bool warn = false; |
| if (MCRegisterInfo::regsOverlap(PhysReg, AArch64::X13) || |
| MCRegisterInfo::regsOverlap(PhysReg, AArch64::X14) || |
| MCRegisterInfo::regsOverlap(PhysReg, AArch64::X23) || |
| MCRegisterInfo::regsOverlap(PhysReg, AArch64::X24) || |
| MCRegisterInfo::regsOverlap(PhysReg, AArch64::X28)) |
| warn = true; |
| |
| for (unsigned i = AArch64::B16; i <= AArch64::B31; ++i) |
| if (MCRegisterInfo::regsOverlap(PhysReg, i)) |
| warn = true; |
| |
| if (warn) |
| return std::string(AArch64InstPrinter::getRegisterName(PhysReg)) + |
| " is clobbered by asynchronous signals when using Arm64EC."; |
| } |
| |
| return {}; |
| } |
| |
| BitVector |
| AArch64RegisterInfo::getStrictlyReservedRegs(const MachineFunction &MF) const { |
| const AArch64FrameLowering *TFI = getFrameLowering(MF); |
| |
| // FIXME: avoid re-calculating this every time. |
| BitVector Reserved(getNumRegs()); |
| markSuperRegs(Reserved, AArch64::WSP); |
| markSuperRegs(Reserved, AArch64::WZR); |
| |
| if (TFI->hasFP(MF) || TT.isOSDarwin()) |
| markSuperRegs(Reserved, AArch64::W29); |
| |
| if (MF.getSubtarget<AArch64Subtarget>().isWindowsArm64EC()) { |
| // x13, x14, x23, x24, x28, and v16-v31 are clobbered by asynchronous |
| // signals, so we can't ever use them. |
| markSuperRegs(Reserved, AArch64::W13); |
| markSuperRegs(Reserved, AArch64::W14); |
| markSuperRegs(Reserved, AArch64::W23); |
| markSuperRegs(Reserved, AArch64::W24); |
| markSuperRegs(Reserved, AArch64::W28); |
| for (unsigned i = AArch64::B16; i <= AArch64::B31; ++i) |
| markSuperRegs(Reserved, i); |
| } |
| |
| for (size_t i = 0; i < AArch64::GPR32commonRegClass.getNumRegs(); ++i) { |
| if (MF.getSubtarget<AArch64Subtarget>().isXRegisterReserved(i)) |
| markSuperRegs(Reserved, AArch64::GPR32commonRegClass.getRegister(i)); |
| } |
| |
| if (hasBasePointer(MF)) |
| markSuperRegs(Reserved, AArch64::W19); |
| |
| // SLH uses register W16/X16 as the taint register. |
| if (MF.getFunction().hasFnAttribute(Attribute::SpeculativeLoadHardening)) |
| markSuperRegs(Reserved, AArch64::W16); |
| |
| // FFR is modelled as global state that cannot be allocated. |
| if (MF.getSubtarget<AArch64Subtarget>().hasSVE()) |
| Reserved.set(AArch64::FFR); |
| |
| // SME tiles are not allocatable. |
| if (MF.getSubtarget<AArch64Subtarget>().hasSME()) { |
| for (MCPhysReg SubReg : subregs_inclusive(AArch64::ZA)) |
| Reserved.set(SubReg); |
| } |
| |
| // VG cannot be allocated |
| Reserved.set(AArch64::VG); |
| |
| if (MF.getSubtarget<AArch64Subtarget>().hasSME2()) { |
| for (MCSubRegIterator SubReg(AArch64::ZT0, this, /*self=*/true); |
| SubReg.isValid(); ++SubReg) |
| Reserved.set(*SubReg); |
| } |
| |
| markSuperRegs(Reserved, AArch64::FPCR); |
| markSuperRegs(Reserved, AArch64::FPMR); |
| markSuperRegs(Reserved, AArch64::FPSR); |
| |
| if (MF.getFunction().getCallingConv() == CallingConv::GRAAL) { |
| markSuperRegs(Reserved, AArch64::X27); |
| markSuperRegs(Reserved, AArch64::X28); |
| markSuperRegs(Reserved, AArch64::W27); |
| markSuperRegs(Reserved, AArch64::W28); |
| } |
| |
| assert(checkAllSuperRegsMarked(Reserved)); |
| |
| // Add _HI registers after checkAllSuperRegsMarked as this check otherwise |
| // becomes considerably more expensive. |
| Reserved.set(AArch64::WSP_HI); |
| Reserved.set(AArch64::WZR_HI); |
| static_assert(AArch64::W30_HI - AArch64::W0_HI == 30, |
| "Unexpected order of registers"); |
| Reserved.set(AArch64::W0_HI, AArch64::W30_HI); |
| static_assert(AArch64::B31_HI - AArch64::B0_HI == 31, |
| "Unexpected order of registers"); |
| Reserved.set(AArch64::B0_HI, AArch64::B31_HI); |
| static_assert(AArch64::H31_HI - AArch64::H0_HI == 31, |
| "Unexpected order of registers"); |
| Reserved.set(AArch64::H0_HI, AArch64::H31_HI); |
| static_assert(AArch64::S31_HI - AArch64::S0_HI == 31, |
| "Unexpected order of registers"); |
| Reserved.set(AArch64::S0_HI, AArch64::S31_HI); |
| static_assert(AArch64::D31_HI - AArch64::D0_HI == 31, |
| "Unexpected order of registers"); |
| Reserved.set(AArch64::D0_HI, AArch64::D31_HI); |
| static_assert(AArch64::Q31_HI - AArch64::Q0_HI == 31, |
| "Unexpected order of registers"); |
| Reserved.set(AArch64::Q0_HI, AArch64::Q31_HI); |
| |
| return Reserved; |
| } |
| |
| BitVector |
| AArch64RegisterInfo::getUserReservedRegs(const MachineFunction &MF) const { |
| BitVector Reserved(getNumRegs()); |
| for (size_t i = 0; i < AArch64::GPR32commonRegClass.getNumRegs(); ++i) { |
| // ReserveXRegister is set for registers manually reserved |
| // through +reserve-x#i. |
| if (MF.getSubtarget<AArch64Subtarget>().isXRegisterReserved(i)) |
| markSuperRegs(Reserved, AArch64::GPR32commonRegClass.getRegister(i)); |
| } |
| return Reserved; |
| } |
| |
| BitVector |
| AArch64RegisterInfo::getReservedRegs(const MachineFunction &MF) const { |
| BitVector Reserved(getNumRegs()); |
| for (size_t i = 0; i < AArch64::GPR32commonRegClass.getNumRegs(); ++i) { |
| if (MF.getSubtarget<AArch64Subtarget>().isXRegisterReservedForRA(i)) |
| markSuperRegs(Reserved, AArch64::GPR32commonRegClass.getRegister(i)); |
| } |
| |
| if (MF.getSubtarget<AArch64Subtarget>().isLRReservedForRA()) { |
| // In order to prevent the register allocator from using LR, we need to |
| // mark it as reserved. However we don't want to keep it reserved throughout |
| // the pipeline since it prevents other infrastructure from reasoning about |
| // it's liveness. We use the NoVRegs property instead of IsSSA because |
| // IsSSA is removed before VirtRegRewriter runs. |
| if (!MF.getProperties().hasNoVRegs()) |
| markSuperRegs(Reserved, AArch64::LR); |
| } |
| |
| assert(checkAllSuperRegsMarked(Reserved)); |
| |
| // Handle strictlyReservedRegs separately to avoid re-evaluating the assert, |
| // which becomes considerably expensive when considering the _HI registers. |
| Reserved |= getStrictlyReservedRegs(MF); |
| |
| return Reserved; |
| } |
| |
| bool AArch64RegisterInfo::isReservedReg(const MachineFunction &MF, |
| MCRegister Reg) const { |
| return getReservedRegs(MF)[Reg]; |
| } |
| |
| bool AArch64RegisterInfo::isUserReservedReg(const MachineFunction &MF, |
| MCRegister Reg) const { |
| return getUserReservedRegs(MF)[Reg]; |
| } |
| |
| bool AArch64RegisterInfo::isStrictlyReservedReg(const MachineFunction &MF, |
| MCRegister Reg) const { |
| return getStrictlyReservedRegs(MF)[Reg]; |
| } |
| |
| bool AArch64RegisterInfo::isAnyArgRegReserved(const MachineFunction &MF) const { |
| return llvm::any_of(*AArch64::GPR64argRegClass.MC, [this, &MF](MCPhysReg r) { |
| return isStrictlyReservedReg(MF, r); |
| }); |
| } |
| |
| void AArch64RegisterInfo::emitReservedArgRegCallError( |
| const MachineFunction &MF) const { |
| const Function &F = MF.getFunction(); |
| F.getContext().diagnose(DiagnosticInfoUnsupported{F, ("AArch64 doesn't support" |
| " function calls if any of the argument registers is reserved.")}); |
| } |
| |
| bool AArch64RegisterInfo::isAsmClobberable(const MachineFunction &MF, |
| MCRegister PhysReg) const { |
| // SLH uses register X16 as the taint register but it will fallback to a different |
| // method if the user clobbers it. So X16 is not reserved for inline asm but is |
| // for normal codegen. |
| if (MF.getFunction().hasFnAttribute(Attribute::SpeculativeLoadHardening) && |
| MCRegisterInfo::regsOverlap(PhysReg, AArch64::X16)) |
| return true; |
| |
| // ZA/ZT0 registers are reserved but may be permitted in the clobber list. |
| if (PhysReg == AArch64::ZA || PhysReg == AArch64::ZT0) |
| return true; |
| |
| return !isReservedReg(MF, PhysReg); |
| } |
| |
| const TargetRegisterClass * |
| AArch64RegisterInfo::getPointerRegClass(const MachineFunction &MF, |
| unsigned Kind) const { |
| return &AArch64::GPR64spRegClass; |
| } |
| |
| const TargetRegisterClass * |
| AArch64RegisterInfo::getCrossCopyRegClass(const TargetRegisterClass *RC) const { |
| if (RC == &AArch64::CCRRegClass) |
| return &AArch64::GPR64RegClass; // Only MSR & MRS copy NZCV. |
| return RC; |
| } |
| |
| unsigned AArch64RegisterInfo::getBaseRegister() const { return AArch64::X19; } |
| |
| bool AArch64RegisterInfo::hasBasePointer(const MachineFunction &MF) const { |
| const MachineFrameInfo &MFI = MF.getFrameInfo(); |
| |
| // In the presence of variable sized objects or funclets, if the fixed stack |
| // size is large enough that referencing from the FP won't result in things |
| // being in range relatively often, we can use a base pointer to allow access |
| // from the other direction like the SP normally works. |
| // |
| // Furthermore, if both variable sized objects are present, and the |
| // stack needs to be dynamically re-aligned, the base pointer is the only |
| // reliable way to reference the locals. |
| if (MFI.hasVarSizedObjects() || MF.hasEHFunclets()) { |
| if (hasStackRealignment(MF)) |
| return true; |
| |
| auto &ST = MF.getSubtarget<AArch64Subtarget>(); |
| const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>(); |
| if (ST.hasSVE() || ST.isStreaming()) { |
| // Frames that have variable sized objects and scalable SVE objects, |
| // should always use a basepointer. |
| if (!AFI->hasCalculatedStackSizeSVE() || AFI->getStackSizeSVE()) |
| return true; |
| } |
| |
| // Frames with hazard padding can have a large offset between the frame |
| // pointer and GPR locals, which includes the emergency spill slot. If the |
| // emergency spill slot is not within range of the load/store instructions |
| // (which have a signed 9-bit range), we will fail to compile if it is used. |
| // Since hasBasePointer() is called before we know if we have hazard padding |
| // or an emergency spill slot we need to enable the basepointer |
| // conservatively. |
| if (ST.getStreamingHazardSize() && |
| !AFI->getSMEFnAttrs().hasNonStreamingInterfaceAndBody()) { |
| return true; |
| } |
| |
| // Conservatively estimate whether the negative offset from the frame |
| // pointer will be sufficient to reach. If a function has a smallish |
| // frame, it's less likely to have lots of spills and callee saved |
| // space, so it's all more likely to be within range of the frame pointer. |
| // If it's wrong, we'll materialize the constant and still get to the |
| // object; it's just suboptimal. Negative offsets use the unscaled |
| // load/store instructions, which have a 9-bit signed immediate. |
| return MFI.getLocalFrameSize() >= 256; |
| } |
| |
| return false; |
| } |
| |
| bool AArch64RegisterInfo::isArgumentRegister(const MachineFunction &MF, |
| MCRegister Reg) const { |
| CallingConv::ID CC = MF.getFunction().getCallingConv(); |
| const AArch64Subtarget &STI = MF.getSubtarget<AArch64Subtarget>(); |
| bool IsVarArg = STI.isCallingConvWin64(MF.getFunction().getCallingConv(), |
| MF.getFunction().isVarArg()); |
| |
| auto HasReg = [](ArrayRef<MCRegister> RegList, MCRegister Reg) { |
| return llvm::is_contained(RegList, Reg); |
| }; |
| |
| switch (CC) { |
| default: |
| report_fatal_error("Unsupported calling convention."); |
| case CallingConv::GHC: |
| return HasReg(CC_AArch64_GHC_ArgRegs, Reg); |
| case CallingConv::PreserveNone: |
| if (!MF.getFunction().isVarArg()) |
| return HasReg(CC_AArch64_Preserve_None_ArgRegs, Reg); |
| [[fallthrough]]; |
| case CallingConv::C: |
| case CallingConv::Fast: |
| case CallingConv::PreserveMost: |
| case CallingConv::PreserveAll: |
| case CallingConv::CXX_FAST_TLS: |
| case CallingConv::Swift: |
| case CallingConv::SwiftTail: |
| case CallingConv::Tail: |
| if (STI.isTargetWindows()) { |
| if (IsVarArg) |
| return HasReg(CC_AArch64_Win64_VarArg_ArgRegs, Reg); |
| switch (CC) { |
| default: |
| return HasReg(CC_AArch64_Win64PCS_ArgRegs, Reg); |
| case CallingConv::Swift: |
| case CallingConv::SwiftTail: |
| return HasReg(CC_AArch64_Win64PCS_Swift_ArgRegs, Reg) || |
| HasReg(CC_AArch64_Win64PCS_ArgRegs, Reg); |
| } |
| } |
| if (!STI.isTargetDarwin()) { |
| switch (CC) { |
| default: |
| return HasReg(CC_AArch64_AAPCS_ArgRegs, Reg); |
| case CallingConv::Swift: |
| case CallingConv::SwiftTail: |
| return HasReg(CC_AArch64_AAPCS_ArgRegs, Reg) || |
| HasReg(CC_AArch64_AAPCS_Swift_ArgRegs, Reg); |
| } |
| } |
| if (!IsVarArg) { |
| switch (CC) { |
| default: |
| return HasReg(CC_AArch64_DarwinPCS_ArgRegs, Reg); |
| case CallingConv::Swift: |
| case CallingConv::SwiftTail: |
| return HasReg(CC_AArch64_DarwinPCS_ArgRegs, Reg) || |
| HasReg(CC_AArch64_DarwinPCS_Swift_ArgRegs, Reg); |
| } |
| } |
| if (STI.isTargetILP32()) |
| return HasReg(CC_AArch64_DarwinPCS_ILP32_VarArg_ArgRegs, Reg); |
| return HasReg(CC_AArch64_DarwinPCS_VarArg_ArgRegs, Reg); |
| case CallingConv::Win64: |
| if (IsVarArg) |
| HasReg(CC_AArch64_Win64_VarArg_ArgRegs, Reg); |
| return HasReg(CC_AArch64_Win64PCS_ArgRegs, Reg); |
| case CallingConv::CFGuard_Check: |
| return HasReg(CC_AArch64_Win64_CFGuard_Check_ArgRegs, Reg); |
| case CallingConv::AArch64_VectorCall: |
| case CallingConv::AArch64_SVE_VectorCall: |
| case CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0: |
| case CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1: |
| case CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2: |
| if (STI.isTargetWindows()) |
| return HasReg(CC_AArch64_Win64PCS_ArgRegs, Reg); |
| return HasReg(CC_AArch64_AAPCS_ArgRegs, Reg); |
| } |
| } |
| |
| Register |
| AArch64RegisterInfo::getFrameRegister(const MachineFunction &MF) const { |
| const AArch64FrameLowering *TFI = getFrameLowering(MF); |
| return TFI->hasFP(MF) ? AArch64::FP : AArch64::SP; |
| } |
| |
| bool AArch64RegisterInfo::requiresRegisterScavenging( |
| const MachineFunction &MF) const { |
| return true; |
| } |
| |
| bool AArch64RegisterInfo::requiresVirtualBaseRegisters( |
| const MachineFunction &MF) const { |
| return true; |
| } |
| |
| bool |
| AArch64RegisterInfo::useFPForScavengingIndex(const MachineFunction &MF) const { |
| // This function indicates whether the emergency spillslot should be placed |
| // close to the beginning of the stackframe (closer to FP) or the end |
| // (closer to SP). |
| // |
| // The beginning works most reliably if we have a frame pointer. |
| // In the presence of any non-constant space between FP and locals, |
| // (e.g. in case of stack realignment or a scalable SVE area), it is |
| // better to use SP or BP. |
| const AArch64FrameLowering &TFI = *getFrameLowering(MF); |
| const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>(); |
| assert((!MF.getSubtarget<AArch64Subtarget>().hasSVE() || |
| AFI->hasCalculatedStackSizeSVE()) && |
| "Expected SVE area to be calculated by this point"); |
| return TFI.hasFP(MF) && !hasStackRealignment(MF) && !AFI->getStackSizeSVE() && |
| !AFI->hasStackHazardSlotIndex(); |
| } |
| |
| bool AArch64RegisterInfo::requiresFrameIndexScavenging( |
| const MachineFunction &MF) const { |
| return true; |
| } |
| |
| bool |
| AArch64RegisterInfo::cannotEliminateFrame(const MachineFunction &MF) const { |
| const MachineFrameInfo &MFI = MF.getFrameInfo(); |
| if (MF.getTarget().Options.DisableFramePointerElim(MF) && MFI.adjustsStack()) |
| return true; |
| return MFI.hasVarSizedObjects() || MFI.isFrameAddressTaken(); |
| } |
| |
| /// needsFrameBaseReg - Returns true if the instruction's frame index |
| /// reference would be better served by a base register other than FP |
| /// or SP. Used by LocalStackFrameAllocation to determine which frame index |
| /// references it should create new base registers for. |
| bool AArch64RegisterInfo::needsFrameBaseReg(MachineInstr *MI, |
| int64_t Offset) const { |
| for (unsigned i = 0; !MI->getOperand(i).isFI(); ++i) |
| assert(i < MI->getNumOperands() && |
| "Instr doesn't have FrameIndex operand!"); |
| |
| // It's the load/store FI references that cause issues, as it can be difficult |
| // to materialize the offset if it won't fit in the literal field. Estimate |
| // based on the size of the local frame and some conservative assumptions |
| // about the rest of the stack frame (note, this is pre-regalloc, so |
| // we don't know everything for certain yet) whether this offset is likely |
| // to be out of range of the immediate. Return true if so. |
| |
| // We only generate virtual base registers for loads and stores, so |
| // return false for everything else. |
| if (!MI->mayLoad() && !MI->mayStore()) |
| return false; |
| |
| // Without a virtual base register, if the function has variable sized |
| // objects, all fixed-size local references will be via the frame pointer, |
| // Approximate the offset and see if it's legal for the instruction. |
| // Note that the incoming offset is based on the SP value at function entry, |
| // so it'll be negative. |
| MachineFunction &MF = *MI->getParent()->getParent(); |
| const AArch64FrameLowering *TFI = getFrameLowering(MF); |
| MachineFrameInfo &MFI = MF.getFrameInfo(); |
| |
| // Estimate an offset from the frame pointer. |
| // Conservatively assume all GPR callee-saved registers get pushed. |
| // FP, LR, X19-X28, D8-D15. 64-bits each. |
| int64_t FPOffset = Offset - 16 * 20; |
| // Estimate an offset from the stack pointer. |
| // The incoming offset is relating to the SP at the start of the function, |
| // but when we access the local it'll be relative to the SP after local |
| // allocation, so adjust our SP-relative offset by that allocation size. |
| Offset += MFI.getLocalFrameSize(); |
| // Assume that we'll have at least some spill slots allocated. |
| // FIXME: This is a total SWAG number. We should run some statistics |
| // and pick a real one. |
| Offset += 128; // 128 bytes of spill slots |
| |
| // If there is a frame pointer, try using it. |
| // The FP is only available if there is no dynamic realignment. We |
| // don't know for sure yet whether we'll need that, so we guess based |
| // on whether there are any local variables that would trigger it. |
| if (TFI->hasFP(MF) && isFrameOffsetLegal(MI, AArch64::FP, FPOffset)) |
| return false; |
| |
| // If we can reference via the stack pointer or base pointer, try that. |
| // FIXME: This (and the code that resolves the references) can be improved |
| // to only disallow SP relative references in the live range of |
| // the VLA(s). In practice, it's unclear how much difference that |
| // would make, but it may be worth doing. |
| if (isFrameOffsetLegal(MI, AArch64::SP, Offset)) |
| return false; |
| |
| // If even offset 0 is illegal, we don't want a virtual base register. |
| if (!isFrameOffsetLegal(MI, AArch64::SP, 0)) |
| return false; |
| |
| // The offset likely isn't legal; we want to allocate a virtual base register. |
| return true; |
| } |
| |
| bool AArch64RegisterInfo::isFrameOffsetLegal(const MachineInstr *MI, |
| Register BaseReg, |
| int64_t Offset) const { |
| assert(MI && "Unable to get the legal offset for nil instruction."); |
| StackOffset SaveOffset = StackOffset::getFixed(Offset); |
| return isAArch64FrameOffsetLegal(*MI, SaveOffset) & AArch64FrameOffsetIsLegal; |
| } |
| |
| /// Insert defining instruction(s) for BaseReg to be a pointer to FrameIdx |
| /// at the beginning of the basic block. |
| Register |
| AArch64RegisterInfo::materializeFrameBaseRegister(MachineBasicBlock *MBB, |
| int FrameIdx, |
| int64_t Offset) const { |
| MachineBasicBlock::iterator Ins = MBB->begin(); |
| DebugLoc DL; // Defaults to "unknown" |
| if (Ins != MBB->end()) |
| DL = Ins->getDebugLoc(); |
| const MachineFunction &MF = *MBB->getParent(); |
| const AArch64InstrInfo *TII = |
| MF.getSubtarget<AArch64Subtarget>().getInstrInfo(); |
| const MCInstrDesc &MCID = TII->get(AArch64::ADDXri); |
| MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo(); |
| Register BaseReg = MRI.createVirtualRegister(&AArch64::GPR64spRegClass); |
| MRI.constrainRegClass(BaseReg, TII->getRegClass(MCID, 0, this, MF)); |
| unsigned Shifter = AArch64_AM::getShifterImm(AArch64_AM::LSL, 0); |
| |
| BuildMI(*MBB, Ins, DL, MCID, BaseReg) |
| .addFrameIndex(FrameIdx) |
| .addImm(Offset) |
| .addImm(Shifter); |
| |
| return BaseReg; |
| } |
| |
| void AArch64RegisterInfo::resolveFrameIndex(MachineInstr &MI, Register BaseReg, |
| int64_t Offset) const { |
| // ARM doesn't need the general 64-bit offsets |
| StackOffset Off = StackOffset::getFixed(Offset); |
| |
| unsigned i = 0; |
| while (!MI.getOperand(i).isFI()) { |
| ++i; |
| assert(i < MI.getNumOperands() && "Instr doesn't have FrameIndex operand!"); |
| } |
| |
| const MachineFunction *MF = MI.getParent()->getParent(); |
| const AArch64InstrInfo *TII = |
| MF->getSubtarget<AArch64Subtarget>().getInstrInfo(); |
| bool Done = rewriteAArch64FrameIndex(MI, i, BaseReg, Off, TII); |
| assert(Done && "Unable to resolve frame index!"); |
| (void)Done; |
| } |
| |
| // Create a scratch register for the frame index elimination in an instruction. |
| // This function has special handling of stack tagging loop pseudos, in which |
| // case it can also change the instruction opcode. |
| static Register |
| createScratchRegisterForInstruction(MachineInstr &MI, unsigned FIOperandNum, |
| const AArch64InstrInfo *TII) { |
| // ST*Gloop have a reserved scratch register in operand 1. Use it, and also |
| // replace the instruction with the writeback variant because it will now |
| // satisfy the operand constraints for it. |
| Register ScratchReg; |
| if (MI.getOpcode() == AArch64::STGloop || |
| MI.getOpcode() == AArch64::STZGloop) { |
| assert(FIOperandNum == 3 && |
| "Wrong frame index operand for STGloop/STZGloop"); |
| unsigned Op = MI.getOpcode() == AArch64::STGloop ? AArch64::STGloop_wback |
| : AArch64::STZGloop_wback; |
| ScratchReg = MI.getOperand(1).getReg(); |
| MI.getOperand(3).ChangeToRegister(ScratchReg, false, false, true); |
| MI.setDesc(TII->get(Op)); |
| MI.tieOperands(1, 3); |
| } else { |
| ScratchReg = |
| MI.getMF()->getRegInfo().createVirtualRegister(&AArch64::GPR64RegClass); |
| MI.getOperand(FIOperandNum) |
| .ChangeToRegister(ScratchReg, false, false, true); |
| } |
| return ScratchReg; |
| } |
| |
| void AArch64RegisterInfo::getOffsetOpcodes( |
| const StackOffset &Offset, SmallVectorImpl<uint64_t> &Ops) const { |
| // The smallest scalable element supported by scaled SVE addressing |
| // modes are predicates, which are 2 scalable bytes in size. So the scalable |
| // byte offset must always be a multiple of 2. |
| assert(Offset.getScalable() % 2 == 0 && "Invalid frame offset"); |
| |
| // Add fixed-sized offset using existing DIExpression interface. |
| DIExpression::appendOffset(Ops, Offset.getFixed()); |
| |
| unsigned VG = getDwarfRegNum(AArch64::VG, true); |
| int64_t VGSized = Offset.getScalable() / 2; |
| if (VGSized > 0) { |
| Ops.push_back(dwarf::DW_OP_constu); |
| Ops.push_back(VGSized); |
| Ops.append({dwarf::DW_OP_bregx, VG, 0ULL}); |
| Ops.push_back(dwarf::DW_OP_mul); |
| Ops.push_back(dwarf::DW_OP_plus); |
| } else if (VGSized < 0) { |
| Ops.push_back(dwarf::DW_OP_constu); |
| Ops.push_back(-VGSized); |
| Ops.append({dwarf::DW_OP_bregx, VG, 0ULL}); |
| Ops.push_back(dwarf::DW_OP_mul); |
| Ops.push_back(dwarf::DW_OP_minus); |
| } |
| } |
| |
| bool AArch64RegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II, |
| int SPAdj, unsigned FIOperandNum, |
| RegScavenger *RS) const { |
| assert(SPAdj == 0 && "Unexpected"); |
| |
| MachineInstr &MI = *II; |
| MachineBasicBlock &MBB = *MI.getParent(); |
| MachineFunction &MF = *MBB.getParent(); |
| const MachineFrameInfo &MFI = MF.getFrameInfo(); |
| const AArch64InstrInfo *TII = |
| MF.getSubtarget<AArch64Subtarget>().getInstrInfo(); |
| const AArch64FrameLowering *TFI = getFrameLowering(MF); |
| int FrameIndex = MI.getOperand(FIOperandNum).getIndex(); |
| bool Tagged = |
| MI.getOperand(FIOperandNum).getTargetFlags() & AArch64II::MO_TAGGED; |
| Register FrameReg; |
| |
| // Special handling of dbg_value, stackmap patchpoint statepoint instructions. |
| if (MI.getOpcode() == TargetOpcode::STACKMAP || |
| MI.getOpcode() == TargetOpcode::PATCHPOINT || |
| MI.getOpcode() == TargetOpcode::STATEPOINT) { |
| StackOffset Offset = |
| TFI->resolveFrameIndexReference(MF, FrameIndex, FrameReg, |
| /*PreferFP=*/true, |
| /*ForSimm=*/false); |
| Offset += StackOffset::getFixed(MI.getOperand(FIOperandNum + 1).getImm()); |
| MI.getOperand(FIOperandNum).ChangeToRegister(FrameReg, false /*isDef*/); |
| MI.getOperand(FIOperandNum + 1).ChangeToImmediate(Offset.getFixed()); |
| return false; |
| } |
| |
| if (MI.getOpcode() == TargetOpcode::LOCAL_ESCAPE) { |
| MachineOperand &FI = MI.getOperand(FIOperandNum); |
| StackOffset Offset = TFI->getNonLocalFrameIndexReference(MF, FrameIndex); |
| assert(!Offset.getScalable() && |
| "Frame offsets with a scalable component are not supported"); |
| FI.ChangeToImmediate(Offset.getFixed()); |
| return false; |
| } |
| |
| StackOffset Offset; |
| if (MI.getOpcode() == AArch64::TAGPstack) { |
| // TAGPstack must use the virtual frame register in its 3rd operand. |
| const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>(); |
| FrameReg = MI.getOperand(3).getReg(); |
| Offset = StackOffset::getFixed(MFI.getObjectOffset(FrameIndex) + |
| AFI->getTaggedBasePointerOffset()); |
| } else if (Tagged) { |
| StackOffset SPOffset = StackOffset::getFixed( |
| MFI.getObjectOffset(FrameIndex) + (int64_t)MFI.getStackSize()); |
| if (MFI.hasVarSizedObjects() || |
| isAArch64FrameOffsetLegal(MI, SPOffset, nullptr, nullptr, nullptr) != |
| (AArch64FrameOffsetCanUpdate | AArch64FrameOffsetIsLegal)) { |
| // Can't update to SP + offset in place. Precalculate the tagged pointer |
| // in a scratch register. |
| Offset = TFI->resolveFrameIndexReference( |
| MF, FrameIndex, FrameReg, /*PreferFP=*/false, /*ForSimm=*/true); |
| Register ScratchReg = |
| MF.getRegInfo().createVirtualRegister(&AArch64::GPR64RegClass); |
| emitFrameOffset(MBB, II, MI.getDebugLoc(), ScratchReg, FrameReg, Offset, |
| TII); |
| BuildMI(MBB, MI, MI.getDebugLoc(), TII->get(AArch64::LDG), ScratchReg) |
| .addReg(ScratchReg) |
| .addReg(ScratchReg) |
| .addImm(0); |
| MI.getOperand(FIOperandNum) |
| .ChangeToRegister(ScratchReg, false, false, true); |
| return false; |
| } |
| FrameReg = AArch64::SP; |
| Offset = StackOffset::getFixed(MFI.getObjectOffset(FrameIndex) + |
| (int64_t)MFI.getStackSize()); |
| } else { |
| Offset = TFI->resolveFrameIndexReference( |
| MF, FrameIndex, FrameReg, /*PreferFP=*/false, /*ForSimm=*/true); |
| } |
| |
| // Modify MI as necessary to handle as much of 'Offset' as possible |
| if (rewriteAArch64FrameIndex(MI, FIOperandNum, FrameReg, Offset, TII)) |
| return true; |
| |
| assert((!RS || !RS->isScavengingFrameIndex(FrameIndex)) && |
| "Emergency spill slot is out of reach"); |
| |
| // If we get here, the immediate doesn't fit into the instruction. We folded |
| // as much as possible above. Handle the rest, providing a register that is |
| // SP+LargeImm. |
| Register ScratchReg = |
| createScratchRegisterForInstruction(MI, FIOperandNum, TII); |
| emitFrameOffset(MBB, II, MI.getDebugLoc(), ScratchReg, FrameReg, Offset, TII); |
| return false; |
| } |
| |
| unsigned AArch64RegisterInfo::getRegPressureLimit(const TargetRegisterClass *RC, |
| MachineFunction &MF) const { |
| const AArch64FrameLowering *TFI = getFrameLowering(MF); |
| |
| switch (RC->getID()) { |
| default: |
| return 0; |
| case AArch64::GPR32RegClassID: |
| case AArch64::GPR32spRegClassID: |
| case AArch64::GPR32allRegClassID: |
| case AArch64::GPR64spRegClassID: |
| case AArch64::GPR64allRegClassID: |
| case AArch64::GPR64RegClassID: |
| case AArch64::GPR32commonRegClassID: |
| case AArch64::GPR64commonRegClassID: |
| return 32 - 1 // XZR/SP |
| - (TFI->hasFP(MF) || TT.isOSDarwin()) // FP |
| - MF.getSubtarget<AArch64Subtarget>().getNumXRegisterReserved() |
| - hasBasePointer(MF); // X19 |
| case AArch64::FPR8RegClassID: |
| case AArch64::FPR16RegClassID: |
| case AArch64::FPR32RegClassID: |
| case AArch64::FPR64RegClassID: |
| case AArch64::FPR128RegClassID: |
| return 32; |
| |
| case AArch64::MatrixIndexGPR32_8_11RegClassID: |
| case AArch64::MatrixIndexGPR32_12_15RegClassID: |
| return 4; |
| |
| case AArch64::DDRegClassID: |
| case AArch64::DDDRegClassID: |
| case AArch64::DDDDRegClassID: |
| case AArch64::QQRegClassID: |
| case AArch64::QQQRegClassID: |
| case AArch64::QQQQRegClassID: |
| return 32; |
| |
| case AArch64::FPR128_loRegClassID: |
| case AArch64::FPR64_loRegClassID: |
| case AArch64::FPR16_loRegClassID: |
| return 16; |
| case AArch64::FPR128_0to7RegClassID: |
| return 8; |
| } |
| } |
| |
| // FORM_TRANSPOSED_REG_TUPLE nodes are created to improve register allocation |
| // where a consecutive multi-vector tuple is constructed from the same indices |
| // of multiple strided loads. This may still result in unnecessary copies |
| // between the loads and the tuple. Here we try to return a hint to assign the |
| // contiguous ZPRMulReg starting at the same register as the first operand of |
| // the pseudo, which should be a subregister of the first strided load. |
| // |
| // For example, if the first strided load has been assigned $z16_z20_z24_z28 |
| // and the operands of the pseudo are each accessing subregister zsub2, we |
| // should look through through Order to find a contiguous register which |
| // begins with $z24 (i.e. $z24_z25_z26_z27). |
| // |
| bool AArch64RegisterInfo::getRegAllocationHints( |
| Register VirtReg, ArrayRef<MCPhysReg> Order, |
| SmallVectorImpl<MCPhysReg> &Hints, const MachineFunction &MF, |
| const VirtRegMap *VRM, const LiveRegMatrix *Matrix) const { |
| |
| auto &ST = MF.getSubtarget<AArch64Subtarget>(); |
| if (!ST.hasSME() || !ST.isStreaming()) |
| return TargetRegisterInfo::getRegAllocationHints(VirtReg, Order, Hints, MF, |
| VRM); |
| |
| // The SVE calling convention preserves registers Z8-Z23. As a result, there |
| // are no ZPR2Strided or ZPR4Strided registers that do not overlap with the |
| // callee-saved registers and so by default these will be pushed to the back |
| // of the allocation order for the ZPRStridedOrContiguous classes. |
| // If any of the instructions which define VirtReg are used by the |
| // FORM_TRANSPOSED_REG_TUPLE pseudo, we want to favour reducing copy |
| // instructions over reducing the number of clobbered callee-save registers, |
| // so we add the strided registers as a hint. |
| const MachineRegisterInfo &MRI = MF.getRegInfo(); |
| unsigned RegID = MRI.getRegClass(VirtReg)->getID(); |
| if (RegID == AArch64::ZPR2StridedOrContiguousRegClassID || |
| RegID == AArch64::ZPR4StridedOrContiguousRegClassID) { |
| |
| // Look through uses of the register for FORM_TRANSPOSED_REG_TUPLE. |
| for (const MachineInstr &Use : MRI.use_nodbg_instructions(VirtReg)) { |
| if (Use.getOpcode() != AArch64::FORM_TRANSPOSED_REG_TUPLE_X2_PSEUDO && |
| Use.getOpcode() != AArch64::FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO) |
| continue; |
| |
| unsigned UseOps = Use.getNumOperands() - 1; |
| const TargetRegisterClass *StridedRC; |
| switch (RegID) { |
| case AArch64::ZPR2StridedOrContiguousRegClassID: |
| StridedRC = &AArch64::ZPR2StridedRegClass; |
| break; |
| case AArch64::ZPR4StridedOrContiguousRegClassID: |
| StridedRC = &AArch64::ZPR4StridedRegClass; |
| break; |
| default: |
| llvm_unreachable("Unexpected RegID"); |
| } |
| |
| SmallVector<MCPhysReg, 4> StridedOrder; |
| for (MCPhysReg Reg : Order) |
| if (StridedRC->contains(Reg)) |
| StridedOrder.push_back(Reg); |
| |
| int OpIdx = Use.findRegisterUseOperandIdx(VirtReg, this); |
| assert(OpIdx != -1 && "Expected operand index from register use."); |
| |
| unsigned TupleID = MRI.getRegClass(Use.getOperand(0).getReg())->getID(); |
| bool IsMulZPR = TupleID == AArch64::ZPR2Mul2RegClassID || |
| TupleID == AArch64::ZPR4Mul4RegClassID; |
| |
| const MachineOperand *AssignedRegOp = llvm::find_if( |
| make_range(Use.operands_begin() + 1, Use.operands_end()), |
| [&VRM](const MachineOperand &Op) { |
| return VRM->hasPhys(Op.getReg()); |
| }); |
| |
| // Example: |
| // |
| // When trying to find a suitable register allocation for VirtReg %v2 in: |
| // |
| // %v0:zpr2stridedorcontiguous = ld1 p0/z, [...] |
| // %v1:zpr2stridedorcontiguous = ld1 p0/z, [...] |
| // %v2:zpr2stridedorcontiguous = ld1 p0/z, [...] |
| // %v3:zpr2stridedorcontiguous = ld1 p0/z, [...] |
| // %v4:zpr4mul4 = FORM_TRANSPOSED_X4 %v0:0, %v1:0, %v2:0, %v3:0 |
| // |
| // One such suitable allocation would be: |
| // |
| // { z0, z8 } = ld1 p0/z, [...] |
| // { z1, z9 } = ld1 p0/z, [...] |
| // { z2, z10 } = ld1 p0/z, [...] |
| // { z3, z11 } = ld1 p0/z, [...] |
| // { z0, z1, z2, z3 } = |
| // FORM_TRANSPOSED_X4 {z0, z8}:0, {z1, z9}:0, {z2, z10}:0, {z3, z11}:0 |
| // |
| // Below we distinguish two cases when trying to find a register: |
| // * None of the registers used by FORM_TRANSPOSED_X4 have been assigned |
| // yet. In this case the code muse ensure that there are at least UseOps |
| // free consecutive registers. If IsMulZPR is true, then the first of |
| // registers must also be a multiple of UseOps, e.g. { z0, z1, z2, z3 } |
| // is valid but { z1, z2, z3, z5 } is not. |
| // * One or more of the registers used by FORM_TRANSPOSED_X4 is already |
| // assigned a physical register, which means only checking that a |
| // consecutive range of free tuple registers exists which includes |
| // the assigned register. |
| // e.g. in the example above, if { z0, z8 } is already allocated for |
| // %v0, we just need to ensure that { z1, z9 }, { z2, z10 } and |
| // { z3, z11 } are also free. If so, we add { z2, z10 }. |
| |
| if (AssignedRegOp == Use.operands_end()) { |
| // There are no registers already assigned to any of the pseudo |
| // operands. Look for a valid starting register for the group. |
| for (unsigned I = 0; I < StridedOrder.size(); ++I) { |
| MCPhysReg Reg = StridedOrder[I]; |
| |
| // If the FORM_TRANSPOSE nodes use the ZPRMul classes, the starting |
| // register of the first load should be a multiple of 2 or 4. |
| unsigned SubRegIdx = Use.getOperand(OpIdx).getSubReg(); |
| if (IsMulZPR && (getSubReg(Reg, SubRegIdx) - AArch64::Z0) % UseOps != |
| ((unsigned)OpIdx - 1)) |
| continue; |
| |
| // In the example above, if VirtReg is the third operand of the |
| // tuple (%v2) and Reg == Z2_Z10, then we need to make sure that |
| // Z0_Z8, Z1_Z9 and Z3_Z11 are also available. |
| auto IsFreeConsecutiveReg = [&](unsigned UseOp) { |
| unsigned R = Reg - (OpIdx - 1) + UseOp; |
| return StridedRC->contains(R) && |
| (UseOp == 0 || |
| ((getSubReg(R, AArch64::zsub0) - AArch64::Z0) == |
| (getSubReg(R - 1, AArch64::zsub0) - AArch64::Z0) + 1)) && |
| !Matrix->isPhysRegUsed(R); |
| }; |
| if (all_of(iota_range<unsigned>(0U, UseOps, /*Inclusive=*/false), |
| IsFreeConsecutiveReg)) |
| Hints.push_back(Reg); |
| } |
| } else { |
| // At least one operand already has a physical register assigned. |
| // Find the starting sub-register of this and use it to work out the |
| // correct strided register to suggest based on the current op index. |
| MCPhysReg TargetStartReg = |
| getSubReg(VRM->getPhys(AssignedRegOp->getReg()), AArch64::zsub0) + |
| (OpIdx - AssignedRegOp->getOperandNo()); |
| |
| for (unsigned I = 0; I < StridedOrder.size(); ++I) |
| if (getSubReg(StridedOrder[I], AArch64::zsub0) == TargetStartReg) |
| Hints.push_back(StridedOrder[I]); |
| } |
| |
| if (!Hints.empty()) |
| return TargetRegisterInfo::getRegAllocationHints(VirtReg, Order, Hints, |
| MF, VRM); |
| } |
| } |
| |
| for (MachineInstr &MI : MRI.def_instructions(VirtReg)) { |
| if (MI.getOpcode() != AArch64::FORM_TRANSPOSED_REG_TUPLE_X2_PSEUDO && |
| MI.getOpcode() != AArch64::FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO) |
| return TargetRegisterInfo::getRegAllocationHints(VirtReg, Order, Hints, |
| MF, VRM); |
| |
| unsigned FirstOpSubReg = MI.getOperand(1).getSubReg(); |
| switch (FirstOpSubReg) { |
| case AArch64::zsub0: |
| case AArch64::zsub1: |
| case AArch64::zsub2: |
| case AArch64::zsub3: |
| break; |
| default: |
| continue; |
| } |
| |
| // Look up the physical register mapped to the first operand of the pseudo. |
| Register FirstOpVirtReg = MI.getOperand(1).getReg(); |
| if (!VRM->hasPhys(FirstOpVirtReg)) |
| continue; |
| |
| MCRegister TupleStartReg = |
| getSubReg(VRM->getPhys(FirstOpVirtReg), FirstOpSubReg); |
| for (unsigned I = 0; I < Order.size(); ++I) |
| if (MCRegister R = getSubReg(Order[I], AArch64::zsub0)) |
| if (R == TupleStartReg) |
| Hints.push_back(Order[I]); |
| } |
| |
| return TargetRegisterInfo::getRegAllocationHints(VirtReg, Order, Hints, MF, |
| VRM); |
| } |
| |
| unsigned AArch64RegisterInfo::getLocalAddressRegister( |
| const MachineFunction &MF) const { |
| const auto &MFI = MF.getFrameInfo(); |
| if (!MF.hasEHFunclets() && !MFI.hasVarSizedObjects()) |
| return AArch64::SP; |
| else if (hasStackRealignment(MF)) |
| return getBaseRegister(); |
| return getFrameRegister(MF); |
| } |
| |
| /// SrcRC and DstRC will be morphed into NewRC if this returns true |
| bool AArch64RegisterInfo::shouldCoalesce( |
| MachineInstr *MI, const TargetRegisterClass *SrcRC, unsigned SubReg, |
| const TargetRegisterClass *DstRC, unsigned DstSubReg, |
| const TargetRegisterClass *NewRC, LiveIntervals &LIS) const { |
| MachineRegisterInfo &MRI = MI->getMF()->getRegInfo(); |
| |
| if (MI->isCopy() && |
| ((DstRC->getID() == AArch64::GPR64RegClassID) || |
| (DstRC->getID() == AArch64::GPR64commonRegClassID)) && |
| MI->getOperand(0).getSubReg() && MI->getOperand(1).getSubReg()) |
| // Do not coalesce in the case of a 32-bit subregister copy |
| // which implements a 32 to 64 bit zero extension |
| // which relies on the upper 32 bits being zeroed. |
| return false; |
| |
| auto IsCoalescerBarrier = [](const MachineInstr &MI) { |
| switch (MI.getOpcode()) { |
| case AArch64::COALESCER_BARRIER_FPR16: |
| case AArch64::COALESCER_BARRIER_FPR32: |
| case AArch64::COALESCER_BARRIER_FPR64: |
| case AArch64::COALESCER_BARRIER_FPR128: |
| return true; |
| default: |
| return false; |
| } |
| }; |
| |
| // For calls that temporarily have to toggle streaming mode as part of the |
| // call-sequence, we need to be more careful when coalescing copy instructions |
| // so that we don't end up coalescing the NEON/FP result or argument register |
| // with a whole Z-register, such that after coalescing the register allocator |
| // will try to spill/reload the entire Z register. |
| // |
| // We do this by checking if the node has any defs/uses that are |
| // COALESCER_BARRIER pseudos. These are 'nops' in practice, but they exist to |
| // instruct the coalescer to avoid coalescing the copy. |
| if (MI->isCopy() && SubReg != DstSubReg && |
| (AArch64::ZPRRegClass.hasSubClassEq(DstRC) || |
| AArch64::ZPRRegClass.hasSubClassEq(SrcRC))) { |
| unsigned SrcReg = MI->getOperand(1).getReg(); |
| if (any_of(MRI.def_instructions(SrcReg), IsCoalescerBarrier)) |
| return false; |
| unsigned DstReg = MI->getOperand(0).getReg(); |
| if (any_of(MRI.use_nodbg_instructions(DstReg), IsCoalescerBarrier)) |
| return false; |
| } |
| |
| return true; |
| } |
| |
| bool AArch64RegisterInfo::shouldAnalyzePhysregInMachineLoopInfo( |
| MCRegister R) const { |
| return R == AArch64::VG; |
| } |
