| //=== X86CallingConv.cpp - X86 Custom Calling Convention Impl -*- C++ -*-===// |
| // |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| // See https://llvm.org/LICENSE.txt for license information. |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| // |
| //===----------------------------------------------------------------------===// |
| // |
| // This file contains the implementation of custom routines for the X86 |
| // Calling Convention that aren't done by tablegen. |
| // |
| //===----------------------------------------------------------------------===// |
| |
| #include "X86CallingConv.h" |
| #include "X86Subtarget.h" |
| #include "llvm/ADT/SmallVector.h" |
| #include "llvm/CodeGen/CallingConvLower.h" |
| #include "llvm/IR/CallingConv.h" |
| |
| using namespace llvm; |
| |
| /// When regcall calling convention compiled to 32 bit arch, special treatment |
| /// is required for 64 bit masks. |
| /// The value should be assigned to two GPRs. |
| /// \return true if registers were allocated and false otherwise. |
| static bool CC_X86_32_RegCall_Assign2Regs(unsigned &ValNo, MVT &ValVT, |
| MVT &LocVT, |
| CCValAssign::LocInfo &LocInfo, |
| ISD::ArgFlagsTy &ArgFlags, |
| CCState &State) { |
| // List of GPR registers that are available to store values in regcall |
| // calling convention. |
| static const MCPhysReg RegList[] = {X86::EAX, X86::ECX, X86::EDX, X86::EDI, |
| X86::ESI}; |
| |
| // The vector will save all the available registers for allocation. |
| SmallVector<unsigned, 5> AvailableRegs; |
| |
| // searching for the available registers. |
| for (auto Reg : RegList) { |
| if (!State.isAllocated(Reg)) |
| AvailableRegs.push_back(Reg); |
| } |
| |
| const size_t RequiredGprsUponSplit = 2; |
| if (AvailableRegs.size() < RequiredGprsUponSplit) |
| return false; // Not enough free registers - continue the search. |
| |
| // Allocating the available registers. |
| for (unsigned I = 0; I < RequiredGprsUponSplit; I++) { |
| |
| // Marking the register as located. |
| unsigned Reg = State.AllocateReg(AvailableRegs[I]); |
| |
| // Since we previously made sure that 2 registers are available |
| // we expect that a real register number will be returned. |
| assert(Reg && "Expecting a register will be available"); |
| |
| // Assign the value to the allocated register |
| State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT, LocInfo)); |
| } |
| |
| // Successful in allocating registers - stop scanning next rules. |
| return true; |
| } |
| |
| static ArrayRef<MCPhysReg> CC_X86_VectorCallGetSSEs(const MVT &ValVT) { |
| if (ValVT.is512BitVector()) { |
| static const MCPhysReg RegListZMM[] = {X86::ZMM0, X86::ZMM1, X86::ZMM2, |
| X86::ZMM3, X86::ZMM4, X86::ZMM5}; |
| return makeArrayRef(std::begin(RegListZMM), std::end(RegListZMM)); |
| } |
| |
| if (ValVT.is256BitVector()) { |
| static const MCPhysReg RegListYMM[] = {X86::YMM0, X86::YMM1, X86::YMM2, |
| X86::YMM3, X86::YMM4, X86::YMM5}; |
| return makeArrayRef(std::begin(RegListYMM), std::end(RegListYMM)); |
| } |
| |
| static const MCPhysReg RegListXMM[] = {X86::XMM0, X86::XMM1, X86::XMM2, |
| X86::XMM3, X86::XMM4, X86::XMM5}; |
| return makeArrayRef(std::begin(RegListXMM), std::end(RegListXMM)); |
| } |
| |
| static ArrayRef<MCPhysReg> CC_X86_64_VectorCallGetGPRs() { |
| static const MCPhysReg RegListGPR[] = {X86::RCX, X86::RDX, X86::R8, X86::R9}; |
| return makeArrayRef(std::begin(RegListGPR), std::end(RegListGPR)); |
| } |
| |
| static bool CC_X86_VectorCallAssignRegister(unsigned &ValNo, MVT &ValVT, |
| MVT &LocVT, |
| CCValAssign::LocInfo &LocInfo, |
| ISD::ArgFlagsTy &ArgFlags, |
| CCState &State) { |
| |
| ArrayRef<MCPhysReg> RegList = CC_X86_VectorCallGetSSEs(ValVT); |
| bool Is64bit = static_cast<const X86Subtarget &>( |
| State.getMachineFunction().getSubtarget()) |
| .is64Bit(); |
| |
| for (auto Reg : RegList) { |
| // If the register is not marked as allocated - assign to it. |
| if (!State.isAllocated(Reg)) { |
| unsigned AssigedReg = State.AllocateReg(Reg); |
| assert(AssigedReg == Reg && "Expecting a valid register allocation"); |
| State.addLoc( |
| CCValAssign::getReg(ValNo, ValVT, AssigedReg, LocVT, LocInfo)); |
| return true; |
| } |
| // If the register is marked as shadow allocated - assign to it. |
| if (Is64bit && State.IsShadowAllocatedReg(Reg)) { |
| State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo)); |
| return true; |
| } |
| } |
| |
| llvm_unreachable("Clang should ensure that hva marked vectors will have " |
| "an available register."); |
| return false; |
| } |
| |
| /// Vectorcall calling convention has special handling for vector types or |
| /// HVA for 64 bit arch. |
| /// For HVAs shadow registers might be allocated on the first pass |
| /// and actual XMM registers are allocated on the second pass. |
| /// For vector types, actual XMM registers are allocated on the first pass. |
| /// \return true if registers were allocated and false otherwise. |
| static bool CC_X86_64_VectorCall(unsigned &ValNo, MVT &ValVT, MVT &LocVT, |
| CCValAssign::LocInfo &LocInfo, |
| ISD::ArgFlagsTy &ArgFlags, CCState &State) { |
| // On the second pass, go through the HVAs only. |
| if (ArgFlags.isSecArgPass()) { |
| if (ArgFlags.isHva()) |
| return CC_X86_VectorCallAssignRegister(ValNo, ValVT, LocVT, LocInfo, |
| ArgFlags, State); |
| return true; |
| } |
| |
| // Process only vector types as defined by vectorcall spec: |
| // "A vector type is either a floating-point type, for example, |
| // a float or double, or an SIMD vector type, for example, __m128 or __m256". |
| if (!(ValVT.isFloatingPoint() || |
| (ValVT.isVector() && ValVT.getSizeInBits() >= 128))) { |
| // If R9 was already assigned it means that we are after the fourth element |
| // and because this is not an HVA / Vector type, we need to allocate |
| // shadow XMM register. |
| if (State.isAllocated(X86::R9)) { |
| // Assign shadow XMM register. |
| (void)State.AllocateReg(CC_X86_VectorCallGetSSEs(ValVT)); |
| } |
| |
| return false; |
| } |
| |
| if (!ArgFlags.isHva() || ArgFlags.isHvaStart()) { |
| // Assign shadow GPR register. |
| (void)State.AllocateReg(CC_X86_64_VectorCallGetGPRs()); |
| |
| // Assign XMM register - (shadow for HVA and non-shadow for non HVA). |
| if (unsigned Reg = State.AllocateReg(CC_X86_VectorCallGetSSEs(ValVT))) { |
| // In Vectorcall Calling convention, additional shadow stack can be |
| // created on top of the basic 32 bytes of win64. |
| // It can happen if the fifth or sixth argument is vector type or HVA. |
| // At that case for each argument a shadow stack of 8 bytes is allocated. |
| const TargetRegisterInfo *TRI = |
| State.getMachineFunction().getSubtarget().getRegisterInfo(); |
| if (TRI->regsOverlap(Reg, X86::XMM4) || |
| TRI->regsOverlap(Reg, X86::XMM5)) |
| State.AllocateStack(8, Align(8)); |
| |
| if (!ArgFlags.isHva()) { |
| State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo)); |
| return true; // Allocated a register - Stop the search. |
| } |
| } |
| } |
| |
| // If this is an HVA - Stop the search, |
| // otherwise continue the search. |
| return ArgFlags.isHva(); |
| } |
| |
| /// Vectorcall calling convention has special handling for vector types or |
| /// HVA for 32 bit arch. |
| /// For HVAs actual XMM registers are allocated on the second pass. |
| /// For vector types, actual XMM registers are allocated on the first pass. |
| /// \return true if registers were allocated and false otherwise. |
| static bool CC_X86_32_VectorCall(unsigned &ValNo, MVT &ValVT, MVT &LocVT, |
| CCValAssign::LocInfo &LocInfo, |
| ISD::ArgFlagsTy &ArgFlags, CCState &State) { |
| // On the second pass, go through the HVAs only. |
| if (ArgFlags.isSecArgPass()) { |
| if (ArgFlags.isHva()) |
| return CC_X86_VectorCallAssignRegister(ValNo, ValVT, LocVT, LocInfo, |
| ArgFlags, State); |
| return true; |
| } |
| |
| // Process only vector types as defined by vectorcall spec: |
| // "A vector type is either a floating point type, for example, |
| // a float or double, or an SIMD vector type, for example, __m128 or __m256". |
| if (!(ValVT.isFloatingPoint() || |
| (ValVT.isVector() && ValVT.getSizeInBits() >= 128))) { |
| return false; |
| } |
| |
| if (ArgFlags.isHva()) |
| return true; // If this is an HVA - Stop the search. |
| |
| // Assign XMM register. |
| if (unsigned Reg = State.AllocateReg(CC_X86_VectorCallGetSSEs(ValVT))) { |
| State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo)); |
| return true; |
| } |
| |
| // In case we did not find an available XMM register for a vector - |
| // pass it indirectly. |
| // It is similar to CCPassIndirect, with the addition of inreg. |
| if (!ValVT.isFloatingPoint()) { |
| LocVT = MVT::i32; |
| LocInfo = CCValAssign::Indirect; |
| ArgFlags.setInReg(); |
| } |
| |
| return false; // No register was assigned - Continue the search. |
| } |
| |
| static bool CC_X86_AnyReg_Error(unsigned &, MVT &, MVT &, |
| CCValAssign::LocInfo &, ISD::ArgFlagsTy &, |
| CCState &) { |
| llvm_unreachable("The AnyReg calling convention is only supported by the " |
| "stackmap and patchpoint intrinsics."); |
| // gracefully fallback to X86 C calling convention on Release builds. |
| return false; |
| } |
| |
| static bool CC_X86_32_MCUInReg(unsigned &ValNo, MVT &ValVT, MVT &LocVT, |
| CCValAssign::LocInfo &LocInfo, |
| ISD::ArgFlagsTy &ArgFlags, CCState &State) { |
| // This is similar to CCAssignToReg<[EAX, EDX, ECX]>, but makes sure |
| // not to split i64 and double between a register and stack |
| static const MCPhysReg RegList[] = {X86::EAX, X86::EDX, X86::ECX}; |
| static const unsigned NumRegs = sizeof(RegList) / sizeof(RegList[0]); |
| |
| SmallVectorImpl<CCValAssign> &PendingMembers = State.getPendingLocs(); |
| |
| // If this is the first part of an double/i64/i128, or if we're already |
| // in the middle of a split, add to the pending list. If this is not |
| // the end of the split, return, otherwise go on to process the pending |
| // list |
| if (ArgFlags.isSplit() || !PendingMembers.empty()) { |
| PendingMembers.push_back( |
| CCValAssign::getPending(ValNo, ValVT, LocVT, LocInfo)); |
| if (!ArgFlags.isSplitEnd()) |
| return true; |
| } |
| |
| // If there are no pending members, we are not in the middle of a split, |
| // so do the usual inreg stuff. |
| if (PendingMembers.empty()) { |
| if (unsigned Reg = State.AllocateReg(RegList)) { |
| State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo)); |
| return true; |
| } |
| return false; |
| } |
| |
| assert(ArgFlags.isSplitEnd()); |
| |
| // We now have the entire original argument in PendingMembers, so decide |
| // whether to use registers or the stack. |
| // Per the MCU ABI: |
| // a) To use registers, we need to have enough of them free to contain |
| // the entire argument. |
| // b) We never want to use more than 2 registers for a single argument. |
| |
| unsigned FirstFree = State.getFirstUnallocated(RegList); |
| bool UseRegs = PendingMembers.size() <= std::min(2U, NumRegs - FirstFree); |
| |
| for (auto &It : PendingMembers) { |
| if (UseRegs) |
| It.convertToReg(State.AllocateReg(RegList[FirstFree++])); |
| else |
| It.convertToMem(State.AllocateStack(4, Align(4))); |
| State.addLoc(It); |
| } |
| |
| PendingMembers.clear(); |
| |
| return true; |
| } |
| |
| /// X86 interrupt handlers can only take one or two stack arguments, but if |
| /// there are two arguments, they are in the opposite order from the standard |
| /// convention. Therefore, we have to look at the argument count up front before |
| /// allocating stack for each argument. |
| static bool CC_X86_Intr(unsigned &ValNo, MVT &ValVT, MVT &LocVT, |
| CCValAssign::LocInfo &LocInfo, |
| ISD::ArgFlagsTy &ArgFlags, CCState &State) { |
| const MachineFunction &MF = State.getMachineFunction(); |
| size_t ArgCount = State.getMachineFunction().getFunction().arg_size(); |
| bool Is64Bit = static_cast<const X86Subtarget &>(MF.getSubtarget()).is64Bit(); |
| unsigned SlotSize = Is64Bit ? 8 : 4; |
| unsigned Offset; |
| if (ArgCount == 1 && ValNo == 0) { |
| // If we have one argument, the argument is five stack slots big, at fixed |
| // offset zero. |
| Offset = State.AllocateStack(5 * SlotSize, Align(4)); |
| } else if (ArgCount == 2 && ValNo == 0) { |
| // If we have two arguments, the stack slot is *after* the error code |
| // argument. Pretend it doesn't consume stack space, and account for it when |
| // we assign the second argument. |
| Offset = SlotSize; |
| } else if (ArgCount == 2 && ValNo == 1) { |
| // If this is the second of two arguments, it must be the error code. It |
| // appears first on the stack, and is then followed by the five slot |
| // interrupt struct. |
| Offset = 0; |
| (void)State.AllocateStack(6 * SlotSize, Align(4)); |
| } else { |
| report_fatal_error("unsupported x86 interrupt prototype"); |
| } |
| |
| // FIXME: This should be accounted for in |
| // X86FrameLowering::getFrameIndexReference, not here. |
| if (Is64Bit && ArgCount == 2) |
| Offset += SlotSize; |
| |
| State.addLoc(CCValAssign::getMem(ValNo, ValVT, Offset, LocVT, LocInfo)); |
| return true; |
| } |
| |
| static bool CC_X86_64_Pointer(unsigned &ValNo, MVT &ValVT, MVT &LocVT, |
| CCValAssign::LocInfo &LocInfo, |
| ISD::ArgFlagsTy &ArgFlags, CCState &State) { |
| if (LocVT != MVT::i64) { |
| LocVT = MVT::i64; |
| LocInfo = CCValAssign::ZExt; |
| } |
| return false; |
| } |
| |
| // Provides entry points of CC_X86 and RetCC_X86. |
| #include "X86GenCallingConv.inc" |