blob: 7a58e9ae0692a93cc882374c4ab0cec4f2d1e98d [file] [log] [blame]
//=== X86CallingConv.cpp - X86 Custom Calling Convention Impl -*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file contains the implementation of custom routines for the X86
// Calling Convention that aren't done by tablegen.
//
//===----------------------------------------------------------------------===//
#include "X86CallingConv.h"
#include "X86Subtarget.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/CodeGen/CallingConvLower.h"
#include "llvm/IR/CallingConv.h"
using namespace llvm;
/// When regcall calling convention compiled to 32 bit arch, special treatment
/// is required for 64 bit masks.
/// The value should be assigned to two GPRs.
/// \return true if registers were allocated and false otherwise.
static bool CC_X86_32_RegCall_Assign2Regs(unsigned &ValNo, MVT &ValVT,
MVT &LocVT,
CCValAssign::LocInfo &LocInfo,
ISD::ArgFlagsTy &ArgFlags,
CCState &State) {
// List of GPR registers that are available to store values in regcall
// calling convention.
static const MCPhysReg RegList[] = {X86::EAX, X86::ECX, X86::EDX, X86::EDI,
X86::ESI};
// The vector will save all the available registers for allocation.
SmallVector<unsigned, 5> AvailableRegs;
// searching for the available registers.
for (auto Reg : RegList) {
if (!State.isAllocated(Reg))
AvailableRegs.push_back(Reg);
}
const size_t RequiredGprsUponSplit = 2;
if (AvailableRegs.size() < RequiredGprsUponSplit)
return false; // Not enough free registers - continue the search.
// Allocating the available registers.
for (unsigned I = 0; I < RequiredGprsUponSplit; I++) {
// Marking the register as located.
unsigned Reg = State.AllocateReg(AvailableRegs[I]);
// Since we previously made sure that 2 registers are available
// we expect that a real register number will be returned.
assert(Reg && "Expecting a register will be available");
// Assign the value to the allocated register
State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT, LocInfo));
}
// Successful in allocating regsiters - stop scanning next rules.
return true;
}
static ArrayRef<MCPhysReg> CC_X86_VectorCallGetSSEs(const MVT &ValVT) {
if (ValVT.is512BitVector()) {
static const MCPhysReg RegListZMM[] = {X86::ZMM0, X86::ZMM1, X86::ZMM2,
X86::ZMM3, X86::ZMM4, X86::ZMM5};
return makeArrayRef(std::begin(RegListZMM), std::end(RegListZMM));
}
if (ValVT.is256BitVector()) {
static const MCPhysReg RegListYMM[] = {X86::YMM0, X86::YMM1, X86::YMM2,
X86::YMM3, X86::YMM4, X86::YMM5};
return makeArrayRef(std::begin(RegListYMM), std::end(RegListYMM));
}
static const MCPhysReg RegListXMM[] = {X86::XMM0, X86::XMM1, X86::XMM2,
X86::XMM3, X86::XMM4, X86::XMM5};
return makeArrayRef(std::begin(RegListXMM), std::end(RegListXMM));
}
static ArrayRef<MCPhysReg> CC_X86_64_VectorCallGetGPRs() {
static const MCPhysReg RegListGPR[] = {X86::RCX, X86::RDX, X86::R8, X86::R9};
return makeArrayRef(std::begin(RegListGPR), std::end(RegListGPR));
}
static bool CC_X86_VectorCallAssignRegister(unsigned &ValNo, MVT &ValVT,
MVT &LocVT,
CCValAssign::LocInfo &LocInfo,
ISD::ArgFlagsTy &ArgFlags,
CCState &State) {
ArrayRef<MCPhysReg> RegList = CC_X86_VectorCallGetSSEs(ValVT);
bool Is64bit = static_cast<const X86Subtarget &>(
State.getMachineFunction().getSubtarget())
.is64Bit();
for (auto Reg : RegList) {
// If the register is not marked as allocated - assign to it.
if (!State.isAllocated(Reg)) {
unsigned AssigedReg = State.AllocateReg(Reg);
assert(AssigedReg == Reg && "Expecting a valid register allocation");
State.addLoc(
CCValAssign::getReg(ValNo, ValVT, AssigedReg, LocVT, LocInfo));
return true;
}
// If the register is marked as shadow allocated - assign to it.
if (Is64bit && State.IsShadowAllocatedReg(Reg)) {
State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
return true;
}
}
llvm_unreachable("Clang should ensure that hva marked vectors will have "
"an available register.");
return false;
}
/// Vectorcall calling convention has special handling for vector types or
/// HVA for 64 bit arch.
/// For HVAs shadow registers might be allocated on the first pass
/// and actual XMM registers are allocated on the second pass.
/// For vector types, actual XMM registers are allocated on the first pass.
/// \return true if registers were allocated and false otherwise.
static bool CC_X86_64_VectorCall(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
CCValAssign::LocInfo &LocInfo,
ISD::ArgFlagsTy &ArgFlags, CCState &State) {
// On the second pass, go through the HVAs only.
if (ArgFlags.isSecArgPass()) {
if (ArgFlags.isHva())
return CC_X86_VectorCallAssignRegister(ValNo, ValVT, LocVT, LocInfo,
ArgFlags, State);
return true;
}
// Process only vector types as defined by vectorcall spec:
// "A vector type is either a floating-point type, for example,
// a float or double, or an SIMD vector type, for example, __m128 or __m256".
if (!(ValVT.isFloatingPoint() ||
(ValVT.isVector() && ValVT.getSizeInBits() >= 128))) {
// If R9 was already assigned it means that we are after the fourth element
// and because this is not an HVA / Vector type, we need to allocate
// shadow XMM register.
if (State.isAllocated(X86::R9)) {
// Assign shadow XMM register.
(void)State.AllocateReg(CC_X86_VectorCallGetSSEs(ValVT));
}
return false;
}
if (!ArgFlags.isHva() || ArgFlags.isHvaStart()) {
// Assign shadow GPR register.
(void)State.AllocateReg(CC_X86_64_VectorCallGetGPRs());
// Assign XMM register - (shadow for HVA and non-shadow for non HVA).
if (unsigned Reg = State.AllocateReg(CC_X86_VectorCallGetSSEs(ValVT))) {
// In Vectorcall Calling convention, additional shadow stack can be
// created on top of the basic 32 bytes of win64.
// It can happen if the fifth or sixth argument is vector type or HVA.
// At that case for each argument a shadow stack of 8 bytes is allocated.
if (Reg == X86::XMM4 || Reg == X86::XMM5)
State.AllocateStack(8, 8);
if (!ArgFlags.isHva()) {
State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
return true; // Allocated a register - Stop the search.
}
}
}
// If this is an HVA - Stop the search,
// otherwise continue the search.
return ArgFlags.isHva();
}
/// Vectorcall calling convention has special handling for vector types or
/// HVA for 32 bit arch.
/// For HVAs actual XMM registers are allocated on the second pass.
/// For vector types, actual XMM registers are allocated on the first pass.
/// \return true if registers were allocated and false otherwise.
static bool CC_X86_32_VectorCall(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
CCValAssign::LocInfo &LocInfo,
ISD::ArgFlagsTy &ArgFlags, CCState &State) {
// On the second pass, go through the HVAs only.
if (ArgFlags.isSecArgPass()) {
if (ArgFlags.isHva())
return CC_X86_VectorCallAssignRegister(ValNo, ValVT, LocVT, LocInfo,
ArgFlags, State);
return true;
}
// Process only vector types as defined by vectorcall spec:
// "A vector type is either a floating point type, for example,
// a float or double, or an SIMD vector type, for example, __m128 or __m256".
if (!(ValVT.isFloatingPoint() ||
(ValVT.isVector() && ValVT.getSizeInBits() >= 128))) {
return false;
}
if (ArgFlags.isHva())
return true; // If this is an HVA - Stop the search.
// Assign XMM register.
if (unsigned Reg = State.AllocateReg(CC_X86_VectorCallGetSSEs(ValVT))) {
State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
return true;
}
// In case we did not find an available XMM register for a vector -
// pass it indirectly.
// It is similar to CCPassIndirect, with the addition of inreg.
if (!ValVT.isFloatingPoint()) {
LocVT = MVT::i32;
LocInfo = CCValAssign::Indirect;
ArgFlags.setInReg();
}
return false; // No register was assigned - Continue the search.
}
static bool CC_X86_AnyReg_Error(unsigned &, MVT &, MVT &,
CCValAssign::LocInfo &, ISD::ArgFlagsTy &,
CCState &) {
llvm_unreachable("The AnyReg calling convention is only supported by the "
"stackmap and patchpoint intrinsics.");
// gracefully fallback to X86 C calling convention on Release builds.
return false;
}
static bool CC_X86_32_MCUInReg(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
CCValAssign::LocInfo &LocInfo,
ISD::ArgFlagsTy &ArgFlags, CCState &State) {
// This is similar to CCAssignToReg<[EAX, EDX, ECX]>, but makes sure
// not to split i64 and double between a register and stack
static const MCPhysReg RegList[] = {X86::EAX, X86::EDX, X86::ECX};
static const unsigned NumRegs = sizeof(RegList) / sizeof(RegList[0]);
SmallVectorImpl<CCValAssign> &PendingMembers = State.getPendingLocs();
// If this is the first part of an double/i64/i128, or if we're already
// in the middle of a split, add to the pending list. If this is not
// the end of the split, return, otherwise go on to process the pending
// list
if (ArgFlags.isSplit() || !PendingMembers.empty()) {
PendingMembers.push_back(
CCValAssign::getPending(ValNo, ValVT, LocVT, LocInfo));
if (!ArgFlags.isSplitEnd())
return true;
}
// If there are no pending members, we are not in the middle of a split,
// so do the usual inreg stuff.
if (PendingMembers.empty()) {
if (unsigned Reg = State.AllocateReg(RegList)) {
State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
return true;
}
return false;
}
assert(ArgFlags.isSplitEnd());
// We now have the entire original argument in PendingMembers, so decide
// whether to use registers or the stack.
// Per the MCU ABI:
// a) To use registers, we need to have enough of them free to contain
// the entire argument.
// b) We never want to use more than 2 registers for a single argument.
unsigned FirstFree = State.getFirstUnallocated(RegList);
bool UseRegs = PendingMembers.size() <= std::min(2U, NumRegs - FirstFree);
for (auto &It : PendingMembers) {
if (UseRegs)
It.convertToReg(State.AllocateReg(RegList[FirstFree++]));
else
It.convertToMem(State.AllocateStack(4, 4));
State.addLoc(It);
}
PendingMembers.clear();
return true;
}
// Provides entry points of CC_X86 and RetCC_X86.
#include "X86GenCallingConv.inc"