llvm/lib/Target/X86/X86CallingConv.cpp - llvm-project - Git at Google

 //=== X86CallingConv.cpp - X86 Custom Calling Convention Impl   -*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file contains the implementation of custom routines for the X86
 // Calling Convention that aren't done by tablegen.
 //
 //===----------------------------------------------------------------------===//

 #include "X86CallingConv.h"
 #include "X86Subtarget.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/CodeGen/CallingConvLower.h"
 #include "llvm/IR/CallingConv.h"

 using namespace llvm;

 /// When regcall calling convention compiled to 32 bit arch, special treatment
 /// is required for 64 bit masks.
 /// The value should be assigned to two GPRs.
 /// \return true if registers were allocated and false otherwise.
 static bool CC_X86_32_RegCall_Assign2Regs(unsigned &ValNo, MVT &ValVT,
                                           MVT &LocVT,
                                           CCValAssign::LocInfo &LocInfo,
                                           ISD::ArgFlagsTy &ArgFlags,
                                           CCState &State) {
   // List of GPR registers that are available to store values in regcall
   // calling convention.
   static const MCPhysReg RegList[] = {X86::EAX, X86::ECX, X86::EDX, X86::EDI,
                                       X86::ESI};

   // The vector will save all the available registers for allocation.
   SmallVector<unsigned, 5> AvailableRegs;

   // searching for the available registers.
   for (auto Reg : RegList) {
     if (!State.isAllocated(Reg))
       AvailableRegs.push_back(Reg);
   }

   const size_t RequiredGprsUponSplit = 2;
   if (AvailableRegs.size() < RequiredGprsUponSplit)
     return false; // Not enough free registers - continue the search.

   // Allocating the available registers.
   for (unsigned I = 0; I < RequiredGprsUponSplit; I++) {

     // Marking the register as located.
     unsigned Reg = State.AllocateReg(AvailableRegs[I]);

     // Since we previously made sure that 2 registers are available
     // we expect that a real register number will be returned.
     assert(Reg && "Expecting a register will be available");

     // Assign the value to the allocated register
     State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT, LocInfo));
   }

   // Successful in allocating registers - stop scanning next rules.
   return true;
 }

 static ArrayRef<MCPhysReg> CC_X86_VectorCallGetSSEs(const MVT &ValVT) {
   if (ValVT.is512BitVector()) {
     static const MCPhysReg RegListZMM[] = {X86::ZMM0, X86::ZMM1, X86::ZMM2,
                                            X86::ZMM3, X86::ZMM4, X86::ZMM5};
     return makeArrayRef(std::begin(RegListZMM), std::end(RegListZMM));
   }

   if (ValVT.is256BitVector()) {
     static const MCPhysReg RegListYMM[] = {X86::YMM0, X86::YMM1, X86::YMM2,
                                            X86::YMM3, X86::YMM4, X86::YMM5};
     return makeArrayRef(std::begin(RegListYMM), std::end(RegListYMM));
   }

   static const MCPhysReg RegListXMM[] = {X86::XMM0, X86::XMM1, X86::XMM2,
                                          X86::XMM3, X86::XMM4, X86::XMM5};
   return makeArrayRef(std::begin(RegListXMM), std::end(RegListXMM));
 }

 static ArrayRef<MCPhysReg> CC_X86_64_VectorCallGetGPRs() {
   static const MCPhysReg RegListGPR[] = {X86::RCX, X86::RDX, X86::R8, X86::R9};
   return makeArrayRef(std::begin(RegListGPR), std::end(RegListGPR));
 }

 static bool CC_X86_VectorCallAssignRegister(unsigned &ValNo, MVT &ValVT,
                                             MVT &LocVT,
                                             CCValAssign::LocInfo &LocInfo,
                                             ISD::ArgFlagsTy &ArgFlags,
                                             CCState &State) {

   ArrayRef<MCPhysReg> RegList = CC_X86_VectorCallGetSSEs(ValVT);
   bool Is64bit = static_cast<const X86Subtarget &>(
                      State.getMachineFunction().getSubtarget())
                      .is64Bit();

   for (auto Reg : RegList) {
     // If the register is not marked as allocated - assign to it.
     if (!State.isAllocated(Reg)) {
       unsigned AssigedReg = State.AllocateReg(Reg);
       assert(AssigedReg == Reg && "Expecting a valid register allocation");
       State.addLoc(
           CCValAssign::getReg(ValNo, ValVT, AssigedReg, LocVT, LocInfo));
       return true;
     }
     // If the register is marked as shadow allocated - assign to it.
     if (Is64bit && State.IsShadowAllocatedReg(Reg)) {
       State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
       return true;
     }
   }

   llvm_unreachable("Clang should ensure that hva marked vectors will have "
                    "an available register.");
   return false;
 }

 /// Vectorcall calling convention has special handling for vector types or
 /// HVA for 64 bit arch.
 /// For HVAs shadow registers might be allocated on the first pass
 /// and actual XMM registers are allocated on the second pass.
 /// For vector types, actual XMM registers are allocated on the first pass.
 /// \return true if registers were allocated and false otherwise.
 static bool CC_X86_64_VectorCall(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
                                  CCValAssign::LocInfo &LocInfo,
                                  ISD::ArgFlagsTy &ArgFlags, CCState &State) {
   // On the second pass, go through the HVAs only.
   if (ArgFlags.isSecArgPass()) {
     if (ArgFlags.isHva())
       return CC_X86_VectorCallAssignRegister(ValNo, ValVT, LocVT, LocInfo,
                                              ArgFlags, State);
     return true;
   }

   // Process only vector types as defined by vectorcall spec:
   // "A vector type is either a floating-point type, for example,
   //  a float or double, or an SIMD vector type, for example, __m128 or __m256".
   if (!(ValVT.isFloatingPoint() ||
         (ValVT.isVector() && ValVT.getSizeInBits() >= 128))) {
     // If R9 was already assigned it means that we are after the fourth element
     // and because this is not an HVA / Vector type, we need to allocate
     // shadow XMM register.
     if (State.isAllocated(X86::R9)) {
       // Assign shadow XMM register.
       (void)State.AllocateReg(CC_X86_VectorCallGetSSEs(ValVT));
     }

     return false;
   }

   if (!ArgFlags.isHva() || ArgFlags.isHvaStart()) {
     // Assign shadow GPR register.
     (void)State.AllocateReg(CC_X86_64_VectorCallGetGPRs());

     // Assign XMM register - (shadow for HVA and non-shadow for non HVA).
     if (unsigned Reg = State.AllocateReg(CC_X86_VectorCallGetSSEs(ValVT))) {
       // In Vectorcall Calling convention, additional shadow stack can be
       // created on top of the basic 32 bytes of win64.
       // It can happen if the fifth or sixth argument is vector type or HVA.
       // At that case for each argument a shadow stack of 8 bytes is allocated.
       const TargetRegisterInfo *TRI =
           State.getMachineFunction().getSubtarget().getRegisterInfo();
       if (TRI->regsOverlap(Reg, X86::XMM4) ||
           TRI->regsOverlap(Reg, X86::XMM5))
         State.AllocateStack(8, Align(8));

       if (!ArgFlags.isHva()) {
         State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
         return true; // Allocated a register - Stop the search.
       }
     }
   }

   // If this is an HVA - Stop the search,
   // otherwise continue the search.
   return ArgFlags.isHva();
 }

 /// Vectorcall calling convention has special handling for vector types or
 /// HVA for 32 bit arch.
 /// For HVAs actual XMM registers are allocated on the second pass.
 /// For vector types, actual XMM registers are allocated on the first pass.
 /// \return true if registers were allocated and false otherwise.
 static bool CC_X86_32_VectorCall(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
                                  CCValAssign::LocInfo &LocInfo,
                                  ISD::ArgFlagsTy &ArgFlags, CCState &State) {
   // On the second pass, go through the HVAs only.
   if (ArgFlags.isSecArgPass()) {
     if (ArgFlags.isHva())
       return CC_X86_VectorCallAssignRegister(ValNo, ValVT, LocVT, LocInfo,
                                              ArgFlags, State);
     return true;
   }

   // Process only vector types as defined by vectorcall spec:
   // "A vector type is either a floating point type, for example,
   //  a float or double, or an SIMD vector type, for example, __m128 or __m256".
   if (!(ValVT.isFloatingPoint() ||
         (ValVT.isVector() && ValVT.getSizeInBits() >= 128))) {
     return false;
   }

   if (ArgFlags.isHva())
     return true; // If this is an HVA - Stop the search.

   // Assign XMM register.
   if (unsigned Reg = State.AllocateReg(CC_X86_VectorCallGetSSEs(ValVT))) {
     State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
     return true;
   }

   // In case we did not find an available XMM register for a vector -
   // pass it indirectly.
   // It is similar to CCPassIndirect, with the addition of inreg.
   if (!ValVT.isFloatingPoint()) {
     LocVT = MVT::i32;
     LocInfo = CCValAssign::Indirect;
     ArgFlags.setInReg();
   }

   return false; // No register was assigned - Continue the search.
 }

 static bool CC_X86_AnyReg_Error(unsigned &, MVT &, MVT &,
                                 CCValAssign::LocInfo &, ISD::ArgFlagsTy &,
                                 CCState &) {
   llvm_unreachable("The AnyReg calling convention is only supported by the "
                    "stackmap and patchpoint intrinsics.");
   // gracefully fallback to X86 C calling convention on Release builds.
   return false;
 }

 static bool CC_X86_32_MCUInReg(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
                                CCValAssign::LocInfo &LocInfo,
                                ISD::ArgFlagsTy &ArgFlags, CCState &State) {
   // This is similar to CCAssignToReg<[EAX, EDX, ECX]>, but makes sure
   // not to split i64 and double between a register and stack
   static const MCPhysReg RegList[] = {X86::EAX, X86::EDX, X86::ECX};
   static const unsigned NumRegs = sizeof(RegList) / sizeof(RegList[0]);

   SmallVectorImpl<CCValAssign> &PendingMembers = State.getPendingLocs();

   // If this is the first part of an double/i64/i128, or if we're already
   // in the middle of a split, add to the pending list. If this is not
   // the end of the split, return, otherwise go on to process the pending
   // list
   if (ArgFlags.isSplit() || !PendingMembers.empty()) {
     PendingMembers.push_back(
         CCValAssign::getPending(ValNo, ValVT, LocVT, LocInfo));
     if (!ArgFlags.isSplitEnd())
       return true;
   }

   // If there are no pending members, we are not in the middle of a split,
   // so do the usual inreg stuff.
   if (PendingMembers.empty()) {
     if (unsigned Reg = State.AllocateReg(RegList)) {
       State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
       return true;
     }
     return false;
   }

   assert(ArgFlags.isSplitEnd());

   // We now have the entire original argument in PendingMembers, so decide
   // whether to use registers or the stack.
   // Per the MCU ABI:
   // a) To use registers, we need to have enough of them free to contain
   // the entire argument.
   // b) We never want to use more than 2 registers for a single argument.

   unsigned FirstFree = State.getFirstUnallocated(RegList);
   bool UseRegs = PendingMembers.size() <= std::min(2U, NumRegs - FirstFree);

   for (auto &It : PendingMembers) {
     if (UseRegs)
       It.convertToReg(State.AllocateReg(RegList[FirstFree++]));
     else
       It.convertToMem(State.AllocateStack(4, Align(4)));
     State.addLoc(It);
   }

   PendingMembers.clear();

   return true;
 }

 /// X86 interrupt handlers can only take one or two stack arguments, but if
 /// there are two arguments, they are in the opposite order from the standard
 /// convention. Therefore, we have to look at the argument count up front before
 /// allocating stack for each argument.
 static bool CC_X86_Intr(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
                         CCValAssign::LocInfo &LocInfo,
                         ISD::ArgFlagsTy &ArgFlags, CCState &State) {
   const MachineFunction &MF = State.getMachineFunction();
   size_t ArgCount = State.getMachineFunction().getFunction().arg_size();
   bool Is64Bit = static_cast<const X86Subtarget &>(MF.getSubtarget()).is64Bit();
   unsigned SlotSize = Is64Bit ? 8 : 4;
   unsigned Offset;
   if (ArgCount == 1 && ValNo == 0) {
     // If we have one argument, the argument is five stack slots big, at fixed
     // offset zero.
     Offset = State.AllocateStack(5 * SlotSize, Align(4));
   } else if (ArgCount == 2 && ValNo == 0) {
     // If we have two arguments, the stack slot is *after* the error code
     // argument. Pretend it doesn't consume stack space, and account for it when
     // we assign the second argument.
     Offset = SlotSize;
   } else if (ArgCount == 2 && ValNo == 1) {
     // If this is the second of two arguments, it must be the error code. It
     // appears first on the stack, and is then followed by the five slot
     // interrupt struct.
     Offset = 0;
     (void)State.AllocateStack(6 * SlotSize, Align(4));
   } else {
     report_fatal_error("unsupported x86 interrupt prototype");
   }

   // FIXME: This should be accounted for in
   // X86FrameLowering::getFrameIndexReference, not here.
   if (Is64Bit && ArgCount == 2)
     Offset += SlotSize;

   State.addLoc(CCValAssign::getMem(ValNo, ValVT, Offset, LocVT, LocInfo));
   return true;
 }

 static bool CC_X86_64_Pointer(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
                               CCValAssign::LocInfo &LocInfo,
                               ISD::ArgFlagsTy &ArgFlags, CCState &State) {
   if (LocVT != MVT::i64) {
     LocVT = MVT::i64;
     LocInfo = CCValAssign::ZExt;
   }
   return false;
 }

 // Provides entry points of CC_X86 and RetCC_X86.
 #include "X86GenCallingConv.inc"
	//=== X86CallingConv.cpp - X86 Custom Calling Convention Impl -- C++ --===//
	//
	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	// See https://llvm.org/LICENSE.txt for license information.
	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	//
	//===----------------------------------------------------------------------===//
	//
	// This file contains the implementation of custom routines for the X86
	// Calling Convention that aren't done by tablegen.
	//
	//===----------------------------------------------------------------------===//

	#include "X86CallingConv.h"
	#include "X86Subtarget.h"
	#include "llvm/ADT/SmallVector.h"
	#include "llvm/CodeGen/CallingConvLower.h"
	#include "llvm/IR/CallingConv.h"

	using namespace llvm;

	/// When regcall calling convention compiled to 32 bit arch, special treatment
	/// is required for 64 bit masks.
	/// The value should be assigned to two GPRs.
	/// \return true if registers were allocated and false otherwise.
	static bool CC_X86_32_RegCall_Assign2Regs(unsigned &ValNo, MVT &ValVT,
	MVT &LocVT,
	CCValAssign::LocInfo &LocInfo,
	ISD::ArgFlagsTy &ArgFlags,
	CCState &State) {
	// List of GPR registers that are available to store values in regcall
	// calling convention.
	static const MCPhysReg RegList[] = {X86::EAX, X86::ECX, X86::EDX, X86::EDI,
	X86::ESI};

	// The vector will save all the available registers for allocation.
	SmallVector<unsigned, 5> AvailableRegs;

	// searching for the available registers.
	for (auto Reg : RegList) {
	if (!State.isAllocated(Reg))
	AvailableRegs.push_back(Reg);
	}

	const size_t RequiredGprsUponSplit = 2;
	if (AvailableRegs.size() < RequiredGprsUponSplit)
	return false; // Not enough free registers - continue the search.

	// Allocating the available registers.
	for (unsigned I = 0; I < RequiredGprsUponSplit; I++) {

	// Marking the register as located.
	unsigned Reg = State.AllocateReg(AvailableRegs[I]);

	// Since we previously made sure that 2 registers are available
	// we expect that a real register number will be returned.
	assert(Reg && "Expecting a register will be available");

	// Assign the value to the allocated register
	State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT, LocInfo));
	}

	// Successful in allocating registers - stop scanning next rules.
	return true;
	}

	static ArrayRef<MCPhysReg> CC_X86_VectorCallGetSSEs(const MVT &ValVT) {
	if (ValVT.is512BitVector()) {
	static const MCPhysReg RegListZMM[] = {X86::ZMM0, X86::ZMM1, X86::ZMM2,
	X86::ZMM3, X86::ZMM4, X86::ZMM5};
	return makeArrayRef(std::begin(RegListZMM), std::end(RegListZMM));
	}

	if (ValVT.is256BitVector()) {
	static const MCPhysReg RegListYMM[] = {X86::YMM0, X86::YMM1, X86::YMM2,
	X86::YMM3, X86::YMM4, X86::YMM5};
	return makeArrayRef(std::begin(RegListYMM), std::end(RegListYMM));
	}

	static const MCPhysReg RegListXMM[] = {X86::XMM0, X86::XMM1, X86::XMM2,
	X86::XMM3, X86::XMM4, X86::XMM5};
	return makeArrayRef(std::begin(RegListXMM), std::end(RegListXMM));
	}

	static ArrayRef<MCPhysReg> CC_X86_64_VectorCallGetGPRs() {
	static const MCPhysReg RegListGPR[] = {X86::RCX, X86::RDX, X86::R8, X86::R9};
	return makeArrayRef(std::begin(RegListGPR), std::end(RegListGPR));
	}

	static bool CC_X86_VectorCallAssignRegister(unsigned &ValNo, MVT &ValVT,
	MVT &LocVT,
	CCValAssign::LocInfo &LocInfo,
	ISD::ArgFlagsTy &ArgFlags,
	CCState &State) {

	ArrayRef<MCPhysReg> RegList = CC_X86_VectorCallGetSSEs(ValVT);
	bool Is64bit = static_cast<const X86Subtarget &>(
	State.getMachineFunction().getSubtarget())
	.is64Bit();

	for (auto Reg : RegList) {
	// If the register is not marked as allocated - assign to it.
	if (!State.isAllocated(Reg)) {
	unsigned AssigedReg = State.AllocateReg(Reg);
	assert(AssigedReg == Reg && "Expecting a valid register allocation");
	State.addLoc(
	CCValAssign::getReg(ValNo, ValVT, AssigedReg, LocVT, LocInfo));
	return true;
	}
	// If the register is marked as shadow allocated - assign to it.
	if (Is64bit && State.IsShadowAllocatedReg(Reg)) {
	State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
	return true;
	}
	}

	llvm_unreachable("Clang should ensure that hva marked vectors will have "
	"an available register.");
	return false;
	}

	/// Vectorcall calling convention has special handling for vector types or
	/// HVA for 64 bit arch.
	/// For HVAs shadow registers might be allocated on the first pass
	/// and actual XMM registers are allocated on the second pass.
	/// For vector types, actual XMM registers are allocated on the first pass.
	/// \return true if registers were allocated and false otherwise.
	static bool CC_X86_64_VectorCall(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
	CCValAssign::LocInfo &LocInfo,
	ISD::ArgFlagsTy &ArgFlags, CCState &State) {
	// On the second pass, go through the HVAs only.
	if (ArgFlags.isSecArgPass()) {
	if (ArgFlags.isHva())
	return CC_X86_VectorCallAssignRegister(ValNo, ValVT, LocVT, LocInfo,
	ArgFlags, State);
	return true;
	}

	// Process only vector types as defined by vectorcall spec:
	// "A vector type is either a floating-point type, for example,
	// a float or double, or an SIMD vector type, for example, __m128 or __m256".
	if (!(ValVT.isFloatingPoint() \|\|
	(ValVT.isVector() && ValVT.getSizeInBits() >= 128))) {
	// If R9 was already assigned it means that we are after the fourth element
	// and because this is not an HVA / Vector type, we need to allocate
	// shadow XMM register.
	if (State.isAllocated(X86::R9)) {
	// Assign shadow XMM register.
	(void)State.AllocateReg(CC_X86_VectorCallGetSSEs(ValVT));
	}

	return false;
	}

	if (!ArgFlags.isHva() \|\| ArgFlags.isHvaStart()) {
	// Assign shadow GPR register.
	(void)State.AllocateReg(CC_X86_64_VectorCallGetGPRs());

	// Assign XMM register - (shadow for HVA and non-shadow for non HVA).
	if (unsigned Reg = State.AllocateReg(CC_X86_VectorCallGetSSEs(ValVT))) {
	// In Vectorcall Calling convention, additional shadow stack can be
	// created on top of the basic 32 bytes of win64.
	// It can happen if the fifth or sixth argument is vector type or HVA.
	// At that case for each argument a shadow stack of 8 bytes is allocated.
	const TargetRegisterInfo *TRI =
	State.getMachineFunction().getSubtarget().getRegisterInfo();
	if (TRI->regsOverlap(Reg, X86::XMM4) \|\|
	TRI->regsOverlap(Reg, X86::XMM5))
	State.AllocateStack(8, Align(8));

	if (!ArgFlags.isHva()) {
	State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
	return true; // Allocated a register - Stop the search.
	}
	}
	}

	// If this is an HVA - Stop the search,
	// otherwise continue the search.
	return ArgFlags.isHva();
	}

	/// Vectorcall calling convention has special handling for vector types or
	/// HVA for 32 bit arch.
	/// For HVAs actual XMM registers are allocated on the second pass.
	/// For vector types, actual XMM registers are allocated on the first pass.
	/// \return true if registers were allocated and false otherwise.
	static bool CC_X86_32_VectorCall(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
	CCValAssign::LocInfo &LocInfo,
	ISD::ArgFlagsTy &ArgFlags, CCState &State) {
	// On the second pass, go through the HVAs only.
	if (ArgFlags.isSecArgPass()) {
	if (ArgFlags.isHva())
	return CC_X86_VectorCallAssignRegister(ValNo, ValVT, LocVT, LocInfo,
	ArgFlags, State);
	return true;
	}

	// Process only vector types as defined by vectorcall spec:
	// "A vector type is either a floating point type, for example,
	// a float or double, or an SIMD vector type, for example, __m128 or __m256".
	if (!(ValVT.isFloatingPoint() \|\|
	(ValVT.isVector() && ValVT.getSizeInBits() >= 128))) {
	return false;
	}

	if (ArgFlags.isHva())
	return true; // If this is an HVA - Stop the search.

	// Assign XMM register.
	if (unsigned Reg = State.AllocateReg(CC_X86_VectorCallGetSSEs(ValVT))) {
	State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
	return true;
	}

	// In case we did not find an available XMM register for a vector -
	// pass it indirectly.
	// It is similar to CCPassIndirect, with the addition of inreg.
	if (!ValVT.isFloatingPoint()) {
	LocVT = MVT::i32;
	LocInfo = CCValAssign::Indirect;
	ArgFlags.setInReg();
	}

	return false; // No register was assigned - Continue the search.
	}

	static bool CC_X86_AnyReg_Error(unsigned &, MVT &, MVT &,
	CCValAssign::LocInfo &, ISD::ArgFlagsTy &,
	CCState &) {
	llvm_unreachable("The AnyReg calling convention is only supported by the "
	"stackmap and patchpoint intrinsics.");
	// gracefully fallback to X86 C calling convention on Release builds.
	return false;
	}

	static bool CC_X86_32_MCUInReg(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
	CCValAssign::LocInfo &LocInfo,
	ISD::ArgFlagsTy &ArgFlags, CCState &State) {
	// This is similar to CCAssignToReg<[EAX, EDX, ECX]>, but makes sure
	// not to split i64 and double between a register and stack
	static const MCPhysReg RegList[] = {X86::EAX, X86::EDX, X86::ECX};
	static const unsigned NumRegs = sizeof(RegList) / sizeof(RegList[0]);

	SmallVectorImpl<CCValAssign> &PendingMembers = State.getPendingLocs();

	// If this is the first part of an double/i64/i128, or if we're already
	// in the middle of a split, add to the pending list. If this is not
	// the end of the split, return, otherwise go on to process the pending
	// list
	if (ArgFlags.isSplit() \|\| !PendingMembers.empty()) {
	PendingMembers.push_back(
	CCValAssign::getPending(ValNo, ValVT, LocVT, LocInfo));
	if (!ArgFlags.isSplitEnd())
	return true;
	}

	// If there are no pending members, we are not in the middle of a split,
	// so do the usual inreg stuff.
	if (PendingMembers.empty()) {
	if (unsigned Reg = State.AllocateReg(RegList)) {
	State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
	return true;
	}
	return false;
	}

	assert(ArgFlags.isSplitEnd());

	// We now have the entire original argument in PendingMembers, so decide
	// whether to use registers or the stack.
	// Per the MCU ABI:
	// a) To use registers, we need to have enough of them free to contain
	// the entire argument.
	// b) We never want to use more than 2 registers for a single argument.

	unsigned FirstFree = State.getFirstUnallocated(RegList);
	bool UseRegs = PendingMembers.size() <= std::min(2U, NumRegs - FirstFree);

	for (auto &It : PendingMembers) {
	if (UseRegs)
	It.convertToReg(State.AllocateReg(RegList[FirstFree++]));
	else
	It.convertToMem(State.AllocateStack(4, Align(4)));
	State.addLoc(It);
	}

	PendingMembers.clear();

	return true;
	}

	/// X86 interrupt handlers can only take one or two stack arguments, but if
	/// there are two arguments, they are in the opposite order from the standard
	/// convention. Therefore, we have to look at the argument count up front before
	/// allocating stack for each argument.
	static bool CC_X86_Intr(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
	CCValAssign::LocInfo &LocInfo,
	ISD::ArgFlagsTy &ArgFlags, CCState &State) {
	const MachineFunction &MF = State.getMachineFunction();
	size_t ArgCount = State.getMachineFunction().getFunction().arg_size();
	bool Is64Bit = static_cast<const X86Subtarget &>(MF.getSubtarget()).is64Bit();
	unsigned SlotSize = Is64Bit ? 8 : 4;
	unsigned Offset;
	if (ArgCount == 1 && ValNo == 0) {
	// If we have one argument, the argument is five stack slots big, at fixed
	// offset zero.
	Offset = State.AllocateStack(5 * SlotSize, Align(4));
	} else if (ArgCount == 2 && ValNo == 0) {
	// If we have two arguments, the stack slot is after the error code
	// argument. Pretend it doesn't consume stack space, and account for it when
	// we assign the second argument.
	Offset = SlotSize;
	} else if (ArgCount == 2 && ValNo == 1) {
	// If this is the second of two arguments, it must be the error code. It
	// appears first on the stack, and is then followed by the five slot
	// interrupt struct.
	Offset = 0;
	(void)State.AllocateStack(6 * SlotSize, Align(4));
	} else {
	report_fatal_error("unsupported x86 interrupt prototype");
	}

	// FIXME: This should be accounted for in
	// X86FrameLowering::getFrameIndexReference, not here.
	if (Is64Bit && ArgCount == 2)
	Offset += SlotSize;

	State.addLoc(CCValAssign::getMem(ValNo, ValVT, Offset, LocVT, LocInfo));
	return true;
	}

	static bool CC_X86_64_Pointer(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
	CCValAssign::LocInfo &LocInfo,
	ISD::ArgFlagsTy &ArgFlags, CCState &State) {
	if (LocVT != MVT::i64) {
	LocVT = MVT::i64;
	LocInfo = CCValAssign::ZExt;
	}
	return false;
	}

	// Provides entry points of CC_X86 and RetCC_X86.
	#include "X86GenCallingConv.inc"