| //===- X86.cpp ------------------------------------------------------------===// | |
| // | |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |
| // See https://llvm.org/LICENSE.txt for license information. | |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |
| // | |
| //===----------------------------------------------------------------------===// | |
| #include "llvm/ABI/FunctionInfo.h" | |
| #include "llvm/ABI/TargetInfo.h" | |
| #include "llvm/ABI/Types.h" | |
| #include "llvm/Support/Alignment.h" | |
| #include "llvm/Support/Casting.h" | |
| #include "llvm/Support/ErrorHandling.h" | |
| #include "llvm/Support/MathExtras.h" | |
| #include "llvm/Support/TypeSize.h" | |
| #include <algorithm> | |
| #include <cassert> | |
| #include <cstdint> | |
| namespace llvm { | |
| namespace abi { | |
| static unsigned getNativeVectorSizeForAVXABI(X86AVXABILevel AVXLevel) { | |
| switch (AVXLevel) { | |
| case X86AVXABILevel::AVX512: | |
| return 512; | |
| case X86AVXABILevel::AVX: | |
| return 256; | |
| case X86AVXABILevel::None: | |
| return 128; | |
| } | |
| llvm_unreachable("Unknown AVXLevel"); | |
| } | |
| // The width of an integer's storage container, mirroring Clang's | |
| // ASTContext::getTypeSize. For a plain integer this is its bit width; for a | |
| // _BitInt(N) it is N rounded up to the type's alignment. The x86-64 _BitInt | |
| // max alignment is 64, so this clamp is target-specific and kept file-local. | |
| static uint64_t getClangIntegerWidthInBits(const IntegerType *IT) { | |
| uint64_t NumBits = IT->getSizeInBits().getFixedValue(); | |
| if (!IT->isBitInt()) | |
| return NumBits; | |
| uint64_t BitAlign = | |
| std::max<uint64_t>(8, std::min<uint64_t>(64, llvm::bit_ceil(NumBits))); | |
| return llvm::alignTo(NumBits, BitAlign); | |
| } | |
| static uint64_t getClangVectorWidthInBits(const VectorType *VT) { | |
| const Type *EltTy = VT->getElementType(); | |
| uint64_t EltWidth = EltTy->getSizeInBits().getFixedValue(); | |
| if (const auto *IT = dyn_cast<IntegerType>(EltTy)) | |
| EltWidth = getClangIntegerWidthInBits(IT); | |
| uint64_t Width = | |
| std::max<uint64_t>(8, EltWidth * VT->getNumElements().getKnownMinValue()); | |
| return llvm::bit_ceil(Width); | |
| } | |
| // The storage-container width of a type, mirroring Clang's getTypeSize. Used on | |
| // the stack path so a _BitInt or illegal vector coerces to the integer covering | |
| // its storage, not its raw iN width. | |
| static uint64_t getClangTypeWidthInBits(const Type *Ty) { | |
| if (const auto *VT = dyn_cast<VectorType>(Ty)) | |
| return getClangVectorWidthInBits(VT); | |
| if (const auto *IT = dyn_cast<IntegerType>(Ty)) | |
| return getClangIntegerWidthInBits(IT); | |
| return Ty->getSizeInBits().getFixedValue(); | |
| } | |
| class X86_64TargetInfo : public TargetInfo { | |
| public: | |
| enum Class { Integer, Sse, SseUp, X87, X87Up, ComplexX87, NoClass, Memory }; | |
| private: | |
| X86AVXABILevel AVXLevel; | |
| bool Has64BitPointers; | |
| X86ABICompatInfo X86CompatInfo; | |
| static Class merge(Class Accum, Class Field); | |
| void postMerge(unsigned AggregateSize, Class &Lo, Class &Hi) const; | |
| void classify(const Type *T, uint64_t OffsetBase, Class &Lo, Class &Hi, | |
| bool IsNamedArg, bool IsRegCall = false) const; | |
| const Type *getIntegerTypeAtOffset(const Type *IRType, unsigned IROffset, | |
| const Type *SourceTy, | |
| unsigned SourceOffset, | |
| bool InMemory = false) const; | |
| const Type *getSSETypeAtOffset(const Type *ABIType, unsigned ABIOffset, | |
| const Type *SourceTy, | |
| unsigned SourceOffset) const; | |
| bool isIllegalVectorType(const Type *Ty) const; | |
| bool containsMatrixField(const RecordType *RT) const; | |
| void computeInfo(FunctionInfo &FI) const override; | |
| ArgInfo getIndirectReturnResult(const Type *Ty) const; | |
| const Type *getFPTypeAtOffset(const Type *Ty, unsigned Offset) const; | |
| const Type *getByteVectorType(const Type *Ty) const; | |
| const Type *createPairType(const Type *Lo, const Type *Hi) const; | |
| ArgInfo getIndirectResult(const Type *Ty, unsigned FreeIntRegs) const; | |
| ArgInfo classifyReturnType(const Type *RetTy) const; | |
| ArgInfo classifyArgumentType(const Type *Ty, unsigned FreeIntRegs, | |
| unsigned &NeededInt, unsigned &NeededSse, | |
| bool IsNamedArg, bool IsRegCall = false) const; | |
| public: | |
| X86_64TargetInfo(TypeBuilder &TypeBuilder, X86AVXABILevel AVXABILevel, | |
| bool Has64BitPtrs, const X86ABICompatInfo &Compat) | |
| : TargetInfo(TypeBuilder), AVXLevel(AVXABILevel), | |
| Has64BitPointers(Has64BitPtrs), X86CompatInfo(Compat) {} | |
| bool has64BitPointers() const { return Has64BitPointers; } | |
| const ABICompatInfo &getABICompatInfo() const override { | |
| return X86CompatInfo; | |
| } | |
| const X86ABICompatInfo &getX86ABICompatInfo() const { return X86CompatInfo; } | |
| }; | |
| static bool bitsContainNoUserData(const Type *Ty, unsigned StartBit, | |
| unsigned EndBit); | |
| // Gets the "best" type to represent the union. | |
| static const Type *reduceUnionForX8664(const RecordType *UnionType, | |
| TypeBuilder &TB) { | |
| assert(UnionType->isUnion() && "Expected union type"); | |
| ArrayRef<FieldInfo> Fields = UnionType->getFields(); | |
| if (Fields.empty()) { | |
| return nullptr; | |
| } | |
| const Type *StorageType = nullptr; | |
| for (const auto &Field : Fields) { | |
| if (Field.IsBitField && Field.IsUnnamedBitfield && | |
| Field.BitFieldWidth == 0) { | |
| continue; | |
| } | |
| const Type *FieldType = Field.FieldType; | |
| if (UnionType->isTransparentUnion() && !StorageType) { | |
| StorageType = FieldType; | |
| break; | |
| } | |
| // A member that holds no user data supplies no bytes for a coercion to | |
| // read, so it must not become the storage type however wide or aligned it | |
| // is declared. Clang compares lowered types instead, where an empty class | |
| // is a byte array whose i8 leaf lets getIntegerTypeAtOffset narrow the | |
| // coercion. A record mapped here holds no fields, so there is no such | |
| // leaf and the eightbyte would be sized from the union. | |
| if (bitsContainNoUserData(FieldType, 0, | |
| FieldType->getSizeInBits().getFixedValue())) | |
| continue; | |
| if (!StorageType || | |
| FieldType->getAlignment() > StorageType->getAlignment() || | |
| (FieldType->getAlignment() == StorageType->getAlignment() && | |
| TypeSize::isKnownGT(FieldType->getSizeInBits(), | |
| StorageType->getSizeInBits()))) { | |
| StorageType = FieldType; | |
| } | |
| } | |
| return StorageType; | |
| } | |
| void X86_64TargetInfo::postMerge(unsigned AggregateSize, Class &Lo, | |
| Class &Hi) const { | |
| // AMD64-ABI 3.2.3p2: Rule 5. Then a post merger cleanup is done: | |
| // | |
| // (a) If one of the classes is Memory, the whole argument is passed in | |
| // memory. | |
| // | |
| // (b) If X87Up is not preceded by X87, the whole argument is passed in | |
| // memory. | |
| // | |
| // (c) If the size of the aggregate exceeds two eightbytes and the first | |
| // eightbyte isn't SSE or any other eightbyte isn't SSEUP, the whole | |
| // argument is passed in memory. NOTE: This is necessary to keep the | |
| // ABI working for processors that don't support the __m256 type. | |
| // | |
| // (d) If SSEUP is not preceded by SSE or SSEUP, it is converted to SSE. | |
| // | |
| // Some of these are enforced by the merging logic. Others can arise | |
| // only with unions; for example: | |
| // union { _Complex double; unsigned; } | |
| // | |
| // Note that clauses (b) and (c) were added in 0.98. | |
| if (Hi == Memory) | |
| Lo = Memory; | |
| if (Hi == X87Up && Lo != X87 && getX86ABICompatInfo().HonorsRevision98) | |
| Lo = Memory; | |
| if (AggregateSize > 128 && (Lo != Sse || Hi != SseUp)) | |
| Lo = Memory; | |
| if (Hi == SseUp && Lo != Sse) | |
| Hi = Sse; | |
| } | |
| X86_64TargetInfo::Class X86_64TargetInfo::merge(Class Accum, Class Field) { | |
| // AMD64-ABI 3.2.3p2: Rule 4. Each field of an object is | |
| // classified recursively so that always two fields are | |
| // considered. The resulting class is calculated according to | |
| // the classes of the fields in the eightbyte: | |
| // | |
| // (a) If both classes are equal, this is the resulting class. | |
| // | |
| // (b) If one of the classes is NO_CLASS, the resulting class is | |
| // the other class. | |
| // | |
| // (c) If one of the classes is MEMORY, the result is the MEMORY | |
| // class. | |
| // | |
| // (d) If one of the classes is INTEGER, the result is the | |
| // INTEGER. | |
| // | |
| // (e) If one of the classes is X87, X87Up, COMPLEX_X87 class, | |
| // MEMORY is used as class. | |
| // | |
| // (f) Otherwise class SSE is used. | |
| // Accum should never be memory (we should have returned) or | |
| // ComplexX87 (because this cannot be passed in a structure). | |
| assert((Accum != Memory && Accum != ComplexX87) && | |
| "Invalid accumulated classification during merge."); | |
| if (Accum == Field || Field == NoClass) | |
| return Accum; | |
| if (Field == Memory) | |
| return Memory; | |
| if (Accum == NoClass) | |
| return Field; | |
| if (Accum == Integer || Field == Integer) | |
| return Integer; | |
| if (Field == X87 || Field == X87Up || Field == ComplexX87 || Accum == X87 || | |
| Accum == X87Up) | |
| return Memory; | |
| return Sse; | |
| } | |
| // A record with a matrix-extension field is passed in memory. clang has no | |
| // matrix-specific ABI code: a matrix falls through X86_64ABIInfo::classify to | |
| // the default MEMORY class. We model matrices as arrays, so this check | |
| // reproduces that record-with-matrix -> MEMORY result. | |
| bool X86_64TargetInfo::containsMatrixField(const RecordType *RT) const { | |
| for (const auto &Field : RT->getFields()) { | |
| const Type *FieldType = Field.FieldType; | |
| if (const auto *AT = dyn_cast<ArrayType>(FieldType)) { | |
| if (AT->isMatrixType()) | |
| return true; | |
| continue; | |
| } | |
| if (const auto *NestedRT = dyn_cast<RecordType>(FieldType)) | |
| if (containsMatrixField(NestedRT)) | |
| return true; | |
| } | |
| return false; | |
| } | |
| void X86_64TargetInfo::classify(const Type *T, uint64_t OffsetBase, Class &Lo, | |
| Class &Hi, bool IsNamedArg, | |
| bool IsRegCall) const { | |
| Lo = Hi = NoClass; | |
| Class &Current = OffsetBase < 64 ? Lo : Hi; | |
| Current = Memory; | |
| if (T->isVoid()) { | |
| Current = NoClass; | |
| return; | |
| } | |
| if (const auto *IT = dyn_cast<IntegerType>(T)) { | |
| auto BitWidth = IT->getSizeInBits().getFixedValue(); | |
| if (BitWidth == 128 || | |
| (IT->isBitInt() && BitWidth > 64 && BitWidth <= 128)) { | |
| Lo = Integer; | |
| Hi = Integer; | |
| } else if (BitWidth <= 64) { | |
| Current = Integer; | |
| } | |
| return; | |
| } | |
| if (const auto *FT = dyn_cast<FloatType>(T)) { | |
| const auto *FltSem = FT->getSemantics(); | |
| if (FltSem == &llvm::APFloat::IEEEsingle() || | |
| FltSem == &llvm::APFloat::IEEEdouble() || | |
| FltSem == &llvm::APFloat::IEEEhalf() || | |
| FltSem == &llvm::APFloat::BFloat()) { | |
| Current = Sse; | |
| } else if (FltSem == &llvm::APFloat::IEEEquad()) { | |
| Lo = Sse; | |
| Hi = SseUp; | |
| } else if (FltSem == &llvm::APFloat::x87DoubleExtended()) { | |
| Lo = X87; | |
| Hi = X87Up; | |
| } else { | |
| Current = Sse; | |
| } | |
| return; | |
| } | |
| if (T->isPointer()) { | |
| Current = Integer; | |
| return; | |
| } | |
| if (const auto *MPT = dyn_cast<MemberPointerType>(T)) { | |
| if (MPT->isFunctionPointer()) { | |
| if (Has64BitPointers) { | |
| Lo = Hi = Integer; | |
| } else { | |
| uint64_t EbFuncPtr = OffsetBase / 64; | |
| uint64_t EbThisAdj = (OffsetBase + 64 - 1) / 64; | |
| if (EbFuncPtr != EbThisAdj) { | |
| Lo = Hi = Integer; | |
| } else { | |
| Current = Integer; | |
| } | |
| } | |
| } else { | |
| Current = Integer; | |
| } | |
| return; | |
| } | |
| if (const auto *VT = dyn_cast<VectorType>(T)) { | |
| auto Size = VT->getSizeInBits().getFixedValue(); | |
| const Type *ElementType = VT->getElementType(); | |
| if (Size == 1 || Size == 8 || Size == 16 || Size == 32) { | |
| // gcc passes the following as integer: | |
| // 4 bytes - <4 x char>, <2 x short>, <1 x int>, <1 x float> | |
| // 2 bytes - <2 x char>, <1 x short> | |
| // 1 byte - <1 x char> | |
| Current = Integer; | |
| // If this type crosses an eightbyte boundary, it should be | |
| // split. | |
| uint64_t EbLo = (OffsetBase) / 64; | |
| uint64_t EbHi = (OffsetBase + Size - 1) / 64; | |
| if (EbLo != EbHi) | |
| Hi = Lo; | |
| } else if (Size == 64) { | |
| if (const auto *FT = dyn_cast<FloatType>(ElementType)) { | |
| // gcc passes <1 x double> in memory. :( | |
| if (FT->getSemantics() == &llvm::APFloat::IEEEdouble()) | |
| return; | |
| } | |
| // gcc passes <1 x long long> as SSE but clang used to unconditionally | |
| // pass them as integer. For platforms where clang is the de facto | |
| // platform compiler, we must continue to use integer. | |
| if (const auto *IT = dyn_cast<IntegerType>(ElementType)) { | |
| uint64_t ElemBits = IT->getSizeInBits().getFixedValue(); | |
| if (!getX86ABICompatInfo().ClassifyIntegerMMXAsSSE && ElemBits == 64 && | |
| !IT->isBitInt()) { | |
| Current = Integer; | |
| } else { | |
| Current = Sse; | |
| } | |
| } else { | |
| Current = Sse; | |
| } | |
| // If this type crosses an eightbyte boundary, it should be | |
| // split. | |
| if (OffsetBase && OffsetBase != 64) | |
| Hi = Lo; | |
| } else if (Size == 128 || | |
| (IsNamedArg && Size <= getNativeVectorSizeForAVXABI(AVXLevel))) { | |
| if (const auto *IT = dyn_cast<IntegerType>(ElementType)) { | |
| uint64_t ElemBits = IT->getSizeInBits().getFixedValue(); | |
| // gcc passes 256 and 512 bit <X x __int128> vectors in memory. :( | |
| if (getX86ABICompatInfo().PassInt128VectorsInMem && Size != 128 && | |
| ElemBits == 128 && !IT->isBitInt()) | |
| return; | |
| } | |
| // Arguments of 256-bits are split into four eightbyte chunks. The | |
| // least significant one belongs to class SSE and all the others to class | |
| // SSEUP. The original Lo and Hi design considers that types can't be | |
| // greater than 128-bits, so a 64-bit split in Hi and Lo makes sense. | |
| // This design isn't correct for 256-bits, but since there're no cases | |
| // where the upper parts would need to be inspected, avoid adding | |
| // complexity and just consider Hi to match the 64-256 part. | |
| // | |
| // Note that per 3.5.7 of AMD64-ABI, 256-bit args are only passed in | |
| // registers if they are "named", i.e. not part of the "..." of a | |
| // variadic function. | |
| // | |
| // Similarly, per 3.2.3. of the AVX512 draft, 512-bits ("named") args are | |
| // split into eight eightbyte chunks, one SSE and seven SSEUP. | |
| Lo = Sse; | |
| Hi = SseUp; | |
| } | |
| return; | |
| } | |
| if (const auto *CT = dyn_cast<ComplexType>(T)) { | |
| const Type *ElementType = CT->getElementType(); | |
| uint64_t Size = T->getSizeInBits().getFixedValue(); | |
| if (isa<IntegerType>(ElementType)) { | |
| if (Size <= 64) | |
| Current = Integer; | |
| else if (Size <= 128) | |
| Lo = Hi = Integer; | |
| } else if (const auto *EFT = dyn_cast<FloatType>(ElementType)) { | |
| const auto *FltSem = EFT->getSemantics(); | |
| if (FltSem == &llvm::APFloat::IEEEhalf() || | |
| FltSem == &llvm::APFloat::IEEEsingle() || | |
| FltSem == &llvm::APFloat::BFloat()) | |
| Current = Sse; | |
| else if (FltSem == &llvm::APFloat::IEEEquad()) | |
| Current = Memory; | |
| else if (FltSem == &llvm::APFloat::x87DoubleExtended()) | |
| Current = ComplexX87; | |
| else if (FltSem == &llvm::APFloat::IEEEdouble()) | |
| Lo = Hi = Sse; | |
| else | |
| llvm_unreachable("Unexpected long double representation!"); | |
| } | |
| uint64_t ElementSize = ElementType->getSizeInBits().getFixedValue(); | |
| // If this complex type crosses an eightbyte boundary then it | |
| // should be split. | |
| uint64_t EbReal = OffsetBase / 64; | |
| uint64_t EbImag = (OffsetBase + ElementSize) / 64; | |
| if (Hi == NoClass && EbReal != EbImag) | |
| Hi = Lo; | |
| return; | |
| } | |
| if (const auto *AT = dyn_cast<ArrayType>(T)) { | |
| // A matrix type is modeled as an array but, like Clang, is treated as a | |
| // non-aggregate scalar: it matches no class here and stays in the Memory | |
| // class, so classify*Type later returns it Direct (coerced to its | |
| // flattened vector) rather than classifying it field-by-field. | |
| if (AT->isMatrixType()) | |
| return; | |
| // Arrays are treated like structures. | |
| uint64_t Size = AT->getSizeInBits().getFixedValue(); | |
| // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger | |
| // than eight eightbytes, ..., it has class MEMORY. | |
| // regcall ABI doesn't have limitation to an object. The only limitation | |
| // is the free registers, which will be checked in computeInfo. | |
| if (!IsRegCall && Size > 512) | |
| return; | |
| // AMD64-ABI 3.2.3p2: Rule 1. If ..., or it contains unaligned | |
| // fields, it has class MEMORY. | |
| // | |
| // Only need to check alignment of array base. | |
| const Type *ElementType = AT->getElementType(); | |
| uint64_t ElemAlign = ElementType->getAlignment().value() * 8; | |
| if (OffsetBase % ElemAlign) | |
| return; | |
| // Otherwise implement simplified merge. We could be smarter about | |
| // this, but it isn't worth it and would be harder to verify. | |
| Current = NoClass; | |
| uint64_t EltSize = ElementType->getSizeInBits().getFixedValue(); | |
| uint64_t ArraySize = AT->getNumElements(); | |
| // The only case a 256-bit wide vector could be used is when the array | |
| // contains a single 256-bit element. Since Lo and Hi logic isn't extended | |
| // to work for sizes wider than 128, early check and fallback to memory. | |
| // | |
| if (Size > 128 && | |
| (Size != EltSize || Size > getNativeVectorSizeForAVXABI(AVXLevel))) | |
| return; | |
| for (uint64_t I = 0, Offset = OffsetBase; I < ArraySize; | |
| ++I, Offset += EltSize) { | |
| Class FieldLo, FieldHi; | |
| classify(ElementType, Offset, FieldLo, FieldHi, IsNamedArg); | |
| Lo = merge(Lo, FieldLo); | |
| Hi = merge(Hi, FieldHi); | |
| if (Lo == Memory || Hi == Memory) | |
| break; | |
| } | |
| postMerge(Size, Lo, Hi); | |
| assert((Hi != SseUp || Lo == Sse) && "Invalid SseUp array classification."); | |
| return; | |
| } | |
| if (const auto *RT = dyn_cast<RecordType>(T)) { | |
| uint64_t Size = RT->getSizeInBits().getFixedValue(); | |
| if (containsMatrixField(RT)) { | |
| Lo = Memory; | |
| return; | |
| } | |
| // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger | |
| // than eight eightbytes, ..., it has class MEMORY. | |
| if (Size > 512) | |
| return; | |
| // AMD64-ABI 3.2.3p2: Rule 2. If a C++ object has either a non-trivial | |
| // copy constructor or a non-trivial destructor, it is passed by invisible | |
| // reference. | |
| if (getRecordArgABI(RT)) | |
| return; | |
| // Assume variable sized types are passed in memory. | |
| if (RT->hasFlexibleArrayMember()) | |
| return; | |
| // Reset Lo class, this will be recomputed. | |
| Current = NoClass; | |
| // If this is a C++ record, classify the bases first. | |
| if (RT->isCXXRecord()) { | |
| for (const auto &Base : RT->getBaseClasses()) { | |
| // Classify this field. | |
| // | |
| // AMD64-ABI 3.2.3p2: Rule 3. If the size of the aggregate exceeds a | |
| // single eightbyte, each is classified separately. Each eightbyte gets | |
| // initialized to class NO_CLASS. | |
| Class FieldLo, FieldHi; | |
| uint64_t Offset = OffsetBase + Base.OffsetInBits; | |
| classify(Base.FieldType, Offset, FieldLo, FieldHi, IsNamedArg); | |
| Lo = merge(Lo, FieldLo); | |
| Hi = merge(Hi, FieldHi); | |
| if (getX86ABICompatInfo().ReturnCXXRecordGreaterThan128InMem && | |
| (Size > 128 && | |
| (Size != Base.FieldType->getSizeInBits().getFixedValue() || | |
| Size > getNativeVectorSizeForAVXABI(AVXLevel)))) | |
| Lo = Memory; | |
| if (Lo == Memory || Hi == Memory) { | |
| postMerge(Size, Lo, Hi); | |
| return; | |
| } | |
| } | |
| } | |
| // Classify the fields one at a time, merging the results. | |
| bool IsUnion = RT->isUnion() && !getX86ABICompatInfo().Clang11Compat; | |
| for (const auto &Field : RT->getFields()) { | |
| uint64_t Offset = OffsetBase + Field.OffsetInBits; | |
| bool BitField = Field.IsBitField; | |
| // Ignore padding bit-fields. Normally only zero-length bit-fields are | |
| // padding, but under Clang 23 compatibility every unnamed bit-field is, | |
| // faithfully reproducing Clang 23. | |
| if (BitField && (getX86ABICompatInfo().ClassifyUnnamedBitFields | |
| ? Field.BitFieldWidth == 0 | |
| : Field.IsUnnamedBitfield)) | |
| continue; | |
| if (Size > 128 && | |
| ((!IsUnion && | |
| Size != Field.FieldType->getSizeInBits().getFixedValue()) || | |
| Size > getNativeVectorSizeForAVXABI(AVXLevel))) { | |
| Lo = Memory; | |
| postMerge(Size, Lo, Hi); | |
| return; | |
| } | |
| bool IsInMemory = Offset % (Field.FieldType->getAlignment().value() * 8); | |
| if (!BitField && IsInMemory) { | |
| Lo = Memory; | |
| postMerge(Size, Lo, Hi); | |
| return; | |
| } | |
| Class FieldLo, FieldHi; | |
| if (BitField) { | |
| uint64_t BitFieldSize = Field.BitFieldWidth; | |
| uint64_t EbLo = Offset / 64; | |
| uint64_t EbHi = (Offset + BitFieldSize - 1) / 64; | |
| if (EbLo) { | |
| assert(EbHi == EbLo && "Invalid classification, type > 16 bytes."); | |
| FieldLo = NoClass; | |
| FieldHi = Integer; | |
| } else { | |
| FieldLo = Integer; | |
| FieldHi = EbHi ? Integer : NoClass; | |
| } | |
| } else { | |
| classify(Field.FieldType, Offset, FieldLo, FieldHi, IsNamedArg); | |
| } | |
| Lo = merge(Lo, FieldLo); | |
| Hi = merge(Hi, FieldHi); | |
| if (Lo == Memory || Hi == Memory) | |
| break; | |
| } | |
| postMerge(Size, Lo, Hi); | |
| return; | |
| } | |
| Lo = Memory; | |
| Hi = NoClass; | |
| } | |
| ArgInfo | |
| X86_64TargetInfo::classifyArgumentType(const Type *Ty, unsigned FreeIntRegs, | |
| unsigned &NeededInt, unsigned &NeededSSE, | |
| bool IsNamedArg, bool IsRegCall) const { | |
| Ty = useFirstFieldIfTransparentUnion(Ty); | |
| X86_64TargetInfo::Class Lo, Hi; | |
| classify(Ty, 0, Lo, Hi, IsNamedArg, IsRegCall); | |
| // Check some invariants | |
| assert((Hi != Memory || Lo == Memory) && "Invalid memory classification."); | |
| assert((Hi != SseUp || Lo == Sse) && "Invalid SseUp classification."); | |
| NeededInt = 0; | |
| NeededSSE = 0; | |
| const Type *ResType = nullptr; | |
| switch (Lo) { | |
| case NoClass: | |
| if (Hi == NoClass) | |
| return ArgInfo::getIgnore(); | |
| // If the low part is just padding, it takes no register, leave ResType | |
| // null. | |
| assert((Hi == Sse || Hi == Integer || Hi == X87Up) && | |
| "Unknown missing lo part"); | |
| break; | |
| // AMD64-ABI 3.2.3p3: Rule 1. If the class is MEMORY, pass the argument | |
| // on the stack. | |
| case Memory: | |
| // AMD64-ABI 3.2.3p3: Rule 5. If the class is X87, X87Up or | |
| // COMPLEX_X87, it is passed in memory. | |
| case X87: | |
| case ComplexX87: | |
| if (getRecordArgABI(Ty) == RAA_Indirect) | |
| ++NeededInt; | |
| return getIndirectResult(Ty, FreeIntRegs); | |
| case SseUp: | |
| case X87Up: | |
| llvm_unreachable("Invalid classification for lo word."); | |
| // AMD64-ABI 3.2.3p3: Rule 2. If the class is INTEGER, the next | |
| // available register of the sequence %rdi, %rsi, %rdx, %rcx, %r8 | |
| // and %r9 is used. | |
| case Integer: | |
| ++NeededInt; | |
| // Pick an 8-byte type based on the preferred type. | |
| ResType = getIntegerTypeAtOffset(Ty, 0, Ty, 0); | |
| // If we have a sign or zero extended integer, make sure to return Extend | |
| // so that the parameter gets the right LLVM IR attributes. | |
| if (Hi == NoClass && ResType->isInteger()) { | |
| if (Ty->isInteger() && isPromotableInteger(cast<IntegerType>(Ty))) | |
| return ArgInfo::getExtend(Ty); | |
| } | |
| if (ResType->isInteger() && ResType->getSizeInBits() == 128) { | |
| assert(Hi == Integer); | |
| ++NeededInt; | |
| return ArgInfo::getDirect(ResType); | |
| } | |
| break; | |
| // AMD64-ABI 3.2.3p3: Rule 3. If the class is SSE, the next | |
| // available SSE register is used, the registers are taken in the | |
| // order from %xmm0 to %xmm7. | |
| case Sse: | |
| ResType = getSSETypeAtOffset(Ty, 0, Ty, 0); | |
| ++NeededSSE; | |
| break; | |
| } | |
| const Type *HighPart = nullptr; | |
| switch (Hi) { | |
| // Memory was handled previously, ComplexX87 and X87 should | |
| // never occur as hi classes, and X87Up must be preceded by X87, | |
| // which is passed in memory. | |
| case Memory: | |
| case X87: | |
| case ComplexX87: | |
| llvm_unreachable("Invalid classification for hi word."); | |
| case NoClass: | |
| break; | |
| case Integer: | |
| ++NeededInt; | |
| // Pick an 8-byte type based on the preferred type. | |
| HighPart = getIntegerTypeAtOffset(Ty, 8, Ty, 8); | |
| if (Lo == NoClass) // Pass HighPart at offset 8 in memory. | |
| return ArgInfo::getDirect(HighPart, 8); | |
| break; | |
| // X87Up generally doesn't occur here (long double is passed in | |
| // memory), except in situations involving unions. | |
| case X87Up: | |
| case Sse: | |
| ++NeededSSE; | |
| HighPart = getSSETypeAtOffset(Ty, 8, Ty, 8); | |
| if (Lo == NoClass) // Pass HighPart at offset 8 in memory. | |
| return ArgInfo::getDirect(HighPart, 8); | |
| break; | |
| // AMD64-ABI 3.2.3p3: Rule 4. If the class is SSEUP, the | |
| // eightbyte is passed in the upper half of the last used SSE | |
| // register. This only happens when 128-bit vectors are passed. | |
| case SseUp: | |
| assert(Lo == Sse && "Unexpected SseUp classification"); | |
| ResType = getByteVectorType(Ty); | |
| break; | |
| } | |
| // If a high part was specified, merge it together with the low part. It is | |
| // known to pass in the high eightbyte of the result. We do this by forming a | |
| // first class struct aggregate with the high and low part: {low, high} | |
| if (HighPart) | |
| ResType = createPairType(ResType, HighPart); | |
| return ArgInfo::getDirect(ResType); | |
| } | |
| ArgInfo X86_64TargetInfo::classifyReturnType(const Type *RetTy) const { | |
| // AMD64-ABI 3.2.3p4: Rule 1. Classify the return type with the | |
| // classification algorithm. | |
| X86_64TargetInfo::Class Lo, Hi; | |
| classify(RetTy, 0, Lo, Hi, /*isNamedArg*/ true); | |
| // Check some invariants | |
| assert((Hi != Memory || Lo == Memory) && "Invalid memory classification."); | |
| assert((Hi != SseUp || Lo == Sse) && "Invalid SseUp classification."); | |
| const Type *ResType = nullptr; | |
| switch (Lo) { | |
| case NoClass: | |
| if (Hi == NoClass) | |
| return ArgInfo::getIgnore(); | |
| // If the low part is just padding, it takes no register, leave ResType | |
| // null. | |
| assert((Hi == Sse || Hi == Integer || Hi == X87Up) && | |
| "Unknown missing lo part"); | |
| break; | |
| case SseUp: | |
| case X87Up: | |
| llvm_unreachable("Invalid classification for lo word."); | |
| // AMD64-ABI 3.2.3p4: Rule 2. Types of class memory are returned via | |
| // hidden argument. | |
| case Memory: | |
| return getIndirectReturnResult(RetTy); | |
| // AMD64-ABI 3.2.3p4: Rule 3. If the class is INTEGER, the next | |
| // available register of the sequence %rax, %rdx is used. | |
| case Integer: | |
| ResType = getIntegerTypeAtOffset(RetTy, 0, RetTy, 0); | |
| // If we have a sign or zero extended integer, make sure to return Extend | |
| // so that the parameter gets the right LLVM IR attributes. | |
| if (Hi == NoClass && ResType->isInteger()) { | |
| if (const IntegerType *IntTy = dyn_cast<IntegerType>(RetTy)) { | |
| if (isPromotableInteger(IntTy)) | |
| return ArgInfo::getExtend(RetTy); | |
| } | |
| } | |
| if (ResType->isInteger() && ResType->getSizeInBits() == 128) { | |
| assert(Hi == Integer); | |
| return ArgInfo::getDirect(ResType); | |
| } | |
| break; | |
| // AMD64-ABI 3.2.3p4: Rule 4. If the class is SSE, the next | |
| // available SSE register of the sequence %xmm0, %xmm1 is used. | |
| case Sse: | |
| ResType = getSSETypeAtOffset(RetTy, 0, RetTy, 0); | |
| break; | |
| // AMD64-ABI 3.2.3p4: Rule 6. If the class is X87, the value is | |
| // returned on the X87 stack in %st0 as 80-bit x87 number. | |
| case X87: | |
| ResType = TB.getFloatType(APFloat::x87DoubleExtended(), Align(16)); | |
| break; | |
| // AMD64-ABI 3.2.3p4: Rule 8. If the class is COMPLEX_X87, the real | |
| // part of the value is returned in %st0 and the imaginary part in | |
| // %st1. | |
| case ComplexX87: | |
| assert(Hi == ComplexX87 && "Unexpected ComplexX87 classification."); | |
| { | |
| const Type *X87Type = | |
| TB.getFloatType(APFloat::x87DoubleExtended(), Align(16)); | |
| FieldInfo Fields[] = {FieldInfo(X87Type, 0), FieldInfo(X87Type, 80)}; | |
| ResType = TB.getRecordType(Fields, TypeSize::getFixed(160), Align(16), | |
| /*UnadjustedAlign=*/Align(16)); | |
| } | |
| break; | |
| } | |
| const Type *HighPart = nullptr; | |
| switch (Hi) { | |
| // Memory was handled previously and X87 should | |
| // never occur as a hi class. | |
| case Memory: | |
| case X87: | |
| llvm_unreachable("Invalid classification for hi word."); | |
| case ComplexX87: | |
| case NoClass: | |
| break; | |
| case Integer: | |
| HighPart = getIntegerTypeAtOffset(RetTy, 8, RetTy, 8); | |
| if (Lo == NoClass) | |
| return ArgInfo::getDirect(HighPart, 8); | |
| break; | |
| case Sse: | |
| HighPart = getSSETypeAtOffset(RetTy, 8, RetTy, 8); | |
| if (Lo == NoClass) | |
| return ArgInfo::getDirect(HighPart, 8); | |
| break; | |
| // AMD64-ABI 3.2.3p4: Rule 5. If the class is SSEUP, the eightbyte | |
| // is passed in the next available eightbyte chunk if the last used | |
| // vector register. | |
| // | |
| // SSEUP should always be preceded by SSE, just widen. | |
| case SseUp: | |
| assert(Lo == Sse && "Unexpected SseUp classification."); | |
| ResType = getByteVectorType(RetTy); | |
| break; | |
| // AMD64-ABI 3.2.3p4: Rule 7. If the class is X87Up, the value is | |
| // returned together with the previous X87 value in %st0. | |
| case X87Up: | |
| // If X87Up is preceded by X87, we don't need to do | |
| // anything. However, in some cases with unions it may not be | |
| // preceded by X87. In such situations we follow gcc and pass the | |
| // extra bits in an SSE reg. | |
| if (Lo != X87) { | |
| HighPart = getSSETypeAtOffset(RetTy, 8, RetTy, 8); | |
| if (Lo == NoClass) // Return HighPart at offset 8 in memory. | |
| return ArgInfo::getDirect(HighPart, 8); | |
| } | |
| break; | |
| } | |
| // If a high part was specified, merge it together with the low part. It is | |
| // known to pass in the high eightbyte of the result. We do this by forming a | |
| // first class struct aggregate with the high and low part: {low, high} | |
| if (HighPart) | |
| ResType = createPairType(ResType, HighPart); | |
| return ArgInfo::getDirect(ResType); | |
| } | |
| /// Given a high and low type that can ideally | |
| /// be used as elements of a two register pair to pass or return, return a | |
| /// first class aggregate to represent them. For example, if the low part of | |
| /// a by-value argument should be passed as i32* and the high part as float, | |
| /// return {i32*, float}. | |
| const Type *X86_64TargetInfo::createPairType(const Type *Lo, | |
| const Type *Hi) const { | |
| // In order to correctly satisfy the ABI, we need to the high part to start | |
| // at offset 8. If the high and low parts we inferred are both 4-byte types | |
| // (e.g. i32 and i32) then the resultant struct type ({i32,i32}) won't have | |
| // the second element at offset 8. Check for this: | |
| unsigned LoSize = (unsigned)Lo->getTypeAllocSize(); | |
| llvm::Align HiAlign = Hi->getAlignment(); | |
| unsigned HiStart = alignTo(LoSize, HiAlign); | |
| assert(HiStart != 0 && HiStart <= 8 && "Invalid x86-64 argument pair!"); | |
| // To handle this, we have to increase the size of the low part so that the | |
| // second element will start at an 8 byte offset. We can't increase the size | |
| // of the second element because it might make us access off the end of the | |
| // struct. | |
| const Type *AdjustedLo = Lo; | |
| if (HiStart != 8) { | |
| // There are usually two sorts of types the ABI generation code can produce | |
| // for the low part of a pair that aren't 8 bytes in size: half, float or | |
| // i8/i16/i32. This can also include pointers when they are 32-bit (X32 and | |
| // NaCl). | |
| // Promote these to a larger type. | |
| if (Lo->isFloat()) { | |
| const FloatType *FT = cast<FloatType>(Lo); | |
| if (FT->getSemantics() == &APFloat::IEEEhalf() || | |
| FT->getSemantics() == &APFloat::IEEEsingle() || | |
| FT->getSemantics() == &APFloat::BFloat()) | |
| AdjustedLo = TB.getFloatType(APFloat::IEEEdouble(), Align(8)); | |
| } | |
| // Promote integers and pointers to i64 | |
| else if (Lo->isInteger() || Lo->isPointer()) | |
| AdjustedLo = TB.getIntegerType(64, Align(8), /*Signed=*/false); | |
| else | |
| assert((Lo->isInteger() || Lo->isPointer()) && | |
| "Invalid/unknown low type in pair"); | |
| unsigned AdjustedLoSize = AdjustedLo->getSizeInBits().getFixedValue() / 8; | |
| HiStart = alignTo(AdjustedLoSize, HiAlign); | |
| } | |
| // Create the pair struct | |
| FieldInfo Fields[] = {FieldInfo(AdjustedLo, 0), FieldInfo(Hi, HiStart * 8)}; | |
| // Verify the high part is at offset 8 | |
| assert((8 * 8) == Fields[1].OffsetInBits && | |
| "High part must be at offset 8 bytes"); | |
| uint64_t PairSizeInBits = | |
| Fields[1].OffsetInBits + Hi->getSizeInBits().getFixedValue(); | |
| return TB.getRecordType(Fields, TypeSize::getFixed(PairSizeInBits), Align(8), | |
| /*UnadjustedAlign=*/Align(8), StructPacking::Default); | |
| } | |
| static bool bitsContainNoUserData(const Type *Ty, unsigned StartBit, | |
| unsigned EndBit) { | |
| // If range is completely beyond type size, it's definitely padding | |
| unsigned TySize = Ty->getSizeInBits().getFixedValue(); | |
| if (TySize <= StartBit) | |
| return true; | |
| // Handle arrays - check each element | |
| if (const ArrayType *AT = dyn_cast<ArrayType>(Ty)) { | |
| const Type *EltTy = AT->getElementType(); | |
| unsigned EltSize = EltTy->getSizeInBits().getFixedValue(); | |
| for (unsigned I = 0; I < AT->getNumElements(); ++I) { | |
| unsigned EltOffset = I * EltSize; | |
| if (EltOffset >= EndBit) | |
| break; | |
| unsigned EltStart = (EltOffset < StartBit) ? StartBit - EltOffset : 0; | |
| if (!bitsContainNoUserData(EltTy, EltStart, EndBit - EltOffset)) | |
| return false; | |
| } | |
| return true; | |
| } | |
| // Handle records - check all fields and base classes. getUnionType places a | |
| // union's members at offset zero, so the field loop covers a union too. | |
| if (const RecordType *RT = dyn_cast<RecordType>(Ty)) { | |
| // Check base classes first (for C++ records) | |
| if (RT->isCXXRecord()) { | |
| for (unsigned I = 0; I < RT->getNumBaseClasses(); ++I) { | |
| const FieldInfo &Base = RT->getBaseClasses()[I]; | |
| if (Base.OffsetInBits >= EndBit) | |
| continue; | |
| unsigned BaseStart = | |
| (Base.OffsetInBits < StartBit) ? StartBit - Base.OffsetInBits : 0; | |
| if (!bitsContainNoUserData(Base.FieldType, BaseStart, | |
| EndBit - Base.OffsetInBits)) | |
| return false; | |
| } | |
| } | |
| for (unsigned I = 0; I < RT->getNumFields(); ++I) { | |
| const FieldInfo &Field = RT->getFields()[I]; | |
| if (Field.OffsetInBits >= EndBit) | |
| break; | |
| unsigned FieldStart = | |
| (Field.OffsetInBits < StartBit) ? StartBit - Field.OffsetInBits : 0; | |
| if (!bitsContainNoUserData(Field.FieldType, FieldStart, | |
| EndBit - Field.OffsetInBits)) | |
| return false; | |
| } | |
| return true; | |
| } | |
| // For any other type - assume all bits are user data | |
| return false; | |
| } | |
| const Type *X86_64TargetInfo::getIntegerTypeAtOffset(const Type *ABIType, | |
| unsigned ABIOffset, | |
| const Type *SourceTy, | |
| unsigned SourceOffset, | |
| bool InMemory) const { | |
| const Type *WorkingType = ABIType; | |
| if (InMemory && ABIType->isInteger()) { | |
| const auto *IT = cast<IntegerType>(ABIType); | |
| unsigned OriginalBitWidth = IT->getSizeInBits().getFixedValue(); | |
| unsigned WidenedBitWidth = OriginalBitWidth; | |
| if (OriginalBitWidth <= 8) { | |
| WidenedBitWidth = 8; | |
| } else { | |
| WidenedBitWidth = llvm::bit_ceil(OriginalBitWidth); | |
| } | |
| if (WidenedBitWidth != OriginalBitWidth) { | |
| WorkingType = TB.getIntegerType(WidenedBitWidth, ABIType->getAlignment(), | |
| IT->isSigned()); | |
| } | |
| } | |
| // If we're dealing with an un-offset ABI type, then it means that we're | |
| // returning an 8-byte unit starting with it. See if we can safely use it. | |
| if (ABIOffset == 0) { | |
| // Pointers and int64's always fill the 8-byte unit. Return WorkingType, | |
| // which is the in-memory-widened type (e.g. a _BitInt(37) field widened to | |
| // i64): returning the raw ABIType here would coerce the eightbyte to the | |
| // narrow iN instead of the storage integer clang uses. | |
| if ((WorkingType->isPointer() && Has64BitPointers) || | |
| (WorkingType->isInteger() && | |
| cast<IntegerType>(WorkingType)->getSizeInBits() == 64)) | |
| return WorkingType; | |
| // If we have a 1/2/4-byte integer, we can use it only if the rest of the | |
| // goodness in the source type is just tail padding. This is allowed to | |
| // kick in for struct {double,int} on the int, but not on | |
| // struct{double,int,int} because we wouldn't return the second int. We | |
| // have to do this analysis on the source type because we can't depend on | |
| // unions being lowered a specific way etc. | |
| if ((WorkingType->isInteger() && | |
| (cast<IntegerType>(WorkingType)->getSizeInBits() == 1 || | |
| cast<IntegerType>(WorkingType)->getSizeInBits() == 8 || | |
| cast<IntegerType>(WorkingType)->getSizeInBits() == 16 || | |
| cast<IntegerType>(WorkingType)->getSizeInBits() == 32)) || | |
| (WorkingType->isPointer() && !Has64BitPointers)) { | |
| unsigned BitWidth = WorkingType->isPointer() | |
| ? 32 | |
| : cast<IntegerType>(WorkingType)->getSizeInBits(); | |
| if (bitsContainNoUserData(SourceTy, SourceOffset * 8 + BitWidth, | |
| SourceOffset * 8 + 64)) | |
| return WorkingType; | |
| } | |
| } | |
| if (const auto *RTy = dyn_cast<RecordType>(ABIType)) { | |
| if (RTy->isUnion()) { | |
| const Type *ReducedType = reduceUnionForX8664(RTy, TB); | |
| if (ReducedType) { | |
| if (ABIOffset * 8 < ReducedType->getSizeInBits().getFixedValue()) | |
| return getIntegerTypeAtOffset(ReducedType, ABIOffset, SourceTy, | |
| SourceOffset, true); | |
| // The storage type stops before this offset, so size the coercion | |
| // from the union itself: a byte when the rest of this eightbyte | |
| // holds no data, and the union's remaining bytes otherwise. | |
| if (bitsContainNoUserData(SourceTy, SourceOffset * 8 + 8, | |
| SourceOffset * 8 + 64)) | |
| return TB.getIntegerType(8, Align(1), /*Signed=*/false); | |
| unsigned RemainingBytes = | |
| llvm::divideCeil(SourceTy->getSizeInBits().getFixedValue(), 8) - | |
| SourceOffset; | |
| return TB.getIntegerType(std::min(RemainingBytes, 8U) * 8, Align(1), | |
| /*Signed=*/false); | |
| } | |
| } | |
| if (const FieldInfo *Element = | |
| RTy->getElementContainingOffset(ABIOffset * 8)) { | |
| unsigned ElementOffsetBytes = Element->OffsetInBits / 8; | |
| return getIntegerTypeAtOffset(Element->FieldType, | |
| ABIOffset - ElementOffsetBytes, SourceTy, | |
| SourceOffset, true); | |
| } | |
| } | |
| if (const auto *ATy = dyn_cast<ArrayType>(ABIType)) { | |
| const Type *EltTy = ATy->getElementType(); | |
| unsigned EltSize = EltTy->getSizeInBits() / 8; | |
| if (EltSize > 0) { | |
| unsigned EltOffset = (ABIOffset / EltSize) * EltSize; | |
| return getIntegerTypeAtOffset(EltTy, ABIOffset - EltOffset, SourceTy, | |
| SourceOffset, true); | |
| } | |
| } | |
| // If we have a 128-bit integer, we can pass it safely using an i128 | |
| // so we return that | |
| if (ABIType->isInteger() && ABIType->getSizeInBits() == 128) { | |
| assert(ABIOffset == 0); | |
| return ABIType; | |
| } | |
| unsigned TySizeInBytes = | |
| llvm::divideCeil(SourceTy->getSizeInBits().getFixedValue(), 8); | |
| if (auto *IT = dyn_cast<IntegerType>(SourceTy)) { | |
| if (IT->isBitInt()) | |
| TySizeInBytes = | |
| alignTo(SourceTy->getSizeInBits().getFixedValue(), 64) / 8; | |
| } | |
| assert(TySizeInBytes != SourceOffset && "Empty field?"); | |
| unsigned AvailableSize = TySizeInBytes - SourceOffset; | |
| return TB.getIntegerType(std::min(AvailableSize, 8U) * 8, Align(1), false); | |
| } | |
| /// Returns the floating point type at the specified offset within a type, or | |
| /// nullptr if no floating point type is found at that offset. | |
| const Type *X86_64TargetInfo::getFPTypeAtOffset(const Type *Ty, | |
| unsigned Offset) const { | |
| // Check for direct match at offset 0 | |
| if (Offset == 0 && Ty->isFloat()) | |
| return Ty; | |
| if (const ComplexType *CT = dyn_cast<ComplexType>(Ty)) { | |
| const Type *ElementType = CT->getElementType(); | |
| unsigned ElementSize = ElementType->getSizeInBits().getFixedValue() / 8; | |
| if (Offset == 0 || Offset == ElementSize) | |
| return ElementType; | |
| return nullptr; | |
| } | |
| // Handle struct types by checking each field | |
| if (const RecordType *RT = dyn_cast<RecordType>(Ty)) { | |
| if (const FieldInfo *Element = RT->getElementContainingOffset(Offset * 8)) { | |
| unsigned ElementOffsetBytes = Element->OffsetInBits / 8; | |
| return getFPTypeAtOffset(Element->FieldType, Offset - ElementOffsetBytes); | |
| } | |
| } | |
| // Handle array types | |
| if (const ArrayType *AT = dyn_cast<ArrayType>(Ty)) { | |
| const Type *EltTy = AT->getElementType(); | |
| unsigned EltSize = EltTy->getSizeInBits() / 8; | |
| unsigned EltIndex = Offset / EltSize; | |
| return getFPTypeAtOffset(EltTy, Offset - (EltIndex * EltSize)); | |
| } | |
| // No floating point type found at this offset | |
| return nullptr; | |
| } | |
| /// Helper to check if a floating point type matches specific semantics | |
| static bool isFloatTypeWithSemantics(const Type *Ty, | |
| const fltSemantics &Semantics) { | |
| if (!Ty->isFloat()) | |
| return false; | |
| const FloatType *FT = cast<FloatType>(Ty); | |
| return FT->getSemantics() == &Semantics; | |
| } | |
| /// GetSSETypeAtOffset - Return a type that will be passed by the backend in the | |
| /// low 8 bytes of an XMM register, corresponding to the SSE class. | |
| const Type *X86_64TargetInfo::getSSETypeAtOffset(const Type *ABIType, | |
| unsigned ABIOffset, | |
| const Type *SourceTy, | |
| unsigned SourceOffset) const { | |
| if (const auto *RTy = dyn_cast<RecordType>(ABIType)) { | |
| if (RTy->isUnion()) { | |
| const Type *ReducedType = reduceUnionForX8664(RTy, TB); | |
| if (ReducedType) { | |
| return getSSETypeAtOffset(ReducedType, ABIOffset, SourceTy, | |
| SourceOffset); | |
| } | |
| } | |
| } | |
| auto Is16bitFpTy = [](const Type *T) { | |
| return isFloatTypeWithSemantics(T, APFloat::IEEEhalf()) || | |
| isFloatTypeWithSemantics(T, APFloat::BFloat()); | |
| }; | |
| // Get the floating point type at the requested offset | |
| const Type *T0 = getFPTypeAtOffset(ABIType, ABIOffset); | |
| if (!T0 || isFloatTypeWithSemantics(T0, APFloat::IEEEdouble())) | |
| return TB.getFloatType(APFloat::IEEEdouble(), Align(8)); | |
| // Calculate remaining source size in bytes | |
| unsigned SourceSize = | |
| (SourceTy->getSizeInBits().getFixedValue() / 8) - SourceOffset; | |
| // Try to get adjacent FP type | |
| const Type *T1 = nullptr; | |
| unsigned T0Size = | |
| alignTo(T0->getSizeInBits().getFixedValue(), T0->getAlignment().value()) / | |
| 8; | |
| if (SourceSize > T0Size) | |
| T1 = getFPTypeAtOffset(ABIType, ABIOffset + T0Size); | |
| if (T1 == nullptr) { | |
| if (Is16bitFpTy(T0) && SourceSize > 4) | |
| T1 = getFPTypeAtOffset(ABIType, ABIOffset + 4); | |
| if (T1 == nullptr) | |
| return T0; | |
| } | |
| // Handle vector cases | |
| if (isFloatTypeWithSemantics(T0, APFloat::IEEEsingle()) && | |
| isFloatTypeWithSemantics(T1, APFloat::IEEEsingle())) | |
| return TB.getVectorType(T0, ElementCount::getFixed(2), Align(8)); | |
| if (Is16bitFpTy(T0) && Is16bitFpTy(T1)) { | |
| const Type *T2 = nullptr; | |
| if (SourceSize > 4) | |
| T2 = getFPTypeAtOffset(ABIType, ABIOffset + 4); | |
| if (!T2) | |
| return TB.getVectorType(T0, ElementCount::getFixed(2), Align(8)); | |
| return TB.getVectorType(T0, ElementCount::getFixed(4), Align(8)); | |
| } | |
| // Mixed half-float cases | |
| if (Is16bitFpTy(T0) || Is16bitFpTy(T1)) | |
| return TB.getVectorType(TB.getFloatType(APFloat::IEEEhalf(), Align(2)), | |
| ElementCount::getFixed(4), Align(8)); | |
| // Default to double | |
| return TB.getFloatType(APFloat::IEEEdouble(), Align(8)); | |
| } | |
| /// The ABI specifies that a value should be passed in a full vector XMM/YMM | |
| /// register. Pick an LLVM IR type that will be passed as a vector register. | |
| const Type *X86_64TargetInfo::getByteVectorType(const Type *Ty) const { | |
| // Wrapper structs/arrays that only contain vectors are passed just like | |
| // vectors; strip them off if present. | |
| if (const Type *InnerTy = isSingleElementStruct(Ty)) | |
| Ty = InnerTy; | |
| // Handle vector types | |
| if (const VectorType *VT = dyn_cast<VectorType>(Ty)) { | |
| // Don't pass vXi128 vectors in their native type, the backend can't | |
| // legalize them. | |
| if (getX86ABICompatInfo().PassInt128VectorsInMem && | |
| VT->getElementType()->isInteger() && | |
| cast<IntegerType>(VT->getElementType())->getSizeInBits() == 128) { | |
| unsigned Size = VT->getSizeInBits().getFixedValue(); | |
| return TB.getVectorType(TB.getIntegerType(64, Align(8), /*Signed=*/false), | |
| ElementCount::getFixed(Size / 64), | |
| Align(Size / 8)); | |
| } | |
| return VT; | |
| } | |
| // Handle fp128 | |
| if (isFloatTypeWithSemantics(Ty, APFloat::IEEEquad())) | |
| return Ty; | |
| // We couldn't find the preferred IR vector type for 'Ty'. | |
| unsigned Size = Ty->getSizeInBits().getFixedValue(); | |
| assert((Size == 128 || Size == 256 || Size == 512) && "Invalid vector size"); | |
| return TB.getVectorType(TB.getFloatType(APFloat::IEEEdouble(), Align(8)), | |
| ElementCount::getFixed(Size / 64), Align(Size / 8)); | |
| } | |
| bool X86_64TargetInfo::isIllegalVectorType(const Type *Ty) const { | |
| if (const auto *VecTy = dyn_cast<VectorType>(Ty)) { | |
| uint64_t Size = VecTy->getSizeInBits().getFixedValue(); | |
| unsigned LargestVector = getNativeVectorSizeForAVXABI(AVXLevel); | |
| // Vectors <= 64 bits or > largest supported vector size are illegal | |
| if (Size <= 64 || Size > LargestVector) | |
| return true; | |
| // Check for 128-bit integer element vectors that should be passed in memory | |
| const Type *EltTy = VecTy->getElementType(); | |
| if (getX86ABICompatInfo().PassInt128VectorsInMem && EltTy->isInteger()) { | |
| const auto *IntTy = cast<IntegerType>(EltTy); | |
| if (IntTy->getSizeInBits().getFixedValue() == 128) | |
| return true; | |
| } | |
| } | |
| return false; | |
| } | |
| ArgInfo X86_64TargetInfo::getIndirectResult(const Type *Ty, | |
| unsigned FreeIntRegs) const { | |
| // If this is a scalar LLVM value then assume LLVM will pass it in the right | |
| // place naturally. | |
| // | |
| // This assumption is optimistic, as there could be free registers available | |
| // when we need to pass this argument in memory, and LLVM could try to pass | |
| // the argument in the free register. This does not seem to happen currently, | |
| // but this code would be much safer if we could mark the argument with | |
| // 'onstack'. See PR12193. | |
| if (!isAggregateTypeForABI(Ty) && !isIllegalVectorType(Ty) && | |
| !(Ty->isInteger() && cast<IntegerType>(Ty)->isBitInt())) { | |
| return (Ty->isInteger() && isPromotableInteger(cast<IntegerType>(Ty)) | |
| ? ArgInfo::getExtend(Ty) | |
| : ArgInfo::getDirect()); | |
| } | |
| // Check if this is a record type that needs special handling | |
| if (auto RecordRAA = getRecordArgABI(Ty)) | |
| return getNaturalAlignIndirect(Ty, RecordRAA == | |
| RecordArgABI::RAA_DirectInMemory); | |
| // Compute the byval alignment. We specify the alignment of the byval in all | |
| // cases so that the mid-level optimizer knows the alignment of the byval. | |
| uint64_t AlignVal = std::max<uint64_t>(Ty->getAlignment().value(), 8u); | |
| // Attempt to avoid passing indirect results using byval when possible. This | |
| // is important for good codegen. | |
| // | |
| // We do this by coercing the value into a scalar type which the backend can | |
| // handle naturally (i.e., without using byval). | |
| // | |
| // For simplicity, we currently only do this when we have exhausted all of the | |
| // free integer registers. Doing this when there are free integer registers | |
| // would require more care, as we would have to ensure that the coerced value | |
| // did not claim the unused register. That would require either reording the | |
| // arguments to the function (so that any subsequent inreg values came first), | |
| // or only doing this optimization when there were no following arguments that | |
| // might be inreg. | |
| // | |
| // We currently expect it to be rare (particularly in well written code) for | |
| // arguments to be passed on the stack when there are still free integer | |
| // registers available (this would typically imply large structs being passed | |
| // by value), so this seems like a fair tradeoff for now. | |
| // | |
| // We can revisit this if the backend grows support for 'onstack' parameter | |
| // attributes. See PR12193. | |
| if (FreeIntRegs == 0) { | |
| // Use the storage-container width (like Clang's getTypeSize) so a stack | |
| // _BitInt or illegal vector coerces to the integer covering its storage, | |
| // not its raw iN width. | |
| uint64_t Size = getClangTypeWidthInBits(Ty); | |
| // If this type fits in an eightbyte, coerce it into the matching integral | |
| // type, which will end up on the stack (with alignment 8). | |
| if (AlignVal == 8 && Size <= 64) { | |
| const Type *IntTy = | |
| TB.getIntegerType(Size, llvm::Align(8), /*Signed=*/false); | |
| return ArgInfo::getDirect(IntTy); | |
| } | |
| } | |
| return ArgInfo::getIndirect(llvm::Align(AlignVal), /*ByVal=*/true); | |
| } | |
| ArgInfo X86_64TargetInfo::getIndirectReturnResult(const Type *Ty) const { | |
| if (!isAggregateTypeForABI(Ty)) { | |
| // Bit-precise integers are returned indirectly regardless of size. | |
| if (const auto *IntTy = dyn_cast<IntegerType>(Ty)) { | |
| if (IntTy->isBitInt()) | |
| return getNaturalAlignIndirect(IntTy, /*ByVal=*/true); | |
| if (isPromotableInteger(IntTy)) | |
| return ArgInfo::getExtend(Ty); | |
| } | |
| return ArgInfo::getDirect(); | |
| } | |
| return getNaturalAlignIndirect(Ty, /*ByVal=*/true); | |
| } | |
| void X86_64TargetInfo::computeInfo(FunctionInfo &FI) const { | |
| CallingConv::ID CallingConv = FI.getCallingConvention(); | |
| // Only the standard SysV (C) calling convention is classified here. Any other | |
| // convention must be added explicitly once it has been verified against this | |
| // classifier rather than silently taking the SysV path. | |
| switch (CallingConv) { | |
| case CallingConv::C: | |
| break; | |
| default: | |
| llvm_unreachable( | |
| "calling convention not supported by the LLVMABI X86_64 classifier"); | |
| } | |
| unsigned FreeIntRegs = 6; | |
| unsigned FreeSSERegs = 8; | |
| unsigned NeededInt = 0, NeededSSE = 0; | |
| if (!maybeCommonClassifyReturnType(FI)) { | |
| const Type *RetTy = FI.getReturnType(); | |
| FI.getReturnInfo() = classifyReturnType(RetTy); | |
| } | |
| if (FI.getReturnInfo().isIndirect()) | |
| --FreeIntRegs; | |
| unsigned NumRequiredArgs = FI.getNumRequiredArgs(); | |
| unsigned ArgNo = 0; | |
| for (auto IT = FI.arg_begin(), IE = FI.arg_end(); IT != IE; ++IT, ++ArgNo) { | |
| bool IsNamedArg = ArgNo < NumRequiredArgs; | |
| const Type *ArgTy = IT->ABIType; | |
| NeededInt = 0; | |
| NeededSSE = 0; | |
| ArgInfo AI = classifyArgumentType(ArgTy, FreeIntRegs, NeededInt, NeededSSE, | |
| IsNamedArg); | |
| // AMD64-ABI 3.2.3p3: If there are no registers available for any | |
| // eightbyte of an argument, the whole argument is passed on the | |
| // stack. If registers have already been assigned for some | |
| // eightbytes of such an argument, the assignments get reverted. | |
| if (FreeIntRegs >= NeededInt && FreeSSERegs >= NeededSSE) { | |
| FreeIntRegs -= NeededInt; | |
| FreeSSERegs -= NeededSSE; | |
| AI.setNeededRegs(NeededInt, NeededSSE); | |
| IT->Info = AI; | |
| } else { | |
| // Not enough registers, pass on stack. The demand the classification | |
| // reports is what the argument ends up occupying, which is nothing. | |
| IT->Info = getIndirectResult(ArgTy, FreeIntRegs); | |
| } | |
| } | |
| } | |
| std::unique_ptr<TargetInfo> | |
| createX86_64TargetInfo(TypeBuilder &TB, X86AVXABILevel AVXLevel, | |
| bool Has64BitPointers, const X86ABICompatInfo &Compat) { | |
| return std::make_unique<X86_64TargetInfo>(TB, AVXLevel, Has64BitPointers, | |
| Compat); | |
| } | |
| } // namespace abi | |
| } // namespace llvm |