| //===- AArch64.cpp --------------------------------------------------------===// | |
| // | |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |
| // See https://llvm.org/LICENSE.txt for license information. | |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |
| // | |
| //===----------------------------------------------------------------------===// | |
| #include "ABIInfoImpl.h" | |
| #include "TargetInfo.h" | |
| #include "clang/AST/Decl.h" | |
| #include "clang/Basic/DiagnosticFrontend.h" | |
| #include "llvm/TargetParser/AArch64TargetParser.h" | |
| using namespace clang; | |
| using namespace clang::CodeGen; | |
| //===----------------------------------------------------------------------===// | |
| // AArch64 ABI Implementation | |
| //===----------------------------------------------------------------------===// | |
| namespace { | |
| class AArch64ABIInfo : public ABIInfo { | |
| AArch64ABIKind Kind; | |
| std::unique_ptr<TargetCodeGenInfo> WinX86_64CodegenInfo; | |
| public: | |
| AArch64ABIInfo(CodeGenModule &CGM, AArch64ABIKind Kind) | |
| : ABIInfo(CGM.getTypes()), Kind(Kind) { | |
| if (getTarget().getTriple().isWindowsArm64EC()) { | |
| WinX86_64CodegenInfo = | |
| createWinX86_64TargetCodeGenInfo(CGM, X86AVXABILevel::None); | |
| } | |
| } | |
| bool isSoftFloat() const { return Kind == AArch64ABIKind::AAPCSSoft; } | |
| private: | |
| AArch64ABIKind getABIKind() const { return Kind; } | |
| bool isDarwinPCS() const { return Kind == AArch64ABIKind::DarwinPCS; } | |
| ABIArgInfo classifyReturnType(QualType RetTy, bool IsVariadicFn) const; | |
| ABIArgInfo classifyArgumentType(QualType RetTy, bool IsVariadicFn, | |
| bool IsNamedArg, unsigned CallingConvention, | |
| unsigned &NSRN, unsigned &NPRN) const; | |
| llvm::Type *convertFixedToScalableVectorType(const VectorType *VT) const; | |
| ABIArgInfo coerceIllegalVector(QualType Ty, unsigned &NSRN, | |
| unsigned &NPRN) const; | |
| ABIArgInfo coerceAndExpandPureScalableAggregate( | |
| QualType Ty, bool IsNamedArg, unsigned NVec, unsigned NPred, | |
| const SmallVectorImpl<llvm::Type *> &UnpaddedCoerceToSeq, unsigned &NSRN, | |
| unsigned &NPRN) const; | |
| bool isHomogeneousAggregateBaseType(QualType Ty) const override; | |
| bool isHomogeneousAggregateSmallEnough(const Type *Ty, | |
| uint64_t Members) const override; | |
| bool isZeroLengthBitfieldPermittedInHomogeneousAggregate() const override; | |
| bool isIllegalVectorType(QualType Ty) const; | |
| bool passAsAggregateType(QualType Ty) const; | |
| bool passAsPureScalableType(QualType Ty, unsigned &NV, unsigned &NP, | |
| SmallVectorImpl<llvm::Type *> &CoerceToSeq) const; | |
| void flattenType(llvm::Type *Ty, | |
| SmallVectorImpl<llvm::Type *> &Flattened) const; | |
| void computeInfo(CGFunctionInfo &FI) const override { | |
| if (!::classifyReturnType(getCXXABI(), FI, *this)) | |
| FI.getReturnInfo() = | |
| classifyReturnType(FI.getReturnType(), FI.isVariadic()); | |
| unsigned ArgNo = 0; | |
| unsigned NSRN = 0, NPRN = 0; | |
| for (auto &it : FI.arguments()) { | |
| const bool IsNamedArg = | |
| !FI.isVariadic() || ArgNo < FI.getRequiredArgs().getNumRequiredArgs(); | |
| ++ArgNo; | |
| it.info = classifyArgumentType(it.type, FI.isVariadic(), IsNamedArg, | |
| FI.getCallingConvention(), NSRN, NPRN); | |
| } | |
| } | |
| RValue EmitDarwinVAArg(Address VAListAddr, QualType Ty, CodeGenFunction &CGF, | |
| AggValueSlot Slot) const; | |
| RValue EmitAAPCSVAArg(Address VAListAddr, QualType Ty, CodeGenFunction &CGF, | |
| AArch64ABIKind Kind, AggValueSlot Slot) const; | |
| RValue EmitVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty, | |
| AggValueSlot Slot) const override { | |
| llvm::Type *BaseTy = CGF.ConvertType(Ty); | |
| if (isa<llvm::ScalableVectorType>(BaseTy)) | |
| llvm::report_fatal_error("Passing SVE types to variadic functions is " | |
| "currently not supported"); | |
| return Kind == AArch64ABIKind::Win64 | |
| ? EmitMSVAArg(CGF, VAListAddr, Ty, Slot) | |
| : isDarwinPCS() ? EmitDarwinVAArg(VAListAddr, Ty, CGF, Slot) | |
| : EmitAAPCSVAArg(VAListAddr, Ty, CGF, Kind, Slot); | |
| } | |
| RValue EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr, QualType Ty, | |
| AggValueSlot Slot) const override; | |
| bool allowBFloatArgsAndRet() const override { | |
| return getTarget().hasBFloat16Type(); | |
| } | |
| using ABIInfo::appendAttributeMangling; | |
| void appendAttributeMangling(TargetClonesAttr *Attr, unsigned Index, | |
| raw_ostream &Out) const override; | |
| void appendAttributeMangling(StringRef AttrStr, | |
| raw_ostream &Out) const override; | |
| }; | |
| class AArch64SwiftABIInfo : public SwiftABIInfo { | |
| public: | |
| explicit AArch64SwiftABIInfo(CodeGenTypes &CGT) | |
| : SwiftABIInfo(CGT, /*SwiftErrorInRegister=*/true) {} | |
| bool isLegalVectorType(CharUnits VectorSize, llvm::Type *EltTy, | |
| unsigned NumElts) const override; | |
| }; | |
| class AArch64TargetCodeGenInfo : public TargetCodeGenInfo { | |
| public: | |
| AArch64TargetCodeGenInfo(CodeGenModule &CGM, AArch64ABIKind Kind) | |
| : TargetCodeGenInfo(std::make_unique<AArch64ABIInfo>(CGM, Kind)) { | |
| SwiftInfo = std::make_unique<AArch64SwiftABIInfo>(CGM.getTypes()); | |
| } | |
| StringRef getARCRetainAutoreleasedReturnValueMarker() const override { | |
| return "mov\tfp, fp\t\t// marker for objc_retainAutoreleaseReturnValue"; | |
| } | |
| int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const override { | |
| return 31; | |
| } | |
| bool doesReturnSlotInterfereWithArgs() const override { return false; } | |
| void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV, | |
| CodeGen::CodeGenModule &CGM) const override { | |
| auto *Fn = dyn_cast<llvm::Function>(GV); | |
| if (!Fn) | |
| return; | |
| const auto *FD = dyn_cast_or_null<FunctionDecl>(D); | |
| TargetInfo::BranchProtectionInfo BPI(CGM.getLangOpts()); | |
| if (FD && FD->hasAttr<TargetAttr>()) { | |
| const auto *TA = FD->getAttr<TargetAttr>(); | |
| ParsedTargetAttr Attr = | |
| CGM.getTarget().parseTargetAttr(TA->getFeaturesStr()); | |
| if (!Attr.BranchProtection.empty()) { | |
| StringRef Error; | |
| (void)CGM.getTarget().validateBranchProtection( | |
| Attr.BranchProtection, Attr.CPU, BPI, CGM.getLangOpts(), Error); | |
| assert(Error.empty()); | |
| } | |
| } | |
| setBranchProtectionFnAttributes(BPI, *Fn); | |
| setPointerAuthFnAttributes(CGM.getCodeGenOpts().PointerAuth, *Fn); | |
| } | |
| bool isScalarizableAsmOperand(CodeGen::CodeGenFunction &CGF, | |
| llvm::Type *Ty) const override { | |
| if (CGF.getTarget().hasFeature("ls64")) { | |
| auto *ST = dyn_cast<llvm::StructType>(Ty); | |
| if (ST && ST->getNumElements() == 1) { | |
| auto *AT = dyn_cast<llvm::ArrayType>(ST->getElementType(0)); | |
| if (AT && AT->getNumElements() == 8 && | |
| AT->getElementType()->isIntegerTy(64)) | |
| return true; | |
| } | |
| } | |
| return TargetCodeGenInfo::isScalarizableAsmOperand(CGF, Ty); | |
| } | |
| void checkFunctionABI(CodeGenModule &CGM, | |
| const FunctionDecl *Decl) const override; | |
| void checkFunctionCallABI(CodeGenModule &CGM, SourceLocation CallLoc, | |
| const FunctionDecl *Caller, | |
| const FunctionDecl *Callee, const CallArgList &Args, | |
| QualType ReturnType) const override; | |
| bool wouldInliningViolateFunctionCallABI( | |
| const FunctionDecl *Caller, const FunctionDecl *Callee) const override; | |
| private: | |
| // Diagnose calls between functions with incompatible Streaming SVE | |
| // attributes. | |
| void checkFunctionCallABIStreaming(CodeGenModule &CGM, SourceLocation CallLoc, | |
| const FunctionDecl *Caller, | |
| const FunctionDecl *Callee) const; | |
| // Diagnose calls which must pass arguments in floating-point registers when | |
| // the selected target does not have floating-point registers. | |
| void checkFunctionCallABISoftFloat(CodeGenModule &CGM, SourceLocation CallLoc, | |
| const FunctionDecl *Caller, | |
| const FunctionDecl *Callee, | |
| const CallArgList &Args, | |
| QualType ReturnType) const; | |
| }; | |
| class WindowsAArch64TargetCodeGenInfo : public AArch64TargetCodeGenInfo { | |
| public: | |
| WindowsAArch64TargetCodeGenInfo(CodeGenModule &CGM, AArch64ABIKind K) | |
| : AArch64TargetCodeGenInfo(CGM, K) {} | |
| void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV, | |
| CodeGen::CodeGenModule &CGM) const override; | |
| void getDependentLibraryOption(llvm::StringRef Lib, | |
| llvm::SmallString<24> &Opt) const override { | |
| Opt = "/DEFAULTLIB:" + qualifyWindowsLibrary(Lib); | |
| } | |
| void getDetectMismatchOption(llvm::StringRef Name, llvm::StringRef Value, | |
| llvm::SmallString<32> &Opt) const override { | |
| Opt = "/FAILIFMISMATCH:\"" + Name.str() + "=" + Value.str() + "\""; | |
| } | |
| }; | |
| void WindowsAArch64TargetCodeGenInfo::setTargetAttributes( | |
| const Decl *D, llvm::GlobalValue *GV, CodeGen::CodeGenModule &CGM) const { | |
| AArch64TargetCodeGenInfo::setTargetAttributes(D, GV, CGM); | |
| if (GV->isDeclaration()) | |
| return; | |
| addStackProbeTargetAttributes(D, GV, CGM); | |
| } | |
| } | |
| llvm::Type * | |
| AArch64ABIInfo::convertFixedToScalableVectorType(const VectorType *VT) const { | |
| assert(VT->getElementType()->isBuiltinType() && "expected builtin type!"); | |
| if (VT->getVectorKind() == VectorKind::SveFixedLengthPredicate) { | |
| assert(VT->getElementType()->castAs<BuiltinType>()->getKind() == | |
| BuiltinType::UChar && | |
| "unexpected builtin type for SVE predicate!"); | |
| return llvm::ScalableVectorType::get(llvm::Type::getInt1Ty(getVMContext()), | |
| 16); | |
| } | |
| if (VT->getVectorKind() == VectorKind::SveFixedLengthData) { | |
| const auto *BT = VT->getElementType()->castAs<BuiltinType>(); | |
| switch (BT->getKind()) { | |
| default: | |
| llvm_unreachable("unexpected builtin type for SVE vector!"); | |
| case BuiltinType::SChar: | |
| case BuiltinType::UChar: | |
| case BuiltinType::MFloat8: | |
| return llvm::ScalableVectorType::get( | |
| llvm::Type::getInt8Ty(getVMContext()), 16); | |
| case BuiltinType::Short: | |
| case BuiltinType::UShort: | |
| return llvm::ScalableVectorType::get( | |
| llvm::Type::getInt16Ty(getVMContext()), 8); | |
| case BuiltinType::Int: | |
| case BuiltinType::UInt: | |
| return llvm::ScalableVectorType::get( | |
| llvm::Type::getInt32Ty(getVMContext()), 4); | |
| case BuiltinType::Long: | |
| case BuiltinType::ULong: | |
| return llvm::ScalableVectorType::get( | |
| llvm::Type::getInt64Ty(getVMContext()), 2); | |
| case BuiltinType::Half: | |
| return llvm::ScalableVectorType::get( | |
| llvm::Type::getHalfTy(getVMContext()), 8); | |
| case BuiltinType::Float: | |
| return llvm::ScalableVectorType::get( | |
| llvm::Type::getFloatTy(getVMContext()), 4); | |
| case BuiltinType::Double: | |
| return llvm::ScalableVectorType::get( | |
| llvm::Type::getDoubleTy(getVMContext()), 2); | |
| case BuiltinType::BFloat16: | |
| return llvm::ScalableVectorType::get( | |
| llvm::Type::getBFloatTy(getVMContext()), 8); | |
| } | |
| } | |
| llvm_unreachable("expected fixed-length SVE vector"); | |
| } | |
| ABIArgInfo AArch64ABIInfo::coerceIllegalVector(QualType Ty, unsigned &NSRN, | |
| unsigned &NPRN) const { | |
| assert(Ty->isVectorType() && "expected vector type!"); | |
| const auto *VT = Ty->castAs<VectorType>(); | |
| if (VT->getVectorKind() == VectorKind::SveFixedLengthPredicate) { | |
| assert(VT->getElementType()->isBuiltinType() && "expected builtin type!"); | |
| assert(VT->getElementType()->castAs<BuiltinType>()->getKind() == | |
| BuiltinType::UChar && | |
| "unexpected builtin type for SVE predicate!"); | |
| NPRN = std::min(NPRN + 1, 4u); | |
| return ABIArgInfo::getDirect(llvm::ScalableVectorType::get( | |
| llvm::Type::getInt1Ty(getVMContext()), 16)); | |
| } | |
| if (VT->getVectorKind() == VectorKind::SveFixedLengthData) { | |
| NSRN = std::min(NSRN + 1, 8u); | |
| return ABIArgInfo::getDirect(convertFixedToScalableVectorType(VT)); | |
| } | |
| uint64_t Size = getContext().getTypeSize(Ty); | |
| // Android promotes <2 x i8> to i16, not i32 | |
| if ((isAndroid() || isOHOSFamily()) && (Size <= 16)) { | |
| llvm::Type *ResType = llvm::Type::getInt16Ty(getVMContext()); | |
| return ABIArgInfo::getDirect(ResType); | |
| } | |
| if (Size <= 32) { | |
| llvm::Type *ResType = llvm::Type::getInt32Ty(getVMContext()); | |
| return ABIArgInfo::getDirect(ResType); | |
| } | |
| if (Size == 64) { | |
| NSRN = std::min(NSRN + 1, 8u); | |
| auto *ResType = | |
| llvm::FixedVectorType::get(llvm::Type::getInt32Ty(getVMContext()), 2); | |
| return ABIArgInfo::getDirect(ResType); | |
| } | |
| if (Size == 128) { | |
| NSRN = std::min(NSRN + 1, 8u); | |
| auto *ResType = | |
| llvm::FixedVectorType::get(llvm::Type::getInt32Ty(getVMContext()), 4); | |
| return ABIArgInfo::getDirect(ResType); | |
| } | |
| return getNaturalAlignIndirect(Ty, getDataLayout().getAllocaAddrSpace(), | |
| /*ByVal=*/false); | |
| } | |
| ABIArgInfo AArch64ABIInfo::coerceAndExpandPureScalableAggregate( | |
| QualType Ty, bool IsNamedArg, unsigned NVec, unsigned NPred, | |
| const SmallVectorImpl<llvm::Type *> &UnpaddedCoerceToSeq, unsigned &NSRN, | |
| unsigned &NPRN) const { | |
| if (!IsNamedArg || NSRN + NVec > 8 || NPRN + NPred > 4) | |
| return getNaturalAlignIndirect(Ty, getDataLayout().getAllocaAddrSpace(), | |
| /*ByVal=*/false); | |
| NSRN += NVec; | |
| NPRN += NPred; | |
| // Handle SVE vector tuples. | |
| if (Ty->isSVESizelessBuiltinType()) | |
| return ABIArgInfo::getDirect(); | |
| llvm::Type *UnpaddedCoerceToType = | |
| UnpaddedCoerceToSeq.size() == 1 | |
| ? UnpaddedCoerceToSeq[0] | |
| : llvm::StructType::get(CGT.getLLVMContext(), UnpaddedCoerceToSeq, | |
| true); | |
| SmallVector<llvm::Type *> CoerceToSeq; | |
| flattenType(CGT.ConvertType(Ty), CoerceToSeq); | |
| auto *CoerceToType = | |
| llvm::StructType::get(CGT.getLLVMContext(), CoerceToSeq, false); | |
| return ABIArgInfo::getCoerceAndExpand(CoerceToType, UnpaddedCoerceToType); | |
| } | |
| ABIArgInfo AArch64ABIInfo::classifyArgumentType(QualType Ty, bool IsVariadicFn, | |
| bool IsNamedArg, | |
| unsigned CallingConvention, | |
| unsigned &NSRN, | |
| unsigned &NPRN) const { | |
| Ty = useFirstFieldIfTransparentUnion(Ty); | |
| if (IsVariadicFn && getTarget().getTriple().isWindowsArm64EC()) { | |
| // Arm64EC varargs functions use the x86_64 classification rules, | |
| // not the AArch64 ABI rules. | |
| return WinX86_64CodegenInfo->getABIInfo().classifyArgForArm64ECVarArg(Ty); | |
| } | |
| // Handle illegal vector types here. | |
| if (isIllegalVectorType(Ty)) | |
| return coerceIllegalVector(Ty, NSRN, NPRN); | |
| if (!passAsAggregateType(Ty)) { | |
| // Treat an enum type as its underlying type. | |
| if (const auto *ED = Ty->getAsEnumDecl()) | |
| Ty = ED->getIntegerType(); | |
| if (const auto *EIT = Ty->getAs<BitIntType>()) | |
| if (EIT->getNumBits() > 128) | |
| return getNaturalAlignIndirect(Ty, getDataLayout().getAllocaAddrSpace(), | |
| false); | |
| if (Ty->isVectorType()) | |
| NSRN = std::min(NSRN + 1, 8u); | |
| else if (const auto *BT = Ty->getAs<BuiltinType>()) { | |
| if (BT->isFloatingPoint()) | |
| NSRN = std::min(NSRN + 1, 8u); | |
| else { | |
| switch (BT->getKind()) { | |
| case BuiltinType::SveBool: | |
| case BuiltinType::SveCount: | |
| NPRN = std::min(NPRN + 1, 4u); | |
| break; | |
| case BuiltinType::SveBoolx2: | |
| NPRN = std::min(NPRN + 2, 4u); | |
| break; | |
| case BuiltinType::SveBoolx4: | |
| NPRN = std::min(NPRN + 4, 4u); | |
| break; | |
| default: | |
| if (BT->isSVESizelessBuiltinType()) | |
| NSRN = std::min( | |
| NSRN + getContext().getBuiltinVectorTypeInfo(BT).NumVectors, | |
| 8u); | |
| } | |
| } | |
| } | |
| return (isPromotableIntegerTypeForABI(Ty) && isDarwinPCS() | |
| ? ABIArgInfo::getExtend(Ty, CGT.ConvertType(Ty)) | |
| : ABIArgInfo::getDirect()); | |
| } | |
| // Structures with either a non-trivial destructor or a non-trivial | |
| // copy constructor are always indirect. | |
| if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(Ty, getCXXABI())) { | |
| return getNaturalAlignIndirect( | |
| Ty, /*AddrSpace=*/getDataLayout().getAllocaAddrSpace(), | |
| /*ByVal=*/RAA == CGCXXABI::RAA_DirectInMemory); | |
| } | |
| // Empty records: | |
| // AAPCS64 does not say that empty records are ignored as arguments, | |
| // but other compilers do so in certain situations, and we copy that behavior. | |
| // Those situations are in fact language-mode-specific, which seems really | |
| // unfortunate, but it's something we just have to accept. If this doesn't | |
| // apply, just fall through to the standard argument-handling path. | |
| // Darwin overrides the psABI here to ignore all empty records in all modes. | |
| uint64_t Size = getContext().getTypeSize(Ty); | |
| bool IsEmpty = isEmptyRecord(getContext(), Ty, true); | |
| if (!Ty->isSVESizelessBuiltinType() && (IsEmpty || Size == 0)) { | |
| // Empty records are ignored in C mode, and in C++ on Darwin. | |
| if (!getContext().getLangOpts().CPlusPlus || isDarwinPCS()) | |
| return ABIArgInfo::getIgnore(); | |
| // In C++ mode, arguments which have sizeof() == 0 (which are non-standard | |
| // C++) are ignored. This isn't defined by any standard, so we copy GCC's | |
| // behaviour here. | |
| if (Size == 0) | |
| return ABIArgInfo::getIgnore(); | |
| } | |
| // Homogeneous Floating-point Aggregates (HFAs) need to be expanded. | |
| const Type *Base = nullptr; | |
| uint64_t Members = 0; | |
| bool IsWin64 = Kind == AArch64ABIKind::Win64 || | |
| CallingConvention == llvm::CallingConv::Win64; | |
| bool IsWinVariadic = IsWin64 && IsVariadicFn; | |
| // In variadic functions on Windows, all composite types are treated alike, | |
| // no special handling of HFAs/HVAs. | |
| if (!IsWinVariadic && isHomogeneousAggregate(Ty, Base, Members)) { | |
| NSRN = std::min(NSRN + Members, uint64_t(8)); | |
| if (Kind != AArch64ABIKind::AAPCS) | |
| return ABIArgInfo::getDirect( | |
| llvm::ArrayType::get(CGT.ConvertType(QualType(Base, 0)), Members)); | |
| // For HFAs/HVAs, cap the argument alignment to 16, otherwise | |
| // set it to 8 according to the AAPCS64 document. | |
| unsigned Align = | |
| getContext().getTypeUnadjustedAlignInChars(Ty).getQuantity(); | |
| Align = (Align >= 16) ? 16 : 8; | |
| return ABIArgInfo::getDirect( | |
| llvm::ArrayType::get(CGT.ConvertType(QualType(Base, 0)), Members), 0, | |
| nullptr, true, Align); | |
| } | |
| // In AAPCS named arguments of a Pure Scalable Type are passed expanded in | |
| // registers, or indirectly if there are not enough registers. | |
| if (Kind == AArch64ABIKind::AAPCS) { | |
| unsigned NVec = 0, NPred = 0; | |
| SmallVector<llvm::Type *> UnpaddedCoerceToSeq; | |
| if (passAsPureScalableType(Ty, NVec, NPred, UnpaddedCoerceToSeq) && | |
| (NVec + NPred) > 0) | |
| return coerceAndExpandPureScalableAggregate( | |
| Ty, IsNamedArg, NVec, NPred, UnpaddedCoerceToSeq, NSRN, NPRN); | |
| } | |
| // Aggregates <= 16 bytes are passed directly in registers or on the stack. | |
| if (Size <= 128) { | |
| unsigned Alignment; | |
| if (Kind == AArch64ABIKind::AAPCS) { | |
| Alignment = getContext().getTypeUnadjustedAlign(Ty); | |
| Alignment = Alignment < 128 ? 64 : 128; | |
| } else { | |
| Alignment = | |
| std::max(getContext().getTypeAlign(Ty), | |
| (unsigned)getTarget().getPointerWidth(LangAS::Default)); | |
| } | |
| Size = llvm::alignTo(Size, Alignment); | |
| // If the Aggregate is made up of pointers, use an array of pointers for the | |
| // coerced type. This prevents having to convert ptr2int->int2ptr through | |
| // the call, allowing alias analysis to produce better code. | |
| auto ContainsOnlyPointers = [&](const auto &Self, QualType Ty) { | |
| if (isEmptyRecord(getContext(), Ty, true)) | |
| return false; | |
| const auto *RD = Ty->getAsRecordDecl(); | |
| if (!RD) | |
| return false; | |
| if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) { | |
| for (const auto &I : CXXRD->bases()) | |
| if (!Self(Self, I.getType())) | |
| return false; | |
| } | |
| return all_of(RD->fields(), [&](FieldDecl *FD) { | |
| QualType FDTy = FD->getType(); | |
| if (FDTy->isArrayType()) | |
| FDTy = getContext().getBaseElementType(FDTy); | |
| return (FDTy->isPointerOrReferenceType() && | |
| getContext().getTypeSize(FDTy) == 64 && | |
| !FDTy->getPointeeType().hasAddressSpace()) || | |
| Self(Self, FDTy); | |
| }); | |
| }; | |
| // We use a pair of i64 for 16-byte aggregate with 8-byte alignment. | |
| // For aggregates with 16-byte alignment, we use i128. | |
| llvm::Type *BaseTy = llvm::Type::getIntNTy(getVMContext(), Alignment); | |
| if ((Size == 64 || Size == 128) && Alignment == 64 && | |
| ContainsOnlyPointers(ContainsOnlyPointers, Ty)) | |
| BaseTy = llvm::PointerType::getUnqual(getVMContext()); | |
| return ABIArgInfo::getDirect( | |
| Size == Alignment ? BaseTy | |
| : llvm::ArrayType::get(BaseTy, Size / Alignment)); | |
| } | |
| return getNaturalAlignIndirect(Ty, getDataLayout().getAllocaAddrSpace(), | |
| /*ByVal=*/false); | |
| } | |
| ABIArgInfo AArch64ABIInfo::classifyReturnType(QualType RetTy, | |
| bool IsVariadicFn) const { | |
| if (RetTy->isVoidType()) | |
| return ABIArgInfo::getIgnore(); | |
| if (const auto *VT = RetTy->getAs<VectorType>()) { | |
| if (VT->getVectorKind() == VectorKind::SveFixedLengthData || | |
| VT->getVectorKind() == VectorKind::SveFixedLengthPredicate) { | |
| unsigned NSRN = 0, NPRN = 0; | |
| return coerceIllegalVector(RetTy, NSRN, NPRN); | |
| } | |
| } | |
| // Large vector types should be returned via memory. | |
| if (RetTy->isVectorType() && getContext().getTypeSize(RetTy) > 128) | |
| return getNaturalAlignIndirect(RetTy, getDataLayout().getAllocaAddrSpace()); | |
| if (!passAsAggregateType(RetTy)) { | |
| // Treat an enum type as its underlying type. | |
| if (const auto *ED = RetTy->getAsEnumDecl()) | |
| RetTy = ED->getIntegerType(); | |
| if (const auto *EIT = RetTy->getAs<BitIntType>()) | |
| if (EIT->getNumBits() > 128) | |
| return getNaturalAlignIndirect(RetTy, | |
| getDataLayout().getAllocaAddrSpace()); | |
| return (isPromotableIntegerTypeForABI(RetTy) && isDarwinPCS() | |
| ? ABIArgInfo::getExtend(RetTy) | |
| : ABIArgInfo::getDirect()); | |
| } | |
| uint64_t Size = getContext().getTypeSize(RetTy); | |
| if (!RetTy->isSVESizelessBuiltinType() && | |
| (isEmptyRecord(getContext(), RetTy, true) || Size == 0)) | |
| return ABIArgInfo::getIgnore(); | |
| const Type *Base = nullptr; | |
| uint64_t Members = 0; | |
| if (isHomogeneousAggregate(RetTy, Base, Members) && | |
| !(getTarget().getTriple().getArch() == llvm::Triple::aarch64_32 && | |
| IsVariadicFn)) | |
| // Homogeneous Floating-point Aggregates (HFAs) are returned directly. | |
| return ABIArgInfo::getDirect(); | |
| // In AAPCS return values of a Pure Scalable type are treated as a single | |
| // named argument and passed expanded in registers, or indirectly if there are | |
| // not enough registers. | |
| if (Kind == AArch64ABIKind::AAPCS) { | |
| unsigned NSRN = 0, NPRN = 0; | |
| unsigned NVec = 0, NPred = 0; | |
| SmallVector<llvm::Type *> UnpaddedCoerceToSeq; | |
| if (passAsPureScalableType(RetTy, NVec, NPred, UnpaddedCoerceToSeq) && | |
| (NVec + NPred) > 0) | |
| return coerceAndExpandPureScalableAggregate( | |
| RetTy, /* IsNamedArg */ true, NVec, NPred, UnpaddedCoerceToSeq, NSRN, | |
| NPRN); | |
| } | |
| // Aggregates <= 16 bytes are returned directly in registers or on the stack. | |
| if (Size <= 128) { | |
| if (Size <= 64 && getDataLayout().isLittleEndian()) { | |
| // Composite types are returned in lower bits of a 64-bit register for LE, | |
| // and in higher bits for BE. However, integer types are always returned | |
| // in lower bits for both LE and BE, and they are not rounded up to | |
| // 64-bits. We can skip rounding up of composite types for LE, but not for | |
| // BE, otherwise composite types will be indistinguishable from integer | |
| // types. | |
| return ABIArgInfo::getDirect( | |
| llvm::IntegerType::get(getVMContext(), Size)); | |
| } | |
| unsigned Alignment = getContext().getTypeAlign(RetTy); | |
| Size = llvm::alignTo(Size, 64); // round up to multiple of 8 bytes | |
| // We use a pair of i64 for 16-byte aggregate with 8-byte alignment. | |
| // For aggregates with 16-byte alignment, we use i128. | |
| if (Alignment < 128 && Size == 128) { | |
| llvm::Type *BaseTy = llvm::Type::getInt64Ty(getVMContext()); | |
| return ABIArgInfo::getDirect(llvm::ArrayType::get(BaseTy, Size / 64)); | |
| } | |
| return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(), Size)); | |
| } | |
| return getNaturalAlignIndirect(RetTy, getDataLayout().getAllocaAddrSpace()); | |
| } | |
| /// isIllegalVectorType - check whether the vector type is legal for AArch64. | |
| bool AArch64ABIInfo::isIllegalVectorType(QualType Ty) const { | |
| if (const VectorType *VT = Ty->getAs<VectorType>()) { | |
| // Check whether VT is a fixed-length SVE vector. These types are | |
| // represented as scalable vectors in function args/return and must be | |
| // coerced from fixed vectors. | |
| if (VT->getVectorKind() == VectorKind::SveFixedLengthData || | |
| VT->getVectorKind() == VectorKind::SveFixedLengthPredicate) | |
| return true; | |
| // Check whether VT is legal. | |
| unsigned NumElements = VT->getNumElements(); | |
| uint64_t Size = getContext().getTypeSize(VT); | |
| // NumElements should be power of 2. | |
| if (!llvm::isPowerOf2_32(NumElements)) | |
| return true; | |
| // arm64_32 has to be compatible with the ARM logic here, which allows huge | |
| // vectors for some reason. | |
| llvm::Triple Triple = getTarget().getTriple(); | |
| if (Triple.getArch() == llvm::Triple::aarch64_32 && | |
| Triple.isOSBinFormatMachO()) | |
| return Size <= 32; | |
| return Size != 64 && (Size != 128 || NumElements == 1); | |
| } | |
| return false; | |
| } | |
| bool AArch64SwiftABIInfo::isLegalVectorType(CharUnits VectorSize, | |
| llvm::Type *EltTy, | |
| unsigned NumElts) const { | |
| if (!llvm::isPowerOf2_32(NumElts)) | |
| return false; | |
| if (VectorSize.getQuantity() != 8 && | |
| (VectorSize.getQuantity() != 16 || NumElts == 1)) | |
| return false; | |
| return true; | |
| } | |
| bool AArch64ABIInfo::isHomogeneousAggregateBaseType(QualType Ty) const { | |
| // For the soft-float ABI variant, no types are considered to be homogeneous | |
| // aggregates. | |
| if (isSoftFloat()) | |
| return false; | |
| // Homogeneous aggregates for AAPCS64 must have base types of a floating | |
| // point type or a short-vector type. This is the same as the 32-bit ABI, | |
| // but with the difference that any floating-point type is allowed, | |
| // including __fp16. | |
| if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) { | |
| if (BT->isFloatingPoint()) | |
| return true; | |
| } else if (const VectorType *VT = Ty->getAs<VectorType>()) { | |
| if (auto Kind = VT->getVectorKind(); | |
| Kind == VectorKind::SveFixedLengthData || | |
| Kind == VectorKind::SveFixedLengthPredicate) | |
| return false; | |
| unsigned VecSize = getContext().getTypeSize(VT); | |
| if (VecSize == 64 || VecSize == 128) | |
| return true; | |
| } | |
| return false; | |
| } | |
| bool AArch64ABIInfo::isHomogeneousAggregateSmallEnough(const Type *Base, | |
| uint64_t Members) const { | |
| return Members <= 4; | |
| } | |
| bool AArch64ABIInfo::isZeroLengthBitfieldPermittedInHomogeneousAggregate() | |
| const { | |
| // AAPCS64 says that the rule for whether something is a homogeneous | |
| // aggregate is applied to the output of the data layout decision. So | |
| // anything that doesn't affect the data layout also does not affect | |
| // homogeneity. In particular, zero-length bitfields don't stop a struct | |
| // being homogeneous. | |
| return true; | |
| } | |
| bool AArch64ABIInfo::passAsAggregateType(QualType Ty) const { | |
| if (Kind == AArch64ABIKind::AAPCS && Ty->isSVESizelessBuiltinType()) { | |
| const auto *BT = Ty->castAs<BuiltinType>(); | |
| return !BT->isSVECount() && | |
| getContext().getBuiltinVectorTypeInfo(BT).NumVectors > 1; | |
| } | |
| return isAggregateTypeForABI(Ty); | |
| } | |
| // Check if a type needs to be passed in registers as a Pure Scalable Type (as | |
| // defined by AAPCS64). Return the number of data vectors and the number of | |
| // predicate vectors in the type, into `NVec` and `NPred`, respectively. Upon | |
| // return `CoerceToSeq` contains an expanded sequence of LLVM IR types, one | |
| // element for each non-composite member. For practical purposes, limit the | |
| // length of `CoerceToSeq` to about 12 (the maximum that could possibly fit | |
| // in registers) and return false, the effect of which will be to pass the | |
| // argument under the rules for a large (> 128 bytes) composite. | |
| bool AArch64ABIInfo::passAsPureScalableType( | |
| QualType Ty, unsigned &NVec, unsigned &NPred, | |
| SmallVectorImpl<llvm::Type *> &CoerceToSeq) const { | |
| if (const ConstantArrayType *AT = getContext().getAsConstantArrayType(Ty)) { | |
| uint64_t NElt = AT->getZExtSize(); | |
| if (NElt == 0) | |
| return false; | |
| unsigned NV = 0, NP = 0; | |
| SmallVector<llvm::Type *> EltCoerceToSeq; | |
| if (!passAsPureScalableType(AT->getElementType(), NV, NP, EltCoerceToSeq)) | |
| return false; | |
| if (CoerceToSeq.size() + NElt * EltCoerceToSeq.size() > 12) | |
| return false; | |
| for (uint64_t I = 0; I < NElt; ++I) | |
| llvm::append_range(CoerceToSeq, EltCoerceToSeq); | |
| NVec += NElt * NV; | |
| NPred += NElt * NP; | |
| return true; | |
| } | |
| if (const RecordType *RT = Ty->getAsCanonical<RecordType>()) { | |
| // If the record cannot be passed in registers, then it's not a PST. | |
| if (CGCXXABI::RecordArgABI RAA = getRecordArgABI(RT, getCXXABI()); | |
| RAA != CGCXXABI::RAA_Default) | |
| return false; | |
| // Pure scalable types are never unions and never contain unions. | |
| const RecordDecl *RD = RT->getDecl()->getDefinitionOrSelf(); | |
| if (RD->isUnion()) | |
| return false; | |
| // If this is a C++ record, check the bases. | |
| if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) { | |
| for (const auto &I : CXXRD->bases()) { | |
| if (isEmptyRecord(getContext(), I.getType(), true)) | |
| continue; | |
| if (!passAsPureScalableType(I.getType(), NVec, NPred, CoerceToSeq)) | |
| return false; | |
| } | |
| } | |
| // Check members. | |
| for (const auto *FD : RD->fields()) { | |
| QualType FT = FD->getType(); | |
| if (isEmptyField(getContext(), FD, /* AllowArrays */ true)) | |
| continue; | |
| if (!passAsPureScalableType(FT, NVec, NPred, CoerceToSeq)) | |
| return false; | |
| } | |
| return true; | |
| } | |
| if (const auto *VT = Ty->getAs<VectorType>()) { | |
| if (VT->getVectorKind() == VectorKind::SveFixedLengthPredicate) { | |
| ++NPred; | |
| if (CoerceToSeq.size() + 1 > 12) | |
| return false; | |
| CoerceToSeq.push_back(convertFixedToScalableVectorType(VT)); | |
| return true; | |
| } | |
| if (VT->getVectorKind() == VectorKind::SveFixedLengthData) { | |
| ++NVec; | |
| if (CoerceToSeq.size() + 1 > 12) | |
| return false; | |
| CoerceToSeq.push_back(convertFixedToScalableVectorType(VT)); | |
| return true; | |
| } | |
| return false; | |
| } | |
| if (!Ty->isBuiltinType()) | |
| return false; | |
| bool isPredicate; | |
| switch (Ty->castAs<BuiltinType>()->getKind()) { | |
| #define SVE_VECTOR_TYPE(Name, MangledName, Id, SingletonId) \ | |
| case BuiltinType::Id: \ | |
| isPredicate = false; \ | |
| break; | |
| #define SVE_PREDICATE_TYPE(Name, MangledName, Id, SingletonId) \ | |
| case BuiltinType::Id: \ | |
| isPredicate = true; \ | |
| break; | |
| #include "clang/Basic/AArch64ACLETypes.def" | |
| default: | |
| return false; | |
| } | |
| ASTContext::BuiltinVectorTypeInfo Info = | |
| getContext().getBuiltinVectorTypeInfo(cast<BuiltinType>(Ty)); | |
| assert(Info.NumVectors > 0 && Info.NumVectors <= 4 && | |
| "Expected 1, 2, 3 or 4 vectors!"); | |
| if (isPredicate) | |
| NPred += Info.NumVectors; | |
| else | |
| NVec += Info.NumVectors; | |
| llvm::Type *EltTy = Info.ElementType->isMFloat8Type() | |
| ? llvm::Type::getInt8Ty(getVMContext()) | |
| : CGT.ConvertType(Info.ElementType); | |
| auto *VTy = llvm::ScalableVectorType::get(EltTy, Info.EC.getKnownMinValue()); | |
| if (CoerceToSeq.size() + Info.NumVectors > 12) | |
| return false; | |
| std::fill_n(std::back_inserter(CoerceToSeq), Info.NumVectors, VTy); | |
| return true; | |
| } | |
| // Expand an LLVM IR type into a sequence with a element for each non-struct, | |
| // non-array member of the type, with the exception of the padding types, which | |
| // are retained. | |
| void AArch64ABIInfo::flattenType( | |
| llvm::Type *Ty, SmallVectorImpl<llvm::Type *> &Flattened) const { | |
| if (ABIArgInfo::isPaddingForCoerceAndExpand(Ty)) { | |
| Flattened.push_back(Ty); | |
| return; | |
| } | |
| if (const auto *AT = dyn_cast<llvm::ArrayType>(Ty)) { | |
| uint64_t NElt = AT->getNumElements(); | |
| if (NElt == 0) | |
| return; | |
| SmallVector<llvm::Type *> EltFlattened; | |
| flattenType(AT->getElementType(), EltFlattened); | |
| for (uint64_t I = 0; I < NElt; ++I) | |
| llvm::append_range(Flattened, EltFlattened); | |
| return; | |
| } | |
| if (const auto *ST = dyn_cast<llvm::StructType>(Ty)) { | |
| for (auto *ET : ST->elements()) | |
| flattenType(ET, Flattened); | |
| return; | |
| } | |
| Flattened.push_back(Ty); | |
| } | |
| RValue AArch64ABIInfo::EmitAAPCSVAArg(Address VAListAddr, QualType Ty, | |
| CodeGenFunction &CGF, AArch64ABIKind Kind, | |
| AggValueSlot Slot) const { | |
| // These numbers are not used for variadic arguments, hence it doesn't matter | |
| // they don't retain their values across multiple calls to | |
| // `classifyArgumentType` here. | |
| unsigned NSRN = 0, NPRN = 0; | |
| ABIArgInfo AI = | |
| classifyArgumentType(Ty, /*IsVariadicFn=*/true, /* IsNamedArg */ false, | |
| CGF.CurFnInfo->getCallingConvention(), NSRN, NPRN); | |
| // Empty records are ignored for parameter passing purposes. | |
| if (AI.isIgnore()) | |
| return Slot.asRValue(); | |
| bool IsIndirect = AI.isIndirect(); | |
| llvm::Type *BaseTy = CGF.ConvertType(Ty); | |
| if (IsIndirect) | |
| BaseTy = llvm::PointerType::getUnqual(BaseTy->getContext()); | |
| else if (AI.getCoerceToType()) | |
| BaseTy = AI.getCoerceToType(); | |
| unsigned NumRegs = 1; | |
| if (llvm::ArrayType *ArrTy = dyn_cast<llvm::ArrayType>(BaseTy)) { | |
| BaseTy = ArrTy->getElementType(); | |
| NumRegs = ArrTy->getNumElements(); | |
| } | |
| bool IsFPR = | |
| !isSoftFloat() && (BaseTy->isFloatingPointTy() || BaseTy->isVectorTy()); | |
| // The AArch64 va_list type and handling is specified in the Procedure Call | |
| // Standard, section B.4: | |
| // | |
| // struct { | |
| // void *__stack; | |
| // void *__gr_top; | |
| // void *__vr_top; | |
| // int __gr_offs; | |
| // int __vr_offs; | |
| // }; | |
| llvm::BasicBlock *MaybeRegBlock = CGF.createBasicBlock("vaarg.maybe_reg"); | |
| llvm::BasicBlock *InRegBlock = CGF.createBasicBlock("vaarg.in_reg"); | |
| llvm::BasicBlock *OnStackBlock = CGF.createBasicBlock("vaarg.on_stack"); | |
| llvm::BasicBlock *ContBlock = CGF.createBasicBlock("vaarg.end"); | |
| CharUnits TySize = getContext().getTypeSizeInChars(Ty); | |
| CharUnits TyAlign = getContext().getTypeUnadjustedAlignInChars(Ty); | |
| Address reg_offs_p = Address::invalid(); | |
| llvm::Value *reg_offs = nullptr; | |
| int reg_top_index; | |
| int RegSize = IsIndirect ? 8 : TySize.getQuantity(); | |
| if (!IsFPR) { | |
| // 3 is the field number of __gr_offs | |
| reg_offs_p = CGF.Builder.CreateStructGEP(VAListAddr, 3, "gr_offs_p"); | |
| reg_offs = CGF.Builder.CreateLoad(reg_offs_p, "gr_offs"); | |
| reg_top_index = 1; // field number for __gr_top | |
| RegSize = llvm::alignTo(RegSize, 8); | |
| } else { | |
| // 4 is the field number of __vr_offs. | |
| reg_offs_p = CGF.Builder.CreateStructGEP(VAListAddr, 4, "vr_offs_p"); | |
| reg_offs = CGF.Builder.CreateLoad(reg_offs_p, "vr_offs"); | |
| reg_top_index = 2; // field number for __vr_top | |
| RegSize = 16 * NumRegs; | |
| } | |
| //======================================= | |
| // Find out where argument was passed | |
| //======================================= | |
| // If reg_offs >= 0 we're already using the stack for this type of | |
| // argument. We don't want to keep updating reg_offs (in case it overflows, | |
| // though anyone passing 2GB of arguments, each at most 16 bytes, deserves | |
| // whatever they get). | |
| llvm::Value *UsingStack = nullptr; | |
| UsingStack = CGF.Builder.CreateICmpSGE( | |
| reg_offs, llvm::ConstantInt::get(CGF.Int32Ty, 0)); | |
| CGF.Builder.CreateCondBr(UsingStack, OnStackBlock, MaybeRegBlock); | |
| // Otherwise, at least some kind of argument could go in these registers, the | |
| // question is whether this particular type is too big. | |
| CGF.EmitBlock(MaybeRegBlock); | |
| // Integer arguments may need to correct register alignment (for example a | |
| // "struct { __int128 a; };" gets passed in x_2N, x_{2N+1}). In this case we | |
| // align __gr_offs to calculate the potential address. | |
| if (!IsFPR && !IsIndirect && TyAlign.getQuantity() > 8) { | |
| int Align = TyAlign.getQuantity(); | |
| reg_offs = CGF.Builder.CreateAdd( | |
| reg_offs, llvm::ConstantInt::get(CGF.Int32Ty, Align - 1), | |
| "align_regoffs"); | |
| reg_offs = CGF.Builder.CreateAnd( | |
| reg_offs, llvm::ConstantInt::getSigned(CGF.Int32Ty, -Align), | |
| "aligned_regoffs"); | |
| } | |
| // Update the gr_offs/vr_offs pointer for next call to va_arg on this va_list. | |
| // The fact that this is done unconditionally reflects the fact that | |
| // allocating an argument to the stack also uses up all the remaining | |
| // registers of the appropriate kind. | |
| llvm::Value *NewOffset = nullptr; | |
| NewOffset = CGF.Builder.CreateAdd( | |
| reg_offs, llvm::ConstantInt::get(CGF.Int32Ty, RegSize), "new_reg_offs"); | |
| CGF.Builder.CreateStore(NewOffset, reg_offs_p); | |
| // Now we're in a position to decide whether this argument really was in | |
| // registers or not. | |
| llvm::Value *InRegs = nullptr; | |
| InRegs = CGF.Builder.CreateICmpSLE( | |
| NewOffset, llvm::ConstantInt::get(CGF.Int32Ty, 0), "inreg"); | |
| CGF.Builder.CreateCondBr(InRegs, InRegBlock, OnStackBlock); | |
| //======================================= | |
| // Argument was in registers | |
| //======================================= | |
| // Now we emit the code for if the argument was originally passed in | |
| // registers. First start the appropriate block: | |
| CGF.EmitBlock(InRegBlock); | |
| llvm::Value *reg_top = nullptr; | |
| Address reg_top_p = | |
| CGF.Builder.CreateStructGEP(VAListAddr, reg_top_index, "reg_top_p"); | |
| reg_top = CGF.Builder.CreateLoad(reg_top_p, "reg_top"); | |
| Address BaseAddr(CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, reg_top, reg_offs), | |
| CGF.Int8Ty, CharUnits::fromQuantity(IsFPR ? 16 : 8)); | |
| Address RegAddr = Address::invalid(); | |
| llvm::Type *MemTy = CGF.ConvertTypeForMem(Ty), *ElementTy = MemTy; | |
| if (IsIndirect) { | |
| // If it's been passed indirectly (actually a struct), whatever we find from | |
| // stored registers or on the stack will actually be a struct **. | |
| MemTy = llvm::PointerType::getUnqual(MemTy->getContext()); | |
| } | |
| const Type *Base = nullptr; | |
| uint64_t NumMembers = 0; | |
| bool IsHFA = isHomogeneousAggregate(Ty, Base, NumMembers); | |
| if (IsHFA && NumMembers > 1) { | |
| // Homogeneous aggregates passed in registers will have their elements split | |
| // and stored 16-bytes apart regardless of size (they're notionally in qN, | |
| // qN+1, ...). We reload and store into a temporary local variable | |
| // contiguously. | |
| assert(!IsIndirect && "Homogeneous aggregates should be passed directly"); | |
| auto BaseTyInfo = getContext().getTypeInfoInChars(QualType(Base, 0)); | |
| llvm::Type *BaseTy = CGF.ConvertType(QualType(Base, 0)); | |
| llvm::Type *HFATy = llvm::ArrayType::get(BaseTy, NumMembers); | |
| Address Tmp = CGF.CreateTempAlloca(HFATy, | |
| std::max(TyAlign, BaseTyInfo.Align)); | |
| // On big-endian platforms, the value will be right-aligned in its slot. | |
| int Offset = 0; | |
| if (CGF.CGM.getDataLayout().isBigEndian() && | |
| BaseTyInfo.Width.getQuantity() < 16) | |
| Offset = 16 - BaseTyInfo.Width.getQuantity(); | |
| for (unsigned i = 0; i < NumMembers; ++i) { | |
| CharUnits BaseOffset = CharUnits::fromQuantity(16 * i + Offset); | |
| Address LoadAddr = | |
| CGF.Builder.CreateConstInBoundsByteGEP(BaseAddr, BaseOffset); | |
| LoadAddr = LoadAddr.withElementType(BaseTy); | |
| Address StoreAddr = CGF.Builder.CreateConstArrayGEP(Tmp, i); | |
| llvm::Value *Elem = CGF.Builder.CreateLoad(LoadAddr); | |
| CGF.Builder.CreateStore(Elem, StoreAddr); | |
| } | |
| RegAddr = Tmp.withElementType(MemTy); | |
| } else { | |
| // Otherwise the object is contiguous in memory. | |
| // It might be right-aligned in its slot. | |
| CharUnits SlotSize = BaseAddr.getAlignment(); | |
| if (CGF.CGM.getDataLayout().isBigEndian() && !IsIndirect && | |
| (IsHFA || !isAggregateTypeForABI(Ty)) && | |
| TySize < SlotSize) { | |
| CharUnits Offset = SlotSize - TySize; | |
| BaseAddr = CGF.Builder.CreateConstInBoundsByteGEP(BaseAddr, Offset); | |
| } | |
| RegAddr = BaseAddr.withElementType(MemTy); | |
| } | |
| CGF.EmitBranch(ContBlock); | |
| //======================================= | |
| // Argument was on the stack | |
| //======================================= | |
| CGF.EmitBlock(OnStackBlock); | |
| Address stack_p = CGF.Builder.CreateStructGEP(VAListAddr, 0, "stack_p"); | |
| llvm::Value *OnStackPtr = CGF.Builder.CreateLoad(stack_p, "stack"); | |
| // Again, stack arguments may need realignment. In this case both integer and | |
| // floating-point ones might be affected. | |
| if (!IsIndirect && TyAlign.getQuantity() > 8) { | |
| OnStackPtr = emitRoundPointerUpToAlignment(CGF, OnStackPtr, TyAlign); | |
| } | |
| Address OnStackAddr = Address(OnStackPtr, CGF.Int8Ty, | |
| std::max(CharUnits::fromQuantity(8), TyAlign)); | |
| // All stack slots are multiples of 8 bytes. | |
| CharUnits StackSlotSize = CharUnits::fromQuantity(8); | |
| CharUnits StackSize; | |
| if (IsIndirect) | |
| StackSize = StackSlotSize; | |
| else | |
| StackSize = TySize.alignTo(StackSlotSize); | |
| llvm::Value *StackSizeC = CGF.Builder.getSize(StackSize); | |
| llvm::Value *NewStack = CGF.Builder.CreateInBoundsGEP( | |
| CGF.Int8Ty, OnStackPtr, StackSizeC, "new_stack"); | |
| // Write the new value of __stack for the next call to va_arg | |
| CGF.Builder.CreateStore(NewStack, stack_p); | |
| if (CGF.CGM.getDataLayout().isBigEndian() && !isAggregateTypeForABI(Ty) && | |
| TySize < StackSlotSize) { | |
| CharUnits Offset = StackSlotSize - TySize; | |
| OnStackAddr = CGF.Builder.CreateConstInBoundsByteGEP(OnStackAddr, Offset); | |
| } | |
| OnStackAddr = OnStackAddr.withElementType(MemTy); | |
| CGF.EmitBranch(ContBlock); | |
| //======================================= | |
| // Tidy up | |
| //======================================= | |
| CGF.EmitBlock(ContBlock); | |
| Address ResAddr = emitMergePHI(CGF, RegAddr, InRegBlock, OnStackAddr, | |
| OnStackBlock, "vaargs.addr"); | |
| if (IsIndirect) | |
| return CGF.EmitLoadOfAnyValue( | |
| CGF.MakeAddrLValue( | |
| Address(CGF.Builder.CreateLoad(ResAddr, "vaarg.addr"), ElementTy, | |
| TyAlign), | |
| Ty), | |
| Slot); | |
| return CGF.EmitLoadOfAnyValue(CGF.MakeAddrLValue(ResAddr, Ty), Slot); | |
| } | |
| RValue AArch64ABIInfo::EmitDarwinVAArg(Address VAListAddr, QualType Ty, | |
| CodeGenFunction &CGF, | |
| AggValueSlot Slot) const { | |
| // The backend's lowering doesn't support va_arg for aggregates or | |
| // illegal vector types. Lower VAArg here for these cases and use | |
| // the LLVM va_arg instruction for everything else. | |
| if (!isAggregateTypeForABI(Ty) && !isIllegalVectorType(Ty)) | |
| return CGF.EmitLoadOfAnyValue( | |
| CGF.MakeAddrLValue( | |
| EmitVAArgInstr(CGF, VAListAddr, Ty, ABIArgInfo::getDirect()), Ty), | |
| Slot); | |
| uint64_t PointerSize = getTarget().getPointerWidth(LangAS::Default) / 8; | |
| CharUnits SlotSize = CharUnits::fromQuantity(PointerSize); | |
| // Empty records are ignored for parameter passing purposes. | |
| if (isEmptyRecord(getContext(), Ty, true)) | |
| return Slot.asRValue(); | |
| // The size of the actual thing passed, which might end up just | |
| // being a pointer for indirect types. | |
| auto TyInfo = getContext().getTypeInfoInChars(Ty); | |
| // Arguments bigger than 16 bytes which aren't homogeneous | |
| // aggregates should be passed indirectly. | |
| bool IsIndirect = false; | |
| if (TyInfo.Width.getQuantity() > 16) { | |
| const Type *Base = nullptr; | |
| uint64_t Members = 0; | |
| IsIndirect = !isHomogeneousAggregate(Ty, Base, Members); | |
| } | |
| return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect, TyInfo, SlotSize, | |
| /*AllowHigherAlign*/ true, Slot); | |
| } | |
| RValue AArch64ABIInfo::EmitMSVAArg(CodeGenFunction &CGF, Address VAListAddr, | |
| QualType Ty, AggValueSlot Slot) const { | |
| bool IsIndirect = false; | |
| if (getTarget().getTriple().isWindowsArm64EC()) { | |
| // MS x64 ABI requirement: "Any argument that doesn't fit in 8 bytes, or is | |
| // not 1, 2, 4, or 8 bytes, must be passed by reference." | |
| uint64_t Width = getContext().getTypeSize(Ty); | |
| IsIndirect = Width > 64 || !llvm::isPowerOf2_64(Width); | |
| } else { | |
| // Composites larger than 16 bytes are passed by reference. | |
| if (isAggregateTypeForABI(Ty) && getContext().getTypeSize(Ty) > 128) | |
| IsIndirect = true; | |
| } | |
| return emitVoidPtrVAArg(CGF, VAListAddr, Ty, IsIndirect, | |
| CGF.getContext().getTypeInfoInChars(Ty), | |
| CharUnits::fromQuantity(8), | |
| /*allowHigherAlign*/ false, Slot); | |
| } | |
| static bool isStreamingCompatible(const FunctionDecl *F) { | |
| if (const auto *T = F->getType()->getAs<FunctionProtoType>()) | |
| return T->getAArch64SMEAttributes() & | |
| FunctionType::SME_PStateSMCompatibleMask; | |
| return false; | |
| } | |
| // Report an error if an argument or return value of type Ty would need to be | |
| // passed in a floating-point register. | |
| static void diagnoseIfNeedsFPReg(DiagnosticsEngine &Diags, | |
| const StringRef ABIName, | |
| const AArch64ABIInfo &ABIInfo, | |
| const QualType &Ty, const NamedDecl *D, | |
| SourceLocation loc) { | |
| const Type *HABase = nullptr; | |
| uint64_t HAMembers = 0; | |
| if (Ty->isFloatingType() || Ty->isVectorType() || | |
| ABIInfo.isHomogeneousAggregate(Ty, HABase, HAMembers)) { | |
| Diags.Report(loc, diag::err_target_unsupported_type_for_abi) | |
| << D->getDeclName() << Ty << ABIName; | |
| } | |
| } | |
| // If we are using a hard-float ABI, but do not have floating point registers, | |
| // then report an error for any function arguments or returns which would be | |
| // passed in floating-pint registers. | |
| void AArch64TargetCodeGenInfo::checkFunctionABI( | |
| CodeGenModule &CGM, const FunctionDecl *FuncDecl) const { | |
| const AArch64ABIInfo &ABIInfo = getABIInfo<AArch64ABIInfo>(); | |
| const TargetInfo &TI = ABIInfo.getContext().getTargetInfo(); | |
| if (!TI.hasFeature("fp") && !ABIInfo.isSoftFloat()) { | |
| diagnoseIfNeedsFPReg(CGM.getDiags(), TI.getABI(), ABIInfo, | |
| FuncDecl->getReturnType(), FuncDecl, | |
| FuncDecl->getLocation()); | |
| for (ParmVarDecl *PVD : FuncDecl->parameters()) { | |
| diagnoseIfNeedsFPReg(CGM.getDiags(), TI.getABI(), ABIInfo, PVD->getType(), | |
| PVD, FuncDecl->getLocation()); | |
| } | |
| } | |
| } | |
| enum class ArmSMEInlinability : uint8_t { | |
| Ok = 0, | |
| ErrorCalleeRequiresNewZA = 1 << 0, | |
| ErrorCalleeRequiresNewZT0 = 1 << 1, | |
| WarnIncompatibleStreamingModes = 1 << 2, | |
| ErrorIncompatibleStreamingModes = 1 << 3, | |
| IncompatibleStreamingModes = | |
| WarnIncompatibleStreamingModes | ErrorIncompatibleStreamingModes, | |
| LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/ErrorIncompatibleStreamingModes), | |
| }; | |
| /// Determines if there are any Arm SME ABI issues with inlining \p Callee into | |
| /// \p Caller. Returns the issue (if any) in the ArmSMEInlinability bit enum. | |
| static ArmSMEInlinability GetArmSMEInlinability(const FunctionDecl *Caller, | |
| const FunctionDecl *Callee) { | |
| bool CallerIsStreaming = | |
| IsArmStreamingFunction(Caller, /*IncludeLocallyStreaming=*/true); | |
| bool CalleeIsStreaming = | |
| IsArmStreamingFunction(Callee, /*IncludeLocallyStreaming=*/true); | |
| bool CallerIsStreamingCompatible = isStreamingCompatible(Caller); | |
| bool CalleeIsStreamingCompatible = isStreamingCompatible(Callee); | |
| ArmSMEInlinability Inlinability = ArmSMEInlinability::Ok; | |
| if (!CalleeIsStreamingCompatible && | |
| (CallerIsStreaming != CalleeIsStreaming || CallerIsStreamingCompatible)) { | |
| if (CalleeIsStreaming) | |
| Inlinability |= ArmSMEInlinability::ErrorIncompatibleStreamingModes; | |
| else | |
| Inlinability |= ArmSMEInlinability::WarnIncompatibleStreamingModes; | |
| } | |
| if (auto *NewAttr = Callee->getAttr<ArmNewAttr>()) { | |
| if (NewAttr->isNewZA()) | |
| Inlinability |= ArmSMEInlinability::ErrorCalleeRequiresNewZA; | |
| if (NewAttr->isNewZT0()) | |
| Inlinability |= ArmSMEInlinability::ErrorCalleeRequiresNewZT0; | |
| } | |
| return Inlinability; | |
| } | |
| void AArch64TargetCodeGenInfo::checkFunctionCallABIStreaming( | |
| CodeGenModule &CGM, SourceLocation CallLoc, const FunctionDecl *Caller, | |
| const FunctionDecl *Callee) const { | |
| if (!Caller || !Callee || !Callee->hasAttr<AlwaysInlineAttr>()) | |
| return; | |
| ArmSMEInlinability Inlinability = GetArmSMEInlinability(Caller, Callee); | |
| if ((Inlinability & ArmSMEInlinability::IncompatibleStreamingModes) != | |
| ArmSMEInlinability::Ok) | |
| CGM.getDiags().Report( | |
| CallLoc, | |
| (Inlinability & ArmSMEInlinability::ErrorIncompatibleStreamingModes) == | |
| ArmSMEInlinability::ErrorIncompatibleStreamingModes | |
| ? diag::err_function_always_inline_attribute_mismatch | |
| : diag::warn_function_always_inline_attribute_mismatch) | |
| << Caller->getDeclName() << Callee->getDeclName() << "streaming"; | |
| if ((Inlinability & ArmSMEInlinability::ErrorCalleeRequiresNewZA) == | |
| ArmSMEInlinability::ErrorCalleeRequiresNewZA) | |
| CGM.getDiags().Report(CallLoc, diag::err_function_always_inline_new_za) | |
| << Callee->getDeclName(); | |
| if ((Inlinability & ArmSMEInlinability::ErrorCalleeRequiresNewZT0) == | |
| ArmSMEInlinability::ErrorCalleeRequiresNewZT0) | |
| CGM.getDiags().Report(CallLoc, diag::err_function_always_inline_new_zt0) | |
| << Callee->getDeclName(); | |
| } | |
| // If the target does not have floating-point registers, but we are using a | |
| // hard-float ABI, there is no way to pass floating-point, vector or HFA values | |
| // to functions, so we report an error. | |
| void AArch64TargetCodeGenInfo::checkFunctionCallABISoftFloat( | |
| CodeGenModule &CGM, SourceLocation CallLoc, const FunctionDecl *Caller, | |
| const FunctionDecl *Callee, const CallArgList &Args, | |
| QualType ReturnType) const { | |
| const AArch64ABIInfo &ABIInfo = getABIInfo<AArch64ABIInfo>(); | |
| const TargetInfo &TI = ABIInfo.getContext().getTargetInfo(); | |
| if (!Caller || TI.hasFeature("fp") || ABIInfo.isSoftFloat()) | |
| return; | |
| diagnoseIfNeedsFPReg(CGM.getDiags(), TI.getABI(), ABIInfo, ReturnType, | |
| Callee ? Callee : Caller, CallLoc); | |
| for (const CallArg &Arg : Args) | |
| diagnoseIfNeedsFPReg(CGM.getDiags(), TI.getABI(), ABIInfo, Arg.getType(), | |
| Callee ? Callee : Caller, CallLoc); | |
| } | |
| void AArch64TargetCodeGenInfo::checkFunctionCallABI(CodeGenModule &CGM, | |
| SourceLocation CallLoc, | |
| const FunctionDecl *Caller, | |
| const FunctionDecl *Callee, | |
| const CallArgList &Args, | |
| QualType ReturnType) const { | |
| checkFunctionCallABIStreaming(CGM, CallLoc, Caller, Callee); | |
| checkFunctionCallABISoftFloat(CGM, CallLoc, Caller, Callee, Args, ReturnType); | |
| } | |
| bool AArch64TargetCodeGenInfo::wouldInliningViolateFunctionCallABI( | |
| const FunctionDecl *Caller, const FunctionDecl *Callee) const { | |
| return Caller && Callee && | |
| GetArmSMEInlinability(Caller, Callee) != ArmSMEInlinability::Ok; | |
| } | |
| void AArch64ABIInfo::appendAttributeMangling(TargetClonesAttr *Attr, | |
| unsigned Index, | |
| raw_ostream &Out) const { | |
| appendAttributeMangling(Attr->getFeatureStr(Index), Out); | |
| } | |
| void AArch64ABIInfo::appendAttributeMangling(StringRef AttrStr, | |
| raw_ostream &Out) const { | |
| if (AttrStr == "default") { | |
| Out << ".default"; | |
| return; | |
| } | |
| Out << "._"; | |
| SmallVector<StringRef, 8> Features; | |
| AttrStr.split(Features, "+"); | |
| for (auto &Feat : Features) | |
| Feat = Feat.trim(); | |
| llvm::sort(Features, [](const StringRef LHS, const StringRef RHS) { | |
| return LHS.compare(RHS) < 0; | |
| }); | |
| llvm::SmallDenseSet<StringRef, 8> UniqueFeats; | |
| for (auto &Feat : Features) | |
| if (getTarget().doesFeatureAffectCodeGen(Feat)) | |
| if (auto Ext = llvm::AArch64::parseFMVExtension(Feat)) | |
| if (UniqueFeats.insert(Ext->Name).second) | |
| Out << 'M' << Ext->Name; | |
| } | |
| std::unique_ptr<TargetCodeGenInfo> | |
| CodeGen::createAArch64TargetCodeGenInfo(CodeGenModule &CGM, | |
| AArch64ABIKind Kind) { | |
| return std::make_unique<AArch64TargetCodeGenInfo>(CGM, Kind); | |
| } | |
| std::unique_ptr<TargetCodeGenInfo> | |
| CodeGen::createWindowsAArch64TargetCodeGenInfo(CodeGenModule &CGM, | |
| AArch64ABIKind K) { | |
| return std::make_unique<WindowsAArch64TargetCodeGenInfo>(CGM, K); | |
| } |