| //===---- CIRGenBuiltinAArch64.cpp - Emit CIR for AArch64 builtins --------===// |
| // |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| // See https://llvm.org/LICENSE.txt for license information. |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| // |
| //===----------------------------------------------------------------------===// |
| // |
| // This contains code to emit ARM64 Builtin calls as CIR or a function call |
| // to be later resolved. |
| // |
| //===----------------------------------------------------------------------===// |
| |
| #include "CIRGenBuilder.h" |
| #include "CIRGenFunction.h" |
| #include "clang/Basic/TargetBuiltins.h" |
| #include "clang/CIR/MissingFeatures.h" |
| |
| // TODO(cir): once all builtins are covered, decide whether we still |
| // need to use LLVM intrinsics or if there's a better approach to follow. Right |
| // now the intrinsics are reused to make it convenient to encode all thousands |
| // of them and passing down to LLVM lowering. |
| #include "llvm/IR/Intrinsics.h" |
| #include "llvm/IR/IntrinsicsAArch64.h" |
| |
| #include "mlir/IR/Value.h" |
| #include "clang/AST/GlobalDecl.h" |
| #include "clang/Basic/Builtins.h" |
| |
| using namespace clang; |
| using namespace clang::CIRGen; |
| using namespace llvm; |
| |
| template <typename... Operands> |
| static mlir::Value emitIntrinsicCallOp(CIRGenBuilderTy &builder, |
| mlir::Location loc, const StringRef str, |
| const mlir::Type &resTy, |
| Operands &&...op) { |
| return cir::LLVMIntrinsicCallOp::create(builder, loc, |
| builder.getStringAttr(str), resTy, |
| std::forward<Operands>(op)...) |
| .getResult(); |
| } |
| |
| // Generate vscale * scalingFactor |
| static mlir::Value genVscaleTimesFactor(mlir::Location loc, |
| CIRGenBuilderTy builder, |
| mlir::Type cirTy, |
| int32_t scalingFactor) { |
| mlir::Value vscale = emitIntrinsicCallOp(builder, loc, "vscale", cirTy); |
| return builder.createNUWAMul(loc, vscale, |
| builder.getUInt64(scalingFactor, loc)); |
| } |
| |
| static bool aarch64SVEIntrinsicsProvenSorted = false; |
| |
| namespace { |
| struct AArch64BuiltinInfo { |
| unsigned builtinID; |
| unsigned llvmIntrinsic; |
| uint64_t typeModifier; |
| |
| bool operator<(unsigned rhsBuiltinID) const { |
| return builtinID < rhsBuiltinID; |
| } |
| bool operator<(const AArch64BuiltinInfo &te) const { |
| return builtinID < te.builtinID; |
| } |
| }; |
| } // end anonymous namespace |
| |
| #define SVEMAP1(NameBase, llvmIntrinsic, TypeModifier) \ |
| {SVE::BI__builtin_sve_##NameBase, Intrinsic::llvmIntrinsic, TypeModifier} |
| |
| #define SVEMAP2(NameBase, TypeModifier) \ |
| {SVE::BI__builtin_sve_##NameBase, 0, TypeModifier} |
| static const AArch64BuiltinInfo aarch64SVEIntrinsicMap[] = { |
| #define GET_SVE_LLVM_INTRINSIC_MAP |
| #include "clang/Basic/arm_sve_builtin_cg.inc" |
| #undef GET_SVE_LLVM_INTRINSIC_MAP |
| }; |
| |
| static const AArch64BuiltinInfo * |
| findARMVectorIntrinsicInMap(ArrayRef<AArch64BuiltinInfo> intrinsicMap, |
| unsigned builtinID, bool &mapProvenSorted) { |
| |
| #ifndef NDEBUG |
| if (!mapProvenSorted) { |
| assert(llvm::is_sorted(intrinsicMap)); |
| mapProvenSorted = true; |
| } |
| #endif |
| |
| const AArch64BuiltinInfo *info = llvm::lower_bound(intrinsicMap, builtinID); |
| |
| if (info != intrinsicMap.end() && info->builtinID == builtinID) |
| return info; |
| |
| return nullptr; |
| } |
| |
| bool CIRGenFunction::getAArch64SVEProcessedOperands( |
| unsigned builtinID, const CallExpr *expr, SmallVectorImpl<mlir::Value> &ops, |
| SVETypeFlags typeFlags) { |
| // Find out if any arguments are required to be integer constant expressions. |
| unsigned iceArguments = 0; |
| ASTContext::GetBuiltinTypeError error; |
| getContext().GetBuiltinType(builtinID, error, &iceArguments); |
| assert(error == ASTContext::GE_None && "Should not codegen an error"); |
| |
| for (unsigned i = 0, e = expr->getNumArgs(); i != e; i++) { |
| bool isIce = iceArguments & (1 << i); |
| mlir::Value arg = emitScalarExpr(expr->getArg(i)); |
| |
| if (isIce) { |
| cgm.errorNYI(expr->getSourceRange(), |
| std::string("unimplemented AArch64 builtin call: ") + |
| getContext().BuiltinInfo.getName(builtinID)); |
| } |
| |
| // FIXME: Handle types like svint16x2_t, which are currently incorrectly |
| // converted to i32. These should be treated as structs and unpacked. |
| |
| ops.push_back(arg); |
| } |
| return true; |
| } |
| |
| std::optional<mlir::Value> |
| CIRGenFunction::emitAArch64SVEBuiltinExpr(unsigned builtinID, |
| const CallExpr *expr) { |
| if (builtinID >= SVE::BI__builtin_sve_reinterpret_s8_s8 && |
| builtinID <= SVE::BI__builtin_sve_reinterpret_f64_f64_x4) { |
| cgm.errorNYI(expr->getSourceRange(), |
| std::string("unimplemented AArch64 builtin call: ") + |
| getContext().BuiltinInfo.getName(builtinID)); |
| return mlir::Value{}; |
| } |
| |
| assert(!cir::MissingFeatures::aarch64SVEIntrinsics()); |
| |
| auto *builtinIntrInfo = findARMVectorIntrinsicInMap( |
| aarch64SVEIntrinsicMap, builtinID, aarch64SVEIntrinsicsProvenSorted); |
| |
| // The operands of the builtin call |
| llvm::SmallVector<mlir::Value> ops; |
| |
| SVETypeFlags typeFlags(builtinIntrInfo->typeModifier); |
| if (!CIRGenFunction::getAArch64SVEProcessedOperands(builtinID, expr, ops, |
| typeFlags)) |
| return mlir::Value{}; |
| |
| if (typeFlags.isLoad() || typeFlags.isStore() || typeFlags.isGatherLoad() || |
| typeFlags.isScatterStore() || typeFlags.isPrefetch() || |
| typeFlags.isGatherPrefetch() || typeFlags.isStructLoad() || |
| typeFlags.isStructStore() || typeFlags.isTupleSet() || |
| typeFlags.isTupleGet() || typeFlags.isTupleCreate() || |
| typeFlags.isUndef()) |
| cgm.errorNYI(expr->getSourceRange(), |
| std::string("unimplemented AArch64 builtin call: ") + |
| getContext().BuiltinInfo.getName(builtinID)); |
| |
| mlir::Location loc = getLoc(expr->getExprLoc()); |
| |
| if (builtinIntrInfo->llvmIntrinsic != 0) { |
| std::string llvmIntrName(Intrinsic::getBaseName( |
| (llvm::Intrinsic::ID)builtinIntrInfo->llvmIntrinsic)); |
| |
| llvmIntrName.erase(0, /*std::strlen(".llvm")=*/5); |
| |
| return emitIntrinsicCallOp(builder, loc, llvmIntrName, |
| convertType(expr->getType()), |
| mlir::ValueRange{ops}); |
| } |
| |
| switch (builtinID) { |
| default: |
| return std::nullopt; |
| |
| case SVE::BI__builtin_sve_svreinterpret_b: |
| case SVE::BI__builtin_sve_svreinterpret_c: |
| case SVE::BI__builtin_sve_svpsel_lane_b8: |
| case SVE::BI__builtin_sve_svpsel_lane_b16: |
| case SVE::BI__builtin_sve_svpsel_lane_b32: |
| case SVE::BI__builtin_sve_svpsel_lane_b64: |
| case SVE::BI__builtin_sve_svpsel_lane_c8: |
| case SVE::BI__builtin_sve_svpsel_lane_c16: |
| case SVE::BI__builtin_sve_svpsel_lane_c32: |
| case SVE::BI__builtin_sve_svpsel_lane_c64: |
| case SVE::BI__builtin_sve_svmov_b_z: |
| case SVE::BI__builtin_sve_svnot_b_z: |
| case SVE::BI__builtin_sve_svmovlb_u16: |
| case SVE::BI__builtin_sve_svmovlb_u32: |
| case SVE::BI__builtin_sve_svmovlb_u64: |
| case SVE::BI__builtin_sve_svmovlb_s16: |
| case SVE::BI__builtin_sve_svmovlb_s32: |
| case SVE::BI__builtin_sve_svmovlb_s64: |
| case SVE::BI__builtin_sve_svmovlt_u16: |
| case SVE::BI__builtin_sve_svmovlt_u32: |
| case SVE::BI__builtin_sve_svmovlt_u64: |
| case SVE::BI__builtin_sve_svmovlt_s16: |
| case SVE::BI__builtin_sve_svmovlt_s32: |
| case SVE::BI__builtin_sve_svmovlt_s64: |
| case SVE::BI__builtin_sve_svpmullt_u16: |
| case SVE::BI__builtin_sve_svpmullt_u64: |
| case SVE::BI__builtin_sve_svpmullt_n_u16: |
| case SVE::BI__builtin_sve_svpmullt_n_u64: |
| case SVE::BI__builtin_sve_svpmullb_u16: |
| case SVE::BI__builtin_sve_svpmullb_u64: |
| case SVE::BI__builtin_sve_svpmullb_n_u16: |
| case SVE::BI__builtin_sve_svpmullb_n_u64: |
| |
| case SVE::BI__builtin_sve_svdup_n_b8: |
| case SVE::BI__builtin_sve_svdup_n_b16: |
| case SVE::BI__builtin_sve_svdup_n_b32: |
| case SVE::BI__builtin_sve_svdup_n_b64: |
| |
| case SVE::BI__builtin_sve_svdupq_n_b8: |
| case SVE::BI__builtin_sve_svdupq_n_b16: |
| case SVE::BI__builtin_sve_svdupq_n_b32: |
| case SVE::BI__builtin_sve_svdupq_n_b64: |
| case SVE::BI__builtin_sve_svdupq_n_u8: |
| case SVE::BI__builtin_sve_svdupq_n_s8: |
| case SVE::BI__builtin_sve_svdupq_n_u64: |
| case SVE::BI__builtin_sve_svdupq_n_f64: |
| case SVE::BI__builtin_sve_svdupq_n_s64: |
| case SVE::BI__builtin_sve_svdupq_n_u16: |
| case SVE::BI__builtin_sve_svdupq_n_f16: |
| case SVE::BI__builtin_sve_svdupq_n_bf16: |
| case SVE::BI__builtin_sve_svdupq_n_s16: |
| case SVE::BI__builtin_sve_svdupq_n_u32: |
| case SVE::BI__builtin_sve_svdupq_n_f32: |
| case SVE::BI__builtin_sve_svdupq_n_s32: |
| case SVE::BI__builtin_sve_svpfalse_b: |
| case SVE::BI__builtin_sve_svpfalse_c: |
| cgm.errorNYI(expr->getSourceRange(), |
| std::string("unimplemented AArch64 builtin call: ") + |
| getContext().BuiltinInfo.getName(builtinID)); |
| return mlir::Value{}; |
| |
| case SVE::BI__builtin_sve_svlen_u8: |
| case SVE::BI__builtin_sve_svlen_s8: |
| return genVscaleTimesFactor(loc, builder, convertType(expr->getType()), 16); |
| |
| case SVE::BI__builtin_sve_svlen_u16: |
| case SVE::BI__builtin_sve_svlen_s16: |
| case SVE::BI__builtin_sve_svlen_f16: |
| case SVE::BI__builtin_sve_svlen_bf16: |
| return genVscaleTimesFactor(loc, builder, convertType(expr->getType()), 8); |
| |
| case SVE::BI__builtin_sve_svlen_u32: |
| case SVE::BI__builtin_sve_svlen_s32: |
| case SVE::BI__builtin_sve_svlen_f32: |
| return genVscaleTimesFactor(loc, builder, convertType(expr->getType()), 4); |
| |
| case SVE::BI__builtin_sve_svlen_u64: |
| case SVE::BI__builtin_sve_svlen_s64: |
| case SVE::BI__builtin_sve_svlen_f64: |
| return genVscaleTimesFactor(loc, builder, convertType(expr->getType()), 2); |
| |
| case SVE::BI__builtin_sve_svtbl2_u8: |
| case SVE::BI__builtin_sve_svtbl2_s8: |
| case SVE::BI__builtin_sve_svtbl2_u16: |
| case SVE::BI__builtin_sve_svtbl2_s16: |
| case SVE::BI__builtin_sve_svtbl2_u32: |
| case SVE::BI__builtin_sve_svtbl2_s32: |
| case SVE::BI__builtin_sve_svtbl2_u64: |
| case SVE::BI__builtin_sve_svtbl2_s64: |
| case SVE::BI__builtin_sve_svtbl2_f16: |
| case SVE::BI__builtin_sve_svtbl2_bf16: |
| case SVE::BI__builtin_sve_svtbl2_f32: |
| case SVE::BI__builtin_sve_svtbl2_f64: |
| case SVE::BI__builtin_sve_svset_neonq_s8: |
| case SVE::BI__builtin_sve_svset_neonq_s16: |
| case SVE::BI__builtin_sve_svset_neonq_s32: |
| case SVE::BI__builtin_sve_svset_neonq_s64: |
| case SVE::BI__builtin_sve_svset_neonq_u8: |
| case SVE::BI__builtin_sve_svset_neonq_u16: |
| case SVE::BI__builtin_sve_svset_neonq_u32: |
| case SVE::BI__builtin_sve_svset_neonq_u64: |
| case SVE::BI__builtin_sve_svset_neonq_f16: |
| case SVE::BI__builtin_sve_svset_neonq_f32: |
| case SVE::BI__builtin_sve_svset_neonq_f64: |
| case SVE::BI__builtin_sve_svset_neonq_bf16: |
| case SVE::BI__builtin_sve_svget_neonq_s8: |
| case SVE::BI__builtin_sve_svget_neonq_s16: |
| case SVE::BI__builtin_sve_svget_neonq_s32: |
| case SVE::BI__builtin_sve_svget_neonq_s64: |
| case SVE::BI__builtin_sve_svget_neonq_u8: |
| case SVE::BI__builtin_sve_svget_neonq_u16: |
| case SVE::BI__builtin_sve_svget_neonq_u32: |
| case SVE::BI__builtin_sve_svget_neonq_u64: |
| case SVE::BI__builtin_sve_svget_neonq_f16: |
| case SVE::BI__builtin_sve_svget_neonq_f32: |
| case SVE::BI__builtin_sve_svget_neonq_f64: |
| case SVE::BI__builtin_sve_svget_neonq_bf16: |
| case SVE::BI__builtin_sve_svdup_neonq_s8: |
| case SVE::BI__builtin_sve_svdup_neonq_s16: |
| case SVE::BI__builtin_sve_svdup_neonq_s32: |
| case SVE::BI__builtin_sve_svdup_neonq_s64: |
| case SVE::BI__builtin_sve_svdup_neonq_u8: |
| case SVE::BI__builtin_sve_svdup_neonq_u16: |
| case SVE::BI__builtin_sve_svdup_neonq_u32: |
| case SVE::BI__builtin_sve_svdup_neonq_u64: |
| case SVE::BI__builtin_sve_svdup_neonq_f16: |
| case SVE::BI__builtin_sve_svdup_neonq_f32: |
| case SVE::BI__builtin_sve_svdup_neonq_f64: |
| case SVE::BI__builtin_sve_svdup_neonq_bf16: |
| cgm.errorNYI(expr->getSourceRange(), |
| std::string("unimplemented AArch64 builtin call: ") + |
| getContext().BuiltinInfo.getName(builtinID)); |
| return mlir::Value{}; |
| } |
| |
| // Unreachable: All cases in the switch above return. |
| } |
| |
| std::optional<mlir::Value> |
| CIRGenFunction::emitAArch64SMEBuiltinExpr(unsigned builtinID, |
| const CallExpr *expr) { |
| assert(!cir::MissingFeatures::aarch64SMEIntrinsics()); |
| |
| cgm.errorNYI(expr->getSourceRange(), |
| std::string("unimplemented AArch64 builtin call: ") + |
| getContext().BuiltinInfo.getName(builtinID)); |
| return mlir::Value{}; |
| } |
| |
| // Some intrinsics are equivalent for codegen. |
| static const std::pair<unsigned, unsigned> neonEquivalentIntrinsicMap[] = { |
| { |
| NEON::BI__builtin_neon_splat_lane_bf16, |
| NEON::BI__builtin_neon_splat_lane_v, |
| }, |
| { |
| NEON::BI__builtin_neon_splat_laneq_bf16, |
| NEON::BI__builtin_neon_splat_laneq_v, |
| }, |
| { |
| NEON::BI__builtin_neon_splatq_lane_bf16, |
| NEON::BI__builtin_neon_splatq_lane_v, |
| }, |
| { |
| NEON::BI__builtin_neon_splatq_laneq_bf16, |
| NEON::BI__builtin_neon_splatq_laneq_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vabd_f16, |
| NEON::BI__builtin_neon_vabd_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vabdq_f16, |
| NEON::BI__builtin_neon_vabdq_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vabs_f16, |
| NEON::BI__builtin_neon_vabs_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vabsq_f16, |
| NEON::BI__builtin_neon_vabsq_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vcage_f16, |
| NEON::BI__builtin_neon_vcage_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vcageq_f16, |
| NEON::BI__builtin_neon_vcageq_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vcagt_f16, |
| NEON::BI__builtin_neon_vcagt_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vcagtq_f16, |
| NEON::BI__builtin_neon_vcagtq_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vcale_f16, |
| NEON::BI__builtin_neon_vcale_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vcaleq_f16, |
| NEON::BI__builtin_neon_vcaleq_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vcalt_f16, |
| NEON::BI__builtin_neon_vcalt_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vcaltq_f16, |
| NEON::BI__builtin_neon_vcaltq_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vceqz_f16, |
| NEON::BI__builtin_neon_vceqz_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vceqzq_f16, |
| NEON::BI__builtin_neon_vceqzq_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vcgez_f16, |
| NEON::BI__builtin_neon_vcgez_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vcgezq_f16, |
| NEON::BI__builtin_neon_vcgezq_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vcgtz_f16, |
| NEON::BI__builtin_neon_vcgtz_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vcgtzq_f16, |
| NEON::BI__builtin_neon_vcgtzq_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vclez_f16, |
| NEON::BI__builtin_neon_vclez_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vclezq_f16, |
| NEON::BI__builtin_neon_vclezq_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vcltz_f16, |
| NEON::BI__builtin_neon_vcltz_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vcltzq_f16, |
| NEON::BI__builtin_neon_vcltzq_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vfma_f16, |
| NEON::BI__builtin_neon_vfma_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vfma_lane_f16, |
| NEON::BI__builtin_neon_vfma_lane_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vfma_laneq_f16, |
| NEON::BI__builtin_neon_vfma_laneq_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vfmaq_f16, |
| NEON::BI__builtin_neon_vfmaq_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vfmaq_lane_f16, |
| NEON::BI__builtin_neon_vfmaq_lane_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vfmaq_laneq_f16, |
| NEON::BI__builtin_neon_vfmaq_laneq_v, |
| }, |
| {NEON::BI__builtin_neon_vld1_bf16_x2, NEON::BI__builtin_neon_vld1_x2_v}, |
| {NEON::BI__builtin_neon_vld1_bf16_x3, NEON::BI__builtin_neon_vld1_x3_v}, |
| {NEON::BI__builtin_neon_vld1_bf16_x4, NEON::BI__builtin_neon_vld1_x4_v}, |
| {NEON::BI__builtin_neon_vld1_bf16, NEON::BI__builtin_neon_vld1_v}, |
| {NEON::BI__builtin_neon_vld1_dup_bf16, NEON::BI__builtin_neon_vld1_dup_v}, |
| {NEON::BI__builtin_neon_vld1_lane_bf16, NEON::BI__builtin_neon_vld1_lane_v}, |
| {NEON::BI__builtin_neon_vld1q_bf16_x2, NEON::BI__builtin_neon_vld1q_x2_v}, |
| {NEON::BI__builtin_neon_vld1q_bf16_x3, NEON::BI__builtin_neon_vld1q_x3_v}, |
| {NEON::BI__builtin_neon_vld1q_bf16_x4, NEON::BI__builtin_neon_vld1q_x4_v}, |
| {NEON::BI__builtin_neon_vld1q_bf16, NEON::BI__builtin_neon_vld1q_v}, |
| {NEON::BI__builtin_neon_vld1q_dup_bf16, NEON::BI__builtin_neon_vld1q_dup_v}, |
| {NEON::BI__builtin_neon_vld1q_lane_bf16, |
| NEON::BI__builtin_neon_vld1q_lane_v}, |
| {NEON::BI__builtin_neon_vld2_bf16, NEON::BI__builtin_neon_vld2_v}, |
| {NEON::BI__builtin_neon_vld2_dup_bf16, NEON::BI__builtin_neon_vld2_dup_v}, |
| {NEON::BI__builtin_neon_vld2_lane_bf16, NEON::BI__builtin_neon_vld2_lane_v}, |
| {NEON::BI__builtin_neon_vld2q_bf16, NEON::BI__builtin_neon_vld2q_v}, |
| {NEON::BI__builtin_neon_vld2q_dup_bf16, NEON::BI__builtin_neon_vld2q_dup_v}, |
| {NEON::BI__builtin_neon_vld2q_lane_bf16, |
| NEON::BI__builtin_neon_vld2q_lane_v}, |
| {NEON::BI__builtin_neon_vld3_bf16, NEON::BI__builtin_neon_vld3_v}, |
| {NEON::BI__builtin_neon_vld3_dup_bf16, NEON::BI__builtin_neon_vld3_dup_v}, |
| {NEON::BI__builtin_neon_vld3_lane_bf16, NEON::BI__builtin_neon_vld3_lane_v}, |
| {NEON::BI__builtin_neon_vld3q_bf16, NEON::BI__builtin_neon_vld3q_v}, |
| {NEON::BI__builtin_neon_vld3q_dup_bf16, NEON::BI__builtin_neon_vld3q_dup_v}, |
| {NEON::BI__builtin_neon_vld3q_lane_bf16, |
| NEON::BI__builtin_neon_vld3q_lane_v}, |
| {NEON::BI__builtin_neon_vld4_bf16, NEON::BI__builtin_neon_vld4_v}, |
| {NEON::BI__builtin_neon_vld4_dup_bf16, NEON::BI__builtin_neon_vld4_dup_v}, |
| {NEON::BI__builtin_neon_vld4_lane_bf16, NEON::BI__builtin_neon_vld4_lane_v}, |
| {NEON::BI__builtin_neon_vld4q_bf16, NEON::BI__builtin_neon_vld4q_v}, |
| {NEON::BI__builtin_neon_vld4q_dup_bf16, NEON::BI__builtin_neon_vld4q_dup_v}, |
| {NEON::BI__builtin_neon_vld4q_lane_bf16, |
| NEON::BI__builtin_neon_vld4q_lane_v}, |
| { |
| NEON::BI__builtin_neon_vmax_f16, |
| NEON::BI__builtin_neon_vmax_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vmaxnm_f16, |
| NEON::BI__builtin_neon_vmaxnm_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vmaxnmq_f16, |
| NEON::BI__builtin_neon_vmaxnmq_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vmaxq_f16, |
| NEON::BI__builtin_neon_vmaxq_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vmin_f16, |
| NEON::BI__builtin_neon_vmin_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vminnm_f16, |
| NEON::BI__builtin_neon_vminnm_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vminnmq_f16, |
| NEON::BI__builtin_neon_vminnmq_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vminq_f16, |
| NEON::BI__builtin_neon_vminq_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vmulx_f16, |
| NEON::BI__builtin_neon_vmulx_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vmulxq_f16, |
| NEON::BI__builtin_neon_vmulxq_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vpadd_f16, |
| NEON::BI__builtin_neon_vpadd_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vpaddq_f16, |
| NEON::BI__builtin_neon_vpaddq_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vpmax_f16, |
| NEON::BI__builtin_neon_vpmax_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vpmaxnm_f16, |
| NEON::BI__builtin_neon_vpmaxnm_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vpmaxnmq_f16, |
| NEON::BI__builtin_neon_vpmaxnmq_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vpmaxq_f16, |
| NEON::BI__builtin_neon_vpmaxq_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vpmin_f16, |
| NEON::BI__builtin_neon_vpmin_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vpminnm_f16, |
| NEON::BI__builtin_neon_vpminnm_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vpminnmq_f16, |
| NEON::BI__builtin_neon_vpminnmq_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vpminq_f16, |
| NEON::BI__builtin_neon_vpminq_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vrecpe_f16, |
| NEON::BI__builtin_neon_vrecpe_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vrecpeq_f16, |
| NEON::BI__builtin_neon_vrecpeq_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vrecps_f16, |
| NEON::BI__builtin_neon_vrecps_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vrecpsq_f16, |
| NEON::BI__builtin_neon_vrecpsq_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vrnd_f16, |
| NEON::BI__builtin_neon_vrnd_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vrnda_f16, |
| NEON::BI__builtin_neon_vrnda_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vrndaq_f16, |
| NEON::BI__builtin_neon_vrndaq_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vrndi_f16, |
| NEON::BI__builtin_neon_vrndi_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vrndiq_f16, |
| NEON::BI__builtin_neon_vrndiq_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vrndm_f16, |
| NEON::BI__builtin_neon_vrndm_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vrndmq_f16, |
| NEON::BI__builtin_neon_vrndmq_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vrndn_f16, |
| NEON::BI__builtin_neon_vrndn_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vrndnq_f16, |
| NEON::BI__builtin_neon_vrndnq_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vrndp_f16, |
| NEON::BI__builtin_neon_vrndp_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vrndpq_f16, |
| NEON::BI__builtin_neon_vrndpq_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vrndq_f16, |
| NEON::BI__builtin_neon_vrndq_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vrndx_f16, |
| NEON::BI__builtin_neon_vrndx_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vrndxq_f16, |
| NEON::BI__builtin_neon_vrndxq_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vrsqrte_f16, |
| NEON::BI__builtin_neon_vrsqrte_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vrsqrteq_f16, |
| NEON::BI__builtin_neon_vrsqrteq_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vrsqrts_f16, |
| NEON::BI__builtin_neon_vrsqrts_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vrsqrtsq_f16, |
| NEON::BI__builtin_neon_vrsqrtsq_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vsqrt_f16, |
| NEON::BI__builtin_neon_vsqrt_v, |
| }, |
| { |
| NEON::BI__builtin_neon_vsqrtq_f16, |
| NEON::BI__builtin_neon_vsqrtq_v, |
| }, |
| {NEON::BI__builtin_neon_vst1_bf16_x2, NEON::BI__builtin_neon_vst1_x2_v}, |
| {NEON::BI__builtin_neon_vst1_bf16_x3, NEON::BI__builtin_neon_vst1_x3_v}, |
| {NEON::BI__builtin_neon_vst1_bf16_x4, NEON::BI__builtin_neon_vst1_x4_v}, |
| {NEON::BI__builtin_neon_vst1_bf16, NEON::BI__builtin_neon_vst1_v}, |
| {NEON::BI__builtin_neon_vst1_lane_bf16, NEON::BI__builtin_neon_vst1_lane_v}, |
| {NEON::BI__builtin_neon_vst1q_bf16_x2, NEON::BI__builtin_neon_vst1q_x2_v}, |
| {NEON::BI__builtin_neon_vst1q_bf16_x3, NEON::BI__builtin_neon_vst1q_x3_v}, |
| {NEON::BI__builtin_neon_vst1q_bf16_x4, NEON::BI__builtin_neon_vst1q_x4_v}, |
| {NEON::BI__builtin_neon_vst1q_bf16, NEON::BI__builtin_neon_vst1q_v}, |
| {NEON::BI__builtin_neon_vst1q_lane_bf16, |
| NEON::BI__builtin_neon_vst1q_lane_v}, |
| {NEON::BI__builtin_neon_vst2_bf16, NEON::BI__builtin_neon_vst2_v}, |
| {NEON::BI__builtin_neon_vst2_lane_bf16, NEON::BI__builtin_neon_vst2_lane_v}, |
| {NEON::BI__builtin_neon_vst2q_bf16, NEON::BI__builtin_neon_vst2q_v}, |
| {NEON::BI__builtin_neon_vst2q_lane_bf16, |
| NEON::BI__builtin_neon_vst2q_lane_v}, |
| {NEON::BI__builtin_neon_vst3_bf16, NEON::BI__builtin_neon_vst3_v}, |
| {NEON::BI__builtin_neon_vst3_lane_bf16, NEON::BI__builtin_neon_vst3_lane_v}, |
| {NEON::BI__builtin_neon_vst3q_bf16, NEON::BI__builtin_neon_vst3q_v}, |
| {NEON::BI__builtin_neon_vst3q_lane_bf16, |
| NEON::BI__builtin_neon_vst3q_lane_v}, |
| {NEON::BI__builtin_neon_vst4_bf16, NEON::BI__builtin_neon_vst4_v}, |
| {NEON::BI__builtin_neon_vst4_lane_bf16, NEON::BI__builtin_neon_vst4_lane_v}, |
| {NEON::BI__builtin_neon_vst4q_bf16, NEON::BI__builtin_neon_vst4q_v}, |
| {NEON::BI__builtin_neon_vst4q_lane_bf16, |
| NEON::BI__builtin_neon_vst4q_lane_v}, |
| // The mangling rules cause us to have one ID for each type for |
| // vldap1(q)_lane and vstl1(q)_lane, but codegen is equivalent for all of |
| // them. Choose an arbitrary one to be handled as tha canonical variation. |
| {NEON::BI__builtin_neon_vldap1_lane_u64, |
| NEON::BI__builtin_neon_vldap1_lane_s64}, |
| {NEON::BI__builtin_neon_vldap1_lane_f64, |
| NEON::BI__builtin_neon_vldap1_lane_s64}, |
| {NEON::BI__builtin_neon_vldap1_lane_p64, |
| NEON::BI__builtin_neon_vldap1_lane_s64}, |
| {NEON::BI__builtin_neon_vldap1q_lane_u64, |
| NEON::BI__builtin_neon_vldap1q_lane_s64}, |
| {NEON::BI__builtin_neon_vldap1q_lane_f64, |
| NEON::BI__builtin_neon_vldap1q_lane_s64}, |
| {NEON::BI__builtin_neon_vldap1q_lane_p64, |
| NEON::BI__builtin_neon_vldap1q_lane_s64}, |
| {NEON::BI__builtin_neon_vstl1_lane_u64, |
| NEON::BI__builtin_neon_vstl1_lane_s64}, |
| {NEON::BI__builtin_neon_vstl1_lane_f64, |
| NEON::BI__builtin_neon_vstl1_lane_s64}, |
| {NEON::BI__builtin_neon_vstl1_lane_p64, |
| NEON::BI__builtin_neon_vstl1_lane_s64}, |
| {NEON::BI__builtin_neon_vstl1q_lane_u64, |
| NEON::BI__builtin_neon_vstl1q_lane_s64}, |
| {NEON::BI__builtin_neon_vstl1q_lane_f64, |
| NEON::BI__builtin_neon_vstl1q_lane_s64}, |
| {NEON::BI__builtin_neon_vstl1q_lane_p64, |
| NEON::BI__builtin_neon_vstl1q_lane_s64}, |
| }; |
| |
| std::optional<mlir::Value> |
| CIRGenFunction::emitAArch64BuiltinExpr(unsigned builtinID, const CallExpr *expr, |
| ReturnValueSlot returnValue, |
| llvm::Triple::ArchType arch) { |
| if (builtinID >= clang::AArch64::FirstSVEBuiltin && |
| builtinID <= clang::AArch64::LastSVEBuiltin) |
| return emitAArch64SVEBuiltinExpr(builtinID, expr); |
| |
| if (builtinID >= clang::AArch64::FirstSMEBuiltin && |
| builtinID <= clang::AArch64::LastSMEBuiltin) |
| return emitAArch64SMEBuiltinExpr(builtinID, expr); |
| |
| if (builtinID == Builtin::BI__builtin_cpu_supports) { |
| cgm.errorNYI(expr->getSourceRange(), |
| std::string("unimplemented AArch64 builtin call: ") + |
| getContext().BuiltinInfo.getName(builtinID)); |
| return mlir::Value{}; |
| } |
| |
| switch (builtinID) { |
| default: |
| break; |
| case clang::AArch64::BI__builtin_arm_nop: |
| case clang::AArch64::BI__builtin_arm_yield: |
| case clang::AArch64::BI__yield: |
| case clang::AArch64::BI__builtin_arm_wfe: |
| case clang::AArch64::BI__wfe: |
| case clang::AArch64::BI__builtin_arm_wfi: |
| case clang::AArch64::BI__wfi: |
| case clang::AArch64::BI__builtin_arm_sev: |
| case clang::AArch64::BI__sev: |
| case clang::AArch64::BI__builtin_arm_sevl: |
| case clang::AArch64::BI__sevl: |
| cgm.errorNYI(expr->getSourceRange(), |
| std::string("unimplemented AArch64 builtin call: ") + |
| getContext().BuiltinInfo.getName(builtinID)); |
| return mlir::Value{}; |
| } |
| |
| if (builtinID == clang::AArch64::BI__builtin_arm_trap) { |
| cgm.errorNYI(expr->getSourceRange(), |
| std::string("unimplemented AArch64 builtin call: ") + |
| getContext().BuiltinInfo.getName(builtinID)); |
| return mlir::Value{}; |
| } |
| |
| if (builtinID == clang::AArch64::BI__builtin_arm_get_sme_state) { |
| cgm.errorNYI(expr->getSourceRange(), |
| std::string("unimplemented AArch64 builtin call: ") + |
| getContext().BuiltinInfo.getName(builtinID)); |
| return mlir::Value{}; |
| } |
| |
| if (builtinID == clang::AArch64::BI__builtin_arm_rbit) { |
| cgm.errorNYI(expr->getSourceRange(), |
| std::string("unimplemented AArch64 builtin call: ") + |
| getContext().BuiltinInfo.getName(builtinID)); |
| return mlir::Value{}; |
| } |
| if (builtinID == clang::AArch64::BI__builtin_arm_rbit64) { |
| cgm.errorNYI(expr->getSourceRange(), |
| std::string("unimplemented AArch64 builtin call: ") + |
| getContext().BuiltinInfo.getName(builtinID)); |
| return mlir::Value{}; |
| } |
| |
| if (builtinID == clang::AArch64::BI__builtin_arm_clz || |
| builtinID == clang::AArch64::BI__builtin_arm_clz64) { |
| cgm.errorNYI(expr->getSourceRange(), |
| std::string("unimplemented AArch64 builtin call: ") + |
| getContext().BuiltinInfo.getName(builtinID)); |
| return mlir::Value{}; |
| } |
| |
| if (builtinID == clang::AArch64::BI__builtin_arm_cls) { |
| cgm.errorNYI(expr->getSourceRange(), |
| std::string("unimplemented AArch64 builtin call: ") + |
| getContext().BuiltinInfo.getName(builtinID)); |
| return mlir::Value{}; |
| } |
| if (builtinID == clang::AArch64::BI__builtin_arm_cls64) { |
| cgm.errorNYI(expr->getSourceRange(), |
| std::string("unimplemented AArch64 builtin call: ") + |
| getContext().BuiltinInfo.getName(builtinID)); |
| return mlir::Value{}; |
| } |
| |
| if (builtinID == clang::AArch64::BI__builtin_arm_rint32zf || |
| builtinID == clang::AArch64::BI__builtin_arm_rint32z) { |
| cgm.errorNYI(expr->getSourceRange(), |
| std::string("unimplemented AArch64 builtin call: ") + |
| getContext().BuiltinInfo.getName(builtinID)); |
| return mlir::Value{}; |
| } |
| |
| if (builtinID == clang::AArch64::BI__builtin_arm_rint64zf || |
| builtinID == clang::AArch64::BI__builtin_arm_rint64z) { |
| cgm.errorNYI(expr->getSourceRange(), |
| std::string("unimplemented AArch64 builtin call: ") + |
| getContext().BuiltinInfo.getName(builtinID)); |
| return mlir::Value{}; |
| } |
| |
| if (builtinID == clang::AArch64::BI__builtin_arm_rint32xf || |
| builtinID == clang::AArch64::BI__builtin_arm_rint32x) { |
| cgm.errorNYI(expr->getSourceRange(), |
| std::string("unimplemented AArch64 builtin call: ") + |
| getContext().BuiltinInfo.getName(builtinID)); |
| return mlir::Value{}; |
| } |
| |
| if (builtinID == clang::AArch64::BI__builtin_arm_rint64xf || |
| builtinID == clang::AArch64::BI__builtin_arm_rint64x) { |
| cgm.errorNYI(expr->getSourceRange(), |
| std::string("unimplemented AArch64 builtin call: ") + |
| getContext().BuiltinInfo.getName(builtinID)); |
| return mlir::Value{}; |
| } |
| |
| if (builtinID == clang::AArch64::BI__builtin_arm_jcvt) { |
| cgm.errorNYI(expr->getSourceRange(), |
| std::string("unimplemented AArch64 builtin call: ") + |
| getContext().BuiltinInfo.getName(builtinID)); |
| return mlir::Value{}; |
| } |
| |
| if (builtinID == clang::AArch64::BI__builtin_arm_ld64b || |
| builtinID == clang::AArch64::BI__builtin_arm_st64b || |
| builtinID == clang::AArch64::BI__builtin_arm_st64bv || |
| builtinID == clang::AArch64::BI__builtin_arm_st64bv0) { |
| cgm.errorNYI(expr->getSourceRange(), |
| std::string("unimplemented AArch64 builtin call: ") + |
| getContext().BuiltinInfo.getName(builtinID)); |
| return mlir::Value{}; |
| } |
| |
| if (builtinID == clang::AArch64::BI__builtin_arm_rndr || |
| builtinID == clang::AArch64::BI__builtin_arm_rndrrs) { |
| cgm.errorNYI(expr->getSourceRange(), |
| std::string("unimplemented AArch64 builtin call: ") + |
| getContext().BuiltinInfo.getName(builtinID)); |
| return mlir::Value{}; |
| } |
| |
| if (builtinID == clang::AArch64::BI__clear_cache) { |
| cgm.errorNYI(expr->getSourceRange(), |
| std::string("unimplemented AArch64 builtin call: ") + |
| getContext().BuiltinInfo.getName(builtinID)); |
| return mlir::Value{}; |
| } |
| |
| if ((builtinID == clang::AArch64::BI__builtin_arm_ldrex || |
| builtinID == clang::AArch64::BI__builtin_arm_ldaex) && |
| getContext().getTypeSize(expr->getType()) == 128) { |
| cgm.errorNYI(expr->getSourceRange(), |
| std::string("unimplemented AArch64 builtin call: ") + |
| getContext().BuiltinInfo.getName(builtinID)); |
| return mlir::Value{}; |
| } |
| if (builtinID == clang::AArch64::BI__builtin_arm_ldrex || |
| builtinID == clang::AArch64::BI__builtin_arm_ldaex) { |
| cgm.errorNYI(expr->getSourceRange(), |
| std::string("unimplemented AArch64 builtin call: ") + |
| getContext().BuiltinInfo.getName(builtinID)); |
| return mlir::Value{}; |
| } |
| |
| if ((builtinID == clang::AArch64::BI__builtin_arm_strex || |
| builtinID == clang::AArch64::BI__builtin_arm_stlex) && |
| getContext().getTypeSize(expr->getArg(0)->getType()) == 128) { |
| cgm.errorNYI(expr->getSourceRange(), |
| std::string("unimplemented AArch64 builtin call: ") + |
| getContext().BuiltinInfo.getName(builtinID)); |
| return mlir::Value{}; |
| } |
| |
| if (builtinID == clang::AArch64::BI__builtin_arm_strex || |
| builtinID == clang::AArch64::BI__builtin_arm_stlex) { |
| cgm.errorNYI(expr->getSourceRange(), |
| std::string("unimplemented AArch64 builtin call: ") + |
| getContext().BuiltinInfo.getName(builtinID)); |
| return mlir::Value{}; |
| } |
| |
| if (builtinID == clang::AArch64::BI__getReg) { |
| cgm.errorNYI(expr->getSourceRange(), |
| std::string("unimplemented AArch64 builtin call: ") + |
| getContext().BuiltinInfo.getName(builtinID)); |
| return mlir::Value{}; |
| } |
| |
| if (builtinID == clang::AArch64::BI__break) { |
| cgm.errorNYI(expr->getSourceRange(), |
| std::string("unimplemented AArch64 builtin call: ") + |
| getContext().BuiltinInfo.getName(builtinID)); |
| return mlir::Value{}; |
| } |
| |
| if (builtinID == clang::AArch64::BI__builtin_arm_clrex) { |
| cgm.errorNYI(expr->getSourceRange(), |
| std::string("unimplemented AArch64 builtin call: ") + |
| getContext().BuiltinInfo.getName(builtinID)); |
| return mlir::Value{}; |
| } |
| |
| if (builtinID == clang::AArch64::BI_ReadWriteBarrier) { |
| cgm.errorNYI(expr->getSourceRange(), |
| std::string("unimplemented AArch64 builtin call: ") + |
| getContext().BuiltinInfo.getName(builtinID)); |
| return mlir::Value{}; |
| } |
| |
| // CRC32 |
| Intrinsic::ID crcIntrinsicID = Intrinsic::not_intrinsic; |
| switch (builtinID) { |
| case clang::AArch64::BI__builtin_arm_crc32b: |
| crcIntrinsicID = Intrinsic::aarch64_crc32b; |
| break; |
| case clang::AArch64::BI__builtin_arm_crc32cb: |
| crcIntrinsicID = Intrinsic::aarch64_crc32cb; |
| break; |
| case clang::AArch64::BI__builtin_arm_crc32h: |
| crcIntrinsicID = Intrinsic::aarch64_crc32h; |
| break; |
| case clang::AArch64::BI__builtin_arm_crc32ch: |
| crcIntrinsicID = Intrinsic::aarch64_crc32ch; |
| break; |
| case clang::AArch64::BI__builtin_arm_crc32w: |
| crcIntrinsicID = Intrinsic::aarch64_crc32w; |
| break; |
| case clang::AArch64::BI__builtin_arm_crc32cw: |
| crcIntrinsicID = Intrinsic::aarch64_crc32cw; |
| break; |
| case clang::AArch64::BI__builtin_arm_crc32d: |
| crcIntrinsicID = Intrinsic::aarch64_crc32x; |
| break; |
| case clang::AArch64::BI__builtin_arm_crc32cd: |
| crcIntrinsicID = Intrinsic::aarch64_crc32cx; |
| break; |
| } |
| |
| if (crcIntrinsicID != Intrinsic::not_intrinsic) { |
| cgm.errorNYI(expr->getSourceRange(), |
| std::string("unimplemented AArch64 builtin call: ") + |
| getContext().BuiltinInfo.getName(builtinID)); |
| return mlir::Value{}; |
| } |
| |
| // Memory Operations (MOPS) |
| if (builtinID == AArch64::BI__builtin_arm_mops_memset_tag) { |
| cgm.errorNYI(expr->getSourceRange(), |
| std::string("unimplemented AArch64 builtin call: ") + |
| getContext().BuiltinInfo.getName(builtinID)); |
| return mlir::Value{}; |
| } |
| |
| // Memory Tagging Extensions (MTE) Intrinsics |
| Intrinsic::ID mteIntrinsicID = Intrinsic::not_intrinsic; |
| switch (builtinID) { |
| case clang::AArch64::BI__builtin_arm_irg: |
| mteIntrinsicID = Intrinsic::aarch64_irg; |
| break; |
| case clang::AArch64::BI__builtin_arm_addg: |
| mteIntrinsicID = Intrinsic::aarch64_addg; |
| break; |
| case clang::AArch64::BI__builtin_arm_gmi: |
| mteIntrinsicID = Intrinsic::aarch64_gmi; |
| break; |
| case clang::AArch64::BI__builtin_arm_ldg: |
| mteIntrinsicID = Intrinsic::aarch64_ldg; |
| break; |
| case clang::AArch64::BI__builtin_arm_stg: |
| mteIntrinsicID = Intrinsic::aarch64_stg; |
| break; |
| case clang::AArch64::BI__builtin_arm_subp: |
| mteIntrinsicID = Intrinsic::aarch64_subp; |
| break; |
| } |
| |
| if (mteIntrinsicID != Intrinsic::not_intrinsic) { |
| cgm.errorNYI(expr->getSourceRange(), |
| std::string("unimplemented AArch64 builtin call: ") + |
| getContext().BuiltinInfo.getName(builtinID)); |
| return mlir::Value{}; |
| } |
| |
| if (builtinID == clang::AArch64::BI__builtin_arm_rsr || |
| builtinID == clang::AArch64::BI__builtin_arm_rsr64 || |
| builtinID == clang::AArch64::BI__builtin_arm_rsr128 || |
| builtinID == clang::AArch64::BI__builtin_arm_rsrp || |
| builtinID == clang::AArch64::BI__builtin_arm_wsr || |
| builtinID == clang::AArch64::BI__builtin_arm_wsr64 || |
| builtinID == clang::AArch64::BI__builtin_arm_wsr128 || |
| builtinID == clang::AArch64::BI__builtin_arm_wsrp) { |
| cgm.errorNYI(expr->getSourceRange(), |
| std::string("unimplemented AArch64 builtin call: ") + |
| getContext().BuiltinInfo.getName(builtinID)); |
| return mlir::Value{}; |
| } |
| |
| if (builtinID == clang::AArch64::BI_ReadStatusReg || |
| builtinID == clang::AArch64::BI_WriteStatusReg || |
| builtinID == clang::AArch64::BI__sys) { |
| cgm.errorNYI(expr->getSourceRange(), |
| std::string("unimplemented AArch64 builtin call: ") + |
| getContext().BuiltinInfo.getName(builtinID)); |
| return mlir::Value{}; |
| } |
| |
| if (builtinID == clang::AArch64::BI_AddressOfReturnAddress) { |
| cgm.errorNYI(expr->getSourceRange(), |
| std::string("unimplemented AArch64 builtin call: ") + |
| getContext().BuiltinInfo.getName(builtinID)); |
| return mlir::Value{}; |
| } |
| |
| if (builtinID == clang::AArch64::BI__builtin_sponentry) { |
| cgm.errorNYI(expr->getSourceRange(), |
| std::string("unimplemented AArch64 builtin call: ") + |
| getContext().BuiltinInfo.getName(builtinID)); |
| return mlir::Value{}; |
| } |
| |
| if (builtinID == clang::AArch64::BI__mulh || |
| builtinID == clang::AArch64::BI__umulh) { |
| cgm.errorNYI(expr->getSourceRange(), |
| std::string("unimplemented AArch64 builtin call: ") + |
| getContext().BuiltinInfo.getName(builtinID)); |
| return mlir::Value{}; |
| } |
| |
| if (builtinID == AArch64::BI__writex18byte || |
| builtinID == AArch64::BI__writex18word || |
| builtinID == AArch64::BI__writex18dword || |
| builtinID == AArch64::BI__writex18qword) { |
| cgm.errorNYI(expr->getSourceRange(), |
| std::string("unimplemented AArch64 builtin call: ") + |
| getContext().BuiltinInfo.getName(builtinID)); |
| return mlir::Value{}; |
| } |
| |
| if (builtinID == AArch64::BI__readx18byte || |
| builtinID == AArch64::BI__readx18word || |
| builtinID == AArch64::BI__readx18dword || |
| builtinID == AArch64::BI__readx18qword) { |
| cgm.errorNYI(expr->getSourceRange(), |
| std::string("unimplemented AArch64 builtin call: ") + |
| getContext().BuiltinInfo.getName(builtinID)); |
| return mlir::Value{}; |
| } |
| |
| if (builtinID == AArch64::BI__addx18byte || |
| builtinID == AArch64::BI__addx18word || |
| builtinID == AArch64::BI__addx18dword || |
| builtinID == AArch64::BI__addx18qword || |
| builtinID == AArch64::BI__incx18byte || |
| builtinID == AArch64::BI__incx18word || |
| builtinID == AArch64::BI__incx18dword || |
| builtinID == AArch64::BI__incx18qword) { |
| cgm.errorNYI(expr->getSourceRange(), |
| std::string("unimplemented AArch64 builtin call: ") + |
| getContext().BuiltinInfo.getName(builtinID)); |
| return mlir::Value{}; |
| } |
| |
| if (builtinID == AArch64::BI_CopyDoubleFromInt64 || |
| builtinID == AArch64::BI_CopyFloatFromInt32 || |
| builtinID == AArch64::BI_CopyInt32FromFloat || |
| builtinID == AArch64::BI_CopyInt64FromDouble) { |
| cgm.errorNYI(expr->getSourceRange(), |
| std::string("unimplemented AArch64 builtin call: ") + |
| getContext().BuiltinInfo.getName(builtinID)); |
| return mlir::Value{}; |
| } |
| |
| if (builtinID == AArch64::BI_CountLeadingOnes || |
| builtinID == AArch64::BI_CountLeadingOnes64 || |
| builtinID == AArch64::BI_CountLeadingZeros || |
| builtinID == AArch64::BI_CountLeadingZeros64) { |
| cgm.errorNYI(expr->getSourceRange(), |
| std::string("unimplemented AArch64 builtin call: ") + |
| getContext().BuiltinInfo.getName(builtinID)); |
| return mlir::Value{}; |
| } |
| |
| if (builtinID == AArch64::BI_CountLeadingSigns || |
| builtinID == AArch64::BI_CountLeadingSigns64) { |
| cgm.errorNYI(expr->getSourceRange(), |
| std::string("unimplemented AArch64 builtin call: ") + |
| getContext().BuiltinInfo.getName(builtinID)); |
| return mlir::Value{}; |
| } |
| |
| if (builtinID == AArch64::BI_CountOneBits || |
| builtinID == AArch64::BI_CountOneBits64) { |
| cgm.errorNYI(expr->getSourceRange(), |
| std::string("unimplemented AArch64 builtin call: ") + |
| getContext().BuiltinInfo.getName(builtinID)); |
| return mlir::Value{}; |
| } |
| |
| if (builtinID == AArch64::BI__prefetch) { |
| cgm.errorNYI(expr->getSourceRange(), |
| std::string("unimplemented AArch64 builtin call: ") + |
| getContext().BuiltinInfo.getName(builtinID)); |
| return mlir::Value{}; |
| } |
| |
| if (builtinID == AArch64::BI__hlt) { |
| cgm.errorNYI(expr->getSourceRange(), |
| std::string("unimplemented AArch64 builtin call: ") + |
| getContext().BuiltinInfo.getName(builtinID)); |
| return mlir::Value{}; |
| } |
| |
| if (builtinID == NEON::BI__builtin_neon_vcvth_bf16_f32) { |
| cgm.errorNYI(expr->getSourceRange(), |
| std::string("unimplemented AArch64 builtin call: ") + |
| getContext().BuiltinInfo.getName(builtinID)); |
| return mlir::Value{}; |
| } |
| |
| // Handle MSVC intrinsics before argument evaluation to prevent double |
| // evaluation. |
| assert(!cir::MissingFeatures::msvcBuiltins()); |
| |
| // Some intrinsics are equivalent - if they are use the base intrinsic ID. |
| auto it = llvm::find_if(neonEquivalentIntrinsicMap, [builtinID](auto &p) { |
| return p.first == builtinID; |
| }); |
| if (it != end(neonEquivalentIntrinsicMap)) |
| builtinID = it->second; |
| |
| // Find out if any arguments are required to be integer constant |
| // expressions. |
| assert(!cir::MissingFeatures::handleBuiltinICEArguments()); |
| |
| assert(!cir::MissingFeatures::neonSISDIntrinsics()); |
| |
| // Handle non-overloaded intrinsics first. |
| switch (builtinID) { |
| default: |
| break; |
| case NEON::BI__builtin_neon_vabsh_f16: |
| case NEON::BI__builtin_neon_vaddq_p128: |
| case NEON::BI__builtin_neon_vldrq_p128: |
| case NEON::BI__builtin_neon_vstrq_p128: |
| case NEON::BI__builtin_neon_vcvts_f32_u32: |
| case NEON::BI__builtin_neon_vcvtd_f64_u64: |
| case NEON::BI__builtin_neon_vcvts_f32_s32: |
| case NEON::BI__builtin_neon_vcvtd_f64_s64: |
| case NEON::BI__builtin_neon_vcvth_f16_u16: |
| case NEON::BI__builtin_neon_vcvth_f16_u32: |
| case NEON::BI__builtin_neon_vcvth_f16_u64: |
| case NEON::BI__builtin_neon_vcvth_f16_s16: |
| case NEON::BI__builtin_neon_vcvth_f16_s32: |
| case NEON::BI__builtin_neon_vcvth_f16_s64: |
| case NEON::BI__builtin_neon_vcvtah_u16_f16: |
| case NEON::BI__builtin_neon_vcvtmh_u16_f16: |
| case NEON::BI__builtin_neon_vcvtnh_u16_f16: |
| case NEON::BI__builtin_neon_vcvtph_u16_f16: |
| case NEON::BI__builtin_neon_vcvth_u16_f16: |
| case NEON::BI__builtin_neon_vcvtah_s16_f16: |
| case NEON::BI__builtin_neon_vcvtmh_s16_f16: |
| case NEON::BI__builtin_neon_vcvtnh_s16_f16: |
| case NEON::BI__builtin_neon_vcvtph_s16_f16: |
| case NEON::BI__builtin_neon_vcvth_s16_f16: |
| case NEON::BI__builtin_neon_vcaleh_f16: |
| case NEON::BI__builtin_neon_vcalth_f16: |
| case NEON::BI__builtin_neon_vcageh_f16: |
| case NEON::BI__builtin_neon_vcagth_f16: |
| case NEON::BI__builtin_neon_vcvth_n_s16_f16: |
| case NEON::BI__builtin_neon_vcvth_n_u16_f16: |
| case NEON::BI__builtin_neon_vcvth_n_f16_s16: |
| case NEON::BI__builtin_neon_vcvth_n_f16_u16: |
| case NEON::BI__builtin_neon_vpaddd_s64: |
| case NEON::BI__builtin_neon_vpaddd_f64: |
| case NEON::BI__builtin_neon_vpadds_f32: |
| case NEON::BI__builtin_neon_vceqzd_s64: |
| case NEON::BI__builtin_neon_vceqzd_f64: |
| case NEON::BI__builtin_neon_vceqzs_f32: |
| case NEON::BI__builtin_neon_vceqzh_f16: |
| case NEON::BI__builtin_neon_vcgezd_s64: |
| case NEON::BI__builtin_neon_vcgezd_f64: |
| case NEON::BI__builtin_neon_vcgezs_f32: |
| case NEON::BI__builtin_neon_vcgezh_f16: |
| case NEON::BI__builtin_neon_vclezd_s64: |
| case NEON::BI__builtin_neon_vclezd_f64: |
| case NEON::BI__builtin_neon_vclezs_f32: |
| case NEON::BI__builtin_neon_vclezh_f16: |
| case NEON::BI__builtin_neon_vcgtzd_s64: |
| case NEON::BI__builtin_neon_vcgtzd_f64: |
| case NEON::BI__builtin_neon_vcgtzs_f32: |
| case NEON::BI__builtin_neon_vcgtzh_f16: |
| case NEON::BI__builtin_neon_vcltzd_s64: |
| case NEON::BI__builtin_neon_vcltzd_f64: |
| case NEON::BI__builtin_neon_vcltzs_f32: |
| case NEON::BI__builtin_neon_vcltzh_f16: |
| case NEON::BI__builtin_neon_vceqzd_u64: |
| case NEON::BI__builtin_neon_vceqd_f64: |
| case NEON::BI__builtin_neon_vcled_f64: |
| case NEON::BI__builtin_neon_vcltd_f64: |
| case NEON::BI__builtin_neon_vcged_f64: |
| case NEON::BI__builtin_neon_vcgtd_f64: |
| case NEON::BI__builtin_neon_vceqs_f32: |
| case NEON::BI__builtin_neon_vcles_f32: |
| case NEON::BI__builtin_neon_vclts_f32: |
| case NEON::BI__builtin_neon_vcges_f32: |
| case NEON::BI__builtin_neon_vcgts_f32: |
| case NEON::BI__builtin_neon_vceqh_f16: |
| case NEON::BI__builtin_neon_vcleh_f16: |
| case NEON::BI__builtin_neon_vclth_f16: |
| case NEON::BI__builtin_neon_vcgeh_f16: |
| case NEON::BI__builtin_neon_vcgth_f16: |
| case NEON::BI__builtin_neon_vceqd_s64: |
| case NEON::BI__builtin_neon_vceqd_u64: |
| case NEON::BI__builtin_neon_vcgtd_s64: |
| case NEON::BI__builtin_neon_vcgtd_u64: |
| case NEON::BI__builtin_neon_vcltd_s64: |
| case NEON::BI__builtin_neon_vcltd_u64: |
| case NEON::BI__builtin_neon_vcged_u64: |
| case NEON::BI__builtin_neon_vcged_s64: |
| case NEON::BI__builtin_neon_vcled_u64: |
| case NEON::BI__builtin_neon_vcled_s64: |
| case NEON::BI__builtin_neon_vtstd_s64: |
| case NEON::BI__builtin_neon_vtstd_u64: |
| case NEON::BI__builtin_neon_vset_lane_i8: |
| case NEON::BI__builtin_neon_vset_lane_i16: |
| case NEON::BI__builtin_neon_vset_lane_i32: |
| case NEON::BI__builtin_neon_vset_lane_i64: |
| case NEON::BI__builtin_neon_vset_lane_bf16: |
| case NEON::BI__builtin_neon_vset_lane_f32: |
| case NEON::BI__builtin_neon_vsetq_lane_i8: |
| case NEON::BI__builtin_neon_vsetq_lane_i16: |
| case NEON::BI__builtin_neon_vsetq_lane_i32: |
| case NEON::BI__builtin_neon_vsetq_lane_i64: |
| case NEON::BI__builtin_neon_vsetq_lane_bf16: |
| case NEON::BI__builtin_neon_vsetq_lane_f32: |
| case NEON::BI__builtin_neon_vset_lane_f64: |
| case NEON::BI__builtin_neon_vset_lane_mf8: |
| case NEON::BI__builtin_neon_vsetq_lane_mf8: |
| case NEON::BI__builtin_neon_vsetq_lane_f64: |
| case NEON::BI__builtin_neon_vget_lane_i8: |
| case NEON::BI__builtin_neon_vdupb_lane_i8: |
| case NEON::BI__builtin_neon_vgetq_lane_i8: |
| case NEON::BI__builtin_neon_vdupb_laneq_i8: |
| case NEON::BI__builtin_neon_vget_lane_mf8: |
| case NEON::BI__builtin_neon_vdupb_lane_mf8: |
| case NEON::BI__builtin_neon_vgetq_lane_mf8: |
| case NEON::BI__builtin_neon_vdupb_laneq_mf8: |
| case NEON::BI__builtin_neon_vget_lane_i16: |
| case NEON::BI__builtin_neon_vduph_lane_i16: |
| case NEON::BI__builtin_neon_vgetq_lane_i16: |
| case NEON::BI__builtin_neon_vduph_laneq_i16: |
| case NEON::BI__builtin_neon_vget_lane_i32: |
| case NEON::BI__builtin_neon_vdups_lane_i32: |
| case NEON::BI__builtin_neon_vdups_lane_f32: |
| case NEON::BI__builtin_neon_vgetq_lane_i32: |
| case NEON::BI__builtin_neon_vdups_laneq_i32: |
| case NEON::BI__builtin_neon_vget_lane_i64: |
| case NEON::BI__builtin_neon_vdupd_lane_i64: |
| case NEON::BI__builtin_neon_vdupd_lane_f64: |
| case NEON::BI__builtin_neon_vgetq_lane_i64: |
| case NEON::BI__builtin_neon_vdupd_laneq_i64: |
| case NEON::BI__builtin_neon_vget_lane_f32: |
| case NEON::BI__builtin_neon_vget_lane_f64: |
| case NEON::BI__builtin_neon_vgetq_lane_f32: |
| case NEON::BI__builtin_neon_vdups_laneq_f32: |
| case NEON::BI__builtin_neon_vgetq_lane_f64: |
| case NEON::BI__builtin_neon_vdupd_laneq_f64: |
| case NEON::BI__builtin_neon_vaddh_f16: |
| case NEON::BI__builtin_neon_vsubh_f16: |
| case NEON::BI__builtin_neon_vmulh_f16: |
| case NEON::BI__builtin_neon_vdivh_f16: |
| case NEON::BI__builtin_neon_vfmah_f16: |
| case NEON::BI__builtin_neon_vfmsh_f16: |
| case NEON::BI__builtin_neon_vaddd_s64: |
| case NEON::BI__builtin_neon_vaddd_u64: |
| case NEON::BI__builtin_neon_vsubd_s64: |
| case NEON::BI__builtin_neon_vsubd_u64: |
| case NEON::BI__builtin_neon_vqdmlalh_s16: |
| case NEON::BI__builtin_neon_vqdmlslh_s16: |
| case NEON::BI__builtin_neon_vqshlud_n_s64: |
| case NEON::BI__builtin_neon_vqshld_n_u64: |
| case NEON::BI__builtin_neon_vqshld_n_s64: |
| case NEON::BI__builtin_neon_vrshrd_n_u64: |
| case NEON::BI__builtin_neon_vrshrd_n_s64: |
| case NEON::BI__builtin_neon_vrsrad_n_u64: |
| case NEON::BI__builtin_neon_vrsrad_n_s64: |
| case NEON::BI__builtin_neon_vshld_n_s64: |
| case NEON::BI__builtin_neon_vshld_n_u64: |
| case NEON::BI__builtin_neon_vshrd_n_s64: |
| case NEON::BI__builtin_neon_vshrd_n_u64: |
| case NEON::BI__builtin_neon_vsrad_n_s64: |
| case NEON::BI__builtin_neon_vsrad_n_u64: |
| case NEON::BI__builtin_neon_vqdmlalh_lane_s16: |
| case NEON::BI__builtin_neon_vqdmlalh_laneq_s16: |
| case NEON::BI__builtin_neon_vqdmlslh_lane_s16: |
| case NEON::BI__builtin_neon_vqdmlslh_laneq_s16: |
| case NEON::BI__builtin_neon_vqdmlals_s32: |
| case NEON::BI__builtin_neon_vqdmlsls_s32: |
| case NEON::BI__builtin_neon_vqdmlals_lane_s32: |
| case NEON::BI__builtin_neon_vqdmlals_laneq_s32: |
| case NEON::BI__builtin_neon_vqdmlsls_lane_s32: |
| case NEON::BI__builtin_neon_vqdmlsls_laneq_s32: |
| case NEON::BI__builtin_neon_vget_lane_bf16: |
| case NEON::BI__builtin_neon_vduph_lane_bf16: |
| case NEON::BI__builtin_neon_vduph_lane_f16: |
| case NEON::BI__builtin_neon_vgetq_lane_bf16: |
| case NEON::BI__builtin_neon_vduph_laneq_bf16: |
| case NEON::BI__builtin_neon_vduph_laneq_f16: |
| case NEON::BI__builtin_neon_vcvt_bf16_f32: |
| case NEON::BI__builtin_neon_vcvtq_low_bf16_f32: |
| case NEON::BI__builtin_neon_vcvtq_high_bf16_f32: |
| case clang::AArch64::BI_InterlockedAdd: |
| case clang::AArch64::BI_InterlockedAdd_acq: |
| case clang::AArch64::BI_InterlockedAdd_rel: |
| case clang::AArch64::BI_InterlockedAdd_nf: |
| case clang::AArch64::BI_InterlockedAdd64: |
| case clang::AArch64::BI_InterlockedAdd64_acq: |
| case clang::AArch64::BI_InterlockedAdd64_rel: |
| case clang::AArch64::BI_InterlockedAdd64_nf: |
| cgm.errorNYI(expr->getSourceRange(), |
| std::string("unimplemented AArch64 builtin call: ") + |
| getContext().BuiltinInfo.getName(builtinID)); |
| return mlir::Value{}; |
| } |
| |
| // Not all intrinsics handled by the common case work for AArch64 yet, so only |
| // defer to common code if it's been added to our special map. |
| assert(!cir::MissingFeatures::aarch64SIMDIntrinsics()); |
| |
| assert(!cir::MissingFeatures::aarch64TblBuiltinExpr()); |
| |
| switch (builtinID) { |
| default: |
| return std::nullopt; |
| case NEON::BI__builtin_neon_vbsl_v: |
| case NEON::BI__builtin_neon_vbslq_v: |
| case NEON::BI__builtin_neon_vfma_lane_v: |
| case NEON::BI__builtin_neon_vfmaq_lane_v: |
| case NEON::BI__builtin_neon_vfma_laneq_v: |
| case NEON::BI__builtin_neon_vfmaq_laneq_v: |
| case NEON::BI__builtin_neon_vfmah_lane_f16: |
| case NEON::BI__builtin_neon_vfmas_lane_f32: |
| case NEON::BI__builtin_neon_vfmah_laneq_f16: |
| case NEON::BI__builtin_neon_vfmas_laneq_f32: |
| case NEON::BI__builtin_neon_vfmad_lane_f64: |
| case NEON::BI__builtin_neon_vfmad_laneq_f64: |
| case NEON::BI__builtin_neon_vmull_v: |
| case NEON::BI__builtin_neon_vmax_v: |
| case NEON::BI__builtin_neon_vmaxq_v: |
| case NEON::BI__builtin_neon_vmaxh_f16: |
| case NEON::BI__builtin_neon_vmin_v: |
| case NEON::BI__builtin_neon_vminq_v: |
| case NEON::BI__builtin_neon_vminh_f16: |
| case NEON::BI__builtin_neon_vabd_v: |
| case NEON::BI__builtin_neon_vabdq_v: |
| case NEON::BI__builtin_neon_vpadal_v: |
| case NEON::BI__builtin_neon_vpadalq_v: |
| case NEON::BI__builtin_neon_vpmin_v: |
| case NEON::BI__builtin_neon_vpminq_v: |
| case NEON::BI__builtin_neon_vpmax_v: |
| case NEON::BI__builtin_neon_vpmaxq_v: |
| case NEON::BI__builtin_neon_vminnm_v: |
| case NEON::BI__builtin_neon_vminnmq_v: |
| case NEON::BI__builtin_neon_vminnmh_f16: |
| case NEON::BI__builtin_neon_vmaxnm_v: |
| case NEON::BI__builtin_neon_vmaxnmq_v: |
| case NEON::BI__builtin_neon_vmaxnmh_f16: |
| case NEON::BI__builtin_neon_vrecpss_f32: |
| case NEON::BI__builtin_neon_vrecpsd_f64: |
| case NEON::BI__builtin_neon_vrecpsh_f16: |
| case NEON::BI__builtin_neon_vqshrun_n_v: |
| case NEON::BI__builtin_neon_vqrshrun_n_v: |
| case NEON::BI__builtin_neon_vqshrn_n_v: |
| case NEON::BI__builtin_neon_vrshrn_n_v: |
| case NEON::BI__builtin_neon_vqrshrn_n_v: |
| case NEON::BI__builtin_neon_vrndah_f16: |
| case NEON::BI__builtin_neon_vrnda_v: |
| case NEON::BI__builtin_neon_vrndaq_v: |
| case NEON::BI__builtin_neon_vrndih_f16: |
| case NEON::BI__builtin_neon_vrndmh_f16: |
| case NEON::BI__builtin_neon_vrndm_v: |
| case NEON::BI__builtin_neon_vrndmq_v: |
| case NEON::BI__builtin_neon_vrndnh_f16: |
| case NEON::BI__builtin_neon_vrndn_v: |
| case NEON::BI__builtin_neon_vrndnq_v: |
| case NEON::BI__builtin_neon_vrndns_f32: |
| case NEON::BI__builtin_neon_vrndph_f16: |
| case NEON::BI__builtin_neon_vrndp_v: |
| case NEON::BI__builtin_neon_vrndpq_v: |
| case NEON::BI__builtin_neon_vrndxh_f16: |
| case NEON::BI__builtin_neon_vrndx_v: |
| case NEON::BI__builtin_neon_vrndxq_v: |
| case NEON::BI__builtin_neon_vrndh_f16: |
| case NEON::BI__builtin_neon_vrnd32x_f32: |
| case NEON::BI__builtin_neon_vrnd32xq_f32: |
| case NEON::BI__builtin_neon_vrnd32x_f64: |
| case NEON::BI__builtin_neon_vrnd32xq_f64: |
| case NEON::BI__builtin_neon_vrnd32z_f32: |
| case NEON::BI__builtin_neon_vrnd32zq_f32: |
| case NEON::BI__builtin_neon_vrnd32z_f64: |
| case NEON::BI__builtin_neon_vrnd32zq_f64: |
| case NEON::BI__builtin_neon_vrnd64x_f32: |
| case NEON::BI__builtin_neon_vrnd64xq_f32: |
| case NEON::BI__builtin_neon_vrnd64x_f64: |
| case NEON::BI__builtin_neon_vrnd64xq_f64: |
| case NEON::BI__builtin_neon_vrnd64z_f32: |
| case NEON::BI__builtin_neon_vrnd64zq_f32: |
| case NEON::BI__builtin_neon_vrnd64z_f64: |
| case NEON::BI__builtin_neon_vrnd64zq_f64: |
| case NEON::BI__builtin_neon_vrnd_v: |
| case NEON::BI__builtin_neon_vrndq_v: |
| case NEON::BI__builtin_neon_vcvt_f64_v: |
| case NEON::BI__builtin_neon_vcvtq_f64_v: |
| case NEON::BI__builtin_neon_vcvt_f64_f32: |
| case NEON::BI__builtin_neon_vcvt_f32_f64: |
| case NEON::BI__builtin_neon_vcvt_s32_v: |
| case NEON::BI__builtin_neon_vcvt_u32_v: |
| case NEON::BI__builtin_neon_vcvt_s64_v: |
| case NEON::BI__builtin_neon_vcvt_u64_v: |
| case NEON::BI__builtin_neon_vcvt_s16_f16: |
| case NEON::BI__builtin_neon_vcvt_u16_f16: |
| case NEON::BI__builtin_neon_vcvtq_s32_v: |
| case NEON::BI__builtin_neon_vcvtq_u32_v: |
| case NEON::BI__builtin_neon_vcvtq_s64_v: |
| case NEON::BI__builtin_neon_vcvtq_u64_v: |
| case NEON::BI__builtin_neon_vcvtq_s16_f16: |
| case NEON::BI__builtin_neon_vcvtq_u16_f16: |
| case NEON::BI__builtin_neon_vcvta_s16_f16: |
| case NEON::BI__builtin_neon_vcvta_u16_f16: |
| case NEON::BI__builtin_neon_vcvta_s32_v: |
| case NEON::BI__builtin_neon_vcvtaq_s16_f16: |
| case NEON::BI__builtin_neon_vcvtaq_s32_v: |
| case NEON::BI__builtin_neon_vcvta_u32_v: |
| case NEON::BI__builtin_neon_vcvtaq_u16_f16: |
| case NEON::BI__builtin_neon_vcvtaq_u32_v: |
| case NEON::BI__builtin_neon_vcvta_s64_v: |
| case NEON::BI__builtin_neon_vcvtaq_s64_v: |
| case NEON::BI__builtin_neon_vcvta_u64_v: |
| case NEON::BI__builtin_neon_vcvtaq_u64_v: |
| case NEON::BI__builtin_neon_vcvtm_s16_f16: |
| case NEON::BI__builtin_neon_vcvtm_s32_v: |
| case NEON::BI__builtin_neon_vcvtmq_s16_f16: |
| case NEON::BI__builtin_neon_vcvtmq_s32_v: |
| case NEON::BI__builtin_neon_vcvtm_u16_f16: |
| case NEON::BI__builtin_neon_vcvtm_u32_v: |
| case NEON::BI__builtin_neon_vcvtmq_u16_f16: |
| case NEON::BI__builtin_neon_vcvtmq_u32_v: |
| case NEON::BI__builtin_neon_vcvtm_s64_v: |
| case NEON::BI__builtin_neon_vcvtmq_s64_v: |
| case NEON::BI__builtin_neon_vcvtm_u64_v: |
| case NEON::BI__builtin_neon_vcvtmq_u64_v: |
| case NEON::BI__builtin_neon_vcvtn_s16_f16: |
| case NEON::BI__builtin_neon_vcvtn_s32_v: |
| case NEON::BI__builtin_neon_vcvtnq_s16_f16: |
| case NEON::BI__builtin_neon_vcvtnq_s32_v: |
| case NEON::BI__builtin_neon_vcvtn_u16_f16: |
| case NEON::BI__builtin_neon_vcvtn_u32_v: |
| case NEON::BI__builtin_neon_vcvtnq_u16_f16: |
| case NEON::BI__builtin_neon_vcvtnq_u32_v: |
| case NEON::BI__builtin_neon_vcvtn_s64_v: |
| case NEON::BI__builtin_neon_vcvtnq_s64_v: |
| case NEON::BI__builtin_neon_vcvtn_u64_v: |
| case NEON::BI__builtin_neon_vcvtnq_u64_v: |
| case NEON::BI__builtin_neon_vcvtp_s16_f16: |
| case NEON::BI__builtin_neon_vcvtp_s32_v: |
| case NEON::BI__builtin_neon_vcvtpq_s16_f16: |
| case NEON::BI__builtin_neon_vcvtpq_s32_v: |
| case NEON::BI__builtin_neon_vcvtp_u16_f16: |
| case NEON::BI__builtin_neon_vcvtp_u32_v: |
| case NEON::BI__builtin_neon_vcvtpq_u16_f16: |
| case NEON::BI__builtin_neon_vcvtpq_u32_v: |
| case NEON::BI__builtin_neon_vcvtp_s64_v: |
| case NEON::BI__builtin_neon_vcvtpq_s64_v: |
| case NEON::BI__builtin_neon_vcvtp_u64_v: |
| case NEON::BI__builtin_neon_vcvtpq_u64_v: |
| case NEON::BI__builtin_neon_vmulx_v: |
| case NEON::BI__builtin_neon_vmulxq_v: |
| case NEON::BI__builtin_neon_vmulxh_lane_f16: |
| case NEON::BI__builtin_neon_vmulxh_laneq_f16: |
| case NEON::BI__builtin_neon_vmul_lane_v: |
| case NEON::BI__builtin_neon_vmul_laneq_v: |
| case NEON::BI__builtin_neon_vnegd_s64: |
| case NEON::BI__builtin_neon_vnegh_f16: |
| case NEON::BI__builtin_neon_vpmaxnm_v: |
| case NEON::BI__builtin_neon_vpmaxnmq_v: |
| case NEON::BI__builtin_neon_vpminnm_v: |
| case NEON::BI__builtin_neon_vpminnmq_v: |
| case NEON::BI__builtin_neon_vsqrth_f16: |
| case NEON::BI__builtin_neon_vsqrt_v: |
| case NEON::BI__builtin_neon_vsqrtq_v: |
| case NEON::BI__builtin_neon_vrbit_v: |
| case NEON::BI__builtin_neon_vrbitq_v: |
| case NEON::BI__builtin_neon_vmaxv_f16: |
| case NEON::BI__builtin_neon_vmaxvq_f16: |
| case NEON::BI__builtin_neon_vminv_f16: |
| case NEON::BI__builtin_neon_vminvq_f16: |
| case NEON::BI__builtin_neon_vmaxnmv_f16: |
| case NEON::BI__builtin_neon_vmaxnmvq_f16: |
| case NEON::BI__builtin_neon_vminnmv_f16: |
| case NEON::BI__builtin_neon_vminnmvq_f16: |
| case NEON::BI__builtin_neon_vmul_n_f64: |
| case NEON::BI__builtin_neon_vaddlv_u8: |
| case NEON::BI__builtin_neon_vaddlv_u16: |
| case NEON::BI__builtin_neon_vaddlvq_u8: |
| case NEON::BI__builtin_neon_vaddlvq_u16: |
| case NEON::BI__builtin_neon_vaddlv_s8: |
| case NEON::BI__builtin_neon_vaddlv_s16: |
| case NEON::BI__builtin_neon_vaddlvq_s8: |
| case NEON::BI__builtin_neon_vaddlvq_s16: |
| case NEON::BI__builtin_neon_vsri_n_v: |
| case NEON::BI__builtin_neon_vsriq_n_v: |
| case NEON::BI__builtin_neon_vsli_n_v: |
| case NEON::BI__builtin_neon_vsliq_n_v: |
| case NEON::BI__builtin_neon_vsra_n_v: |
| case NEON::BI__builtin_neon_vsraq_n_v: |
| case NEON::BI__builtin_neon_vrsra_n_v: |
| case NEON::BI__builtin_neon_vrsraq_n_v: |
| case NEON::BI__builtin_neon_vld1_v: |
| case NEON::BI__builtin_neon_vld1q_v: |
| case NEON::BI__builtin_neon_vst1_v: |
| case NEON::BI__builtin_neon_vst1q_v: |
| case NEON::BI__builtin_neon_vld1_lane_v: |
| case NEON::BI__builtin_neon_vld1q_lane_v: |
| case NEON::BI__builtin_neon_vldap1_lane_s64: |
| case NEON::BI__builtin_neon_vldap1q_lane_s64: |
| case NEON::BI__builtin_neon_vld1_dup_v: |
| case NEON::BI__builtin_neon_vld1q_dup_v: |
| case NEON::BI__builtin_neon_vst1_lane_v: |
| case NEON::BI__builtin_neon_vst1q_lane_v: |
| case NEON::BI__builtin_neon_vstl1_lane_s64: |
| case NEON::BI__builtin_neon_vstl1q_lane_s64: |
| case NEON::BI__builtin_neon_vld2_v: |
| case NEON::BI__builtin_neon_vld2q_v: |
| case NEON::BI__builtin_neon_vld3_v: |
| case NEON::BI__builtin_neon_vld3q_v: |
| case NEON::BI__builtin_neon_vld4_v: |
| case NEON::BI__builtin_neon_vld4q_v: |
| case NEON::BI__builtin_neon_vld2_dup_v: |
| case NEON::BI__builtin_neon_vld2q_dup_v: |
| case NEON::BI__builtin_neon_vld3_dup_v: |
| case NEON::BI__builtin_neon_vld3q_dup_v: |
| case NEON::BI__builtin_neon_vld4_dup_v: |
| case NEON::BI__builtin_neon_vld4q_dup_v: |
| case NEON::BI__builtin_neon_vld2_lane_v: |
| case NEON::BI__builtin_neon_vld2q_lane_v: |
| case NEON::BI__builtin_neon_vld3_lane_v: |
| case NEON::BI__builtin_neon_vld3q_lane_v: |
| case NEON::BI__builtin_neon_vld4_lane_v: |
| case NEON::BI__builtin_neon_vld4q_lane_v: |
| case NEON::BI__builtin_neon_vst2_v: |
| case NEON::BI__builtin_neon_vst2q_v: |
| case NEON::BI__builtin_neon_vst2_lane_v: |
| case NEON::BI__builtin_neon_vst2q_lane_v: |
| case NEON::BI__builtin_neon_vst3_v: |
| case NEON::BI__builtin_neon_vst3q_v: |
| case NEON::BI__builtin_neon_vst3_lane_v: |
| case NEON::BI__builtin_neon_vst3q_lane_v: |
| case NEON::BI__builtin_neon_vst4_v: |
| case NEON::BI__builtin_neon_vst4q_v: |
| case NEON::BI__builtin_neon_vst4_lane_v: |
| case NEON::BI__builtin_neon_vst4q_lane_v: |
| case NEON::BI__builtin_neon_vtrn_v: |
| case NEON::BI__builtin_neon_vtrnq_v: |
| case NEON::BI__builtin_neon_vuzp_v: |
| case NEON::BI__builtin_neon_vuzpq_v: |
| case NEON::BI__builtin_neon_vzip_v: |
| case NEON::BI__builtin_neon_vzipq_v: |
| case NEON::BI__builtin_neon_vqtbl1q_v: |
| case NEON::BI__builtin_neon_vqtbl2q_v: |
| case NEON::BI__builtin_neon_vqtbl3q_v: |
| case NEON::BI__builtin_neon_vqtbl4q_v: |
| case NEON::BI__builtin_neon_vqtbx1q_v: |
| case NEON::BI__builtin_neon_vqtbx2q_v: |
| case NEON::BI__builtin_neon_vqtbx3q_v: |
| case NEON::BI__builtin_neon_vqtbx4q_v: |
| case NEON::BI__builtin_neon_vsqadd_v: |
| case NEON::BI__builtin_neon_vsqaddq_v: |
| case NEON::BI__builtin_neon_vuqadd_v: |
| case NEON::BI__builtin_neon_vuqaddq_v: |
| case NEON::BI__builtin_neon_vluti2_laneq_mf8: |
| case NEON::BI__builtin_neon_vluti2_laneq_bf16: |
| case NEON::BI__builtin_neon_vluti2_laneq_f16: |
| case NEON::BI__builtin_neon_vluti2_laneq_p16: |
| case NEON::BI__builtin_neon_vluti2_laneq_p8: |
| case NEON::BI__builtin_neon_vluti2_laneq_s16: |
| case NEON::BI__builtin_neon_vluti2_laneq_s8: |
| case NEON::BI__builtin_neon_vluti2_laneq_u16: |
| case NEON::BI__builtin_neon_vluti2_laneq_u8: |
| case NEON::BI__builtin_neon_vluti2q_laneq_mf8: |
| case NEON::BI__builtin_neon_vluti2q_laneq_bf16: |
| case NEON::BI__builtin_neon_vluti2q_laneq_f16: |
| case NEON::BI__builtin_neon_vluti2q_laneq_p16: |
| case NEON::BI__builtin_neon_vluti2q_laneq_p8: |
| case NEON::BI__builtin_neon_vluti2q_laneq_s16: |
| case NEON::BI__builtin_neon_vluti2q_laneq_s8: |
| case NEON::BI__builtin_neon_vluti2q_laneq_u16: |
| case NEON::BI__builtin_neon_vluti2q_laneq_u8: |
| case NEON::BI__builtin_neon_vluti2_lane_mf8: |
| case NEON::BI__builtin_neon_vluti2_lane_bf16: |
| case NEON::BI__builtin_neon_vluti2_lane_f16: |
| case NEON::BI__builtin_neon_vluti2_lane_p16: |
| case NEON::BI__builtin_neon_vluti2_lane_p8: |
| case NEON::BI__builtin_neon_vluti2_lane_s16: |
| case NEON::BI__builtin_neon_vluti2_lane_s8: |
| case NEON::BI__builtin_neon_vluti2_lane_u16: |
| case NEON::BI__builtin_neon_vluti2_lane_u8: |
| case NEON::BI__builtin_neon_vluti2q_lane_mf8: |
| case NEON::BI__builtin_neon_vluti2q_lane_bf16: |
| case NEON::BI__builtin_neon_vluti2q_lane_f16: |
| case NEON::BI__builtin_neon_vluti2q_lane_p16: |
| case NEON::BI__builtin_neon_vluti2q_lane_p8: |
| case NEON::BI__builtin_neon_vluti2q_lane_s16: |
| case NEON::BI__builtin_neon_vluti2q_lane_s8: |
| case NEON::BI__builtin_neon_vluti2q_lane_u16: |
| case NEON::BI__builtin_neon_vluti2q_lane_u8: |
| case NEON::BI__builtin_neon_vluti4q_lane_mf8: |
| case NEON::BI__builtin_neon_vluti4q_lane_p8: |
| case NEON::BI__builtin_neon_vluti4q_lane_s8: |
| case NEON::BI__builtin_neon_vluti4q_lane_u8: |
| case NEON::BI__builtin_neon_vluti4q_laneq_mf8: |
| case NEON::BI__builtin_neon_vluti4q_laneq_p8: |
| case NEON::BI__builtin_neon_vluti4q_laneq_s8: |
| case NEON::BI__builtin_neon_vluti4q_laneq_u8: |
| case NEON::BI__builtin_neon_vluti4q_lane_bf16_x2: |
| case NEON::BI__builtin_neon_vluti4q_lane_f16_x2: |
| case NEON::BI__builtin_neon_vluti4q_lane_p16_x2: |
| case NEON::BI__builtin_neon_vluti4q_lane_s16_x2: |
| case NEON::BI__builtin_neon_vluti4q_lane_u16_x2: |
| case NEON::BI__builtin_neon_vluti4q_laneq_bf16_x2: |
| case NEON::BI__builtin_neon_vluti4q_laneq_f16_x2: |
| case NEON::BI__builtin_neon_vluti4q_laneq_p16_x2: |
| case NEON::BI__builtin_neon_vluti4q_laneq_s16_x2: |
| case NEON::BI__builtin_neon_vluti4q_laneq_u16_x2: |
| case NEON::BI__builtin_neon_vmmlaq_f16_mf8_fpm: |
| case NEON::BI__builtin_neon_vmmlaq_f32_mf8_fpm: |
| case NEON::BI__builtin_neon_vcvt1_low_bf16_mf8_fpm: |
| case NEON::BI__builtin_neon_vcvt1_bf16_mf8_fpm: |
| case NEON::BI__builtin_neon_vcvt1_high_bf16_mf8_fpm: |
| case NEON::BI__builtin_neon_vcvt2_low_bf16_mf8_fpm: |
| case NEON::BI__builtin_neon_vcvt2_bf16_mf8_fpm: |
| case NEON::BI__builtin_neon_vcvt2_high_bf16_mf8_fpm: |
| case NEON::BI__builtin_neon_vcvt1_low_f16_mf8_fpm: |
| case NEON::BI__builtin_neon_vcvt1_f16_mf8_fpm: |
| case NEON::BI__builtin_neon_vcvt1_high_f16_mf8_fpm: |
| case NEON::BI__builtin_neon_vcvt2_low_f16_mf8_fpm: |
| case NEON::BI__builtin_neon_vcvt2_f16_mf8_fpm: |
| case NEON::BI__builtin_neon_vcvt2_high_f16_mf8_fpm: |
| case NEON::BI__builtin_neon_vcvt_mf8_f32_fpm: |
| case NEON::BI__builtin_neon_vcvt_mf8_f16_fpm: |
| case NEON::BI__builtin_neon_vcvtq_mf8_f16_fpm: |
| case NEON::BI__builtin_neon_vcvt_high_mf8_f32_fpm: |
| case NEON::BI__builtin_neon_vdot_f16_mf8_fpm: |
| case NEON::BI__builtin_neon_vdotq_f16_mf8_fpm: |
| case NEON::BI__builtin_neon_vdot_lane_f16_mf8_fpm: |
| case NEON::BI__builtin_neon_vdotq_lane_f16_mf8_fpm: |
| case NEON::BI__builtin_neon_vdot_laneq_f16_mf8_fpm: |
| case NEON::BI__builtin_neon_vdotq_laneq_f16_mf8_fpm: |
| case NEON::BI__builtin_neon_vdot_f32_mf8_fpm: |
| case NEON::BI__builtin_neon_vdotq_f32_mf8_fpm: |
| case NEON::BI__builtin_neon_vdot_lane_f32_mf8_fpm: |
| case NEON::BI__builtin_neon_vdotq_lane_f32_mf8_fpm: |
| case NEON::BI__builtin_neon_vdot_laneq_f32_mf8_fpm: |
| case NEON::BI__builtin_neon_vdotq_laneq_f32_mf8_fpm: |
| case NEON::BI__builtin_neon_vmlalbq_f16_mf8_fpm: |
| case NEON::BI__builtin_neon_vmlaltq_f16_mf8_fpm: |
| case NEON::BI__builtin_neon_vmlallbbq_f32_mf8_fpm: |
| case NEON::BI__builtin_neon_vmlallbtq_f32_mf8_fpm: |
| case NEON::BI__builtin_neon_vmlalltbq_f32_mf8_fpm: |
| case NEON::BI__builtin_neon_vmlallttq_f32_mf8_fpm: |
| case NEON::BI__builtin_neon_vmlalbq_lane_f16_mf8_fpm: |
| case NEON::BI__builtin_neon_vmlalbq_laneq_f16_mf8_fpm: |
| case NEON::BI__builtin_neon_vmlaltq_lane_f16_mf8_fpm: |
| case NEON::BI__builtin_neon_vmlaltq_laneq_f16_mf8_fpm: |
| case NEON::BI__builtin_neon_vmlallbbq_lane_f32_mf8_fpm: |
| case NEON::BI__builtin_neon_vmlallbbq_laneq_f32_mf8_fpm: |
| case NEON::BI__builtin_neon_vmlallbtq_lane_f32_mf8_fpm: |
| case NEON::BI__builtin_neon_vmlallbtq_laneq_f32_mf8_fpm: |
| case NEON::BI__builtin_neon_vmlalltbq_lane_f32_mf8_fpm: |
| case NEON::BI__builtin_neon_vmlalltbq_laneq_f32_mf8_fpm: |
| case NEON::BI__builtin_neon_vmlallttq_lane_f32_mf8_fpm: |
| case NEON::BI__builtin_neon_vmlallttq_laneq_f32_mf8_fpm: |
| case NEON::BI__builtin_neon_vamin_f16: |
| case NEON::BI__builtin_neon_vaminq_f16: |
| case NEON::BI__builtin_neon_vamin_f32: |
| case NEON::BI__builtin_neon_vaminq_f32: |
| case NEON::BI__builtin_neon_vaminq_f64: |
| case NEON::BI__builtin_neon_vamax_f16: |
| case NEON::BI__builtin_neon_vamaxq_f16: |
| case NEON::BI__builtin_neon_vamax_f32: |
| case NEON::BI__builtin_neon_vamaxq_f32: |
| case NEON::BI__builtin_neon_vamaxq_f64: |
| case NEON::BI__builtin_neon_vscale_f16: |
| case NEON::BI__builtin_neon_vscaleq_f16: |
| case NEON::BI__builtin_neon_vscale_f32: |
| case NEON::BI__builtin_neon_vscaleq_f32: |
| case NEON::BI__builtin_neon_vscaleq_f64: |
| cgm.errorNYI(expr->getSourceRange(), |
| std::string("unimplemented AArch64 builtin call: ") + |
| getContext().BuiltinInfo.getName(builtinID)); |
| return mlir::Value{}; |
| } |
| |
| // Unreachable: All cases in the switch above return. |
| } |
