| //===---- CIRGenBuiltinNVPTX.cpp - Emit CIR for NVPTX builtins ------------===// | |
| // | |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |
| // See https://llvm.org/LICENSE.txt for license information. | |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |
| // | |
| //===----------------------------------------------------------------------===// | |
| // | |
| // This contains code to emit NVPTX Builtin calls. | |
| // | |
| //===----------------------------------------------------------------------===// | |
| #include "CIRGenFunction.h" | |
| #include "mlir/IR/Value.h" | |
| #include "clang/Basic/TargetBuiltins.h" | |
| #include "clang/CIR/Dialect/IR/CIRDataLayout.h" | |
| #include "clang/CIR/Dialect/IR/CIRDialect.h" | |
| #include "llvm/Support/NVPTXAddrSpace.h" | |
| using namespace clang; | |
| using namespace clang::CIRGen; | |
| static mlir::Value makeLdu(CIRGenFunction &cgf, const CallExpr *expr, | |
| llvm::StringRef intrinsicName) { | |
| auto &builder = cgf.getBuilder(); | |
| Address ptr = cgf.emitPointerWithAlignment(expr->getArg(0)); | |
| QualType argType = expr->getArg(0)->getType(); | |
| mlir::Type elemTy = cgf.convertTypeForMem(argType->getPointeeType()); | |
| clang::CharUnits align = ptr.getAlignment(); | |
| mlir::Location loc = cgf.getLoc(expr->getExprLoc()); | |
| mlir::Value alignVal = | |
| builder.getConstantInt(loc, builder.getSInt32Ty(), align.getQuantity()); | |
| return cir::LLVMIntrinsicCallOp::create( | |
| builder, loc, builder.getStringAttr(intrinsicName), elemTy, | |
| {ptr.emitRawPointer(), alignVal}) | |
| .getResult(); | |
| } | |
| static mlir::Value makeLdg(CIRGenFunction &cgf, const CallExpr *expr) { | |
| auto &builder = cgf.getBuilder(); | |
| Address ptr = cgf.emitPointerWithAlignment(expr->getArg(0)); | |
| QualType argType = expr->getArg(0)->getType(); | |
| mlir::Type elemTy = cgf.convertTypeForMem(argType->getPointeeType()); | |
| mlir::Location loc = cgf.getLoc(expr->getExprLoc()); | |
| // Use addrspace(1) for NVPTX ADDRESS_SPACE_GLOBAL. | |
| mlir::Type globalPtrTy = cir::PointerType::get( | |
| elemTy, cir::TargetAddressSpaceAttr::get( | |
| builder.getContext(), llvm::NVPTXAS::ADDRESS_SPACE_GLOBAL)); | |
| mlir::Value asc = | |
| builder.createAddrSpaceCast(loc, ptr.getPointer(), globalPtrTy); | |
| cir::LoadOp load = | |
| builder.createAlignedLoad(loc, elemTy, asc, ptr.getAlignment()); | |
| load.setInvariant(true); | |
| return load.getResult(); | |
| } | |
| /// Emit a CIR LLVMIntrinsicCallOp for a unary NVVM intrinsic. | |
| /// The result type is inferred from the single argument. | |
| static mlir::Value emitUnaryNVVMIntrinsic(CIRGenFunction &cgf, | |
| const CallExpr *expr, | |
| llvm::StringRef intrinsicName) { | |
| auto &builder = cgf.getBuilder(); | |
| mlir::Value arg = cgf.emitScalarExpr(expr->getArg(0)); | |
| return cir::LLVMIntrinsicCallOp::create( | |
| builder, cgf.getLoc(expr->getExprLoc()), | |
| builder.getStringAttr(intrinsicName), arg.getType(), {arg}) | |
| .getResult(); | |
| } | |
| /// Emit a CIR LLVMIntrinsicCallOp for an NVVM fadd intrinsic, which takes the | |
| /// rounding mode as a trailing operand. | |
| static mlir::Value emitNVVMFAdd(CIRGenFunction &cgf, const CallExpr *expr, | |
| llvm::StringRef intrinsicName, | |
| llvm::APFloat::roundingMode rm) { | |
| auto &builder = cgf.getBuilder(); | |
| mlir::Location loc = cgf.getLoc(expr->getExprLoc()); | |
| mlir::Value lhs = cgf.emitScalarExpr(expr->getArg(0)); | |
| mlir::Value rhs = cgf.emitScalarExpr(expr->getArg(1)); | |
| mlir::Value rnd = | |
| builder.getConstInt(loc, builder.getSInt32Ty(), static_cast<int>(rm)); | |
| return cir::LLVMIntrinsicCallOp::create(builder, loc, | |
| builder.getStringAttr(intrinsicName), | |
| lhs.getType(), {lhs, rhs, rnd}) | |
| .getResult(); | |
| } | |
| static mlir::Value emitBar0Reduction(CIRGenFunction &cgf, const CallExpr *expr, | |
| llvm::StringRef intrinsicName, | |
| bool returnsPred) { | |
| CIRGenBuilderTy &builder = cgf.getBuilder(); | |
| mlir::Location loc = cgf.getLoc(expr->getExprLoc()); | |
| mlir::Type si32Ty = builder.getSInt32Ty(); | |
| mlir::Value zero = builder.getNullValue(si32Ty, loc); | |
| mlir::Value pred = builder.createCompare( | |
| loc, cir::CmpOpKind::ne, cgf.emitScalarExpr(expr->getArg(0)), zero); | |
| mlir::Type resultTy = returnsPred ? mlir::Type(builder.getBoolTy()) : si32Ty; | |
| mlir::Value result = builder.emitIntrinsicCallOp( | |
| loc, intrinsicName, resultTy, mlir::ValueRange{zero, pred}); | |
| if (returnsPred) | |
| result = builder.createBoolToInt(result, si32Ty); | |
| return result; | |
| } | |
| static mlir::Value makeScopedAtomicRMW(CIRGenFunction &cgf, | |
| const CallExpr *expr, | |
| cir::AtomicFetchKind kind, | |
| cir::SyncScopeKind scope) { | |
| auto &builder = cgf.getBuilder(); | |
| Address destAddr = cgf.emitPointerWithAlignment(expr->getArg(0)); | |
| mlir::Value destValue = destAddr.emitRawPointer(); | |
| mlir::Value val = cgf.emitScalarExpr(expr->getArg(1)); | |
| auto rmwi = cir::AtomicFetchOp::create( | |
| builder, cgf.getLoc(expr->getSourceRange()), destValue, val, kind, | |
| cir::MemOrder::Relaxed, scope, /*is_volatile=*/false, | |
| /*fetch_first=*/true); | |
| return rmwi->getResult(0); | |
| } | |
| static mlir::Value makeScopedAtomicXchg(CIRGenFunction &cgf, | |
| const CallExpr *expr, | |
| cir::SyncScopeKind scope) { | |
| auto &builder = cgf.getBuilder(); | |
| Address destAddr = cgf.emitPointerWithAlignment(expr->getArg(0)); | |
| mlir::Value destValue = destAddr.emitRawPointer(); | |
| mlir::Value val = cgf.emitScalarExpr(expr->getArg(1)); | |
| auto xchg = cir::AtomicXchgOp::create( | |
| builder, cgf.getLoc(expr->getSourceRange()), destValue, val, | |
| cir::MemOrder::Relaxed, scope, /*is_volatile=*/false); | |
| return xchg.getResult(); | |
| } | |
| std::optional<mlir::Value> | |
| CIRGenFunction::emitNVPTXBuiltinExpr(unsigned builtinId, const CallExpr *expr) { | |
| switch (builtinId) { | |
| case NVPTX::BI__nvvm_atom_add_gen_i: | |
| case NVPTX::BI__nvvm_atom_add_gen_l: | |
| case NVPTX::BI__nvvm_atom_add_gen_ll: | |
| return makeBinaryAtomicValue(cir::AtomicFetchKind::Add, expr, | |
| /*originalArgType=*/nullptr, | |
| /*emittedArgValue=*/nullptr, | |
| cir::MemOrder::Relaxed); | |
| case NVPTX::BI__nvvm_atom_sub_gen_i: | |
| case NVPTX::BI__nvvm_atom_sub_gen_l: | |
| case NVPTX::BI__nvvm_atom_sub_gen_ll: | |
| return makeBinaryAtomicValue(cir::AtomicFetchKind::Sub, expr, | |
| /*originalArgType=*/nullptr, | |
| /*emittedArgValue=*/nullptr, | |
| cir::MemOrder::Relaxed); | |
| case NVPTX::BI__nvvm_atom_and_gen_i: | |
| case NVPTX::BI__nvvm_atom_and_gen_l: | |
| case NVPTX::BI__nvvm_atom_and_gen_ll: | |
| return makeBinaryAtomicValue(cir::AtomicFetchKind::And, expr, | |
| /*originalArgType=*/nullptr, | |
| /*emittedArgValue=*/nullptr, | |
| cir::MemOrder::Relaxed); | |
| case NVPTX::BI__nvvm_atom_or_gen_i: | |
| case NVPTX::BI__nvvm_atom_or_gen_l: | |
| case NVPTX::BI__nvvm_atom_or_gen_ll: | |
| return makeBinaryAtomicValue(cir::AtomicFetchKind::Or, expr, | |
| /*originalArgType=*/nullptr, | |
| /*emittedArgValue=*/nullptr, | |
| cir::MemOrder::Relaxed); | |
| case NVPTX::BI__nvvm_atom_xor_gen_i: | |
| case NVPTX::BI__nvvm_atom_xor_gen_l: | |
| case NVPTX::BI__nvvm_atom_xor_gen_ll: | |
| return makeBinaryAtomicValue(cir::AtomicFetchKind::Xor, expr, | |
| /*originalArgType=*/nullptr, | |
| /*emittedArgValue=*/nullptr, | |
| cir::MemOrder::Relaxed); | |
| case NVPTX::BI__nvvm_atom_xchg_gen_i: | |
| case NVPTX::BI__nvvm_atom_xchg_gen_l: | |
| case NVPTX::BI__nvvm_atom_xchg_gen_ll: | |
| return makeScopedAtomicXchg(*this, expr, cir::SyncScopeKind::System); | |
| case NVPTX::BI__nvvm_atom_max_gen_i: | |
| case NVPTX::BI__nvvm_atom_max_gen_l: | |
| case NVPTX::BI__nvvm_atom_max_gen_ll: | |
| return makeBinaryAtomicValue(cir::AtomicFetchKind::Max, expr, | |
| /*originalArgType=*/nullptr, | |
| /*emittedArgValue=*/nullptr, | |
| cir::MemOrder::Relaxed); | |
| case NVPTX::BI__nvvm_atom_max_gen_ui: | |
| case NVPTX::BI__nvvm_atom_max_gen_ul: | |
| case NVPTX::BI__nvvm_atom_max_gen_ull: | |
| return makeBinaryAtomicValue(cir::AtomicFetchKind::Max, expr, | |
| /*originalArgType=*/nullptr, | |
| /*emittedArgValue=*/nullptr, | |
| cir::MemOrder::Relaxed); | |
| case NVPTX::BI__nvvm_atom_min_gen_i: | |
| case NVPTX::BI__nvvm_atom_min_gen_l: | |
| case NVPTX::BI__nvvm_atom_min_gen_ll: | |
| return makeBinaryAtomicValue(cir::AtomicFetchKind::Min, expr, | |
| /*originalArgType=*/nullptr, | |
| /*emittedArgValue=*/nullptr, | |
| cir::MemOrder::Relaxed); | |
| case NVPTX::BI__nvvm_atom_min_gen_ui: | |
| case NVPTX::BI__nvvm_atom_min_gen_ul: | |
| case NVPTX::BI__nvvm_atom_min_gen_ull: | |
| return makeBinaryAtomicValue(cir::AtomicFetchKind::Min, expr, | |
| /*originalArgType=*/nullptr, | |
| /*emittedArgValue=*/nullptr, | |
| cir::MemOrder::Relaxed); | |
| case NVPTX::BI__nvvm_atom_cas_gen_us: | |
| case NVPTX::BI__nvvm_atom_cas_gen_i: | |
| case NVPTX::BI__nvvm_atom_cas_gen_l: | |
| case NVPTX::BI__nvvm_atom_cas_gen_ll: | |
| return emitAtomicCmpXchg(expr, /*returnBool=*/false, cir::MemOrder::Relaxed, | |
| cir::MemOrder::Relaxed, | |
| cir::SyncScopeKind::System); | |
| case NVPTX::BI__nvvm_atom_add_gen_f: | |
| case NVPTX::BI__nvvm_atom_add_gen_d: | |
| return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::Add, | |
| cir::SyncScopeKind::System); | |
| case NVPTX::BI__nvvm_atom_inc_gen_ui: | |
| return makeBinaryAtomicValue(cir::AtomicFetchKind::UIncWrap, expr, | |
| /*originalArgType=*/nullptr, | |
| /*emittedArgValue=*/nullptr, | |
| cir::MemOrder::Relaxed); | |
| case NVPTX::BI__nvvm_atom_dec_gen_ui: | |
| return makeBinaryAtomicValue(cir::AtomicFetchKind::UDecWrap, expr, | |
| /*originalArgType=*/nullptr, | |
| /*emittedArgValue=*/nullptr, | |
| cir::MemOrder::Relaxed); | |
| case NVPTX::BI__nvvm_ldg_c: | |
| case NVPTX::BI__nvvm_ldg_sc: | |
| case NVPTX::BI__nvvm_ldg_c2: | |
| case NVPTX::BI__nvvm_ldg_sc2: | |
| case NVPTX::BI__nvvm_ldg_c4: | |
| case NVPTX::BI__nvvm_ldg_sc4: | |
| case NVPTX::BI__nvvm_ldg_s: | |
| case NVPTX::BI__nvvm_ldg_s2: | |
| case NVPTX::BI__nvvm_ldg_s4: | |
| case NVPTX::BI__nvvm_ldg_i: | |
| case NVPTX::BI__nvvm_ldg_i2: | |
| case NVPTX::BI__nvvm_ldg_i4: | |
| case NVPTX::BI__nvvm_ldg_l: | |
| case NVPTX::BI__nvvm_ldg_l2: | |
| case NVPTX::BI__nvvm_ldg_ll: | |
| case NVPTX::BI__nvvm_ldg_ll2: | |
| case NVPTX::BI__nvvm_ldg_uc: | |
| case NVPTX::BI__nvvm_ldg_uc2: | |
| case NVPTX::BI__nvvm_ldg_uc4: | |
| case NVPTX::BI__nvvm_ldg_us: | |
| case NVPTX::BI__nvvm_ldg_us2: | |
| case NVPTX::BI__nvvm_ldg_us4: | |
| case NVPTX::BI__nvvm_ldg_ui: | |
| case NVPTX::BI__nvvm_ldg_ui2: | |
| case NVPTX::BI__nvvm_ldg_ui4: | |
| case NVPTX::BI__nvvm_ldg_ul: | |
| case NVPTX::BI__nvvm_ldg_ul2: | |
| case NVPTX::BI__nvvm_ldg_ull: | |
| case NVPTX::BI__nvvm_ldg_ull2: | |
| case NVPTX::BI__nvvm_ldg_f: | |
| case NVPTX::BI__nvvm_ldg_f2: | |
| case NVPTX::BI__nvvm_ldg_f4: | |
| case NVPTX::BI__nvvm_ldg_d: | |
| case NVPTX::BI__nvvm_ldg_d2: | |
| return makeLdg(*this, expr); | |
| case NVPTX::BI__nvvm_ldu_c: | |
| case NVPTX::BI__nvvm_ldu_sc: | |
| case NVPTX::BI__nvvm_ldu_c2: | |
| case NVPTX::BI__nvvm_ldu_sc2: | |
| case NVPTX::BI__nvvm_ldu_c4: | |
| case NVPTX::BI__nvvm_ldu_sc4: | |
| case NVPTX::BI__nvvm_ldu_s: | |
| case NVPTX::BI__nvvm_ldu_s2: | |
| case NVPTX::BI__nvvm_ldu_s4: | |
| case NVPTX::BI__nvvm_ldu_i: | |
| case NVPTX::BI__nvvm_ldu_i2: | |
| case NVPTX::BI__nvvm_ldu_i4: | |
| case NVPTX::BI__nvvm_ldu_l: | |
| case NVPTX::BI__nvvm_ldu_l2: | |
| case NVPTX::BI__nvvm_ldu_ll: | |
| case NVPTX::BI__nvvm_ldu_ll2: | |
| case NVPTX::BI__nvvm_ldu_uc: | |
| case NVPTX::BI__nvvm_ldu_uc2: | |
| case NVPTX::BI__nvvm_ldu_uc4: | |
| case NVPTX::BI__nvvm_ldu_us: | |
| case NVPTX::BI__nvvm_ldu_us2: | |
| case NVPTX::BI__nvvm_ldu_us4: | |
| case NVPTX::BI__nvvm_ldu_ui: | |
| case NVPTX::BI__nvvm_ldu_ui2: | |
| case NVPTX::BI__nvvm_ldu_ui4: | |
| case NVPTX::BI__nvvm_ldu_ul: | |
| case NVPTX::BI__nvvm_ldu_ul2: | |
| case NVPTX::BI__nvvm_ldu_ull: | |
| case NVPTX::BI__nvvm_ldu_ull2: | |
| return makeLdu(*this, expr, "nvvm.ldu.global.i"); | |
| case NVPTX::BI__nvvm_ldu_f: | |
| case NVPTX::BI__nvvm_ldu_f2: | |
| case NVPTX::BI__nvvm_ldu_f4: | |
| case NVPTX::BI__nvvm_ldu_d: | |
| case NVPTX::BI__nvvm_ldu_d2: | |
| return makeLdu(*this, expr, "nvvm.ldu.global.f"); | |
| case NVPTX::BI__nvvm_atom_cta_add_gen_i: | |
| case NVPTX::BI__nvvm_atom_cta_add_gen_l: | |
| case NVPTX::BI__nvvm_atom_cta_add_gen_ll: | |
| return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::Add, | |
| cir::SyncScopeKind::Workgroup); | |
| case NVPTX::BI__nvvm_atom_sys_add_gen_i: | |
| case NVPTX::BI__nvvm_atom_sys_add_gen_l: | |
| case NVPTX::BI__nvvm_atom_sys_add_gen_ll: | |
| return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::Add, | |
| cir::SyncScopeKind::System); | |
| case NVPTX::BI__nvvm_atom_cta_add_gen_f: | |
| case NVPTX::BI__nvvm_atom_cta_add_gen_d: | |
| return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::Add, | |
| cir::SyncScopeKind::Workgroup); | |
| case NVPTX::BI__nvvm_atom_sys_add_gen_f: | |
| case NVPTX::BI__nvvm_atom_sys_add_gen_d: | |
| return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::Add, | |
| cir::SyncScopeKind::System); | |
| case NVPTX::BI__nvvm_atom_cta_xchg_gen_i: | |
| case NVPTX::BI__nvvm_atom_cta_xchg_gen_l: | |
| case NVPTX::BI__nvvm_atom_cta_xchg_gen_ll: | |
| return makeScopedAtomicXchg(*this, expr, cir::SyncScopeKind::Workgroup); | |
| case NVPTX::BI__nvvm_atom_sys_xchg_gen_i: | |
| case NVPTX::BI__nvvm_atom_sys_xchg_gen_l: | |
| case NVPTX::BI__nvvm_atom_sys_xchg_gen_ll: | |
| return makeScopedAtomicXchg(*this, expr, cir::SyncScopeKind::System); | |
| case NVPTX::BI__nvvm_atom_cta_max_gen_i: | |
| case NVPTX::BI__nvvm_atom_cta_max_gen_ui: | |
| case NVPTX::BI__nvvm_atom_cta_max_gen_l: | |
| case NVPTX::BI__nvvm_atom_cta_max_gen_ul: | |
| case NVPTX::BI__nvvm_atom_cta_max_gen_ll: | |
| case NVPTX::BI__nvvm_atom_cta_max_gen_ull: | |
| return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::Max, | |
| cir::SyncScopeKind::Workgroup); | |
| case NVPTX::BI__nvvm_atom_sys_max_gen_i: | |
| case NVPTX::BI__nvvm_atom_sys_max_gen_ui: | |
| case NVPTX::BI__nvvm_atom_sys_max_gen_l: | |
| case NVPTX::BI__nvvm_atom_sys_max_gen_ul: | |
| case NVPTX::BI__nvvm_atom_sys_max_gen_ll: | |
| case NVPTX::BI__nvvm_atom_sys_max_gen_ull: | |
| return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::Max, | |
| cir::SyncScopeKind::System); | |
| case NVPTX::BI__nvvm_atom_cta_min_gen_i: | |
| case NVPTX::BI__nvvm_atom_cta_min_gen_ui: | |
| case NVPTX::BI__nvvm_atom_cta_min_gen_l: | |
| case NVPTX::BI__nvvm_atom_cta_min_gen_ul: | |
| case NVPTX::BI__nvvm_atom_cta_min_gen_ll: | |
| case NVPTX::BI__nvvm_atom_cta_min_gen_ull: | |
| return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::Min, | |
| cir::SyncScopeKind::Workgroup); | |
| case NVPTX::BI__nvvm_atom_sys_min_gen_i: | |
| case NVPTX::BI__nvvm_atom_sys_min_gen_ui: | |
| case NVPTX::BI__nvvm_atom_sys_min_gen_l: | |
| case NVPTX::BI__nvvm_atom_sys_min_gen_ul: | |
| case NVPTX::BI__nvvm_atom_sys_min_gen_ll: | |
| case NVPTX::BI__nvvm_atom_sys_min_gen_ull: | |
| return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::Min, | |
| cir::SyncScopeKind::System); | |
| case NVPTX::BI__nvvm_atom_cta_inc_gen_ui: | |
| return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::UIncWrap, | |
| cir::SyncScopeKind::Workgroup); | |
| case NVPTX::BI__nvvm_atom_cta_dec_gen_ui: | |
| return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::UDecWrap, | |
| cir::SyncScopeKind::Workgroup); | |
| case NVPTX::BI__nvvm_atom_sys_inc_gen_ui: | |
| return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::UIncWrap, | |
| cir::SyncScopeKind::System); | |
| case NVPTX::BI__nvvm_atom_sys_dec_gen_ui: | |
| return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::UDecWrap, | |
| cir::SyncScopeKind::System); | |
| case NVPTX::BI__nvvm_atom_cta_and_gen_i: | |
| case NVPTX::BI__nvvm_atom_cta_and_gen_l: | |
| case NVPTX::BI__nvvm_atom_cta_and_gen_ll: | |
| return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::And, | |
| cir::SyncScopeKind::Workgroup); | |
| case NVPTX::BI__nvvm_atom_sys_and_gen_i: | |
| case NVPTX::BI__nvvm_atom_sys_and_gen_l: | |
| case NVPTX::BI__nvvm_atom_sys_and_gen_ll: | |
| return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::And, | |
| cir::SyncScopeKind::System); | |
| case NVPTX::BI__nvvm_atom_cta_or_gen_i: | |
| case NVPTX::BI__nvvm_atom_cta_or_gen_l: | |
| case NVPTX::BI__nvvm_atom_cta_or_gen_ll: | |
| return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::Or, | |
| cir::SyncScopeKind::Workgroup); | |
| case NVPTX::BI__nvvm_atom_sys_or_gen_i: | |
| case NVPTX::BI__nvvm_atom_sys_or_gen_l: | |
| case NVPTX::BI__nvvm_atom_sys_or_gen_ll: | |
| return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::Or, | |
| cir::SyncScopeKind::System); | |
| case NVPTX::BI__nvvm_atom_cta_xor_gen_i: | |
| case NVPTX::BI__nvvm_atom_cta_xor_gen_l: | |
| case NVPTX::BI__nvvm_atom_cta_xor_gen_ll: | |
| return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::Xor, | |
| cir::SyncScopeKind::Workgroup); | |
| case NVPTX::BI__nvvm_atom_sys_xor_gen_i: | |
| case NVPTX::BI__nvvm_atom_sys_xor_gen_l: | |
| case NVPTX::BI__nvvm_atom_sys_xor_gen_ll: | |
| return makeScopedAtomicRMW(*this, expr, cir::AtomicFetchKind::Xor, | |
| cir::SyncScopeKind::System); | |
| case NVPTX::BI__nvvm_atom_cta_cas_gen_us: | |
| case NVPTX::BI__nvvm_atom_cta_cas_gen_i: | |
| case NVPTX::BI__nvvm_atom_cta_cas_gen_l: | |
| case NVPTX::BI__nvvm_atom_cta_cas_gen_ll: | |
| return emitAtomicCmpXchg(expr, /*returnBool=*/false, cir::MemOrder::Relaxed, | |
| cir::MemOrder::Relaxed, | |
| cir::SyncScopeKind::Workgroup); | |
| case NVPTX::BI__nvvm_atom_sys_cas_gen_us: | |
| case NVPTX::BI__nvvm_atom_sys_cas_gen_i: | |
| case NVPTX::BI__nvvm_atom_sys_cas_gen_l: | |
| case NVPTX::BI__nvvm_atom_sys_cas_gen_ll: | |
| return emitAtomicCmpXchg(expr, /*returnBool=*/false, cir::MemOrder::Relaxed, | |
| cir::MemOrder::Relaxed, | |
| cir::SyncScopeKind::System); | |
| case NVPTX::BI__nvvm_match_all_sync_i32p: | |
| case NVPTX::BI__nvvm_match_all_sync_i64p: | |
| cgm.errorNYI(expr->getSourceRange(), | |
| std::string("unimplemented NVPTX builtin call: ") + | |
| getContext().BuiltinInfo.getName(builtinId)); | |
| return mlir::Value{}; | |
| // FP MMA loads | |
| case NVPTX::BI__hmma_m16n16k16_ld_a: | |
| case NVPTX::BI__hmma_m16n16k16_ld_b: | |
| case NVPTX::BI__hmma_m16n16k16_ld_c_f16: | |
| case NVPTX::BI__hmma_m16n16k16_ld_c_f32: | |
| case NVPTX::BI__hmma_m32n8k16_ld_a: | |
| case NVPTX::BI__hmma_m32n8k16_ld_b: | |
| case NVPTX::BI__hmma_m32n8k16_ld_c_f16: | |
| case NVPTX::BI__hmma_m32n8k16_ld_c_f32: | |
| case NVPTX::BI__hmma_m8n32k16_ld_a: | |
| case NVPTX::BI__hmma_m8n32k16_ld_b: | |
| case NVPTX::BI__hmma_m8n32k16_ld_c_f16: | |
| case NVPTX::BI__hmma_m8n32k16_ld_c_f32: | |
| cgm.errorNYI(expr->getSourceRange(), | |
| std::string("unimplemented NVPTX builtin call: ") + | |
| getContext().BuiltinInfo.getName(builtinId)); | |
| return mlir::Value{}; | |
| case NVPTX::BI__imma_m16n16k16_ld_a_s8: | |
| case NVPTX::BI__imma_m16n16k16_ld_a_u8: | |
| case NVPTX::BI__imma_m16n16k16_ld_b_s8: | |
| case NVPTX::BI__imma_m16n16k16_ld_b_u8: | |
| case NVPTX::BI__imma_m16n16k16_ld_c: | |
| case NVPTX::BI__imma_m32n8k16_ld_a_s8: | |
| case NVPTX::BI__imma_m32n8k16_ld_a_u8: | |
| case NVPTX::BI__imma_m32n8k16_ld_b_s8: | |
| case NVPTX::BI__imma_m32n8k16_ld_b_u8: | |
| case NVPTX::BI__imma_m32n8k16_ld_c: | |
| case NVPTX::BI__imma_m8n32k16_ld_a_s8: | |
| case NVPTX::BI__imma_m8n32k16_ld_a_u8: | |
| case NVPTX::BI__imma_m8n32k16_ld_b_s8: | |
| case NVPTX::BI__imma_m8n32k16_ld_b_u8: | |
| case NVPTX::BI__imma_m8n32k16_ld_c: | |
| cgm.errorNYI(expr->getSourceRange(), | |
| std::string("unimplemented NVPTX builtin call: ") + | |
| getContext().BuiltinInfo.getName(builtinId)); | |
| return mlir::Value{}; | |
| case NVPTX::BI__imma_m8n8k32_ld_a_s4: | |
| case NVPTX::BI__imma_m8n8k32_ld_a_u4: | |
| case NVPTX::BI__imma_m8n8k32_ld_b_s4: | |
| case NVPTX::BI__imma_m8n8k32_ld_b_u4: | |
| case NVPTX::BI__imma_m8n8k32_ld_c: | |
| case NVPTX::BI__bmma_m8n8k128_ld_a_b1: | |
| case NVPTX::BI__bmma_m8n8k128_ld_b_b1: | |
| case NVPTX::BI__bmma_m8n8k128_ld_c: | |
| cgm.errorNYI(expr->getSourceRange(), | |
| std::string("unimplemented NVPTX builtin call: ") + | |
| getContext().BuiltinInfo.getName(builtinId)); | |
| return mlir::Value{}; | |
| case NVPTX::BI__dmma_m8n8k4_ld_a: | |
| case NVPTX::BI__dmma_m8n8k4_ld_b: | |
| case NVPTX::BI__dmma_m8n8k4_ld_c: | |
| cgm.errorNYI(expr->getSourceRange(), | |
| std::string("unimplemented NVPTX builtin call: ") + | |
| getContext().BuiltinInfo.getName(builtinId)); | |
| return mlir::Value{}; | |
| case NVPTX::BI__mma_bf16_m16n16k16_ld_a: | |
| case NVPTX::BI__mma_bf16_m16n16k16_ld_b: | |
| case NVPTX::BI__mma_bf16_m8n32k16_ld_a: | |
| case NVPTX::BI__mma_bf16_m8n32k16_ld_b: | |
| case NVPTX::BI__mma_bf16_m32n8k16_ld_a: | |
| case NVPTX::BI__mma_bf16_m32n8k16_ld_b: | |
| case NVPTX::BI__mma_tf32_m16n16k8_ld_a: | |
| case NVPTX::BI__mma_tf32_m16n16k8_ld_b: | |
| case NVPTX::BI__mma_tf32_m16n16k8_ld_c: | |
| cgm.errorNYI(expr->getSourceRange(), | |
| std::string("unimplemented NVPTX builtin call: ") + | |
| getContext().BuiltinInfo.getName(builtinId)); | |
| return mlir::Value{}; | |
| case NVPTX::BI__hmma_m16n16k16_st_c_f16: | |
| case NVPTX::BI__hmma_m16n16k16_st_c_f32: | |
| case NVPTX::BI__hmma_m32n8k16_st_c_f16: | |
| case NVPTX::BI__hmma_m32n8k16_st_c_f32: | |
| case NVPTX::BI__hmma_m8n32k16_st_c_f16: | |
| case NVPTX::BI__hmma_m8n32k16_st_c_f32: | |
| case NVPTX::BI__imma_m16n16k16_st_c_i32: | |
| case NVPTX::BI__imma_m32n8k16_st_c_i32: | |
| case NVPTX::BI__imma_m8n32k16_st_c_i32: | |
| case NVPTX::BI__imma_m8n8k32_st_c_i32: | |
| case NVPTX::BI__bmma_m8n8k128_st_c_i32: | |
| case NVPTX::BI__dmma_m8n8k4_st_c_f64: | |
| case NVPTX::BI__mma_m16n16k8_st_c_f32: | |
| cgm.errorNYI(expr->getSourceRange(), | |
| std::string("unimplemented NVPTX builtin call: ") + | |
| getContext().BuiltinInfo.getName(builtinId)); | |
| return mlir::Value{}; | |
| // BI__hmma_m16n16k16_mma_<Dtype><CType>(d, a, b, c, layout, satf) --> | |
| // Intrinsic::nvvm_wmma_m16n16k16_mma_sync<layout A,B><DType><CType><Satf> | |
| case NVPTX::BI__hmma_m16n16k16_mma_f16f16: | |
| case NVPTX::BI__hmma_m16n16k16_mma_f32f16: | |
| case NVPTX::BI__hmma_m16n16k16_mma_f32f32: | |
| case NVPTX::BI__hmma_m16n16k16_mma_f16f32: | |
| case NVPTX::BI__hmma_m32n8k16_mma_f16f16: | |
| case NVPTX::BI__hmma_m32n8k16_mma_f32f16: | |
| case NVPTX::BI__hmma_m32n8k16_mma_f32f32: | |
| case NVPTX::BI__hmma_m32n8k16_mma_f16f32: | |
| case NVPTX::BI__hmma_m8n32k16_mma_f16f16: | |
| case NVPTX::BI__hmma_m8n32k16_mma_f32f16: | |
| case NVPTX::BI__hmma_m8n32k16_mma_f32f32: | |
| case NVPTX::BI__hmma_m8n32k16_mma_f16f32: | |
| case NVPTX::BI__imma_m16n16k16_mma_s8: | |
| case NVPTX::BI__imma_m16n16k16_mma_u8: | |
| case NVPTX::BI__imma_m32n8k16_mma_s8: | |
| case NVPTX::BI__imma_m32n8k16_mma_u8: | |
| case NVPTX::BI__imma_m8n32k16_mma_s8: | |
| case NVPTX::BI__imma_m8n32k16_mma_u8: | |
| case NVPTX::BI__imma_m8n8k32_mma_s4: | |
| case NVPTX::BI__imma_m8n8k32_mma_u4: | |
| case NVPTX::BI__bmma_m8n8k128_mma_xor_popc_b1: | |
| case NVPTX::BI__bmma_m8n8k128_mma_and_popc_b1: | |
| case NVPTX::BI__dmma_m8n8k4_mma_f64: | |
| case NVPTX::BI__mma_bf16_m16n16k16_mma_f32: | |
| case NVPTX::BI__mma_bf16_m8n32k16_mma_f32: | |
| case NVPTX::BI__mma_bf16_m32n8k16_mma_f32: | |
| case NVPTX::BI__mma_tf32_m16n16k8_mma_f32: | |
| cgm.errorNYI(expr->getSourceRange(), | |
| std::string("unimplemented NVPTX builtin call: ") + | |
| getContext().BuiltinInfo.getName(builtinId)); | |
| return mlir::Value{}; | |
| // The following builtins require half type support | |
| case NVPTX::BI__nvvm_ff2f16x2_rn: | |
| cgm.errorNYI(expr->getSourceRange(), | |
| std::string("unimplemented NVPTX builtin call: ") + | |
| getContext().BuiltinInfo.getName(builtinId)); | |
| return mlir::Value{}; | |
| case NVPTX::BI__nvvm_ff2f16x2_rn_relu: | |
| cgm.errorNYI(expr->getSourceRange(), | |
| std::string("unimplemented NVPTX builtin call: ") + | |
| getContext().BuiltinInfo.getName(builtinId)); | |
| return mlir::Value{}; | |
| case NVPTX::BI__nvvm_ff2f16x2_rz: | |
| cgm.errorNYI(expr->getSourceRange(), | |
| std::string("unimplemented NVPTX builtin call: ") + | |
| getContext().BuiltinInfo.getName(builtinId)); | |
| return mlir::Value{}; | |
| case NVPTX::BI__nvvm_ff2f16x2_rz_relu: | |
| cgm.errorNYI(expr->getSourceRange(), | |
| std::string("unimplemented NVPTX builtin call: ") + | |
| getContext().BuiltinInfo.getName(builtinId)); | |
| return mlir::Value{}; | |
| case NVPTX::BI__nvvm_fma_rn_f16: | |
| return emitBuiltinWithOneOverloadedType<3>(expr, "nvvm.fma.rn.f16") | |
| .getValue(); | |
| case NVPTX::BI__nvvm_fma_rn_f16x2: | |
| return emitBuiltinWithOneOverloadedType<3>(expr, "nvvm.fma.rn.f16x2") | |
| .getValue(); | |
| case NVPTX::BI__nvvm_fma_rn_ftz_f16: | |
| return emitBuiltinWithOneOverloadedType<3>(expr, "nvvm.fma.rn.ftz.f16") | |
| .getValue(); | |
| case NVPTX::BI__nvvm_fma_rn_ftz_f16x2: | |
| return emitBuiltinWithOneOverloadedType<3>(expr, "nvvm.fma.rn.ftz.f16x2") | |
| .getValue(); | |
| case NVPTX::BI__nvvm_fma_rn_ftz_relu_f16: | |
| return emitBuiltinWithOneOverloadedType<3>(expr, "nvvm.fma.rn.ftz.relu.f16") | |
| .getValue(); | |
| case NVPTX::BI__nvvm_fma_rn_ftz_relu_f16x2: | |
| return emitBuiltinWithOneOverloadedType<3>(expr, | |
| "nvvm.fma.rn.ftz.relu.f16x2") | |
| .getValue(); | |
| case NVPTX::BI__nvvm_fma_rn_ftz_sat_f16: | |
| return emitBuiltinWithOneOverloadedType<3>(expr, "nvvm.fma.rn.ftz.sat.f16") | |
| .getValue(); | |
| case NVPTX::BI__nvvm_fma_rn_ftz_sat_f16x2: | |
| return emitBuiltinWithOneOverloadedType<3>(expr, | |
| "nvvm.fma.rn.ftz.sat.f16x2") | |
| .getValue(); | |
| case NVPTX::BI__nvvm_fma_rn_relu_f16: | |
| return emitBuiltinWithOneOverloadedType<3>(expr, "nvvm.fma.rn.relu.f16") | |
| .getValue(); | |
| case NVPTX::BI__nvvm_fma_rn_relu_f16x2: | |
| return emitBuiltinWithOneOverloadedType<3>(expr, "nvvm.fma.rn.relu.f16x2") | |
| .getValue(); | |
| case NVPTX::BI__nvvm_fma_rn_sat_f16: | |
| return emitBuiltinWithOneOverloadedType<3>(expr, "nvvm.fma.rn.sat.f16") | |
| .getValue(); | |
| case NVPTX::BI__nvvm_fma_rn_sat_f16x2: | |
| return emitBuiltinWithOneOverloadedType<3>(expr, "nvvm.fma.rn.sat.f16x2") | |
| .getValue(); | |
| case NVPTX::BI__nvvm_fma_rn_oob_f16: | |
| case NVPTX::BI__nvvm_fma_rn_oob_f16x2: | |
| case NVPTX::BI__nvvm_fma_rn_oob_bf16: | |
| case NVPTX::BI__nvvm_fma_rn_oob_bf16x2: | |
| return emitBuiltinWithOneOverloadedType<3>(expr, "nvvm.fma.rn.oob") | |
| .getValue(); | |
| case NVPTX::BI__nvvm_fma_rn_oob_relu_f16: | |
| case NVPTX::BI__nvvm_fma_rn_oob_relu_f16x2: | |
| case NVPTX::BI__nvvm_fma_rn_oob_relu_bf16: | |
| case NVPTX::BI__nvvm_fma_rn_oob_relu_bf16x2: | |
| return emitBuiltinWithOneOverloadedType<3>(expr, "nvvm.fma.rn.oob.relu") | |
| .getValue(); | |
| case NVPTX::BI__nvvm_fmax_f16: | |
| return emitBuiltinWithOneOverloadedType<2>(expr, "nvvm.fmax.f16") | |
| .getValue(); | |
| case NVPTX::BI__nvvm_fmax_f16x2: | |
| return emitBuiltinWithOneOverloadedType<2>(expr, "nvvm.fmax.f16x2") | |
| .getValue(); | |
| case NVPTX::BI__nvvm_fmax_ftz_f16: | |
| return emitBuiltinWithOneOverloadedType<2>(expr, "nvvm.fmax.ftz.f16") | |
| .getValue(); | |
| case NVPTX::BI__nvvm_fmax_ftz_f16x2: | |
| return emitBuiltinWithOneOverloadedType<2>(expr, "nvvm.fmax.ftz.f16x2") | |
| .getValue(); | |
| case NVPTX::BI__nvvm_fmax_ftz_nan_f16: | |
| return emitBuiltinWithOneOverloadedType<2>(expr, "nvvm.fmax.ftz.nan.f16") | |
| .getValue(); | |
| case NVPTX::BI__nvvm_fmax_ftz_nan_f16x2: | |
| return emitBuiltinWithOneOverloadedType<2>(expr, "nvvm.fmax.ftz.nan.f16x2") | |
| .getValue(); | |
| case NVPTX::BI__nvvm_fmax_ftz_nan_xorsign_abs_f16: | |
| return emitBuiltinWithOneOverloadedType<2>( | |
| expr, "nvvm.fmax.ftz.nan.xorsign.abs.f16") | |
| .getValue(); | |
| case NVPTX::BI__nvvm_fmax_ftz_nan_xorsign_abs_f16x2: | |
| return emitBuiltinWithOneOverloadedType<2>( | |
| expr, "nvvm.fmax.ftz.nan.xorsign.abs.f16x2") | |
| .getValue(); | |
| case NVPTX::BI__nvvm_fmax_ftz_xorsign_abs_f16: | |
| return emitBuiltinWithOneOverloadedType<2>(expr, | |
| "nvvm.fmax.ftz.xorsign.abs.f16") | |
| .getValue(); | |
| case NVPTX::BI__nvvm_fmax_ftz_xorsign_abs_f16x2: | |
| return emitBuiltinWithOneOverloadedType<2>( | |
| expr, "nvvm.fmax.ftz.xorsign.abs.f16x2") | |
| .getValue(); | |
| case NVPTX::BI__nvvm_fmax_nan_f16: | |
| return emitBuiltinWithOneOverloadedType<2>(expr, "nvvm.fmax.nan.f16") | |
| .getValue(); | |
| case NVPTX::BI__nvvm_fmax_nan_f16x2: | |
| return emitBuiltinWithOneOverloadedType<2>(expr, "nvvm.fmax.nan.f16x2") | |
| .getValue(); | |
| case NVPTX::BI__nvvm_fmax_nan_xorsign_abs_f16: | |
| return emitBuiltinWithOneOverloadedType<2>(expr, | |
| "nvvm.fmax.nan.xorsign.abs.f16") | |
| .getValue(); | |
| case NVPTX::BI__nvvm_fmax_nan_xorsign_abs_f16x2: | |
| return emitBuiltinWithOneOverloadedType<2>( | |
| expr, "nvvm.fmax.nan.xorsign.abs.f16x2") | |
| .getValue(); | |
| case NVPTX::BI__nvvm_fmax_xorsign_abs_f16: | |
| return emitBuiltinWithOneOverloadedType<2>(expr, | |
| "nvvm.fmax.xorsign.abs.f16") | |
| .getValue(); | |
| case NVPTX::BI__nvvm_fmax_xorsign_abs_f16x2: | |
| return emitBuiltinWithOneOverloadedType<2>(expr, | |
| "nvvm.fmax.xorsign.abs.f16x2") | |
| .getValue(); | |
| case NVPTX::BI__nvvm_fmin_f16: | |
| return emitBuiltinWithOneOverloadedType<2>(expr, "nvvm.fmin.f16") | |
| .getValue(); | |
| case NVPTX::BI__nvvm_fmin_f16x2: | |
| return emitBuiltinWithOneOverloadedType<2>(expr, "nvvm.fmin.f16x2") | |
| .getValue(); | |
| case NVPTX::BI__nvvm_fmin_ftz_f16: | |
| return emitBuiltinWithOneOverloadedType<2>(expr, "nvvm.fmin.ftz.f16") | |
| .getValue(); | |
| case NVPTX::BI__nvvm_fmin_ftz_f16x2: | |
| return emitBuiltinWithOneOverloadedType<2>(expr, "nvvm.fmin.ftz.f16x2") | |
| .getValue(); | |
| case NVPTX::BI__nvvm_fmin_ftz_nan_f16: | |
| return emitBuiltinWithOneOverloadedType<2>(expr, "nvvm.fmin.ftz.nan.f16") | |
| .getValue(); | |
| case NVPTX::BI__nvvm_fmin_ftz_nan_f16x2: | |
| return emitBuiltinWithOneOverloadedType<2>(expr, "nvvm.fmin.ftz.nan.f16x2") | |
| .getValue(); | |
| case NVPTX::BI__nvvm_fmin_ftz_nan_xorsign_abs_f16: | |
| return emitBuiltinWithOneOverloadedType<2>( | |
| expr, "nvvm.fmin.ftz.nan.xorsign.abs.f16") | |
| .getValue(); | |
| case NVPTX::BI__nvvm_fmin_ftz_nan_xorsign_abs_f16x2: | |
| return emitBuiltinWithOneOverloadedType<2>( | |
| expr, "nvvm.fmin.ftz.nan.xorsign.abs.f16x2") | |
| .getValue(); | |
| case NVPTX::BI__nvvm_fmin_ftz_xorsign_abs_f16: | |
| return emitBuiltinWithOneOverloadedType<2>(expr, | |
| "nvvm.fmin.ftz.xorsign.abs.f16") | |
| .getValue(); | |
| case NVPTX::BI__nvvm_fmin_ftz_xorsign_abs_f16x2: | |
| return emitBuiltinWithOneOverloadedType<2>( | |
| expr, "nvvm.fmin.ftz.xorsign.abs.f16x2") | |
| .getValue(); | |
| case NVPTX::BI__nvvm_fmin_nan_f16: | |
| return emitBuiltinWithOneOverloadedType<2>(expr, "nvvm.fmin.nan.f16") | |
| .getValue(); | |
| case NVPTX::BI__nvvm_fmin_nan_f16x2: | |
| return emitBuiltinWithOneOverloadedType<2>(expr, "nvvm.fmin.nan.f16x2") | |
| .getValue(); | |
| case NVPTX::BI__nvvm_fmin_nan_xorsign_abs_f16: | |
| return emitBuiltinWithOneOverloadedType<2>(expr, | |
| "nvvm.fmin.nan.xorsign.abs.f16") | |
| .getValue(); | |
| case NVPTX::BI__nvvm_fmin_nan_xorsign_abs_f16x2: | |
| return emitBuiltinWithOneOverloadedType<2>( | |
| expr, "nvvm.fmin.nan.xorsign.abs.f16x2") | |
| .getValue(); | |
| case NVPTX::BI__nvvm_fmin_xorsign_abs_f16: | |
| return emitBuiltinWithOneOverloadedType<2>(expr, | |
| "nvvm.fmin.xorsign.abs.f16") | |
| .getValue(); | |
| case NVPTX::BI__nvvm_fmin_xorsign_abs_f16x2: | |
| return emitBuiltinWithOneOverloadedType<2>(expr, | |
| "nvvm.fmin.xorsign.abs.f16x2") | |
| .getValue(); | |
| case NVPTX::BI__nvvm_fabs_f: | |
| case NVPTX::BI__nvvm_abs_bf16: | |
| case NVPTX::BI__nvvm_abs_bf16x2: | |
| case NVPTX::BI__nvvm_fabs_f16: | |
| case NVPTX::BI__nvvm_fabs_f16x2: | |
| return emitUnaryNVVMIntrinsic(*this, expr, "nvvm.fabs"); | |
| case NVPTX::BI__nvvm_fabs_ftz_f: | |
| case NVPTX::BI__nvvm_fabs_ftz_f16: | |
| case NVPTX::BI__nvvm_fabs_ftz_f16x2: | |
| return emitUnaryNVVMIntrinsic(*this, expr, "nvvm.fabs.ftz"); | |
| case NVPTX::BI__nvvm_fabs_d: | |
| return emitUnaryNVVMIntrinsic(*this, expr, "fabs"); | |
| case NVPTX::BI__nvvm_ex2_approx_d: | |
| case NVPTX::BI__nvvm_ex2_approx_f: | |
| case NVPTX::BI__nvvm_ex2_approx_f16: | |
| case NVPTX::BI__nvvm_ex2_approx_f16x2: | |
| return emitUnaryNVVMIntrinsic(*this, expr, "nvvm.ex2.approx"); | |
| case NVPTX::BI__nvvm_ex2_approx_ftz_f: | |
| return emitUnaryNVVMIntrinsic(*this, expr, "nvvm.ex2.approx.ftz"); | |
| case NVPTX::BI__nvvm_add_rn_f: | |
| case NVPTX::BI__nvvm_add_rn_d: | |
| return emitNVVMFAdd(*this, expr, "nvvm.fadd", | |
| llvm::APFloat::rmNearestTiesToEven); | |
| case NVPTX::BI__nvvm_add_rz_f: | |
| case NVPTX::BI__nvvm_add_rz_d: | |
| return emitNVVMFAdd(*this, expr, "nvvm.fadd", llvm::APFloat::rmTowardZero); | |
| case NVPTX::BI__nvvm_add_rm_f: | |
| case NVPTX::BI__nvvm_add_rm_d: | |
| return emitNVVMFAdd(*this, expr, "nvvm.fadd", | |
| llvm::APFloat::rmTowardNegative); | |
| case NVPTX::BI__nvvm_add_rp_f: | |
| case NVPTX::BI__nvvm_add_rp_d: | |
| return emitNVVMFAdd(*this, expr, "nvvm.fadd", | |
| llvm::APFloat::rmTowardPositive); | |
| case NVPTX::BI__nvvm_add_rn_ftz_f: | |
| return emitNVVMFAdd(*this, expr, "nvvm.fadd.ftz", | |
| llvm::APFloat::rmNearestTiesToEven); | |
| case NVPTX::BI__nvvm_add_rz_ftz_f: | |
| return emitNVVMFAdd(*this, expr, "nvvm.fadd.ftz", | |
| llvm::APFloat::rmTowardZero); | |
| case NVPTX::BI__nvvm_add_rm_ftz_f: | |
| return emitNVVMFAdd(*this, expr, "nvvm.fadd.ftz", | |
| llvm::APFloat::rmTowardNegative); | |
| case NVPTX::BI__nvvm_add_rp_ftz_f: | |
| return emitNVVMFAdd(*this, expr, "nvvm.fadd.ftz", | |
| llvm::APFloat::rmTowardPositive); | |
| case NVPTX::BI__nvvm_add_rn_sat_f: | |
| case NVPTX::BI__nvvm_add_rn_sat_f16: | |
| case NVPTX::BI__nvvm_add_rn_sat_v2f16: | |
| return emitNVVMFAdd(*this, expr, "nvvm.fadd.sat", | |
| llvm::APFloat::rmNearestTiesToEven); | |
| case NVPTX::BI__nvvm_add_rz_sat_f: | |
| return emitNVVMFAdd(*this, expr, "nvvm.fadd.sat", | |
| llvm::APFloat::rmTowardZero); | |
| case NVPTX::BI__nvvm_add_rm_sat_f: | |
| return emitNVVMFAdd(*this, expr, "nvvm.fadd.sat", | |
| llvm::APFloat::rmTowardNegative); | |
| case NVPTX::BI__nvvm_add_rp_sat_f: | |
| return emitNVVMFAdd(*this, expr, "nvvm.fadd.sat", | |
| llvm::APFloat::rmTowardPositive); | |
| case NVPTX::BI__nvvm_add_rn_ftz_sat_f: | |
| case NVPTX::BI__nvvm_add_rn_ftz_sat_f16: | |
| case NVPTX::BI__nvvm_add_rn_ftz_sat_v2f16: | |
| return emitNVVMFAdd(*this, expr, "nvvm.fadd.ftz.sat", | |
| llvm::APFloat::rmNearestTiesToEven); | |
| case NVPTX::BI__nvvm_add_rz_ftz_sat_f: | |
| return emitNVVMFAdd(*this, expr, "nvvm.fadd.ftz.sat", | |
| llvm::APFloat::rmTowardZero); | |
| case NVPTX::BI__nvvm_add_rm_ftz_sat_f: | |
| return emitNVVMFAdd(*this, expr, "nvvm.fadd.ftz.sat", | |
| llvm::APFloat::rmTowardNegative); | |
| case NVPTX::BI__nvvm_add_rp_ftz_sat_f: | |
| return emitNVVMFAdd(*this, expr, "nvvm.fadd.ftz.sat", | |
| llvm::APFloat::rmTowardPositive); | |
| case NVPTX::BI__nvvm_ldg_h: | |
| case NVPTX::BI__nvvm_ldg_h2: | |
| cgm.errorNYI(expr->getSourceRange(), | |
| std::string("unimplemented NVPTX builtin call: ") + | |
| getContext().BuiltinInfo.getName(builtinId)); | |
| return mlir::Value{}; | |
| case NVPTX::BI__nvvm_ldu_h: | |
| case NVPTX::BI__nvvm_ldu_h2: | |
| return makeLdu(*this, expr, "nvvm.ldu.global.f"); | |
| case NVPTX::BI__nvvm_cp_async_ca_shared_global_4: | |
| cgm.errorNYI(expr->getSourceRange(), | |
| std::string("unimplemented NVPTX builtin call: ") + | |
| getContext().BuiltinInfo.getName(builtinId)); | |
| return mlir::Value{}; | |
| case NVPTX::BI__nvvm_cp_async_ca_shared_global_8: | |
| cgm.errorNYI(expr->getSourceRange(), | |
| std::string("unimplemented NVPTX builtin call: ") + | |
| getContext().BuiltinInfo.getName(builtinId)); | |
| return mlir::Value{}; | |
| case NVPTX::BI__nvvm_cp_async_ca_shared_global_16: | |
| cgm.errorNYI(expr->getSourceRange(), | |
| std::string("unimplemented NVPTX builtin call: ") + | |
| getContext().BuiltinInfo.getName(builtinId)); | |
| return mlir::Value{}; | |
| case NVPTX::BI__nvvm_cp_async_cg_shared_global_16: | |
| cgm.errorNYI(expr->getSourceRange(), | |
| std::string("unimplemented NVPTX builtin call: ") + | |
| getContext().BuiltinInfo.getName(builtinId)); | |
| return mlir::Value{}; | |
| case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_x: | |
| cgm.errorNYI(expr->getSourceRange(), | |
| std::string("unimplemented NVPTX builtin call: ") + | |
| getContext().BuiltinInfo.getName(builtinId)); | |
| return mlir::Value{}; | |
| case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_y: | |
| cgm.errorNYI(expr->getSourceRange(), | |
| std::string("unimplemented NVPTX builtin call: ") + | |
| getContext().BuiltinInfo.getName(builtinId)); | |
| return mlir::Value{}; | |
| case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_z: | |
| cgm.errorNYI(expr->getSourceRange(), | |
| std::string("unimplemented NVPTX builtin call: ") + | |
| getContext().BuiltinInfo.getName(builtinId)); | |
| return mlir::Value{}; | |
| case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_w: | |
| cgm.errorNYI(expr->getSourceRange(), | |
| std::string("unimplemented NVPTX builtin call: ") + | |
| getContext().BuiltinInfo.getName(builtinId)); | |
| return mlir::Value{}; | |
| case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_x: | |
| cgm.errorNYI(expr->getSourceRange(), | |
| std::string("unimplemented NVPTX builtin call: ") + | |
| getContext().BuiltinInfo.getName(builtinId)); | |
| return mlir::Value{}; | |
| case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_y: | |
| cgm.errorNYI(expr->getSourceRange(), | |
| std::string("unimplemented NVPTX builtin call: ") + | |
| getContext().BuiltinInfo.getName(builtinId)); | |
| return mlir::Value{}; | |
| case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_z: | |
| cgm.errorNYI(expr->getSourceRange(), | |
| std::string("unimplemented NVPTX builtin call: ") + | |
| getContext().BuiltinInfo.getName(builtinId)); | |
| return mlir::Value{}; | |
| case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_w: | |
| cgm.errorNYI(expr->getSourceRange(), | |
| std::string("unimplemented NVPTX builtin call: ") + | |
| getContext().BuiltinInfo.getName(builtinId)); | |
| return mlir::Value{}; | |
| case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_x: | |
| cgm.errorNYI(expr->getSourceRange(), | |
| std::string("unimplemented NVPTX builtin call: ") + | |
| getContext().BuiltinInfo.getName(builtinId)); | |
| return mlir::Value{}; | |
| case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_y: | |
| cgm.errorNYI(expr->getSourceRange(), | |
| std::string("unimplemented NVPTX builtin call: ") + | |
| getContext().BuiltinInfo.getName(builtinId)); | |
| return mlir::Value{}; | |
| case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_z: | |
| cgm.errorNYI(expr->getSourceRange(), | |
| std::string("unimplemented NVPTX builtin call: ") + | |
| getContext().BuiltinInfo.getName(builtinId)); | |
| return mlir::Value{}; | |
| case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_w: | |
| cgm.errorNYI(expr->getSourceRange(), | |
| std::string("unimplemented NVPTX builtin call: ") + | |
| getContext().BuiltinInfo.getName(builtinId)); | |
| return mlir::Value{}; | |
| case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_x: | |
| cgm.errorNYI(expr->getSourceRange(), | |
| std::string("unimplemented NVPTX builtin call: ") + | |
| getContext().BuiltinInfo.getName(builtinId)); | |
| return mlir::Value{}; | |
| case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_y: | |
| cgm.errorNYI(expr->getSourceRange(), | |
| std::string("unimplemented NVPTX builtin call: ") + | |
| getContext().BuiltinInfo.getName(builtinId)); | |
| return mlir::Value{}; | |
| case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_z: | |
| cgm.errorNYI(expr->getSourceRange(), | |
| std::string("unimplemented NVPTX builtin call: ") + | |
| getContext().BuiltinInfo.getName(builtinId)); | |
| return mlir::Value{}; | |
| case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_w: | |
| cgm.errorNYI(expr->getSourceRange(), | |
| std::string("unimplemented NVPTX builtin call: ") + | |
| getContext().BuiltinInfo.getName(builtinId)); | |
| return mlir::Value{}; | |
| case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctarank: | |
| cgm.errorNYI(expr->getSourceRange(), | |
| std::string("unimplemented NVPTX builtin call: ") + | |
| getContext().BuiltinInfo.getName(builtinId)); | |
| return mlir::Value{}; | |
| case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctarank: | |
| cgm.errorNYI(expr->getSourceRange(), | |
| std::string("unimplemented NVPTX builtin call: ") + | |
| getContext().BuiltinInfo.getName(builtinId)); | |
| return mlir::Value{}; | |
| case NVPTX::BI__nvvm_is_explicit_cluster: | |
| cgm.errorNYI(expr->getSourceRange(), | |
| std::string("unimplemented NVPTX builtin call: ") + | |
| getContext().BuiltinInfo.getName(builtinId)); | |
| return mlir::Value{}; | |
| case NVPTX::BI__nvvm_isspacep_shared_cluster: | |
| cgm.errorNYI(expr->getSourceRange(), | |
| std::string("unimplemented NVPTX builtin call: ") + | |
| getContext().BuiltinInfo.getName(builtinId)); | |
| return mlir::Value{}; | |
| case NVPTX::BI__nvvm_mapa: | |
| cgm.errorNYI(expr->getSourceRange(), | |
| std::string("unimplemented NVPTX builtin call: ") + | |
| getContext().BuiltinInfo.getName(builtinId)); | |
| return mlir::Value{}; | |
| case NVPTX::BI__nvvm_mapa_shared_cluster: | |
| cgm.errorNYI(expr->getSourceRange(), | |
| std::string("unimplemented NVPTX builtin call: ") + | |
| getContext().BuiltinInfo.getName(builtinId)); | |
| return mlir::Value{}; | |
| case NVPTX::BI__nvvm_getctarank: | |
| cgm.errorNYI(expr->getSourceRange(), | |
| std::string("unimplemented NVPTX builtin call: ") + | |
| getContext().BuiltinInfo.getName(builtinId)); | |
| return mlir::Value{}; | |
| case NVPTX::BI__nvvm_getctarank_shared_cluster: | |
| cgm.errorNYI(expr->getSourceRange(), | |
| std::string("unimplemented NVPTX builtin call: ") + | |
| getContext().BuiltinInfo.getName(builtinId)); | |
| return mlir::Value{}; | |
| case NVPTX::BI__nvvm_barrier_cluster_arrive: | |
| return builder.emitIntrinsicCallOp(getLoc(expr->getExprLoc()), | |
| "nvvm.barrier.cluster.arrive", | |
| builder.getVoidTy()); | |
| case NVPTX::BI__nvvm_barrier_cluster_arrive_relaxed: | |
| return builder.emitIntrinsicCallOp(getLoc(expr->getExprLoc()), | |
| "nvvm.barrier.cluster.arrive.relaxed", | |
| builder.getVoidTy()); | |
| case NVPTX::BI__nvvm_barrier_cluster_wait: | |
| return builder.emitIntrinsicCallOp(getLoc(expr->getExprLoc()), | |
| "nvvm.barrier.cluster.wait", | |
| builder.getVoidTy()); | |
| case NVPTX::BI__nvvm_fence_sc_cluster: | |
| return builder.emitIntrinsicCallOp(getLoc(expr->getExprLoc()), | |
| "nvvm.fence.sc.cluster", | |
| builder.getVoidTy()); | |
| case NVPTX::BI__nvvm_bar_sync: | |
| return builder.emitIntrinsicCallOp( | |
| getLoc(expr->getExprLoc()), "nvvm.barrier.cta.sync.aligned.all", | |
| builder.getVoidTy(), mlir::ValueRange{emitScalarExpr(expr->getArg(0))}); | |
| case NVPTX::BI__syncthreads: | |
| return builder.emitIntrinsicCallOp( | |
| getLoc(expr->getExprLoc()), "nvvm.barrier.cta.sync.aligned.all", | |
| builder.getVoidTy(), | |
| mlir::ValueRange{builder.getConstInt(getLoc(expr->getExprLoc()), | |
| builder.getSInt32Ty(), 0)}); | |
| case NVPTX::BI__nvvm_barrier_sync: | |
| return builder.emitIntrinsicCallOp( | |
| getLoc(expr->getExprLoc()), "nvvm.barrier.cta.sync.all", | |
| builder.getVoidTy(), mlir::ValueRange{emitScalarExpr(expr->getArg(0))}); | |
| case NVPTX::BI__nvvm_barrier_sync_cnt: | |
| return builder.emitIntrinsicCallOp( | |
| getLoc(expr->getExprLoc()), "nvvm.barrier.cta.sync.count", | |
| builder.getVoidTy(), | |
| mlir::ValueRange{emitScalarExpr(expr->getArg(0)), | |
| emitScalarExpr(expr->getArg(1))}); | |
| case NVPTX::BI__nvvm_bar0_and: | |
| return emitBar0Reduction(*this, expr, | |
| "nvvm.barrier.cta.red.and.aligned.all", | |
| /*returnsPred=*/true); | |
| case NVPTX::BI__nvvm_bar0_or: | |
| return emitBar0Reduction(*this, expr, "nvvm.barrier.cta.red.or.aligned.all", | |
| /*returnsPred=*/true); | |
| case NVPTX::BI__nvvm_bar0_popc: | |
| return emitBar0Reduction(*this, expr, | |
| "nvvm.barrier.cta.red.popc.aligned.all", | |
| /*returnsPred=*/false); | |
| default: | |
| return std::nullopt; | |
| } | |
| } | |
| // vprintf takes two args: A format string, and a pointer to a buffer containing | |
| // the varargs. | |
| // | |
| // For example, the call | |
| // | |
| // printf("format string", arg1, arg2, arg3); | |
| // | |
| // is converted into something resembling | |
| // | |
| // struct Tmp { | |
| // Arg1 a1; | |
| // Arg2 a2; | |
| // Arg3 a3; | |
| // }; | |
| // char* buf = alloca(sizeof(Tmp)); | |
| // *(Tmp*)buf = {a1, a2, a3}; | |
| // vprintf("format string", buf); | |
| // | |
| // `buf` is aligned to the max of {alignof(Arg1), ...}. Furthermore, each of | |
| // the args is itself aligned to its preferred alignment. | |
| // | |
| // Note that by the time this function runs, the arguments have already | |
| // undergone the standard C vararg promotion (short -> int, float -> double | |
| // etc). In this function we pack the arguments into the buffer described above. | |
| static mlir::Value packArgsIntoNVPTXFormatBuffer(CIRGenFunction &cgf, | |
| const CallArgList &args, | |
| mlir::Location loc) { | |
| const cir::CIRDataLayout dataLayout = cgf.cgm.getDataLayout(); | |
| CIRGenBuilderTy &builder = cgf.getBuilder(); | |
| if (args.size() <= 1) | |
| // If there are no arguments other than the format string, | |
| // pass a nullptr to vprintf. | |
| return builder.getNullPtr(builder.getVoidPtrTy(), loc); | |
| llvm::SmallVector<mlir::Type> argTypes; | |
| for (const auto &arg : llvm::drop_begin(args)) | |
| argTypes.push_back(arg.getKnownRValue().getValue().getType()); | |
| // We can directly store the arguments into a struct, and the alignment | |
| // would automatically be correct. That's because vprintf does not | |
| // accept aggregates. | |
| mlir::Type allocaTy = builder.getAnonRecordTy( | |
| argTypes, /*packed=*/false, cir::RecordType::getAllDataKinds(argTypes)); | |
| auto allocaAlign = clang::CharUnits::fromQuantity( | |
| dataLayout.getABITypeAlign(allocaTy).value()); | |
| Address allocaAddr = | |
| cgf.createTempAlloca(allocaTy, allocaAlign, loc, "printf_args"); | |
| mlir::Value alloca = allocaAddr.getPointer(); | |
| for (auto [i, arg] : llvm::enumerate(llvm::drop_begin(args))) { | |
| mlir::Value member = builder.createGetMember( | |
| loc, cir::PointerType::get(argTypes[i]), alloca, /*name=*/"", | |
| /*index=*/i); | |
| auto abiAlign = clang::CharUnits::fromQuantity( | |
| dataLayout.getABITypeAlign(argTypes[i]).value()); | |
| cir::StoreOp::create(builder, loc, arg.getKnownRValue().getValue(), member, | |
| /*is_volatile=*/false, | |
| /*isNontemporal=*/false, | |
| builder.getAlignmentAttr(abiAlign), | |
| /*sync_scope=*/cir::SyncScopeKindAttr{}, | |
| /*mem_order=*/cir::MemOrderAttr{}); | |
| } | |
| return builder.createBitcast(alloca, builder.getVoidPtrTy()); | |
| } | |
| mlir::Value | |
| CIRGenFunction::emitNVPTXDevicePrintfCallExpr(const CallExpr *expr) { | |
| assert(cgm.getTriple().isNVPTX()); | |
| assert(expr->getBuiltinCallee() == Builtin::BIprintf || | |
| expr->getBuiltinCallee() == Builtin::BI__builtin_printf); | |
| assert(expr->getNumArgs() >= 1); // printf always has at least one arg. | |
| CallArgList args; | |
| emitCallArgs(args, | |
| expr->getDirectCallee()->getType()->getAs<FunctionProtoType>(), | |
| expr->arguments(), expr->getDirectCallee()); | |
| mlir::Location loc = getLoc(expr->getBeginLoc()); | |
| // We don't know how to emit non-scalar varargs. | |
| bool hasNonScalar = | |
| llvm::any_of(llvm::drop_begin(args), [&](const CallArg &a) { | |
| return a.hasLValue() || !a.getKnownRValue().isScalar(); | |
| }); | |
| if (hasNonScalar) { | |
| cgm.errorUnsupported(expr, "non-scalar args to printf"); | |
| return builder.getConstInt(loc, builder.getSInt32Ty(), 0); | |
| } | |
| mlir::Value packedData = packArgsIntoNVPTXFormatBuffer(*this, args, loc); | |
| // int vprintf(char *format, void *packedData); | |
| auto vprintf = cgm.createRuntimeFunction( | |
| cir::FuncType::get( | |
| {cir::PointerType::get(builder.getSInt8Ty()), builder.getVoidPtrTy()}, | |
| builder.getSInt32Ty()), | |
| "vprintf"); | |
| auto formatString = args[0].getKnownRValue().getValue(); | |
| return builder | |
| .createCallOp(loc, vprintf, mlir::ValueRange{formatString, packedData}) | |
| .getResult(); | |
| } |