| //===-- VerifierAMDGPU.cpp - AMDGPU-specific IR verification ---------------==// | |
| // | |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |
| // See https://llvm.org/LICENSE.txt for license information. | |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |
| // | |
| //===----------------------------------------------------------------------===// | |
| // | |
| // This file contains AMDGPU-specific IR verification logic that was extracted | |
| // from Verifier.cpp for code organization purposes only. These checks are | |
| // always compiled and linked as part of LLVMCore — this is not a target- | |
| // dependent IR verifier, which would require a different design. | |
| // | |
| // This file should only contain checks for AMDGPU-specific IR constructs | |
| // (e.g. amdgcn intrinsics, AMDGPU address spaces). It must not contain | |
| // checks for generic IR that might behave differently under AMDGPU. | |
| // | |
| //===----------------------------------------------------------------------===// | |
| #include "VerifierInternal.h" | |
| #include "llvm/ADT/StringExtras.h" | |
| #include "llvm/IR/CallingConv.h" | |
| #include "llvm/IR/Constants.h" | |
| #include "llvm/IR/DerivedTypes.h" | |
| #include "llvm/IR/Function.h" | |
| #include "llvm/IR/GlobalVariable.h" | |
| #include "llvm/IR/IntrinsicInst.h" | |
| #include "llvm/IR/IntrinsicsAMDGPU.h" | |
| #include "llvm/Support/AMDGPUAddrSpace.h" | |
| using namespace llvm; | |
| #define Check(C, ...) \ | |
| do { \ | |
| if (!(C)) { \ | |
| VS.CheckFailed(__VA_ARGS__); \ | |
| return; \ | |
| } \ | |
| } while (false) | |
| void llvm::verifyAMDGPUModuleFlag(VerifierSupport &VS, const MDString *ID, | |
| Module::ModFlagBehavior MFB, | |
| const MDNode *Op) { | |
| StringRef FlagName = ID->getString(); | |
| if (!FlagName.consume_front("amdgpu.")) | |
| return; | |
| if (FlagName == "buffer.oob.mode" || FlagName == "tbuffer.oob.mode") { | |
| Check(MFB == Module::Max, | |
| "'" + ID->getString() + | |
| "' module flag must use 'max' merge behaviour"); | |
| ConstantInt *Value = | |
| mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(2)); | |
| Check(Value, "'" + ID->getString() + | |
| "' module flag must have a constant integer value"); | |
| Check(Value->getZExtValue() <= 2, | |
| "'" + ID->getString() + "' module flag must be 0, 1, or 2"); | |
| return; | |
| } | |
| if (FlagName == "xnack" || FlagName == "sramecc") { | |
| Check(MFB == Module::Error, | |
| "'" + ID->getString() + | |
| "' module flag must use 'error' merge behaviour"); | |
| ConstantInt *Value = | |
| mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(2)); | |
| Check(Value, "'" + ID->getString() + | |
| "' module flag must have a constant integer value"); | |
| Check(Value->getZExtValue() <= 1, | |
| "'" + ID->getString() + "' module flag must be 0 or 1"); | |
| return; | |
| } | |
| } | |
| // Verify that when a function has !reqd_work_group_size metadata, it also has | |
| // an amdgpu-flat-work-group-size attribute that matches the product of the | |
| // reqd_work_group_size operands. | |
| static void verifyAMDGPUReqdWorkGroupSize(VerifierSupport &VS, | |
| const Function &F) { | |
| // This is not required for other targets so we only check for AMDGPU. | |
| if (!VS.TT.isAMDGPU()) | |
| return; | |
| MDNode *ReqdWorkGroupSize = F.getMetadata("reqd_work_group_size"); | |
| if (!ReqdWorkGroupSize || ReqdWorkGroupSize->getNumOperands() != 3) | |
| return; | |
| uint64_t Product = 1; | |
| for (const MDOperand &Op : ReqdWorkGroupSize->operands()) { | |
| ConstantInt *C = mdconst::dyn_extract<ConstantInt>(Op); | |
| if (!C || C->getValue().getActiveBits() > 64) | |
| return; | |
| uint64_t Dim = C->getZExtValue(); | |
| if (Dim != 0 && Product > std::numeric_limits<uint64_t>::max() / Dim) | |
| return; | |
| Product *= Dim; | |
| } | |
| Attribute FlatWorkGroupSize = F.getFnAttribute("amdgpu-flat-work-group-size"); | |
| if (!FlatWorkGroupSize.isValid()) { | |
| VS.CheckFailed("reqd_work_group_size requires amdgpu-flat-work-group-size", | |
| &F, ReqdWorkGroupSize); | |
| return; | |
| } | |
| if (!FlatWorkGroupSize.isStringAttribute()) { | |
| VS.CheckFailed("amdgpu-flat-work-group-size must be a string attribute", | |
| &F); | |
| return; | |
| } | |
| StringRef AttrValue = FlatWorkGroupSize.getValueAsString(); | |
| std::pair<StringRef, StringRef> Values = AttrValue.split(','); | |
| uint64_t Min = 0; | |
| uint64_t Max = 0; | |
| bool Parsed = !Values.second.contains(',') && | |
| llvm::to_integer(Values.first.trim(), Min) && | |
| llvm::to_integer(Values.second.trim(), Max); | |
| if (!Parsed) { | |
| VS.CheckFailed("amdgpu-flat-work-group-size must be a pair of unsigned " | |
| "integers", | |
| &F); | |
| return; | |
| } | |
| if (Min != Product || Max != Product) { | |
| VS.CheckFailed("amdgpu-flat-work-group-size must equal the product of " | |
| "reqd_work_group_size operands", | |
| &F, ReqdWorkGroupSize); | |
| } | |
| } | |
| void llvm::verifyAMDGPUFunctionMetadata(VerifierSupport &VS, | |
| const Function &F) { | |
| verifyAMDGPUReqdWorkGroupSize(VS, F); | |
| } | |
| void llvm::verifyAMDGPUGlobalVariable(VerifierSupport &VS, | |
| const GlobalVariable &GV) { | |
| // This is not required for other targets so we only check for AMDGPU. | |
| if (!VS.TT.isAMDGPU()) | |
| return; | |
| // The VGPR address space is a view of one wave's own vector registers, which | |
| // exist only while that wave runs. A global variable needs storage that | |
| // outlives any particular wave, so there is nothing here for it to name. | |
| if (GV.getAddressSpace() == AMDGPUAS::VGPR) | |
| VS.CheckFailed("global variable on amdgpu must not be in addrspace(13)", | |
| &GV); | |
| } | |
| void llvm::verifyAMDGPUAlloca(VerifierSupport &VS, const AllocaInst &AI) { | |
| // This is not required for other targets so we only check for AMDGPU. | |
| if (!VS.TT.isAMDGPU()) | |
| return; | |
| if (AI.getAddressSpace() != AMDGPUAS::PRIVATE_ADDRESS && | |
| AI.getAddressSpace() != AMDGPUAS::VGPR) | |
| VS.CheckFailed("alloca on amdgpu must be in addrspace(5) or addrspace(13)", | |
| &AI); | |
| // Only static allocas can live in VGPRs; a dynamically sized one has no | |
| // register-file representation. (Other address spaces are already rejected | |
| // above, so this only adds the more specific diagnostic for addrspace(13).) | |
| if (!AI.isStaticAlloca() && AI.getAddressSpace() == AMDGPUAS::VGPR) | |
| VS.CheckFailed("dynamic alloca on amdgpu must be in addrspace(5)", &AI); | |
| } | |
| bool llvm::isAMDGPUCallBrIntrinsic(Intrinsic::ID ID) { | |
| switch (ID) { | |
| default: | |
| return false; | |
| case Intrinsic::amdgcn_kill: | |
| return true; | |
| } | |
| } | |
| void llvm::verifyAMDGPUIntrinsicCall(VerifierSupport &VS, Intrinsic::ID ID, | |
| CallBase &Call) { | |
| switch (ID) { | |
| default: | |
| return; | |
| case Intrinsic::amdgcn_kill: { | |
| if (auto *CBI = dyn_cast<CallBrInst>(&Call)) { | |
| Check(CBI->getNumIndirectDests() == 1, | |
| "callbr amdgcn_kill only supports one indirect dest"); | |
| // We assume that amdgcn_unreachable is only introduced by | |
| // AMDGPUUnifyDivergentExitNodes, which replaces the block's original | |
| // unreachable terminator by a call to amdgcn_unreachable + a return. | |
| const Instruction *Term = CBI->getIndirectDest(0)->getTerminator(); | |
| const CallInst *CI = dyn_cast_if_present<CallInst>(Term->getPrevNode()); | |
| Check(isa<UnreachableInst>(Term) || | |
| (CI && CI->getIntrinsicID() == Intrinsic::amdgcn_unreachable), | |
| "callbr amdgcn_kill indirect dest needs to be unreachable"); | |
| } | |
| break; | |
| } | |
| case Intrinsic::amdgcn_cs_chain: { | |
| CallingConv::ID CallerCC = Call.getCaller()->getCallingConv(); | |
| switch (CallerCC) { | |
| case CallingConv::AMDGPU_CS: | |
| case CallingConv::AMDGPU_CS_Chain: | |
| case CallingConv::AMDGPU_CS_ChainPreserve: | |
| case CallingConv::AMDGPU_ES: | |
| case CallingConv::AMDGPU_GS: | |
| case CallingConv::AMDGPU_HS: | |
| case CallingConv::AMDGPU_LS: | |
| case CallingConv::AMDGPU_VS: | |
| break; | |
| default: | |
| VS.CheckFailed("Intrinsic cannot be called from functions with this " | |
| "calling convention", | |
| &Call); | |
| break; | |
| } | |
| Check(Call.paramHasAttr(2, Attribute::InReg), | |
| "SGPR arguments must have the `inreg` attribute", &Call); | |
| Check(!Call.paramHasAttr(3, Attribute::InReg), | |
| "VGPR arguments must not have the `inreg` attribute", &Call); | |
| ConstantInt *FlagsArg = cast<ConstantInt>(Call.getArgOperand(4)); | |
| Check(FlagsArg->getValue().ult(2), | |
| "flags must be 0 or 1 for llvm.amdgcn.cs.chain", &Call); | |
| Instruction *Next = Call.getNextNode(); | |
| bool IsAMDUnreachable = isa_and_nonnull<IntrinsicInst>(Next) && | |
| cast<IntrinsicInst>(Next)->getIntrinsicID() == | |
| Intrinsic::amdgcn_unreachable; | |
| Check(Next && (isa<UnreachableInst>(Next) || IsAMDUnreachable), | |
| "llvm.amdgcn.cs.chain must be followed by unreachable", &Call); | |
| break; | |
| } | |
| case Intrinsic::amdgcn_init_exec_from_input: { | |
| const Argument *Arg = dyn_cast<Argument>(Call.getOperand(0)); | |
| Check(Arg && Arg->hasInRegAttr(), | |
| "only inreg arguments to the parent function are valid as inputs to " | |
| "this intrinsic", | |
| &Call); | |
| break; | |
| } | |
| case Intrinsic::amdgcn_set_inactive_chain_arg: { | |
| CallingConv::ID CallerCC = Call.getCaller()->getCallingConv(); | |
| switch (CallerCC) { | |
| case CallingConv::AMDGPU_CS_Chain: | |
| case CallingConv::AMDGPU_CS_ChainPreserve: | |
| break; | |
| default: | |
| VS.CheckFailed("Intrinsic can only be used from functions with the " | |
| "amdgpu_cs_chain or amdgpu_cs_chain_preserve " | |
| "calling conventions", | |
| &Call); | |
| break; | |
| } | |
| unsigned InactiveIdx = 1; | |
| Check(!Call.paramHasAttr(InactiveIdx, Attribute::InReg), | |
| "Value for inactive lanes must not have the `inreg` attribute", | |
| &Call); | |
| Check(isa<Argument>(Call.getArgOperand(InactiveIdx)), | |
| "Value for inactive lanes must be a function argument", &Call); | |
| Check(!cast<Argument>(Call.getArgOperand(InactiveIdx))->hasInRegAttr(), | |
| "Value for inactive lanes must be a VGPR function argument", &Call); | |
| break; | |
| } | |
| case Intrinsic::amdgcn_call_whole_wave: { | |
| Function *F = dyn_cast<Function>(Call.getArgOperand(0)); | |
| Check(F, "Indirect whole wave calls are not allowed", &Call); | |
| CallingConv::ID CC = F->getCallingConv(); | |
| Check(CC == CallingConv::AMDGPU_Gfx_WholeWave, | |
| "Callee must have the amdgpu_gfx_whole_wave calling convention", | |
| &Call); | |
| Check(!F->isVarArg(), "Variadic whole wave calls are not allowed", &Call); | |
| Check(Call.arg_size() == F->arg_size(), | |
| "Call argument count must match callee argument count", &Call); | |
| Check(F->arg_begin()->getType()->isIntegerTy(1), | |
| "Callee must have i1 as its first argument", &Call); | |
| for (auto [CallArg, FuncArg] : | |
| drop_begin(zip_equal(Call.args(), F->args()))) { | |
| Check(CallArg->getType() == FuncArg.getType(), | |
| "Argument types must match", &Call); | |
| Check(Call.paramHasAttr(FuncArg.getArgNo(), Attribute::InReg) == | |
| FuncArg.hasInRegAttr(), | |
| "Argument inreg attributes must match", &Call); | |
| } | |
| break; | |
| } | |
| case Intrinsic::amdgcn_s_prefetch_data: { | |
| Check( | |
| AMDGPU::isFlatGlobalAddrSpace( | |
| Call.getArgOperand(0)->getType()->getPointerAddressSpace()), | |
| "llvm.amdgcn.s.prefetch.data only supports global or constant memory"); | |
| break; | |
| } | |
| case Intrinsic::amdgcn_load_to_lds: | |
| case Intrinsic::amdgcn_load_async_to_lds: | |
| case Intrinsic::amdgcn_global_load_lds: | |
| case Intrinsic::amdgcn_global_load_async_lds: | |
| case Intrinsic::amdgcn_raw_buffer_load_lds: | |
| case Intrinsic::amdgcn_raw_buffer_load_async_lds: | |
| case Intrinsic::amdgcn_raw_ptr_buffer_load_lds: | |
| case Intrinsic::amdgcn_raw_ptr_buffer_load_async_lds: | |
| case Intrinsic::amdgcn_struct_buffer_load_lds: | |
| case Intrinsic::amdgcn_struct_buffer_load_async_lds: | |
| case Intrinsic::amdgcn_struct_ptr_buffer_load_lds: | |
| case Intrinsic::amdgcn_struct_ptr_buffer_load_async_lds: { | |
| uint64_t Size = cast<ConstantInt>(Call.getArgOperand(2))->getZExtValue(); | |
| Check(Size == 1 || Size == 2 || Size == 4 || Size == 12 || Size == 16, | |
| "invalid data size for load-to-LDS intrinsic; must be 1, 2, 4, 12, " | |
| "or 16", | |
| &Call); | |
| break; | |
| } | |
| case Intrinsic::amdgcn_mfma_scale_f32_16x16x128_f8f6f4: | |
| case Intrinsic::amdgcn_mfma_scale_f32_32x32x64_f8f6f4: { | |
| Value *Src0 = Call.getArgOperand(0); | |
| Value *Src1 = Call.getArgOperand(1); | |
| uint64_t CBSZ = cast<ConstantInt>(Call.getArgOperand(3))->getZExtValue(); | |
| uint64_t BLGP = cast<ConstantInt>(Call.getArgOperand(4))->getZExtValue(); | |
| Check(CBSZ <= 4, "invalid value for cbsz format", Call, | |
| Call.getArgOperand(3)); | |
| Check(BLGP <= 4, "invalid value for blgp format", Call, | |
| Call.getArgOperand(4)); | |
| auto GetFormatNumRegs = [](unsigned FormatVal) { | |
| switch (FormatVal) { | |
| case 0: | |
| case 1: | |
| return 8u; | |
| case 2: | |
| case 3: | |
| return 6u; | |
| case 4: | |
| return 4u; | |
| default: | |
| llvm_unreachable("invalid format value"); | |
| } | |
| }; | |
| auto IsValidSrcASrcBVector = [](FixedVectorType *Ty) { | |
| if (!Ty || !Ty->getElementType()->isIntegerTy(32)) | |
| return false; | |
| unsigned NumElts = Ty->getNumElements(); | |
| return NumElts == 4 || NumElts == 6 || NumElts == 8; | |
| }; | |
| FixedVectorType *Src0Ty = dyn_cast<FixedVectorType>(Src0->getType()); | |
| FixedVectorType *Src1Ty = dyn_cast<FixedVectorType>(Src1->getType()); | |
| Check(IsValidSrcASrcBVector(Src0Ty), | |
| "operand 0 must be 4, 6 or 8 element i32 vector", &Call, Src0); | |
| Check(IsValidSrcASrcBVector(Src1Ty), | |
| "operand 1 must be 4, 6 or 8 element i32 vector", &Call, Src1); | |
| Check(Src0Ty->getNumElements() >= GetFormatNumRegs(CBSZ), | |
| "invalid vector type for format", &Call, Src0, Call.getArgOperand(3)); | |
| Check(Src1Ty->getNumElements() >= GetFormatNumRegs(BLGP), | |
| "invalid vector type for format", &Call, Src1, Call.getArgOperand(5)); | |
| break; | |
| } | |
| case Intrinsic::amdgcn_wmma_f32_16x16x128_f8f6f4: | |
| case Intrinsic::amdgcn_wmma_scale_f32_16x16x128_f8f6f4: | |
| case Intrinsic::amdgcn_wmma_scale16_f32_16x16x128_f8f6f4: { | |
| Value *Src0 = Call.getArgOperand(1); | |
| Value *Src1 = Call.getArgOperand(3); | |
| unsigned FmtA = cast<ConstantInt>(Call.getArgOperand(0))->getZExtValue(); | |
| unsigned FmtB = cast<ConstantInt>(Call.getArgOperand(2))->getZExtValue(); | |
| Check(FmtA <= 4, "invalid value for matrix format", Call, | |
| Call.getArgOperand(0)); | |
| Check(FmtB <= 4, "invalid value for matrix format", Call, | |
| Call.getArgOperand(2)); | |
| auto GetFormatNumRegs = [](unsigned FormatVal) { | |
| switch (FormatVal) { | |
| case 0: | |
| case 1: | |
| return 16u; | |
| case 2: | |
| case 3: | |
| return 12u; | |
| case 4: | |
| return 8u; | |
| default: | |
| llvm_unreachable("invalid format value"); | |
| } | |
| }; | |
| auto IsValidSrcASrcBVector = [](FixedVectorType *Ty) { | |
| if (!Ty || !Ty->getElementType()->isIntegerTy(32)) | |
| return false; | |
| unsigned NumElts = Ty->getNumElements(); | |
| return NumElts == 16 || NumElts == 12 || NumElts == 8; | |
| }; | |
| FixedVectorType *Src0Ty = dyn_cast<FixedVectorType>(Src0->getType()); | |
| FixedVectorType *Src1Ty = dyn_cast<FixedVectorType>(Src1->getType()); | |
| Check(IsValidSrcASrcBVector(Src0Ty), | |
| "operand 1 must be 8, 12 or 16 element i32 vector", &Call, Src0); | |
| Check(IsValidSrcASrcBVector(Src1Ty), | |
| "operand 3 must be 8, 12 or 16 element i32 vector", &Call, Src1); | |
| Check(Src0Ty->getNumElements() >= GetFormatNumRegs(FmtA), | |
| "invalid vector type for format", &Call, Src0, Call.getArgOperand(0)); | |
| Check(Src1Ty->getNumElements() >= GetFormatNumRegs(FmtB), | |
| "invalid vector type for format", &Call, Src1, Call.getArgOperand(2)); | |
| break; | |
| } | |
| case Intrinsic::amdgcn_cooperative_atomic_load_32x4B: | |
| case Intrinsic::amdgcn_cooperative_atomic_load_16x8B: | |
| case Intrinsic::amdgcn_cooperative_atomic_load_8x16B: | |
| case Intrinsic::amdgcn_cooperative_atomic_store_32x4B: | |
| case Intrinsic::amdgcn_cooperative_atomic_store_16x8B: | |
| case Intrinsic::amdgcn_cooperative_atomic_store_8x16B: { | |
| Value *PtrArg = Call.getArgOperand(0); | |
| const unsigned AS = PtrArg->getType()->getPointerAddressSpace(); | |
| Check(AS == AMDGPUAS::FLAT_ADDRESS || AS == AMDGPUAS::GLOBAL_ADDRESS, | |
| "cooperative atomic intrinsics require a generic or global pointer", | |
| &Call, PtrArg); | |
| MetadataAsValue *Op = | |
| cast<MetadataAsValue>(Call.getArgOperand(Call.arg_size() - 1)); | |
| MDNode *MD = cast<MDNode>(Op->getMetadata()); | |
| Check((MD->getNumOperands() == 1) && isa<MDString>(MD->getOperand(0)), | |
| "cooperative atomic intrinsics require that the last argument is a " | |
| "metadata string", | |
| &Call, Op); | |
| break; | |
| } | |
| case Intrinsic::amdgcn_av_load_b128: | |
| case Intrinsic::amdgcn_av_store_b128: { | |
| MetadataAsValue *Op = | |
| cast<MetadataAsValue>(Call.getArgOperand(Call.arg_size() - 1)); | |
| MDNode *MD = dyn_cast<MDNode>(Op->getMetadata()); | |
| Check(MD && (MD->getNumOperands() == 1) && isa<MDString>(MD->getOperand(0)), | |
| "the last argument to av load/store intrinsics must be a " | |
| "metadata string", | |
| &Call, Op); | |
| break; | |
| } | |
| } | |
| } | |
| #undef Check |