| //===--- SPIRVCallLowering.cpp - Call lowering ------------------*- C++ -*-===// | |
| // | |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |
| // See https://llvm.org/LICENSE.txt for license information. | |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |
| // | |
| //===----------------------------------------------------------------------===// | |
| // | |
| // This file implements the lowering of LLVM calls to machine code calls for | |
| // GlobalISel. | |
| // | |
| //===----------------------------------------------------------------------===// | |
| #include "SPIRVCallLowering.h" | |
| #include "MCTargetDesc/SPIRVBaseInfo.h" | |
| #include "SPIRV.h" | |
| #include "SPIRVBuiltins.h" | |
| #include "SPIRVGlobalRegistry.h" | |
| #include "SPIRVISelLowering.h" | |
| #include "SPIRVMetadata.h" | |
| #include "SPIRVRegisterInfo.h" | |
| #include "SPIRVSubtarget.h" | |
| #include "SPIRVUtils.h" | |
| #include "llvm/ADT/STLExtras.h" | |
| #include "llvm/CodeGen/FunctionLoweringInfo.h" | |
| #include "llvm/IR/IntrinsicInst.h" | |
| #include "llvm/IR/IntrinsicsSPIRV.h" | |
| #include "llvm/Support/ModRef.h" | |
| using namespace llvm; | |
| SPIRVCallLowering::SPIRVCallLowering(const SPIRVTargetLowering &TLI, | |
| SPIRVGlobalRegistry *GR) | |
| : CallLowering(&TLI), GR(GR) {} | |
| bool SPIRVCallLowering::lowerReturn(MachineIRBuilder &MIRBuilder, | |
| const Value *Val, ArrayRef<Register> VRegs, | |
| FunctionLoweringInfo &FLI, | |
| Register SwiftErrorVReg) const { | |
| // Ignore if called from the internal service function | |
| if (MIRBuilder.getMF() | |
| .getFunction() | |
| .getFnAttribute(SPIRV_BACKEND_SERVICE_FUN_NAME) | |
| .isValid()) | |
| return true; | |
| // Currently all return types should use a single register. | |
| // TODO: handle the case of multiple registers. | |
| if (VRegs.size() > 1) | |
| return false; | |
| if (Val) { | |
| const auto &STI = MIRBuilder.getMF().getSubtarget(); | |
| MIRBuilder.buildInstr(SPIRV::OpReturnValue) | |
| .addUse(VRegs[0]) | |
| .constrainAllUses(MIRBuilder.getTII(), *STI.getRegisterInfo(), | |
| *STI.getRegBankInfo()); | |
| return true; | |
| } | |
| MIRBuilder.buildInstr(SPIRV::OpReturn); | |
| return true; | |
| } | |
| // Based on the LLVM function attributes, get a SPIR-V FunctionControl. | |
| static uint32_t getFunctionControl(const Function &F, | |
| const SPIRVSubtarget *ST) { | |
| MemoryEffects MemEffects = F.getMemoryEffects(); | |
| uint32_t FuncControl = static_cast<uint32_t>(SPIRV::FunctionControl::None); | |
| if (F.hasFnAttribute(Attribute::AttrKind::NoInline)) | |
| FuncControl |= static_cast<uint32_t>(SPIRV::FunctionControl::DontInline); | |
| else if (F.hasFnAttribute(Attribute::AttrKind::AlwaysInline)) | |
| FuncControl |= static_cast<uint32_t>(SPIRV::FunctionControl::Inline); | |
| if (MemEffects.doesNotAccessMemory()) | |
| FuncControl |= static_cast<uint32_t>(SPIRV::FunctionControl::Pure); | |
| else if (MemEffects.onlyReadsMemory()) | |
| FuncControl |= static_cast<uint32_t>(SPIRV::FunctionControl::Const); | |
| if (ST->canUseExtension(SPIRV::Extension::SPV_INTEL_optnone) || | |
| ST->canUseExtension(SPIRV::Extension::SPV_EXT_optnone)) | |
| if (F.hasFnAttribute(Attribute::OptimizeNone)) | |
| FuncControl |= static_cast<uint32_t>(SPIRV::FunctionControl::OptNoneEXT); | |
| return FuncControl; | |
| } | |
| // If the function has pointer arguments, we are forced to re-create this | |
| // function type from the very beginning, changing PointerType by | |
| // TypedPointerType for each pointer argument. Otherwise, the same `Type*` | |
| // potentially corresponds to different SPIR-V function type, effectively | |
| // invalidating logic behind global registry and duplicates tracker. | |
| static FunctionType * | |
| fixFunctionTypeIfPtrArgs(SPIRVGlobalRegistry *GR, const Function &F, | |
| FunctionType *FTy, SPIRVTypeInst SRetTy, | |
| const SmallVector<SPIRVTypeInst, 4> &SArgTys) { | |
| bool hasArgPtrs = any_of(F.args(), [](const Argument &Arg) { | |
| // check if it's an instance of a non-typed PointerType | |
| return Arg.getType()->isPointerTy(); | |
| }); | |
| if (!hasArgPtrs) { | |
| Type *RetTy = FTy->getReturnType(); | |
| // check if it's an instance of a non-typed PointerType | |
| if (!RetTy->isPointerTy()) | |
| return FTy; | |
| } | |
| // re-create function type, using TypedPointerType instead of PointerType to | |
| // properly trace argument types | |
| const Type *RetTy = GR->getTypeForSPIRVType(SRetTy); | |
| SmallVector<Type *, 4> ArgTys; | |
| for (auto SArgTy : SArgTys) | |
| ArgTys.push_back(const_cast<Type *>(GR->getTypeForSPIRVType(SArgTy))); | |
| return FunctionType::get(const_cast<Type *>(RetTy), ArgTys, false); | |
| } | |
| static SPIRV::AccessQualifier::AccessQualifier | |
| getArgAccessQual(const Function &F, unsigned ArgIdx) { | |
| if (F.getCallingConv() != CallingConv::SPIR_KERNEL) | |
| return SPIRV::AccessQualifier::ReadWrite; | |
| MDString *ArgAttribute = getOCLKernelArgAccessQual(F, ArgIdx); | |
| if (!ArgAttribute) | |
| return SPIRV::AccessQualifier::ReadWrite; | |
| if (ArgAttribute->getString() == "read_only") | |
| return SPIRV::AccessQualifier::ReadOnly; | |
| if (ArgAttribute->getString() == "write_only") | |
| return SPIRV::AccessQualifier::WriteOnly; | |
| return SPIRV::AccessQualifier::ReadWrite; | |
| } | |
| static SPIRVTypeInst getArgSPIRVType(const Function &F, unsigned ArgIdx, | |
| SPIRVGlobalRegistry *GR, | |
| MachineIRBuilder &MIRBuilder, | |
| const SPIRVSubtarget &ST) { | |
| // Read argument's access qualifier from metadata or default. | |
| SPIRV::AccessQualifier::AccessQualifier ArgAccessQual = | |
| getArgAccessQual(F, ArgIdx); | |
| Type *OriginalArgType = | |
| SPIRV::getOriginalFunctionType(F)->getParamType(ArgIdx); | |
| // Vector of untyped pointers: build with the deduced pointee instead of | |
| // the default i8 (mismatches typed uses downstream). | |
| Argument *Arg = F.getArg(ArgIdx); | |
| if (auto *VTy = dyn_cast<FixedVectorType>(OriginalArgType); | |
| VTy && isUntypedPointerTy(VTy->getElementType())) | |
| if (Type *ElemTy = GR->findDeducedElementType(Arg)) | |
| return GR->getOrCreateSPIRVVectorType( | |
| GR->getOrCreateSPIRVPointerType( | |
| ElemTy, MIRBuilder, | |
| addressSpaceToStorageClass( | |
| getPointerAddressSpace(OriginalArgType), ST)), | |
| VTy->getNumElements(), MIRBuilder, true); | |
| // If OriginalArgType is non-pointer, use the OriginalArgType (the type cannot | |
| // be legally reassigned later). | |
| if (!isPointerTy(OriginalArgType)) | |
| return GR->getOrCreateSPIRVType(OriginalArgType, MIRBuilder, ArgAccessQual, | |
| true); | |
| Type *ArgType = Arg->getType(); | |
| // In case OriginalArgType is of untyped pointer type, there are three | |
| // possibilities: | |
| // 1) This is a pointer of an LLVM IR element type, passed byval/byref. | |
| // 2) This is an OpenCL/SPIR-V builtin type if there is spv_assign_type | |
| // intrinsic assigning a TargetExtType. | |
| // 3) This is a pointer, try to retrieve pointer element type from a | |
| // spv_assign_ptr_type intrinsic or otherwise use default pointer element | |
| // type. | |
| if (hasPointeeTypeAttr(Arg)) { | |
| // byval/byref/sret carry the aggregate layout in the pointee type, so keep | |
| // a typed pointer here. An untyped one drops the type and breaks the | |
| // argument ABI on the way back from SPIR-V. | |
| return GR->getOrCreateSPIRVTypedPointerType( | |
| getPointeeTypeByAttr(Arg), MIRBuilder, | |
| addressSpaceToStorageClass(getPointerAddressSpace(ArgType), ST)); | |
| } | |
| for (auto User : Arg->users()) { | |
| auto *II = dyn_cast<IntrinsicInst>(User); | |
| // Check if this is spv_assign_type assigning OpenCL/SPIR-V builtin type. | |
| if (II && II->getIntrinsicID() == Intrinsic::spv_assign_type) { | |
| MetadataAsValue *VMD = cast<MetadataAsValue>(II->getOperand(1)); | |
| Type *BuiltinType = | |
| cast<ConstantAsMetadata>(VMD->getMetadata())->getType(); | |
| assert(BuiltinType->isTargetExtTy() && "Expected TargetExtType"); | |
| return GR->getOrCreateSPIRVType(BuiltinType, MIRBuilder, ArgAccessQual, | |
| true); | |
| } | |
| // Check if this is spv_assign_ptr_type assigning pointer element type. | |
| if (!II || II->getIntrinsicID() != Intrinsic::spv_assign_ptr_type) | |
| continue; | |
| MetadataAsValue *VMD = cast<MetadataAsValue>(II->getOperand(1)); | |
| Type *ElementTy = | |
| toTypedPointer(cast<ConstantAsMetadata>(VMD->getMetadata())->getType()); | |
| return GR->getOrCreateSPIRVPointerType( | |
| ElementTy, MIRBuilder, | |
| addressSpaceToStorageClass( | |
| cast<ConstantInt>(II->getOperand(2))->getZExtValue(), ST)); | |
| } | |
| // Use the recorded pointee type for declarations. | |
| if (F.isDeclaration()) | |
| if (Type *ElemTy = GR->findDeducedElementType(Arg)) | |
| return GR->getOrCreateSPIRVPointerType( | |
| ElemTy, MIRBuilder, | |
| addressSpaceToStorageClass(getPointerAddressSpace(ArgType), ST)); | |
| // Replace PointerType with TypedPointerType to be able to map SPIR-V types to | |
| // LLVM types in a consistent manner | |
| return GR->getOrCreateSPIRVType(toTypedPointer(OriginalArgType), MIRBuilder, | |
| ArgAccessQual, true); | |
| } | |
| static SPIRV::ExecutionModel::ExecutionModel | |
| getExecutionModel(const SPIRVSubtarget &STI, const Function &F) { | |
| assert(STI.getEnv() != SPIRVSubtarget::Unknown && | |
| "Environment must be resolved before lowering entry points."); | |
| if (STI.isKernel()) | |
| return SPIRV::ExecutionModel::Kernel; | |
| auto attribute = F.getFnAttribute("hlsl.shader"); | |
| if (!attribute.isValid()) { | |
| report_fatal_error( | |
| "This entry point lacks mandatory hlsl.shader attribute."); | |
| } | |
| const auto value = attribute.getValueAsString(); | |
| if (value == "compute") | |
| return SPIRV::ExecutionModel::GLCompute; | |
| if (value == "vertex") | |
| return SPIRV::ExecutionModel::Vertex; | |
| if (value == "pixel") | |
| return SPIRV::ExecutionModel::Fragment; | |
| report_fatal_error("This HLSL entry point is not supported by this backend."); | |
| } | |
| bool SPIRVCallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder, | |
| const Function &F, | |
| ArrayRef<ArrayRef<Register>> VRegs, | |
| FunctionLoweringInfo &FLI) const { | |
| // Discard the internal service function | |
| if (F.getFnAttribute(SPIRV_BACKEND_SERVICE_FUN_NAME).isValid()) | |
| return true; | |
| assert(GR && "Must initialize the SPIRV type registry before lowering args."); | |
| GR->setCurrentFunc(MIRBuilder.getMF()); | |
| // Get access to information about available extensions | |
| const SPIRVSubtarget *ST = | |
| static_cast<const SPIRVSubtarget *>(&MIRBuilder.getMF().getSubtarget()); | |
| // Assign types and names to all args, and store their types for later. | |
| SmallVector<SPIRVTypeInst, 4> ArgTypeVRegs; | |
| if (VRegs.size() > 0) { | |
| unsigned i = 0; | |
| for (const auto &Arg : F.args()) { | |
| // Currently formal args should use single registers. | |
| // TODO: handle the case of multiple registers. | |
| if (VRegs[i].size() > 1) | |
| return false; | |
| SPIRVTypeInst SpirvTy = getArgSPIRVType(F, i, GR, MIRBuilder, *ST); | |
| GR->assignSPIRVTypeToVReg(SpirvTy, VRegs[i][0], MIRBuilder.getMF()); | |
| ArgTypeVRegs.push_back(SpirvTy); | |
| if (Arg.hasName()) | |
| buildOpName(VRegs[i][0], Arg.getName(), MIRBuilder); | |
| if (isPointerTyOrWrapper(Arg.getType()) && !ST->isShader()) { | |
| auto DerefBytes = static_cast<unsigned>(Arg.getDereferenceableBytes()); | |
| if (DerefBytes != 0 && ST->isAtLeastSPIRVVer(VersionTuple(1, 1))) | |
| buildOpDecorate(VRegs[i][0], MIRBuilder, | |
| SPIRV::Decoration::MaxByteOffset, {DerefBytes}); | |
| } | |
| if (Arg.hasAttribute(Attribute::Alignment) && !ST->isShader()) { | |
| auto Alignment = static_cast<unsigned>( | |
| Arg.getAttribute(Attribute::Alignment).getValueAsInt()); | |
| buildOpDecorate(VRegs[i][0], MIRBuilder, SPIRV::Decoration::Alignment, | |
| {Alignment}); | |
| } | |
| if (ST->isKernel()) { | |
| if (Arg.hasAttribute(Attribute::ReadOnly)) { | |
| auto Attr = | |
| static_cast<unsigned>(SPIRV::FunctionParameterAttribute::NoWrite); | |
| buildOpDecorate(VRegs[i][0], MIRBuilder, | |
| SPIRV::Decoration::FuncParamAttr, {Attr}); | |
| } | |
| if (Arg.hasAttribute(Attribute::ReadNone)) { | |
| auto Attr = static_cast<unsigned>( | |
| SPIRV::FunctionParameterAttribute::NoReadWrite); | |
| buildOpDecorate(VRegs[i][0], MIRBuilder, | |
| SPIRV::Decoration::FuncParamAttr, {Attr}); | |
| } | |
| if (Arg.hasAttribute(Attribute::ZExt)) { | |
| auto Attr = | |
| static_cast<unsigned>(SPIRV::FunctionParameterAttribute::Zext); | |
| buildOpDecorate(VRegs[i][0], MIRBuilder, | |
| SPIRV::Decoration::FuncParamAttr, {Attr}); | |
| } | |
| if (Arg.hasAttribute(Attribute::SExt)) { | |
| auto Attr = | |
| static_cast<unsigned>(SPIRV::FunctionParameterAttribute::Sext); | |
| buildOpDecorate(VRegs[i][0], MIRBuilder, | |
| SPIRV::Decoration::FuncParamAttr, {Attr}); | |
| } | |
| if (Arg.hasAttribute(Attribute::NoAlias)) { | |
| auto Attr = | |
| static_cast<unsigned>(SPIRV::FunctionParameterAttribute::NoAlias); | |
| buildOpDecorate(VRegs[i][0], MIRBuilder, | |
| SPIRV::Decoration::FuncParamAttr, {Attr}); | |
| } | |
| if (Arg.hasNoCaptureAttr()) { | |
| auto Attr = static_cast<unsigned>( | |
| SPIRV::FunctionParameterAttribute::NoCapture); | |
| buildOpDecorate(VRegs[i][0], MIRBuilder, | |
| SPIRV::Decoration::FuncParamAttr, {Attr}); | |
| } | |
| // TODO: the AMDGPU BE only supports ByRef argument passing, thus for | |
| // AMDGCN flavoured SPIRV we CodeGen for ByRef, but lower it to | |
| // ByVal, handling the impedance mismatch during reverse | |
| // translation from SPIRV to LLVM IR; the vendor check should be | |
| // removed once / if SPIRV adds ByRef support. | |
| if (Arg.hasAttribute(Attribute::ByVal) || | |
| (Arg.hasAttribute(Attribute::ByRef) && | |
| F.getParent()->getTargetTriple().getVendor() == | |
| Triple::VendorType::AMD)) { | |
| auto Attr = | |
| static_cast<unsigned>(SPIRV::FunctionParameterAttribute::ByVal); | |
| buildOpDecorate(VRegs[i][0], MIRBuilder, | |
| SPIRV::Decoration::FuncParamAttr, {Attr}); | |
| } | |
| if (Arg.hasAttribute(Attribute::StructRet)) { | |
| auto Attr = | |
| static_cast<unsigned>(SPIRV::FunctionParameterAttribute::Sret); | |
| buildOpDecorate(VRegs[i][0], MIRBuilder, | |
| SPIRV::Decoration::FuncParamAttr, {Attr}); | |
| } | |
| } | |
| MDNode *Node = F.getMetadata("spirv.ParameterDecorations"); | |
| if (Node && i < Node->getNumOperands() && | |
| isa<MDNode>(Node->getOperand(i))) { | |
| MDNode *MD = cast<MDNode>(Node->getOperand(i)); | |
| for (const MDOperand &MDOp : MD->operands()) { | |
| MDNode *MD2 = dyn_cast<MDNode>(MDOp); | |
| assert(MD2 && "Metadata operand is expected"); | |
| ConstantInt *Const = getMDOperandAsConstInt(MD2, 0); | |
| assert(Const && "MDOperand should be ConstantInt"); | |
| auto Dec = | |
| static_cast<SPIRV::Decoration::Decoration>(Const->getZExtValue()); | |
| std::vector<uint32_t> DecVec; | |
| for (unsigned j = 1; j < MD2->getNumOperands(); j++) { | |
| ConstantInt *Const = getMDOperandAsConstInt(MD2, j); | |
| assert(Const && "MDOperand should be ConstantInt"); | |
| DecVec.push_back(static_cast<uint32_t>(Const->getZExtValue())); | |
| } | |
| buildOpDecorate(VRegs[i][0], MIRBuilder, Dec, DecVec); | |
| } | |
| } | |
| ++i; | |
| } | |
| } | |
| auto MRI = MIRBuilder.getMRI(); | |
| Register FuncVReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); | |
| MRI->setRegClass(FuncVReg, &SPIRV::iIDRegClass); | |
| FunctionType *FTy = SPIRV::getOriginalFunctionType(F); | |
| Type *FRetTy = FTy->getReturnType(); | |
| if (isUntypedPointerTy(FRetTy)) { | |
| if (Type *FRetElemTy = GR->findDeducedElementType(&F)) { | |
| TypedPointerType *DerivedTy = TypedPointerType::get( | |
| toTypedPointer(FRetElemTy), getPointerAddressSpace(FRetTy)); | |
| GR->addReturnType(&F, DerivedTy); | |
| FRetTy = DerivedTy; | |
| } | |
| } | |
| SPIRVTypeInst RetTy = GR->getOrCreateSPIRVType( | |
| FRetTy, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, true); | |
| FTy = fixFunctionTypeIfPtrArgs(GR, F, FTy, RetTy, ArgTypeVRegs); | |
| SPIRVTypeInst FuncTy = GR->getOrCreateOpTypeFunctionWithArgs( | |
| FTy, RetTy, ArgTypeVRegs, MIRBuilder); | |
| uint32_t FuncControl = getFunctionControl(F, ST); | |
| // Add OpFunction instruction | |
| MachineInstrBuilder MB = MIRBuilder.buildInstr(SPIRV::OpFunction) | |
| .addDef(FuncVReg) | |
| .addUse(GR->getSPIRVTypeID(RetTy)) | |
| .addImm(FuncControl) | |
| .addUse(GR->getSPIRVTypeID(FuncTy)); | |
| GR->recordFunctionDefinition(&F, &MB.getInstr()->getOperand(0)); | |
| GR->addGlobalObject(&F, &MIRBuilder.getMF(), FuncVReg); | |
| if (F.isDeclaration()) | |
| GR->add(&F, MB); | |
| // Add OpFunctionParameter instructions | |
| int i = 0; | |
| for (const auto &Arg : F.args()) { | |
| assert(VRegs[i].size() == 1 && "Formal arg has multiple vregs"); | |
| Register ArgReg = VRegs[i][0]; | |
| MRI->setRegClass(ArgReg, GR->getRegClass(ArgTypeVRegs[i])); | |
| auto MIB = MIRBuilder.buildInstr(SPIRV::OpFunctionParameter) | |
| .addDef(ArgReg) | |
| .addUse(GR->getSPIRVTypeID(ArgTypeVRegs[i])); | |
| if (F.isDeclaration()) | |
| GR->add(&Arg, MIB); | |
| GR->addGlobalObject(&Arg, &MIRBuilder.getMF(), ArgReg); | |
| i++; | |
| } | |
| if (!ST->isShader()) { | |
| if (F.hasRetAttribute(Attribute::ZExt)) { | |
| auto Attr = | |
| static_cast<unsigned>(SPIRV::FunctionParameterAttribute::Zext); | |
| buildOpDecorate(FuncVReg, MIRBuilder, SPIRV::Decoration::FuncParamAttr, | |
| {Attr}); | |
| } | |
| if (F.hasRetAttribute(Attribute::SExt)) { | |
| auto Attr = | |
| static_cast<unsigned>(SPIRV::FunctionParameterAttribute::Sext); | |
| buildOpDecorate(FuncVReg, MIRBuilder, SPIRV::Decoration::FuncParamAttr, | |
| {Attr}); | |
| } | |
| } | |
| // Name the function. | |
| if (F.hasName()) | |
| buildOpName(FuncVReg, F.getName(), MIRBuilder); | |
| // Handle entry points and function linkage. | |
| if (isEntryPoint(F)) { | |
| if (F.getName().empty()) | |
| report_fatal_error("SPIR-V entry point function must have a name"); | |
| auto MIB = MIRBuilder.buildInstr(SPIRV::OpEntryPoint) | |
| .addImm(static_cast<uint32_t>(getExecutionModel(*ST, F))) | |
| .addUse(FuncVReg); | |
| addStringImm(F.getName(), MIB); | |
| } else if (const auto LnkTy = getSpirvLinkageTypeFor(*ST, F); | |
| LnkTy && !F.getName().empty()) { | |
| buildOpDecorate(FuncVReg, MIRBuilder, SPIRV::Decoration::LinkageAttributes, | |
| {static_cast<uint32_t>(*LnkTy)}, F.getName()); | |
| } | |
| // Handle function pointers decoration | |
| bool hasFunctionPointers = | |
| ST->canUseExtension(SPIRV::Extension::SPV_INTEL_function_pointers); | |
| if (hasFunctionPointers) { | |
| if (F.hasFnAttribute("referenced-indirectly")) { | |
| assert((F.getCallingConv() != CallingConv::SPIR_KERNEL) && | |
| "Unexpected 'referenced-indirectly' attribute of the kernel " | |
| "function"); | |
| buildOpDecorate(FuncVReg, MIRBuilder, | |
| SPIRV::Decoration::ReferencedIndirectlyINTEL, {}); | |
| } | |
| } | |
| if (MDNode *FuncMD = F.getMetadata("spirv.Decorations")) | |
| buildOpSpirvDecorations(FuncVReg, MIRBuilder, FuncMD, *ST); | |
| return true; | |
| } | |
| // TODO: | |
| // - add a topological sort of IndirectCalls to ensure the best types knowledge | |
| // - we may need to fix function formal parameter types if they are opaque | |
| // pointers used as function pointers in these indirect calls | |
| // - defaulting to StorageClass::Function in the absence of the | |
| // SPV_INTEL_function_pointers extension seems wrong, as that might not be | |
| // able to hold a full width pointer to function, and it also does not model | |
| // the semantics of a pointer to function in a generic fashion. | |
| void SPIRVCallLowering::produceIndirectPtrType( | |
| MachineIRBuilder &MIRBuilder, | |
| const SPIRVCallLowering::SPIRVIndirectCall &IC) const { | |
| // Create indirect call data type if any | |
| MachineFunction &MF = MIRBuilder.getMF(); | |
| const SPIRVSubtarget &ST = MF.getSubtarget<SPIRVSubtarget>(); | |
| SPIRVTypeInst SpirvRetTy = GR->getOrCreateSPIRVType( | |
| IC.RetTy, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, true); | |
| SmallVector<SPIRVTypeInst, 4> SpirvArgTypes; | |
| for (size_t i = 0; i < IC.ArgTys.size(); ++i) { | |
| SPIRVTypeInst SPIRVTy = GR->getOrCreateSPIRVType( | |
| IC.ArgTys[i], MIRBuilder, SPIRV::AccessQualifier::ReadWrite, true); | |
| SpirvArgTypes.push_back(SPIRVTy); | |
| if (!GR->getSPIRVTypeForVReg(IC.ArgRegs[i])) | |
| GR->assignSPIRVTypeToVReg(SPIRVTy, IC.ArgRegs[i], MF); | |
| } | |
| // SPIR-V function type: | |
| FunctionType *FTy = | |
| FunctionType::get(const_cast<Type *>(IC.RetTy), IC.ArgTys, false); | |
| SPIRVTypeInst SpirvFuncTy = GR->getOrCreateOpTypeFunctionWithArgs( | |
| FTy, SpirvRetTy, SpirvArgTypes, MIRBuilder); | |
| // SPIR-V pointer to function type: | |
| auto SC = ST.canUseExtension(SPIRV::Extension::SPV_INTEL_function_pointers) | |
| ? SPIRV::StorageClass::CodeSectionINTEL | |
| : SPIRV::StorageClass::Function; | |
| SPIRVTypeInst IndirectFuncPtrTy = | |
| GR->getOrCreateSPIRVPointerType(SpirvFuncTy, MIRBuilder, SC); | |
| // Correct the Callee type | |
| GR->assignSPIRVTypeToVReg(IndirectFuncPtrTy, IC.Callee, MF); | |
| } | |
| bool SPIRVCallLowering::lowerCall(MachineIRBuilder &MIRBuilder, | |
| CallLoweringInfo &Info) const { | |
| // Currently call returns should have single vregs. | |
| // TODO: handle the case of multiple registers. | |
| if (Info.OrigRet.Regs.size() > 1) | |
| return false; | |
| MachineFunction &MF = MIRBuilder.getMF(); | |
| GR->setCurrentFunc(MF); | |
| const Function *CF = nullptr; | |
| std::string DemangledName; | |
| const Type *OrigRetTy = Info.OrigRet.Ty; | |
| // Emit a regular OpFunctionCall. If it's an externally declared function, | |
| // be sure to emit its type and function declaration here. It will be hoisted | |
| // globally later. | |
| if (Info.Callee.isGlobal()) { | |
| std::string FuncName = Info.Callee.getGlobal()->getName().str(); | |
| DemangledName = getOclOrSpirvBuiltinDemangledName(FuncName); | |
| CF = dyn_cast_or_null<const Function>(Info.Callee.getGlobal()); | |
| // TODO: support constexpr casts and indirect calls. | |
| if (CF == nullptr) | |
| return false; | |
| FunctionType *FTy = SPIRV::getOriginalFunctionType(*CF); | |
| OrigRetTy = FTy->getReturnType(); | |
| if (isUntypedPointerTy(OrigRetTy)) { | |
| if (auto *DerivedRetTy = GR->findReturnType(CF)) | |
| OrigRetTy = DerivedRetTy; | |
| } | |
| } | |
| MachineRegisterInfo *MRI = MIRBuilder.getMRI(); | |
| Register ResVReg = | |
| Info.OrigRet.Regs.empty() ? Register(0) : Info.OrigRet.Regs[0]; | |
| const auto *ST = static_cast<const SPIRVSubtarget *>(&MF.getSubtarget()); | |
| bool isFunctionDecl = CF && CF->isDeclaration(); | |
| if (isFunctionDecl && !DemangledName.empty()) { | |
| if (ResVReg.isValid()) { | |
| if (!GR->getSPIRVTypeForVReg(ResVReg)) { | |
| const Type *RetTy = OrigRetTy; | |
| if (auto *PtrRetTy = dyn_cast<PointerType>(OrigRetTy)) { | |
| const Value *OrigValue = Info.OrigRet.OrigValue; | |
| if (!OrigValue) | |
| OrigValue = Info.CB; | |
| if (OrigValue) | |
| if (Type *ElemTy = GR->findDeducedElementType(OrigValue)) | |
| RetTy = | |
| TypedPointerType::get(ElemTy, PtrRetTy->getAddressSpace()); | |
| } | |
| setRegClassType(ResVReg, RetTy, GR, MIRBuilder, | |
| SPIRV::AccessQualifier::ReadWrite, true); | |
| } | |
| } else { | |
| ResVReg = createVirtualRegister(OrigRetTy, GR, MIRBuilder, | |
| SPIRV::AccessQualifier::ReadWrite, true); | |
| } | |
| SmallVector<Register, 8> ArgVRegs; | |
| for (auto Arg : Info.OrigArgs) { | |
| assert(Arg.Regs.size() == 1 && "Call arg has multiple VRegs"); | |
| Register ArgReg = Arg.Regs[0]; | |
| ArgVRegs.push_back(ArgReg); | |
| SPIRVTypeInst SpvType = GR->getSPIRVTypeForVReg(ArgReg); | |
| if (!SpvType) { | |
| Type *ArgTy = nullptr; | |
| if (auto *PtrArgTy = dyn_cast<PointerType>(Arg.Ty)) { | |
| // If Arg.Ty is an untyped pointer (i.e., ptr [addrspace(...)]) and we | |
| // don't have access to original value in LLVM IR or info about | |
| // deduced pointee type, then we should wait with setting the type for | |
| // the virtual register until pre-legalizer step when we access | |
| // @llvm.spv.assign.ptr.type.p...(...)'s info. | |
| if (Arg.OrigValue) | |
| if (Type *ElemTy = GR->findDeducedElementType(Arg.OrigValue)) | |
| ArgTy = | |
| TypedPointerType::get(ElemTy, PtrArgTy->getAddressSpace()); | |
| } else { | |
| ArgTy = Arg.Ty; | |
| } | |
| if (ArgTy) { | |
| SpvType = GR->getOrCreateSPIRVType( | |
| ArgTy, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, true); | |
| GR->assignSPIRVTypeToVReg(SpvType, ArgReg, MF); | |
| } | |
| } | |
| if (!MRI->getRegClassOrNull(ArgReg)) { | |
| // Either we have SpvType created, or Arg.Ty is an untyped pointer and | |
| // we know its virtual register's class and type even if we don't know | |
| // pointee type. | |
| MRI->setRegClass(ArgReg, SpvType ? GR->getRegClass(SpvType) | |
| : &SPIRV::pIDRegClass); | |
| MRI->setType( | |
| ArgReg, | |
| SpvType ? GR->getRegType(SpvType) | |
| : LLT::pointer(cast<PointerType>(Arg.Ty)->getAddressSpace(), | |
| GR->getPointerSize())); | |
| } | |
| } | |
| if (auto Res = SPIRV::lowerBuiltin( | |
| DemangledName, ST->getPreferredInstructionSet(), MIRBuilder, | |
| ResVReg, OrigRetTy, ArgVRegs, GR, *Info.CB)) | |
| return *Res; | |
| } | |
| if (isFunctionDecl && !GR->find(CF, &MF).isValid()) { | |
| // Emit the type info and forward function declaration to the first MBB | |
| // to ensure VReg definition dependencies are valid across all MBBs. | |
| MachineIRBuilder FirstBlockBuilder; | |
| FirstBlockBuilder.setMF(MF); | |
| FirstBlockBuilder.setMBB(*MF.getBlockNumbered(0)); | |
| SmallVector<ArrayRef<Register>, 8> VRegArgs; | |
| SmallVector<SmallVector<Register, 1>, 8> ToInsert; | |
| for (const Argument &Arg : CF->args()) { | |
| if (MIRBuilder.getDataLayout().getTypeStoreSize(Arg.getType()).isZero()) | |
| continue; // Don't handle zero sized types. | |
| Register Reg = MRI->createGenericVirtualRegister(LLT::scalar(64)); | |
| MRI->setRegClass(Reg, &SPIRV::iIDRegClass); | |
| ToInsert.push_back({Reg}); | |
| VRegArgs.push_back(ToInsert.back()); | |
| } | |
| // TODO: Reuse FunctionLoweringInfo | |
| FunctionLoweringInfo FuncInfo; | |
| lowerFormalArguments(FirstBlockBuilder, *CF, VRegArgs, FuncInfo); | |
| } | |
| // Ignore the call if it's called from the internal service function | |
| if (MIRBuilder.getMF() | |
| .getFunction() | |
| .getFnAttribute(SPIRV_BACKEND_SERVICE_FUN_NAME) | |
| .isValid()) { | |
| // insert a no-op | |
| MIRBuilder.buildTrap(); | |
| return true; | |
| } | |
| unsigned CallOp; | |
| if (Info.CB && Info.CB->isIndirectCall()) { | |
| if (!ST->canUseExtension(SPIRV::Extension::SPV_INTEL_function_pointers)) | |
| report_fatal_error("An indirect call is encountered but SPIR-V without " | |
| "extensions does not support it", | |
| false); | |
| // Set instruction operation according to SPV_INTEL_function_pointers | |
| CallOp = SPIRV::OpFunctionPointerCallINTEL; | |
| // Collect information about the indirect call to create correct types. | |
| Register CalleeReg = Info.Callee.getReg(); | |
| if (CalleeReg.isValid()) { | |
| SPIRVCallLowering::SPIRVIndirectCall IndirectCall; | |
| IndirectCall.Callee = CalleeReg; | |
| FunctionType *FTy = SPIRV::getOriginalFunctionType(*Info.CB); | |
| IndirectCall.RetTy = OrigRetTy = FTy->getReturnType(); | |
| assert(FTy->getNumParams() == Info.OrigArgs.size() && | |
| "Function types mismatch"); | |
| for (unsigned I = 0; I != Info.OrigArgs.size(); ++I) { | |
| assert(Info.OrigArgs[I].Regs.size() == 1 && | |
| "Call arg has multiple VRegs"); | |
| IndirectCall.ArgTys.push_back(FTy->getParamType(I)); | |
| IndirectCall.ArgRegs.push_back(Info.OrigArgs[I].Regs[0]); | |
| } | |
| produceIndirectPtrType(MIRBuilder, IndirectCall); | |
| } | |
| } else { | |
| // Emit a regular OpFunctionCall | |
| CallOp = SPIRV::OpFunctionCall; | |
| } | |
| // Make sure there's a valid return reg, even for functions returning void. | |
| if (!ResVReg.isValid()) | |
| ResVReg = MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::iIDRegClass); | |
| SPIRVTypeInst RetType = GR->assignTypeToVReg( | |
| OrigRetTy, ResVReg, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, true); | |
| // Emit the call instruction and its args. | |
| auto MIB = MIRBuilder.buildInstr(CallOp) | |
| .addDef(ResVReg) | |
| .addUse(GR->getSPIRVTypeID(RetType)) | |
| .add(Info.Callee); | |
| if (Info.NoMerge) | |
| MIB.setMIFlag(MachineInstr::MIFlag::NoMerge); | |
| for (const auto &Arg : Info.OrigArgs) { | |
| // Currently call args should have single vregs. | |
| if (Arg.Regs.size() > 1) | |
| return false; | |
| MIB.addUse(Arg.Regs[0]); | |
| } | |
| if (Info.CB) | |
| MIB.getInstr()->copyIRFlags(*Info.CB); | |
| if (ST->canUseExtension(SPIRV::Extension::SPV_INTEL_memory_access_aliasing)) { | |
| // Process aliasing metadata. | |
| const CallBase *CI = Info.CB; | |
| if (CI && CI->hasMetadata()) { | |
| if (MDNode *MD = CI->getMetadata(LLVMContext::MD_alias_scope)) | |
| GR->buildMemAliasingOpDecorate(ResVReg, MIRBuilder, | |
| SPIRV::Decoration::AliasScopeINTEL, MD); | |
| if (MDNode *MD = CI->getMetadata(LLVMContext::MD_noalias)) | |
| GR->buildMemAliasingOpDecorate(ResVReg, MIRBuilder, | |
| SPIRV::Decoration::NoAliasINTEL, MD); | |
| } | |
| } | |
| MIB.constrainAllUses(MIRBuilder.getTII(), *ST->getRegisterInfo(), | |
| *ST->getRegBankInfo()); | |
| return true; | |
| } |