| //===- SPIRVInstructionSelector.cpp ------------------------------*- C++ -*-==// | |
| // | |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |
| // See https://llvm.org/LICENSE.txt for license information. | |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |
| // | |
| //===----------------------------------------------------------------------===// | |
| // | |
| // This file implements the targeting of the InstructionSelector class for | |
| // SPIRV. | |
| // TODO: This should be generated by TableGen. | |
| // | |
| //===----------------------------------------------------------------------===// | |
| #include "MCTargetDesc/SPIRVBaseInfo.h" | |
| #include "MCTargetDesc/SPIRVMCTargetDesc.h" | |
| #include "SPIRV.h" | |
| #include "SPIRVGlobalRegistry.h" | |
| #include "SPIRVInstrInfo.h" | |
| #include "SPIRVRegisterInfo.h" | |
| #include "SPIRVTargetMachine.h" | |
| #include "SPIRVTypeInst.h" | |
| #include "SPIRVUtils.h" | |
| #include "llvm/ADT/APFloat.h" | |
| #include "llvm/ADT/STLExtras.h" | |
| #include "llvm/ADT/STLFunctionalExtras.h" | |
| #include "llvm/ADT/SmallSet.h" | |
| #include "llvm/ADT/StringExtras.h" | |
| #include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h" | |
| #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" | |
| #include "llvm/CodeGen/GlobalISel/InstructionSelector.h" | |
| #include "llvm/CodeGen/MachineInstrBuilder.h" | |
| #include "llvm/CodeGen/MachineRegisterInfo.h" | |
| #include "llvm/CodeGen/Register.h" | |
| #include "llvm/CodeGen/TargetOpcodes.h" | |
| #include "llvm/IR/DiagnosticInfo.h" | |
| #include "llvm/IR/IntrinsicsSPIRV.h" | |
| #include "llvm/Support/Debug.h" | |
| #include "llvm/Support/ErrorHandling.h" | |
| #include <functional> | |
| #include <optional> | |
| #define DEBUG_TYPE "spirv-isel" | |
| using namespace llvm; | |
| namespace CL = SPIRV::OpenCLExtInst; | |
| namespace GL = SPIRV::GLSLExtInst; | |
| using ExtInstList = | |
| std::vector<std::pair<SPIRV::InstructionSet::InstructionSet, uint32_t>>; | |
| namespace { | |
| struct ImageOperands { | |
| std::optional<Register> Bias; | |
| std::optional<Register> Offset; | |
| std::optional<Register> MinLod; | |
| std::optional<Register> GradX; | |
| std::optional<Register> GradY; | |
| std::optional<Register> Lod; | |
| std::optional<Register> Compare; | |
| }; | |
| llvm::SPIRV::SelectionControl::SelectionControl | |
| getSelectionOperandForImm(int Imm) { | |
| if (Imm == 2) | |
| return SPIRV::SelectionControl::Flatten; | |
| if (Imm == 1) | |
| return SPIRV::SelectionControl::DontFlatten; | |
| if (Imm == 0) | |
| return SPIRV::SelectionControl::None; | |
| llvm_unreachable("Invalid immediate"); | |
| } | |
| #define GET_GLOBALISEL_PREDICATE_BITSET | |
| #include "SPIRVGenGlobalISel.inc" | |
| #undef GET_GLOBALISEL_PREDICATE_BITSET | |
| class SPIRVInstructionSelector : public InstructionSelector { | |
| const SPIRVSubtarget &STI; | |
| const SPIRVInstrInfo &TII; | |
| const SPIRVRegisterInfo &TRI; | |
| const RegisterBankInfo &RBI; | |
| SPIRVGlobalRegistry &GR; | |
| MachineRegisterInfo *MRI; | |
| MachineFunction *HasVRegsReset = nullptr; | |
| /// We need to keep track of the number we give to anonymous global values to | |
| /// generate the same name every time when this is needed. | |
| mutable DenseMap<const GlobalValue *, unsigned> UnnamedGlobalIDs; | |
| SmallPtrSet<MachineInstr *, 8> DeadMIs; | |
| public: | |
| SPIRVInstructionSelector(const SPIRVTargetMachine &TM, | |
| const SPIRVSubtarget &ST, | |
| const RegisterBankInfo &RBI); | |
| void setupMF(MachineFunction &MF, GISelValueTracking *VT, | |
| CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI, | |
| BlockFrequencyInfo *BFI) override; | |
| // Common selection code. Instruction-specific selection occurs in spvSelect. | |
| bool select(MachineInstr &I) override; | |
| static const char *getName() { return DEBUG_TYPE; } | |
| #define GET_GLOBALISEL_PREDICATES_DECL | |
| #include "SPIRVGenGlobalISel.inc" | |
| #undef GET_GLOBALISEL_PREDICATES_DECL | |
| #define GET_GLOBALISEL_TEMPORARIES_DECL | |
| #include "SPIRVGenGlobalISel.inc" | |
| #undef GET_GLOBALISEL_TEMPORARIES_DECL | |
| private: | |
| void resetVRegsType(MachineFunction &MF); | |
| void removeDeadInstruction(MachineInstr &MI) const; | |
| void removeOpNamesForDeadMI(MachineInstr &MI) const; | |
| // tblgen-erated 'select' implementation, used as the initial selector for | |
| // the patterns that don't require complex C++. | |
| bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const; | |
| // All instruction-specific selection that didn't happen in "select()". | |
| // Is basically a large Switch/Case delegating to all other select method. | |
| bool spvSelect(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectFirstBitHigh(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I, bool IsSigned) const; | |
| bool selectFirstBitLow(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectFirstBitSet16(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I, unsigned ExtendOpcode, | |
| unsigned BitSetOpcode) const; | |
| bool selectFirstBitSet32(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I, Register SrcReg, | |
| unsigned BitSetOpcode) const; | |
| bool selectFirstBitSet64(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I, Register SrcReg, | |
| unsigned BitSetOpcode, bool SwapPrimarySide) const; | |
| bool selectGlobalValue(Register ResVReg, MachineInstr &I, | |
| const MachineInstr *Init = nullptr) const; | |
| bool selectOpWithSrcs(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I, ArrayRef<Register> SrcRegs, | |
| unsigned Opcode) const; | |
| bool selectUnOp(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I, | |
| unsigned Opcode) const; | |
| bool selectBitcast(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectLoad(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectAtomicLoad(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectStore(MachineInstr &I) const; | |
| bool selectAtomicStore(MachineInstr &I) const; | |
| bool selectStackSave(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectStackRestore(MachineInstr &I) const; | |
| bool selectMemOperation(Register ResVReg, MachineInstr &I) const; | |
| Register getOrCreateMemSetGlobal(MachineInstr &I) const; | |
| bool selectCopyMemory(MachineInstr &I, Register SrcReg) const; | |
| bool selectAtomicRMW(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I, | |
| unsigned NewOpcode, unsigned NegateOpcode = 0) const; | |
| // Creates an integer-typed register with bitwidth equal to pointer size. | |
| Register createPtrSizedIntReg(MachineIRBuilder &MIRBuilder) const; | |
| // Emit an OpConvertPtrToU that converts the pointer value in \p PtrVal into | |
| // an integer of equal bitwidth, returning the register holding the result. | |
| Register convertPtrToInt(Register PtrVal, MachineIRBuilder &MIRBuilder) const; | |
| // Emit an OpBitcast that reinterprets the pointer \p Ptr as a pointer to an | |
| // integer of pointer size in storage class \p SC, returning the result. | |
| Register castPtrToPtrToInt(Register Ptr, SPIRV::StorageClass::StorageClass SC, | |
| MachineIRBuilder &MIRBuilder) const; | |
| // Handle atomic loads, stores and exchanges of pointer types by casting | |
| // to/from integer types as needed. | |
| bool selectAtomicPtrValue( | |
| Register ResVReg, SPIRVTypeInst ResType, MachineIRBuilder &MIRBuilder, | |
| function_ref<Register(SPIRVTypeInst IntType)> EmitAtomic) const; | |
| bool selectAtomicCmpXchg(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectFence(MachineInstr &I) const; | |
| bool selectAddrSpaceCast(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectPtrMask(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectAnyOrAll(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I, | |
| unsigned OpType) const; | |
| bool selectAll(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectAny(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectBitreverse(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectBitreverseViaI32(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I, Register Op) const; | |
| bool selectBitreverse64(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I, Register SrcReg) const; | |
| bool selectBitreverseNative(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I, Register Op) const; | |
| bool selectBuildVector(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectSplatVector(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectConcatVectors(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectCmp(Register ResVReg, SPIRVTypeInst ResType, | |
| unsigned comparisonOpcode, MachineInstr &I) const; | |
| bool selectDiscard(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectICmp(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectFCmp(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectSign(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectFloatDot(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectOverflowArith(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I, unsigned Opcode) const; | |
| bool selectDebugTrap(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectPrefetch(MachineInstr &I) const; | |
| bool selectIntegerDot(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I, bool Signed) const; | |
| bool selectIntegerDotExpansion(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectOpIsInf(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectOpIsNan(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectOpIsFinite(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectOpIsNormal(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectPopCount(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I, | |
| unsigned Opcode) const; | |
| bool selectPopCount16(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I, unsigned ExtOpcode, | |
| unsigned Opcode) const; | |
| bool selectPopCount32(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I, Register SrcReg, | |
| unsigned Opcode) const; | |
| bool selectPopCount64(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I, Register SrcReg, | |
| unsigned Opcode) const; | |
| template <bool Signed> | |
| bool selectDot4AddPacked(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| template <bool Signed> | |
| bool selectDot4AddPackedExpansion(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectWavePrefixBitCount(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| template <typename PickOpcodeFn> | |
| bool selectWaveReduce(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I, bool IsUnsigned, | |
| PickOpcodeFn &&PickOpcode) const; | |
| bool selectWaveReduceOp(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I, unsigned Opcode) const; | |
| bool selectWaveReduceMax(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I, bool IsUnsigned) const; | |
| bool selectWaveReduceMin(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I, bool IsUnsigned) const; | |
| bool selectWaveReduceSum(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectWaveReduceProduct(Register ResVReg, const SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| template <typename PickOpcodeFn> | |
| bool selectWaveExclusiveScan(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I, bool IsUnsigned, | |
| PickOpcodeFn &&PickOpcode) const; | |
| bool selectWaveExclusiveScanSum(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectWaveExclusiveScanProduct(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectQuadSwap(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I, | |
| unsigned Direction) const; | |
| bool selectConst(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectSelect(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectBoolToInt(Register ResVReg, SPIRVTypeInst ResType, | |
| Register BooleanVReg, MachineInstr &InsertAt, | |
| bool IsSigned) const; | |
| bool selectIToF(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I, | |
| bool IsSigned, unsigned Opcode) const; | |
| bool selectExt(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I, | |
| bool IsSigned) const; | |
| bool selectTrunc(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectSUCmp(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I, | |
| bool IsSigned) const; | |
| bool selectIntToBool(Register IntReg, Register ResVReg, MachineInstr &I, | |
| SPIRVTypeInst intTy, SPIRVTypeInst boolTy) const; | |
| bool selectOpUndef(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectFreeze(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectIntrinsic(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectExtractVal(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectInsertVal(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectExtractElt(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectInsertElt(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectGEP(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectMaskedGather(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectMaskedScatter(MachineInstr &I) const; | |
| bool diagnoseUnsupported(const MachineInstr &I, const Twine &Msg) const; | |
| bool selectAbort(MachineInstr &I) const; | |
| bool selectTrap(MachineInstr &I) const; | |
| bool selectFrameIndex(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectAllocaArray(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectBranch(MachineInstr &I) const; | |
| bool selectBranchCond(MachineInstr &I) const; | |
| bool selectPhi(Register ResVReg, MachineInstr &I) const; | |
| bool selectExtInst(Register ResVReg, SPIRVTypeInst RestType, MachineInstr &I, | |
| GL::GLSLExtInst GLInst, bool setMIFlags = true, | |
| bool useMISrc = true, | |
| ArrayRef<Register> SrcRegs = {}) const; | |
| bool selectExtInst(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I, | |
| CL::OpenCLExtInst CLInst, bool setMIFlags = true, | |
| bool useMISrc = true, | |
| ArrayRef<Register> SrcRegs = {}) const; | |
| bool selectExtInst(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I, | |
| CL::OpenCLExtInst CLInst, GL::GLSLExtInst GLInst, | |
| bool setMIFlags = true, bool useMISrc = true, | |
| ArrayRef<Register> SrcRegs = {}) const; | |
| bool selectExtInst(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I, | |
| const ExtInstList &ExtInsts, bool setMIFlags = true, | |
| bool useMISrc = true, | |
| ArrayRef<Register> SrcRegs = {}) const; | |
| bool selectLog10(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectFpowi(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectSaturate(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectWaveOpInst(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I, unsigned Opcode) const; | |
| bool selectBarrierInst(MachineInstr &I, unsigned Scope, unsigned MemSem, | |
| bool WithGroupSync) const; | |
| bool selectWaveActiveCountBits(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectWaveActiveAllEqual(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectUnmergeValues(MachineInstr &I) const; | |
| bool selectHandleFromBinding(Register &ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectCounterHandleFromBinding(Register &ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectReadImageIntrinsic(Register &ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectGetDimensionsIntrinsic(Register &ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectGetDimensionsLevelsIntrinsic(Register &ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectGetDimensionsMSIntrinsic(Register &ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool | |
| selectImageQuerySize(Register ImageReg, Register &ResVReg, MachineInstr &I, | |
| std::optional<Register> LodReg = std::nullopt) const; | |
| bool selectSampleBasicIntrinsic(Register &ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectCalculateLodIntrinsic(Register &ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectSampleBiasIntrinsic(Register &ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectSampleGradIntrinsic(Register &ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectSampleLevelIntrinsic(Register &ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectLoadLevelIntrinsic(Register &ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectSampleCmpIntrinsic(Register &ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectSampleCmpLevelZeroIntrinsic(Register &ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectGatherIntrinsic(Register &ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectImageWriteIntrinsic(MachineInstr &I) const; | |
| bool selectResourceGetPointer(Register &ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectPushConstantGetPointer(Register &ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectResourceNonUniformIndex(Register &ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectModf(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectUpdateCounter(Register &ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectFrexp(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectCopySign(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectLdexp(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectSincos(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectExp10(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool selectDerivativeInst(Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I, const unsigned DPdOpCode) const; | |
| // Utilities | |
| Register buildI32Constant(uint32_t Val, MachineInstr &I, | |
| SPIRVTypeInst ResType = nullptr) const; | |
| Register buildI32ConstantInEntryBlock(uint32_t Val, MachineInstr &I, | |
| SPIRVTypeInst ResType = nullptr) const; | |
| Register buildZerosVal(SPIRVTypeInst ResType, MachineInstr &I) const; | |
| bool isScalarOrVectorIntConstantZero(Register Reg) const; | |
| Register buildZerosValF(SPIRVTypeInst ResType, MachineInstr &I) const; | |
| Register buildOnesVal(bool AllOnes, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| Register buildOnesValF(SPIRVTypeInst ResType, MachineInstr &I) const; | |
| Register buildVectorSplat(Register ScalarReg, unsigned NumElts, | |
| MachineInstr &I) const; | |
| bool wrapIntoSpecConstantOp(MachineInstr &I, | |
| SmallVector<Register> &CompositeArgs) const; | |
| Register getUcharPtrTypeReg(MachineInstr &I, | |
| SPIRV::StorageClass::StorageClass SC) const; | |
| MachineInstrBuilder buildSpecConstantOp(MachineInstr &I, Register Dest, | |
| Register Src, Register DestType, | |
| uint32_t Opcode) const; | |
| MachineInstrBuilder buildConstGenericPtr(MachineInstr &I, Register SrcPtr, | |
| SPIRVTypeInst SrcPtrTy) const; | |
| Register buildPointerToResource(SPIRVTypeInst ResType, | |
| SPIRV::StorageClass::StorageClass SC, | |
| uint32_t Set, uint32_t Binding, | |
| uint32_t ArraySize, Register IndexReg, | |
| StringRef Name, | |
| MachineIRBuilder MIRBuilder) const; | |
| SPIRVTypeInst widenTypeToVec4(SPIRVTypeInst Type, MachineInstr &I) const; | |
| bool extractSubvector(Register &ResVReg, SPIRVTypeInst ResType, | |
| Register &ReadReg, MachineInstr &InsertionPoint) const; | |
| bool generateImageReadOrFetch(Register &ResVReg, SPIRVTypeInst ResType, | |
| Register ImageReg, Register IdxReg, | |
| DebugLoc Loc, MachineInstr &Pos, | |
| const ImageOperands *ImOps = nullptr) const; | |
| bool generateSampleImage(Register ResVReg, SPIRVTypeInst ResType, | |
| Register ImageReg, Register SamplerReg, | |
| Register CoordinateReg, const ImageOperands &ImOps, | |
| DebugLoc Loc, MachineInstr &I) const; | |
| bool BuildCOPY(Register DestReg, Register SrcReg, MachineInstr &I) const; | |
| bool loadVec3BuiltinInputID(SPIRV::BuiltIn::BuiltIn BuiltInValue, | |
| Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool loadBuiltinInputID(SPIRV::BuiltIn::BuiltIn BuiltInValue, | |
| Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I) const; | |
| bool loadHandleBeforePosition(Register &HandleReg, SPIRVTypeInst ResType, | |
| GIntrinsic &HandleDef, MachineInstr &Pos) const; | |
| void decorateUsesAsNonUniform(Register &NonUniformReg) const; | |
| bool errorIfInstrOutsideShader(MachineInstr &I) const; | |
| bool | |
| handle64BitOverflow(Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I, | |
| Register SrcReg, unsigned int Opcode, | |
| std::function<bool(Register, SPIRVTypeInst, | |
| MachineInstr &, Register, unsigned)> | |
| CallbackFunction) const; | |
| }; | |
| bool sampledTypeIsSignedInteger(const llvm::Type *HandleType) { | |
| const TargetExtType *TET = cast<TargetExtType>(HandleType); | |
| if (TET->getTargetExtName() == "spirv.Image") { | |
| return false; | |
| } | |
| assert(TET->getTargetExtName() == "spirv.SignedImage"); | |
| return TET->getTypeParameter(0)->isIntegerTy(); | |
| } | |
| } // end anonymous namespace | |
| #define GET_GLOBALISEL_IMPL | |
| #include "SPIRVGenGlobalISel.inc" | |
| #undef GET_GLOBALISEL_IMPL | |
| SPIRVInstructionSelector::SPIRVInstructionSelector(const SPIRVTargetMachine &TM, | |
| const SPIRVSubtarget &ST, | |
| const RegisterBankInfo &RBI) | |
| : InstructionSelector(), STI(ST), TII(*ST.getInstrInfo()), | |
| TRI(*ST.getRegisterInfo()), RBI(RBI), GR(*ST.getSPIRVGlobalRegistry()), | |
| MRI(nullptr), | |
| #define GET_GLOBALISEL_PREDICATES_INIT | |
| #include "SPIRVGenGlobalISel.inc" | |
| #undef GET_GLOBALISEL_PREDICATES_INIT | |
| #define GET_GLOBALISEL_TEMPORARIES_INIT | |
| #include "SPIRVGenGlobalISel.inc" | |
| #undef GET_GLOBALISEL_TEMPORARIES_INIT | |
| { | |
| } | |
| void SPIRVInstructionSelector::setupMF(MachineFunction &MF, | |
| GISelValueTracking *VT, | |
| CodeGenCoverage *CoverageInfo, | |
| ProfileSummaryInfo *PSI, | |
| BlockFrequencyInfo *BFI) { | |
| MRI = &MF.getRegInfo(); | |
| GR.setCurrentFunc(MF); | |
| InstructionSelector::setupMF(MF, VT, CoverageInfo, PSI, BFI); | |
| } | |
| // Ensure that register classes correspond to pattern matching rules. | |
| void SPIRVInstructionSelector::resetVRegsType(MachineFunction &MF) { | |
| if (HasVRegsReset == &MF) | |
| return; | |
| HasVRegsReset = &MF; | |
| MachineRegisterInfo &MRI = MF.getRegInfo(); | |
| for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) { | |
| Register Reg = Register::index2VirtReg(I); | |
| LLT RegType = MRI.getType(Reg); | |
| if (RegType.isScalar()) | |
| MRI.setType(Reg, LLT::scalar(64)); | |
| else if (RegType.isPointer()) | |
| MRI.setType(Reg, LLT::pointer(0, 64)); | |
| else if (RegType.isVector()) | |
| MRI.setType(Reg, LLT::fixed_vector(2, LLT::scalar(64))); | |
| } | |
| for (const auto &MBB : MF) { | |
| for (const auto &MI : MBB) { | |
| if (isPreISelGenericOpcode(MI.getOpcode())) | |
| GR.erase(&MI); | |
| if (MI.getOpcode() != SPIRV::ASSIGN_TYPE) | |
| continue; | |
| Register DstReg = MI.getOperand(0).getReg(); | |
| LLT DstType = MRI.getType(DstReg); | |
| Register SrcReg = MI.getOperand(1).getReg(); | |
| LLT SrcType = MRI.getType(SrcReg); | |
| if (DstType != SrcType) | |
| MRI.setType(DstReg, MRI.getType(SrcReg)); | |
| const TargetRegisterClass *DstRC = MRI.getRegClassOrNull(DstReg); | |
| const TargetRegisterClass *SrcRC = MRI.getRegClassOrNull(SrcReg); | |
| if (DstRC != SrcRC && SrcRC) | |
| MRI.setRegClass(DstReg, SrcRC); | |
| } | |
| } | |
| } | |
| // Return true if the MachineInstr represents a constant register | |
| static bool isConstReg(MachineRegisterInfo *MRI, MachineInstr *OpDef) { | |
| SmallVector<MachineInstr *> Stack = {OpDef}; | |
| SmallPtrSet<MachineInstr *, 4> Visited; | |
| while (!Stack.empty()) { | |
| MachineInstr *MI = Stack.pop_back_val(); | |
| MI = passCopy(MI, MRI); | |
| if (!Visited.insert(MI).second) | |
| continue; | |
| switch (MI->getOpcode()) { | |
| case TargetOpcode::G_INTRINSIC: | |
| case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: | |
| case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: { | |
| GIntrinsic *GIntr = cast<GIntrinsic>(MI); | |
| unsigned IntrID = GIntr->getIntrinsicID(); | |
| if (IntrID != Intrinsic::spv_const_composite && | |
| IntrID != Intrinsic::spv_undef && IntrID != Intrinsic::spv_poison) | |
| return false; | |
| continue; | |
| } | |
| case TargetOpcode::G_BUILD_VECTOR: | |
| case TargetOpcode::G_SPLAT_VECTOR: | |
| for (unsigned i = OpDef->getNumExplicitDefs(); | |
| i < OpDef->getNumOperands(); i++) { | |
| if (!OpDef->getOperand(i).isReg()) | |
| continue; | |
| MachineInstr *OpNestedDef = | |
| MRI->getVRegDef(OpDef->getOperand(i).getReg()); | |
| Stack.push_back(OpNestedDef); | |
| } | |
| continue; | |
| case TargetOpcode::G_CONSTANT: | |
| case TargetOpcode::G_FCONSTANT: | |
| case TargetOpcode::G_IMPLICIT_DEF: | |
| case SPIRV::OpConstantTrue: | |
| case SPIRV::OpConstantFalse: | |
| case SPIRV::OpConstantI: | |
| case SPIRV::OpConstantF: | |
| case SPIRV::OpConstantComposite: | |
| case SPIRV::OpConstantCompositeContinuedINTEL: | |
| case SPIRV::OpConstantSampler: | |
| case SPIRV::OpConstantNull: | |
| case SPIRV::OpUndef: | |
| case SPIRV::OpPoisonKHR: | |
| case SPIRV::OpConstantFunctionPointerINTEL: | |
| continue; | |
| default: | |
| return false; | |
| } | |
| } | |
| return true; | |
| } | |
| // Return true if the virtual register represents a constant | |
| static bool isConstReg(MachineRegisterInfo *MRI, Register OpReg) { | |
| if (MachineInstr *OpDef = MRI->getVRegDef(OpReg)) | |
| return isConstReg(MRI, OpDef); | |
| return false; | |
| } | |
| // TODO(168736): We should make this either a flag in tabelgen | |
| // or reduce our dependence on the global registry, so we can remove this | |
| // function. It can easily be missed when new intrinsics are added. | |
| // Most SPIR-V intrinsics are considered to have side-effects in their tablegen | |
| // definition because they are referenced in the global registry. This is a list | |
| // of intrinsics that have no side effects other than their references in the | |
| // global registry. | |
| static bool intrinsicHasSideEffects(Intrinsic::ID ID) { | |
| switch (ID) { | |
| // This is not an exhaustive list and may need to be updated. | |
| case Intrinsic::spv_all: | |
| case Intrinsic::spv_alloca: | |
| case Intrinsic::spv_any: | |
| case Intrinsic::spv_bitcast: | |
| case Intrinsic::spv_const_composite: | |
| case Intrinsic::spv_degrees: | |
| case Intrinsic::spv_distance: | |
| case Intrinsic::spv_extractelt: | |
| case Intrinsic::spv_extractv: | |
| case Intrinsic::spv_faceforward: | |
| case Intrinsic::spv_fdot: | |
| case Intrinsic::spv_firstbitlow: | |
| case Intrinsic::spv_firstbitshigh: | |
| case Intrinsic::spv_firstbituhigh: | |
| case Intrinsic::spv_frac: | |
| case Intrinsic::spv_gep: | |
| case Intrinsic::spv_global_offset: | |
| case Intrinsic::spv_global_size: | |
| case Intrinsic::spv_group_id: | |
| case Intrinsic::spv_insertelt: | |
| case Intrinsic::spv_insertv: | |
| case Intrinsic::spv_isinf: | |
| case Intrinsic::spv_isnan: | |
| case Intrinsic::spv_isfinite: | |
| case Intrinsic::spv_isnormal: | |
| case Intrinsic::spv_lerp: | |
| case Intrinsic::spv_length: | |
| case Intrinsic::spv_normalize: | |
| case Intrinsic::spv_num_subgroups: | |
| case Intrinsic::spv_num_workgroups: | |
| case Intrinsic::spv_ptrcast: | |
| case Intrinsic::spv_radians: | |
| case Intrinsic::spv_reflect: | |
| case Intrinsic::spv_refract: | |
| case Intrinsic::spv_resource_getbasepointer: | |
| case Intrinsic::spv_resource_getpointer: | |
| case Intrinsic::spv_resource_handlefrombinding: | |
| case Intrinsic::spv_resource_handlefromimplicitbinding: | |
| case Intrinsic::spv_resource_nonuniformindex: | |
| case Intrinsic::spv_resource_sample: | |
| case Intrinsic::spv_rsqrt: | |
| case Intrinsic::spv_saturate: | |
| case Intrinsic::spv_sdot: | |
| case Intrinsic::spv_sign: | |
| case Intrinsic::spv_smoothstep: | |
| case Intrinsic::spv_subgroup_id: | |
| case Intrinsic::spv_subgroup_local_invocation_id: | |
| case Intrinsic::spv_subgroup_max_size: | |
| case Intrinsic::spv_subgroup_size: | |
| case Intrinsic::spv_thread_id: | |
| case Intrinsic::spv_thread_id_in_group: | |
| case Intrinsic::spv_udot: | |
| case Intrinsic::spv_undef: | |
| case Intrinsic::spv_value_md: | |
| case Intrinsic::spv_workgroup_size: | |
| return false; | |
| default: | |
| return true; | |
| } | |
| } | |
| // TODO(168736): We should make this either a flag in tabelgen | |
| // or reduce our dependence on the global registry, so we can remove this | |
| // function. It can easily be missed when new intrinsics are added. | |
| static bool isOpcodeWithNoSideEffects(unsigned Opcode) { | |
| switch (Opcode) { | |
| case SPIRV::OpTypeVoid: | |
| case SPIRV::OpTypeBool: | |
| case SPIRV::OpTypeInt: | |
| case SPIRV::OpTypeFloat: | |
| case SPIRV::OpTypeVector: | |
| case SPIRV::OpTypeVectorIdEXT: | |
| case SPIRV::OpTypeMatrix: | |
| case SPIRV::OpTypeImage: | |
| case SPIRV::OpTypeSampler: | |
| case SPIRV::OpTypeSampledImage: | |
| case SPIRV::OpTypeArray: | |
| case SPIRV::OpTypeRuntimeArray: | |
| case SPIRV::OpTypeStruct: | |
| case SPIRV::OpTypeOpaque: | |
| case SPIRV::OpTypePointer: | |
| case SPIRV::OpTypeFunction: | |
| case SPIRV::OpTypeEvent: | |
| case SPIRV::OpTypeDeviceEvent: | |
| case SPIRV::OpTypeReserveId: | |
| case SPIRV::OpTypeQueue: | |
| case SPIRV::OpTypePipe: | |
| case SPIRV::OpTypeForwardPointer: | |
| case SPIRV::OpTypePipeStorage: | |
| case SPIRV::OpTypeNamedBarrier: | |
| case SPIRV::OpTypeAccelerationStructureNV: | |
| case SPIRV::OpTypeCooperativeMatrixNV: | |
| case SPIRV::OpTypeCooperativeMatrixKHR: | |
| return true; | |
| default: | |
| return false; | |
| } | |
| } | |
| bool isDead(const MachineInstr &MI, const MachineRegisterInfo &MRI) { | |
| // If there are no definitions, then assume there is some other | |
| // side-effect that makes this instruction live. | |
| if (MI.getNumDefs() == 0) | |
| return false; | |
| for (const auto &MO : MI.all_defs()) { | |
| Register Reg = MO.getReg(); | |
| if (Reg.isPhysical()) { | |
| LLVM_DEBUG(dbgs() << "Not dead: def of physical register " << Reg); | |
| return false; | |
| } | |
| for (const auto &UseMI : MRI.use_nodbg_instructions(Reg)) { | |
| if (UseMI.getOpcode() != SPIRV::OpName) { | |
| LLVM_DEBUG(dbgs() << "Not dead: def " << MO << " has use in " << UseMI); | |
| return false; | |
| } | |
| } | |
| } | |
| if (MI.getOpcode() == TargetOpcode::LOCAL_ESCAPE || MI.isFakeUse() || | |
| MI.isLifetimeMarker()) { | |
| LLVM_DEBUG( | |
| dbgs() | |
| << "Not dead: Opcode is LOCAL_ESCAPE, fake use, or lifetime marker.\n"); | |
| return false; | |
| } | |
| if (MI.isPHI()) { | |
| LLVM_DEBUG(dbgs() << "Dead: Phi instruction with no uses.\n"); | |
| return true; | |
| } | |
| // It is possible that the only side effect is that the instruction is | |
| // referenced in the global registry. If that is the only side effect, the | |
| // intrinsic is dead. | |
| if (MI.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS || | |
| MI.getOpcode() == TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS) { | |
| const auto &Intr = cast<GIntrinsic>(MI); | |
| if (!intrinsicHasSideEffects(Intr.getIntrinsicID())) { | |
| LLVM_DEBUG(dbgs() << "Dead: Intrinsic with no real side effects.\n"); | |
| return true; | |
| } | |
| } | |
| if (MI.mayStore() || MI.isCall() || | |
| (MI.mayLoad() && MI.hasOrderedMemoryRef()) || MI.isPosition() || | |
| MI.isDebugInstr() || MI.isTerminator() || MI.isJumpTableDebugInfo()) { | |
| LLVM_DEBUG(dbgs() << "Not dead: instruction has side effects.\n"); | |
| return false; | |
| } | |
| if (isPreISelGenericOpcode(MI.getOpcode())) { | |
| // TODO: Is there a generic way to check if the opcode has side effects? | |
| LLVM_DEBUG(dbgs() << "Dead: Generic opcode with no uses.\n"); | |
| return true; | |
| } | |
| if (isOpcodeWithNoSideEffects(MI.getOpcode())) { | |
| LLVM_DEBUG(dbgs() << "Dead: known opcode with no side effects\n"); | |
| return true; | |
| } | |
| return false; | |
| } | |
| void SPIRVInstructionSelector::removeOpNamesForDeadMI(MachineInstr &MI) const { | |
| // Delete the OpName that uses the result if there is one. | |
| for (const auto &MO : MI.all_defs()) { | |
| Register Reg = MO.getReg(); | |
| if (Reg.isPhysical()) | |
| continue; | |
| SmallVector<MachineInstr *, 4> UselessOpNames; | |
| for (MachineInstr &UseMI : MRI->use_nodbg_instructions(Reg)) { | |
| assert(UseMI.getOpcode() == SPIRV::OpName && | |
| "There is still a use of the dead function."); | |
| UselessOpNames.push_back(&UseMI); | |
| } | |
| for (MachineInstr *OpNameMI : UselessOpNames) { | |
| GR.invalidateMachineInstr(OpNameMI); | |
| OpNameMI->eraseFromParent(); | |
| } | |
| } | |
| } | |
| void SPIRVInstructionSelector::removeDeadInstruction(MachineInstr &MI) const { | |
| salvageDebugInfo(*MRI, MI); | |
| GR.invalidateMachineInstr(&MI); | |
| removeOpNamesForDeadMI(MI); | |
| MI.eraseFromParent(); | |
| } | |
| bool SPIRVInstructionSelector::select(MachineInstr &I) { | |
| resetVRegsType(*I.getParent()->getParent()); | |
| assert(I.getParent() && "Instruction should be in a basic block!"); | |
| assert(I.getParent()->getParent() && "Instruction should be in a function!"); | |
| LLVM_DEBUG(dbgs() << "Checking if instruction is dead: " << I;); | |
| if (isDead(I, *MRI)) { | |
| LLVM_DEBUG(dbgs() << "Instruction is dead.\n"); | |
| removeDeadInstruction(I); | |
| return true; | |
| } | |
| Register Opcode = I.getOpcode(); | |
| // If it's not a GMIR instruction, we've selected it already. | |
| if (!isPreISelGenericOpcode(Opcode)) { | |
| if (Opcode == SPIRV::ASSIGN_TYPE) { // These pseudos aren't needed any more. | |
| Register DstReg = I.getOperand(0).getReg(); | |
| Register SrcReg = I.getOperand(1).getReg(); | |
| auto *Def = MRI->getVRegDef(SrcReg); | |
| if (isTypeFoldingSupported(Def->getOpcode()) && | |
| Def->getOpcode() != TargetOpcode::G_CONSTANT && | |
| Def->getOpcode() != TargetOpcode::G_FCONSTANT) { | |
| if (Def->getOpcode() == TargetOpcode::G_SELECT) { | |
| Register SelectDstReg = Def->getOperand(0).getReg(); | |
| bool SuccessToSelectSelect [[maybe_unused]] = selectSelect( | |
| SelectDstReg, GR.getSPIRVTypeForVReg(SelectDstReg), *Def); | |
| assert(SuccessToSelectSelect); | |
| GR.invalidateMachineInstr(Def); | |
| Def->eraseFromParent(); | |
| MRI->replaceRegWith(DstReg, SelectDstReg); | |
| GR.invalidateMachineInstr(&I); | |
| I.eraseFromParent(); | |
| return true; | |
| } | |
| bool Res = selectImpl(I, *CoverageInfo); | |
| LLVM_DEBUG({ | |
| if (!Res && Def->getOpcode() != TargetOpcode::G_CONSTANT) { | |
| dbgs() << "Unexpected pattern in ASSIGN_TYPE.\nInstruction: "; | |
| I.print(dbgs()); | |
| } | |
| }); | |
| assert(Res || Def->getOpcode() == TargetOpcode::G_CONSTANT); | |
| if (Res) { | |
| if (!isTriviallyDead(*Def, *MRI) && isDead(*Def, *MRI)) | |
| DeadMIs.insert(Def); | |
| return Res; | |
| } | |
| } | |
| MRI->setRegClass(SrcReg, MRI->getRegClass(DstReg)); | |
| MRI->replaceRegWith(SrcReg, DstReg); | |
| GR.invalidateMachineInstr(&I); | |
| I.eraseFromParent(); | |
| return true; | |
| } else if (I.getNumDefs() == 1) { | |
| // Make all vregs 64 bits (for SPIR-V IDs). | |
| MRI->setType(I.getOperand(0).getReg(), LLT::scalar(64)); | |
| } | |
| constrainSelectedInstRegOperands(I, TII, TRI, RBI); | |
| return true; | |
| } | |
| if (DeadMIs.contains(&I)) { | |
| // if the instruction has been already made dead by folding it away | |
| // erase it | |
| LLVM_DEBUG(dbgs() << "Instruction is folded and dead.\n"); | |
| removeDeadInstruction(I); | |
| DeadMIs.erase(&I); | |
| return true; | |
| } | |
| if (I.getNumOperands() != I.getNumExplicitOperands()) { | |
| LLVM_DEBUG(errs() << "Generic instr has unexpected implicit operands\n"); | |
| return false; | |
| } | |
| // Common code for getting return reg+type, and removing selected instr | |
| // from parent occurs here. Instr-specific selection happens in spvSelect(). | |
| bool HasDefs = I.getNumDefs() > 0; | |
| Register ResVReg = HasDefs ? I.getOperand(0).getReg() : Register(0); | |
| SPIRVTypeInst ResType = HasDefs ? GR.getSPIRVTypeForVReg(ResVReg) : nullptr; | |
| assert(!HasDefs || ResType || I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE || | |
| I.getOpcode() == TargetOpcode::G_IMPLICIT_DEF); | |
| if (spvSelect(ResVReg, ResType, I)) { | |
| if (HasDefs) // Make all vregs 64 bits (for SPIR-V IDs). | |
| for (unsigned i = 0; i < I.getNumDefs(); ++i) | |
| MRI->setType(I.getOperand(i).getReg(), LLT::scalar(64)); | |
| GR.invalidateMachineInstr(&I); | |
| I.eraseFromParent(); | |
| return true; | |
| } | |
| return false; | |
| } | |
| static bool mayApplyGenericSelection(unsigned Opcode) { | |
| switch (Opcode) { | |
| case TargetOpcode::G_CONSTANT: | |
| case TargetOpcode::G_FCONSTANT: | |
| return false; | |
| } | |
| return isTypeFoldingSupported(Opcode); | |
| } | |
| bool SPIRVInstructionSelector::BuildCOPY(Register DestReg, Register SrcReg, | |
| MachineInstr &I) const { | |
| const TargetRegisterClass *DstRC = MRI->getRegClassOrNull(DestReg); | |
| const TargetRegisterClass *SrcRC = MRI->getRegClassOrNull(SrcReg); | |
| if (DstRC != SrcRC && SrcRC) | |
| MRI->setRegClass(DestReg, SrcRC); | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(TargetOpcode::COPY)) | |
| .addDef(DestReg) | |
| .addUse(SrcReg) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::spvSelect(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| const unsigned Opcode = I.getOpcode(); | |
| if (mayApplyGenericSelection(Opcode)) | |
| return selectImpl(I, *CoverageInfo); | |
| switch (Opcode) { | |
| case TargetOpcode::G_CONSTANT: | |
| case TargetOpcode::G_FCONSTANT: | |
| return selectConst(ResVReg, ResType, I); | |
| case TargetOpcode::G_GLOBAL_VALUE: | |
| return selectGlobalValue(ResVReg, I); | |
| case TargetOpcode::G_IMPLICIT_DEF: | |
| return selectOpUndef(ResVReg, ResType, I); | |
| case TargetOpcode::G_FREEZE: | |
| return selectFreeze(ResVReg, ResType, I); | |
| case TargetOpcode::G_INTRINSIC: | |
| case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: | |
| case TargetOpcode::G_INTRINSIC_CONVERGENT: | |
| case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: | |
| return selectIntrinsic(ResVReg, ResType, I); | |
| case TargetOpcode::G_BITREVERSE: | |
| return selectBitreverse(ResVReg, ResType, I); | |
| case TargetOpcode::G_BUILD_VECTOR: | |
| return selectBuildVector(ResVReg, ResType, I); | |
| case TargetOpcode::G_SPLAT_VECTOR: | |
| return selectSplatVector(ResVReg, ResType, I); | |
| case TargetOpcode::G_CONCAT_VECTORS: | |
| return selectConcatVectors(ResVReg, ResType, I); | |
| case TargetOpcode::G_SHUFFLE_VECTOR: { | |
| MachineBasicBlock &BB = *I.getParent(); | |
| auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorShuffle)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(I.getOperand(1).getReg()) | |
| .addUse(I.getOperand(2).getReg()); | |
| for (auto V : I.getOperand(3).getShuffleMask()) | |
| MIB.addImm(V); | |
| MIB.constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| case TargetOpcode::G_MEMMOVE: | |
| case TargetOpcode::G_MEMCPY: | |
| case TargetOpcode::G_MEMCPY_INLINE: | |
| case TargetOpcode::G_MEMSET: | |
| case TargetOpcode::G_MEMSET_INLINE: | |
| return selectMemOperation(ResVReg, I); | |
| case TargetOpcode::G_ICMP: | |
| return selectICmp(ResVReg, ResType, I); | |
| case TargetOpcode::G_FCMP: | |
| return selectFCmp(ResVReg, ResType, I); | |
| case TargetOpcode::G_FRAME_INDEX: | |
| return selectFrameIndex(ResVReg, ResType, I); | |
| case TargetOpcode::G_LOAD: | |
| return selectLoad(ResVReg, ResType, I); | |
| case TargetOpcode::G_STORE: | |
| return selectStore(I); | |
| case TargetOpcode::G_BR: | |
| return selectBranch(I); | |
| case TargetOpcode::G_BRCOND: | |
| return selectBranchCond(I); | |
| case TargetOpcode::G_PHI: | |
| return selectPhi(ResVReg, I); | |
| case TargetOpcode::G_FPTOSI: | |
| return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToS); | |
| case TargetOpcode::G_FPTOUI: | |
| return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToU); | |
| case TargetOpcode::G_FPTOSI_SAT: | |
| return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToS); | |
| case TargetOpcode::G_FPTOUI_SAT: | |
| return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToU); | |
| case TargetOpcode::G_SITOFP: | |
| return selectIToF(ResVReg, ResType, I, true, SPIRV::OpConvertSToF); | |
| case TargetOpcode::G_UITOFP: | |
| return selectIToF(ResVReg, ResType, I, false, SPIRV::OpConvertUToF); | |
| case TargetOpcode::G_CTPOP: | |
| return selectPopCount(ResVReg, ResType, I, SPIRV::OpBitCount); | |
| case TargetOpcode::G_SMIN: | |
| return selectExtInst(ResVReg, ResType, I, CL::s_min, GL::SMin); | |
| case TargetOpcode::G_UMIN: | |
| return selectExtInst(ResVReg, ResType, I, CL::u_min, GL::UMin); | |
| case TargetOpcode::G_SMAX: | |
| return selectExtInst(ResVReg, ResType, I, CL::s_max, GL::SMax); | |
| case TargetOpcode::G_UMAX: | |
| return selectExtInst(ResVReg, ResType, I, CL::u_max, GL::UMax); | |
| case TargetOpcode::G_SCMP: | |
| return selectSUCmp(ResVReg, ResType, I, true); | |
| case TargetOpcode::G_UCMP: | |
| return selectSUCmp(ResVReg, ResType, I, false); | |
| case TargetOpcode::G_LROUND: | |
| case TargetOpcode::G_LLROUND: { | |
| Register regForLround = | |
| MRI->createVirtualRegister(MRI->getRegClass(ResVReg), "lround"); | |
| MRI->setRegClass(regForLround, &SPIRV::iIDRegClass); | |
| GR.assignSPIRVTypeToVReg(GR.getSPIRVTypeForVReg(I.getOperand(1).getReg()), | |
| regForLround, *(I.getParent()->getParent())); | |
| selectExtInst(regForLround, GR.getSPIRVTypeForVReg(regForLround), I, | |
| CL::round, GL::Round, /* setMIFlags */ false); | |
| MachineBasicBlock &BB = *I.getParent(); | |
| auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConvertFToS)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(regForLround); | |
| MIB.constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| case TargetOpcode::G_STRICT_FMA: | |
| case TargetOpcode::G_FMA: { | |
| if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_fma)) { | |
| MachineBasicBlock &BB = *I.getParent(); | |
| auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpFmaKHR)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(I.getOperand(1).getReg()) | |
| .addUse(I.getOperand(2).getReg()) | |
| .addUse(I.getOperand(3).getReg()) | |
| .setMIFlags(I.getFlags()); | |
| MIB.constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| return selectExtInst(ResVReg, ResType, I, CL::fma, GL::Fma); | |
| } | |
| case TargetOpcode::G_FLDEXP: | |
| case TargetOpcode::G_STRICT_FLDEXP: | |
| return selectLdexp(ResVReg, ResType, I); | |
| case TargetOpcode::G_FPOW: | |
| return selectExtInst(ResVReg, ResType, I, CL::pow, GL::Pow); | |
| case TargetOpcode::G_FPOWI: | |
| return selectFpowi(ResVReg, ResType, I); | |
| case TargetOpcode::G_FEXP: | |
| return selectExtInst(ResVReg, ResType, I, CL::exp, GL::Exp); | |
| case TargetOpcode::G_FEXP2: | |
| return selectExtInst(ResVReg, ResType, I, CL::exp2, GL::Exp2); | |
| case TargetOpcode::G_FEXP10: | |
| return selectExp10(ResVReg, ResType, I); | |
| case TargetOpcode::G_FMODF: | |
| return selectModf(ResVReg, ResType, I); | |
| case TargetOpcode::G_FSINCOS: | |
| return selectSincos(ResVReg, ResType, I); | |
| case TargetOpcode::G_FLOG: | |
| return selectExtInst(ResVReg, ResType, I, CL::log, GL::Log); | |
| case TargetOpcode::G_FLOG2: | |
| return selectExtInst(ResVReg, ResType, I, CL::log2, GL::Log2); | |
| case TargetOpcode::G_FLOG10: | |
| return selectLog10(ResVReg, ResType, I); | |
| case TargetOpcode::G_FABS: | |
| return selectExtInst(ResVReg, ResType, I, CL::fabs, GL::FAbs); | |
| case TargetOpcode::G_ABS: | |
| return selectExtInst(ResVReg, ResType, I, CL::s_abs, GL::SAbs); | |
| case TargetOpcode::G_FMINNUM: | |
| return selectExtInst(ResVReg, ResType, I, CL::fmin, GL::NMin); | |
| case TargetOpcode::G_FMAXNUM: | |
| return selectExtInst(ResVReg, ResType, I, CL::fmax, GL::NMax); | |
| case TargetOpcode::G_FCOPYSIGN: | |
| return selectCopySign(ResVReg, ResType, I); | |
| case TargetOpcode::G_FCEIL: | |
| return selectExtInst(ResVReg, ResType, I, CL::ceil, GL::Ceil); | |
| case TargetOpcode::G_FFLOOR: | |
| return selectExtInst(ResVReg, ResType, I, CL::floor, GL::Floor); | |
| case TargetOpcode::G_FCOS: | |
| return selectExtInst(ResVReg, ResType, I, CL::cos, GL::Cos); | |
| case TargetOpcode::G_FSIN: | |
| return selectExtInst(ResVReg, ResType, I, CL::sin, GL::Sin); | |
| case TargetOpcode::G_FTAN: | |
| return selectExtInst(ResVReg, ResType, I, CL::tan, GL::Tan); | |
| case TargetOpcode::G_FACOS: | |
| return selectExtInst(ResVReg, ResType, I, CL::acos, GL::Acos); | |
| case TargetOpcode::G_FASIN: | |
| return selectExtInst(ResVReg, ResType, I, CL::asin, GL::Asin); | |
| case TargetOpcode::G_FATAN: | |
| return selectExtInst(ResVReg, ResType, I, CL::atan, GL::Atan); | |
| case TargetOpcode::G_FATAN2: | |
| return selectExtInst(ResVReg, ResType, I, CL::atan2, GL::Atan2); | |
| case TargetOpcode::G_FCOSH: | |
| return selectExtInst(ResVReg, ResType, I, CL::cosh, GL::Cosh); | |
| case TargetOpcode::G_FSINH: | |
| return selectExtInst(ResVReg, ResType, I, CL::sinh, GL::Sinh); | |
| case TargetOpcode::G_FTANH: | |
| return selectExtInst(ResVReg, ResType, I, CL::tanh, GL::Tanh); | |
| case TargetOpcode::G_STRICT_FSQRT: | |
| case TargetOpcode::G_FSQRT: | |
| return selectExtInst(ResVReg, ResType, I, CL::sqrt, GL::Sqrt); | |
| case TargetOpcode::G_CTTZ: | |
| case TargetOpcode::G_CTTZ_ZERO_POISON: | |
| return selectExtInst(ResVReg, ResType, I, CL::ctz); | |
| case TargetOpcode::G_CTLZ: | |
| case TargetOpcode::G_CTLZ_ZERO_POISON: | |
| return selectExtInst(ResVReg, ResType, I, CL::clz); | |
| case TargetOpcode::G_INTRINSIC_ROUND: | |
| return selectExtInst(ResVReg, ResType, I, CL::round, GL::Round); | |
| case TargetOpcode::G_INTRINSIC_ROUNDEVEN: | |
| return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven); | |
| case TargetOpcode::G_INTRINSIC_TRUNC: | |
| return selectExtInst(ResVReg, ResType, I, CL::trunc, GL::Trunc); | |
| case TargetOpcode::G_FRINT: | |
| case TargetOpcode::G_FNEARBYINT: | |
| return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven); | |
| case TargetOpcode::G_SMULH: | |
| return selectExtInst(ResVReg, ResType, I, CL::s_mul_hi); | |
| case TargetOpcode::G_UMULH: | |
| return selectExtInst(ResVReg, ResType, I, CL::u_mul_hi); | |
| case TargetOpcode::G_SADDSAT: | |
| return selectExtInst(ResVReg, ResType, I, CL::s_add_sat); | |
| case TargetOpcode::G_UADDSAT: | |
| return selectExtInst(ResVReg, ResType, I, CL::u_add_sat); | |
| case TargetOpcode::G_SSUBSAT: | |
| return selectExtInst(ResVReg, ResType, I, CL::s_sub_sat); | |
| case TargetOpcode::G_USUBSAT: | |
| return selectExtInst(ResVReg, ResType, I, CL::u_sub_sat); | |
| case TargetOpcode::G_FFREXP: | |
| return selectFrexp(ResVReg, ResType, I); | |
| case TargetOpcode::G_UADDO: | |
| return selectOverflowArith(ResVReg, ResType, I, | |
| isVectorType(ResType) ? SPIRV::OpIAddCarryV | |
| : SPIRV::OpIAddCarryS); | |
| case TargetOpcode::G_USUBO: | |
| return selectOverflowArith(ResVReg, ResType, I, | |
| isVectorType(ResType) ? SPIRV::OpISubBorrowV | |
| : SPIRV::OpISubBorrowS); | |
| case TargetOpcode::G_UMULO: | |
| return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpUMulExtended); | |
| case TargetOpcode::G_SMULO: | |
| return selectOverflowArith(ResVReg, ResType, I, SPIRV::OpSMulExtended); | |
| case TargetOpcode::G_SEXT: | |
| return selectExt(ResVReg, ResType, I, true); | |
| case TargetOpcode::G_ANYEXT: | |
| case TargetOpcode::G_ZEXT: | |
| return selectExt(ResVReg, ResType, I, false); | |
| case TargetOpcode::G_TRUNC: | |
| return selectTrunc(ResVReg, ResType, I); | |
| case TargetOpcode::G_FPTRUNC: | |
| case TargetOpcode::G_FPEXT: | |
| return selectUnOp(ResVReg, ResType, I, SPIRV::OpFConvert); | |
| case TargetOpcode::G_PTRTOINT: | |
| return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertPtrToU); | |
| case TargetOpcode::G_INTTOPTR: | |
| return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertUToPtr); | |
| case TargetOpcode::G_BITCAST: | |
| return selectBitcast(ResVReg, ResType, I); | |
| case TargetOpcode::G_ADDRSPACE_CAST: | |
| return selectAddrSpaceCast(ResVReg, ResType, I); | |
| case TargetOpcode::G_PTRMASK: | |
| return selectPtrMask(ResVReg, ResType, I); | |
| case TargetOpcode::G_PTR_ADD: { | |
| // Currently, we get G_PTR_ADD only applied to global variables. | |
| assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); | |
| Register GV = I.getOperand(1).getReg(); | |
| MachineRegisterInfo::def_instr_iterator II = MRI->def_instr_begin(GV); | |
| (void)II; | |
| assert(((*II).getOpcode() == TargetOpcode::G_GLOBAL_VALUE || | |
| (*II).getOpcode() == TargetOpcode::COPY || | |
| (*II).getOpcode() == SPIRV::OpVariable || | |
| (*II).getOpcode() == SPIRV::OpUntypedVariableKHR) && | |
| getImm(I.getOperand(2), MRI)); | |
| // It may be the initialization of a global variable. | |
| bool IsGVInit = false; | |
| for (MachineRegisterInfo::use_instr_iterator | |
| UseIt = MRI->use_instr_begin(I.getOperand(0).getReg()), | |
| UseEnd = MRI->use_instr_end(); | |
| UseIt != UseEnd; UseIt = std::next(UseIt)) { | |
| if ((*UseIt).getOpcode() == TargetOpcode::G_GLOBAL_VALUE || | |
| (*UseIt).getOpcode() == SPIRV::OpSpecConstantOp || | |
| (*UseIt).getOpcode() == SPIRV::OpVariable || | |
| (*UseIt).getOpcode() == SPIRV::OpUntypedVariableKHR) { | |
| IsGVInit = true; | |
| break; | |
| } | |
| } | |
| MachineBasicBlock &BB = *I.getParent(); | |
| // An untyped result needs OpUntypedInBoundsPtrAccessChainKHR, which spells | |
| // out the Base Type. The offset of G_PTR_ADD is a byte count, so the Base | |
| // Type is i8 and the offset is the Element index as is. The opcode is | |
| // picked by the result type alone because an untyped access chain accepts | |
| // a typed base while a typed access chain rejects an untyped result. | |
| SPIRVTypeInst GVType = GR.getSPIRVTypeForVReg(GV); | |
| const bool UseUntypedPointers = | |
| ResType->getOpcode() == SPIRV::OpTypeUntypedPointerKHR; | |
| if (UseUntypedPointers) { | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addImm(static_cast<uint32_t>( | |
| SPIRV::Opcode::UntypedInBoundsPtrAccessChainKHR)) | |
| .addUse(GR.getSPIRVTypeID(GR.getOrCreateSPIRVIntegerType(8, I, TII))) | |
| .addUse(GV) | |
| .addUse(I.getOperand(2).getReg()) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| if (!IsGVInit) { | |
| SPIRVTypeInst GVPointeeType = GR.getPointeeType(GVType); | |
| SPIRVTypeInst ResPointeeType = GR.getPointeeType(ResType); | |
| if (GVPointeeType && ResPointeeType && GVPointeeType != ResPointeeType) { | |
| // Build a new virtual register that is associated with the required | |
| // data type. | |
| Register NewVReg = MRI->createGenericVirtualRegister(MRI->getType(GV)); | |
| MRI->setRegClass(NewVReg, MRI->getRegClass(GV)); | |
| // Having a correctly typed base we are ready to build the actually | |
| // required GEP. It may not be a constant though, because all Operands | |
| // of OpSpecConstantOp is to originate from other const instructions, | |
| // and only the AccessChain named opcodes accept a global OpVariable | |
| // instruction. We can't use an AccessChain opcode because of the type | |
| // mismatch between result and base types. | |
| if (!GR.isBitcastCompatible(ResType, GVType)) | |
| return diagnoseUnsupported( | |
| I, "incompatible result and operand types in a bitcast"); | |
| Register ResTypeReg = GR.getSPIRVTypeID(ResType); | |
| MachineInstrBuilder MIB = | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitcast)) | |
| .addDef(NewVReg) | |
| .addUse(ResTypeReg) | |
| .addUse(GV); | |
| MIB.constrainAllUses(TII, TRI, RBI); | |
| BuildMI(BB, I, I.getDebugLoc(), | |
| TII.get(STI.isLogicalSPIRV() ? SPIRV::OpInBoundsAccessChain | |
| : SPIRV::OpInBoundsPtrAccessChain)) | |
| .addDef(ResVReg) | |
| .addUse(ResTypeReg) | |
| .addUse(NewVReg) | |
| .addUse(I.getOperand(2).getReg()) | |
| .constrainAllUses(TII, TRI, RBI); | |
| } else { | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addImm( | |
| static_cast<uint32_t>(SPIRV::Opcode::InBoundsPtrAccessChain)) | |
| .addUse(GV) | |
| .addUse(I.getOperand(2).getReg()) | |
| .constrainAllUses(TII, TRI, RBI); | |
| } | |
| return true; | |
| } | |
| // It's possible to translate G_PTR_ADD to OpSpecConstantOp: either to | |
| // initialize a global variable with a constant expression (e.g., the test | |
| // case opencl/basic/progvar_prog_scope_init.ll), or for another use case | |
| Register Idx = buildZerosVal(GR.getOrCreateSPIRVIntegerType(32, I, TII), I); | |
| auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addImm(static_cast<uint32_t>( | |
| SPIRV::Opcode::InBoundsPtrAccessChain)) | |
| .addUse(GV) | |
| .addUse(Idx) | |
| .addUse(I.getOperand(2).getReg()); | |
| MIB.constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| case TargetOpcode::G_ATOMICRMW_OR: | |
| return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicOr); | |
| case TargetOpcode::G_ATOMICRMW_ADD: | |
| return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicIAdd); | |
| case TargetOpcode::G_ATOMICRMW_AND: | |
| return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicAnd); | |
| case TargetOpcode::G_ATOMICRMW_MAX: | |
| return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMax); | |
| case TargetOpcode::G_ATOMICRMW_MIN: | |
| return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMin); | |
| case TargetOpcode::G_ATOMICRMW_SUB: | |
| return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicISub); | |
| case TargetOpcode::G_ATOMICRMW_XOR: | |
| return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicXor); | |
| case TargetOpcode::G_ATOMICRMW_UMAX: | |
| return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMax); | |
| case TargetOpcode::G_ATOMICRMW_UMIN: | |
| return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMin); | |
| case TargetOpcode::G_ATOMICRMW_XCHG: | |
| return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicExchange); | |
| case TargetOpcode::G_ATOMICRMW_FADD: | |
| return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT); | |
| case TargetOpcode::G_ATOMICRMW_FSUB: | |
| // Translate G_ATOMICRMW_FSUB to OpAtomicFAddEXT with negative value operand | |
| return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT, | |
| isVectorType(ResType) ? SPIRV::OpFNegateV | |
| : SPIRV::OpFNegate); | |
| case TargetOpcode::G_ATOMICRMW_FMIN: | |
| return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMinEXT); | |
| case TargetOpcode::G_ATOMICRMW_FMAX: | |
| return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMaxEXT); | |
| case TargetOpcode::G_FENCE: | |
| return selectFence(I); | |
| case TargetOpcode::G_STACKSAVE: | |
| return selectStackSave(ResVReg, ResType, I); | |
| case TargetOpcode::G_STACKRESTORE: | |
| return selectStackRestore(I); | |
| case TargetOpcode::G_UNMERGE_VALUES: | |
| return selectUnmergeValues(I); | |
| case TargetOpcode::G_TRAP: | |
| case TargetOpcode::G_UBSANTRAP: | |
| return selectTrap(I); | |
| // Discard gen opcodes for intrinsics which we do not expect to actually | |
| // represent code after lowering or intrinsics which are not implemented but | |
| // should not crash when found in a customer's LLVM IR input. | |
| case TargetOpcode::DBG_LABEL: | |
| return true; | |
| case TargetOpcode::G_DEBUGTRAP: | |
| return selectDebugTrap(ResVReg, ResType, I); | |
| case TargetOpcode::G_PREFETCH: | |
| return selectPrefetch(I); | |
| default: | |
| return false; | |
| } | |
| } | |
| bool SPIRVInstructionSelector::selectDebugTrap(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| unsigned Opcode = SPIRV::OpNop; | |
| MachineBasicBlock &BB = *I.getParent(); | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectPrefetch(MachineInstr &I) const { | |
| // llvm.prefetch is missing info about how many bytes to prefetch that | |
| // OpUntypedPrefetchKHR instruction requires, so just drop the instruction | |
| // in case untyped pointers are enabled since OCL prefetch cannot take | |
| // untyped pointers. | |
| if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_untyped_pointers)) | |
| return true; | |
| if (STI.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) { | |
| MachineIRBuilder MIRBuilder(I); | |
| const unsigned PointerSize = GR.getPointerSize(); | |
| const SPIRVTypeInst PointerSizeType = | |
| GR.getOrCreateSPIRVIntegerType(PointerSize, MIRBuilder); | |
| bool ZeroAsNull = !STI.isShader(); | |
| // Assume that a single element needs prefetch. | |
| Register ConstIntOne = | |
| GR.getOrCreateConstInt(1, I, PointerSizeType, TII, ZeroAsNull); | |
| Register AddrVal = I.getOperand(0).getReg(); | |
| Register ExtReg = MRI->createVirtualRegister( | |
| GR.getRegClass(GR.getOpTypeVoid(MIRBuilder))); | |
| return selectExtInst(ExtReg, GR.getOpTypeVoid(MIRBuilder), I, CL::prefetch, | |
| /*setMIFlags=*/false, /*useMSSrc=*/false, | |
| {AddrVal, ConstIntOne}); | |
| } | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I, | |
| GL::GLSLExtInst GLInst, | |
| bool setMIFlags, bool useMISrc, | |
| ArrayRef<Register> SrcRegs) const { | |
| if (!STI.canUseExtInstSet( | |
| SPIRV::InstructionSet::InstructionSet::GLSL_std_450)) | |
| return diagnoseUnsupported( | |
| I, | |
| "this instruction is only supported with the GLSL extended instruction " | |
| "set."); | |
| return selectExtInst(ResVReg, ResType, I, | |
| {{SPIRV::InstructionSet::GLSL_std_450, GLInst}}, | |
| setMIFlags, useMISrc, SrcRegs); | |
| } | |
| bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I, | |
| CL::OpenCLExtInst CLInst, | |
| bool setMIFlags, bool useMISrc, | |
| ArrayRef<Register> SrcRegs) const { | |
| return selectExtInst(ResVReg, ResType, I, | |
| {{SPIRV::InstructionSet::OpenCL_std, CLInst}}, | |
| setMIFlags, useMISrc, SrcRegs); | |
| } | |
| bool SPIRVInstructionSelector::selectExtInst( | |
| Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I, | |
| CL::OpenCLExtInst CLInst, GL::GLSLExtInst GLInst, bool setMIFlags, | |
| bool useMISrc, ArrayRef<Register> SrcRegs) const { | |
| ExtInstList ExtInsts = {{SPIRV::InstructionSet::OpenCL_std, CLInst}, | |
| {SPIRV::InstructionSet::GLSL_std_450, GLInst}}; | |
| return selectExtInst(ResVReg, ResType, I, ExtInsts, setMIFlags, useMISrc, | |
| SrcRegs); | |
| } | |
| bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I, | |
| const ExtInstList &Insts, | |
| bool setMIFlags, bool useMISrc, | |
| ArrayRef<Register> SrcRegs) const { | |
| for (const auto &[InstructionSet, Opcode] : Insts) { | |
| if (!STI.canUseExtInstSet(InstructionSet)) | |
| continue; | |
| MachineBasicBlock &BB = *I.getParent(); | |
| auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addImm(static_cast<uint32_t>(InstructionSet)) | |
| .addImm(Opcode); | |
| if (setMIFlags) | |
| MIB.setMIFlags(I.getFlags()); | |
| if (useMISrc) { | |
| const unsigned NumOps = I.getNumOperands(); | |
| unsigned Index = 1; | |
| if (Index < NumOps && | |
| I.getOperand(Index).getType() == | |
| MachineOperand::MachineOperandType::MO_IntrinsicID) | |
| Index = 2; | |
| for (; Index < NumOps; ++Index) | |
| MIB.add(I.getOperand(Index)); | |
| } else { | |
| for (Register SReg : SrcRegs) { | |
| MIB.addUse(SReg); | |
| } | |
| } | |
| MIB.constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| return false; | |
| } | |
| bool SPIRVInstructionSelector::selectCopySign(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| if (STI.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) | |
| return selectExtInst(ResVReg, ResType, I, CL::copysign); | |
| // There is no copysign instruction in the GLSL Extended Instruction set, so | |
| // it is implemented with bit manipulation: | |
| // bitcast((bitcast(magnitude) & ~signBit) | (bitcast(sign) & signBit)) | |
| Register MagnitudeReg = I.getOperand(1).getReg(); | |
| Register SignReg = I.getOperand(2).getReg(); | |
| const unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType); | |
| const unsigned ComponentCount = GR.getScalarOrVectorComponentCount(ResType); | |
| SPIRVTypeInst IntType = GR.getOrCreateSPIRVIntegerType(BitWidth, I, TII); | |
| const APInt SignMaskVal = APInt::getSignMask(BitWidth); | |
| Register SignMask, NotSignMask; | |
| unsigned AndOpcode, OrOpcode; | |
| if (ComponentCount > 1) { | |
| IntType = GR.getOrCreateSPIRVVectorType(IntType, ComponentCount, I, TII); | |
| SignMask = GR.getOrCreateConstVector(SignMaskVal, I, IntType, TII); | |
| NotSignMask = GR.getOrCreateConstVector(~SignMaskVal, I, IntType, TII); | |
| AndOpcode = SPIRV::OpBitwiseAndV; | |
| OrOpcode = SPIRV::OpBitwiseOrV; | |
| } else { | |
| SignMask = GR.getOrCreateConstInt(SignMaskVal, I, IntType, TII); | |
| NotSignMask = GR.getOrCreateConstInt(~SignMaskVal, I, IntType, TII); | |
| AndOpcode = SPIRV::OpBitwiseAndS; | |
| OrOpcode = SPIRV::OpBitwiseOrS; | |
| } | |
| auto EmitBitOp = [&](Register &ResReg, ArrayRef<Register> SrcRegs, | |
| unsigned Opcode) { | |
| ResReg = MRI->createVirtualRegister(GR.getRegClass(IntType)); | |
| return selectOpWithSrcs(ResReg, IntType, I, SrcRegs, Opcode); | |
| }; | |
| Register MagnitudeInt, SignInt, MagnitudeBits, SignBits, CombinedInt; | |
| if (!EmitBitOp(MagnitudeInt, {MagnitudeReg}, SPIRV::OpBitcast) || | |
| !EmitBitOp(SignInt, {SignReg}, SPIRV::OpBitcast) || | |
| !EmitBitOp(MagnitudeBits, {MagnitudeInt, NotSignMask}, AndOpcode) || | |
| !EmitBitOp(SignBits, {SignInt, SignMask}, AndOpcode) || | |
| !EmitBitOp(CombinedInt, {MagnitudeBits, SignBits}, OrOpcode)) | |
| return false; | |
| return selectOpWithSrcs(ResVReg, ResType, I, {CombinedInt}, SPIRV::OpBitcast); | |
| } | |
| bool SPIRVInstructionSelector::selectFrexp(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| ExtInstList ExtInsts = {{SPIRV::InstructionSet::OpenCL_std, CL::frexp}, | |
| {SPIRV::InstructionSet::GLSL_std_450, GL::Frexp}}; | |
| for (const auto &Ex : ExtInsts) { | |
| SPIRV::InstructionSet::InstructionSet Set = Ex.first; | |
| uint32_t Opcode = Ex.second; | |
| if (!STI.canUseExtInstSet(Set)) | |
| continue; | |
| MachineIRBuilder MIRBuilder(I); | |
| SPIRVTypeInst PointeeTy = GR.getSPIRVTypeForVReg(I.getOperand(1).getReg()); | |
| const SPIRVTypeInst PointerType = GR.getOrCreateSPIRVPointerType( | |
| PointeeTy, MIRBuilder, SPIRV::StorageClass::Function); | |
| Register PointerVReg = | |
| createVirtualRegister(PointerType, &GR, MRI, MRI->getMF()); | |
| auto It = getOpVariableMBBIt(*I.getMF()); | |
| // An untyped pointer result type is only legal on OpUntypedVariableKHR, | |
| // but not on OpVariable. | |
| const bool IsUntyped = | |
| PointerType->getOpcode() == SPIRV::OpTypeUntypedPointerKHR; | |
| auto VarMIB = | |
| BuildMI(*It->getParent(), It, It->getDebugLoc(), | |
| TII.get(IsUntyped ? SPIRV::OpUntypedVariableKHR | |
| : SPIRV::OpVariable)) | |
| .addDef(PointerVReg) | |
| .addUse(GR.getSPIRVTypeID(PointerType)) | |
| .addImm(static_cast<uint32_t>(SPIRV::StorageClass::Function)); | |
| if (IsUntyped) | |
| VarMIB.addUse(GR.getSPIRVTypeID(PointeeTy)); // Data Type | |
| VarMIB.constrainAllUses(TII, TRI, RBI); | |
| SPIRVTypeInst MantissaTy = GR.getSPIRVTypeForVReg(I.getOperand(2).getReg()); | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(MantissaTy)) | |
| .addImm(static_cast<uint32_t>(Ex.first)) | |
| .addImm(Opcode) | |
| .add(I.getOperand(2)) | |
| .addUse(PointerVReg) | |
| .constrainAllUses(TII, TRI, RBI); | |
| Register ExpResReg = I.getOperand(1).getReg(); | |
| if (!MRI->use_nodbg_empty(ExpResReg)) | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) | |
| .addDef(ExpResReg) | |
| .addUse(GR.getSPIRVTypeID(PointeeTy)) | |
| .addUse(PointerVReg) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| return false; | |
| } | |
| bool SPIRVInstructionSelector::selectLdexp(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| Register XReg = I.getOperand(1).getReg(); | |
| Register ExpReg = I.getOperand(2).getReg(); | |
| // Both OpenCL.std ldexp and GLSL.std.450 Ldexp require the exponent to have | |
| // the same number of components as the result, but G_FLDEXP's exponent is | |
| // scalar even for vector operands. Splat it when the result is a vector. | |
| SPIRVTypeInst ExpType = GR.getSPIRVTypeForVReg(ExpReg); | |
| if (ResType->getOpcode() == SPIRV::OpTypeVector && | |
| ExpType->getOpcode() != SPIRV::OpTypeVector) { | |
| unsigned NumElts = ResType->getOperand(2).getImm(); | |
| ExpReg = buildVectorSplat(ExpReg, NumElts, I); | |
| } | |
| return selectExtInst(ResVReg, ResType, I, CL::ldexp, GL::Ldexp, | |
| /*setMIFlags=*/true, /*useMISrc=*/false, {XReg, ExpReg}); | |
| } | |
| bool SPIRVInstructionSelector::selectSincos(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| Register CosResVReg = I.getOperand(1).getReg(); | |
| unsigned SrcIdx = I.getNumExplicitDefs(); | |
| Register ResTypeReg = GR.getSPIRVTypeID(ResType); | |
| if (STI.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) { | |
| // OpenCL.std sincos(x, cosval*) -> returns sin(x), writes cos(x) to ptr. | |
| MachineIRBuilder MIRBuilder(I); | |
| const SPIRVTypeInst PointerType = GR.getOrCreateSPIRVPointerType( | |
| ResType, MIRBuilder, SPIRV::StorageClass::Function); | |
| Register PointerVReg = | |
| createVirtualRegister(PointerType, &GR, MRI, MRI->getMF()); | |
| auto It = getOpVariableMBBIt(*I.getMF()); | |
| // An untyped pointer result type is only legal on OpUntypedVariableKHR, | |
| // but not on OpVariable. | |
| const bool IsUntyped = | |
| PointerType->getOpcode() == SPIRV::OpTypeUntypedPointerKHR; | |
| auto VarMIB = | |
| BuildMI(*It->getParent(), It, It->getDebugLoc(), | |
| TII.get(IsUntyped ? SPIRV::OpUntypedVariableKHR | |
| : SPIRV::OpVariable)) | |
| .addDef(PointerVReg) | |
| .addUse(GR.getSPIRVTypeID(PointerType)) | |
| .addImm(static_cast<uint32_t>(SPIRV::StorageClass::Function)); | |
| if (IsUntyped) | |
| VarMIB.addUse(GR.getSPIRVTypeID(ResType)); // Data Type | |
| VarMIB.constrainAllUses(TII, TRI, RBI); | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) | |
| .addDef(ResVReg) | |
| .addUse(ResTypeReg) | |
| .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::OpenCL_std)) | |
| .addImm(CL::sincos) | |
| .add(I.getOperand(SrcIdx)) | |
| .addUse(PointerVReg) | |
| .constrainAllUses(TII, TRI, RBI); | |
| if (!MRI->use_nodbg_empty(CosResVReg)) | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) | |
| .addDef(CosResVReg) | |
| .addUse(ResTypeReg) | |
| .addUse(PointerVReg) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } else if (STI.canUseExtInstSet(SPIRV::InstructionSet::GLSL_std_450)) { | |
| // GLSL.std.450 has no combined sincos; emit separate Sin and Cos. | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) | |
| .addDef(ResVReg) | |
| .addUse(ResTypeReg) | |
| .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) | |
| .addImm(GL::Sin) | |
| .add(I.getOperand(SrcIdx)) | |
| .constrainAllUses(TII, TRI, RBI); | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) | |
| .addDef(CosResVReg) | |
| .addUse(ResTypeReg) | |
| .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) | |
| .addImm(GL::Cos) | |
| .add(I.getOperand(SrcIdx)) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| return false; | |
| } | |
| bool SPIRVInstructionSelector::selectOpWithSrcs(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I, | |
| ArrayRef<Register> Srcs, | |
| unsigned Opcode) const { | |
| auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)); | |
| for (Register SReg : Srcs) { | |
| MIB.addUse(SReg); | |
| } | |
| MIB.constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectPopCount16(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I, | |
| unsigned ExtOpcode, | |
| unsigned Opcode) const { | |
| MachineIRBuilder MIRBuilder(I); | |
| Register OpReg = I.getOperand(1).getReg(); | |
| SPIRVTypeInst SrcType = GR.getSPIRVTypeForVReg(OpReg); | |
| unsigned ComponentCount = GR.getScalarOrVectorComponentCount(SrcType); | |
| bool IsScalar = !isVectorType(SrcType); | |
| SPIRVTypeInst I32Type = GR.getOrCreateSPIRVIntegerType(32, MIRBuilder); | |
| SPIRVTypeInst ExtType = | |
| IsScalar ? I32Type | |
| : GR.getOrCreateSPIRVVectorType(I32Type, ComponentCount, | |
| MIRBuilder, /*IsSigned=*/false); | |
| Register ExtReg = MRI->createVirtualRegister(GR.getRegClass(ExtType)); | |
| // Always use OpUConvert to always use a 0 extend | |
| if (!selectOpWithSrcs(ExtReg, ExtType, I, {OpReg}, SPIRV::OpUConvert)) | |
| return false; | |
| Register PopCountReg = MRI->createVirtualRegister(GR.getRegClass(ExtType)); | |
| if (!selectPopCount32(PopCountReg, ExtType, I, ExtReg, Opcode)) | |
| return false; | |
| return selectOpWithSrcs(ResVReg, ResType, I, {PopCountReg}, ExtOpcode); | |
| } | |
| bool SPIRVInstructionSelector::selectPopCount32(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I, | |
| Register SrcReg, | |
| unsigned Opcode) const { | |
| return selectOpWithSrcs(ResVReg, ResType, I, {SrcReg}, Opcode); | |
| } | |
| bool SPIRVInstructionSelector::selectPopCount64(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I, | |
| Register SrcReg, | |
| unsigned Opcode) const { | |
| MachineIRBuilder MIRBuilder(I); | |
| SPIRVTypeInst SrcType = GR.getSPIRVTypeForVReg(SrcReg); | |
| unsigned ComponentCount = GR.getScalarOrVectorComponentCount(SrcType); | |
| bool IsScalar = !isVectorType(SrcType); | |
| SPIRVTypeInst I32Type = GR.getOrCreateSPIRVIntegerType(32, MIRBuilder); | |
| // We need to work with a type matching the shape of the input but using 32 | |
| // bit int instead of 64. | |
| SPIRVTypeInst WorkingType = | |
| IsScalar ? I32Type | |
| : GR.getOrCreateSPIRVVectorType(I32Type, ComponentCount, | |
| MIRBuilder, /*IsSigned=*/false); | |
| // Truncate and count the low bits. | |
| Register Trunc = MRI->createVirtualRegister(GR.getRegClass(WorkingType)); | |
| if (!selectOpWithSrcs(Trunc, WorkingType, I, {SrcReg}, SPIRV::OpUConvert)) | |
| return false; | |
| Register LowCount = MRI->createVirtualRegister(GR.getRegClass(WorkingType)); | |
| if (!selectOpWithSrcs(LowCount, WorkingType, I, {Trunc}, SPIRV::OpBitCount)) | |
| return false; | |
| // Shift the high bits over and count them too. | |
| Register ShiftAmount = IsScalar | |
| ? GR.getOrCreateConstInt(32, I, SrcType, TII) | |
| : GR.getOrCreateConstVector(32, I, SrcType, TII); | |
| unsigned ShiftOp = | |
| IsScalar ? SPIRV::OpShiftRightLogicalS : SPIRV::OpShiftRightLogicalV; | |
| Register Shift = MRI->createVirtualRegister(GR.getRegClass(SrcType)); | |
| if (!selectOpWithSrcs(Shift, SrcType, I, {SrcReg, ShiftAmount}, ShiftOp)) | |
| return false; | |
| Trunc = MRI->createVirtualRegister(GR.getRegClass(WorkingType)); | |
| if (!selectOpWithSrcs(Trunc, WorkingType, I, {Shift}, SPIRV::OpUConvert)) | |
| return false; | |
| Register HighCount = MRI->createVirtualRegister(GR.getRegClass(WorkingType)); | |
| if (!selectOpWithSrcs(HighCount, WorkingType, I, {Trunc}, SPIRV::OpBitCount)) | |
| return false; | |
| // Add them up and zext or sext back to 64 bit values. | |
| Register Sum = MRI->createVirtualRegister(GR.getRegClass(WorkingType)); | |
| if (!selectOpWithSrcs(Sum, WorkingType, I, {HighCount, LowCount}, | |
| IsScalar ? SPIRV::OpIAddS : SPIRV::OpIAddV)) | |
| return false; | |
| bool IsSigned = GR.isScalarOrVectorSigned(ResType); | |
| unsigned ConvOp = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; | |
| return selectOpWithSrcs(ResVReg, ResType, I, {Sum}, ConvOp); | |
| } | |
| bool SPIRVInstructionSelector::selectPopCount(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I, | |
| unsigned Opcode) const { | |
| // Vulkan restricts OpBitCount to 32-bit integers or vectors of 32-bit | |
| // integers unless VK_KHR_maintenance9 is enabled. Until VK_KHR_maintenance9 | |
| // is core we will not generate OpBitCount with any other types when | |
| // targeting Vulkan. | |
| if (!STI.getTargetTriple().isVulkanOS()) | |
| return selectUnOp(ResVReg, ResType, I, Opcode); | |
| Register OpReg = I.getOperand(1).getReg(); | |
| SPIRVTypeInst OpType = GR.getSPIRVTypeForVReg(OpReg); | |
| unsigned ExtOpcode = GR.isScalarOrVectorSigned(ResType) ? SPIRV::OpSConvert | |
| : SPIRV::OpUConvert; | |
| switch (GR.getScalarOrVectorBitWidth(OpType)) { | |
| case 8: | |
| case 16: | |
| return selectPopCount16(ResVReg, ResType, I, ExtOpcode, Opcode); | |
| case 32: | |
| return selectPopCount32(ResVReg, ResType, I, OpReg, Opcode); | |
| case 64: | |
| return selectPopCount64(ResVReg, ResType, I, OpReg, Opcode); | |
| default: | |
| return diagnoseUnsupported(I, "unsupported operand bit width for popcount"); | |
| } | |
| } | |
| bool SPIRVInstructionSelector::selectUnOp(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I, | |
| unsigned Opcode) const { | |
| if (STI.isPhysicalSPIRV() && I.getOperand(1).isReg()) { | |
| Register SrcReg = I.getOperand(1).getReg(); | |
| bool IsGV = false; | |
| for (MachineRegisterInfo::def_instr_iterator DefIt = | |
| MRI->def_instr_begin(SrcReg); | |
| DefIt != MRI->def_instr_end(); DefIt = std::next(DefIt)) { | |
| unsigned DefOpCode = DefIt->getOpcode(); | |
| if (DefOpCode == SPIRV::ASSIGN_TYPE || DefOpCode == TargetOpcode::COPY) { | |
| // We need special handling to look through the type assignment or the | |
| // COPY pseudo-op and see if this is a constant or a global. | |
| if (auto *VRD = getVRegDef(*MRI, DefIt->getOperand(1).getReg())) | |
| DefOpCode = VRD->getOpcode(); | |
| } | |
| if (DefOpCode == TargetOpcode::G_GLOBAL_VALUE || | |
| DefOpCode == TargetOpcode::G_CONSTANT || | |
| DefOpCode == SPIRV::OpVariable || | |
| DefOpCode == SPIRV::OpUntypedVariableKHR || | |
| DefOpCode == SPIRV::OpConstantI) { | |
| IsGV = true; | |
| break; | |
| } | |
| } | |
| if (IsGV) { | |
| uint32_t SpecOpcode = 0; | |
| switch (Opcode) { | |
| case SPIRV::OpConvertPtrToU: | |
| SpecOpcode = static_cast<uint32_t>(SPIRV::Opcode::ConvertPtrToU); | |
| break; | |
| case SPIRV::OpConvertUToPtr: | |
| SpecOpcode = static_cast<uint32_t>(SPIRV::Opcode::ConvertUToPtr); | |
| break; | |
| } | |
| if (SpecOpcode) { | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), | |
| TII.get(SPIRV::OpSpecConstantOp)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addImm(SpecOpcode) | |
| .addUse(SrcReg) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| } | |
| } | |
| return selectOpWithSrcs(ResVReg, ResType, I, {I.getOperand(1).getReg()}, | |
| Opcode); | |
| } | |
| bool SPIRVInstructionSelector::selectBitcast(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| Register OpReg = I.getOperand(1).getReg(); | |
| SPIRVTypeInst OpType = | |
| OpReg.isValid() ? GR.getSPIRVTypeForVReg(OpReg) : nullptr; | |
| if (!GR.isBitcastCompatible(ResType, OpType)) | |
| return diagnoseUnsupported( | |
| I, "incompatible result and operand types in a bitcast"); | |
| return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitcast); | |
| } | |
| static void | |
| addMemoryOperands(const MachineMemOperand *MemOp, MachineInstrBuilder &MIB, | |
| MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry &GR, | |
| std::optional<Align> AlignOverride = std::nullopt) { | |
| const SPIRVSubtarget *ST = | |
| static_cast<const SPIRVSubtarget *>(&MIRBuilder.getMF().getSubtarget()); | |
| uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None); | |
| if (MemOp->isVolatile()) | |
| SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile); | |
| if (MemOp->isNonTemporal()) | |
| SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal); | |
| // Aligned memory operand requires the Kernel capability. | |
| if (!ST->isShader() && MemOp->getAlign().value()) | |
| SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned); | |
| [[maybe_unused]] MachineInstr *AliasList = nullptr; | |
| [[maybe_unused]] MachineInstr *NoAliasList = nullptr; | |
| if (ST->canUseExtension(SPIRV::Extension::SPV_INTEL_memory_access_aliasing)) { | |
| if (auto *MD = MemOp->getAAInfo().Scope) { | |
| AliasList = GR.getOrAddMemAliasingINTELInst(MIRBuilder, MD); | |
| if (AliasList) | |
| SpvMemOp |= | |
| static_cast<uint32_t>(SPIRV::MemoryOperand::AliasScopeINTELMask); | |
| } | |
| if (auto *MD = MemOp->getAAInfo().NoAlias) { | |
| NoAliasList = GR.getOrAddMemAliasingINTELInst(MIRBuilder, MD); | |
| if (NoAliasList) | |
| SpvMemOp |= | |
| static_cast<uint32_t>(SPIRV::MemoryOperand::NoAliasINTELMask); | |
| } | |
| } | |
| if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) { | |
| MIB.addImm(SpvMemOp); | |
| if (SpvMemOp & static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned)) | |
| MIB.addImm(AlignOverride.value_or(MemOp->getAlign()).value()); | |
| if (AliasList) | |
| MIB.addUse(AliasList->getOperand(0).getReg()); | |
| if (NoAliasList) | |
| MIB.addUse(NoAliasList->getOperand(0).getReg()); | |
| } | |
| } | |
| static void addMemoryOperands(uint64_t Flags, MachineInstrBuilder &MIB) { | |
| uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None); | |
| if (Flags & MachineMemOperand::Flags::MOVolatile) | |
| SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile); | |
| if (Flags & MachineMemOperand::Flags::MONonTemporal) | |
| SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal); | |
| if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) | |
| MIB.addImm(SpvMemOp); | |
| } | |
| bool SPIRVInstructionSelector::selectLoad(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0; | |
| Register Ptr = I.getOperand(1 + OpOffset).getReg(); | |
| auto *PtrDef = getVRegDef(*MRI, Ptr); | |
| auto *IntPtrDef = dyn_cast<GIntrinsic>(PtrDef); | |
| if (IntPtrDef && | |
| (IntPtrDef->getIntrinsicID() == Intrinsic::spv_resource_getbasepointer || | |
| IntPtrDef->getIntrinsicID() == Intrinsic::spv_resource_getpointer)) { | |
| Register HandleReg = IntPtrDef->getOperand(2).getReg(); | |
| SPIRVTypeInst HandleType = GR.getSPIRVTypeForVReg(HandleReg); | |
| if (HandleType->getOpcode() == SPIRV::OpTypeImage) { | |
| Register NewHandleReg = | |
| MRI->createVirtualRegister(MRI->getRegClass(HandleReg)); | |
| auto *HandleDef = cast<GIntrinsic>(getVRegDef(*MRI, HandleReg)); | |
| if (!loadHandleBeforePosition(NewHandleReg, HandleType, *HandleDef, I)) { | |
| return false; | |
| } | |
| Register IdxReg = IntPtrDef->getOperand(3).getReg(); | |
| return generateImageReadOrFetch(ResVReg, ResType, NewHandleReg, IdxReg, | |
| I.getDebugLoc(), I); | |
| } | |
| } | |
| MachineIRBuilder MIRBuilder(I); | |
| if (I.getNumMemOperands()) { | |
| const MachineMemOperand *MemOp = *I.memoperands_begin(); | |
| if (MemOp->isAtomic()) | |
| return selectAtomicLoad(ResVReg, ResType, I); | |
| } | |
| auto MIB = MIRBuilder.buildInstr(SPIRV::OpLoad) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(Ptr); | |
| if (!I.getNumMemOperands()) { | |
| assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS || | |
| I.getOpcode() == | |
| TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS); | |
| addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB); | |
| } else { | |
| addMemoryOperands(*I.memoperands_begin(), MIB, MIRBuilder, GR); | |
| } | |
| MIB.constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| Register SPIRVInstructionSelector::createPtrSizedIntReg( | |
| MachineIRBuilder &MIRBuilder) const { | |
| SPIRVTypeInst IntType = | |
| GR.getOrCreateSPIRVIntegerType(GR.getPointerSize(), MIRBuilder); | |
| Register Reg = | |
| MRI->createGenericVirtualRegister(LLT::scalar(GR.getPointerSize())); | |
| MRI->setRegClass(Reg, GR.getRegClass(IntType)); | |
| GR.assignSPIRVTypeToVReg(IntType, Reg, MIRBuilder.getMF()); | |
| return Reg; | |
| } | |
| Register | |
| SPIRVInstructionSelector::convertPtrToInt(Register PtrVal, | |
| MachineIRBuilder &MIRBuilder) const { | |
| SPIRVTypeInst IntType = | |
| GR.getOrCreateSPIRVIntegerType(GR.getPointerSize(), MIRBuilder); | |
| Register IntReg = createPtrSizedIntReg(MIRBuilder); | |
| MIRBuilder.buildInstr(SPIRV::OpConvertPtrToU) | |
| .addDef(IntReg) | |
| .addUse(GR.getSPIRVTypeID(IntType)) // Result type | |
| .addUse(PtrVal) // Pointer operand | |
| .constrainAllUses(TII, TRI, RBI); | |
| return IntReg; | |
| } | |
| Register SPIRVInstructionSelector::castPtrToPtrToInt( | |
| Register Ptr, SPIRV::StorageClass::StorageClass SC, | |
| MachineIRBuilder &MIRBuilder) const { | |
| SPIRVTypeInst IntType = | |
| GR.getOrCreateSPIRVIntegerType(GR.getPointerSize(), MIRBuilder); | |
| SPIRVTypeInst PtrType = | |
| GR.getOrCreateSPIRVPointerType(IntType, MIRBuilder, SC); | |
| Register CastedPtr = | |
| MRI->createGenericVirtualRegister(LLT::scalar(GR.getPointerSize())); | |
| MRI->setRegClass(CastedPtr, GR.getRegClass(PtrType)); | |
| GR.assignSPIRVTypeToVReg(PtrType, CastedPtr, MIRBuilder.getMF()); | |
| MIRBuilder.buildInstr(SPIRV::OpBitcast) | |
| .addDef(CastedPtr) | |
| .addUse(GR.getSPIRVTypeID(PtrType)) | |
| .addUse(Ptr) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return CastedPtr; | |
| } | |
| bool SPIRVInstructionSelector::selectAtomicPtrValue( | |
| Register ResVReg, SPIRVTypeInst ResType, MachineIRBuilder &MIRBuilder, | |
| function_ref<Register(SPIRVTypeInst IntType)> EmitAtomic) const { | |
| // Pointer-typed atomics are lowered by bitcasting the Ptr operand to a | |
| // pointer to an integer of the same size as the pointer, so that the actual | |
| // atomic instruction operates on integers as required by the spec. Value | |
| // operands and results are converted with OpConvertPtrToU/OpConvertUToPtr. | |
| unsigned PtrSize = GR.getPointerSize(); | |
| SPIRVTypeInst IntType = GR.getOrCreateSPIRVIntegerType(PtrSize, MIRBuilder); | |
| Register IntResult = EmitAtomic(IntType); | |
| if (IntResult.isValid()) | |
| MIRBuilder.buildInstr(SPIRV::OpConvertUToPtr) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(IntResult) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectAtomicLoad(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| LLVMContext &Context = I.getMF()->getFunction().getContext(); | |
| unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0; | |
| Register Ptr = I.getOperand(1 + OpOffset).getReg(); | |
| if (!ResType.isTypeIntOrFloat() && !ResType.isTypePtr()) | |
| return diagnoseUnsupported( | |
| I, "Lowering to SPIR-V of atomic load is only " | |
| "allowed for integer, floating point or pointer types"); | |
| assert(I.getNumMemOperands()); | |
| const MachineMemOperand &MemOp = **I.memoperands_begin(); | |
| assert(MemOp.isAtomic()); | |
| uint32_t Scope = static_cast<uint32_t>( | |
| getMemScope(STI.getTargetTriple(), Context, MemOp.getSyncScopeID())); | |
| Register ScopeReg = buildI32Constant(Scope, I); | |
| AtomicOrdering AO = MemOp.getSuccessOrdering(); | |
| uint32_t StorageClass = static_cast<uint32_t>(getMemSemanticsForStorageClass( | |
| addressSpaceToStorageClass(MemOp.getAddrSpace(), STI))); | |
| uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO)); | |
| if (MemOp.isVolatile() && STI.getTargetTriple().isVulkanOS()) | |
| MemSem |= static_cast<uint32_t>(SPIRV::MemorySemantics::Volatile); | |
| uint32_t Sem = getMemSemanticsWithStorageClass(STI.getTargetTriple(), MemSem, | |
| StorageClass); | |
| Register MemSemReg = buildI32Constant(Sem, I); | |
| MachineIRBuilder MIRBuilder(I); | |
| if (ResType.isTypePtr()) { | |
| if (!STI.isPhysicalSPIRV()) | |
| return diagnoseUnsupported( | |
| I, "Lowering to SPIR-V of atomic load is only " | |
| "allowed for pointer types for physical addressing model"); | |
| // If data to load is a pointer type we bitcast the Ptr parameter to pointer | |
| // to an integer type of the same size as the pointer size and then generate | |
| // OpAtomicLoad the return value of that OpAtomicLoad is an integer that is | |
| // converted back to a pointer type using OpConvertUToPtr. | |
| SPIRV::StorageClass::StorageClass SC = | |
| addressSpaceToStorageClass(MemOp.getAddrSpace(), STI); | |
| return selectAtomicPtrValue( | |
| ResVReg, ResType, MIRBuilder, [&](SPIRVTypeInst IntType) { | |
| Register CastedPtr = castPtrToPtrToInt(Ptr, SC, MIRBuilder); | |
| Register IntResult = createPtrSizedIntReg(MIRBuilder); | |
| MIRBuilder.buildInstr(SPIRV::OpAtomicLoad) | |
| .addDef(IntResult) | |
| .addUse(GR.getSPIRVTypeID(IntType)) | |
| .addUse(CastedPtr) | |
| .addUse(ScopeReg) | |
| .addUse(MemSemReg) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return IntResult; | |
| }); | |
| } | |
| auto AtomicLoad = MIRBuilder.buildInstr(SPIRV::OpAtomicLoad) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(Ptr) | |
| .addUse(ScopeReg) | |
| .addUse(MemSemReg); | |
| AtomicLoad.constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectStore(MachineInstr &I) const { | |
| unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0; | |
| Register StoreVal = I.getOperand(0 + OpOffset).getReg(); | |
| Register Ptr = I.getOperand(1 + OpOffset).getReg(); | |
| auto *PtrDef = getVRegDef(*MRI, Ptr); | |
| auto *IntPtrDef = dyn_cast<GIntrinsic>(PtrDef); | |
| if (IntPtrDef && | |
| (IntPtrDef->getIntrinsicID() == Intrinsic::spv_resource_getbasepointer || | |
| IntPtrDef->getIntrinsicID() == Intrinsic::spv_resource_getpointer)) { | |
| Register HandleReg = IntPtrDef->getOperand(2).getReg(); | |
| Register NewHandleReg = | |
| MRI->createVirtualRegister(MRI->getRegClass(HandleReg)); | |
| auto *HandleDef = cast<GIntrinsic>(getVRegDef(*MRI, HandleReg)); | |
| SPIRVTypeInst HandleType = GR.getSPIRVTypeForVReg(HandleReg); | |
| if (!loadHandleBeforePosition(NewHandleReg, HandleType, *HandleDef, I)) { | |
| return false; | |
| } | |
| Register IdxReg = IntPtrDef->getOperand(3).getReg(); | |
| if (HandleType->getOpcode() == SPIRV::OpTypeImage) { | |
| SPIRVTypeInst SampledType = | |
| GR.getSPIRVTypeForVReg(HandleType->getOperand(1).getReg()); | |
| SPIRVTypeInst StoreValCompType = | |
| GR.getScalarOrVectorComponentType(GR.getSPIRVTypeForVReg(StoreVal)); | |
| if (StoreValCompType && StoreValCompType != SampledType) { | |
| // A wide element (e.g. int64_t2) is emulated with a narrower packed | |
| // image. This bitcasts the value to match the format. | |
| SPIRVTypeInst PackedType = widenTypeToVec4(SampledType, I); | |
| Register PackedReg = | |
| MRI->createVirtualRegister(GR.getRegClass(PackedType)); | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpBitcast)) | |
| .addDef(PackedReg) | |
| .addUse(GR.getSPIRVTypeID(PackedType)) | |
| .addUse(StoreVal) | |
| .constrainAllUses(TII, TRI, RBI); | |
| StoreVal = PackedReg; | |
| } | |
| auto BMI = BuildMI(*I.getParent(), I, I.getDebugLoc(), | |
| TII.get(SPIRV::OpImageWrite)) | |
| .addUse(NewHandleReg) | |
| .addUse(IdxReg) | |
| .addUse(StoreVal); | |
| const llvm::Type *LLVMHandleType = GR.getTypeForSPIRVType(HandleType); | |
| if (sampledTypeIsSignedInteger(LLVMHandleType)) | |
| BMI.addImm(0x1000); // SignExtend | |
| BMI.constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| } | |
| SPIRVTypeInst PointeeTy = GR.getPointeeType(GR.getSPIRVTypeForVReg(Ptr)); | |
| SPIRVTypeInst StoreTy = GR.getSPIRVTypeForVReg(StoreVal); | |
| if (PointeeTy && PointeeTy->getOpcode() == SPIRV::OpTypeVectorIdEXT && | |
| StoreTy->getOpcode() != SPIRV::OpTypeVectorIdEXT && | |
| GR.getScalarOrVectorComponentCount(PointeeTy) == 1) { | |
| MachineInstr *StoreValDef = getVRegDef(*MRI, StoreVal); | |
| Register Reg = StoreValDef->getOperand(0).getReg(); | |
| if (auto FC = GFConstant::getConstant(Reg, *MRI)) | |
| StoreVal = | |
| GR.getOrCreateConstVector(FC->getScalarValue(), I, PointeeTy, TII); | |
| else if (auto IC = GIConstant::getConstant(Reg, *MRI)) | |
| StoreVal = | |
| GR.getOrCreateConstVector(IC->getScalarValue(), I, PointeeTy, TII); | |
| else | |
| llvm_unreachable("Unexpected <1 x T> Store type!"); | |
| } | |
| if (I.getNumMemOperands()) { | |
| const MachineMemOperand *MemOp = *I.memoperands_begin(); | |
| if (MemOp->isAtomic()) | |
| return selectAtomicStore(I); | |
| } | |
| // Stores into a read-only storage class produce invalid SPIR-V. Reject such | |
| // input with a diagnostic rather than silently emitting an OpStore that | |
| // validation rejects. | |
| SPIRV::StorageClass::StorageClass PtrSC = GR.getPointerStorageClass(Ptr); | |
| if (PtrSC == SPIRV::StorageClass::UniformConstant || | |
| PtrSC == SPIRV::StorageClass::Input || | |
| PtrSC == SPIRV::StorageClass::PushConstant) | |
| return diagnoseUnsupported( | |
| I, "store into a read-only SPIR-V storage class is not allowed"); | |
| MachineIRBuilder MIRBuilder(I); | |
| auto MIB = MIRBuilder.buildInstr(SPIRV::OpStore).addUse(Ptr).addUse(StoreVal); | |
| if (!I.getNumMemOperands()) { | |
| assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS || | |
| I.getOpcode() == | |
| TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS); | |
| addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB); | |
| } else { | |
| addMemoryOperands(*I.memoperands_begin(), MIB, MIRBuilder, GR); | |
| } | |
| MIB.constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectAtomicStore(MachineInstr &I) const { | |
| LLVMContext &Context = I.getMF()->getFunction().getContext(); | |
| unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0; | |
| Register StoreVal = I.getOperand(0 + OpOffset).getReg(); | |
| Register Ptr = I.getOperand(1 + OpOffset).getReg(); | |
| SPIRVTypeInst PtrType = GR.getSPIRVTypeForVReg(Ptr); | |
| SPIRVTypeInst PointeeType = GR.getPointeeType(PtrType); | |
| // For an untyped pointer the data type is the stored value's type. | |
| if (!PointeeType && PtrType && | |
| PtrType->getOpcode() == SPIRV::OpTypeUntypedPointerKHR) | |
| PointeeType = GR.getSPIRVTypeForVReg(StoreVal); | |
| if (!PointeeType) | |
| return diagnoseUnsupported(I, | |
| "Lowering to SPIR-V of atomic store is only " | |
| "allowed for integer or floating point types"); | |
| assert(I.getNumMemOperands()); | |
| const MachineMemOperand &MemOp = **I.memoperands_begin(); | |
| assert(MemOp.isAtomic()); | |
| uint32_t Scope = static_cast<uint32_t>( | |
| getMemScope(STI.getTargetTriple(), Context, MemOp.getSyncScopeID())); | |
| Register ScopeReg = buildI32Constant(Scope, I); | |
| AtomicOrdering AO = MemOp.getSuccessOrdering(); | |
| uint32_t StorageClass = static_cast<uint32_t>(getMemSemanticsForStorageClass( | |
| addressSpaceToStorageClass(MemOp.getAddrSpace(), STI))); | |
| uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO)); | |
| if (MemOp.isVolatile() && STI.getTargetTriple().isVulkanOS()) | |
| MemSem |= static_cast<uint32_t>(SPIRV::MemorySemantics::Volatile); | |
| uint32_t Sem = getMemSemanticsWithStorageClass(STI.getTargetTriple(), MemSem, | |
| StorageClass); | |
| Register MemSemReg = buildI32Constant(Sem, I); | |
| MachineIRBuilder MIRBuilder(I); | |
| if (PointeeType.isTypePtr()) { | |
| if (!STI.isPhysicalSPIRV()) | |
| return diagnoseUnsupported( | |
| I, "Lowering to SPIR-V of atomic store is only " | |
| "allowed for pointer types for physical addressing model"); | |
| // If data to store is a pointer type we cast it to an integer type of the | |
| // same size as the pointer size using OpConvertPtrToU, bitcast Ptr | |
| // parameter to pointer to integer type and then generate OpAtomicStore | |
| // with casted values as required by spec. | |
| SPIRV::StorageClass::StorageClass SC = | |
| addressSpaceToStorageClass(MemOp.getAddrSpace(), STI); | |
| return selectAtomicPtrValue( | |
| Register(), SPIRVTypeInst(), MIRBuilder, [&](SPIRVTypeInst IntType) { | |
| Register ValueAsInt = convertPtrToInt(StoreVal, MIRBuilder); | |
| Register CastedPtr = castPtrToPtrToInt(Ptr, SC, MIRBuilder); | |
| MIRBuilder.buildInstr(SPIRV::OpAtomicStore) | |
| .addUse(CastedPtr) | |
| .addUse(ScopeReg) | |
| .addUse(MemSemReg) | |
| .addUse(ValueAsInt) | |
| .constrainAllUses(TII, TRI, RBI); | |
| // Stores produce no result, so no OpConvertUToPtr is needed. | |
| return Register(); | |
| }); | |
| } | |
| if (!PointeeType.isTypeIntOrFloat()) | |
| return diagnoseUnsupported(I, | |
| "Lowering to SPIR-V of atomic store is only " | |
| "allowed for integer or floating point types"); | |
| auto AtomicStore = MIRBuilder.buildInstr(SPIRV::OpAtomicStore) | |
| .addUse(Ptr) | |
| .addUse(ScopeReg) | |
| .addUse(MemSemReg) | |
| .addUse(StoreVal); | |
| AtomicStore.constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectMaskedGather(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| assert(I.getNumExplicitDefs() == 1 && "Expected single def for gather"); | |
| // Operand indices: | |
| // 0: result (def) | |
| // 1: intrinsic ID | |
| // 2: vector of pointers | |
| // 3: alignment (i32 immediate) | |
| // 4: mask (vector of i1) | |
| // 5: passthru/fill value | |
| const Register PtrsReg = I.getOperand(2).getReg(); | |
| const uint32_t Alignment = I.getOperand(3).getImm(); | |
| const Register MaskReg = I.getOperand(4).getReg(); | |
| const Register PassthruReg = I.getOperand(5).getReg(); | |
| const Register AlignmentReg = buildI32Constant(Alignment, I); | |
| MachineBasicBlock &BB = *I.getParent(); | |
| auto MIB = | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpMaskedGatherINTEL)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(PtrsReg) | |
| .addUse(AlignmentReg) | |
| .addUse(MaskReg) | |
| .addUse(PassthruReg); | |
| MIB.constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectMaskedScatter(MachineInstr &I) const { | |
| assert(I.getNumExplicitDefs() == 0 && "Expected no defs for scatter"); | |
| // Operand indices (no explicit defs): | |
| // 0: intrinsic ID | |
| // 1: value vector | |
| // 2: vector of pointers | |
| // 3: alignment (i32 immediate) | |
| // 4: mask (vector of i1) | |
| const Register ValuesReg = I.getOperand(1).getReg(); | |
| const Register PtrsReg = I.getOperand(2).getReg(); | |
| const uint32_t Alignment = I.getOperand(3).getImm(); | |
| const Register MaskReg = I.getOperand(4).getReg(); | |
| const Register AlignmentReg = buildI32Constant(Alignment, I); | |
| MachineBasicBlock &BB = *I.getParent(); | |
| auto MIB = | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpMaskedScatterINTEL)) | |
| .addUse(PtrsReg) | |
| .addUse(AlignmentReg) | |
| .addUse(MaskReg) | |
| .addUse(ValuesReg); | |
| MIB.constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::diagnoseUnsupported(const MachineInstr &I, | |
| const Twine &Msg) const { | |
| const Function &F = I.getMF()->getFunction(); | |
| F.getContext().diagnose( | |
| DiagnosticInfoUnsupported(F, Msg, I.getDebugLoc(), DS_Error)); | |
| return false; | |
| } | |
| bool SPIRVInstructionSelector::selectStackSave(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array)) | |
| return diagnoseUnsupported( | |
| I, "llvm.stacksave intrinsic: this instruction requires the following " | |
| "SPIR-V extension: SPV_INTEL_variable_length_array"); | |
| MachineBasicBlock &BB = *I.getParent(); | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSaveMemoryINTEL)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectStackRestore(MachineInstr &I) const { | |
| if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array)) | |
| return diagnoseUnsupported( | |
| I, | |
| "llvm.stackrestore intrinsic: this instruction requires the following " | |
| "SPIR-V extension: SPV_INTEL_variable_length_array"); | |
| if (!I.getOperand(0).isReg()) | |
| return false; | |
| MachineBasicBlock &BB = *I.getParent(); | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpRestoreMemoryINTEL)) | |
| .addUse(I.getOperand(0).getReg()) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| Register | |
| SPIRVInstructionSelector::getOrCreateMemSetGlobal(MachineInstr &I) const { | |
| MachineIRBuilder MIRBuilder(I); | |
| assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); | |
| // TODO: check if we have such GV, add init, use buildGlobalVariable. | |
| unsigned Num = getIConstVal(I.getOperand(2).getReg(), MRI); | |
| Function &CurFunction = GR.CurMF->getFunction(); | |
| Type *LLVMArrTy = | |
| ArrayType::get(IntegerType::get(CurFunction.getContext(), 8), Num); | |
| GlobalVariable *GV = new GlobalVariable(*CurFunction.getParent(), LLVMArrTy, | |
| true, GlobalValue::InternalLinkage, | |
| Constant::getNullValue(LLVMArrTy)); | |
| Type *ValTy = Type::getInt8Ty(I.getMF()->getFunction().getContext()); | |
| Type *ArrTy = ArrayType::get(ValTy, Num); | |
| SPIRVTypeInst VarTy = GR.getOrCreateSPIRVPointerType( | |
| ArrTy, MIRBuilder, SPIRV::StorageClass::UniformConstant); | |
| SPIRVTypeInst SpvArrTy = GR.getOrCreateSPIRVType( | |
| ArrTy, MIRBuilder, SPIRV::AccessQualifier::None, false); | |
| unsigned Val = getIConstVal(I.getOperand(1).getReg(), MRI); | |
| Register Const = GR.getOrCreateConstIntArray(Val, Num, I, SpvArrTy, TII); | |
| Register VarReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); | |
| // With SPV_KHR_untyped_pointers enabled, getOrCreateSPIRVPointerType returns | |
| // an untyped pointer type. An untyped pointer result type is only legal on | |
| // OpUntypedVariableKHR, not on OpVariable. | |
| // Pick the matching opcode/operands so the synthesized constant global is | |
| // valid SPIR-V. | |
| const bool IsUntyped = VarTy->getOpcode() == SPIRV::OpTypeUntypedPointerKHR; | |
| auto MIBVar = BuildMI(*I.getParent(), I, I.getDebugLoc(), | |
| TII.get(IsUntyped ? SPIRV::OpUntypedVariableKHR | |
| : SPIRV::OpVariable)) | |
| .addDef(VarReg) | |
| .addUse(GR.getSPIRVTypeID(VarTy)) | |
| .addImm(SPIRV::StorageClass::UniformConstant); | |
| if (IsUntyped) | |
| MIBVar.addUse(GR.getSPIRVTypeID(SpvArrTy)); // Data Type | |
| MIBVar.addUse(Const); // Initializer | |
| MIBVar.constrainAllUses(TII, TRI, RBI); | |
| GR.add(GV, MIBVar); | |
| GR.addGlobalObject(GV, GR.CurMF, VarReg); | |
| buildOpDecorate(VarReg, I, TII, SPIRV::Decoration::Constant, {}); | |
| return VarReg; | |
| } | |
| bool SPIRVInstructionSelector::selectCopyMemory(MachineInstr &I, | |
| Register SrcReg) const { | |
| MachineBasicBlock &BB = *I.getParent(); | |
| Register DstReg = I.getOperand(0).getReg(); | |
| const bool IsLogical = STI.isLogicalSPIRV(); | |
| if (IsLogical) { | |
| SPIRVTypeInst DstTy = GR.getSPIRVTypeForVReg(DstReg); | |
| SPIRVTypeInst SrcTy = GR.getSPIRVTypeForVReg(SrcReg); | |
| if (GR.getPointeeType(DstTy) != GR.getPointeeType(SrcTy)) | |
| return diagnoseUnsupported( | |
| I, "OpCopyMemory requires operands to have the same type"); | |
| uint64_t CopySize = getIConstVal(I.getOperand(2).getReg(), MRI); | |
| SPIRVTypeInst PointeeTy = GR.getPointeeType(DstTy); | |
| const Type *LLVMPointeeTy = GR.getTypeForSPIRVType(PointeeTy); | |
| if (!LLVMPointeeTy) | |
| return diagnoseUnsupported( | |
| I, "Unable to determine pointee type size for OpCopyMemory"); | |
| const DataLayout &DL = I.getMF()->getFunction().getDataLayout(); | |
| if (CopySize != DL.getTypeStoreSize(const_cast<Type *>(LLVMPointeeTy))) | |
| return diagnoseUnsupported( | |
| I, "OpCopyMemory requires the size to match the pointee type size"); | |
| } | |
| const unsigned Opcode = | |
| IsLogical ? SPIRV::OpCopyMemory : SPIRV::OpCopyMemorySized; | |
| auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) | |
| .addUse(DstReg) | |
| .addUse(SrcReg); | |
| if (!IsLogical) | |
| MIB.addUse(I.getOperand(2).getReg()); | |
| if (I.getNumMemOperands()) { | |
| MachineIRBuilder MIRBuilder(I); | |
| const MachineMemOperand *DstMemOp = *I.memoperands_begin(); | |
| const Align DstAlign = DstMemOp->getAlign(); | |
| Align SrcAlign = DstAlign; | |
| // Copies carry destination and source MMOs; memset carries only the | |
| // destination. Shader memory operands do not include alignment. | |
| if (I.getNumMemOperands() > 1 && !STI.isShader()) | |
| SrcAlign = (*std::next(I.memoperands_begin()))->getAlign(); | |
| // A single mask applies to both pointers. SPIR-V 1.4 allows separate | |
| // destination and source masks, preserving their individual alignments. | |
| if (DstAlign != SrcAlign && STI.isAtLeastSPIRVVer(VersionTuple(1, 4))) { | |
| // Preserve the existing flags and metadata on both accesses. | |
| addMemoryOperands(DstMemOp, MIB, MIRBuilder, GR); | |
| addMemoryOperands(DstMemOp, MIB, MIRBuilder, GR, SrcAlign); | |
| } else { | |
| addMemoryOperands(DstMemOp, MIB, MIRBuilder, GR, | |
| std::min(DstAlign, SrcAlign)); | |
| } | |
| } | |
| MIB.constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectMemOperation(Register ResVReg, | |
| MachineInstr &I) const { | |
| // Zero-sized memcpy/memmove/memset are no-ops. | |
| Register SizeReg = I.getOperand(2).getReg(); | |
| if (MachineInstr *SizeDef = getDefInstrMaybeConstant(SizeReg, MRI); | |
| SizeDef && SizeDef->getOpcode() == TargetOpcode::G_CONSTANT && | |
| getIConstVal(SizeReg, MRI) == 0) | |
| return true; | |
| Register SrcReg = I.getOperand(1).getReg(); | |
| if (I.getOpcode() == TargetOpcode::G_MEMSET || | |
| I.getOpcode() == TargetOpcode::G_MEMSET_INLINE) { | |
| Register VarReg = getOrCreateMemSetGlobal(I); | |
| if (!VarReg.isValid()) | |
| return false; | |
| Type *ValTy = Type::getInt8Ty(I.getMF()->getFunction().getContext()); | |
| SPIRVTypeInst SourceTy = GR.getOrCreateSPIRVPointerType( | |
| ValTy, I, SPIRV::StorageClass::UniformConstant); | |
| SrcReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); | |
| if (!selectOpWithSrcs(SrcReg, SourceTy, I, {VarReg}, SPIRV::OpBitcast)) | |
| return false; | |
| } | |
| if (!selectCopyMemory(I, SrcReg)) | |
| return false; | |
| if (ResVReg.isValid() && ResVReg != I.getOperand(0).getReg()) | |
| if (!BuildCOPY(ResVReg, I.getOperand(0).getReg(), I)) | |
| return false; | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectAtomicRMW(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I, | |
| unsigned NewOpcode, | |
| unsigned NegateOpcode) const { | |
| assert(I.hasOneMemOperand()); | |
| const MachineMemOperand *MemOp = *I.memoperands_begin(); | |
| uint32_t Scope = static_cast<uint32_t>( | |
| getMemScope(STI.getTargetTriple(), GR.CurMF->getFunction().getContext(), | |
| MemOp->getSyncScopeID())); | |
| Register ScopeReg = buildI32Constant(Scope, I); | |
| Register Ptr = I.getOperand(1).getReg(); | |
| uint32_t ScSem = static_cast<uint32_t>( | |
| getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr))); | |
| AtomicOrdering AO = MemOp->getSuccessOrdering(); | |
| uint32_t OrderSem = static_cast<uint32_t>(getMemSemantics(AO)); | |
| Register MemSemReg = buildI32Constant( | |
| getMemSemanticsWithStorageClass(STI.getTargetTriple(), OrderSem, ScSem), | |
| I); | |
| Register ValueReg = I.getOperand(2).getReg(); | |
| if (NegateOpcode != 0) { | |
| // Translation with negative value operand is requested | |
| Register TmpReg = createVirtualRegister(ResType, &GR, MRI, MRI->getMF()); | |
| if (!selectOpWithSrcs(TmpReg, ResType, I, {ValueReg}, NegateOpcode)) | |
| return false; | |
| ValueReg = TmpReg; | |
| } | |
| if (ResType.isTypePtr()) { | |
| if (NewOpcode != SPIRV::OpAtomicExchange) | |
| return diagnoseUnsupported( | |
| I, "Lowering to SPIR-V of this atomic operation is not " | |
| "allowed for pointer types"); | |
| if (!STI.isPhysicalSPIRV()) | |
| return diagnoseUnsupported( | |
| I, "Lowering to SPIR-V of atomic exchange is only " | |
| "allowed for pointer types for physical addressing model"); | |
| // If the exchanged value is a pointer type we convert the value operand to | |
| // an integer type of the same size as the pointer size using | |
| // OpConvertPtrToU, bitcast the Ptr parameter to pointer to integer type and | |
| // then generate OpAtomicExchange on integers. The integer result is | |
| // converted back to a pointer type using OpConvertUToPtr, similar to atomic | |
| // load and store. | |
| MachineIRBuilder MIRBuilder(I); | |
| SPIRV::StorageClass::StorageClass SC = GR.getPointerStorageClass(Ptr); | |
| return selectAtomicPtrValue( | |
| ResVReg, ResType, MIRBuilder, [&](SPIRVTypeInst IntType) { | |
| Register ValueAsInt = convertPtrToInt(ValueReg, MIRBuilder); | |
| Register CastedPtr = castPtrToPtrToInt(Ptr, SC, MIRBuilder); | |
| Register ExchangeResReg = createPtrSizedIntReg(MIRBuilder); | |
| MIRBuilder.buildInstr(SPIRV::OpAtomicExchange) | |
| .addDef(ExchangeResReg) | |
| .addUse(GR.getSPIRVTypeID(IntType)) | |
| .addUse(CastedPtr) | |
| .addUse(ScopeReg) | |
| .addUse(MemSemReg) | |
| .addUse(ValueAsInt) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return ExchangeResReg; | |
| }); | |
| } | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(NewOpcode)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(Ptr) | |
| .addUse(ScopeReg) | |
| .addUse(MemSemReg) | |
| .addUse(ValueReg) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectUnmergeValues(MachineInstr &I) const { | |
| unsigned ArgI = I.getNumOperands() - 1; | |
| Register SrcReg = | |
| I.getOperand(ArgI).isReg() ? I.getOperand(ArgI).getReg() : Register(0); | |
| SPIRVTypeInst SrcType = | |
| SrcReg.isValid() ? GR.getSPIRVTypeForVReg(SrcReg) : nullptr; | |
| if (!SrcType || !isVectorType(SrcType)) | |
| report_fatal_error( | |
| "cannot select G_UNMERGE_VALUES with a non-vector argument"); | |
| SPIRVTypeInst ScalarType = GR.getScalarOrVectorComponentType(SrcType); | |
| MachineBasicBlock &BB = *I.getParent(); | |
| unsigned CurrentIndex = 0; | |
| for (unsigned i = 0; i < I.getNumDefs(); ++i) { | |
| Register ResVReg = I.getOperand(i).getReg(); | |
| SPIRVTypeInst ResType = GR.getSPIRVTypeForVReg(ResVReg); | |
| if (!ResType) { | |
| LLT ResLLT = MRI->getType(ResVReg); | |
| assert(ResLLT.isValid()); | |
| if (ResLLT.isVector()) { | |
| ResType = GR.getOrCreateSPIRVVectorType( | |
| ScalarType, ResLLT.getNumElements(), I, TII); | |
| } else { | |
| ResType = ScalarType; | |
| } | |
| MRI->setRegClass(ResVReg, GR.getRegClass(ResType)); | |
| GR.assignSPIRVTypeToVReg(ResType, ResVReg, *GR.CurMF); | |
| } | |
| if (isVectorType(ResType)) { | |
| Register UndefReg = GR.getOrCreateUndef(I, SrcType, TII); | |
| auto MIB = | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorShuffle)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(SrcReg) | |
| .addUse(UndefReg); | |
| unsigned NumElements = GR.getScalarOrVectorComponentCount(ResType); | |
| for (unsigned j = 0; j < NumElements; ++j) { | |
| MIB.addImm(CurrentIndex + j); | |
| } | |
| CurrentIndex += NumElements; | |
| MIB.constrainAllUses(TII, TRI, RBI); | |
| } else { | |
| auto MIB = | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(SrcReg) | |
| .addImm(CurrentIndex); | |
| CurrentIndex++; | |
| MIB.constrainAllUses(TII, TRI, RBI); | |
| } | |
| } | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectFence(MachineInstr &I) const { | |
| AtomicOrdering AO = AtomicOrdering(I.getOperand(0).getImm()); | |
| uint32_t ScSem = STI.isShader() | |
| ? SPIRV::MemorySemantics::UniformMemory | | |
| SPIRV::MemorySemantics::WorkgroupMemory | | |
| SPIRV::MemorySemantics::ImageMemory | |
| : SPIRV::MemorySemantics::WorkgroupMemory | | |
| SPIRV::MemorySemantics::CrossWorkgroupMemory | | |
| SPIRV::MemorySemantics::ImageMemory; | |
| uint32_t MemSem = getMemSemanticsWithStorageClass( | |
| STI.getTargetTriple(), static_cast<uint32_t>(getMemSemantics(AO)), ScSem); | |
| Register MemSemReg = buildI32ConstantInEntryBlock(MemSem, I); | |
| SyncScope::ID Ord = SyncScope::ID(I.getOperand(1).getImm()); | |
| uint32_t Scope = static_cast<uint32_t>(getMemScope( | |
| STI.getTargetTriple(), GR.CurMF->getFunction().getContext(), Ord)); | |
| Register ScopeReg = buildI32ConstantInEntryBlock(Scope, I); | |
| MachineBasicBlock &BB = *I.getParent(); | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpMemoryBarrier)) | |
| .addUse(ScopeReg) | |
| .addUse(MemSemReg) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectOverflowArith(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I, | |
| unsigned Opcode) const { | |
| Type *ResTy = nullptr; | |
| StringRef ResName; | |
| if (!GR.findValueAttrs(&I, ResTy, ResName)) | |
| return diagnoseUnsupported( | |
| I, | |
| "Not enough info to select the arithmetic with overflow instruction"); | |
| if (!ResTy || !ResTy->isStructTy()) | |
| return diagnoseUnsupported(I, | |
| "Expect struct type result for the arithmetic " | |
| "with overflow instruction"); | |
| // "Result Type must be from OpTypeStruct. The struct must have two members, | |
| // and the two members must be the same type." | |
| Type *ResElemTy = cast<StructType>(ResTy)->getElementType(0); | |
| ResTy = StructType::get(ResElemTy, ResElemTy); | |
| // Build SPIR-V types and constant(s) if needed. | |
| MachineIRBuilder MIRBuilder(I); | |
| SPIRVTypeInst StructType = GR.getOrCreateSPIRVType( | |
| ResTy, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false); | |
| assert(I.getNumDefs() > 1 && "Not enought operands"); | |
| SPIRVTypeInst BoolType = GR.getOrCreateSPIRVBoolType(I, TII); | |
| unsigned N = GR.getScalarOrVectorComponentCount(ResType); | |
| if (N > 1 || (isVectorType(ResType) && | |
| STI.canUseExtension(SPIRV::Extension::SPV_EXT_long_vector))) | |
| BoolType = GR.getOrCreateSPIRVVectorType(BoolType, N, I, TII); | |
| Register BoolTypeReg = GR.getSPIRVTypeID(BoolType); | |
| Register ZeroReg = buildZerosVal(ResType, I); | |
| // A new virtual register to store the result struct. | |
| Register StructVReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); | |
| MRI->setRegClass(StructVReg, &SPIRV::IDRegClass); | |
| // Build the result name if needed. | |
| if (ResName.size() > 0) | |
| buildOpName(StructVReg, ResName, MIRBuilder); | |
| // Build the arithmetic with overflow instruction. | |
| MachineBasicBlock &BB = *I.getParent(); | |
| auto MIB = | |
| BuildMI(BB, MIRBuilder.getInsertPt(), I.getDebugLoc(), TII.get(Opcode)) | |
| .addDef(StructVReg) | |
| .addUse(GR.getSPIRVTypeID(StructType)); | |
| for (unsigned i = I.getNumDefs(); i < I.getNumOperands(); ++i) | |
| MIB.addUse(I.getOperand(i).getReg()); | |
| MIB.constrainAllUses(TII, TRI, RBI); | |
| // Build instructions to extract fields of the instruction's result. | |
| // A new virtual register to store the higher part of the result struct. | |
| Register HigherVReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); | |
| MRI->setRegClass(HigherVReg, &SPIRV::iIDRegClass); | |
| for (unsigned i = 0; i < I.getNumDefs(); ++i) { | |
| auto MIB = | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) | |
| .addDef(i == 1 ? HigherVReg : I.getOperand(i).getReg()) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(StructVReg) | |
| .addImm(i); | |
| MIB.constrainAllUses(TII, TRI, RBI); | |
| } | |
| // Build boolean value from the higher part. | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual)) | |
| .addDef(I.getOperand(1).getReg()) | |
| .addUse(BoolTypeReg) | |
| .addUse(HigherVReg) | |
| .addUse(ZeroReg) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectAtomicCmpXchg(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| assert(isa<GIntrinsic>(I) && | |
| "selectAtomicCmpXchg only handles the spv_cmpxchg intrinsic"); | |
| Register Ptr = I.getOperand(2).getReg(); | |
| Register ScopeReg = I.getOperand(5).getReg(); | |
| Register MemSemEqReg = I.getOperand(6).getReg(); | |
| Register MemSemNeqReg = I.getOperand(7).getReg(); | |
| Register Cmp = I.getOperand(3).getReg(); | |
| Register Val = I.getOperand(4).getReg(); | |
| SPIRVTypeInst SpvValTy = GR.getSPIRVTypeForVReg(Val); | |
| Register ACmpRes = createVirtualRegister(SpvValTy, &GR, MRI, *I.getMF()); | |
| const DebugLoc &DL = I.getDebugLoc(); | |
| BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpAtomicCompareExchange)) | |
| .addDef(ACmpRes) | |
| .addUse(GR.getSPIRVTypeID(SpvValTy)) | |
| .addUse(Ptr) | |
| .addUse(ScopeReg) | |
| .addUse(MemSemEqReg) | |
| .addUse(MemSemNeqReg) | |
| .addUse(Val) | |
| .addUse(Cmp) | |
| .constrainAllUses(TII, TRI, RBI); | |
| SPIRVTypeInst BoolTy = GR.getOrCreateSPIRVBoolType(I, TII); | |
| Register CmpSuccReg = createVirtualRegister(BoolTy, &GR, MRI, *I.getMF()); | |
| BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpIEqual)) | |
| .addDef(CmpSuccReg) | |
| .addUse(GR.getSPIRVTypeID(BoolTy)) | |
| .addUse(ACmpRes) | |
| .addUse(Cmp) | |
| .constrainAllUses(TII, TRI, RBI); | |
| Register TmpReg = createVirtualRegister(ResType, &GR, MRI, *I.getMF()); | |
| BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert)) | |
| .addDef(TmpReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(ACmpRes) | |
| .addUse(GR.getOrCreateUndef(I, ResType, TII)) | |
| .addImm(0) | |
| .constrainAllUses(TII, TRI, RBI); | |
| BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(CmpSuccReg) | |
| .addUse(TmpReg) | |
| .addImm(1) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| static bool isUSMStorageClass(SPIRV::StorageClass::StorageClass SC) { | |
| switch (SC) { | |
| case SPIRV::StorageClass::DeviceOnlyINTEL: | |
| case SPIRV::StorageClass::HostOnlyINTEL: | |
| return true; | |
| default: | |
| return false; | |
| } | |
| } | |
| // Returns true ResVReg is referred only from global vars and OpName's. | |
| static bool isASCastInGVar(MachineRegisterInfo *MRI, Register ResVReg) { | |
| bool IsGRef = false; | |
| bool IsAllowedRefs = | |
| llvm::all_of(MRI->use_instructions(ResVReg), [&IsGRef](auto const &It) { | |
| unsigned Opcode = It.getOpcode(); | |
| if (Opcode == SPIRV::OpConstantComposite || | |
| Opcode == SPIRV::OpSpecConstantComposite || | |
| Opcode == SPIRV::OpVariable || | |
| Opcode == SPIRV::OpUntypedVariableKHR || | |
| isSpvIntrinsic(It, Intrinsic::spv_init_global)) | |
| return IsGRef = true; | |
| return Opcode == SPIRV::OpName; | |
| }); | |
| return IsAllowedRefs && IsGRef; | |
| } | |
| Register SPIRVInstructionSelector::getUcharPtrTypeReg( | |
| MachineInstr &I, SPIRV::StorageClass::StorageClass SC) const { | |
| return GR.getSPIRVTypeID(GR.getOrCreateSPIRVPointerType( | |
| Type::getInt8Ty(I.getMF()->getFunction().getContext()), I, SC)); | |
| } | |
| MachineInstrBuilder | |
| SPIRVInstructionSelector::buildSpecConstantOp(MachineInstr &I, Register Dest, | |
| Register Src, Register DestType, | |
| uint32_t Opcode) const { | |
| return BuildMI(*I.getParent(), I, I.getDebugLoc(), | |
| TII.get(SPIRV::OpSpecConstantOp)) | |
| .addDef(Dest) | |
| .addUse(DestType) | |
| .addImm(Opcode) | |
| .addUse(Src); | |
| } | |
| MachineInstrBuilder | |
| SPIRVInstructionSelector::buildConstGenericPtr(MachineInstr &I, Register SrcPtr, | |
| SPIRVTypeInst SrcPtrTy) const { | |
| SPIRVTypeInst GenericPtrTy = | |
| GR.changePointerStorageClass(SrcPtrTy, SPIRV::StorageClass::Generic, I); | |
| Register Tmp = MRI->createVirtualRegister(&SPIRV::pIDRegClass); | |
| MRI->setType(Tmp, LLT::pointer(storageClassToAddressSpace( | |
| SPIRV::StorageClass::Generic), | |
| GR.getPointerSize())); | |
| MachineFunction *MF = I.getParent()->getParent(); | |
| GR.assignSPIRVTypeToVReg(GenericPtrTy, Tmp, *MF); | |
| MachineInstrBuilder MIB = buildSpecConstantOp( | |
| I, Tmp, SrcPtr, GR.getSPIRVTypeID(GenericPtrTy), | |
| static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric)); | |
| GR.add(MIB.getInstr(), MIB); | |
| return MIB; | |
| } | |
| // In SPIR-V address space casting can only happen to and from the Generic | |
| // storage class. We can also only cast Workgroup, CrossWorkgroup, or Function | |
| // pointers to and from Generic pointers. As such, we can convert e.g. from | |
| // Workgroup to Function by going via a Generic pointer as an intermediary. All | |
| // other combinations can only be done by a bitcast, and are probably not safe. | |
| bool SPIRVInstructionSelector::selectAddrSpaceCast(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| MachineBasicBlock &BB = *I.getParent(); | |
| const DebugLoc &DL = I.getDebugLoc(); | |
| Register SrcPtr = I.getOperand(1).getReg(); | |
| SPIRVTypeInst SrcPtrTy = GR.getSPIRVTypeForVReg(SrcPtr); | |
| // don't generate a cast for a null that may be represented by OpTypeInt | |
| if (!SrcPtrTy || !SrcPtrTy.isPointer() || !ResType || !ResType.isPointer()) | |
| return BuildCOPY(ResVReg, SrcPtr, I); | |
| SPIRV::StorageClass::StorageClass SrcSC = GR.getPointerStorageClass(SrcPtrTy); | |
| SPIRV::StorageClass::StorageClass DstSC = GR.getPointerStorageClass(ResType); | |
| if (isASCastInGVar(MRI, ResVReg)) { | |
| // AddrSpaceCast uses within OpVariable and OpConstantComposite instructions | |
| // are expressed by OpSpecConstantOp with an Opcode. | |
| // TODO: maybe insert a check whether the Kernel capability was declared and | |
| // so PtrCastToGeneric/GenericCastToPtr are available. | |
| unsigned SpecOpcode = [&]() -> unsigned { | |
| if (SrcSC == SPIRV::StorageClass::CodeSectionINTEL) | |
| return static_cast<uint32_t>(SPIRV::Opcode::Bitcast); | |
| if (DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC)) | |
| return static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric); | |
| if (SrcSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(DstSC)) | |
| return static_cast<uint32_t>(SPIRV::Opcode::GenericCastToPtr); | |
| return 0u; | |
| }(); | |
| // TODO: OpConstantComposite expects i8*, so we are forced to forget a | |
| // correct value of ResType and use general i8* instead. Maybe this should | |
| // be addressed in the emit-intrinsic step to infer a correct | |
| // OpConstantComposite type. | |
| if (SpecOpcode) { | |
| buildSpecConstantOp(I, ResVReg, SrcPtr, getUcharPtrTypeReg(I, DstSC), | |
| SpecOpcode) | |
| .constrainAllUses(TII, TRI, RBI); | |
| } else if (isGenericCastablePtr(SrcSC) && isGenericCastablePtr(DstSC)) { | |
| MachineInstrBuilder MIB = buildConstGenericPtr(I, SrcPtr, SrcPtrTy); | |
| MIB.constrainAllUses(TII, TRI, RBI); | |
| buildSpecConstantOp( | |
| I, ResVReg, MIB->getOperand(0).getReg(), getUcharPtrTypeReg(I, DstSC), | |
| static_cast<uint32_t>(SPIRV::Opcode::GenericCastToPtr)) | |
| .constrainAllUses(TII, TRI, RBI); | |
| } | |
| return true; | |
| } | |
| // don't generate a cast between identical storage classes | |
| if (SrcSC == DstSC) | |
| return BuildCOPY(ResVReg, SrcPtr, I); | |
| if ((SrcSC == SPIRV::StorageClass::Function && | |
| DstSC == SPIRV::StorageClass::Private) || | |
| (DstSC == SPIRV::StorageClass::Function && | |
| SrcSC == SPIRV::StorageClass::Private)) | |
| return BuildCOPY(ResVReg, SrcPtr, I); | |
| // Casting from an eligible pointer to Generic. | |
| if (DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC)) | |
| return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric); | |
| // Casting from Generic to an eligible pointer. | |
| if (SrcSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(DstSC)) | |
| return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr); | |
| // Casting between 2 eligible pointers using Generic as an intermediary. | |
| if (isGenericCastablePtr(SrcSC) && isGenericCastablePtr(DstSC)) { | |
| SPIRVTypeInst GenericPtrTy = | |
| GR.changePointerStorageClass(SrcPtrTy, SPIRV::StorageClass::Generic, I); | |
| Register Tmp = createVirtualRegister(GenericPtrTy, &GR, MRI, MRI->getMF()); | |
| BuildMI(BB, I, DL, TII.get(SPIRV::OpPtrCastToGeneric)) | |
| .addDef(Tmp) | |
| .addUse(GR.getSPIRVTypeID(GenericPtrTy)) | |
| .addUse(SrcPtr) | |
| .constrainAllUses(TII, TRI, RBI); | |
| BuildMI(BB, I, DL, TII.get(SPIRV::OpGenericCastToPtr)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(Tmp) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| // Check if instructions from the SPV_INTEL_usm_storage_classes extension may | |
| // be applied | |
| if (isUSMStorageClass(SrcSC) && DstSC == SPIRV::StorageClass::CrossWorkgroup) | |
| return selectUnOp(ResVReg, ResType, I, | |
| SPIRV::OpPtrCastToCrossWorkgroupINTEL); | |
| if (SrcSC == SPIRV::StorageClass::CrossWorkgroup && isUSMStorageClass(DstSC)) | |
| return selectUnOp(ResVReg, ResType, I, | |
| SPIRV::OpCrossWorkgroupCastToPtrINTEL); | |
| if (isUSMStorageClass(SrcSC) && DstSC == SPIRV::StorageClass::Generic) | |
| return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric); | |
| if (SrcSC == SPIRV::StorageClass::Generic && isUSMStorageClass(DstSC)) | |
| return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr); | |
| // Bitcast for pointers requires that the address spaces must match | |
| return false; | |
| } | |
| // G_PTRMASK - Apply a bitmask to a pointer value. | |
| // Result = Ptr & Mask | |
| // We need to convert the pointer to an integer, perform the AND operation, | |
| // and convert back to a pointer. | |
| bool SPIRVInstructionSelector::selectPtrMask(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| if (STI.isLogicalSPIRV()) | |
| return diagnoseUnsupported( | |
| I, "G_PTRMASK is not supported with logical SPIR-V"); | |
| MachineBasicBlock &BB = *I.getParent(); | |
| MachineFunction &MF = *BB.getParent(); | |
| const DebugLoc &DL = I.getDebugLoc(); | |
| Register PtrReg = I.getOperand(1).getReg(); | |
| Register MaskReg = I.getOperand(2).getReg(); | |
| SPIRVTypeInst MaskType = GR.getSPIRVTypeForVReg(MaskReg); | |
| // Convert pointer to integer. | |
| Register PtrAsInt = MRI->createVirtualRegister(GR.getRegClass(MaskType)); | |
| GR.assignSPIRVTypeToVReg(MaskType, PtrAsInt, MF); | |
| BuildMI(BB, I, DL, TII.get(SPIRV::OpConvertPtrToU)) | |
| .addDef(PtrAsInt) | |
| .addUse(GR.getSPIRVTypeID(MaskType)) | |
| .addUse(PtrReg) | |
| .constrainAllUses(TII, TRI, RBI); | |
| // Perform bitwise AND. | |
| Register MaskedInt = MRI->createVirtualRegister(GR.getRegClass(MaskType)); | |
| GR.assignSPIRVTypeToVReg(MaskType, MaskedInt, MF); | |
| unsigned AndOpcode = GR.getScalarOrVectorComponentCount(MaskType) > 1 | |
| ? SPIRV::OpBitwiseAndV | |
| : SPIRV::OpBitwiseAndS; | |
| BuildMI(BB, I, DL, TII.get(AndOpcode)) | |
| .addDef(MaskedInt) | |
| .addUse(GR.getSPIRVTypeID(MaskType)) | |
| .addUse(PtrAsInt) | |
| .addUse(MaskReg) | |
| .constrainAllUses(TII, TRI, RBI); | |
| // Convert integer back to pointer. | |
| BuildMI(BB, I, DL, TII.get(SPIRV::OpConvertUToPtr)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(MaskedInt) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| static unsigned getFCmpOpcode(unsigned PredNum) { | |
| auto Pred = static_cast<CmpInst::Predicate>(PredNum); | |
| switch (Pred) { | |
| case CmpInst::FCMP_OEQ: | |
| return SPIRV::OpFOrdEqual; | |
| case CmpInst::FCMP_OGE: | |
| return SPIRV::OpFOrdGreaterThanEqual; | |
| case CmpInst::FCMP_OGT: | |
| return SPIRV::OpFOrdGreaterThan; | |
| case CmpInst::FCMP_OLE: | |
| return SPIRV::OpFOrdLessThanEqual; | |
| case CmpInst::FCMP_OLT: | |
| return SPIRV::OpFOrdLessThan; | |
| case CmpInst::FCMP_ONE: | |
| return SPIRV::OpFOrdNotEqual; | |
| case CmpInst::FCMP_ORD: | |
| return SPIRV::OpOrdered; | |
| case CmpInst::FCMP_UEQ: | |
| return SPIRV::OpFUnordEqual; | |
| case CmpInst::FCMP_UGE: | |
| return SPIRV::OpFUnordGreaterThanEqual; | |
| case CmpInst::FCMP_UGT: | |
| return SPIRV::OpFUnordGreaterThan; | |
| case CmpInst::FCMP_ULE: | |
| return SPIRV::OpFUnordLessThanEqual; | |
| case CmpInst::FCMP_ULT: | |
| return SPIRV::OpFUnordLessThan; | |
| case CmpInst::FCMP_UNE: | |
| return SPIRV::OpFUnordNotEqual; | |
| case CmpInst::FCMP_UNO: | |
| return SPIRV::OpUnordered; | |
| default: | |
| llvm_unreachable("Unknown predicate type for FCmp"); | |
| } | |
| } | |
| static unsigned getICmpOpcode(unsigned PredNum) { | |
| auto Pred = static_cast<CmpInst::Predicate>(PredNum); | |
| switch (Pred) { | |
| case CmpInst::ICMP_EQ: | |
| return SPIRV::OpIEqual; | |
| case CmpInst::ICMP_NE: | |
| return SPIRV::OpINotEqual; | |
| case CmpInst::ICMP_SGE: | |
| return SPIRV::OpSGreaterThanEqual; | |
| case CmpInst::ICMP_SGT: | |
| return SPIRV::OpSGreaterThan; | |
| case CmpInst::ICMP_SLE: | |
| return SPIRV::OpSLessThanEqual; | |
| case CmpInst::ICMP_SLT: | |
| return SPIRV::OpSLessThan; | |
| case CmpInst::ICMP_UGE: | |
| return SPIRV::OpUGreaterThanEqual; | |
| case CmpInst::ICMP_UGT: | |
| return SPIRV::OpUGreaterThan; | |
| case CmpInst::ICMP_ULE: | |
| return SPIRV::OpULessThanEqual; | |
| case CmpInst::ICMP_ULT: | |
| return SPIRV::OpULessThan; | |
| default: | |
| llvm_unreachable("Unknown predicate type for ICmp"); | |
| } | |
| } | |
| static unsigned getPtrCmpOpcode(unsigned Pred) { | |
| switch (static_cast<CmpInst::Predicate>(Pred)) { | |
| case CmpInst::ICMP_EQ: | |
| return SPIRV::OpPtrEqual; | |
| case CmpInst::ICMP_NE: | |
| return SPIRV::OpPtrNotEqual; | |
| default: | |
| llvm_unreachable("Unknown predicate type for pointer comparison"); | |
| } | |
| } | |
| // Return the logical operation, or abort if none exists. | |
| static unsigned getBoolCmpOpcode(unsigned PredNum) { | |
| auto Pred = static_cast<CmpInst::Predicate>(PredNum); | |
| switch (Pred) { | |
| case CmpInst::ICMP_EQ: | |
| return SPIRV::OpLogicalEqual; | |
| case CmpInst::ICMP_NE: | |
| return SPIRV::OpLogicalNotEqual; | |
| default: | |
| llvm_unreachable("Unknown predicate type for Bool comparison"); | |
| } | |
| } | |
| static APFloat getZeroFP(const Type *LLVMFloatTy) { | |
| if (!LLVMFloatTy) | |
| return APFloat::getZero(APFloat::IEEEsingle()); | |
| switch (LLVMFloatTy->getScalarType()->getTypeID()) { | |
| case Type::HalfTyID: | |
| return APFloat::getZero(APFloat::IEEEhalf()); | |
| case Type::BFloatTyID: | |
| return APFloat::getZero(APFloat::BFloat()); | |
| default: | |
| case Type::FloatTyID: | |
| return APFloat::getZero(APFloat::IEEEsingle()); | |
| case Type::DoubleTyID: | |
| return APFloat::getZero(APFloat::IEEEdouble()); | |
| } | |
| } | |
| static APFloat getOneFP(const Type *LLVMFloatTy) { | |
| if (!LLVMFloatTy) | |
| return APFloat::getOne(APFloat::IEEEsingle()); | |
| switch (LLVMFloatTy->getScalarType()->getTypeID()) { | |
| case Type::HalfTyID: | |
| return APFloat::getOne(APFloat::IEEEhalf()); | |
| case Type::BFloatTyID: | |
| return APFloat::getOne(APFloat::BFloat()); | |
| default: | |
| case Type::FloatTyID: | |
| return APFloat::getOne(APFloat::IEEEsingle()); | |
| case Type::DoubleTyID: | |
| return APFloat::getOne(APFloat::IEEEdouble()); | |
| } | |
| } | |
| bool SPIRVInstructionSelector::selectAnyOrAll(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I, | |
| unsigned OpAnyOrAll) const { | |
| assert(I.getNumOperands() == 3); | |
| assert(I.getOperand(2).isReg()); | |
| MachineBasicBlock &BB = *I.getParent(); | |
| Register InputRegister = I.getOperand(2).getReg(); | |
| SPIRVTypeInst InputType = GR.getSPIRVTypeForVReg(InputRegister); | |
| assert(InputType && "VReg has no type assigned"); | |
| bool IsBoolTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeBool); | |
| if (IsBoolTy && !isVectorType(InputType)) { | |
| assert(ResVReg == I.getOperand(0).getReg()); | |
| return BuildCOPY(ResVReg, InputRegister, I); | |
| } | |
| bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat); | |
| unsigned SpirvNotEqualId = | |
| IsFloatTy ? SPIRV::OpFOrdNotEqual : SPIRV::OpINotEqual; | |
| SPIRVTypeInst SpvBoolScalarTy = GR.getOrCreateSPIRVBoolType(I, TII); | |
| SPIRVTypeInst SpvBoolTy = SpvBoolScalarTy; | |
| Register NotEqualReg = ResVReg; | |
| if (isVectorType(InputType)) { | |
| NotEqualReg = | |
| IsBoolTy ? InputRegister | |
| : createVirtualRegister(SpvBoolTy, &GR, MRI, MRI->getMF()); | |
| const unsigned NumElts = GR.getScalarOrVectorComponentCount(InputType); | |
| SpvBoolTy = GR.getOrCreateSPIRVVectorType(SpvBoolTy, NumElts, I, TII); | |
| } | |
| if (!IsBoolTy) { | |
| Register ConstZeroReg = | |
| IsFloatTy ? buildZerosValF(InputType, I) : buildZerosVal(InputType, I); | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(SpirvNotEqualId)) | |
| .addDef(NotEqualReg) | |
| .addUse(GR.getSPIRVTypeID(SpvBoolTy)) | |
| .addUse(InputRegister) | |
| .addUse(ConstZeroReg) | |
| .constrainAllUses(TII, TRI, RBI); | |
| } | |
| if (isVectorType(InputType)) | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(OpAnyOrAll)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(SpvBoolScalarTy)) | |
| .addUse(NotEqualReg) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectAll(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAll); | |
| } | |
| bool SPIRVInstructionSelector::selectAny(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAny); | |
| } | |
| // Select the OpDot instruction for the given float dot | |
| bool SPIRVInstructionSelector::selectFloatDot(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| assert(I.getNumOperands() == 4); | |
| assert(I.getOperand(2).isReg()); | |
| assert(I.getOperand(3).isReg()); | |
| [[maybe_unused]] SPIRVTypeInst VecType = | |
| GR.getSPIRVTypeForVReg(I.getOperand(2).getReg()); | |
| assert(((VecType->getOpcode() == SPIRV::OpTypeVector && | |
| GR.getScalarOrVectorComponentCount(VecType) > 1) || | |
| VecType->getOpcode() == SPIRV::OpTypeVectorIdEXT) && | |
| "dot product requires either a vector of at least 2 components or" | |
| " the SPV_EXT_long vector extension."); | |
| [[maybe_unused]] SPIRVTypeInst EltType = | |
| GR.getScalarOrVectorComponentType(VecType); | |
| assert(EltType->getOpcode() == SPIRV::OpTypeFloat); | |
| MachineBasicBlock &BB = *I.getParent(); | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpDot)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(I.getOperand(2).getReg()) | |
| .addUse(I.getOperand(3).getReg()) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectIntegerDot(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I, | |
| bool Signed) const { | |
| assert(I.getNumOperands() == 4); | |
| assert(I.getOperand(2).isReg()); | |
| assert(I.getOperand(3).isReg()); | |
| MachineBasicBlock &BB = *I.getParent(); | |
| auto DotOp = Signed ? SPIRV::OpSDot : SPIRV::OpUDot; | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(DotOp)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(I.getOperand(2).getReg()) | |
| .addUse(I.getOperand(3).getReg()) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| // Since pre-1.6 SPIRV has no integer dot implementation, | |
| // expand by piecewise multiplying and adding the results | |
| bool SPIRVInstructionSelector::selectIntegerDotExpansion( | |
| Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const { | |
| assert(I.getNumOperands() == 4); | |
| assert(I.getOperand(2).isReg()); | |
| assert(I.getOperand(3).isReg()); | |
| MachineBasicBlock &BB = *I.getParent(); | |
| // Multiply the vectors, then sum the results | |
| Register Vec0 = I.getOperand(2).getReg(); | |
| Register Vec1 = I.getOperand(3).getReg(); | |
| Register TmpVec = MRI->createVirtualRegister(GR.getRegClass(ResType)); | |
| SPIRVTypeInst VecType = GR.getSPIRVTypeForVReg(Vec0); | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIMulV)) | |
| .addDef(TmpVec) | |
| .addUse(GR.getSPIRVTypeID(VecType)) | |
| .addUse(Vec0) | |
| .addUse(Vec1) | |
| .constrainAllUses(TII, TRI, RBI); | |
| assert(((VecType->getOpcode() == SPIRV::OpTypeVector && | |
| GR.getScalarOrVectorComponentCount(VecType) > 1) || | |
| VecType->getOpcode() == SPIRV::OpTypeVectorIdEXT) && | |
| "dot product requires either a vector of at least 2 components " | |
| "or the SPV_EXT_long_vector extension."); | |
| Register Res = MRI->createVirtualRegister(GR.getRegClass(ResType)); | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) | |
| .addDef(Res) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(TmpVec) | |
| .addImm(0) | |
| .constrainAllUses(TII, TRI, RBI); | |
| for (unsigned i = 1; i < GR.getScalarOrVectorComponentCount(VecType); i++) { | |
| Register Elt = MRI->createVirtualRegister(GR.getRegClass(ResType)); | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) | |
| .addDef(Elt) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(TmpVec) | |
| .addImm(i) | |
| .constrainAllUses(TII, TRI, RBI); | |
| Register Sum = i < GR.getScalarOrVectorComponentCount(VecType) - 1 | |
| ? MRI->createVirtualRegister(GR.getRegClass(ResType)) | |
| : ResVReg; | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS)) | |
| .addDef(Sum) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(Res) | |
| .addUse(Elt) | |
| .constrainAllUses(TII, TRI, RBI); | |
| Res = Sum; | |
| } | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectOpIsInf(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| MachineBasicBlock &BB = *I.getParent(); | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIsInf)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(I.getOperand(2).getReg()) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectOpIsNan(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| MachineBasicBlock &BB = *I.getParent(); | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIsNan)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(I.getOperand(2).getReg()) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectOpIsFinite(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| // OpIsFinite requires Kernel capability; emulate with OpIsInf & OpIsNan | |
| MachineBasicBlock &BB = *I.getParent(); | |
| Register Src = I.getOperand(2).getReg(); | |
| Register TypeID = GR.getSPIRVTypeID(ResType); | |
| const DebugLoc &DL = I.getDebugLoc(); | |
| if (!STI.isShader()) { | |
| BuildMI(BB, I, DL, TII.get(SPIRV::OpIsFinite)) | |
| .addDef(ResVReg) | |
| .addUse(TypeID) | |
| .addUse(Src) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| Register IsInfReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); | |
| BuildMI(BB, I, DL, TII.get(SPIRV::OpIsInf)) | |
| .addDef(IsInfReg) | |
| .addUse(TypeID) | |
| .addUse(Src) | |
| .constrainAllUses(TII, TRI, RBI); | |
| Register IsNanReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); | |
| BuildMI(BB, I, DL, TII.get(SPIRV::OpIsNan)) | |
| .addDef(IsNanReg) | |
| .addUse(TypeID) | |
| .addUse(Src) | |
| .constrainAllUses(TII, TRI, RBI); | |
| Register OrReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); | |
| BuildMI(BB, I, DL, TII.get(SPIRV::OpLogicalOr)) | |
| .addDef(OrReg) | |
| .addUse(TypeID) | |
| .addUse(IsInfReg) | |
| .addUse(IsNanReg) | |
| .constrainAllUses(TII, TRI, RBI); | |
| BuildMI(BB, I, DL, TII.get(SPIRV::OpLogicalNot)) | |
| .addDef(ResVReg) | |
| .addUse(TypeID) | |
| .addUse(OrReg) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectOpIsNormal(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| MachineBasicBlock &BB = *I.getParent(); | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIsNormal)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(I.getOperand(2).getReg()) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| template <bool Signed> | |
| bool SPIRVInstructionSelector::selectDot4AddPacked(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| assert(I.getNumOperands() == 5); | |
| assert(I.getOperand(2).isReg()); | |
| assert(I.getOperand(3).isReg()); | |
| assert(I.getOperand(4).isReg()); | |
| MachineBasicBlock &BB = *I.getParent(); | |
| Register Acc = I.getOperand(2).getReg(); | |
| Register X = I.getOperand(3).getReg(); | |
| Register Y = I.getOperand(4).getReg(); | |
| auto DotOp = Signed ? SPIRV::OpSDot : SPIRV::OpUDot; | |
| Register Dot = MRI->createVirtualRegister(GR.getRegClass(ResType)); | |
| auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(DotOp)) | |
| .addDef(Dot) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(X) | |
| .addUse(Y); | |
| MIB.addImm(SPIRV::BuiltIn::PackedVectorFormat4x8Bit); | |
| MIB.constrainAllUses(TII, TRI, RBI); | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(Dot) | |
| .addUse(Acc) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| // Since pre-1.6 SPIRV has no DotProductInput4x8BitPacked implementation, | |
| // extract the elements of the packed inputs, multiply them and add the result | |
| // to the accumulator. | |
| template <bool Signed> | |
| bool SPIRVInstructionSelector::selectDot4AddPackedExpansion( | |
| Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const { | |
| assert(I.getNumOperands() == 5); | |
| assert(I.getOperand(2).isReg()); | |
| assert(I.getOperand(3).isReg()); | |
| assert(I.getOperand(4).isReg()); | |
| MachineBasicBlock &BB = *I.getParent(); | |
| Register Acc = I.getOperand(2).getReg(); | |
| Register X = I.getOperand(3).getReg(); | |
| Register Y = I.getOperand(4).getReg(); | |
| SPIRVTypeInst EltType = GR.getOrCreateSPIRVIntegerType(8, I, TII); | |
| auto ExtractOp = | |
| Signed ? SPIRV::OpBitFieldSExtract : SPIRV::OpBitFieldUExtract; | |
| bool ZeroAsNull = !STI.isShader(); | |
| // Extract the i8 element, multiply and add it to the accumulator | |
| for (unsigned i = 0; i < 4; i++) { | |
| // A[i] | |
| Register AElt = MRI->createVirtualRegister(&SPIRV::IDRegClass); | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp)) | |
| .addDef(AElt) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(X) | |
| .addUse(GR.getOrCreateConstInt(i * 8, I, EltType, TII, ZeroAsNull)) | |
| .addUse(GR.getOrCreateConstInt(8, I, EltType, TII, ZeroAsNull)) | |
| .constrainAllUses(TII, TRI, RBI); | |
| // B[i] | |
| Register BElt = MRI->createVirtualRegister(&SPIRV::IDRegClass); | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp)) | |
| .addDef(BElt) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(Y) | |
| .addUse(GR.getOrCreateConstInt(i * 8, I, EltType, TII, ZeroAsNull)) | |
| .addUse(GR.getOrCreateConstInt(8, I, EltType, TII, ZeroAsNull)) | |
| .constrainAllUses(TII, TRI, RBI); | |
| // A[i] * B[i] | |
| Register Mul = MRI->createVirtualRegister(&SPIRV::IDRegClass); | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIMulS)) | |
| .addDef(Mul) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(AElt) | |
| .addUse(BElt) | |
| .constrainAllUses(TII, TRI, RBI); | |
| // Discard 24 highest-bits so that stored i32 register is i8 equivalent | |
| Register MaskMul = MRI->createVirtualRegister(&SPIRV::IDRegClass); | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(ExtractOp)) | |
| .addDef(MaskMul) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(Mul) | |
| .addUse(GR.getOrCreateConstInt(0, I, EltType, TII, ZeroAsNull)) | |
| .addUse(GR.getOrCreateConstInt(8, I, EltType, TII, ZeroAsNull)) | |
| .constrainAllUses(TII, TRI, RBI); | |
| // Acc = Acc + A[i] * B[i] | |
| Register Sum = | |
| i < 3 ? MRI->createVirtualRegister(&SPIRV::IDRegClass) : ResVReg; | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS)) | |
| .addDef(Sum) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(Acc) | |
| .addUse(MaskMul) | |
| .constrainAllUses(TII, TRI, RBI); | |
| Acc = Sum; | |
| } | |
| return true; | |
| } | |
| /// Transform saturate(x) to clamp(x, 0.0f, 1.0f) as SPIRV | |
| /// does not have a saturate builtin. | |
| bool SPIRVInstructionSelector::selectSaturate(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| assert(I.getNumOperands() == 3); | |
| assert(I.getOperand(2).isReg()); | |
| MachineBasicBlock &BB = *I.getParent(); | |
| Register VZero = buildZerosValF(ResType, I); | |
| Register VOne = buildOnesValF(ResType, I); | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) | |
| .addImm(GL::FClamp) | |
| .addUse(I.getOperand(2).getReg()) | |
| .addUse(VZero) | |
| .addUse(VOne) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectSign(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| assert(I.getNumOperands() == 3); | |
| assert(I.getOperand(2).isReg()); | |
| MachineBasicBlock &BB = *I.getParent(); | |
| Register InputRegister = I.getOperand(2).getReg(); | |
| SPIRVTypeInst InputType = GR.getSPIRVTypeForVReg(InputRegister); | |
| auto &DL = I.getDebugLoc(); | |
| if (!InputType) | |
| return diagnoseUnsupported(I, "Input Type could not be determined."); | |
| bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat); | |
| unsigned SignBitWidth = GR.getScalarOrVectorBitWidth(InputType); | |
| unsigned ResBitWidth = GR.getScalarOrVectorBitWidth(ResType); | |
| bool NeedsConversion = IsFloatTy || SignBitWidth != ResBitWidth; | |
| auto SignOpcode = IsFloatTy ? GL::FSign : GL::SSign; | |
| Register SignReg = NeedsConversion | |
| ? MRI->createVirtualRegister(&SPIRV::IDRegClass) | |
| : ResVReg; | |
| BuildMI(BB, I, DL, TII.get(SPIRV::OpExtInst)) | |
| .addDef(SignReg) | |
| .addUse(GR.getSPIRVTypeID(InputType)) | |
| .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) | |
| .addImm(SignOpcode) | |
| .addUse(InputRegister) | |
| .constrainAllUses(TII, TRI, RBI); | |
| if (NeedsConversion) { | |
| auto ConvertOpcode = IsFloatTy ? SPIRV::OpConvertFToS : SPIRV::OpSConvert; | |
| BuildMI(*I.getParent(), I, DL, TII.get(ConvertOpcode)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(SignReg) | |
| .constrainAllUses(TII, TRI, RBI); | |
| } | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectWaveOpInst(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I, | |
| unsigned Opcode) const { | |
| MachineBasicBlock &BB = *I.getParent(); | |
| SPIRVTypeInst IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII); | |
| auto BMI = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, | |
| IntTy, TII, !STI.isShader())); | |
| for (unsigned J = 2; J < I.getNumOperands(); J++) { | |
| BMI.addUse(I.getOperand(J).getReg()); | |
| } | |
| BMI.constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectBarrierInst(MachineInstr &I, | |
| unsigned Scope, | |
| unsigned MemSem, | |
| bool WithGroupSync) const { | |
| auto BarrierType = | |
| WithGroupSync ? SPIRV::OpControlBarrier : SPIRV::OpMemoryBarrier; | |
| MemSem |= SPIRV::MemorySemantics::AcquireRelease; | |
| assert(((Scope != SPIRV::Scope::Workgroup) || | |
| ((MemSem & SPIRV::MemorySemantics::WorkgroupMemory) > 0)) && | |
| "Workgroup Scope must set WorkGroupMemory semantic " | |
| "in Barrier instruction"); | |
| assert(((Scope != SPIRV::Scope::Device) || | |
| ((MemSem & SPIRV::MemorySemantics::UniformMemory) > 0 && | |
| (MemSem & SPIRV::MemorySemantics::ImageMemory) > 0)) && | |
| "Device Scope must set UniformMemory and ImageMemory semantic " | |
| "in Barrier instruction"); | |
| MachineBasicBlock &BB = *I.getParent(); | |
| auto MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(BarrierType)); | |
| // OpControlBarrier needs to also set Execution Scope | |
| if (WithGroupSync) { | |
| Register ExecReg = buildI32Constant(SPIRV::Scope::Workgroup, I); | |
| MI.addUse(ExecReg); | |
| } | |
| Register ScopeReg = buildI32Constant(Scope, I); | |
| Register MemSemReg = buildI32Constant(MemSem, I); | |
| MI.addUse(ScopeReg).addUse(MemSemReg).constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectWaveActiveCountBits( | |
| Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const { | |
| SPIRVTypeInst IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII); | |
| SPIRVTypeInst BallotType = GR.getOrCreateSPIRVVectorType(IntTy, 4, I, TII); | |
| Register BallotReg = MRI->createVirtualRegister(GR.getRegClass(BallotType)); | |
| if (!selectWaveOpInst(BallotReg, BallotType, I, | |
| SPIRV::OpGroupNonUniformBallot)) | |
| return false; | |
| MachineBasicBlock &BB = *I.getParent(); | |
| BuildMI(BB, I, I.getDebugLoc(), | |
| TII.get(SPIRV::OpGroupNonUniformBallotBitCount)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, IntTy, TII, | |
| !STI.isShader())) | |
| .addImm(SPIRV::GroupOperation::Reduce) | |
| .addUse(BallotReg) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectWaveActiveAllEqual(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| MachineBasicBlock &BB = *I.getParent(); | |
| const DebugLoc &DL = I.getDebugLoc(); | |
| // Input to the intrinsic | |
| Register InputReg = I.getOperand(2).getReg(); | |
| SPIRVTypeInst InputType = GR.getSPIRVTypeForVReg(InputReg); | |
| // Determine if input is vector | |
| unsigned NumElems = GR.getScalarOrVectorComponentCount(InputType); | |
| bool IsVector = NumElems > 1 || | |
| (InputType->getOpcode() == SPIRV::OpTypeVectorIdEXT && | |
| STI.canUseExtension(SPIRV::Extension::SPV_EXT_long_vector)); | |
| // Determine element types | |
| SPIRVTypeInst ElemInputType = GR.getScalarOrVectorComponentType(InputType); | |
| SPIRVTypeInst ElemBoolType = GR.getScalarOrVectorComponentType(ResType); | |
| // Subgroup scope constant | |
| SPIRVTypeInst IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII); | |
| Register ScopeConst = GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, IntTy, | |
| TII, !STI.isShader()); | |
| // Scalar case | |
| if (!IsVector) { | |
| return selectWaveOpInst(ResVReg, ElemBoolType, I, | |
| SPIRV::OpGroupNonUniformAllEqual); | |
| } | |
| // Vector case | |
| SmallVector<Register, 4> ElementResults; | |
| ElementResults.reserve(NumElems); | |
| for (unsigned Idx = 0; Idx < NumElems; ++Idx) { | |
| // Extract element | |
| Register ElemInput = InputReg; | |
| Register Extracted = | |
| MRI->createVirtualRegister(GR.getRegClass(ElemInputType)); | |
| BuildMI(BB, I, DL, TII.get(SPIRV::OpCompositeExtract)) | |
| .addDef(Extracted) | |
| .addUse(GR.getSPIRVTypeID(ElemInputType)) | |
| .addUse(InputReg) | |
| .addImm(Idx) | |
| .constrainAllUses(TII, TRI, RBI); | |
| ElemInput = Extracted; | |
| // Emit per-element AllEqual | |
| Register ElemResult = | |
| MRI->createVirtualRegister(GR.getRegClass(ElemBoolType)); | |
| BuildMI(BB, I, DL, TII.get(SPIRV::OpGroupNonUniformAllEqual)) | |
| .addDef(ElemResult) | |
| .addUse(GR.getSPIRVTypeID(ElemBoolType)) | |
| .addUse(ScopeConst) | |
| .addUse(ElemInput) | |
| .constrainAllUses(TII, TRI, RBI); | |
| ElementResults.push_back(ElemResult); | |
| } | |
| // Reconstruct vector<bool> | |
| auto MIB = BuildMI(BB, I, DL, TII.get(SPIRV::OpCompositeConstruct)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)); | |
| for (Register R : ElementResults) | |
| MIB.addUse(R); | |
| MIB.constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectWavePrefixBitCount(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| assert(I.getNumOperands() == 3); | |
| auto Op = I.getOperand(2); | |
| assert(Op.isReg()); | |
| MachineBasicBlock &BB = *I.getParent(); | |
| DebugLoc DL = I.getDebugLoc(); | |
| Register InputRegister = Op.getReg(); | |
| SPIRVTypeInst InputType = GR.getSPIRVTypeForVReg(InputRegister); | |
| if (!InputType) | |
| return diagnoseUnsupported(I, "Input Type could not be determined."); | |
| if (InputType->getOpcode() != SPIRV::OpTypeBool) | |
| return diagnoseUnsupported(I, "WavePrefixBitCount requires boolean input"); | |
| // Types | |
| SPIRVTypeInst Int32Ty = GR.getOrCreateSPIRVIntegerType(32, I, TII); | |
| // Ballot result type: vector<uint32> | |
| // Match DXC: %v4uint for Subgroup size | |
| SPIRVTypeInst BallotTy = GR.getOrCreateSPIRVVectorType(Int32Ty, 4, I, TII); | |
| // Create a vreg for the ballot result | |
| Register BallotVReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); | |
| // 1. OpGroupNonUniformBallot | |
| BuildMI(BB, I, DL, TII.get(SPIRV::OpGroupNonUniformBallot)) | |
| .addDef(BallotVReg) | |
| .addUse(GR.getSPIRVTypeID(BallotTy)) | |
| .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, Int32Ty, TII)) | |
| .addUse(InputRegister) | |
| .constrainAllUses(TII, TRI, RBI); | |
| // 2. OpGroupNonUniformBallotBitCount | |
| BuildMI(BB, I, DL, TII.get(SPIRV::OpGroupNonUniformBallotBitCount)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, Int32Ty, TII)) | |
| .addImm(SPIRV::GroupOperation::ExclusiveScan) | |
| .addUse(BallotVReg) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectWaveReduceMax(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I, | |
| bool IsUnsigned) const { | |
| return selectWaveReduce( | |
| ResVReg, ResType, I, IsUnsigned, | |
| [&](Register InputRegister, bool IsUnsigned) { | |
| const bool IsFloatTy = | |
| GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat); | |
| const auto IntOp = IsUnsigned ? SPIRV::OpGroupNonUniformUMax | |
| : SPIRV::OpGroupNonUniformSMax; | |
| return IsFloatTy ? SPIRV::OpGroupNonUniformFMax : IntOp; | |
| }); | |
| } | |
| bool SPIRVInstructionSelector::selectWaveReduceMin(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I, | |
| bool IsUnsigned) const { | |
| return selectWaveReduce( | |
| ResVReg, ResType, I, IsUnsigned, | |
| [&](Register InputRegister, bool IsUnsigned) { | |
| const bool IsFloatTy = | |
| GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat); | |
| const auto IntOp = IsUnsigned ? SPIRV::OpGroupNonUniformUMin | |
| : SPIRV::OpGroupNonUniformSMin; | |
| return IsFloatTy ? SPIRV::OpGroupNonUniformFMin : IntOp; | |
| }); | |
| } | |
| bool SPIRVInstructionSelector::selectWaveReduceSum(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| return selectWaveReduce(ResVReg, ResType, I, /*IsUnsigned*/ false, | |
| [&](Register InputRegister, bool IsUnsigned) { | |
| bool IsFloatTy = GR.isScalarOrVectorOfType( | |
| InputRegister, SPIRV::OpTypeFloat); | |
| return IsFloatTy ? SPIRV::OpGroupNonUniformFAdd | |
| : SPIRV::OpGroupNonUniformIAdd; | |
| }); | |
| } | |
| bool SPIRVInstructionSelector::selectWaveReduceProduct(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| return selectWaveReduce(ResVReg, ResType, I, /*IsUnsigned*/ false, | |
| [&](Register InputRegister, bool IsUnsigned) { | |
| bool IsFloatTy = GR.isScalarOrVectorOfType( | |
| InputRegister, SPIRV::OpTypeFloat); | |
| return IsFloatTy ? SPIRV::OpGroupNonUniformFMul | |
| : SPIRV::OpGroupNonUniformIMul; | |
| }); | |
| } | |
| template <typename PickOpcodeFn> | |
| bool SPIRVInstructionSelector::selectWaveReduce( | |
| Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I, bool IsUnsigned, | |
| PickOpcodeFn &&PickOpcode) const { | |
| assert(I.getNumOperands() == 3); | |
| assert(I.getOperand(2).isReg()); | |
| MachineBasicBlock &BB = *I.getParent(); | |
| Register InputRegister = I.getOperand(2).getReg(); | |
| SPIRVTypeInst InputType = GR.getSPIRVTypeForVReg(InputRegister); | |
| if (!InputType) | |
| return diagnoseUnsupported(I, "Input Type could not be determined."); | |
| SPIRVTypeInst IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII); | |
| const unsigned Opcode = PickOpcode(InputRegister, IsUnsigned); | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, IntTy, TII, | |
| !STI.isShader())) | |
| .addImm(SPIRV::GroupOperation::Reduce) | |
| .addUse(I.getOperand(2).getReg()) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectWaveReduceOp(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I, | |
| unsigned Opcode) const { | |
| return selectWaveReduce( | |
| ResVReg, ResType, I, false, | |
| [&](Register InputRegister, bool IsUnsigned) { return Opcode; }); | |
| } | |
| bool SPIRVInstructionSelector::selectWaveExclusiveScanSum( | |
| Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const { | |
| return selectWaveExclusiveScan(ResVReg, ResType, I, /*IsUnsigned*/ false, | |
| [&](Register InputRegister, bool IsUnsigned) { | |
| bool IsFloatTy = GR.isScalarOrVectorOfType( | |
| InputRegister, SPIRV::OpTypeFloat); | |
| return IsFloatTy | |
| ? SPIRV::OpGroupNonUniformFAdd | |
| : SPIRV::OpGroupNonUniformIAdd; | |
| }); | |
| } | |
| bool SPIRVInstructionSelector::selectWaveExclusiveScanProduct( | |
| Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const { | |
| return selectWaveExclusiveScan(ResVReg, ResType, I, /*IsUnsigned*/ false, | |
| [&](Register InputRegister, bool IsUnsigned) { | |
| bool IsFloatTy = GR.isScalarOrVectorOfType( | |
| InputRegister, SPIRV::OpTypeFloat); | |
| return IsFloatTy | |
| ? SPIRV::OpGroupNonUniformFMul | |
| : SPIRV::OpGroupNonUniformIMul; | |
| }); | |
| } | |
| template <typename PickOpcodeFn> | |
| bool SPIRVInstructionSelector::selectWaveExclusiveScan( | |
| Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I, bool IsUnsigned, | |
| PickOpcodeFn &&PickOpcode) const { | |
| assert(I.getNumOperands() == 3); | |
| assert(I.getOperand(2).isReg()); | |
| MachineBasicBlock &BB = *I.getParent(); | |
| Register InputRegister = I.getOperand(2).getReg(); | |
| SPIRVTypeInst InputType = GR.getSPIRVTypeForVReg(InputRegister); | |
| if (!InputType) | |
| return diagnoseUnsupported(I, "Input Type could not be determined."); | |
| SPIRVTypeInst IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII); | |
| const unsigned Opcode = PickOpcode(InputRegister, IsUnsigned); | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, IntTy, TII, | |
| !STI.isShader())) | |
| .addImm(SPIRV::GroupOperation::ExclusiveScan) | |
| .addUse(I.getOperand(2).getReg()) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectQuadSwap(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I, | |
| unsigned Direction) const { | |
| assert(I.getNumOperands() == 3); | |
| assert(I.getOperand(2).isReg()); | |
| MachineBasicBlock &BB = *I.getParent(); | |
| Register InputRegister = I.getOperand(2).getReg(); | |
| SPIRVTypeInst IntTy = GR.getOrCreateSPIRVIntegerType(32, I, TII); | |
| bool ZeroAsNull = !STI.isShader(); | |
| Register DirectionReg = | |
| GR.getOrCreateConstInt(Direction, I, IntTy, TII, ZeroAsNull); | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpGroupNonUniformQuadSwap)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(GR.getOrCreateConstInt(SPIRV::Scope::Subgroup, I, IntTy, TII, | |
| ZeroAsNull)) | |
| .addUse(InputRegister) | |
| .addUse(DirectionReg) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectBitreverseViaI32(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I, | |
| Register Op) const { | |
| SPIRVTypeInst Int32Type = GR.getOrCreateSPIRVIntegerType(32, I, TII); | |
| const unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType); | |
| Register ShiftConst = | |
| GR.getOrCreateConstInt(32 - BitWidth, I, Int32Type, TII); | |
| unsigned ShiftOp = SPIRV::OpShiftRightLogicalS; | |
| const unsigned N = GR.getScalarOrVectorComponentCount(ResType); | |
| const unsigned ExtendOpcode = GR.isScalarOrVectorSigned(ResType) | |
| ? SPIRV::OpSConvert | |
| : SPIRV::OpUConvert; | |
| if (N > 1 || (ResType->getOpcode() == SPIRV::OpTypeVectorIdEXT && | |
| STI.canUseExtension(SPIRV::Extension::SPV_EXT_long_vector))) { | |
| Int32Type = GR.getOrCreateSPIRVVectorType(Int32Type, N, I, TII); | |
| ShiftOp = SPIRV::OpShiftRightLogicalV; | |
| // Vector shifts require a composite constant | |
| const Register CompositeReg = | |
| MRI->createVirtualRegister(GR.getRegClass(Int32Type)); | |
| auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), | |
| TII.get(SPIRV::OpConstantComposite)) | |
| .addDef(CompositeReg) | |
| .addUse(GR.getSPIRVTypeID(Int32Type)); | |
| for (unsigned It = 0; It < N; ++It) | |
| MIB.addUse(ShiftConst); | |
| MIB.constrainAllUses(TII, TRI, RBI); | |
| ShiftConst = CompositeReg; | |
| } | |
| // Converts the input to i32 (or vector of i32) | |
| Register ExtReg = MRI->createVirtualRegister(GR.getRegClass(Int32Type)); | |
| if (!selectOpWithSrcs(ExtReg, Int32Type, I, {Op}, ExtendOpcode)) | |
| return false; | |
| // Perform bitreverse on the i32 value | |
| Register BitrevReg = MRI->createVirtualRegister(GR.getRegClass(Int32Type)); | |
| if (!selectBitreverseNative(BitrevReg, Int32Type, I, ExtReg)) | |
| return false; | |
| // Shift the bit-reversed value to get the final result. | |
| Register ShiftReg = MRI->createVirtualRegister(GR.getRegClass(Int32Type)); | |
| if (!selectOpWithSrcs(ShiftReg, Int32Type, I, {BitrevReg, ShiftConst}, | |
| ShiftOp)) | |
| return false; | |
| // Finally, convert the result back. | |
| return selectOpWithSrcs(ResVReg, ResType, I, {ShiftReg}, ExtendOpcode); | |
| } | |
| bool SPIRVInstructionSelector::handle64BitOverflow( | |
| Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I, Register SrcReg, | |
| unsigned int Opcode, | |
| std::function<bool(Register, SPIRVTypeInst, MachineInstr &, Register, | |
| unsigned)> | |
| CallbackFunction) const { | |
| SPIRVTypeInst BaseType = GR.retrieveScalarOrVectorIntType(ResType); | |
| assert(BaseType->getOpcode() == SPIRV::OpTypeInt && | |
| "handle64BitOverflow should only be used for integer types"); | |
| unsigned ComponentCount = GR.getScalarOrVectorComponentCount(ResType); | |
| assert(ComponentCount < 5 && "Vec 5+ will generate invalid SPIR-V ops"); | |
| MachineIRBuilder MIRBuilder(I); | |
| SPIRVTypeInst I64Type = GR.getOrCreateSPIRVIntegerType(64, MIRBuilder); | |
| SPIRVTypeInst I64x2Type = | |
| GR.getOrCreateSPIRVVectorType(I64Type, 2, MIRBuilder, false); | |
| SPIRVTypeInst Vec2ResType = | |
| GR.getOrCreateSPIRVVectorType(BaseType, 2, MIRBuilder, false); | |
| std::vector<Register> PartialRegs; | |
| unsigned CurrentComponent = 0; | |
| for (; CurrentComponent + 1 < ComponentCount; CurrentComponent += 2) { | |
| Register PopCountResult = | |
| MRI->createVirtualRegister(GR.getRegClass(I64x2Type)); | |
| auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), | |
| TII.get(SPIRV::OpVectorShuffle)) | |
| .addDef(PopCountResult) | |
| .addUse(GR.getSPIRVTypeID(I64x2Type)) | |
| .addUse(SrcReg) | |
| .addUse(SrcReg) | |
| .addImm(CurrentComponent) | |
| .addImm(CurrentComponent + 1); | |
| MIB.constrainAllUses(TII, TRI, RBI); | |
| Register SubVecReg = | |
| MRI->createVirtualRegister(GR.getRegClass(Vec2ResType)); | |
| if (!CallbackFunction(SubVecReg, Vec2ResType, I, PopCountResult, Opcode)) | |
| return false; | |
| PartialRegs.push_back(SubVecReg); | |
| } | |
| // On odd component counts we need to handle one more component | |
| if (CurrentComponent != ComponentCount) { | |
| bool ZeroAsNull = !STI.isShader(); | |
| Register FinalElemReg = MRI->createVirtualRegister(GR.getRegClass(I64Type)); | |
| Register ConstIntLastIdx = GR.getOrCreateConstInt( | |
| ComponentCount - 1, I, BaseType, TII, ZeroAsNull); | |
| if (!selectOpWithSrcs(FinalElemReg, I64Type, I, {SrcReg, ConstIntLastIdx}, | |
| SPIRV::OpVectorExtractDynamic)) | |
| return false; | |
| Register FinalElemResReg = | |
| MRI->createVirtualRegister(GR.getRegClass(BaseType)); | |
| if (!CallbackFunction(FinalElemResReg, BaseType, I, FinalElemReg, Opcode)) | |
| return false; | |
| PartialRegs.push_back(FinalElemResReg); | |
| } | |
| // Join all the resulting registers back into the return type in order | |
| // (ie i32x2, i32x2, i32x1 -> i32x5) | |
| return selectOpWithSrcs(ResVReg, ResType, I, PartialRegs, | |
| SPIRV::OpCompositeConstruct); | |
| } | |
| bool SPIRVInstructionSelector::selectBitreverse64(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I, | |
| Register SrcReg) const { | |
| unsigned ComponentCount = GR.getScalarOrVectorComponentCount(ResType); | |
| if (ComponentCount > 2) | |
| return handle64BitOverflow( | |
| ResVReg, ResType, I, SrcReg, SPIRV::OpBitReverse, | |
| [this](Register R, SPIRVTypeInst T, MachineInstr &I, Register S, | |
| unsigned O) { return this->selectBitreverse64(R, T, I, S); }); | |
| MachineIRBuilder MIRBuilder(I); | |
| SPIRVTypeInst I32Type = GR.getOrCreateSPIRVIntegerType(32, MIRBuilder); | |
| SPIRVTypeInst VecI32Type = GR.getOrCreateSPIRVVectorType( | |
| I32Type, 2 * ComponentCount, MIRBuilder, /*IsSigned=*/false); | |
| // Converts 64 bit into and array of 32 bit, containing 2 elements. | |
| Register Vec32 = MRI->createVirtualRegister(GR.getRegClass(VecI32Type)); | |
| if (!selectOpWithSrcs(Vec32, VecI32Type, I, {SrcReg}, SPIRV::OpBitcast)) | |
| return false; | |
| // Apply bitreverse on each 32 bit lane | |
| Register Reverse32 = MRI->createVirtualRegister(GR.getRegClass(VecI32Type)); | |
| if (!selectBitreverseNative(Reverse32, VecI32Type, I, Vec32)) | |
| return false; | |
| // Reversing a 64-bit value = reverse each 32-bit half AND swap them, | |
| // so the old High word becomes lane 0 (low) and old Low becomes lane 1 | |
| // (high). | |
| Register SwappedVec = MRI->createVirtualRegister(GR.getRegClass(VecI32Type)); | |
| auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), | |
| TII.get(SPIRV::OpVectorShuffle)) | |
| .addDef(SwappedVec) | |
| .addUse(GR.getSPIRVTypeID(VecI32Type)) | |
| .addUse(Reverse32) | |
| .addUse(Reverse32); | |
| for (unsigned J = 0; J < ComponentCount; ++J) { | |
| MIB.addImm(2 * J + 1); | |
| MIB.addImm(2 * J); | |
| } | |
| MIB.constrainAllUses(TII, TRI, RBI); | |
| // Groups 32 bit vector back to 64 bit scalar. | |
| return selectOpWithSrcs(ResVReg, ResType, I, {SwappedVec}, SPIRV::OpBitcast); | |
| } | |
| bool SPIRVInstructionSelector::selectBitreverseNative(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I, | |
| Register Op) const { | |
| MachineBasicBlock &BB = *I.getParent(); | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitReverse)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(Op) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectBitreverse(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| Register OpReg = I.getOperand(1).getReg(); | |
| // TODO: Fix shader behavior in case of VK_KHR_maintenance9 extension is set | |
| if (STI.isShader()) { | |
| SPIRVTypeInst OpType = GR.getSPIRVTypeForVReg(OpReg); | |
| switch (GR.getScalarOrVectorBitWidth(OpType)) { | |
| case 8: | |
| case 16: | |
| case 24: | |
| return selectBitreverseViaI32(ResVReg, ResType, I, OpReg); | |
| case 32: | |
| return selectBitreverseNative(ResVReg, ResType, I, OpReg); | |
| case 64: | |
| return selectBitreverse64(ResVReg, ResType, I, OpReg); | |
| } | |
| return SPIRVInstructionSelector::diagnoseUnsupported( | |
| I, "G_BITREVERSE only support 16,32,64 bits."); | |
| } | |
| if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_bit_instructions)) | |
| return selectBitreverseNative(ResVReg, ResType, I, OpReg); | |
| // Expansion bitreverse using bit manipulation operations | |
| // Algo: https://graphics.stanford.edu/~seander/bithacks.html#ReverseParallel | |
| const unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType); | |
| // TODO: add support for any bit width and bitwidth more than 64. | |
| if (BitWidth > 64 || !isPowerOf2_32(BitWidth)) | |
| return false; | |
| const unsigned N = GR.getScalarOrVectorComponentCount(ResType); | |
| unsigned AndOp = SPIRV::OpBitwiseAndS; | |
| unsigned OrOp = SPIRV::OpBitwiseOrS; | |
| unsigned ShlOp = SPIRV::OpShiftLeftLogicalS; | |
| unsigned ShrOp = SPIRV::OpShiftRightLogicalS; | |
| if (N > 1 || (ResType->getOpcode() == SPIRV::OpTypeVectorIdEXT && | |
| STI.canUseExtension(SPIRV::Extension::SPV_EXT_long_vector))) { | |
| AndOp = SPIRV::OpBitwiseAndV; | |
| OrOp = SPIRV::OpBitwiseOrV; | |
| ShlOp = SPIRV::OpShiftLeftLogicalV; | |
| ShrOp = SPIRV::OpShiftRightLogicalV; | |
| } | |
| // Helper, one swap per step: ((input>>shift)&mask)|((input&mask)<<shift), | |
| // RPN: input shift >> mask & input mask & shift << | | |
| auto SwapBits = [&](const Register Input, const uint64_t Mask, | |
| const unsigned Shift) -> Register { | |
| auto CreateConst = [&](const uint64_t Value) -> Register { | |
| if (N == 1 && | |
| (ResType->getOpcode() != SPIRV::OpTypeVectorIdEXT || | |
| !STI.canUseExtension(SPIRV::Extension::SPV_EXT_long_vector))) | |
| return GR.getOrCreateConstInt( | |
| Value, I, GR.retrieveScalarOrVectorIntType(ResType), TII); | |
| return GR.getOrCreateConstVector(Value, I, ResType, TII); | |
| }; | |
| Register MaskReg = CreateConst(Mask); | |
| Register ShiftReg = CreateConst(Shift); | |
| Register T1 = MRI->createVirtualRegister(GR.getRegClass(ResType)); | |
| Register T2 = MRI->createVirtualRegister(GR.getRegClass(ResType)); | |
| Register T3 = MRI->createVirtualRegister(GR.getRegClass(ResType)); | |
| Register T4 = MRI->createVirtualRegister(GR.getRegClass(ResType)); | |
| Register Result = MRI->createVirtualRegister(GR.getRegClass(ResType)); | |
| if (!selectOpWithSrcs(T1, ResType, I, {Input, ShiftReg}, ShrOp) || | |
| !selectOpWithSrcs(T2, ResType, I, {T1, MaskReg}, AndOp) || | |
| !selectOpWithSrcs(T3, ResType, I, {Input, MaskReg}, AndOp) || | |
| !selectOpWithSrcs(T4, ResType, I, {T3, ShiftReg}, ShlOp) || | |
| !selectOpWithSrcs(Result, ResType, I, {T2, T4}, OrOp)) | |
| return Register(); | |
| return Result; | |
| }; | |
| unsigned Shift = BitWidth; | |
| Register Result = OpReg; | |
| uint64_t Mask = ~0ull; | |
| while ((Shift >>= 1) > 0) { | |
| Mask ^= (Mask << Shift); | |
| Result = SwapBits(Result, Mask, Shift); | |
| if (!Result.isValid()) | |
| return false; | |
| } | |
| return BuildCOPY(ResVReg, Result, I); | |
| } | |
| bool SPIRVInstructionSelector::selectFreeze(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| assert(I.getOperand(0).isReg() && I.getOperand(1).isReg() && | |
| "G_FREEZE must define and use a register"); | |
| Register OpReg = I.getOperand(1).getReg(); | |
| // With SPV_KHR_poison_freeze, lower `freeze` to OpFreezeKHR. | |
| if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_poison_freeze)) { | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpFreezeKHR)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(OpReg) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| // There is no way to implement `freeze` correctly without support on SPIR-V | |
| // standard side, but we may at least address a simple (static) case when | |
| // undef/poison value presence is obvious. The main benefit of even | |
| // incomplete `freeze` support is preventing of translation from crashing due | |
| // to lack of support on legalization and instruction selection steps. | |
| if (MachineInstr *Def = MRI->getVRegDef(OpReg)) { | |
| if (Def->getOpcode() == TargetOpcode::COPY) | |
| Def = MRI->getVRegDef(Def->getOperand(1).getReg()); | |
| Register Reg; | |
| switch (Def->getOpcode()) { | |
| case SPIRV::ASSIGN_TYPE: | |
| if (MachineInstr *AssignToDef = | |
| MRI->getVRegDef(Def->getOperand(1).getReg())) { | |
| if (AssignToDef->getOpcode() == TargetOpcode::G_IMPLICIT_DEF) | |
| Reg = Def->getOperand(2).getReg(); | |
| } | |
| break; | |
| case SPIRV::OpUndef: | |
| Reg = Def->getOperand(1).getReg(); | |
| break; | |
| } | |
| unsigned DestOpCode; | |
| if (Reg.isValid()) { | |
| DestOpCode = SPIRV::OpConstantNull; | |
| LLVM_DEBUG(dbgs() << "SPV_KHR_poison_freeze is not enabled. freeze of a " | |
| "static undef/poison lowered to OpConstantNull\n"); | |
| } else { | |
| DestOpCode = TargetOpcode::COPY; | |
| Reg = OpReg; | |
| LLVM_DEBUG(dbgs() << "SPV_KHR_poison_freeze is not enabled. freeze " | |
| "skipped, lowered as a copy of the operand\n"); | |
| } | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(DestOpCode)) | |
| .addDef(I.getOperand(0).getReg()) | |
| .addUse(Reg) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| return false; | |
| } | |
| bool SPIRVInstructionSelector::selectBuildVector(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| unsigned N = 0; | |
| if (isVectorType(ResType)) | |
| N = GR.getScalarOrVectorComponentCount(ResType); | |
| else if (ResType->getOpcode() == SPIRV::OpTypeArray) | |
| N = getArrayComponentCount(MRI, ResType); | |
| else | |
| report_fatal_error("Cannot select G_BUILD_VECTOR with a non-vector result"); | |
| if (I.getNumExplicitOperands() - I.getNumExplicitDefs() != N) | |
| report_fatal_error("G_BUILD_VECTOR and the result type are inconsistent"); | |
| // check if we may construct a constant vector | |
| bool IsConst = true; | |
| for (unsigned i = I.getNumExplicitDefs(); | |
| i < I.getNumExplicitOperands() && IsConst; ++i) | |
| if (!isConstReg(MRI, I.getOperand(i).getReg())) | |
| IsConst = false; | |
| if (!IsConst && (N < 2 && ResType->getOpcode() != SPIRV::OpTypeVectorIdEXT)) | |
| return diagnoseUnsupported( | |
| I, "There must be at least two constituent operands in a vector"); | |
| MRI->setRegClass(ResVReg, GR.getRegClass(ResType)); | |
| bool IsNullVector = IsConst && !STI.isShader(); | |
| for (unsigned i = I.getNumExplicitDefs(); | |
| i < I.getNumExplicitOperands() && IsNullVector; ++i) { | |
| MachineInstr *Def = getDef(I.getOperand(i), MRI); | |
| IsNullVector = Def && isNullOrNullSplat(*Def, *MRI); | |
| } | |
| if (IsNullVector) { | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), | |
| TII.get(IsConst ? SPIRV::OpConstantComposite | |
| : SPIRV::OpCompositeConstruct)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)); | |
| for (unsigned i = I.getNumExplicitDefs(); i < I.getNumExplicitOperands(); ++i) | |
| MIB.addUse(I.getOperand(i).getReg()); | |
| MIB.constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectSplatVector(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| unsigned N = 0; | |
| if (isVectorType(ResType)) | |
| N = GR.getScalarOrVectorComponentCount(ResType); | |
| else if (ResType->getOpcode() == SPIRV::OpTypeArray) | |
| N = getArrayComponentCount(MRI, ResType); | |
| else | |
| report_fatal_error("Cannot select G_SPLAT_VECTOR with a non-vector result"); | |
| unsigned OpIdx = I.getNumExplicitDefs(); | |
| if (!I.getOperand(OpIdx).isReg()) | |
| report_fatal_error("Unexpected argument in G_SPLAT_VECTOR"); | |
| // check if we may construct a constant vector | |
| Register OpReg = I.getOperand(OpIdx).getReg(); | |
| bool IsConst = isConstReg(MRI, OpReg); | |
| if (!IsConst && (N < 2 && ResType->getOpcode() != SPIRV::OpTypeVectorIdEXT)) | |
| return diagnoseUnsupported( | |
| I, "There must be at least two constituent operands in a vector"); | |
| MRI->setRegClass(ResVReg, GR.getRegClass(ResType)); | |
| auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), | |
| TII.get(IsConst ? SPIRV::OpConstantComposite | |
| : SPIRV::OpCompositeConstruct)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)); | |
| for (unsigned i = 0; i < N; ++i) | |
| MIB.addUse(OpReg); | |
| MIB.constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectConcatVectors(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| // Implement G_CONCAT_VECTORS using OpCompositeConstruct, which allows vector | |
| // constituents that share the result's component type to be | |
| // concatenated in operand order. | |
| if (!isVectorType(ResType)) | |
| report_fatal_error( | |
| "Cannot select G_CONCAT_VECTORS with a non-vector result"); | |
| auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), | |
| TII.get(SPIRV::OpCompositeConstruct)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)); | |
| for (unsigned OpIdx = I.getNumExplicitDefs(); | |
| OpIdx < I.getNumExplicitOperands(); ++OpIdx) | |
| MIB.addUse(I.getOperand(OpIdx).getReg()); | |
| MIB.constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectDiscard(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| unsigned Opcode; | |
| MachineBasicBlock &BB = *I.getParent(); | |
| if (STI.canUseExtension( | |
| SPIRV::Extension::SPV_EXT_demote_to_helper_invocation) || | |
| STI.isAtLeastSPIRVVer(llvm::VersionTuple(1, 6))) { | |
| Opcode = SPIRV::OpDemoteToHelperInvocation; | |
| } else { | |
| Opcode = SPIRV::OpKill; | |
| // OpKill must be the last operation of any basic block. | |
| for (MachineInstr &ToErase : make_early_inc_range( | |
| make_range(std::next(I.getIterator()), BB.instr_end()))) { | |
| GR.invalidateMachineInstr(&ToErase); | |
| ToErase.eraseFromParent(); | |
| } | |
| } | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectCmp(Register ResVReg, | |
| SPIRVTypeInst ResType, unsigned CmpOpc, | |
| MachineInstr &I) const { | |
| Register Cmp0 = I.getOperand(2).getReg(); | |
| Register Cmp1 = I.getOperand(3).getReg(); | |
| assert(GR.getSPIRVTypeForVReg(Cmp0)->getOpcode() == | |
| GR.getSPIRVTypeForVReg(Cmp1)->getOpcode() && | |
| "CMP operands should have the same type"); | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(CmpOpc)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(Cmp0) | |
| .addUse(Cmp1) | |
| .setMIFlags(I.getFlags()) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectICmp(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| auto Pred = I.getOperand(1).getPredicate(); | |
| unsigned CmpOpc; | |
| Register CmpOperand = I.getOperand(2).getReg(); | |
| SPIRVTypeInst CmpOperandType = GR.getSPIRVTypeForVReg(CmpOperand); | |
| bool IsPtrCmp = CmpOperandType && CmpOperandType.isPointer(); | |
| if (IsPtrCmp) { | |
| CmpOpc = getPtrCmpOpcode(Pred); | |
| // OpPtrEqual/OpPtrNotEqual require both operands to share an identical | |
| // pointer type. If they are not OpBitcast is inserted. | |
| Register Op1 = I.getOperand(3).getReg(); | |
| SPIRVTypeInst Ty0 = GR.getSPIRVTypeForVReg(CmpOperand); | |
| if (Ty0 != GR.getSPIRVTypeForVReg(Op1)) { | |
| Register NewOp1 = createVirtualRegister(Ty0, &GR, MRI, MRI->getMF()); | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpBitcast)) | |
| .addDef(NewOp1) | |
| .addUse(GR.getSPIRVTypeID(Ty0)) | |
| .addUse(Op1) | |
| .constrainAllUses(TII, TRI, RBI); | |
| I.getOperand(3).setReg(NewOp1); | |
| } | |
| } else if (GR.isScalarOrVectorOfType(CmpOperand, SPIRV::OpTypeBool)) | |
| CmpOpc = getBoolCmpOpcode(Pred); | |
| else | |
| CmpOpc = getICmpOpcode(Pred); | |
| return selectCmp(ResVReg, ResType, CmpOpc, I); | |
| } | |
| Register | |
| SPIRVInstructionSelector::buildI32Constant(uint32_t Val, MachineInstr &I, | |
| SPIRVTypeInst ResType) const { | |
| Type *LLVMTy = IntegerType::get(GR.CurMF->getFunction().getContext(), 32); | |
| SPIRVTypeInst SpvI32Ty = | |
| ResType ? ResType : GR.getOrCreateSPIRVIntegerType(32, I, TII); | |
| // Find a constant in DT or build a new one. | |
| auto ConstInt = ConstantInt::get(LLVMTy, Val); | |
| Register NewReg = GR.find(ConstInt, GR.CurMF); | |
| if (!NewReg.isValid()) { | |
| NewReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); | |
| MachineBasicBlock &BB = *I.getParent(); | |
| MachineInstr *MI = | |
| Val == 0 | |
| ? BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) | |
| .addDef(NewReg) | |
| .addUse(GR.getSPIRVTypeID(SpvI32Ty)) | |
| : BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI)) | |
| .addDef(NewReg) | |
| .addUse(GR.getSPIRVTypeID(SpvI32Ty)) | |
| .addImm(APInt(32, Val).getZExtValue()); | |
| constrainSelectedInstRegOperands(*MI, TII, TRI, RBI); | |
| GR.add(ConstInt, MI); | |
| } | |
| return NewReg; | |
| } | |
| // Like buildI32Constant, but always inserts the constant definition in the | |
| // entry block so it dominates all uses regardless of block ordering. | |
| Register SPIRVInstructionSelector::buildI32ConstantInEntryBlock( | |
| uint32_t Val, MachineInstr &I, SPIRVTypeInst ResType) const { | |
| Type *LLVMTy = IntegerType::get(GR.CurMF->getFunction().getContext(), 32); | |
| SPIRVTypeInst SpvI32Ty = | |
| ResType ? ResType : GR.getOrCreateSPIRVIntegerType(32, I, TII); | |
| auto *ConstInt = ConstantInt::get(LLVMTy, Val); | |
| Register NewReg = GR.find(ConstInt, GR.CurMF); | |
| if (!NewReg.isValid()) { | |
| NewReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); | |
| auto InsertIt = getOpVariableMBBIt(*I.getMF()); | |
| MachineBasicBlock &EntryBB = *InsertIt->getParent(); | |
| MachineInstr *MI = nullptr; | |
| Register TypeReg = GR.getSPIRVTypeID(SpvI32Ty); | |
| DebugLoc DbgLoc = I.getDebugLoc(); | |
| if (Val == 0) { | |
| MI = BuildMI(EntryBB, InsertIt, DbgLoc, TII.get(SPIRV::OpConstantNull)) | |
| .addDef(NewReg) | |
| .addUse(TypeReg); | |
| } else { | |
| uint64_t ImmVal = APInt(32, Val).getZExtValue(); | |
| MI = BuildMI(EntryBB, InsertIt, DbgLoc, TII.get(SPIRV::OpConstantI)) | |
| .addDef(NewReg) | |
| .addUse(TypeReg) | |
| .addImm(ImmVal); | |
| } | |
| constrainSelectedInstRegOperands(*MI, TII, TRI, RBI); | |
| GR.add(ConstInt, MI); | |
| } | |
| return NewReg; | |
| } | |
| bool SPIRVInstructionSelector::selectFCmp(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| auto Pred = I.getOperand(1).getPredicate(); | |
| // OpOrdered and OpUnordered require the Kernel capability. For shaders, | |
| // express them as uno(a, b) = isnan(a) || isnan(b), ord(a, b) = !uno(a, b). | |
| if (STI.isShader() && | |
| (Pred == CmpInst::FCMP_ORD || Pred == CmpInst::FCMP_UNO)) { | |
| MachineBasicBlock &BB = *I.getParent(); | |
| Register ResTypeID = GR.getSPIRVTypeID(ResType); | |
| Register IsNan[2]; | |
| for (unsigned Idx = 0; Idx < 2; ++Idx) { | |
| IsNan[Idx] = createVirtualRegister(ResType, &GR, MRI, *I.getMF()); | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpIsNan)) | |
| .addDef(IsNan[Idx]) | |
| .addUse(ResTypeID) | |
| .addUse(I.getOperand(2 + Idx).getReg()) | |
| .constrainAllUses(TII, TRI, RBI); | |
| } | |
| Register UnoReg = | |
| Pred == CmpInst::FCMP_UNO | |
| ? ResVReg | |
| : createVirtualRegister(ResType, &GR, MRI, *I.getMF()); | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpLogicalOr)) | |
| .addDef(UnoReg) | |
| .addUse(ResTypeID) | |
| .addUse(IsNan[0]) | |
| .addUse(IsNan[1]) | |
| .constrainAllUses(TII, TRI, RBI); | |
| if (Pred == CmpInst::FCMP_ORD) | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpLogicalNot)) | |
| .addDef(ResVReg) | |
| .addUse(ResTypeID) | |
| .addUse(UnoReg) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| unsigned CmpOp = getFCmpOpcode(Pred); | |
| return selectCmp(ResVReg, ResType, CmpOp, I); | |
| } | |
| bool SPIRVInstructionSelector::selectExp10(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| if (STI.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) { | |
| return selectExtInst(ResVReg, ResType, I, CL::exp10); | |
| } | |
| if (STI.canUseExtInstSet(SPIRV::InstructionSet::GLSL_std_450)) { | |
| /// There is no exp10 in GLSL. Use exp10(x) = exp2(x * log2(10)) instead | |
| /// log2(10) ~= 3.3219280948874l | |
| if (!isVectorType(ResType) && !ResType.isAnyTypeFloat()) | |
| return false; | |
| MachineIRBuilder MIRBuilder(I); | |
| SPIRVTypeInst SpirvScalarType = GR.getScalarOrVectorComponentType(ResType); | |
| // Match the literal precision to the scalar type so the OpConstant | |
| // literal does not contain non-zero high-order bits that would fail | |
| // SPIR-V validation when the type is narrower than 32 bits (e.g. half). | |
| APFloat ConstVal(3.3219280948873623); | |
| bool LosesInfo; | |
| ConstVal.convert( | |
| getZeroFP(GR.getTypeForSPIRVType(SpirvScalarType)).getSemantics(), | |
| APFloat::rmNearestTiesToEven, &LosesInfo); | |
| Register ConstReg = | |
| GR.buildConstantFP(ConstVal, MIRBuilder, SpirvScalarType); | |
| Register ArgReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); | |
| auto Opcode = | |
| isVectorType(ResType) ? SPIRV::OpVectorTimesScalar : SPIRV::OpFMulS; | |
| if (!selectOpWithSrcs(ArgReg, ResType, I, | |
| {I.getOperand(1).getReg(), ConstReg}, Opcode)) | |
| return false; | |
| if (!selectExtInst(ResVReg, ResType, I, | |
| {{SPIRV::InstructionSet::GLSL_std_450, GL::Exp2}}, false, | |
| false, {ArgReg})) | |
| return false; | |
| return true; | |
| } | |
| return false; | |
| } | |
| Register SPIRVInstructionSelector::buildZerosVal(SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| // OpenCL uses nulls for Zero. In HLSL we don't use null constants. | |
| bool ZeroAsNull = !STI.isShader(); | |
| if (isVectorType(ResType)) | |
| return GR.getOrCreateConstVector(0UL, I, ResType, TII, ZeroAsNull); | |
| return GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull); | |
| } | |
| bool SPIRVInstructionSelector::isScalarOrVectorIntConstantZero( | |
| Register Reg) const { | |
| SPIRVTypeInst Type = GR.getSPIRVTypeForVReg(Reg); | |
| if (!Type) | |
| return false; | |
| SPIRVTypeInst CompType = GR.getScalarOrVectorComponentType(Type); | |
| if (!CompType || CompType->getOpcode() != SPIRV::OpTypeInt) | |
| return false; | |
| auto IsZero = [this](Register Reg) { | |
| MachineInstr *Def = getDefInstrMaybeConstant(Reg, MRI); | |
| if (!Def) | |
| return false; | |
| if (Def->getOpcode() == SPIRV::OpConstantNull) | |
| return true; | |
| if (Def->getOpcode() == TargetOpcode::G_CONSTANT || | |
| Def->getOpcode() == SPIRV::OpConstantI) | |
| return getIConstVal(Reg, MRI) == 0; | |
| return false; | |
| }; | |
| if (IsZero(Reg)) | |
| return true; | |
| MachineInstr *Def = MRI->getVRegDef(Reg); | |
| if (!Def) | |
| return false; | |
| if (Def->getOpcode() == TargetOpcode::G_BUILD_VECTOR || | |
| (Def->getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS && | |
| cast<GIntrinsic>(Def)->getIntrinsicID() == | |
| Intrinsic::spv_const_composite)) { | |
| unsigned StartOp = Def->getOpcode() == TargetOpcode::G_BUILD_VECTOR ? 1 : 2; | |
| for (unsigned i = StartOp; i < Def->getNumOperands(); ++i) { | |
| if (!IsZero(Def->getOperand(i).getReg())) | |
| return false; | |
| } | |
| return true; | |
| } | |
| return false; | |
| } | |
| Register SPIRVInstructionSelector::buildZerosValF(SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| // OpenCL uses nulls for Zero. In HLSL we don't use null constants. | |
| bool ZeroAsNull = !STI.isShader(); | |
| APFloat VZero = getZeroFP(GR.getTypeForSPIRVType(ResType)); | |
| if (isVectorType(ResType)) | |
| return GR.getOrCreateConstVector(VZero, I, ResType, TII, ZeroAsNull); | |
| return GR.getOrCreateConstFP(VZero, I, ResType, TII, ZeroAsNull); | |
| } | |
| Register SPIRVInstructionSelector::buildOnesValF(SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| // OpenCL uses nulls for Zero. In HLSL we don't use null constants. | |
| bool ZeroAsNull = !STI.isShader(); | |
| APFloat VOne = getOneFP(GR.getTypeForSPIRVType(ResType)); | |
| if (isVectorType(ResType)) | |
| return GR.getOrCreateConstVector(VOne, I, ResType, TII, ZeroAsNull); | |
| return GR.getOrCreateConstFP(VOne, I, ResType, TII, ZeroAsNull); | |
| } | |
| Register SPIRVInstructionSelector::buildOnesVal(bool AllOnes, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType); | |
| APInt One = | |
| AllOnes ? APInt::getAllOnes(BitWidth) : APInt::getOneBitSet(BitWidth, 0); | |
| if (isVectorType(ResType)) | |
| return GR.getOrCreateConstVector(One, I, ResType, TII); | |
| return GR.getOrCreateConstInt(One, I, ResType, TII); | |
| } | |
| Register SPIRVInstructionSelector::buildVectorSplat(Register ScalarReg, | |
| unsigned NumElts, | |
| MachineInstr &I) const { | |
| SPIRVTypeInst VecType = GR.getOrCreateSPIRVVectorType( | |
| GR.getSPIRVTypeForVReg(ScalarReg), NumElts, I, TII); | |
| Register SplatReg = MRI->createVirtualRegister(GR.getRegClass(VecType)); | |
| GR.assignSPIRVTypeToVReg(VecType, SplatReg, MRI->getMF()); | |
| auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), | |
| TII.get(SPIRV::OpCompositeConstruct)) | |
| .addDef(SplatReg) | |
| .addUse(GR.getSPIRVTypeID(VecType)); | |
| for (unsigned J = 0; J < NumElts; ++J) | |
| MIB.addUse(ScalarReg); | |
| MIB.constrainAllUses(TII, TRI, RBI); | |
| return SplatReg; | |
| } | |
| bool SPIRVInstructionSelector::selectSelect(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| Register SelectFirstArg = I.getOperand(2).getReg(); | |
| Register SelectSecondArg = I.getOperand(3).getReg(); | |
| assert(ResType == GR.getSPIRVTypeForVReg(SelectFirstArg) && | |
| ResType == GR.getSPIRVTypeForVReg(SelectSecondArg)); | |
| bool IsFloatTy = | |
| GR.isScalarOrVectorOfType(SelectFirstArg, SPIRV::OpTypeFloat); | |
| bool IsPtrTy = | |
| GR.isScalarOrVectorOfType(SelectFirstArg, SPIRV::OpTypePointer); | |
| Register CondReg = I.getOperand(1).getReg(); | |
| bool IsScalarBool = GR.isScalarOfType(CondReg, SPIRV::OpTypeBool); | |
| unsigned Opcode; | |
| if (isVectorType(GR.getSPIRVTypeForVReg(SelectFirstArg))) { | |
| // Before SPIR-V 1.4, the condition of an OpSelect with a vector result | |
| // must be a vector of Booleans with the same number of components, while | |
| // LLVM IR also allows a scalar i1 condition. Splat it in that case. | |
| if (IsScalarBool && !STI.isAtLeastSPIRVVer(VersionTuple(1, 4))) { | |
| unsigned NumElts = GR.getScalarOrVectorComponentCount(ResType); | |
| CondReg = buildVectorSplat(CondReg, NumElts, I); | |
| IsScalarBool = false; | |
| } | |
| if (IsFloatTy) { | |
| Opcode = IsScalarBool ? SPIRV::OpSelectVFSCond : SPIRV::OpSelectVFVCond; | |
| } else if (IsPtrTy) { | |
| Opcode = IsScalarBool ? SPIRV::OpSelectVPSCond : SPIRV::OpSelectVPVCond; | |
| } else { | |
| Opcode = IsScalarBool ? SPIRV::OpSelectVISCond : SPIRV::OpSelectVIVCond; | |
| } | |
| } else { | |
| assert(IsScalarBool && "OpSelect with a scalar result requires a scalar " | |
| "boolean condition"); | |
| if (IsFloatTy) { | |
| Opcode = SPIRV::OpSelectSFSCond; | |
| } else if (IsPtrTy) { | |
| Opcode = SPIRV::OpSelectSPSCond; | |
| } else { | |
| Opcode = SPIRV::OpSelectSISCond; | |
| } | |
| } | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(CondReg) | |
| .addUse(SelectFirstArg) | |
| .addUse(SelectSecondArg) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| // This function is used to extend a bool or a vector of bools into an integer | |
| // or vector of integers. | |
| bool SPIRVInstructionSelector::selectBoolToInt(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| Register BooleanVReg, | |
| MachineInstr &InsertAt, | |
| bool IsSigned) const { | |
| // To extend a bool, we need to use OpSelect between constants. | |
| Register ZeroReg = buildZerosVal(ResType, InsertAt); | |
| Register OneReg = buildOnesVal(IsSigned, ResType, InsertAt); | |
| bool IsScalarBool = GR.isScalarOfType(BooleanVReg, SPIRV::OpTypeBool); | |
| unsigned Opcode = | |
| IsScalarBool ? SPIRV::OpSelectSISCond : SPIRV::OpSelectVIVCond; | |
| BuildMI(*InsertAt.getParent(), InsertAt, InsertAt.getDebugLoc(), | |
| TII.get(Opcode)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(BooleanVReg) | |
| .addUse(OneReg) | |
| .addUse(ZeroReg) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectIToF(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I, bool IsSigned, | |
| unsigned Opcode) const { | |
| Register SrcReg = I.getOperand(1).getReg(); | |
| // We can convert bool value directly to float type without OpConvert*ToF, | |
| // however the translator generates OpSelect+OpConvert*ToF, so we do the same. | |
| if (GR.isScalarOrVectorOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool)) { | |
| unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType); | |
| SPIRVTypeInst TmpType = GR.getOrCreateSPIRVIntegerType(BitWidth, I, TII); | |
| if (isVectorType(ResType)) { | |
| const unsigned NumElts = GR.getScalarOrVectorComponentCount(ResType); | |
| TmpType = GR.getOrCreateSPIRVVectorType(TmpType, NumElts, I, TII); | |
| } | |
| SrcReg = createVirtualRegister(TmpType, &GR, MRI, MRI->getMF()); | |
| selectBoolToInt(SrcReg, TmpType, I.getOperand(1).getReg(), I, IsSigned); | |
| } | |
| return selectOpWithSrcs(ResVReg, ResType, I, {SrcReg}, Opcode); | |
| } | |
| bool SPIRVInstructionSelector::selectExt(Register ResVReg, | |
| SPIRVTypeInst ResType, MachineInstr &I, | |
| bool IsSigned) const { | |
| Register SrcReg = I.getOperand(1).getReg(); | |
| if (GR.isScalarOrVectorOfType(SrcReg, SPIRV::OpTypeBool)) | |
| return selectBoolToInt(ResVReg, ResType, I.getOperand(1).getReg(), I, | |
| IsSigned); | |
| SPIRVTypeInst SrcType = GR.getSPIRVTypeForVReg(SrcReg); | |
| if (ResType == SrcType) | |
| return BuildCOPY(ResVReg, SrcReg, I); | |
| unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; | |
| return selectUnOp(ResVReg, ResType, I, Opcode); | |
| } | |
| bool SPIRVInstructionSelector::selectSUCmp(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I, | |
| bool IsSigned) const { | |
| MachineIRBuilder MIRBuilder(I); | |
| MachineRegisterInfo *MRI = MIRBuilder.getMRI(); | |
| MachineBasicBlock &BB = *I.getParent(); | |
| // Ensure we have bool. | |
| SPIRVTypeInst BoolType = GR.getOrCreateSPIRVBoolType(I, TII); | |
| unsigned N = GR.getScalarOrVectorComponentCount(ResType); | |
| if (N > 1 || (ResType->getOpcode() == SPIRV::OpTypeVectorIdEXT && | |
| STI.canUseExtension(SPIRV::Extension::SPV_EXT_long_vector))) | |
| BoolType = GR.getOrCreateSPIRVVectorType(BoolType, N, I, TII); | |
| Register BoolTypeReg = GR.getSPIRVTypeID(BoolType); | |
| // Build less-than-equal and less-than. | |
| Register IsLessEqReg = | |
| createVirtualRegister(BoolType, &GR, MRI, MIRBuilder.getMF()); | |
| BuildMI(BB, I, I.getDebugLoc(), | |
| TII.get(IsSigned ? SPIRV::OpSLessThanEqual : SPIRV::OpULessThanEqual)) | |
| .addDef(IsLessEqReg) | |
| .addUse(BoolTypeReg) | |
| .addUse(I.getOperand(1).getReg()) | |
| .addUse(I.getOperand(2).getReg()) | |
| .constrainAllUses(TII, TRI, RBI); | |
| Register IsLessReg = | |
| createVirtualRegister(BoolType, &GR, MRI, MIRBuilder.getMF()); | |
| BuildMI(BB, I, I.getDebugLoc(), | |
| TII.get(IsSigned ? SPIRV::OpSLessThan : SPIRV::OpULessThan)) | |
| .addDef(IsLessReg) | |
| .addUse(BoolTypeReg) | |
| .addUse(I.getOperand(1).getReg()) | |
| .addUse(I.getOperand(2).getReg()) | |
| .constrainAllUses(TII, TRI, RBI); | |
| // Build selects. | |
| Register ResTypeReg = GR.getSPIRVTypeID(ResType); | |
| Register NegOneOrZeroReg = | |
| MRI->createVirtualRegister(GR.getRegClass(ResType)); | |
| MRI->setType(NegOneOrZeroReg, LLT::scalar(64)); | |
| GR.assignSPIRVTypeToVReg(ResType, NegOneOrZeroReg, MIRBuilder.getMF()); | |
| unsigned SelectOpcode = | |
| (N > 1 || (ResType->getOpcode() == SPIRV::OpTypeVectorIdEXT && | |
| STI.canUseExtension(SPIRV::Extension::SPV_EXT_long_vector))) | |
| ? SPIRV::OpSelectVIVCond | |
| : SPIRV::OpSelectSISCond; | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(SelectOpcode)) | |
| .addDef(NegOneOrZeroReg) | |
| .addUse(ResTypeReg) | |
| .addUse(IsLessReg) | |
| .addUse(buildOnesVal(true, ResType, I)) // -1 | |
| .addUse(buildZerosVal(ResType, I)) | |
| .constrainAllUses(TII, TRI, RBI); | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(SelectOpcode)) | |
| .addDef(ResVReg) | |
| .addUse(ResTypeReg) | |
| .addUse(IsLessEqReg) | |
| .addUse(NegOneOrZeroReg) // -1 or 0 | |
| .addUse(buildOnesVal(false, ResType, I)) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectIntToBool(Register IntReg, | |
| Register ResVReg, | |
| MachineInstr &I, | |
| SPIRVTypeInst IntTy, | |
| SPIRVTypeInst BoolTy) const { | |
| // To truncate to a bool, we use OpBitwiseAnd 1 and OpINotEqual to zero. | |
| Register BitIntReg = createVirtualRegister(IntTy, &GR, MRI, MRI->getMF()); | |
| unsigned Opcode = | |
| isVectorType(IntTy) ? SPIRV::OpBitwiseAndV : SPIRV::OpBitwiseAndS; | |
| Register Zero = buildZerosVal(IntTy, I); | |
| Register One = buildOnesVal(false, IntTy, I); | |
| MachineBasicBlock &BB = *I.getParent(); | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) | |
| .addDef(BitIntReg) | |
| .addUse(GR.getSPIRVTypeID(IntTy)) | |
| .addUse(IntReg) | |
| .addUse(One) | |
| .constrainAllUses(TII, TRI, RBI); | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(BoolTy)) | |
| .addUse(BitIntReg) | |
| .addUse(Zero) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectTrunc(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| Register IntReg = I.getOperand(1).getReg(); | |
| const SPIRVTypeInst ArgType = GR.getSPIRVTypeForVReg(IntReg); | |
| if (GR.isScalarOrVectorOfType(ResVReg, SPIRV::OpTypeBool)) | |
| return selectIntToBool(IntReg, ResVReg, I, ArgType, ResType); | |
| if (ArgType == ResType) | |
| return BuildCOPY(ResVReg, IntReg, I); | |
| bool IsSigned = GR.isScalarOrVectorSigned(ResType); | |
| unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; | |
| return selectUnOp(ResVReg, ResType, I, Opcode); | |
| } | |
| bool SPIRVInstructionSelector::selectConst(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| unsigned Opcode = I.getOpcode(); | |
| unsigned TpOpcode = ResType->getOpcode(); | |
| Register Reg; | |
| if (ResType.isPointer() || TpOpcode == SPIRV::OpTypeEvent) { | |
| assert(Opcode == TargetOpcode::G_CONSTANT && | |
| I.getOperand(1).getCImm()->isZero()); | |
| MachineBasicBlock &DepMBB = I.getMF()->front(); | |
| MachineIRBuilder MIRBuilder(DepMBB, DepMBB.getFirstNonPHI()); | |
| Reg = GR.getOrCreateConstNullPtr(MIRBuilder, ResType); | |
| } else if (TpOpcode == SPIRV::OpTypeVectorIdEXT) { | |
| // We ended up here coming from a splat on a <1 x T> type, which | |
| // IRTranslator translated into a scalar, so we have to restore the | |
| // vectorness. | |
| assert(GR.getScalarOrVectorComponentCount(ResType) == 1 && | |
| "Expected <1 x T> Vector!"); | |
| if (Opcode == TargetOpcode::G_FCONSTANT) | |
| Reg = GR.getOrCreateConstVector(I.getOperand(1).getFPImm()->getValue(), I, | |
| ResType, TII); | |
| else // We handle vector of pointer here as well. | |
| Reg = GR.getOrCreateConstVector(I.getOperand(1).getCImm()->getValue(), I, | |
| ResType, TII); | |
| } else if (Opcode == TargetOpcode::G_FCONSTANT) { | |
| Reg = GR.getOrCreateConstFP(I.getOperand(1).getFPImm()->getValue(), I, | |
| ResType, TII, !STI.isShader()); | |
| } else { | |
| Reg = GR.getOrCreateConstInt(I.getOperand(1).getCImm()->getValue(), I, | |
| ResType, TII, !STI.isShader()); | |
| } | |
| return Reg == ResVReg ? true : BuildCOPY(ResVReg, Reg, I); | |
| } | |
| bool SPIRVInstructionSelector::selectOpUndef(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectInsertVal(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| MachineBasicBlock &BB = *I.getParent(); | |
| auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeInsert)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| // object to insert | |
| .addUse(I.getOperand(3).getReg()) | |
| // composite to insert into | |
| .addUse(I.getOperand(2).getReg()); | |
| for (unsigned i = 4; i < I.getNumOperands(); i++) | |
| MIB.addImm(foldImm(I.getOperand(i), MRI)); | |
| MIB.constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectExtractVal(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| Type *MaybeResTy = nullptr; | |
| StringRef ResName; | |
| if (GR.findValueAttrs(&I, MaybeResTy, ResName) && | |
| MaybeResTy != GR.getTypeForSPIRVType(ResType)) { | |
| assert((!MaybeResTy || MaybeResTy->isAggregateType()) && | |
| "Expected aggregate type for extractv instruction"); | |
| ResType = GR.getOrCreateSPIRVType(MaybeResTy, I, | |
| SPIRV::AccessQualifier::ReadWrite, false); | |
| GR.assignSPIRVTypeToVReg(ResType, ResVReg, *I.getMF()); | |
| } | |
| MachineBasicBlock &BB = *I.getParent(); | |
| auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(I.getOperand(2).getReg()); | |
| for (unsigned i = 3; i < I.getNumOperands(); i++) | |
| MIB.addImm(foldImm(I.getOperand(i), MRI)); | |
| MIB.constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectInsertElt(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| if (getImm(I.getOperand(4), MRI)) | |
| return selectInsertVal(ResVReg, ResType, I); | |
| MachineBasicBlock &BB = *I.getParent(); | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorInsertDynamic)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(I.getOperand(2).getReg()) | |
| .addUse(I.getOperand(3).getReg()) | |
| .addUse(I.getOperand(4).getReg()) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectExtractElt(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| if (getImm(I.getOperand(3), MRI)) | |
| return selectExtractVal(ResVReg, ResType, I); | |
| MachineBasicBlock &BB = *I.getParent(); | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorExtractDynamic)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(I.getOperand(2).getReg()) | |
| .addUse(I.getOperand(3).getReg()) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectGEP(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| const bool IsGEPInBounds = I.getOperand(2).getImm(); | |
| // Pointers to opaque types stay typed even with the extension on, so emit the | |
| // untyped variant only when the result is actually an untyped pointer. | |
| const bool UseUntypedPointers = | |
| ResType->getOpcode() == SPIRV::OpTypeUntypedPointerKHR; | |
| // Determine the opcode based on pointer type and bounds checking. | |
| // When using untyped pointers, use OpUntyped*AccessChainKHR variants. | |
| unsigned Opcode; | |
| if (UseUntypedPointers) { | |
| if (STI.isLogicalSPIRV()) { | |
| Opcode = IsGEPInBounds ? SPIRV::OpUntypedInBoundsAccessChainKHR | |
| : SPIRV::OpUntypedAccessChainKHR; | |
| } else { | |
| Opcode = IsGEPInBounds ? SPIRV::OpUntypedInBoundsPtrAccessChainKHR | |
| : SPIRV::OpUntypedPtrAccessChainKHR; | |
| } | |
| } else { | |
| // OpAccessChain could be used for OpenCL, but the SPIRV-LLVM Translator | |
| // only relies on PtrAccessChain, so we'll try not to deviate. For Vulkan | |
| // however, we have to use Op[InBounds]AccessChain. | |
| // FIXME: fix llvm-spirv. | |
| if (STI.isLogicalSPIRV()) { | |
| Opcode = | |
| IsGEPInBounds ? SPIRV::OpInBoundsAccessChain : SPIRV::OpAccessChain; | |
| } else { | |
| Opcode = IsGEPInBounds ? SPIRV::OpInBoundsPtrAccessChain | |
| : SPIRV::OpPtrAccessChain; | |
| } | |
| } | |
| Register BaseReg = I.getOperand(3).getReg(); | |
| auto Res = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)); | |
| // For untyped access chains, we need to add the base type operand. | |
| if (UseUntypedPointers) { | |
| // Get the element type from the base pointer register. | |
| // For untyped pointers, this was stored when processing | |
| // spv_assign_ptr_type. | |
| SPIRVTypeInst BaseType = GR.getUntypedPtrElementType(BaseReg); | |
| if (!BaseType) { | |
| // Otherwise try the pointee type for mixed typed-pointer usage. | |
| SPIRVTypeInst BasePtrType = GR.getSPIRVTypeForVReg(BaseReg); | |
| BaseType = BasePtrType ? GR.getPointeeType(BasePtrType) : nullptr; | |
| } | |
| if (!BaseType) { | |
| // The base may be a not-yet-selected global. Read its value type from | |
| // the defining G_GLOBAL_VALUE, following copies. | |
| Register DefReg = BaseReg; | |
| MachineInstr *Def = MRI->getVRegDef(DefReg); | |
| while (Def && Def->getOpcode() == TargetOpcode::COPY && | |
| Def->getOperand(1).isReg()) | |
| Def = MRI->getVRegDef(Def->getOperand(1).getReg()); | |
| if (Def && Def->getOpcode() == TargetOpcode::G_GLOBAL_VALUE) | |
| if (const auto *GVar = | |
| dyn_cast<GlobalVariable>(Def->getOperand(1).getGlobal())) | |
| BaseType = GR.getOrCreateSPIRVType(GVar->getValueType(), I, | |
| SPIRV::AccessQualifier::ReadWrite, | |
| /*EmitIR=*/false); | |
| } | |
| if (!BaseType) | |
| return diagnoseUnsupported( | |
| I, "could not deduce the base type of an untyped access chain"); | |
| Res.addUse(GR.getSPIRVTypeID(BaseType)); | |
| } | |
| // Object to get a pointer to. | |
| Res.addUse(BaseReg); | |
| const bool IsAccessChainOpcode = | |
| (Opcode == SPIRV::OpAccessChain || | |
| Opcode == SPIRV::OpInBoundsAccessChain || | |
| Opcode == SPIRV::OpUntypedAccessChainKHR || | |
| Opcode == SPIRV::OpUntypedInBoundsAccessChainKHR); | |
| assert((!IsAccessChainOpcode || (getImm(I.getOperand(4), MRI) && | |
| foldImm(I.getOperand(4), MRI) == 0)) && | |
| "Cannot translate GEP to OpAccessChain."); | |
| // Adding indices. | |
| const unsigned StartingIndex = IsAccessChainOpcode ? 5 : 4; | |
| for (unsigned i = StartingIndex; i < I.getNumExplicitOperands(); ++i) | |
| Res.addUse(I.getOperand(i).getReg()); | |
| Res.constrainAllUses(TII, TRI, RBI); | |
| // IRTranslator doesn't like <1 x T> vectors, and treats them as scalars. This | |
| // creates an information loss issue, and also broken code where a | |
| // OpCompositeExtract is applied to the scalar return of an OpPtrAccessChain, | |
| // when it was meant to be applied to the OpCompositeInsert created <1 x T>. | |
| // TODO: should we also re-create the index <1 x T>? | |
| if (MRI->hasOneUse(ResVReg)) { | |
| MachineInstr &Extract = *MRI->use_instr_begin(ResVReg); | |
| if (Extract.getOpcode() == SPIRV::OpCompositeExtract) { | |
| SPIRVTypeInst V = GR.getOrCreateSPIRVVectorType(ResType, 1, Extract, TII); | |
| // We cannot use GlobalRegistry::getOrCreateUndef directly here because it | |
| // tries to use UndefVal::get, which does not work for TypedPointerType, | |
| // which we can get here if we're dealing with <1 x T*>. | |
| Register Tmp = createVirtualRegister(V, &GR, MRI, *I.getMF()); | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) | |
| .addDef(Tmp) | |
| .addUse(GR.getSPIRVTypeID(V)) | |
| .constrainAllUses(TII, TRI, RBI); | |
| Register InsertReg = createVirtualRegister(V, &GR, MRI, *I.getMF()); | |
| BuildMI(*I.getParent(), Extract, Extract.getDebugLoc(), | |
| TII.get(SPIRV::OpCompositeInsert)) | |
| .addDef(InsertReg) | |
| .addUse(GR.getSPIRVTypeID(V)) | |
| .addUse(ResVReg) | |
| .addUse(Tmp) | |
| .addImm(0) | |
| .constrainAllUses(TII, TRI, RBI); | |
| Extract.substituteRegister(ResVReg, InsertReg, 0, TRI); | |
| } | |
| } | |
| return true; | |
| } | |
| // Maybe wrap a value into OpSpecConstantOp | |
| bool SPIRVInstructionSelector::wrapIntoSpecConstantOp( | |
| MachineInstr &I, SmallVector<Register> &CompositeArgs) const { | |
| unsigned Lim = I.getNumExplicitOperands(); | |
| for (unsigned i = I.getNumExplicitDefs() + 1; i < Lim; ++i) { | |
| Register OpReg = I.getOperand(i).getReg(); | |
| MachineInstr *OpDefine = MRI->getVRegDef(OpReg); | |
| SPIRVTypeInst OpType = GR.getSPIRVTypeForVReg(OpReg); | |
| if (!OpDefine || !OpType || isConstReg(MRI, OpDefine) || | |
| OpDefine->getOpcode() == TargetOpcode::G_ADDRSPACE_CAST || | |
| OpDefine->getOpcode() == TargetOpcode::G_INTTOPTR || | |
| GR.isAggregateType(OpType)) { | |
| // The case of G_ADDRSPACE_CAST inside spv_const_composite() is processed | |
| // by selectAddrSpaceCast(), and G_INTTOPTR is processed by selectUnOp() | |
| CompositeArgs.push_back(OpReg); | |
| continue; | |
| } | |
| MachineFunction *MF = I.getMF(); | |
| Register WrapReg = GR.find(OpDefine, MF); | |
| if (WrapReg.isValid()) { | |
| CompositeArgs.push_back(WrapReg); | |
| continue; | |
| } | |
| SPIRVTypeInst WrapType = OpType; | |
| if (OpType->getOpcode() == SPIRV::OpTypePointer && | |
| GR.getPointerStorageClass(OpType) == | |
| SPIRV::StorageClass::CodeSectionINTEL) { | |
| WrapType = GR.changePointerStorageClass(OpType, | |
| SPIRV::StorageClass::Function, I); | |
| } | |
| WrapReg = MRI->createVirtualRegister(GR.getRegClass(WrapType)); | |
| CompositeArgs.push_back(WrapReg); | |
| // Decorate the wrapper register and generate a new instruction | |
| MRI->setType(WrapReg, LLT::pointer(0, 64)); | |
| GR.assignSPIRVTypeToVReg(WrapType, WrapReg, *MF); | |
| auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), | |
| TII.get(SPIRV::OpSpecConstantOp)) | |
| .addDef(WrapReg) | |
| .addUse(GR.getSPIRVTypeID(WrapType)) | |
| .addImm(static_cast<uint32_t>(SPIRV::Opcode::Bitcast)) | |
| .addUse(OpReg); | |
| GR.add(OpDefine, MIB); | |
| MIB.constrainAllUses(TII, TRI, RBI); | |
| } | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectDerivativeInst( | |
| Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I, | |
| const unsigned DPdOpCode) const { | |
| // TODO: This should check specifically for Fragment Execution Model, but STI | |
| // doesn't provide that information yet. See #167562 | |
| if (!errorIfInstrOutsideShader(I)) | |
| return false; | |
| // If the arg/result types are half then we need to wrap the instr in | |
| // conversions to float | |
| // This case occurs because a half arg/result is legal in HLSL but not spirv. | |
| Register SrcReg = I.getOperand(2).getReg(); | |
| SPIRVTypeInst SrcType = GR.getSPIRVTypeForVReg(SrcReg); | |
| unsigned BitWidth = std::min(GR.getScalarOrVectorBitWidth(SrcType), | |
| GR.getScalarOrVectorBitWidth(ResType)); | |
| if (BitWidth == 32) | |
| return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(DPdOpCode)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(I.getOperand(2).getReg()); | |
| MachineIRBuilder MIRBuilder(I); | |
| unsigned componentCount = GR.getScalarOrVectorComponentCount(SrcType); | |
| SPIRVTypeInst F32ConvertTy = GR.getOrCreateSPIRVFloatType(32, I, TII); | |
| if (componentCount != 1) | |
| F32ConvertTy = GR.getOrCreateSPIRVVectorType(F32ConvertTy, componentCount, | |
| MIRBuilder, false); | |
| const TargetRegisterClass *RegClass = GR.getRegClass(SrcType); | |
| Register ConvertToVReg = MRI->createVirtualRegister(RegClass); | |
| Register DpdOpVReg = MRI->createVirtualRegister(RegClass); | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpFConvert)) | |
| .addDef(ConvertToVReg) | |
| .addUse(GR.getSPIRVTypeID(F32ConvertTy)) | |
| .addUse(SrcReg) | |
| .constrainAllUses(TII, TRI, RBI); | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(DPdOpCode)) | |
| .addDef(DpdOpVReg) | |
| .addUse(GR.getSPIRVTypeID(F32ConvertTy)) | |
| .addUse(ConvertToVReg) | |
| .constrainAllUses(TII, TRI, RBI); | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpFConvert)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(DpdOpVReg) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| MachineBasicBlock &BB = *I.getParent(); | |
| Intrinsic::ID IID = cast<GIntrinsic>(I).getIntrinsicID(); | |
| switch (IID) { | |
| case Intrinsic::spv_load: | |
| return selectLoad(ResVReg, ResType, I); | |
| case Intrinsic::spv_atomic_load: | |
| return selectAtomicLoad(ResVReg, ResType, I); | |
| case Intrinsic::spv_store: | |
| return selectStore(I); | |
| case Intrinsic::spv_atomic_store: | |
| return selectAtomicStore(I); | |
| case Intrinsic::spv_extractv: | |
| return selectExtractVal(ResVReg, ResType, I); | |
| case Intrinsic::spv_insertv: | |
| return selectInsertVal(ResVReg, ResType, I); | |
| case Intrinsic::spv_extractelt: | |
| return selectExtractElt(ResVReg, ResType, I); | |
| case Intrinsic::spv_insertelt: | |
| return selectInsertElt(ResVReg, ResType, I); | |
| case Intrinsic::spv_gep: | |
| return selectGEP(ResVReg, ResType, I); | |
| case Intrinsic::spv_bitcast: { | |
| Register OpReg = I.getOperand(2).getReg(); | |
| SPIRVTypeInst OpType = | |
| OpReg.isValid() ? GR.getSPIRVTypeForVReg(OpReg) : nullptr; | |
| if (!GR.isBitcastCompatible(ResType, OpType)) | |
| report_fatal_error("incompatible result and operand types in a bitcast"); | |
| return selectOpWithSrcs(ResVReg, ResType, I, {OpReg}, SPIRV::OpBitcast); | |
| } | |
| case Intrinsic::spv_unref_global: | |
| case Intrinsic::spv_init_global: { | |
| MachineInstr *MI = MRI->getVRegDef(I.getOperand(1).getReg()); | |
| MachineInstr *Init = I.getNumExplicitOperands() > 2 | |
| ? MRI->getVRegDef(I.getOperand(2).getReg()) | |
| : nullptr; | |
| assert(MI); | |
| Register GVarVReg = MI->getOperand(0).getReg(); | |
| if (!selectGlobalValue(GVarVReg, *MI, Init)) | |
| return false; | |
| // We violate SSA form by inserting OpVariable and still having a gMIR | |
| // instruction %vreg = G_GLOBAL_VALUE @gvar. We need to fix this by erasing | |
| // the duplicated definition. | |
| if (MI->getOpcode() == TargetOpcode::G_GLOBAL_VALUE) { | |
| GR.invalidateMachineInstr(MI); | |
| MI->eraseFromParent(); | |
| } | |
| return true; | |
| } | |
| case Intrinsic::spv_undef: { | |
| auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)); | |
| MIB.constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| case Intrinsic::spv_poison: | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpPoisonKHR)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| case Intrinsic::spv_freeze: | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpFreezeKHR)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(I.getOperand(2).getReg()) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| case Intrinsic::spv_named_boolean_spec_constant: { | |
| auto Opcode = I.getOperand(3).getImm() ? SPIRV::OpSpecConstantTrue | |
| : SPIRV::OpSpecConstantFalse; | |
| auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) | |
| .addDef(I.getOperand(0).getReg()) | |
| .addUse(GR.getSPIRVTypeID(ResType)); | |
| MIB.constrainAllUses(TII, TRI, RBI); | |
| unsigned SpecId = I.getOperand(2).getImm(); | |
| buildOpDecorate(I.getOperand(0).getReg(), *++MIB->getIterator(), TII, | |
| SPIRV::Decoration::SpecId, {SpecId}); | |
| return true; | |
| } | |
| case Intrinsic::spv_const_composite: { | |
| // If no values are attached, the composite is null constant. | |
| bool IsNull = I.getNumExplicitDefs() + 1 == I.getNumExplicitOperands(); | |
| SmallVector<Register> CompositeArgs; | |
| MRI->setRegClass(ResVReg, GR.getRegClass(ResType)); | |
| // skip type MD node we already used when generated assign.type for this | |
| if (!IsNull) { | |
| if (!wrapIntoSpecConstantOp(I, CompositeArgs)) | |
| return false; | |
| std::function<bool(Register)> HasSpecConstOperand = | |
| [&](Register Reg) -> bool { | |
| MachineInstr *Def = MRI->getVRegDef(Reg); | |
| if (!Def) | |
| return false; | |
| if (!isConstReg(MRI, Def)) | |
| return true; | |
| // Recurse into not-yet-selected spv_const_composite intrinsics | |
| // to detect transitive spec constant operands. | |
| if (isSpvIntrinsic(*Def, Intrinsic::spv_const_composite)) { | |
| for (unsigned J = Def->getNumExplicitDefs() + 1; | |
| J < Def->getNumExplicitOperands(); ++J) { | |
| if (Def->getOperand(J).isReg() && | |
| HasSpecConstOperand(Def->getOperand(J).getReg())) | |
| return true; | |
| } | |
| } | |
| return false; | |
| }; | |
| bool HasSpecConst = llvm::any_of(CompositeArgs, HasSpecConstOperand); | |
| unsigned CompositeOpc = HasSpecConst ? SPIRV::OpSpecConstantComposite | |
| : SPIRV::OpConstantComposite; | |
| unsigned ContinuedOpc = HasSpecConst | |
| ? SPIRV::OpSpecConstantCompositeContinuedINTEL | |
| : SPIRV::OpConstantCompositeContinuedINTEL; | |
| MachineIRBuilder MIR(I); | |
| SmallVector<MachineInstr *, 4> Instructions = createContinuedInstructions( | |
| MIR, CompositeOpc, 3, ContinuedOpc, CompositeArgs, ResVReg, | |
| GR.getSPIRVTypeID(ResType)); | |
| for (auto *Instr : Instructions) { | |
| Instr->setDebugLoc(I.getDebugLoc()); | |
| constrainSelectedInstRegOperands(*Instr, TII, TRI, RBI); | |
| } | |
| return true; | |
| } else { | |
| auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)); | |
| MIB.constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| } | |
| case Intrinsic::spv_assign_name: { | |
| auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpName)); | |
| MIB.addUse(I.getOperand(I.getNumExplicitDefs() + 1).getReg()); | |
| for (unsigned i = I.getNumExplicitDefs() + 2; | |
| i < I.getNumExplicitOperands(); ++i) { | |
| MIB.addImm(I.getOperand(i).getImm()); | |
| } | |
| MIB.constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| case Intrinsic::spv_switch: { | |
| auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSwitch)); | |
| for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) { | |
| if (I.getOperand(i).isReg()) | |
| MIB.addReg(I.getOperand(i).getReg()); | |
| else if (I.getOperand(i).isCImm()) | |
| addNumImm(I.getOperand(i).getCImm()->getValue(), MIB); | |
| else if (I.getOperand(i).isMBB()) | |
| MIB.addMBB(I.getOperand(i).getMBB()); | |
| else | |
| llvm_unreachable("Unexpected OpSwitch operand"); | |
| } | |
| MIB.constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| case Intrinsic::spv_loop_merge: { | |
| auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpLoopMerge)); | |
| for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) { | |
| if (I.getOperand(i).isMBB()) | |
| MIB.addMBB(I.getOperand(i).getMBB()); | |
| else | |
| MIB.addImm(foldImm(I.getOperand(i), MRI)); | |
| } | |
| MIB.constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| case Intrinsic::spv_loop_control_intel: { | |
| auto MIB = | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpLoopControlINTEL)); | |
| for (unsigned J = 1; J < I.getNumExplicitOperands(); ++J) | |
| MIB.addImm(foldImm(I.getOperand(J), MRI)); | |
| MIB.constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| case Intrinsic::spv_selection_merge: { | |
| auto MIB = | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSelectionMerge)); | |
| assert(I.getOperand(1).isMBB() && | |
| "operand 1 to spv_selection_merge must be a basic block"); | |
| MIB.addMBB(I.getOperand(1).getMBB()); | |
| MIB.addImm(getSelectionOperandForImm(I.getOperand(2).getImm())); | |
| MIB.constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| case Intrinsic::spv_cmpxchg: | |
| return selectAtomicCmpXchg(ResVReg, ResType, I); | |
| case Intrinsic::spv_unreachable: | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUnreachable)) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| case Intrinsic::spv_abort: | |
| return selectAbort(I); | |
| case Intrinsic::spv_alloca: | |
| return selectFrameIndex(ResVReg, ResType, I); | |
| case Intrinsic::spv_alloca_array: | |
| return selectAllocaArray(ResVReg, ResType, I); | |
| case Intrinsic::spv_assume: | |
| if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) { | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpAssumeTrueKHR)) | |
| .addUse(I.getOperand(1).getReg()) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| break; | |
| case Intrinsic::spv_expect: | |
| if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) { | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExpectKHR)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(I.getOperand(2).getReg()) | |
| .addUse(I.getOperand(3).getReg()) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| break; | |
| case Intrinsic::arithmetic_fence: | |
| if (STI.canUseExtension(SPIRV::Extension::SPV_EXT_arithmetic_fence)) { | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpArithmeticFenceEXT)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(I.getOperand(2).getReg()) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } else | |
| return BuildCOPY(ResVReg, I.getOperand(2).getReg(), I); | |
| break; | |
| case Intrinsic::spv_thread_id: | |
| // The HLSL SV_DispatchThreadID semantic is lowered to llvm.spv.thread.id | |
| // intrinsic in LLVM IR for SPIR-V backend. | |
| // | |
| // In SPIR-V backend, llvm.spv.thread.id is now correctly translated to a | |
| // `GlobalInvocationId` builtin variable | |
| return loadVec3BuiltinInputID(SPIRV::BuiltIn::GlobalInvocationId, ResVReg, | |
| ResType, I); | |
| case Intrinsic::spv_thread_id_in_group: | |
| // The HLSL SV_GroupThreadId semantic is lowered to | |
| // llvm.spv.thread.id.in.group intrinsic in LLVM IR for SPIR-V backend. | |
| // | |
| // In SPIR-V backend, llvm.spv.thread.id.in.group is now correctly | |
| // translated to a `LocalInvocationId` builtin variable | |
| return loadVec3BuiltinInputID(SPIRV::BuiltIn::LocalInvocationId, ResVReg, | |
| ResType, I); | |
| case Intrinsic::spv_group_id: | |
| // The HLSL SV_GroupId semantic is lowered to | |
| // llvm.spv.group.id intrinsic in LLVM IR for SPIR-V backend. | |
| // | |
| // In SPIR-V backend, llvm.spv.group.id is now translated to a `WorkgroupId` | |
| // builtin variable | |
| return loadVec3BuiltinInputID(SPIRV::BuiltIn::WorkgroupId, ResVReg, ResType, | |
| I); | |
| case Intrinsic::spv_flattened_thread_id_in_group: | |
| // The HLSL SV_GroupIndex semantic is lowered to | |
| // llvm.spv.flattened.thread.id.in.group() intrinsic in LLVM IR for SPIR-V | |
| // backend. | |
| // | |
| // In SPIR-V backend, llvm.spv.flattened.thread.id.in.group is translated to | |
| // a `LocalInvocationIndex` builtin variable | |
| return loadBuiltinInputID(SPIRV::BuiltIn::LocalInvocationIndex, ResVReg, | |
| ResType, I); | |
| case Intrinsic::spv_workgroup_size: | |
| return loadVec3BuiltinInputID(SPIRV::BuiltIn::WorkgroupSize, ResVReg, | |
| ResType, I); | |
| case Intrinsic::spv_global_size: | |
| return loadVec3BuiltinInputID(SPIRV::BuiltIn::GlobalSize, ResVReg, ResType, | |
| I); | |
| case Intrinsic::spv_global_offset: | |
| return loadVec3BuiltinInputID(SPIRV::BuiltIn::GlobalOffset, ResVReg, | |
| ResType, I); | |
| case Intrinsic::spv_num_workgroups: | |
| return loadVec3BuiltinInputID(SPIRV::BuiltIn::NumWorkgroups, ResVReg, | |
| ResType, I); | |
| case Intrinsic::spv_subgroup_size: | |
| return loadBuiltinInputID(SPIRV::BuiltIn::SubgroupSize, ResVReg, ResType, | |
| I); | |
| case Intrinsic::spv_num_subgroups: | |
| return loadBuiltinInputID(SPIRV::BuiltIn::NumSubgroups, ResVReg, ResType, | |
| I); | |
| case Intrinsic::spv_subgroup_id: | |
| return loadBuiltinInputID(SPIRV::BuiltIn::SubgroupId, ResVReg, ResType, I); | |
| case Intrinsic::spv_subgroup_local_invocation_id: | |
| return loadBuiltinInputID(SPIRV::BuiltIn::SubgroupLocalInvocationId, | |
| ResVReg, ResType, I); | |
| case Intrinsic::spv_subgroup_max_size: | |
| return loadBuiltinInputID(SPIRV::BuiltIn::SubgroupMaxSize, ResVReg, ResType, | |
| I); | |
| case Intrinsic::spv_fdot: | |
| return selectFloatDot(ResVReg, ResType, I); | |
| case Intrinsic::spv_udot: | |
| case Intrinsic::spv_sdot: | |
| if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) || | |
| STI.isAtLeastSPIRVVer(VersionTuple(1, 6))) | |
| return selectIntegerDot(ResVReg, ResType, I, | |
| /*Signed=*/IID == Intrinsic::spv_sdot); | |
| return selectIntegerDotExpansion(ResVReg, ResType, I); | |
| case Intrinsic::spv_dot4add_i8packed: | |
| if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) || | |
| STI.isAtLeastSPIRVVer(VersionTuple(1, 6))) | |
| return selectDot4AddPacked<true>(ResVReg, ResType, I); | |
| return selectDot4AddPackedExpansion<true>(ResVReg, ResType, I); | |
| case Intrinsic::spv_dot4add_u8packed: | |
| if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) || | |
| STI.isAtLeastSPIRVVer(VersionTuple(1, 6))) | |
| return selectDot4AddPacked<false>(ResVReg, ResType, I); | |
| return selectDot4AddPackedExpansion<false>(ResVReg, ResType, I); | |
| case Intrinsic::spv_all: | |
| return selectAll(ResVReg, ResType, I); | |
| case Intrinsic::spv_any: | |
| return selectAny(ResVReg, ResType, I); | |
| case Intrinsic::spv_distance: | |
| return selectExtInst(ResVReg, ResType, I, CL::distance, GL::Distance); | |
| case Intrinsic::spv_lerp: | |
| return selectExtInst(ResVReg, ResType, I, CL::mix, GL::FMix); | |
| case Intrinsic::spv_length: | |
| return selectExtInst(ResVReg, ResType, I, CL::length, GL::Length); | |
| case Intrinsic::spv_degrees: | |
| return selectExtInst(ResVReg, ResType, I, CL::degrees, GL::Degrees); | |
| case Intrinsic::spv_faceforward: | |
| return selectExtInst(ResVReg, ResType, I, GL::FaceForward); | |
| case Intrinsic::spv_frac: | |
| return selectExtInst(ResVReg, ResType, I, CL::fract, GL::Fract); | |
| case Intrinsic::spv_isinf: | |
| return selectOpIsInf(ResVReg, ResType, I); | |
| case Intrinsic::spv_isnan: | |
| return selectOpIsNan(ResVReg, ResType, I); | |
| case Intrinsic::spv_isfinite: | |
| return selectOpIsFinite(ResVReg, ResType, I); | |
| case Intrinsic::spv_isnormal: | |
| return selectOpIsNormal(ResVReg, ResType, I); | |
| case Intrinsic::spv_normalize: | |
| return selectExtInst(ResVReg, ResType, I, CL::normalize, GL::Normalize); | |
| case Intrinsic::spv_refract: | |
| return selectExtInst(ResVReg, ResType, I, GL::Refract); | |
| case Intrinsic::spv_reflect: | |
| return selectExtInst(ResVReg, ResType, I, GL::Reflect); | |
| case Intrinsic::spv_rsqrt: | |
| return selectExtInst(ResVReg, ResType, I, CL::rsqrt, GL::InverseSqrt); | |
| case Intrinsic::spv_sign: | |
| return selectSign(ResVReg, ResType, I); | |
| case Intrinsic::spv_smoothstep: | |
| return selectExtInst(ResVReg, ResType, I, CL::smoothstep, GL::SmoothStep); | |
| case Intrinsic::spv_firstbituhigh: // There is no CL equivalent of FindUMsb | |
| return selectFirstBitHigh(ResVReg, ResType, I, /*IsSigned=*/false); | |
| case Intrinsic::spv_firstbitshigh: // There is no CL equivalent of FindSMsb | |
| return selectFirstBitHigh(ResVReg, ResType, I, /*IsSigned=*/true); | |
| case Intrinsic::spv_firstbitlow: // There is no CL equivlent of FindILsb | |
| return selectFirstBitLow(ResVReg, ResType, I); | |
| case Intrinsic::spv_all_memory_barrier: | |
| return selectBarrierInst(I, SPIRV::Scope::Device, | |
| SPIRV::MemorySemantics::UniformMemory | | |
| SPIRV::MemorySemantics::ImageMemory | | |
| SPIRV::MemorySemantics::WorkgroupMemory, | |
| /*WithGroupSync*/ false); | |
| case Intrinsic::spv_all_memory_barrier_with_group_sync: | |
| return selectBarrierInst(I, SPIRV::Scope::Device, | |
| SPIRV::MemorySemantics::UniformMemory | | |
| SPIRV::MemorySemantics::ImageMemory | | |
| SPIRV::MemorySemantics::WorkgroupMemory, | |
| /*WithGroupSync*/ true); | |
| case Intrinsic::spv_device_memory_barrier: | |
| return selectBarrierInst(I, SPIRV::Scope::Device, | |
| SPIRV::MemorySemantics::UniformMemory | | |
| SPIRV::MemorySemantics::ImageMemory, | |
| /*WithGroupSync*/ false); | |
| case Intrinsic::spv_device_memory_barrier_with_group_sync: | |
| return selectBarrierInst(I, SPIRV::Scope::Device, | |
| SPIRV::MemorySemantics::UniformMemory | | |
| SPIRV::MemorySemantics::ImageMemory, | |
| /*WithGroupSync*/ true); | |
| case Intrinsic::spv_group_memory_barrier: | |
| return selectBarrierInst(I, SPIRV::Scope::Workgroup, | |
| SPIRV::MemorySemantics::WorkgroupMemory, | |
| /*WithGroupSync*/ false); | |
| case Intrinsic::spv_group_memory_barrier_with_group_sync: | |
| return selectBarrierInst(I, SPIRV::Scope::Workgroup, | |
| SPIRV::MemorySemantics::WorkgroupMemory, | |
| /*WithGroupSync*/ true); | |
| case Intrinsic::spv_generic_cast_to_ptr_explicit: { | |
| Register PtrReg = I.getOperand(I.getNumExplicitDefs() + 1).getReg(); | |
| SPIRV::StorageClass::StorageClass ResSC = | |
| GR.getPointerStorageClass(ResType); | |
| if (!isGenericCastablePtr(ResSC)) | |
| return diagnoseUnsupported(I, "The target storage class is not castable " | |
| "from the Generic storage class"); | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpGenericCastToPtrExplicit)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(PtrReg) | |
| .addImm(ResSC) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| case Intrinsic::spv_lifetime_start: | |
| case Intrinsic::spv_lifetime_end: { | |
| unsigned Op = IID == Intrinsic::spv_lifetime_start ? SPIRV::OpLifetimeStart | |
| : SPIRV::OpLifetimeStop; | |
| int64_t Size = I.getOperand(I.getNumExplicitDefs() + 1).getImm(); | |
| Register PtrReg = I.getOperand(I.getNumExplicitDefs() + 2).getReg(); | |
| if (Size == -1) | |
| Size = 0; | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(Op)) | |
| .addUse(PtrReg) | |
| .addImm(Size) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| case Intrinsic::spv_saturate: | |
| return selectSaturate(ResVReg, ResType, I); | |
| case Intrinsic::spv_nclamp: | |
| return selectExtInst(ResVReg, ResType, I, CL::fclamp, GL::NClamp); | |
| case Intrinsic::spv_uclamp: | |
| return selectExtInst(ResVReg, ResType, I, CL::u_clamp, GL::UClamp); | |
| case Intrinsic::spv_sclamp: | |
| return selectExtInst(ResVReg, ResType, I, CL::s_clamp, GL::SClamp); | |
| case Intrinsic::spv_subgroup_prefix_bit_count: | |
| return selectWavePrefixBitCount(ResVReg, ResType, I); | |
| case Intrinsic::spv_wave_active_countbits: | |
| return selectWaveActiveCountBits(ResVReg, ResType, I); | |
| case Intrinsic::spv_wave_all_equal: | |
| return selectWaveActiveAllEqual(ResVReg, ResType, I); | |
| case Intrinsic::spv_wave_all: | |
| return selectWaveOpInst(ResVReg, ResType, I, SPIRV::OpGroupNonUniformAll); | |
| case Intrinsic::spv_wave_any: | |
| return selectWaveOpInst(ResVReg, ResType, I, SPIRV::OpGroupNonUniformAny); | |
| case Intrinsic::spv_subgroup_ballot: | |
| return selectWaveOpInst(ResVReg, ResType, I, | |
| SPIRV::OpGroupNonUniformBallot); | |
| case Intrinsic::spv_wave_is_first_lane: | |
| return selectWaveOpInst(ResVReg, ResType, I, SPIRV::OpGroupNonUniformElect); | |
| case Intrinsic::spv_wave_reduce_or: | |
| return selectWaveReduceOp(ResVReg, ResType, I, | |
| SPIRV::OpGroupNonUniformBitwiseOr); | |
| case Intrinsic::spv_wave_reduce_xor: | |
| return selectWaveReduceOp(ResVReg, ResType, I, | |
| SPIRV::OpGroupNonUniformBitwiseXor); | |
| case Intrinsic::spv_wave_reduce_and: | |
| return selectWaveReduceOp(ResVReg, ResType, I, | |
| SPIRV::OpGroupNonUniformBitwiseAnd); | |
| case Intrinsic::spv_wave_reduce_umax: | |
| return selectWaveReduceMax(ResVReg, ResType, I, /*IsUnsigned*/ true); | |
| case Intrinsic::spv_wave_reduce_max: | |
| return selectWaveReduceMax(ResVReg, ResType, I, /*IsUnsigned*/ false); | |
| case Intrinsic::spv_wave_reduce_umin: | |
| return selectWaveReduceMin(ResVReg, ResType, I, /*IsUnsigned*/ true); | |
| case Intrinsic::spv_wave_reduce_min: | |
| return selectWaveReduceMin(ResVReg, ResType, I, /*IsUnsigned*/ false); | |
| case Intrinsic::spv_wave_reduce_sum: | |
| return selectWaveReduceSum(ResVReg, ResType, I); | |
| case Intrinsic::spv_wave_product: | |
| return selectWaveReduceProduct(ResVReg, ResType, I); | |
| case Intrinsic::spv_wave_readlane: | |
| return selectWaveOpInst(ResVReg, ResType, I, | |
| SPIRV::OpGroupNonUniformShuffle); | |
| case Intrinsic::spv_wave_readlane_first: | |
| return selectWaveOpInst(ResVReg, ResType, I, | |
| SPIRV::OpGroupNonUniformBroadcastFirst); | |
| case Intrinsic::spv_wave_prefix_sum: | |
| return selectWaveExclusiveScanSum(ResVReg, ResType, I); | |
| case Intrinsic::spv_wave_prefix_product: | |
| return selectWaveExclusiveScanProduct(ResVReg, ResType, I); | |
| case Intrinsic::spv_quad_read_across_x: { | |
| return selectQuadSwap(ResVReg, ResType, I, /*Direction*/ 0); | |
| } | |
| case Intrinsic::spv_quad_read_across_y: { | |
| return selectQuadSwap(ResVReg, ResType, I, /*Direction*/ 1); | |
| } | |
| case Intrinsic::spv_quad_read_across_diagonal: { | |
| return selectQuadSwap(ResVReg, ResType, I, /*Direction*/ 2); | |
| } | |
| case Intrinsic::spv_radians: | |
| return selectExtInst(ResVReg, ResType, I, CL::radians, GL::Radians); | |
| // Discard intrinsics which we do not expect to actually represent code after | |
| // lowering or intrinsics which are not implemented but should not crash when | |
| // found in a customer's LLVM IR input. | |
| case Intrinsic::instrprof_increment: | |
| case Intrinsic::instrprof_increment_step: | |
| case Intrinsic::instrprof_value_profile: | |
| break; | |
| // Discard internal intrinsics. | |
| case Intrinsic::spv_value_md: | |
| break; | |
| case Intrinsic::spv_resource_handlefrombinding: { | |
| return selectHandleFromBinding(ResVReg, ResType, I); | |
| } | |
| case Intrinsic::spv_resource_counterhandlefrombinding: | |
| return selectCounterHandleFromBinding(ResVReg, ResType, I); | |
| case Intrinsic::spv_resource_updatecounter: | |
| return selectUpdateCounter(ResVReg, ResType, I); | |
| case Intrinsic::spv_resource_store_typedbuffer: { | |
| return selectImageWriteIntrinsic(I); | |
| } | |
| case Intrinsic::spv_resource_load_typedbuffer: { | |
| return selectReadImageIntrinsic(ResVReg, ResType, I); | |
| } | |
| case Intrinsic::spv_resource_load_level: { | |
| return selectLoadLevelIntrinsic(ResVReg, ResType, I); | |
| } | |
| case Intrinsic::spv_resource_getdimensions_x: | |
| case Intrinsic::spv_resource_getdimensions_xy: | |
| case Intrinsic::spv_resource_getdimensions_xyz: { | |
| return selectGetDimensionsIntrinsic(ResVReg, ResType, I); | |
| } | |
| case Intrinsic::spv_resource_getdimensions_levels_x: | |
| case Intrinsic::spv_resource_getdimensions_levels_xy: | |
| case Intrinsic::spv_resource_getdimensions_levels_xyz: { | |
| return selectGetDimensionsLevelsIntrinsic(ResVReg, ResType, I); | |
| } | |
| case Intrinsic::spv_resource_getdimensions_ms_xy: | |
| case Intrinsic::spv_resource_getdimensions_ms_xyz: { | |
| return selectGetDimensionsMSIntrinsic(ResVReg, ResType, I); | |
| } | |
| case Intrinsic::spv_resource_calculate_lod: | |
| case Intrinsic::spv_resource_calculate_lod_unclamped: | |
| return selectCalculateLodIntrinsic(ResVReg, ResType, I); | |
| case Intrinsic::spv_resource_sample: | |
| case Intrinsic::spv_resource_sample_clamp: | |
| return selectSampleBasicIntrinsic(ResVReg, ResType, I); | |
| case Intrinsic::spv_resource_samplebias: | |
| case Intrinsic::spv_resource_samplebias_clamp: | |
| return selectSampleBiasIntrinsic(ResVReg, ResType, I); | |
| case Intrinsic::spv_resource_samplegrad: | |
| case Intrinsic::spv_resource_samplegrad_clamp: | |
| return selectSampleGradIntrinsic(ResVReg, ResType, I); | |
| case Intrinsic::spv_resource_samplelevel: | |
| return selectSampleLevelIntrinsic(ResVReg, ResType, I); | |
| case Intrinsic::spv_resource_samplecmp: | |
| case Intrinsic::spv_resource_samplecmp_clamp: | |
| return selectSampleCmpIntrinsic(ResVReg, ResType, I); | |
| case Intrinsic::spv_resource_samplecmplevelzero: | |
| return selectSampleCmpLevelZeroIntrinsic(ResVReg, ResType, I); | |
| case Intrinsic::spv_resource_gather: | |
| case Intrinsic::spv_resource_gather_cmp: | |
| return selectGatherIntrinsic(ResVReg, ResType, I); | |
| case Intrinsic::spv_resource_getbasepointer: | |
| case Intrinsic::spv_resource_getpointer: { | |
| return selectResourceGetPointer(ResVReg, ResType, I); | |
| } | |
| case Intrinsic::spv_pushconstant_getpointer: { | |
| return selectPushConstantGetPointer(ResVReg, ResType, I); | |
| } | |
| case Intrinsic::spv_discard: { | |
| return selectDiscard(ResVReg, ResType, I); | |
| } | |
| case Intrinsic::spv_resource_nonuniformindex: { | |
| return selectResourceNonUniformIndex(ResVReg, ResType, I); | |
| } | |
| case Intrinsic::spv_unpackhalf2x16: { | |
| return selectExtInst(ResVReg, ResType, I, GL::UnpackHalf2x16); | |
| } | |
| case Intrinsic::spv_packhalf2x16: { | |
| return selectExtInst(ResVReg, ResType, I, GL::PackHalf2x16); | |
| } | |
| case Intrinsic::spv_ddx: | |
| return selectDerivativeInst(ResVReg, ResType, I, SPIRV::OpDPdx); | |
| case Intrinsic::spv_ddy: | |
| return selectDerivativeInst(ResVReg, ResType, I, SPIRV::OpDPdy); | |
| case Intrinsic::spv_ddx_coarse: | |
| return selectDerivativeInst(ResVReg, ResType, I, SPIRV::OpDPdxCoarse); | |
| case Intrinsic::spv_ddy_coarse: | |
| return selectDerivativeInst(ResVReg, ResType, I, SPIRV::OpDPdyCoarse); | |
| case Intrinsic::spv_ddx_fine: | |
| return selectDerivativeInst(ResVReg, ResType, I, SPIRV::OpDPdxFine); | |
| case Intrinsic::spv_ddy_fine: | |
| return selectDerivativeInst(ResVReg, ResType, I, SPIRV::OpDPdyFine); | |
| case Intrinsic::spv_fwidth: | |
| return selectDerivativeInst(ResVReg, ResType, I, SPIRV::OpFwidth); | |
| case Intrinsic::spv_masked_gather: | |
| if (STI.canUseExtension(SPIRV::Extension::SPV_INTEL_masked_gather_scatter)) | |
| return selectMaskedGather(ResVReg, ResType, I); | |
| return diagnoseUnsupported( | |
| I, "llvm.masked.gather requires SPV_INTEL_masked_gather_scatter"); | |
| case Intrinsic::spv_masked_scatter: | |
| if (STI.canUseExtension(SPIRV::Extension::SPV_INTEL_masked_gather_scatter)) | |
| return selectMaskedScatter(I); | |
| return diagnoseUnsupported( | |
| I, "llvm.masked.scatter requires SPV_INTEL_masked_gather_scatter"); | |
| case Intrinsic::returnaddress: | |
| case Intrinsic::frameaddress: { | |
| // SPIR-V does not have a stack or return address. Lower to null. | |
| auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)); | |
| MIB.constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| default: | |
| return diagnoseUnsupported(I, "intrinsic selection not implemented."); | |
| } | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectHandleFromBinding(Register &ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| // The images need to be loaded in the same basic block as their use. We defer | |
| // loading the image to the intrinsic that uses it. | |
| if (ResType->getOpcode() == SPIRV::OpTypeImage) | |
| return true; | |
| return loadHandleBeforePosition(ResVReg, GR.getSPIRVTypeForVReg(ResVReg), | |
| *cast<GIntrinsic>(&I), I); | |
| } | |
| bool SPIRVInstructionSelector::selectCounterHandleFromBinding( | |
| Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const { | |
| auto &Intr = cast<GIntrinsic>(I); | |
| assert(Intr.getIntrinsicID() == | |
| Intrinsic::spv_resource_counterhandlefrombinding); | |
| // Extract information from the intrinsic call. | |
| Register MainHandleReg = Intr.getOperand(2).getReg(); | |
| auto *MainHandleDef = cast<GIntrinsic>(getVRegDef(*MRI, MainHandleReg)); | |
| assert(MainHandleDef->getIntrinsicID() == | |
| Intrinsic::spv_resource_handlefrombinding); | |
| uint32_t Set = getIConstVal(Intr.getOperand(3).getReg(), MRI); | |
| uint32_t Binding = getIConstVal(Intr.getOperand(4).getReg(), MRI); | |
| uint32_t ArraySize = getIConstVal(MainHandleDef->getOperand(4).getReg(), MRI); | |
| Register IndexReg = MainHandleDef->getOperand(5).getReg(); | |
| std::string CounterName = | |
| getStringValueFromReg(MainHandleDef->getOperand(6).getReg(), *MRI) + | |
| ".counter"; | |
| // Create the counter variable. | |
| MachineIRBuilder MIRBuilder(I); | |
| Register CounterVarReg = | |
| buildPointerToResource(SPIRVTypeInst(GR.getPointeeType(ResType)), | |
| GR.getPointerStorageClass(ResType), Set, Binding, | |
| ArraySize, IndexReg, CounterName, MIRBuilder); | |
| return BuildCOPY(ResVReg, CounterVarReg, I); | |
| } | |
| bool SPIRVInstructionSelector::selectUpdateCounter(Register &ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| auto &Intr = cast<GIntrinsic>(I); | |
| assert(Intr.getIntrinsicID() == Intrinsic::spv_resource_updatecounter); | |
| Register CounterHandleReg = Intr.getOperand(2).getReg(); | |
| Register IncrReg = Intr.getOperand(3).getReg(); | |
| // The counter handle is a pointer to the counter variable (which is a struct | |
| // containing an i32). We need to get a pointer to that i32 member to do the | |
| // atomic operation. | |
| #ifndef NDEBUG | |
| SPIRVTypeInst CounterVarType = GR.getSPIRVTypeForVReg(CounterHandleReg); | |
| SPIRVTypeInst CounterVarPointeeType = GR.getPointeeType(CounterVarType); | |
| assert(CounterVarPointeeType && | |
| CounterVarPointeeType->getOpcode() == SPIRV::OpTypeStruct && | |
| "Counter variable must be a struct"); | |
| assert(GR.getPointerStorageClass(CounterVarType) == | |
| SPIRV::StorageClass::StorageBuffer && | |
| "Counter variable must be in the storage buffer storage class"); | |
| assert(CounterVarPointeeType->getNumOperands() == 2 && | |
| "Counter variable must have exactly 1 member in the struct"); | |
| const SPIRVTypeInst MemberType = | |
| GR.getSPIRVTypeForVReg(CounterVarPointeeType->getOperand(1).getReg()); | |
| assert(MemberType->getOpcode() == SPIRV::OpTypeInt && | |
| "Counter variable struct must have a single i32 member"); | |
| #endif | |
| // The struct has a single i32 member. | |
| MachineIRBuilder MIRBuilder(I); | |
| const Type *LLVMIntType = | |
| Type::getInt32Ty(I.getMF()->getFunction().getContext()); | |
| SPIRVTypeInst IntPtrType = GR.getOrCreateSPIRVPointerType( | |
| LLVMIntType, MIRBuilder, SPIRV::StorageClass::StorageBuffer); | |
| Register Zero = buildI32Constant(0, I); | |
| Register PtrToCounter = | |
| MRI->createVirtualRegister(GR.getRegClass(IntPtrType)); | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpAccessChain)) | |
| .addDef(PtrToCounter) | |
| .addUse(GR.getSPIRVTypeID(IntPtrType)) | |
| .addUse(CounterHandleReg) | |
| .addUse(Zero) | |
| .constrainAllUses(TII, TRI, RBI); | |
| // For UAV/SSBO counters, the scope is Device. The counter variable is not | |
| // used as a flag. So the memory semantics can be None. | |
| Register Scope = buildI32Constant(SPIRV::Scope::Device, I); | |
| Register Semantics = buildI32Constant(SPIRV::MemorySemantics::None, I); | |
| int64_t IncrVal = getIConstValSext(IncrReg, MRI); | |
| Register Incr = buildI32Constant(static_cast<uint32_t>(IncrVal), I); | |
| Register AtomicRes = MRI->createVirtualRegister(GR.getRegClass(ResType)); | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpAtomicIAdd)) | |
| .addDef(AtomicRes) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(PtrToCounter) | |
| .addUse(Scope) | |
| .addUse(Semantics) | |
| .addUse(Incr) | |
| .constrainAllUses(TII, TRI, RBI); | |
| if (IncrVal >= 0) { | |
| return BuildCOPY(ResVReg, AtomicRes, I); | |
| } | |
| // In HLSL, IncrementCounter returns the value *before* the increment, while | |
| // DecrementCounter returns the value *after* the decrement. Both are lowered | |
| // to the same atomic intrinsic which returns the value *before* the | |
| // operation. So for decrements (negative IncrVal), we must subtract the | |
| // increment value from the result to get the post-decrement value. | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpIAddS)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(AtomicRes) | |
| .addUse(Incr) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectReadImageIntrinsic(Register &ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| // If the load of the image is in a different basic block, then | |
| // this will generate invalid code. A proper solution is to move | |
| // the OpLoad from selectHandleFromBinding here. However, to do | |
| // that we will need to change the return type of the intrinsic. | |
| // We will do that when we can, but for now trying to move forward with other | |
| // issues. | |
| Register ImageReg = I.getOperand(2).getReg(); | |
| auto *ImageDef = cast<GIntrinsic>(getVRegDef(*MRI, ImageReg)); | |
| Register NewImageReg = MRI->createVirtualRegister(MRI->getRegClass(ImageReg)); | |
| if (!loadHandleBeforePosition(NewImageReg, GR.getSPIRVTypeForVReg(ImageReg), | |
| *ImageDef, I)) { | |
| return false; | |
| } | |
| Register IdxReg = I.getOperand(3).getReg(); | |
| DebugLoc Loc = I.getDebugLoc(); | |
| MachineInstr &Pos = I; | |
| return generateImageReadOrFetch(ResVReg, ResType, NewImageReg, IdxReg, Loc, | |
| Pos); | |
| } | |
| bool SPIRVInstructionSelector::generateSampleImage( | |
| Register ResVReg, SPIRVTypeInst ResType, Register ImageReg, | |
| Register SamplerReg, Register CoordinateReg, const ImageOperands &ImOps, | |
| DebugLoc Loc, MachineInstr &Pos) const { | |
| auto *ImageDef = cast<GIntrinsic>(getVRegDef(*MRI, ImageReg)); | |
| Register NewImageReg = MRI->createVirtualRegister(MRI->getRegClass(ImageReg)); | |
| if (!loadHandleBeforePosition(NewImageReg, GR.getSPIRVTypeForVReg(ImageReg), | |
| *ImageDef, Pos)) { | |
| return false; | |
| } | |
| auto *SamplerDef = cast<GIntrinsic>(getVRegDef(*MRI, SamplerReg)); | |
| Register NewSamplerReg = | |
| MRI->createVirtualRegister(MRI->getRegClass(SamplerReg)); | |
| if (!loadHandleBeforePosition(NewSamplerReg, | |
| GR.getSPIRVTypeForVReg(SamplerReg), *SamplerDef, | |
| Pos)) { | |
| return false; | |
| } | |
| MachineIRBuilder MIRBuilder(Pos); | |
| SPIRVTypeInst SampledImageType = GR.getOrCreateOpTypeSampledImage( | |
| GR.getSPIRVTypeForVReg(ImageReg), MIRBuilder); | |
| Register SampledImageReg = | |
| MRI->createVirtualRegister(GR.getRegClass(SampledImageType)); | |
| BuildMI(*Pos.getParent(), Pos, Loc, TII.get(SPIRV::OpSampledImage)) | |
| .addDef(SampledImageReg) | |
| .addUse(GR.getSPIRVTypeID(SampledImageType)) | |
| .addUse(NewImageReg) | |
| .addUse(NewSamplerReg) | |
| .constrainAllUses(TII, TRI, RBI); | |
| bool IsExplicitLod = ImOps.GradX.has_value() || ImOps.GradY.has_value() || | |
| ImOps.Lod.has_value(); | |
| unsigned Opcode = IsExplicitLod ? SPIRV::OpImageSampleExplicitLod | |
| : SPIRV::OpImageSampleImplicitLod; | |
| if (ImOps.Compare) | |
| Opcode = IsExplicitLod ? SPIRV::OpImageSampleDrefExplicitLod | |
| : SPIRV::OpImageSampleDrefImplicitLod; | |
| auto MIB = BuildMI(*Pos.getParent(), Pos, Loc, TII.get(Opcode)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(SampledImageReg) | |
| .addUse(CoordinateReg); | |
| if (ImOps.Compare) | |
| MIB.addUse(*ImOps.Compare); | |
| uint32_t ImageOperands = 0; | |
| if (ImOps.Bias) | |
| ImageOperands |= SPIRV::ImageOperand::Bias; | |
| if (ImOps.Lod) | |
| ImageOperands |= SPIRV::ImageOperand::Lod; | |
| if (ImOps.GradX && ImOps.GradY) | |
| ImageOperands |= SPIRV::ImageOperand::Grad; | |
| if (ImOps.Offset && !isScalarOrVectorIntConstantZero(*ImOps.Offset)) { | |
| if (isConstReg(MRI, *ImOps.Offset)) | |
| ImageOperands |= SPIRV::ImageOperand::ConstOffset; | |
| else { | |
| Pos.emitGenericError( | |
| "Non-constant offsets are not supported in sample instructions."); | |
| return false; | |
| } | |
| } | |
| if (ImOps.MinLod) | |
| ImageOperands |= SPIRV::ImageOperand::MinLod; | |
| if (ImageOperands != 0) { | |
| MIB.addImm(ImageOperands); | |
| if (ImageOperands & SPIRV::ImageOperand::Bias) | |
| MIB.addUse(*ImOps.Bias); | |
| if (ImageOperands & SPIRV::ImageOperand::Lod) | |
| MIB.addUse(*ImOps.Lod); | |
| if (ImageOperands & SPIRV::ImageOperand::Grad) { | |
| MIB.addUse(*ImOps.GradX); | |
| MIB.addUse(*ImOps.GradY); | |
| } | |
| if (ImageOperands & | |
| (SPIRV::ImageOperand::ConstOffset | SPIRV::ImageOperand::Offset)) | |
| MIB.addUse(*ImOps.Offset); | |
| if (ImageOperands & SPIRV::ImageOperand::MinLod) | |
| MIB.addUse(*ImOps.MinLod); | |
| } | |
| MIB.constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectImageQuerySize( | |
| Register ImageReg, Register &ResVReg, MachineInstr &I, | |
| std::optional<Register> LodReg) const { | |
| unsigned Opcode = | |
| LodReg ? SPIRV::OpImageQuerySizeLod : SPIRV::OpImageQuerySize; | |
| SPIRVTypeInst ImageType = GR.getSPIRVTypeForVReg(ImageReg); | |
| assert(ImageType && ImageType->getOpcode() == SPIRV::OpTypeImage && | |
| "ImageReg is not an image type."); | |
| auto Dim = static_cast<SPIRV::Dim::Dim>(ImageType->getOperand(2).getImm()); | |
| bool IsArray = ImageType->getOperand(4).getImm() != 0; | |
| unsigned NumComponents = 0; | |
| switch (Dim) { | |
| case SPIRV::Dim::DIM_1D: | |
| case SPIRV::Dim::DIM_Buffer: | |
| NumComponents = IsArray ? 2 : 1; | |
| break; | |
| case SPIRV::Dim::DIM_2D: | |
| case SPIRV::Dim::DIM_Cube: | |
| case SPIRV::Dim::DIM_Rect: | |
| NumComponents = IsArray ? 3 : 2; | |
| break; | |
| case SPIRV::Dim::DIM_3D: | |
| NumComponents = 3; | |
| break; | |
| default: | |
| I.emitGenericError("Unsupported image dimension for OpImageQuerySize."); | |
| return false; | |
| } | |
| SPIRVTypeInst I32Ty = GR.getOrCreateSPIRVIntegerType(32, I, TII); | |
| SPIRVTypeInst ResType = | |
| NumComponents == 1 | |
| ? I32Ty | |
| : GR.getOrCreateSPIRVVectorType(I32Ty, NumComponents, I, TII); | |
| auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(ImageReg); | |
| if (LodReg) | |
| MIB.addUse(*LodReg); | |
| MIB.constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectGetDimensionsIntrinsic( | |
| Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const { | |
| Register ImageReg = I.getOperand(2).getReg(); | |
| auto *ImageDef = cast<GIntrinsic>(getVRegDef(*MRI, ImageReg)); | |
| Register NewImageReg = MRI->createVirtualRegister(MRI->getRegClass(ImageReg)); | |
| if (!loadHandleBeforePosition(NewImageReg, GR.getSPIRVTypeForVReg(ImageReg), | |
| *ImageDef, I)) { | |
| return false; | |
| } | |
| return selectImageQuerySize(NewImageReg, ResVReg, I); | |
| } | |
| bool SPIRVInstructionSelector::selectGetDimensionsLevelsIntrinsic( | |
| Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const { | |
| Register ImageReg = I.getOperand(2).getReg(); | |
| auto *ImageDef = cast<GIntrinsic>(getVRegDef(*MRI, ImageReg)); | |
| Register NewImageReg = MRI->createVirtualRegister(MRI->getRegClass(ImageReg)); | |
| if (!loadHandleBeforePosition(NewImageReg, GR.getSPIRVTypeForVReg(ImageReg), | |
| *ImageDef, I)) { | |
| return false; | |
| } | |
| Register SizeReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); | |
| Register LodReg = I.getOperand(3).getReg(); | |
| assert(GR.getSPIRVTypeForVReg(NewImageReg)->getOperand(6).getImm() == 1 && | |
| "OpImageQuerySizeLod and OpImageQueryLevels require a sampled image"); | |
| if (!selectImageQuerySize(NewImageReg, SizeReg, I, LodReg)) { | |
| return false; | |
| } | |
| SPIRVTypeInst I32Ty = GR.getOrCreateSPIRVIntegerType(32, I, TII); | |
| Register LevelsReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), | |
| TII.get(SPIRV::OpImageQueryLevels)) | |
| .addDef(LevelsReg) | |
| .addUse(GR.getSPIRVTypeID(I32Ty)) | |
| .addUse(NewImageReg) | |
| .constrainAllUses(TII, TRI, RBI); | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), | |
| TII.get(SPIRV::OpCompositeConstruct)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(SizeReg) | |
| .addUse(LevelsReg) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectGetDimensionsMSIntrinsic( | |
| Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const { | |
| Register ImageReg = I.getOperand(2).getReg(); | |
| auto *ImageDef = cast<GIntrinsic>(getVRegDef(*MRI, ImageReg)); | |
| Register NewImageReg = MRI->createVirtualRegister(MRI->getRegClass(ImageReg)); | |
| if (!loadHandleBeforePosition(NewImageReg, GR.getSPIRVTypeForVReg(ImageReg), | |
| *ImageDef, I)) { | |
| return false; | |
| } | |
| Register SizeReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); | |
| assert(GR.getSPIRVTypeForVReg(NewImageReg)->getOperand(5).getImm() == 1 && | |
| "OpImageQuerySamples requires a multisampled image"); | |
| if (!selectImageQuerySize(NewImageReg, SizeReg, I)) { | |
| return false; | |
| } | |
| Register SamplesReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass); | |
| SPIRVTypeInst I32Ty = GR.getOrCreateSPIRVIntegerType(32, I, TII); | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), | |
| TII.get(SPIRV::OpImageQuerySamples)) | |
| .addDef(SamplesReg) | |
| .addUse(GR.getSPIRVTypeID(I32Ty)) | |
| .addUse(NewImageReg) | |
| .constrainAllUses(TII, TRI, RBI); | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), | |
| TII.get(SPIRV::OpCompositeConstruct)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(SizeReg) | |
| .addUse(SamplesReg) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectCalculateLodIntrinsic( | |
| Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const { | |
| Register ImageReg = I.getOperand(2).getReg(); | |
| Register SamplerReg = I.getOperand(3).getReg(); | |
| Register CoordinateReg = I.getOperand(4).getReg(); | |
| auto *ImageDef = dyn_cast<GIntrinsic>(getVRegDef(*MRI, ImageReg)); | |
| if (!ImageDef) | |
| return false; | |
| Register NewImageReg = MRI->createVirtualRegister(MRI->getRegClass(ImageReg)); | |
| if (!loadHandleBeforePosition(NewImageReg, GR.getSPIRVTypeForVReg(ImageReg), | |
| *ImageDef, I)) { | |
| return false; | |
| } | |
| auto *SamplerDef = dyn_cast<GIntrinsic>(getVRegDef(*MRI, SamplerReg)); | |
| if (!SamplerDef) | |
| return false; | |
| Register NewSamplerReg = | |
| MRI->createVirtualRegister(MRI->getRegClass(SamplerReg)); | |
| if (!loadHandleBeforePosition( | |
| NewSamplerReg, GR.getSPIRVTypeForVReg(SamplerReg), *SamplerDef, I)) { | |
| return false; | |
| } | |
| MachineIRBuilder MIRBuilder(I); | |
| SPIRVTypeInst SampledImageType = GR.getOrCreateOpTypeSampledImage( | |
| GR.getSPIRVTypeForVReg(ImageReg), MIRBuilder); | |
| Register SampledImageReg = | |
| MRI->createVirtualRegister(GR.getRegClass(SampledImageType)); | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpSampledImage)) | |
| .addDef(SampledImageReg) | |
| .addUse(GR.getSPIRVTypeID(SampledImageType)) | |
| .addUse(NewImageReg) | |
| .addUse(NewSamplerReg) | |
| .constrainAllUses(TII, TRI, RBI); | |
| SPIRVTypeInst Vec2Ty = GR.getOrCreateSPIRVVectorType(ResType, 2, I, TII); | |
| Register QueryResultReg = MRI->createVirtualRegister(GR.getRegClass(Vec2Ty)); | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpImageQueryLod)) | |
| .addDef(QueryResultReg) | |
| .addUse(GR.getSPIRVTypeID(Vec2Ty)) | |
| .addUse(SampledImageReg) | |
| .addUse(CoordinateReg) | |
| .constrainAllUses(TII, TRI, RBI); | |
| unsigned ExtractedIndex = | |
| cast<GIntrinsic>(I).getIntrinsicID() == | |
| Intrinsic::spv_resource_calculate_lod_unclamped | |
| ? 1 | |
| : 0; | |
| MachineInstrBuilder MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), | |
| TII.get(SPIRV::OpCompositeExtract)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(QueryResultReg) | |
| .addImm(ExtractedIndex); | |
| MIB.constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectSampleBasicIntrinsic( | |
| Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const { | |
| Register ImageReg = I.getOperand(2).getReg(); | |
| Register SamplerReg = I.getOperand(3).getReg(); | |
| Register CoordinateReg = I.getOperand(4).getReg(); | |
| ImageOperands ImOps; | |
| if (I.getNumOperands() > 5) | |
| ImOps.Offset = I.getOperand(5).getReg(); | |
| if (I.getNumOperands() > 6) | |
| ImOps.MinLod = I.getOperand(6).getReg(); | |
| return generateSampleImage(ResVReg, ResType, ImageReg, SamplerReg, | |
| CoordinateReg, ImOps, I.getDebugLoc(), I); | |
| } | |
| bool SPIRVInstructionSelector::selectSampleBiasIntrinsic( | |
| Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const { | |
| Register ImageReg = I.getOperand(2).getReg(); | |
| Register SamplerReg = I.getOperand(3).getReg(); | |
| Register CoordinateReg = I.getOperand(4).getReg(); | |
| ImageOperands ImOps; | |
| ImOps.Bias = I.getOperand(5).getReg(); | |
| if (I.getNumOperands() > 6) | |
| ImOps.Offset = I.getOperand(6).getReg(); | |
| if (I.getNumOperands() > 7) | |
| ImOps.MinLod = I.getOperand(7).getReg(); | |
| return generateSampleImage(ResVReg, ResType, ImageReg, SamplerReg, | |
| CoordinateReg, ImOps, I.getDebugLoc(), I); | |
| } | |
| bool SPIRVInstructionSelector::selectSampleGradIntrinsic( | |
| Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const { | |
| Register ImageReg = I.getOperand(2).getReg(); | |
| Register SamplerReg = I.getOperand(3).getReg(); | |
| Register CoordinateReg = I.getOperand(4).getReg(); | |
| ImageOperands ImOps; | |
| ImOps.GradX = I.getOperand(5).getReg(); | |
| ImOps.GradY = I.getOperand(6).getReg(); | |
| if (I.getNumOperands() > 7) | |
| ImOps.Offset = I.getOperand(7).getReg(); | |
| if (I.getNumOperands() > 8) | |
| ImOps.MinLod = I.getOperand(8).getReg(); | |
| return generateSampleImage(ResVReg, ResType, ImageReg, SamplerReg, | |
| CoordinateReg, ImOps, I.getDebugLoc(), I); | |
| } | |
| bool SPIRVInstructionSelector::selectSampleLevelIntrinsic( | |
| Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const { | |
| Register ImageReg = I.getOperand(2).getReg(); | |
| Register SamplerReg = I.getOperand(3).getReg(); | |
| Register CoordinateReg = I.getOperand(4).getReg(); | |
| ImageOperands ImOps; | |
| ImOps.Lod = I.getOperand(5).getReg(); | |
| if (I.getNumOperands() > 6) | |
| ImOps.Offset = I.getOperand(6).getReg(); | |
| return generateSampleImage(ResVReg, ResType, ImageReg, SamplerReg, | |
| CoordinateReg, ImOps, I.getDebugLoc(), I); | |
| } | |
| bool SPIRVInstructionSelector::selectSampleCmpIntrinsic(Register &ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| Register ImageReg = I.getOperand(2).getReg(); | |
| Register SamplerReg = I.getOperand(3).getReg(); | |
| Register CoordinateReg = I.getOperand(4).getReg(); | |
| ImageOperands ImOps; | |
| ImOps.Compare = I.getOperand(5).getReg(); | |
| if (I.getNumOperands() > 6) | |
| ImOps.Offset = I.getOperand(6).getReg(); | |
| if (I.getNumOperands() > 7) | |
| ImOps.MinLod = I.getOperand(7).getReg(); | |
| return generateSampleImage(ResVReg, ResType, ImageReg, SamplerReg, | |
| CoordinateReg, ImOps, I.getDebugLoc(), I); | |
| } | |
| bool SPIRVInstructionSelector::selectLoadLevelIntrinsic(Register &ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| Register ImageReg = I.getOperand(2).getReg(); | |
| Register CoordinateReg = I.getOperand(3).getReg(); | |
| Register LodReg = I.getOperand(4).getReg(); | |
| ImageOperands ImOps; | |
| ImOps.Lod = LodReg; | |
| if (I.getNumOperands() > 5) | |
| ImOps.Offset = I.getOperand(5).getReg(); | |
| auto *ImageDef = dyn_cast<GIntrinsic>(getVRegDef(*MRI, ImageReg)); | |
| if (!ImageDef) | |
| return false; | |
| Register NewImageReg = MRI->createVirtualRegister(MRI->getRegClass(ImageReg)); | |
| if (!loadHandleBeforePosition(NewImageReg, GR.getSPIRVTypeForVReg(ImageReg), | |
| *ImageDef, I)) { | |
| return false; | |
| } | |
| return generateImageReadOrFetch(ResVReg, ResType, NewImageReg, CoordinateReg, | |
| I.getDebugLoc(), I, &ImOps); | |
| } | |
| bool SPIRVInstructionSelector::selectSampleCmpLevelZeroIntrinsic( | |
| Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const { | |
| Register ImageReg = I.getOperand(2).getReg(); | |
| Register SamplerReg = I.getOperand(3).getReg(); | |
| Register CoordinateReg = I.getOperand(4).getReg(); | |
| ImageOperands ImOps; | |
| ImOps.Compare = I.getOperand(5).getReg(); | |
| if (I.getNumOperands() > 6) | |
| ImOps.Offset = I.getOperand(6).getReg(); | |
| SPIRVTypeInst FloatTy = GR.getOrCreateSPIRVFloatType(32, I, TII); | |
| ImOps.Lod = GR.getOrCreateConstFP(APFloat(0.0f), I, FloatTy, TII); | |
| return generateSampleImage(ResVReg, ResType, ImageReg, SamplerReg, | |
| CoordinateReg, ImOps, I.getDebugLoc(), I); | |
| } | |
| bool SPIRVInstructionSelector::selectGatherIntrinsic(Register &ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| Register ImageReg = I.getOperand(2).getReg(); | |
| Register SamplerReg = I.getOperand(3).getReg(); | |
| Register CoordinateReg = I.getOperand(4).getReg(); | |
| SPIRVTypeInst ImageType = GR.getSPIRVTypeForVReg(ImageReg); | |
| assert(ImageType && ImageType->getOpcode() == SPIRV::OpTypeImage && | |
| "ImageReg is not an image type."); | |
| Register ComponentOrCompareReg; | |
| Register OffsetReg; | |
| ComponentOrCompareReg = I.getOperand(5).getReg(); | |
| OffsetReg = I.getOperand(6).getReg(); | |
| auto *ImageDef = cast<GIntrinsic>(getVRegDef(*MRI, ImageReg)); | |
| Register NewImageReg = MRI->createVirtualRegister(MRI->getRegClass(ImageReg)); | |
| if (!loadHandleBeforePosition(NewImageReg, ImageType, *ImageDef, I)) { | |
| return false; | |
| } | |
| auto Dim = static_cast<SPIRV::Dim::Dim>(ImageType->getOperand(2).getImm()); | |
| if (Dim != SPIRV::Dim::DIM_2D && Dim != SPIRV::Dim::DIM_Cube && | |
| Dim != SPIRV::Dim::DIM_Rect) { | |
| I.emitGenericError( | |
| "Gather operations are only supported for 2D, Cube, and Rect images."); | |
| return false; | |
| } | |
| auto *SamplerDef = cast<GIntrinsic>(getVRegDef(*MRI, SamplerReg)); | |
| Register NewSamplerReg = | |
| MRI->createVirtualRegister(MRI->getRegClass(SamplerReg)); | |
| if (!loadHandleBeforePosition( | |
| NewSamplerReg, GR.getSPIRVTypeForVReg(SamplerReg), *SamplerDef, I)) { | |
| return false; | |
| } | |
| MachineIRBuilder MIRBuilder(I); | |
| SPIRVTypeInst SampledImageType = | |
| GR.getOrCreateOpTypeSampledImage(ImageType, MIRBuilder); | |
| Register SampledImageReg = | |
| MRI->createVirtualRegister(GR.getRegClass(SampledImageType)); | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpSampledImage)) | |
| .addDef(SampledImageReg) | |
| .addUse(GR.getSPIRVTypeID(SampledImageType)) | |
| .addUse(NewImageReg) | |
| .addUse(NewSamplerReg) | |
| .constrainAllUses(TII, TRI, RBI); | |
| auto IntrId = cast<GIntrinsic>(I).getIntrinsicID(); | |
| bool IsGatherCmp = IntrId == Intrinsic::spv_resource_gather_cmp; | |
| unsigned Opcode = | |
| IsGatherCmp ? SPIRV::OpImageDrefGather : SPIRV::OpImageGather; | |
| auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(SampledImageReg) | |
| .addUse(CoordinateReg) | |
| .addUse(ComponentOrCompareReg); | |
| uint32_t ImageOperands = 0; | |
| if (OffsetReg && !isScalarOrVectorIntConstantZero(OffsetReg)) { | |
| if (Dim == SPIRV::Dim::DIM_Cube) { | |
| I.emitGenericError( | |
| "Gather operations with offset are not supported for Cube images."); | |
| return false; | |
| } | |
| if (isConstReg(MRI, OffsetReg)) | |
| ImageOperands |= SPIRV::ImageOperand::ConstOffset; | |
| else { | |
| ImageOperands |= SPIRV::ImageOperand::Offset; | |
| } | |
| } | |
| if (ImageOperands != 0) { | |
| MIB.addImm(ImageOperands); | |
| if (ImageOperands & | |
| (SPIRV::ImageOperand::ConstOffset | SPIRV::ImageOperand::Offset)) | |
| MIB.addUse(OffsetReg); | |
| } | |
| MIB.constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::generateImageReadOrFetch( | |
| Register &ResVReg, SPIRVTypeInst ResType, Register ImageReg, | |
| Register IdxReg, DebugLoc Loc, MachineInstr &Pos, | |
| const ImageOperands *ImOps) const { | |
| SPIRVTypeInst ImageType = GR.getSPIRVTypeForVReg(ImageReg); | |
| assert(ImageType && ImageType->getOpcode() == SPIRV::OpTypeImage && | |
| "ImageReg is not an image type."); | |
| bool IsSignedInteger = | |
| sampledTypeIsSignedInteger(GR.getTypeForSPIRVType(ImageType)); | |
| // Check if the "sampled" operand of the image type is 1. | |
| // https://registry.khronos.org/SPIR-V/specs/unified1/SPIRV.html#OpImageFetch | |
| auto SampledOp = ImageType->getOperand(6); | |
| bool IsFetch = (SampledOp.getImm() == 1); | |
| auto AddOperands = [&](MachineInstrBuilder &MIB) { | |
| uint32_t ImageOperandsMask = 0; | |
| if (IsSignedInteger) | |
| ImageOperandsMask |= 0x1000; // SignExtend | |
| if (IsFetch && ImOps) { | |
| if (ImOps->Lod) | |
| ImageOperandsMask |= SPIRV::ImageOperand::Lod; | |
| if (ImOps->Offset && !isScalarOrVectorIntConstantZero(*ImOps->Offset)) { | |
| if (isConstReg(MRI, *ImOps->Offset)) | |
| ImageOperandsMask |= SPIRV::ImageOperand::ConstOffset; | |
| else | |
| ImageOperandsMask |= SPIRV::ImageOperand::Offset; | |
| } | |
| } | |
| if (ImageOperandsMask != 0) { | |
| MIB.addImm(ImageOperandsMask); | |
| if (IsFetch && ImOps) { | |
| if (ImOps->Lod) | |
| MIB.addUse(*ImOps->Lod); | |
| if (ImOps->Offset && | |
| (ImageOperandsMask & | |
| (SPIRV::ImageOperand::Offset | SPIRV::ImageOperand::ConstOffset))) | |
| MIB.addUse(*ImOps->Offset); | |
| } | |
| } | |
| }; | |
| uint64_t ResultSize = GR.getScalarOrVectorComponentCount(ResType); | |
| // A wide element (e.g. int64_t2) is emulated with a narrower packed image, so | |
| // its sampled type is different from the result. | |
| SPIRVTypeInst SampledType = | |
| GR.getSPIRVTypeForVReg(ImageType->getOperand(1).getReg()); | |
| bool IsPacked = SampledType != GR.getScalarOrVectorComponentType(ResType); | |
| SPIRVTypeInst ReadType = | |
| widenTypeToVec4(IsPacked ? SampledType : ResType, Pos); | |
| bool ReadTypeMatchesResult = ReadType == ResType; | |
| // Read directly into the result, or into a temporary to bitcast/extract. | |
| Register ReadReg = ReadTypeMatchesResult | |
| ? ResVReg | |
| : MRI->createVirtualRegister(GR.getRegClass(ReadType)); | |
| auto BMI = | |
| BuildMI(*Pos.getParent(), Pos, Loc, | |
| TII.get(IsFetch ? SPIRV::OpImageFetch : SPIRV::OpImageRead)) | |
| .addDef(ReadReg) | |
| .addUse(GR.getSPIRVTypeID(ReadType)) | |
| .addUse(ImageReg) | |
| .addUse(IdxReg); | |
| AddOperands(BMI); | |
| BMI.constrainAllUses(TII, TRI, RBI); | |
| if (ReadTypeMatchesResult) | |
| return true; | |
| if (IsPacked) { | |
| // Reinterpret the packed vector as the wide result type. | |
| BuildMI(*Pos.getParent(), Pos, Loc, TII.get(SPIRV::OpBitcast)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(ReadReg) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| if (ResultSize == 1) { | |
| BuildMI(*Pos.getParent(), Pos, Loc, TII.get(SPIRV::OpCompositeExtract)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(ReadReg) | |
| .addImm(0) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| return extractSubvector(ResVReg, ResType, ReadReg, Pos); | |
| } | |
| bool SPIRVInstructionSelector::selectResourceGetPointer(Register &ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| Register ResourcePtr = I.getOperand(2).getReg(); | |
| SPIRVTypeInst RegType = GR.getSPIRVTypeForVReg(ResourcePtr, I.getMF()); | |
| if (RegType->getOpcode() == SPIRV::OpTypeImage) { | |
| // For texel buffers, the index into the image is part of the OpImageRead or | |
| // OpImageWrite instructions. So we will do nothing in this case. This | |
| // intrinsic will be combined with the load or store when selecting the load | |
| // or store. | |
| return true; | |
| } | |
| assert(ResType->getOpcode() == SPIRV::OpTypePointer); | |
| MachineIRBuilder MIRBuilder(I); | |
| Register ZeroReg = | |
| buildZerosVal(GR.getOrCreateSPIRVIntegerType(32, I, TII), I); | |
| auto MIB = | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpAccessChain)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(ResourcePtr) | |
| .addUse(ZeroReg); | |
| if (I.getNumExplicitOperands() > 3) { | |
| Register IndexReg = I.getOperand(3).getReg(); | |
| MIB.addUse(IndexReg); | |
| } | |
| MIB.constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectPushConstantGetPointer( | |
| Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const { | |
| MRI->replaceRegWith(ResVReg, I.getOperand(2).getReg()); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectResourceNonUniformIndex( | |
| Register &ResVReg, SPIRVTypeInst ResType, MachineInstr &I) const { | |
| Register ObjReg = I.getOperand(2).getReg(); | |
| if (!BuildCOPY(ResVReg, ObjReg, I)) | |
| return false; | |
| buildOpDecorate(ResVReg, I, TII, SPIRV::Decoration::NonUniformEXT, {}); | |
| // Check for the registers that use the index marked as non-uniform | |
| // and recursively mark them as non-uniform. | |
| // Per the spec, it's necessary that the final argument used for | |
| // load/store/sample/atomic must be decorated, so we need to propagate the | |
| // decoration through access chains and copies. | |
| // https://docs.vulkan.org/samples/latest/samples/extensions/descriptor_indexing/README.html#_when_to_use_non_uniform_indexing_qualifier | |
| decorateUsesAsNonUniform(ResVReg); | |
| return true; | |
| } | |
| void SPIRVInstructionSelector::decorateUsesAsNonUniform( | |
| Register &NonUniformReg) const { | |
| llvm::SmallVector<std::pair<Register, MachineInstr *>> WorkList = { | |
| {NonUniformReg, nullptr}}; | |
| llvm::SmallSet<Register, 8> Visited; | |
| while (WorkList.size() > 0) { | |
| auto [CurrentReg, DefMI] = WorkList.pop_back_val(); | |
| if (!Visited.insert(CurrentReg).second) | |
| continue; | |
| bool IsDecorated = false; | |
| for (MachineInstr &Use : MRI->use_instructions(CurrentReg)) { | |
| if (Use.getOpcode() == SPIRV::OpDecorate && | |
| Use.getOperand(1).getImm() == SPIRV::Decoration::NonUniformEXT) { | |
| IsDecorated = true; | |
| continue; | |
| } | |
| // Check if the instruction has the result register and add it to the | |
| // worklist. | |
| if (Use.getOperand(0).isReg() && Use.getOperand(0).isDef()) { | |
| Register ResultReg = Use.getOperand(0).getReg(); | |
| if (ResultReg == CurrentReg) | |
| continue; | |
| WorkList.push_back({ResultReg, &Use}); | |
| } | |
| } | |
| if (!IsDecorated) { | |
| MachineBasicBlock &MBB = *DefMI->getParent(); | |
| MachineInstr &InsertPt = | |
| DefMI->isPHI() ? *MBB.getFirstNonPHI() : *DefMI->getNextNode(); | |
| buildOpDecorate(CurrentReg, InsertPt, TII, | |
| SPIRV::Decoration::NonUniformEXT, {}); | |
| } | |
| } | |
| } | |
| bool SPIRVInstructionSelector::extractSubvector( | |
| Register &ResVReg, SPIRVTypeInst ResType, Register &ReadReg, | |
| MachineInstr &InsertionPoint) const { | |
| SPIRVTypeInst InputType = GR.getResultType(ReadReg); | |
| [[maybe_unused]] uint64_t InputSize = | |
| GR.getScalarOrVectorComponentCount(InputType); | |
| uint64_t ResultSize = GR.getScalarOrVectorComponentCount(ResType); | |
| [[maybe_unused]] bool IsLongVectorEXT = | |
| STI.canUseExtension(SPIRV::Extension::SPV_EXT_long_vector); | |
| assert((InputSize > 1 || IsLongVectorEXT) && "The input must be a vector."); | |
| assert((ResultSize > 1 || IsLongVectorEXT) && "The result must be a vector."); | |
| assert(ResultSize < InputSize && | |
| "Cannot extract more element than there are in the input."); | |
| SmallVector<Register> ComponentRegisters; | |
| SPIRVTypeInst ScalarType = GR.getScalarOrVectorComponentType(ResType); | |
| const TargetRegisterClass *ScalarRegClass = GR.getRegClass(ScalarType); | |
| for (uint64_t I = 0; I < ResultSize; I++) { | |
| Register ComponentReg = MRI->createVirtualRegister(ScalarRegClass); | |
| BuildMI(*InsertionPoint.getParent(), InsertionPoint, | |
| InsertionPoint.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) | |
| .addDef(ComponentReg) | |
| .addUse(ScalarType->getOperand(0).getReg()) | |
| .addUse(ReadReg) | |
| .addImm(I) | |
| .constrainAllUses(TII, TRI, RBI); | |
| ComponentRegisters.emplace_back(ComponentReg); | |
| } | |
| MachineInstrBuilder MIB = BuildMI(*InsertionPoint.getParent(), InsertionPoint, | |
| InsertionPoint.getDebugLoc(), | |
| TII.get(SPIRV::OpCompositeConstruct)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)); | |
| for (Register ComponentReg : ComponentRegisters) | |
| MIB.addUse(ComponentReg); | |
| MIB.constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectImageWriteIntrinsic( | |
| MachineInstr &I) const { | |
| // If the load of the image is in a different basic block, then | |
| // this will generate invalid code. A proper solution is to move | |
| // the OpLoad from selectHandleFromBinding here. However, to do | |
| // that we will need to change the return type of the intrinsic. | |
| // We will do that when we can, but for now trying to move forward with other | |
| // issues. | |
| Register ImageReg = I.getOperand(1).getReg(); | |
| auto *ImageDef = cast<GIntrinsic>(getVRegDef(*MRI, ImageReg)); | |
| Register NewImageReg = MRI->createVirtualRegister(MRI->getRegClass(ImageReg)); | |
| if (!loadHandleBeforePosition(NewImageReg, GR.getSPIRVTypeForVReg(ImageReg), | |
| *ImageDef, I)) { | |
| return false; | |
| } | |
| Register CoordinateReg = I.getOperand(2).getReg(); | |
| Register DataReg = I.getOperand(3).getReg(); | |
| assert(isVectorType(GR.getResultType(DataReg))); | |
| assert(GR.getScalarOrVectorComponentCount(GR.getResultType(DataReg)) == 4); | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpImageWrite)) | |
| .addUse(NewImageReg) | |
| .addUse(CoordinateReg) | |
| .addUse(DataReg) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| Register SPIRVInstructionSelector::buildPointerToResource( | |
| SPIRVTypeInst SpirvResType, SPIRV::StorageClass::StorageClass SC, | |
| uint32_t Set, uint32_t Binding, uint32_t ArraySize, Register IndexReg, | |
| StringRef Name, MachineIRBuilder MIRBuilder) const { | |
| const Type *ResType = GR.getTypeForSPIRVType(SpirvResType); | |
| if (ArraySize == 1) { | |
| SPIRVTypeInst PtrType = | |
| GR.getOrCreateSPIRVPointerType(ResType, MIRBuilder, SC); | |
| assert(GR.getPointeeType(PtrType) == SpirvResType && | |
| "SpirvResType did not have an explicit layout."); | |
| return GR.getOrCreateGlobalVariableWithBinding(PtrType, Set, Binding, Name, | |
| MIRBuilder); | |
| } | |
| const Type *VarType = ArrayType::get(const_cast<Type *>(ResType), ArraySize); | |
| SPIRVTypeInst VarPointerType = | |
| GR.getOrCreateSPIRVPointerType(VarType, MIRBuilder, SC); | |
| Register VarReg = GR.getOrCreateGlobalVariableWithBinding( | |
| VarPointerType, Set, Binding, Name, MIRBuilder); | |
| SPIRVTypeInst ResPointerType = | |
| GR.getOrCreateSPIRVPointerType(ResType, MIRBuilder, SC); | |
| Register AcReg = MRI->createVirtualRegister(GR.getRegClass(ResPointerType)); | |
| MIRBuilder.buildInstr(SPIRV::OpAccessChain) | |
| .addDef(AcReg) | |
| .addUse(GR.getSPIRVTypeID(ResPointerType)) | |
| .addUse(VarReg) | |
| .addUse(IndexReg); | |
| return AcReg; | |
| } | |
| bool SPIRVInstructionSelector::selectFirstBitSet16( | |
| Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I, | |
| unsigned ExtendOpcode, unsigned BitSetOpcode) const { | |
| Register ExtReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); | |
| if (!selectOpWithSrcs(ExtReg, ResType, I, {I.getOperand(2).getReg()}, | |
| ExtendOpcode)) | |
| return false; | |
| return selectFirstBitSet32(ResVReg, ResType, I, ExtReg, BitSetOpcode); | |
| } | |
| bool SPIRVInstructionSelector::selectFirstBitSet32( | |
| Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I, Register SrcReg, | |
| unsigned BitSetOpcode) const { | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) | |
| .addImm(BitSetOpcode) | |
| .addUse(SrcReg) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectFirstBitSet64( | |
| Register ResVReg, SPIRVTypeInst ResType, MachineInstr &I, Register SrcReg, | |
| unsigned BitSetOpcode, bool SwapPrimarySide) const { | |
| unsigned ComponentCount = GR.getScalarOrVectorComponentCount(ResType); | |
| SPIRVTypeInst BaseType = GR.retrieveScalarOrVectorIntType(ResType); | |
| bool ZeroAsNull = !STI.isShader(); | |
| Register ConstIntZero = | |
| GR.getOrCreateConstInt(0, I, BaseType, TII, ZeroAsNull); | |
| Register ConstIntOne = | |
| GR.getOrCreateConstInt(1, I, BaseType, TII, ZeroAsNull); | |
| // SPIRV doesn't support vectors with more than 4 components. Since the | |
| // algoritm below converts i64 -> i32x2 and i64x4 -> i32x8 it can only | |
| // operate on vectors with 2 or less components. When largers vectors are | |
| // seen. Split them, recurse, then recombine them. | |
| // TODO: handle the case where SPV_EXT_long_vector is enabled. | |
| if (ComponentCount > 2) { | |
| auto Func = [this, SwapPrimarySide](Register ResVReg, SPIRVTypeInst ResType, | |
| MachineInstr &I, Register SrcReg, | |
| unsigned Opcode) -> bool { | |
| return this->selectFirstBitSet64(ResVReg, ResType, I, SrcReg, Opcode, | |
| SwapPrimarySide); | |
| }; | |
| return handle64BitOverflow(ResVReg, ResType, I, SrcReg, BitSetOpcode, Func); | |
| } | |
| // 1. Split int64 into 2 pieces using a bitcast | |
| MachineIRBuilder MIRBuilder(I); | |
| SPIRVTypeInst PostCastType = GR.getOrCreateSPIRVVectorType( | |
| BaseType, 2 * ComponentCount, MIRBuilder, false); | |
| Register BitcastReg = | |
| MRI->createVirtualRegister(GR.getRegClass(PostCastType)); | |
| if (!selectOpWithSrcs(BitcastReg, PostCastType, I, {SrcReg}, | |
| SPIRV::OpBitcast)) | |
| return false; | |
| // 2. Find the first set bit from the primary side for all the pieces in #1 | |
| Register FBSReg = MRI->createVirtualRegister(GR.getRegClass(PostCastType)); | |
| if (!selectFirstBitSet32(FBSReg, PostCastType, I, BitcastReg, BitSetOpcode)) | |
| return false; | |
| // 3. Split result vector into high bits and low bits | |
| Register HighReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); | |
| Register LowReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); | |
| bool IsScalarRes = !isVectorType(ResType); | |
| if (IsScalarRes) { | |
| // if scalar do a vector extract | |
| if (!selectOpWithSrcs(HighReg, ResType, I, {FBSReg, ConstIntOne}, | |
| SPIRV::OpVectorExtractDynamic)) | |
| return false; | |
| if (!selectOpWithSrcs(LowReg, ResType, I, {FBSReg, ConstIntZero}, | |
| SPIRV::OpVectorExtractDynamic)) | |
| return false; | |
| } else { | |
| // if vector do a shufflevector | |
| auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), | |
| TII.get(SPIRV::OpVectorShuffle)) | |
| .addDef(HighReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(FBSReg) | |
| // Per the spec, repeat the vector if only one vec is needed | |
| .addUse(FBSReg); | |
| // high bits are stored in even natural indexes. Extract them from FBSReg | |
| for (unsigned J = 1; J < ComponentCount * 2; J += 2) { | |
| MIB.addImm(J); | |
| } | |
| MIB.constrainAllUses(TII, TRI, RBI); | |
| MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), | |
| TII.get(SPIRV::OpVectorShuffle)) | |
| .addDef(LowReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(FBSReg) | |
| // Per the spec, repeat the vector if only one vec is needed | |
| .addUse(FBSReg); | |
| // low bits are stored in odd natural indices. Extract them from FBSReg | |
| for (unsigned J = 0; J < ComponentCount * 2; J += 2) { | |
| MIB.addImm(J); | |
| } | |
| MIB.constrainAllUses(TII, TRI, RBI); | |
| } | |
| // 4. Check the result. When primary bits == -1 use secondary, otherwise use | |
| // primary | |
| SPIRVTypeInst BoolType = GR.getOrCreateSPIRVBoolType(I, TII); | |
| Register NegOneReg; | |
| Register Reg0; | |
| Register Reg32; | |
| unsigned SelectOp; | |
| unsigned AddOp; | |
| if (IsScalarRes) { | |
| NegOneReg = | |
| GR.getOrCreateConstInt((unsigned)-1, I, ResType, TII, ZeroAsNull); | |
| Reg0 = GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull); | |
| Reg32 = GR.getOrCreateConstInt(32, I, ResType, TII, ZeroAsNull); | |
| SelectOp = SPIRV::OpSelectSISCond; | |
| AddOp = SPIRV::OpIAddS; | |
| } else { | |
| BoolType = GR.getOrCreateSPIRVVectorType(BoolType, ComponentCount, | |
| MIRBuilder, false); | |
| NegOneReg = | |
| GR.getOrCreateConstVector((unsigned)-1, I, ResType, TII, ZeroAsNull); | |
| Reg0 = GR.getOrCreateConstVector(0, I, ResType, TII, ZeroAsNull); | |
| Reg32 = GR.getOrCreateConstVector(32, I, ResType, TII, ZeroAsNull); | |
| SelectOp = SPIRV::OpSelectVIVCond; | |
| AddOp = SPIRV::OpIAddV; | |
| } | |
| Register PrimaryReg = HighReg; | |
| Register SecondaryReg = LowReg; | |
| Register RegPrimaryOffset = Reg32; | |
| Register RegSecondaryOffset = Reg0; | |
| // By default the emitted opcodes check for the set bit from the MSB side. | |
| // Setting SwapPrimarySide checks the set bit from the LSB side | |
| if (SwapPrimarySide) { | |
| PrimaryReg = LowReg; | |
| SecondaryReg = HighReg; | |
| RegPrimaryOffset = Reg0; | |
| RegSecondaryOffset = Reg32; | |
| } | |
| Register RegSecondaryHasVal = | |
| MRI->createVirtualRegister(GR.getRegClass(BoolType)); | |
| if (!selectOpWithSrcs(RegSecondaryHasVal, BoolType, I, | |
| {SecondaryReg, NegOneReg}, SPIRV::OpINotEqual)) | |
| return false; | |
| Register RegPrimaryHasVal = | |
| MRI->createVirtualRegister(GR.getRegClass(BoolType)); | |
| if (!selectOpWithSrcs(RegPrimaryHasVal, BoolType, I, {PrimaryReg, NegOneReg}, | |
| SPIRV::OpINotEqual)) | |
| return false; | |
| // Pass 1: seed with secondary (lower-priority fallback) | |
| // ReturnBits = secondaryHasVal ? SecondaryBits : -1 | |
| // Add = secondaryHasVal ? SecondaryOffset : 0 | |
| Register RegReturnBits = MRI->createVirtualRegister(GR.getRegClass(ResType)); | |
| if (!selectOpWithSrcs(RegReturnBits, ResType, I, | |
| {RegSecondaryHasVal, SecondaryReg, NegOneReg}, | |
| SelectOp)) | |
| return false; | |
| Register RegAdd; | |
| if (SwapPrimarySide) { | |
| RegAdd = MRI->createVirtualRegister(GR.getRegClass(ResType)); | |
| if (!selectOpWithSrcs(RegAdd, ResType, I, | |
| {RegSecondaryHasVal, RegSecondaryOffset, Reg0}, | |
| SelectOp)) | |
| return false; | |
| } else { | |
| RegAdd = Reg0; | |
| } | |
| // Pass 2: override with primary (higher priority) if it has a valid result | |
| // ReturnBits2 = primaryHasVal ? PrimaryBits : ReturnBits | |
| // Add2 = primaryHasVal ? PrimaryOffset : Add | |
| Register RegReturnBits2 = MRI->createVirtualRegister(GR.getRegClass(ResType)); | |
| if (!selectOpWithSrcs(RegReturnBits2, ResType, I, | |
| {RegPrimaryHasVal, PrimaryReg, RegReturnBits}, | |
| SelectOp)) | |
| return false; | |
| Register RegAdd2 = MRI->createVirtualRegister(GR.getRegClass(ResType)); | |
| if (!selectOpWithSrcs(RegAdd2, ResType, I, | |
| {RegPrimaryHasVal, RegPrimaryOffset, RegAdd}, SelectOp)) | |
| return false; | |
| return selectOpWithSrcs(ResVReg, ResType, I, {RegReturnBits2, RegAdd2}, | |
| AddOp); | |
| } | |
| bool SPIRVInstructionSelector::selectFirstBitHigh(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I, | |
| bool IsSigned) const { | |
| // FindUMsb and FindSMsb intrinsics only support 32 bit integers | |
| Register OpReg = I.getOperand(2).getReg(); | |
| SPIRVTypeInst OpType = GR.getSPIRVTypeForVReg(OpReg); | |
| // zero or sign extend | |
| unsigned ExtendOpcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; | |
| unsigned BitSetOpcode = IsSigned ? GL::FindSMsb : GL::FindUMsb; | |
| switch (GR.getScalarOrVectorBitWidth(OpType)) { | |
| case 16: | |
| return selectFirstBitSet16(ResVReg, ResType, I, ExtendOpcode, BitSetOpcode); | |
| case 32: | |
| return selectFirstBitSet32(ResVReg, ResType, I, OpReg, BitSetOpcode); | |
| case 64: | |
| return selectFirstBitSet64(ResVReg, ResType, I, OpReg, BitSetOpcode, | |
| /*SwapPrimarySide=*/false); | |
| default: | |
| return diagnoseUnsupported( | |
| I, | |
| "spv_firstbituhigh and spv_firstbitshigh only support 16,32,64 bits."); | |
| } | |
| } | |
| bool SPIRVInstructionSelector::selectFirstBitLow(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| // FindILsb intrinsic only supports 32 bit integers | |
| Register OpReg = I.getOperand(2).getReg(); | |
| SPIRVTypeInst OpType = GR.getSPIRVTypeForVReg(OpReg); | |
| // OpUConvert treats the operand bits as an unsigned i16 and zero extends it | |
| // to an unsigned i32. As this leaves all the least significant bits unchanged | |
| // so the first set bit from the LSB side doesn't change. | |
| unsigned ExtendOpcode = SPIRV::OpUConvert; | |
| unsigned BitSetOpcode = GL::FindILsb; | |
| switch (GR.getScalarOrVectorBitWidth(OpType)) { | |
| case 16: | |
| return selectFirstBitSet16(ResVReg, ResType, I, ExtendOpcode, BitSetOpcode); | |
| case 32: | |
| return selectFirstBitSet32(ResVReg, ResType, I, OpReg, BitSetOpcode); | |
| case 64: | |
| return selectFirstBitSet64(ResVReg, ResType, I, OpReg, BitSetOpcode, | |
| /*SwapPrimarySide=*/true); | |
| default: | |
| return diagnoseUnsupported(I, | |
| "spv_firstbitlow only supports 16,32,64 bits."); | |
| } | |
| } | |
| bool SPIRVInstructionSelector::selectAllocaArray(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| // there was an allocation size parameter to the allocation instruction | |
| // that is not 1 | |
| MachineBasicBlock &BB = *I.getParent(); | |
| bool UseUntypedPointers = | |
| ResType->getOpcode() == SPIRV::OpTypeUntypedPointerKHR; | |
| unsigned Opcode = UseUntypedPointers | |
| ? SPIRV::OpUntypedVariableLengthArrayINTEL | |
| : SPIRV::OpVariableLengthArrayINTEL; | |
| auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)); | |
| // OpUntypedVariableLengthArrayINTEL takes an explicit Element Type <id> | |
| // right after the result type | |
| if (UseUntypedPointers) { | |
| SPIRVTypeInst ElementType = GR.getUntypedPtrElementType(ResVReg); | |
| assert(ElementType && | |
| "untyped variable length array result must have a recorded element " | |
| "type"); | |
| MIB.addUse(GR.getSPIRVTypeID(ElementType)); | |
| } | |
| MIB.addUse(I.getOperand(2).getReg()); | |
| MIB.constrainAllUses(TII, TRI, RBI); | |
| if (!STI.isShader()) { | |
| unsigned Alignment = I.getOperand(3).getImm(); | |
| buildOpDecorate(ResVReg, I, TII, SPIRV::Decoration::Alignment, {Alignment}); | |
| } | |
| return true; | |
| } | |
| // Returns true iff `Ty` is a concrete SPIR-V type per the SPV_KHR_abort | |
| // definition: a numerical scalar (int/float), a (physical) pointer, a vector, | |
| // matrix or any aggregate (array/struct) recursively containing only such | |
| // types. OpTypeBool, OpTypeVoid, opaque handles and similar abstract | |
| // non-concrete types are rejected. | |
| static bool isConcreteSPIRVType(SPIRVTypeInst Ty, | |
| const SPIRVGlobalRegistry &GR) { | |
| SmallVector<SPIRVTypeInst, 4> Worklist{Ty}; | |
| while (!Worklist.empty()) { | |
| SPIRVTypeInst T = Worklist.pop_back_val(); | |
| switch (T->getOpcode()) { | |
| case SPIRV::OpTypeInt: | |
| case SPIRV::OpTypeFloat: | |
| case SPIRV::OpTypePointer: | |
| break; | |
| case SPIRV::OpTypeVector: | |
| case SPIRV::OpTypeVectorIdEXT: | |
| case SPIRV::OpTypeMatrix: | |
| case SPIRV::OpTypeArray: { | |
| Register OperandReg = T->getOperand(1).getReg(); | |
| SPIRVTypeInst ElementT = GR.getSPIRVTypeForVReg(OperandReg); | |
| Worklist.push_back(ElementT); | |
| } break; | |
| case SPIRV::OpTypeStruct: | |
| for (unsigned Idx = 1, E = T->getNumOperands(); Idx < E; ++Idx) { | |
| Register OperandReg = T->getOperand(Idx).getReg(); | |
| SPIRVTypeInst ElementT = GR.getSPIRVTypeForVReg(OperandReg); | |
| Worklist.push_back(ElementT); | |
| } | |
| break; | |
| default: | |
| return false; | |
| } | |
| } | |
| return true; | |
| } | |
| static bool containsStorageBufferPointer(SPIRVTypeInst Ty, | |
| const SPIRVGlobalRegistry &GR, | |
| SmallSet<Register, 8> &Visited) { | |
| Register TypeReg = Ty->getOperand(0).getReg(); | |
| if (!Visited.insert(TypeReg).second) | |
| return false; | |
| switch (Ty->getOpcode()) { | |
| case SPIRV::OpTypePointer: | |
| if (Ty->getOperand(1).getImm() == SPIRV::StorageClass::StorageBuffer) | |
| return true; | |
| return containsStorageBufferPointer( | |
| GR.getSPIRVTypeForVReg(Ty->getOperand(2).getReg()), GR, Visited); | |
| case SPIRV::OpTypeArray: | |
| case SPIRV::OpTypeRuntimeArray: | |
| return containsStorageBufferPointer( | |
| GR.getSPIRVTypeForVReg(Ty->getOperand(1).getReg()), GR, Visited); | |
| case SPIRV::OpTypeStruct: | |
| for (unsigned I = 1; I < Ty->getNumOperands(); ++I) | |
| if (containsStorageBufferPointer( | |
| GR.getSPIRVTypeForVReg(Ty->getOperand(I).getReg()), GR, Visited)) | |
| return true; | |
| return false; | |
| default: | |
| return false; | |
| } | |
| } | |
| static bool containsStorageBufferPointer(SPIRVTypeInst Ty, | |
| const SPIRVGlobalRegistry &GR) { | |
| SmallSet<Register, 8> Visited; | |
| return containsStorageBufferPointer(Ty, GR, Visited); | |
| } | |
| bool SPIRVInstructionSelector::selectAbort(MachineInstr &I) const { | |
| assert(I.getNumExplicitOperands() == 2); | |
| Register MsgReg = I.getOperand(1).getReg(); | |
| SPIRVTypeInst MsgType = GR.getSPIRVTypeForVReg(MsgReg); | |
| assert(MsgType && "Message argument of llvm.spv.abort has no SPIR-V type"); | |
| if (!isConcreteSPIRVType(MsgType, GR)) | |
| return diagnoseUnsupported( | |
| I, | |
| "llvm.spv.abort message type must be a concrete SPIR-V type (numerical " | |
| "scalar, pointer, vector, matrix, or aggregate of such types)"); | |
| MachineBasicBlock &BB = *I.getParent(); | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpAbortKHR)) | |
| .addUse(GR.getSPIRVTypeID(MsgType)) | |
| .addUse(MsgReg) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectTrap(MachineInstr &I) const { | |
| // When the SPV_KHR_abort extension is disabled, drop the G_TRAP and | |
| // G_UBSANTRAP silently. Also drop the no-op placeholder of the service | |
| // function. | |
| if (!STI.canUseExtension(SPIRV::Extension::SPV_KHR_abort) || | |
| I.getMF()->getFunction().hasFnAttribute(SPIRV_BACKEND_SERVICE_FUN_NAME)) | |
| return true; | |
| // Use the 32-bit integer constant for the abort "message" argument: | |
| // - G_UBSANTRAP operand is zero-extended to 32 bits. | |
| // - "All ones" constant is used for G_TRAP. | |
| uint32_t MsgVal = ~0u; | |
| if (I.getOpcode() == TargetOpcode::G_UBSANTRAP) | |
| MsgVal = static_cast<uint32_t>(I.getOperand(0).getImm()); | |
| SPIRVTypeInst MsgType = GR.getOrCreateSPIRVIntegerType(32, I, TII); | |
| Register MsgReg = buildI32ConstantInEntryBlock(MsgVal, I, MsgType); | |
| MachineBasicBlock &BB = *I.getParent(); | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpAbortKHR)) | |
| .addUse(GR.getSPIRVTypeID(MsgType)) | |
| .addUse(MsgReg) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectFrameIndex(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| // Change order of instructions if needed: all OpVariable instructions in a | |
| // function must be the first instructions in the first block | |
| auto It = getOpVariableMBBIt(*I.getMF()); | |
| // Pointers to opaque types stay typed even with the extension on, so emit the | |
| // untyped variant only when the result is actually an untyped pointer. | |
| bool UseUntypedPointers = | |
| ResType->getOpcode() == SPIRV::OpTypeUntypedPointerKHR; | |
| unsigned Opcode = | |
| UseUntypedPointers ? SPIRV::OpUntypedVariableKHR : SPIRV::OpVariable; | |
| if (!UseUntypedPointers && containsStorageBufferPointer(ResType, GR)) { | |
| MachineIRBuilder MIRBuilder(I); | |
| if (!STI.isAtLeastSPIRVVer(VersionTuple(1, 3))) | |
| MIRBuilder.buildInstr(SPIRV::OpExtension) | |
| .addImm(SPIRV::Extension::SPV_KHR_variable_pointers); | |
| MIRBuilder.buildInstr(SPIRV::OpCapability) | |
| .addImm(SPIRV::Capability::VariablePointersStorageBuffer); | |
| } | |
| auto MIB = BuildMI(*It->getParent(), It, It->getDebugLoc(), TII.get(Opcode)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addImm(static_cast<uint32_t>(SPIRV::StorageClass::Function)); | |
| // OpUntypedVariableKHR takes an extra Data Type operand right after the | |
| // storage class. | |
| if (UseUntypedPointers) { | |
| // Get the element type that was stored when processing spv_assign_ptr_type. | |
| SPIRVTypeInst DataType = GR.getUntypedPtrElementType(ResVReg); | |
| if (!DataType) | |
| return diagnoseUnsupported( | |
| I, "could not deduce the data type of an untyped variable"); | |
| MIB.addUse(GR.getSPIRVTypeID(DataType)); | |
| } | |
| MIB.constrainAllUses(TII, TRI, RBI); | |
| if (!STI.isShader()) { | |
| unsigned Alignment = I.getOperand(2).getImm(); | |
| buildOpDecorate(ResVReg, *It, TII, SPIRV::Decoration::Alignment, | |
| {Alignment}); | |
| } | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectBranch(MachineInstr &I) const { | |
| // InstructionSelector walks backwards through the instructions. We can use | |
| // both a G_BR and a G_BRCOND to create an OpBranchConditional. We hit G_BR | |
| // first, so can generate an OpBranchConditional here. If there is no | |
| // G_BRCOND, we just use OpBranch for a regular unconditional branch. | |
| const MachineInstr *PrevI = I.getPrevNode(); | |
| MachineBasicBlock &MBB = *I.getParent(); | |
| if (PrevI != nullptr && PrevI->getOpcode() == TargetOpcode::G_BRCOND) { | |
| BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional)) | |
| .addUse(PrevI->getOperand(0).getReg()) | |
| .addMBB(PrevI->getOperand(1).getMBB()) | |
| .addMBB(I.getOperand(0).getMBB()) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranch)) | |
| .addMBB(I.getOperand(0).getMBB()) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectBranchCond(MachineInstr &I) const { | |
| // InstructionSelector walks backwards through the instructions. For an | |
| // explicit conditional branch with no fallthrough, we use both a G_BR and a | |
| // G_BRCOND to create an OpBranchConditional. We should hit G_BR first, and | |
| // generate the OpBranchConditional in selectBranch above. | |
| // | |
| // If an OpBranchConditional has been generated, we simply return, as the work | |
| // is alread done. If there is no OpBranchConditional, LLVM must be relying on | |
| // implicit fallthrough to the next basic block, so we need to create an | |
| // OpBranchConditional with an explicit "false" argument pointing to the next | |
| // basic block that LLVM would fall through to. | |
| const MachineInstr *NextI = I.getNextNode(); | |
| // Check if this has already been successfully selected. | |
| if (NextI != nullptr && NextI->getOpcode() == SPIRV::OpBranchConditional) | |
| return true; | |
| // Must be relying on implicit block fallthrough, so generate an | |
| // OpBranchConditional with the "next" basic block as the "false" target. | |
| MachineBasicBlock &MBB = *I.getParent(); | |
| unsigned NextMBBNum = MBB.getNextNode()->getNumber(); | |
| MachineBasicBlock *NextMBB = I.getMF()->getBlockNumbered(NextMBBNum); | |
| BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional)) | |
| .addUse(I.getOperand(0).getReg()) | |
| .addMBB(I.getOperand(1).getMBB()) | |
| .addMBB(NextMBB) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectPhi(Register ResVReg, | |
| MachineInstr &I) const { | |
| auto MIB = | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(TargetOpcode::PHI)) | |
| .addDef(ResVReg); | |
| const unsigned NumOps = I.getNumOperands(); | |
| for (unsigned i = 1; i < NumOps; i += 2) { | |
| MIB.addUse(I.getOperand(i + 0).getReg()); | |
| MIB.addMBB(I.getOperand(i + 1).getMBB()); | |
| } | |
| MIB.constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectGlobalValue( | |
| Register ResVReg, MachineInstr &I, const MachineInstr *Init) const { | |
| // FIXME: don't use MachineIRBuilder here, replace it with BuildMI. | |
| MachineIRBuilder MIRBuilder(I); | |
| const GlobalValue *GV = I.getOperand(1).getGlobal(); | |
| Type *GVType = toTypedPointer(GR.getDeducedGlobalValueType(GV)); | |
| std::string GlobalIdent; | |
| if (!GV->hasName()) { | |
| unsigned &ID = UnnamedGlobalIDs[GV]; | |
| if (ID == 0) | |
| ID = UnnamedGlobalIDs.size(); | |
| GlobalIdent = "__unnamed_" + Twine(ID).str(); | |
| } else { | |
| GlobalIdent = GV->getName(); | |
| } | |
| // Behaviour of functions as operands depends on availability of the | |
| // corresponding extension (SPV_INTEL_function_pointers): | |
| // - If there is an extension to operate with functions as operands: | |
| // We create a proper constant operand and evaluate a correct type for a | |
| // function pointer. | |
| // - Without the required extension: | |
| // We have functions as operands in tests with blocks of instruction e.g. in | |
| // transcoding/global_block.ll. These operands are not used and should be | |
| // substituted by zero constants. Their type is expected to be always | |
| // OpTypePointer Function %uchar. | |
| if (isa<Function>(GV)) { | |
| const Constant *ConstVal = GV; | |
| MachineBasicBlock &BB = *I.getParent(); | |
| Register NewReg = GR.find(ConstVal, GR.CurMF); | |
| if (!NewReg.isValid()) { | |
| const Function *GVFun = | |
| STI.canUseExtension(SPIRV::Extension::SPV_INTEL_function_pointers) | |
| ? dyn_cast<Function>(GV) | |
| : nullptr; | |
| SPIRVTypeInst ResType = GR.getOrCreateSPIRVPointerType( | |
| GVType, I, | |
| GVFun ? SPIRV::StorageClass::CodeSectionINTEL | |
| : addressSpaceToStorageClass(GV->getAddressSpace(), STI)); | |
| if (GVFun) { | |
| // References to a function via function pointers generate virtual | |
| // registers without a definition. We will resolve it later, during | |
| // module analysis stage. | |
| Register ResTypeReg = GR.getSPIRVTypeID(ResType); | |
| MachineRegisterInfo *MRI = MIRBuilder.getMRI(); | |
| Register FuncVReg = | |
| MRI->createGenericVirtualRegister(GR.getRegType(ResType)); | |
| MRI->setRegClass(FuncVReg, &SPIRV::pIDRegClass); | |
| GR.assignSPIRVTypeToVReg(ResType, FuncVReg, *GR.CurMF); | |
| MachineInstrBuilder MIB1 = | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) | |
| .addDef(FuncVReg) | |
| .addUse(ResTypeReg); | |
| MachineInstrBuilder MIB2 = | |
| BuildMI(BB, I, I.getDebugLoc(), | |
| TII.get(SPIRV::OpConstantFunctionPointerINTEL)) | |
| .addDef(ResVReg) | |
| .addUse(ResTypeReg) | |
| .addUse(FuncVReg); | |
| GR.add(ConstVal, MIB2); | |
| // mapping the function pointer to the used Function | |
| GR.recordFunctionPointer(&MIB2.getInstr()->getOperand(2), GVFun); | |
| GR.assignSPIRVTypeToVReg(ResType, ResVReg, *GR.CurMF); | |
| MIB1.constrainAllUses(TII, TRI, RBI); | |
| MIB2.constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| MachineInstrBuilder MIB3 = | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)); | |
| GR.add(ConstVal, MIB3); | |
| GR.recordFunctionPointer(&MIB3.getInstr()->getOperand(0), | |
| cast<Function>(GV)); | |
| MIB3.constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| assert(NewReg != ResVReg); | |
| return BuildCOPY(ResVReg, NewReg, I); | |
| } | |
| auto GlobalVar = cast<GlobalVariable>(GV); | |
| assert(GlobalVar->getName() != "llvm.global.annotations"); | |
| // Skip empty declaration for GVs with initializers till we get the decl with | |
| // passed initializer. | |
| if (hasInitializer(GlobalVar) && !Init) | |
| return true; | |
| const std::optional<SPIRV::LinkageType::LinkageType> LnkType = | |
| getSpirvLinkageTypeFor(STI, *GV); | |
| if (LnkType && *LnkType == SPIRV::LinkageType::Import) | |
| Init = nullptr; | |
| const unsigned AddrSpace = GV->getAddressSpace(); | |
| SPIRV::StorageClass::StorageClass StorageClass = | |
| addressSpaceToStorageClass(AddrSpace, STI); | |
| SPIRVTypeInst ResType = | |
| GR.getOrCreateSPIRVPointerType(GVType, I, StorageClass); | |
| Register Reg = GR.buildGlobalVariable( | |
| ResVReg, ResType, GlobalIdent, GV, StorageClass, Init, | |
| GlobalVar->isConstant(), LnkType, MIRBuilder, true); | |
| // TODO: For AMDGCN, we pipe externally_initialized through via | |
| // HostAccessINTEL, with ReadWrite (3) access, which is we then handle during | |
| // reverse translation. We should remove this once SPIR-V gains the ability to | |
| // express the concept. | |
| if (GlobalVar->isExternallyInitialized() && | |
| STI.getTargetTriple().getVendor() == Triple::AMD) { | |
| constexpr unsigned ReadWriteINTEL = 3u; | |
| buildOpDecorate(Reg, MIRBuilder, SPIRV::Decoration::HostAccessINTEL, | |
| {ReadWriteINTEL}); | |
| MachineInstrBuilder MIB(*MF, --MIRBuilder.getInsertPt()); | |
| addStringImm(GV->getName(), MIB); | |
| } | |
| return Reg.isValid(); | |
| } | |
| bool SPIRVInstructionSelector::selectLog10(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| if (STI.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) { | |
| return selectExtInst(ResVReg, ResType, I, CL::log10); | |
| } | |
| // There is no log10 instruction in the GLSL Extended Instruction set, so it | |
| // is implemented as: | |
| // log10(x) = log2(x) * (1 / log2(10)) | |
| // = log2(x) * 0.30103 | |
| MachineIRBuilder MIRBuilder(I); | |
| MachineBasicBlock &BB = *I.getParent(); | |
| // Build log2(x). | |
| Register VarReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) | |
| .addDef(VarReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450)) | |
| .addImm(GL::Log2) | |
| .add(I.getOperand(1)) | |
| .constrainAllUses(TII, TRI, RBI); | |
| // Build 0.30103. | |
| assert(isVectorType(ResType) || ResType.isAnyTypeFloat()); | |
| // TODO: Add matrix implementation once supported by the HLSL frontend. | |
| SPIRVTypeInst SpirvScalarType = GR.getScalarOrVectorComponentType(ResType); | |
| // The literal must match the precision of the scalar type, otherwise the | |
| // OpConstant will contain non-zero high-order bits and fail SPIR-V | |
| // validation when the type is narrower than 32 bits (e.g. half). | |
| APFloat ScaleVal(0.30103); | |
| bool LosesInfo; | |
| ScaleVal.convert( | |
| getZeroFP(GR.getTypeForSPIRVType(SpirvScalarType)).getSemantics(), | |
| APFloat::rmNearestTiesToEven, &LosesInfo); | |
| Register ScaleReg = GR.buildConstantFP(ScaleVal, MIRBuilder, SpirvScalarType); | |
| // Multiply log2(x) by 0.30103 to get log10(x) result. | |
| auto Opcode = | |
| isVectorType(ResType) ? SPIRV::OpVectorTimesScalar : SPIRV::OpFMulS; | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(VarReg) | |
| .addUse(ScaleReg) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::selectFpowi(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| // On OpenCL targets, pown(gentype x, intn n) maps directly. | |
| if (STI.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) | |
| return selectExtInst(ResVReg, ResType, I, CL::pown); | |
| // On GLSL (Vulkan) targets, there is no integer-exponent power instruction. | |
| // Lower as: Pow(base, OpConvertSToF(exp)). | |
| if (STI.canUseExtInstSet(SPIRV::InstructionSet::GLSL_std_450)) { | |
| Register BaseReg = I.getOperand(1).getReg(); | |
| Register ExpReg = I.getOperand(2).getReg(); | |
| Register FloatExpReg = MRI->createVirtualRegister(GR.getRegClass(ResType)); | |
| if (!selectOpWithSrcs(FloatExpReg, ResType, I, {ExpReg}, | |
| SPIRV::OpConvertSToF)) | |
| return false; | |
| return selectExtInst(ResVReg, ResType, I, GL::Pow, | |
| /*setMIFlags=*/true, /*useMISrc=*/false, | |
| {BaseReg, FloatExpReg}); | |
| } | |
| return false; | |
| } | |
| bool SPIRVInstructionSelector::selectModf(Register ResVReg, | |
| SPIRVTypeInst ResType, | |
| MachineInstr &I) const { | |
| // llvm.modf has a single arg --the number to be decomposed-- and returns a | |
| // struct { restype, restype }, while OpenCLLIB::modf has two args --the | |
| // number to be decomposed and a pointer--, returns the fractional part and | |
| // the integral part is stored in the pointer argument. Therefore, we can't | |
| // use directly the OpenCLLIB::modf intrinsic. However, we can do some | |
| // scaffolding to make it work. The idea is to create an alloca instruction | |
| // to get a ptr, pass this ptr to OpenCL::modf, and then load the value | |
| // from this ptr to place it in the struct. llvm.modf returns the fractional | |
| // part as the first element of the result, and the integral part as the | |
| // second element of the result. | |
| // At this point, the return type is not a struct anymore, but rather two | |
| // independent elements of SPIRVResType. We can get each independent element | |
| // from I.getDefs() or I.getOperands(). | |
| if (STI.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) { | |
| MachineIRBuilder MIRBuilder(I); | |
| SPIRVTypeInst FloatType = | |
| GR.getSPIRVTypeForVReg(I.getOperand(I.getNumExplicitDefs()).getReg()); | |
| // Get pointer type for alloca variable. | |
| const SPIRVTypeInst PtrType = GR.getOrCreateSPIRVPointerType( | |
| FloatType, MIRBuilder, SPIRV::StorageClass::Function); | |
| // Create new register for the pointer type of alloca variable. | |
| Register PtrTyReg = | |
| MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::iIDRegClass); | |
| MIRBuilder.getMRI()->setType( | |
| PtrTyReg, | |
| LLT::pointer(storageClassToAddressSpace(SPIRV::StorageClass::Function), | |
| GR.getPointerSize())); | |
| // Assign SPIR-V type of the pointer type of the alloca variable to the | |
| // new register. | |
| GR.assignSPIRVTypeToVReg(PtrType, PtrTyReg, MIRBuilder.getMF()); | |
| MachineBasicBlock::iterator VarPos = getOpVariableMBBIt(*I.getMF()); | |
| MachineBasicBlock &EntryBB = I.getMF()->front(); | |
| const bool IsUntyped = | |
| PtrType->getOpcode() == SPIRV::OpTypeUntypedPointerKHR; | |
| auto AllocaMIB = | |
| BuildMI(EntryBB, VarPos, I.getDebugLoc(), | |
| TII.get(IsUntyped ? SPIRV::OpUntypedVariableKHR | |
| : SPIRV::OpVariable)) | |
| .addDef(PtrTyReg) | |
| .addUse(GR.getSPIRVTypeID(PtrType)) | |
| .addImm(static_cast<uint32_t>(SPIRV::StorageClass::Function)); | |
| if (IsUntyped) | |
| AllocaMIB.addUse(GR.getSPIRVTypeID(ResType)); // Data Type | |
| Register Variable = AllocaMIB->getOperand(0).getReg(); | |
| MachineBasicBlock &BB = *I.getParent(); | |
| // Create the OpenCLLIB::modf instruction. | |
| auto MIB = | |
| BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(FloatType)) | |
| .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::OpenCL_std)) | |
| .addImm(CL::modf) | |
| .setMIFlags(I.getFlags()) | |
| .add(I.getOperand(I.getNumExplicitDefs())) // Floating point value. | |
| .addUse(Variable); // Pointer to integral part. | |
| // Assign the integral part stored in the ptr to the second element of the | |
| // result. | |
| Register IntegralPartReg = I.getOperand(1).getReg(); | |
| if (IntegralPartReg.isValid() && !MRI->use_nodbg_empty(IntegralPartReg)) { | |
| // Load the value from the pointer to integral part. | |
| auto LoadMIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) | |
| .addDef(IntegralPartReg) | |
| .addUse(GR.getSPIRVTypeID(FloatType)) | |
| .addUse(Variable); | |
| LoadMIB.constrainAllUses(TII, TRI, RBI); | |
| } | |
| MIB.constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } else if (STI.canUseExtInstSet(SPIRV::InstructionSet::GLSL_std_450)) { | |
| assert(false && "GLSL::Modf is deprecated."); | |
| // FIXME: GL::Modf is deprecated, use Modfstruct instead. | |
| return false; | |
| } | |
| return false; | |
| } | |
| // Generate the instructions to load 3-element vector builtin input | |
| // IDs/Indices. | |
| // Like: GlobalInvocationId, LocalInvocationId, etc.... | |
| bool SPIRVInstructionSelector::loadVec3BuiltinInputID( | |
| SPIRV::BuiltIn::BuiltIn BuiltInValue, Register ResVReg, | |
| SPIRVTypeInst ResType, MachineInstr &I) const { | |
| MachineIRBuilder MIRBuilder(I); | |
| const SPIRVTypeInst Vec3Ty = | |
| GR.getOrCreateSPIRVVectorType(ResType, 3, MIRBuilder, false); | |
| const SPIRVTypeInst PtrType = GR.getOrCreateSPIRVPointerType( | |
| Vec3Ty, MIRBuilder, SPIRV::StorageClass::Input); | |
| // Create new register for the input ID builtin variable. | |
| Register NewRegister = | |
| MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::iIDRegClass); | |
| MIRBuilder.getMRI()->setType(NewRegister, LLT::pointer(0, 64)); | |
| GR.assignSPIRVTypeToVReg(PtrType, NewRegister, MIRBuilder.getMF()); | |
| // Build global variable with the necessary decorations for the input ID | |
| // builtin variable. | |
| Register Variable = GR.buildGlobalVariable( | |
| NewRegister, PtrType, getLinkStringForBuiltIn(BuiltInValue), nullptr, | |
| SPIRV::StorageClass::Input, nullptr, true, std::nullopt, MIRBuilder, | |
| false); | |
| // Create new register for loading value. | |
| MachineRegisterInfo *MRI = MIRBuilder.getMRI(); | |
| Register LoadedRegister = MRI->createVirtualRegister(&SPIRV::iIDRegClass); | |
| MIRBuilder.getMRI()->setType(LoadedRegister, LLT::pointer(0, 64)); | |
| GR.assignSPIRVTypeToVReg(Vec3Ty, LoadedRegister, MIRBuilder.getMF()); | |
| // Load v3uint value from the global variable. | |
| BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) | |
| .addDef(LoadedRegister) | |
| .addUse(GR.getSPIRVTypeID(Vec3Ty)) | |
| .addUse(Variable); | |
| // Get the input ID index. Expecting operand is a constant immediate value, | |
| // wrapped in a type assignment. | |
| assert(I.getOperand(2).isReg()); | |
| const uint32_t ThreadId = foldImm(I.getOperand(2), MRI); | |
| // Extract the input ID from the loaded vector value. | |
| MachineBasicBlock &BB = *I.getParent(); | |
| auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(LoadedRegister) | |
| .addImm(ThreadId); | |
| MIB.constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| // Generate the instructions to load 32-bit integer builtin input IDs/Indices. | |
| // Like LocalInvocationIndex | |
| bool SPIRVInstructionSelector::loadBuiltinInputID( | |
| SPIRV::BuiltIn::BuiltIn BuiltInValue, Register ResVReg, | |
| SPIRVTypeInst ResType, MachineInstr &I) const { | |
| MachineIRBuilder MIRBuilder(I); | |
| const SPIRVTypeInst PtrType = GR.getOrCreateSPIRVPointerType( | |
| ResType, MIRBuilder, SPIRV::StorageClass::Input); | |
| // Create new register for the input ID builtin variable. | |
| Register NewRegister = | |
| MIRBuilder.getMRI()->createVirtualRegister(GR.getRegClass(PtrType)); | |
| MIRBuilder.getMRI()->setType( | |
| NewRegister, | |
| LLT::pointer(storageClassToAddressSpace(SPIRV::StorageClass::Input), | |
| GR.getPointerSize())); | |
| GR.assignSPIRVTypeToVReg(PtrType, NewRegister, MIRBuilder.getMF()); | |
| // Build global variable with the necessary decorations for the input ID | |
| // builtin variable. | |
| Register Variable = GR.buildGlobalVariable( | |
| NewRegister, PtrType, getLinkStringForBuiltIn(BuiltInValue), nullptr, | |
| SPIRV::StorageClass::Input, nullptr, true, std::nullopt, MIRBuilder, | |
| false); | |
| // Load uint value from the global variable. | |
| auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) | |
| .addDef(ResVReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(Variable); | |
| MIB.constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| SPIRVTypeInst SPIRVInstructionSelector::widenTypeToVec4(SPIRVTypeInst Type, | |
| MachineInstr &I) const { | |
| MachineIRBuilder MIRBuilder(I); | |
| if (!isVectorType(Type)) | |
| return GR.getOrCreateSPIRVVectorType(Type, 4, MIRBuilder, false); | |
| if (GR.getScalarOrVectorComponentCount(Type) == 4) | |
| return Type; | |
| SPIRVTypeInst ScalarType = GR.getScalarOrVectorComponentType(Type); | |
| return GR.getOrCreateSPIRVVectorType(ScalarType, 4, MIRBuilder, false); | |
| } | |
| bool SPIRVInstructionSelector::loadHandleBeforePosition( | |
| Register &HandleReg, SPIRVTypeInst ResType, GIntrinsic &HandleDef, | |
| MachineInstr &Pos) const { | |
| assert(HandleDef.getIntrinsicID() == | |
| Intrinsic::spv_resource_handlefrombinding); | |
| uint32_t Set = foldImm(HandleDef.getOperand(2), MRI); | |
| uint32_t Binding = foldImm(HandleDef.getOperand(3), MRI); | |
| uint32_t ArraySize = foldImm(HandleDef.getOperand(4), MRI); | |
| Register IndexReg = HandleDef.getOperand(5).getReg(); | |
| std::string Name = | |
| getStringValueFromReg(HandleDef.getOperand(6).getReg(), *MRI); | |
| bool IsStructuredBuffer = ResType->getOpcode() == SPIRV::OpTypePointer; | |
| MachineIRBuilder MIRBuilder(HandleDef); | |
| SPIRVTypeInst VarType = ResType; | |
| SPIRV::StorageClass::StorageClass SC = SPIRV::StorageClass::UniformConstant; | |
| if (IsStructuredBuffer) { | |
| VarType = GR.getPointeeType(ResType); | |
| SC = GR.getPointerStorageClass(ResType); | |
| } | |
| // ArraySize 0 means an unbounded array and we need to set to required | |
| // capability. | |
| if (ArraySize == 0) | |
| MIRBuilder.buildInstr(SPIRV::OpCapability) | |
| .addImm(SPIRV::Capability::RuntimeDescriptorArrayEXT); | |
| Register VarReg = | |
| buildPointerToResource(SPIRVTypeInst(VarType), SC, Set, Binding, | |
| ArraySize, IndexReg, Name, MIRBuilder); | |
| // The handle for the buffer is the pointer to the resource. For an image, the | |
| // handle is the image object. So images get an extra load. | |
| uint32_t LoadOpcode = | |
| IsStructuredBuffer ? SPIRV::OpCopyObject : SPIRV::OpLoad; | |
| GR.assignSPIRVTypeToVReg(ResType, HandleReg, *Pos.getMF()); | |
| BuildMI(*Pos.getParent(), Pos, HandleDef.getDebugLoc(), TII.get(LoadOpcode)) | |
| .addDef(HandleReg) | |
| .addUse(GR.getSPIRVTypeID(ResType)) | |
| .addUse(VarReg) | |
| .constrainAllUses(TII, TRI, RBI); | |
| return true; | |
| } | |
| bool SPIRVInstructionSelector::errorIfInstrOutsideShader( | |
| MachineInstr &I) const { | |
| if (!STI.isShader()) | |
| return diagnoseUnsupported( | |
| I, "this instruction is only supported in shaders."); | |
| return true; | |
| } | |
| namespace llvm { | |
| InstructionSelector * | |
| createSPIRVInstructionSelector(const SPIRVTargetMachine &TM, | |
| const SPIRVSubtarget &Subtarget, | |
| const RegisterBankInfo &RBI) { | |
| return new SPIRVInstructionSelector(TM, Subtarget, RBI); | |
| } | |
| } // namespace llvm |