| //===-- SPIRVPreLegalizer.cpp - prepare IR for legalization -----*- C++ -*-===// | |
| // | |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |
| // See https://llvm.org/LICENSE.txt for license information. | |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |
| // | |
| //===----------------------------------------------------------------------===// | |
| // | |
| // The pass prepares IR for legalization: it assigns SPIR-V types to registers | |
| // and removes intrinsics which holded these types during IR translation. | |
| // Also it processes constants and registers them in GR to avoid duplication. | |
| // | |
| //===----------------------------------------------------------------------===// | |
| #include "SPIRV.h" | |
| #include "SPIRVSubtarget.h" | |
| #include "SPIRVUtils.h" | |
| #include "llvm/ADT/PostOrderIterator.h" | |
| #include "llvm/CodeGen/GlobalISel/CSEInfo.h" | |
| #include "llvm/CodeGen/GlobalISel/GISelValueTracking.h" | |
| #include "llvm/IR/Attributes.h" | |
| #include "llvm/IR/Constants.h" | |
| #include "llvm/IR/IntrinsicsSPIRV.h" | |
| #define DEBUG_TYPE "spirv-prelegalizer" | |
| using namespace llvm; | |
| namespace { | |
| class SPIRVPreLegalizer : public MachineFunctionPass { | |
| public: | |
| static char ID; | |
| SPIRVPreLegalizer() : MachineFunctionPass(ID) {} | |
| bool runOnMachineFunction(MachineFunction &MF) override; | |
| void getAnalysisUsage(AnalysisUsage &AU) const override; | |
| }; | |
| } // namespace | |
| void SPIRVPreLegalizer::getAnalysisUsage(AnalysisUsage &AU) const { | |
| AU.addPreserved<GISelValueTrackingAnalysisLegacy>(); | |
| MachineFunctionPass::getAnalysisUsage(AU); | |
| } | |
| static inline void invalidateAndEraseMI(SPIRVGlobalRegistry *GR, | |
| MachineInstr *MI) { | |
| GR->invalidateMachineInstr(MI); | |
| MI->eraseFromParent(); | |
| } | |
| static void | |
| addConstantsToTrack(MachineFunction &MF, SPIRVGlobalRegistry *GR, | |
| const SPIRVSubtarget &STI, | |
| DenseMap<MachineInstr *, Type *> &TargetExtConstTypes) { | |
| MachineRegisterInfo &MRI = MF.getRegInfo(); | |
| DenseMap<MachineInstr *, Register> RegsAlreadyAddedToDT; | |
| SmallVector<MachineInstr *, 10> ToErase, ToEraseComposites; | |
| for (MachineBasicBlock &MBB : MF) { | |
| for (MachineInstr &MI : MBB) { | |
| if (!isSpvIntrinsic(MI, Intrinsic::spv_track_constant)) | |
| continue; | |
| ToErase.push_back(&MI); | |
| Register SrcReg = MI.getOperand(2).getReg(); | |
| auto *Const = | |
| cast<Constant>(cast<ConstantAsMetadata>( | |
| MI.getOperand(3).getMetadata()->getOperand(0)) | |
| ->getValue()); | |
| if (auto *GV = dyn_cast<GlobalValue>(Const)) { | |
| Register Reg = GR->find(GV, &MF); | |
| if (!Reg.isValid()) { | |
| GR->add(GV, MRI.getVRegDef(SrcReg)); | |
| GR->addGlobalObject(GV, &MF, SrcReg); | |
| } else | |
| RegsAlreadyAddedToDT[&MI] = Reg; | |
| } else { | |
| Register Reg = GR->find(Const, &MF); | |
| if (!Reg.isValid()) { | |
| if (auto *ConstVec = dyn_cast<ConstantDataVector>(Const)) { | |
| auto *BuildVec = MRI.getVRegDef(SrcReg); | |
| assert(BuildVec && | |
| BuildVec->getOpcode() == TargetOpcode::G_BUILD_VECTOR); | |
| GR->add(Const, BuildVec); | |
| for (unsigned i = 0; i < ConstVec->getNumElements(); ++i) { | |
| // Ensure that OpConstantComposite reuses a constant when it's | |
| // already created and available in the same machine function. | |
| Constant *ElemConst = ConstVec->getElementAsConstant(i); | |
| Register ElemReg = GR->find(ElemConst, &MF); | |
| if (!ElemReg.isValid()) | |
| GR->add(ElemConst, | |
| MRI.getVRegDef(BuildVec->getOperand(1 + i).getReg())); | |
| else | |
| BuildVec->getOperand(1 + i).setReg(ElemReg); | |
| } | |
| } | |
| if (Const->getType()->isTargetExtTy()) { | |
| // remember association so that we can restore it when assign types | |
| MachineInstr *SrcMI = MRI.getVRegDef(SrcReg); | |
| if (SrcMI) | |
| GR->add(Const, SrcMI); | |
| if (SrcMI && (SrcMI->getOpcode() == TargetOpcode::G_CONSTANT || | |
| SrcMI->getOpcode() == TargetOpcode::G_IMPLICIT_DEF)) | |
| TargetExtConstTypes[SrcMI] = Const->getType(); | |
| if (Const->isNullValue()) { | |
| MachineBasicBlock &DepMBB = MF.front(); | |
| MachineIRBuilder MIB(DepMBB, DepMBB.getFirstNonPHI()); | |
| SPIRVTypeInst ExtType = GR->getOrCreateSPIRVType( | |
| Const->getType(), MIB, SPIRV::AccessQualifier::ReadWrite, | |
| true); | |
| assert(SrcMI && "Expected source instruction to be valid"); | |
| SrcMI->setDesc(STI.getInstrInfo()->get(SPIRV::OpConstantNull)); | |
| SrcMI->addOperand(MachineOperand::CreateReg( | |
| GR->getSPIRVTypeID(ExtType), false)); | |
| } | |
| } | |
| } else { | |
| RegsAlreadyAddedToDT[&MI] = Reg; | |
| // This MI is unused and will be removed. If the MI uses | |
| // const_composite, it will be unused and should be removed too. | |
| assert(MI.getOperand(2).isReg() && "Reg operand is expected"); | |
| MachineInstr *SrcMI = MRI.getVRegDef(MI.getOperand(2).getReg()); | |
| if (SrcMI && isSpvIntrinsic(*SrcMI, Intrinsic::spv_const_composite)) | |
| ToEraseComposites.push_back(SrcMI); | |
| } | |
| } | |
| } | |
| } | |
| for (MachineInstr *MI : ToErase) { | |
| Register Reg = MI->getOperand(2).getReg(); | |
| auto It = RegsAlreadyAddedToDT.find(MI); | |
| if (It != RegsAlreadyAddedToDT.end()) | |
| Reg = It->second; | |
| auto *RC = MRI.getRegClassOrNull(MI->getOperand(0).getReg()); | |
| if (!MRI.getRegClassOrNull(Reg) && RC) | |
| MRI.setRegClass(Reg, RC); | |
| MRI.replaceRegWith(MI->getOperand(0).getReg(), Reg); | |
| invalidateAndEraseMI(GR, MI); | |
| } | |
| for (MachineInstr *MI : ToEraseComposites) | |
| invalidateAndEraseMI(GR, MI); | |
| } | |
| static void foldConstantsIntoIntrinsics(MachineFunction &MF, | |
| SPIRVGlobalRegistry *GR, | |
| MachineIRBuilder MIB) { | |
| SmallVector<MachineInstr *, 64> ToErase; | |
| for (MachineBasicBlock &MBB : MF) { | |
| for (MachineInstr &MI : MBB) { | |
| if (!isSpvIntrinsic(MI, Intrinsic::spv_assign_name)) | |
| continue; | |
| const MDNode *MD = MI.getOperand(2).getMetadata(); | |
| StringRef ValueName = cast<MDString>(MD->getOperand(0))->getString(); | |
| if (ValueName.size() > 0) { | |
| MIB.setInsertPt(*MI.getParent(), MI); | |
| buildOpName(MI.getOperand(1).getReg(), ValueName, MIB); | |
| } | |
| ToErase.push_back(&MI); | |
| } | |
| for (MachineInstr *MI : ToErase) | |
| invalidateAndEraseMI(GR, MI); | |
| ToErase.clear(); | |
| } | |
| } | |
| static MachineInstr *findAssignTypeInstr(Register Reg, | |
| MachineRegisterInfo *MRI) { | |
| for (MachineRegisterInfo::use_instr_iterator I = MRI->use_instr_begin(Reg), | |
| IE = MRI->use_instr_end(); | |
| I != IE; ++I) { | |
| MachineInstr *UseMI = &*I; | |
| if ((isSpvIntrinsic(*UseMI, Intrinsic::spv_assign_ptr_type) || | |
| isSpvIntrinsic(*UseMI, Intrinsic::spv_assign_type)) && | |
| UseMI->getOperand(1).getReg() == Reg) | |
| return UseMI; | |
| } | |
| return nullptr; | |
| } | |
| static void buildOpBitcast(SPIRVGlobalRegistry *GR, MachineIRBuilder &MIB, | |
| Register ResVReg, Register OpReg) { | |
| SPIRVTypeInst ResType = GR->getSPIRVTypeForVReg(ResVReg); | |
| SPIRVTypeInst OpType = GR->getSPIRVTypeForVReg(OpReg); | |
| assert(ResType && OpType && "Operand types are expected"); | |
| if (!GR->isBitcastCompatible(ResType, OpType)) | |
| report_fatal_error("incompatible result and operand types in a bitcast"); | |
| MachineRegisterInfo *MRI = MIB.getMRI(); | |
| if (!MRI->getRegClassOrNull(ResVReg)) | |
| MRI->setRegClass(ResVReg, GR->getRegClass(ResType)); | |
| if (ResType == OpType) | |
| MIB.buildInstr(TargetOpcode::COPY).addDef(ResVReg).addUse(OpReg); | |
| else | |
| MIB.buildInstr(SPIRV::OpBitcast) | |
| .addDef(ResVReg) | |
| .addUse(GR->getSPIRVTypeID(ResType)) | |
| .addUse(OpReg); | |
| } | |
| // We lower G_BITCAST to OpBitcast here to avoid a MachineVerifier error. | |
| // The verifier checks if the source and destination LLTs of a G_BITCAST are | |
| // different, but this check is too strict for SPIR-V's typed pointers, which | |
| // may have the same LLT but different SPIRV type (e.g. pointers to different | |
| // pointee types). By lowering to OpBitcast here, we bypass the verifier's | |
| // check. See discussion in https://github.com/llvm/llvm-project/pull/110270 | |
| // for more context. | |
| // | |
| // We also handle the llvm.spv.bitcast intrinsic here. If the source and | |
| // destination SPIR-V types are the same, we lower it to a COPY to enable | |
| // further optimizations like copy propagation. | |
| static void lowerBitcasts(MachineFunction &MF, SPIRVGlobalRegistry *GR, | |
| MachineIRBuilder MIB) { | |
| SmallVector<MachineInstr *, 16> ToErase; | |
| for (MachineBasicBlock &MBB : MF) { | |
| for (MachineInstr &MI : MBB) { | |
| if (isSpvIntrinsic(MI, Intrinsic::spv_bitcast)) { | |
| Register DstReg = MI.getOperand(0).getReg(); | |
| Register SrcReg = MI.getOperand(2).getReg(); | |
| SPIRVTypeInst DstType = GR->getSPIRVTypeForVReg(DstReg); | |
| assert( | |
| DstType && | |
| "Expected destination SPIR-V type to have been assigned already."); | |
| SPIRVTypeInst SrcType = GR->getSPIRVTypeForVReg(SrcReg); | |
| assert(SrcType && | |
| "Expected source SPIR-V type to have been assigned already."); | |
| if (DstType == SrcType) { | |
| MIB.setInsertPt(*MI.getParent(), MI); | |
| MIB.buildCopy(DstReg, SrcReg); | |
| ToErase.push_back(&MI); | |
| continue; | |
| } | |
| } | |
| if (MI.getOpcode() != TargetOpcode::G_BITCAST) | |
| continue; | |
| MIB.setInsertPt(*MI.getParent(), MI); | |
| buildOpBitcast(GR, MIB, MI.getOperand(0).getReg(), | |
| MI.getOperand(1).getReg()); | |
| ToErase.push_back(&MI); | |
| } | |
| } | |
| for (MachineInstr *MI : ToErase) | |
| invalidateAndEraseMI(GR, MI); | |
| } | |
| static void insertBitcasts(MachineFunction &MF, SPIRVGlobalRegistry *GR, | |
| MachineIRBuilder MIB) { | |
| // Get access to information about available extensions | |
| const SPIRVSubtarget *ST = | |
| static_cast<const SPIRVSubtarget *>(&MIB.getMF().getSubtarget()); | |
| SmallVector<MachineInstr *, 10> ToErase; | |
| for (MachineBasicBlock &MBB : MF) { | |
| for (MachineInstr &MI : MBB) { | |
| if (!isSpvIntrinsic(MI, Intrinsic::spv_ptrcast)) | |
| continue; | |
| assert(MI.getOperand(2).isReg()); | |
| MIB.setInsertPt(*MI.getParent(), MI); | |
| ToErase.push_back(&MI); | |
| Register Def = MI.getOperand(0).getReg(); | |
| Register Source = MI.getOperand(2).getReg(); | |
| Type *ElemTy = getMDOperandAsType(MI.getOperand(3).getMetadata(), 0); | |
| auto SC = | |
| isa<FunctionType>(ElemTy) | |
| ? SPIRV::StorageClass::CodeSectionINTEL | |
| : addressSpaceToStorageClass(MI.getOperand(4).getImm(), *ST); | |
| SPIRVTypeInst AssignedPtrType = | |
| GR->getOrCreateSPIRVPointerType(ElemTy, MI, SC); | |
| // If the ptrcast would be redundant, replace all uses with the source | |
| // register. | |
| MachineRegisterInfo *MRI = MIB.getMRI(); | |
| if (GR->getSPIRVTypeForVReg(Source) == AssignedPtrType) { | |
| // Erase Def's assign type instruction if we are going to replace Def. | |
| if (MachineInstr *AssignMI = findAssignTypeInstr(Def, MRI)) | |
| ToErase.push_back(AssignMI); | |
| MRI->replaceRegWith(Def, Source); | |
| } else { | |
| if (!GR->getSPIRVTypeForVReg(Def, &MF)) | |
| GR->assignSPIRVTypeToVReg(AssignedPtrType, Def, MF); | |
| MIB.buildBitcast(Def, Source); | |
| } | |
| } | |
| } | |
| for (MachineInstr *MI : ToErase) | |
| invalidateAndEraseMI(GR, MI); | |
| } | |
| // Translating GV, IRTranslator sometimes generates following IR: | |
| // %1 = G_GLOBAL_VALUE | |
| // %2 = COPY %1 | |
| // %3 = G_ADDRSPACE_CAST %2 | |
| // | |
| // or | |
| // | |
| // %1 = G_ZEXT %2 | |
| // G_MEMCPY ... %2 ... | |
| // | |
| // New registers have no SPIRV type and no register class info. | |
| // | |
| // Set SPIRV type for GV, propagate it from GV to other instructions, | |
| // also set register classes. | |
| static SPIRVTypeInst propagateSPIRVType(MachineInstr *MI, | |
| SPIRVGlobalRegistry *GR, | |
| MachineRegisterInfo &MRI, | |
| MachineIRBuilder &MIB) { | |
| SPIRVTypeInst SpvType = nullptr; | |
| assert(MI && "Machine instr is expected"); | |
| if (MI->getOperand(0).isReg()) { | |
| Register Reg = MI->getOperand(0).getReg(); | |
| SpvType = GR->getSPIRVTypeForVReg(Reg); | |
| if (!SpvType) { | |
| switch (MI->getOpcode()) { | |
| case TargetOpcode::G_FCONSTANT: | |
| case TargetOpcode::G_CONSTANT: { | |
| MIB.setInsertPt(*MI->getParent(), MI); | |
| Type *Ty = MI->getOperand(1).getCImm()->getType(); | |
| SpvType = GR->getOrCreateSPIRVType( | |
| Ty, MIB, SPIRV::AccessQualifier::ReadWrite, true); | |
| break; | |
| } | |
| case TargetOpcode::G_GLOBAL_VALUE: { | |
| MIB.setInsertPt(*MI->getParent(), MI); | |
| const GlobalValue *Global = MI->getOperand(1).getGlobal(); | |
| Type *ElementTy = toTypedPointer(GR->getDeducedGlobalValueType(Global)); | |
| auto *Ty = TypedPointerType::get(ElementTy, | |
| Global->getType()->getAddressSpace()); | |
| SpvType = GR->getOrCreateSPIRVType( | |
| Ty, MIB, SPIRV::AccessQualifier::ReadWrite, true); | |
| break; | |
| } | |
| case TargetOpcode::G_ANYEXT: | |
| case TargetOpcode::G_SEXT: | |
| case TargetOpcode::G_ZEXT: { | |
| if (MI->getOperand(1).isReg()) { | |
| if (MachineInstr *DefInstr = | |
| MRI.getVRegDef(MI->getOperand(1).getReg())) { | |
| if (SPIRVTypeInst Def = | |
| propagateSPIRVType(DefInstr, GR, MRI, MIB)) { | |
| unsigned CurrentBW = GR->getScalarOrVectorBitWidth(Def); | |
| unsigned ExpectedBW = | |
| std::max(MRI.getType(Reg).getScalarSizeInBits(), CurrentBW); | |
| unsigned NumElements = GR->getScalarOrVectorComponentCount(Def); | |
| SpvType = GR->getOrCreateSPIRVIntegerType(ExpectedBW, MIB); | |
| if (NumElements > 1) | |
| SpvType = GR->getOrCreateSPIRVVectorType(SpvType, NumElements, | |
| MIB, true); | |
| } | |
| } | |
| } | |
| break; | |
| } | |
| case TargetOpcode::G_PTRTOINT: | |
| SpvType = GR->getOrCreateSPIRVIntegerType( | |
| MRI.getType(Reg).getScalarSizeInBits(), MIB); | |
| break; | |
| case TargetOpcode::G_TRUNC: | |
| case TargetOpcode::G_ADDRSPACE_CAST: | |
| case TargetOpcode::G_PTR_ADD: | |
| case TargetOpcode::COPY: { | |
| MachineOperand &Op = MI->getOperand(1); | |
| MachineInstr *Def = Op.isReg() ? MRI.getVRegDef(Op.getReg()) : nullptr; | |
| if (Def) | |
| SpvType = propagateSPIRVType(Def, GR, MRI, MIB); | |
| break; | |
| } | |
| default: | |
| break; | |
| } | |
| if (SpvType) { | |
| // check if the address space needs correction | |
| LLT RegType = MRI.getType(Reg); | |
| if (SpvType->getOpcode() == SPIRV::OpTypePointer && | |
| RegType.isPointer() && | |
| storageClassToAddressSpace(GR->getPointerStorageClass(SpvType)) != | |
| RegType.getAddressSpace()) { | |
| const SPIRVSubtarget &ST = | |
| MI->getParent()->getParent()->getSubtarget<SPIRVSubtarget>(); | |
| auto TSC = addressSpaceToStorageClass(RegType.getAddressSpace(), ST); | |
| SpvType = GR->changePointerStorageClass(SpvType, TSC, *MI); | |
| } | |
| GR->assignSPIRVTypeToVReg(SpvType, Reg, MIB.getMF()); | |
| } | |
| if (!MRI.getRegClassOrNull(Reg)) | |
| MRI.setRegClass(Reg, SpvType ? GR->getRegClass(SpvType) | |
| : &SPIRV::iIDRegClass); | |
| } | |
| } | |
| return SpvType; | |
| } | |
| // To support current approach and limitations wrt. bit width here we widen a | |
| // scalar register with a bit width greater than 1 to valid sizes and cap it to | |
| // 128 width. | |
| static unsigned widenBitWidthToNextPow2(unsigned BitWidth) { | |
| if (BitWidth == 1) | |
| return 1; // No need to widen 1-bit values | |
| return std::min(std::max(1u << Log2_32_Ceil(BitWidth), 8u), 128u); | |
| } | |
| static void widenScalarType(Register Reg, MachineRegisterInfo &MRI) { | |
| LLT RegType = MRI.getType(Reg); | |
| if (!RegType.isScalar()) | |
| return; | |
| unsigned CurrentWidth = RegType.getScalarSizeInBits(); | |
| unsigned NewWidth = widenBitWidthToNextPow2(CurrentWidth); | |
| if (NewWidth != CurrentWidth) | |
| MRI.setType(Reg, LLT::scalar(NewWidth)); | |
| } | |
| static void widenCImmType(MachineOperand &MOP) { | |
| const ConstantInt *CImmVal = MOP.getCImm(); | |
| unsigned CurrentWidth = CImmVal->getBitWidth(); | |
| unsigned NewWidth = widenBitWidthToNextPow2(CurrentWidth); | |
| if (NewWidth != CurrentWidth) { | |
| // Replace the immediate value with the widened version | |
| MOP.setCImm(ConstantInt::get(CImmVal->getType()->getContext(), | |
| CImmVal->getValue().zextOrTrunc(NewWidth))); | |
| } | |
| } | |
| static void setInsertPtAfterDef(MachineIRBuilder &MIB, MachineInstr *Def) { | |
| MachineBasicBlock &MBB = *Def->getParent(); | |
| MachineBasicBlock::iterator DefIt = | |
| Def->getNextNode() ? Def->getNextNode()->getIterator() : MBB.end(); | |
| // Skip all the PHI and debug instructions. | |
| while (DefIt != MBB.end() && | |
| (DefIt->isPHI() || DefIt->isDebugOrPseudoInstr())) | |
| DefIt = std::next(DefIt); | |
| MIB.setInsertPt(MBB, DefIt); | |
| } | |
| namespace llvm { | |
| void updateRegType(Register Reg, Type *Ty, SPIRVTypeInst SpvType, | |
| SPIRVGlobalRegistry *GR, MachineIRBuilder &MIB, | |
| MachineRegisterInfo &MRI) { | |
| assert((Ty || SpvType) && "Either LLVM or SPIRV type is expected."); | |
| MachineInstr *Def = MRI.getVRegDef(Reg); | |
| setInsertPtAfterDef(MIB, Def); | |
| if (!SpvType) | |
| SpvType = GR->getOrCreateSPIRVType(Ty, MIB, | |
| SPIRV::AccessQualifier::ReadWrite, true); | |
| if (!MRI.getRegClassOrNull(Reg)) | |
| MRI.setRegClass(Reg, GR->getRegClass(SpvType)); | |
| if (!MRI.getType(Reg).isValid()) | |
| MRI.setType(Reg, GR->getRegType(SpvType)); | |
| GR->assignSPIRVTypeToVReg(SpvType, Reg, MIB.getMF()); | |
| } | |
| void processInstr(MachineInstr &MI, MachineIRBuilder &MIB, | |
| MachineRegisterInfo &MRI, SPIRVGlobalRegistry *GR, | |
| SPIRVTypeInst KnownResType) { | |
| MIB.setInsertPt(*MI.getParent(), MI.getIterator()); | |
| for (auto &Op : MI.operands()) { | |
| if (!Op.isReg() || Op.isDef()) | |
| continue; | |
| Register OpReg = Op.getReg(); | |
| SPIRVTypeInst SpvType = GR->getSPIRVTypeForVReg(OpReg); | |
| if (!SpvType && KnownResType) { | |
| SpvType = KnownResType; | |
| GR->assignSPIRVTypeToVReg(KnownResType, OpReg, *MI.getMF()); | |
| } | |
| assert(SpvType); | |
| if (!MRI.getRegClassOrNull(OpReg)) | |
| MRI.setRegClass(OpReg, GR->getRegClass(SpvType)); | |
| if (!MRI.getType(OpReg).isValid()) | |
| MRI.setType(OpReg, GR->getRegType(SpvType)); | |
| } | |
| } | |
| } // namespace llvm | |
| static void | |
| generateAssignInstrs(MachineFunction &MF, SPIRVGlobalRegistry *GR, | |
| MachineIRBuilder MIB, | |
| DenseMap<MachineInstr *, Type *> &TargetExtConstTypes) { | |
| // Get access to information about available extensions | |
| const SPIRVSubtarget *ST = | |
| static_cast<const SPIRVSubtarget *>(&MIB.getMF().getSubtarget()); | |
| MachineRegisterInfo &MRI = MF.getRegInfo(); | |
| SmallVector<MachineInstr *, 10> ToErase; | |
| DenseMap<MachineInstr *, Register> RegsAlreadyAddedToDT; | |
| bool IsExtendedInts = | |
| ST->canUseExtension( | |
| SPIRV::Extension::SPV_ALTERA_arbitrary_precision_integers) || | |
| ST->canUseExtension(SPIRV::Extension::SPV_KHR_bit_instructions) || | |
| ST->canUseExtension(SPIRV::Extension::SPV_INTEL_int4); | |
| if (!IsExtendedInts) { | |
| // Without arbitrary precision integer extensions, SPIR-V only supports | |
| // integer widths of 8, 16, 32, 64. Non-standard widths (e.g., i24, i40) | |
| // must be widened to the next power of two. | |
| // | |
| // G_TRUNC requires special handling because its semantics depend on the | |
| // original destination width. For example: | |
| // %dst:s24 = G_TRUNC %src:s64 | |
| // After widening s24 to s32, we cannot simply do: | |
| // %dst:s32 = G_TRUNC %src:s64 | |
| // because this would keep 32 bits instead of 24. Instead, we insert a | |
| // G_AND to mask the value to the original width: | |
| // %mask:s64 = G_CONSTANT 0xFFFFFF ; 24-bit mask | |
| // %masked:s64 = G_AND %src:s64, %mask | |
| // %dst:s32 = G_TRUNC %masked:s64 | |
| // If src and dst widen to the same size, G_TRUNC is replaced entirely: | |
| // %mask:s64 = G_CONSTANT 0xFFFFFFFFFF ; 40-bit mask | |
| // %dst:s64 = G_AND %src:s64, %mask | |
| SmallVector<MachineInstr *, 8> TruncToRemove; | |
| for (MachineBasicBlock &MBB : MF) { | |
| for (MachineInstr &MI : MBB) { | |
| unsigned MIOp = MI.getOpcode(); | |
| if (MIOp != TargetOpcode::G_TRUNC) | |
| continue; | |
| assert(MI.getNumOperands() == 2); | |
| assert(MI.getOperand(0).isReg()); | |
| assert(MI.getOperand(1).isReg()); | |
| Register DstReg = MI.getOperand(0).getReg(); | |
| Register SrcReg = MI.getOperand(1).getReg(); | |
| // TODO: handle vector types. | |
| if (!MRI.getType(DstReg).isScalar()) { | |
| assert(!MRI.getType(SrcReg).isScalar()); | |
| continue; | |
| } | |
| unsigned OriginalDstWidth = MRI.getType(DstReg).getScalarSizeInBits(); | |
| unsigned OriginalSrcWidth = MRI.getType(SrcReg).getScalarSizeInBits(); | |
| unsigned NewDstWidth = widenBitWidthToNextPow2(OriginalDstWidth); | |
| unsigned NewSrcWidth = widenBitWidthToNextPow2(OriginalSrcWidth); | |
| // No Dst width change means no truncation semantics change. | |
| if (OriginalDstWidth == NewDstWidth) | |
| continue; | |
| MRI.setType(SrcReg, LLT::scalar(NewSrcWidth)); | |
| MRI.setType(DstReg, LLT::scalar(NewDstWidth)); | |
| MIB.setInsertPt(MBB, MI.getIterator()); | |
| APInt Mask = APInt::getLowBitsSet(NewSrcWidth, OriginalDstWidth); | |
| auto MaskReg = MIB.buildConstant(LLT::scalar(NewSrcWidth), Mask); | |
| Register MaskedReg = | |
| MRI.createGenericVirtualRegister(LLT::scalar(NewSrcWidth)); | |
| MIB.buildAnd(MaskedReg, SrcReg, MaskReg); | |
| if (NewSrcWidth == NewDstWidth) { | |
| MRI.replaceRegWith(DstReg, MaskedReg); | |
| TruncToRemove.push_back(&MI); | |
| } else { | |
| MI.getOperand(1).setReg(MaskedReg); | |
| } | |
| } | |
| } | |
| for (MachineInstr *MI : TruncToRemove) | |
| MI->eraseFromParent(); | |
| } | |
| for (MachineBasicBlock *MBB : post_order(&MF)) { | |
| if (MBB->empty()) | |
| continue; | |
| bool ReachedBegin = false; | |
| for (auto MII = std::prev(MBB->end()), Begin = MBB->begin(); | |
| !ReachedBegin;) { | |
| MachineInstr &MI = *MII; | |
| unsigned MIOp = MI.getOpcode(); | |
| if (!IsExtendedInts) { | |
| // validate bit width of scalar registers and constant immediates | |
| for (auto &MOP : MI.operands()) { | |
| if (MOP.isReg()) | |
| widenScalarType(MOP.getReg(), MRI); | |
| else if (MOP.isCImm()) | |
| widenCImmType(MOP); | |
| } | |
| } | |
| if (isSpvIntrinsic(MI, Intrinsic::spv_assign_ptr_type)) { | |
| Register Reg = MI.getOperand(1).getReg(); | |
| MIB.setInsertPt(*MI.getParent(), MI.getIterator()); | |
| Type *ElementTy = getMDOperandAsType(MI.getOperand(2).getMetadata(), 0); | |
| SPIRVTypeInst AssignedPtrType = GR->getOrCreateSPIRVPointerType( | |
| ElementTy, MI, | |
| addressSpaceToStorageClass(MI.getOperand(3).getImm(), *ST)); | |
| MachineInstr *Def = MRI.getVRegDef(Reg); | |
| assert(Def && "Expecting an instruction that defines the register"); | |
| // G_GLOBAL_VALUE already has type info. | |
| if (Def->getOpcode() != TargetOpcode::G_GLOBAL_VALUE) | |
| updateRegType(Reg, nullptr, AssignedPtrType, GR, MIB, | |
| MF.getRegInfo()); | |
| ToErase.push_back(&MI); | |
| } else if (isSpvIntrinsic(MI, Intrinsic::spv_assign_type)) { | |
| Register Reg = MI.getOperand(1).getReg(); | |
| Type *Ty = getMDOperandAsType(MI.getOperand(2).getMetadata(), 0); | |
| MachineInstr *Def = MRI.getVRegDef(Reg); | |
| assert(Def && "Expecting an instruction that defines the register"); | |
| // G_GLOBAL_VALUE already has type info. | |
| if (Def->getOpcode() != TargetOpcode::G_GLOBAL_VALUE) | |
| updateRegType(Reg, Ty, nullptr, GR, MIB, MF.getRegInfo()); | |
| ToErase.push_back(&MI); | |
| } else if (MIOp == TargetOpcode::FAKE_USE && MI.getNumOperands() > 0) { | |
| MachineInstr *MdMI = MI.getPrevNode(); | |
| if (MdMI && isSpvIntrinsic(*MdMI, Intrinsic::spv_value_md)) { | |
| // It's an internal service info from before IRTranslator passes. | |
| MachineInstr *Def = getVRegDef(MRI, MI.getOperand(0).getReg()); | |
| for (unsigned I = 1, E = MI.getNumOperands(); I != E && Def; ++I) | |
| if (getVRegDef(MRI, MI.getOperand(I).getReg()) != Def) | |
| Def = nullptr; | |
| if (Def) { | |
| const MDNode *MD = MdMI->getOperand(1).getMetadata(); | |
| StringRef ValueName = | |
| cast<MDString>(MD->getOperand(1))->getString(); | |
| const MDNode *TypeMD = cast<MDNode>(MD->getOperand(0)); | |
| Type *ValueTy = getMDOperandAsType(TypeMD, 0); | |
| GR->addValueAttrs(Def, std::make_pair(ValueTy, ValueName.str())); | |
| } | |
| ToErase.push_back(MdMI); | |
| } | |
| ToErase.push_back(&MI); | |
| } else if (MIOp == TargetOpcode::G_CONSTANT || | |
| MIOp == TargetOpcode::G_FCONSTANT || | |
| MIOp == TargetOpcode::G_BUILD_VECTOR) { | |
| // %rc = G_CONSTANT ty Val | |
| // Ensure %rc has a valid SPIR-V type assigned in the Global Registry. | |
| Register Reg = MI.getOperand(0).getReg(); | |
| bool NeedAssignType = !GR->getSPIRVTypeForVReg(Reg); | |
| Type *Ty = nullptr; | |
| if (MIOp == TargetOpcode::G_CONSTANT) { | |
| auto TargetExtIt = TargetExtConstTypes.find(&MI); | |
| Ty = TargetExtIt == TargetExtConstTypes.end() | |
| ? MI.getOperand(1).getCImm()->getType() | |
| : TargetExtIt->second; | |
| const ConstantInt *OpCI = MI.getOperand(1).getCImm(); | |
| // TODO: we may wish to analyze here if OpCI is zero and LLT RegType = | |
| // MRI.getType(Reg); RegType.isPointer() is true, so that we observe | |
| // at this point not i64/i32 constant but null pointer in the | |
| // corresponding address space of RegType.getAddressSpace(). This may | |
| // help to successfully validate the case when a OpConstantComposite's | |
| // constituent has type that does not match Result Type of | |
| // OpConstantComposite (see, for example, | |
| // pointers/PtrCast-null-in-OpSpecConstantOp.ll). | |
| Register PrimaryReg = GR->find(OpCI, &MF); | |
| if (!PrimaryReg.isValid()) { | |
| GR->add(OpCI, &MI); | |
| } else if (PrimaryReg != Reg && | |
| MRI.getType(Reg) == MRI.getType(PrimaryReg)) { | |
| auto *RCReg = MRI.getRegClassOrNull(Reg); | |
| auto *RCPrimary = MRI.getRegClassOrNull(PrimaryReg); | |
| if (!RCReg || RCPrimary == RCReg) { | |
| RegsAlreadyAddedToDT[&MI] = PrimaryReg; | |
| ToErase.push_back(&MI); | |
| NeedAssignType = false; | |
| } | |
| } | |
| } else if (MIOp == TargetOpcode::G_FCONSTANT) { | |
| Ty = MI.getOperand(1).getFPImm()->getType(); | |
| } else { | |
| assert(MIOp == TargetOpcode::G_BUILD_VECTOR); | |
| Type *ElemTy = nullptr; | |
| MachineInstr *ElemMI = MRI.getVRegDef(MI.getOperand(1).getReg()); | |
| assert(ElemMI); | |
| if (ElemMI->getOpcode() == TargetOpcode::G_CONSTANT) { | |
| ElemTy = ElemMI->getOperand(1).getCImm()->getType(); | |
| } else if (ElemMI->getOpcode() == TargetOpcode::G_FCONSTANT) { | |
| ElemTy = ElemMI->getOperand(1).getFPImm()->getType(); | |
| } else { | |
| if (SPIRVTypeInst ElemSpvType = | |
| GR->getSPIRVTypeForVReg(MI.getOperand(1).getReg(), &MF)) | |
| ElemTy = const_cast<Type *>(GR->getTypeForSPIRVType(ElemSpvType)); | |
| } | |
| if (ElemTy) | |
| Ty = VectorType::get( | |
| ElemTy, MI.getNumExplicitOperands() - MI.getNumExplicitDefs(), | |
| false); | |
| else | |
| NeedAssignType = false; | |
| } | |
| if (NeedAssignType) | |
| updateRegType(Reg, Ty, nullptr, GR, MIB, MRI); | |
| } else if (MIOp == TargetOpcode::G_GLOBAL_VALUE) { | |
| propagateSPIRVType(&MI, GR, MRI, MIB); | |
| } | |
| if (MII == Begin) | |
| ReachedBegin = true; | |
| else | |
| --MII; | |
| } | |
| } | |
| for (MachineInstr *MI : ToErase) { | |
| auto It = RegsAlreadyAddedToDT.find(MI); | |
| if (It != RegsAlreadyAddedToDT.end()) | |
| MRI.replaceRegWith(MI->getOperand(0).getReg(), It->second); | |
| invalidateAndEraseMI(GR, MI); | |
| } | |
| // Address the case when IRTranslator introduces instructions with new | |
| // registers without associated SPIRV type. | |
| for (MachineBasicBlock &MBB : MF) { | |
| for (MachineInstr &MI : MBB) { | |
| switch (MI.getOpcode()) { | |
| case TargetOpcode::G_TRUNC: | |
| case TargetOpcode::G_ANYEXT: | |
| case TargetOpcode::G_SEXT: | |
| case TargetOpcode::G_ZEXT: | |
| case TargetOpcode::G_PTRTOINT: | |
| case TargetOpcode::COPY: | |
| case TargetOpcode::G_ADDRSPACE_CAST: | |
| propagateSPIRVType(&MI, GR, MRI, MIB); | |
| break; | |
| } | |
| } | |
| } | |
| } | |
| static void processInstrsWithTypeFolding(MachineFunction &MF, | |
| SPIRVGlobalRegistry *GR, | |
| MachineIRBuilder MIB) { | |
| MachineRegisterInfo &MRI = MF.getRegInfo(); | |
| for (MachineBasicBlock &MBB : MF) | |
| for (MachineInstr &MI : MBB) | |
| if (isTypeFoldingSupported(MI.getOpcode())) | |
| processInstr(MI, MIB, MRI, GR, nullptr); | |
| } | |
| static Register | |
| collectInlineAsmInstrOperands(MachineInstr *MI, | |
| SmallVector<unsigned, 4> *Ops = nullptr) { | |
| Register DefReg; | |
| unsigned StartOp = InlineAsm::MIOp_FirstOperand, | |
| AsmDescOp = InlineAsm::MIOp_FirstOperand; | |
| for (unsigned Idx = StartOp, MISz = MI->getNumOperands(); Idx != MISz; | |
| ++Idx) { | |
| const MachineOperand &MO = MI->getOperand(Idx); | |
| if (MO.isMetadata()) | |
| continue; | |
| if (Idx == AsmDescOp && MO.isImm()) { | |
| // compute the index of the next operand descriptor | |
| const InlineAsm::Flag F(MO.getImm()); | |
| AsmDescOp += 1 + F.getNumOperandRegisters(); | |
| continue; | |
| } | |
| if (MO.isReg() && MO.isDef()) { | |
| if (!Ops) | |
| return MO.getReg(); | |
| DefReg = MO.getReg(); | |
| } else if (Ops) { | |
| Ops->push_back(Idx); | |
| } | |
| } | |
| return DefReg; | |
| } | |
| static void | |
| insertInlineAsmProcess(MachineFunction &MF, SPIRVGlobalRegistry *GR, | |
| const SPIRVSubtarget &ST, MachineIRBuilder MIRBuilder, | |
| const SmallVector<MachineInstr *> &ToProcess) { | |
| MachineRegisterInfo &MRI = MF.getRegInfo(); | |
| Register AsmTargetReg; | |
| for (unsigned i = 0, Sz = ToProcess.size(); i + 1 < Sz; i += 2) { | |
| MachineInstr *I1 = ToProcess[i], *I2 = ToProcess[i + 1]; | |
| assert(isSpvIntrinsic(*I1, Intrinsic::spv_inline_asm) && I2->isInlineAsm()); | |
| MIRBuilder.setInsertPt(*I2->getParent(), *I2); | |
| if (!AsmTargetReg.isValid()) { | |
| // define vendor specific assembly target or dialect | |
| AsmTargetReg = MRI.createGenericVirtualRegister(LLT::scalar(32)); | |
| MRI.setRegClass(AsmTargetReg, &SPIRV::iIDRegClass); | |
| auto AsmTargetMIB = | |
| MIRBuilder.buildInstr(SPIRV::OpAsmTargetINTEL).addDef(AsmTargetReg); | |
| addStringImm(ST.getTargetTripleAsStr(), AsmTargetMIB); | |
| GR->add(AsmTargetMIB.getInstr(), AsmTargetMIB); | |
| } | |
| // create types | |
| const MDNode *IAMD = I1->getOperand(1).getMetadata(); | |
| FunctionType *FTy = cast<FunctionType>(getMDOperandAsType(IAMD, 0)); | |
| SmallVector<SPIRVTypeInst, 4> ArgTypes; | |
| for (const auto &ArgTy : FTy->params()) | |
| ArgTypes.push_back(GR->getOrCreateSPIRVType( | |
| ArgTy, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, true)); | |
| SPIRVTypeInst RetType = | |
| GR->getOrCreateSPIRVType(FTy->getReturnType(), MIRBuilder, | |
| SPIRV::AccessQualifier::ReadWrite, true); | |
| SPIRVTypeInst FuncType = GR->getOrCreateOpTypeFunctionWithArgs( | |
| FTy, RetType, ArgTypes, MIRBuilder); | |
| // define vendor specific assembly instructions string | |
| Register AsmReg = MRI.createGenericVirtualRegister(LLT::scalar(32)); | |
| MRI.setRegClass(AsmReg, &SPIRV::iIDRegClass); | |
| auto AsmMIB = MIRBuilder.buildInstr(SPIRV::OpAsmINTEL) | |
| .addDef(AsmReg) | |
| .addUse(GR->getSPIRVTypeID(RetType)) | |
| .addUse(GR->getSPIRVTypeID(FuncType)) | |
| .addUse(AsmTargetReg); | |
| // inline asm string: | |
| addStringImm(I2->getOperand(InlineAsm::MIOp_AsmString).getSymbolName(), | |
| AsmMIB); | |
| // inline asm constraint string: | |
| addStringImm(cast<MDString>(I1->getOperand(2).getMetadata()->getOperand(0)) | |
| ->getString(), | |
| AsmMIB); | |
| GR->add(AsmMIB.getInstr(), AsmMIB); | |
| // calls the inline assembly instruction | |
| unsigned ExtraInfo = I2->getOperand(InlineAsm::MIOp_ExtraInfo).getImm(); | |
| if (ExtraInfo & InlineAsm::Extra_HasSideEffects) | |
| MIRBuilder.buildInstr(SPIRV::OpDecorate) | |
| .addUse(AsmReg) | |
| .addImm(static_cast<uint32_t>(SPIRV::Decoration::SideEffectsINTEL)); | |
| Register DefReg = collectInlineAsmInstrOperands(I2); | |
| if (!DefReg.isValid()) { | |
| DefReg = MRI.createGenericVirtualRegister(LLT::scalar(32)); | |
| MRI.setRegClass(DefReg, &SPIRV::iIDRegClass); | |
| SPIRVTypeInst VoidType = GR->getOrCreateSPIRVType( | |
| Type::getVoidTy(MF.getFunction().getContext()), MIRBuilder, | |
| SPIRV::AccessQualifier::ReadWrite, true); | |
| GR->assignSPIRVTypeToVReg(VoidType, DefReg, MF); | |
| } | |
| auto AsmCall = MIRBuilder.buildInstr(SPIRV::OpAsmCallINTEL) | |
| .addDef(DefReg) | |
| .addUse(GR->getSPIRVTypeID(RetType)) | |
| .addUse(AsmReg); | |
| for (unsigned IntrIdx = 3; IntrIdx < I1->getNumOperands(); ++IntrIdx) | |
| AsmCall.addUse(I1->getOperand(IntrIdx).getReg()); | |
| // IRTranslator gets a bit confused when lowering inline ASM with outputs | |
| // and inserts a spurious COPY & TRUNC as registers are assumed to be i64; | |
| // we have to clean that up here to prevent erroneous trunc casts either on | |
| // a struct (for multiple outputs) or same width integers to get lowered | |
| // into SPIR-V | |
| if (MRI.hasOneUse(DefReg)) { | |
| MachineInstr &CopyMI = *MRI.use_instr_begin(DefReg); | |
| if (CopyMI.getOpcode() == TargetOpcode::COPY) { | |
| Register CopyDst = CopyMI.getOperand(0).getReg(); | |
| if (MRI.hasOneUse(CopyDst)) { | |
| MachineInstr &TruncMI = *MRI.use_instr_begin(CopyDst); | |
| if (TruncMI.getOpcode() == TargetOpcode::G_TRUNC) { | |
| MRI.setType(DefReg, GR->getRegType(RetType)); | |
| Register TruncReg = TruncMI.defs().begin()->getReg(); | |
| MRI.replaceRegWith(TruncReg, DefReg); | |
| invalidateAndEraseMI(GR, &TruncMI); | |
| invalidateAndEraseMI(GR, &CopyMI); | |
| } | |
| } | |
| } | |
| } | |
| } | |
| for (MachineInstr *MI : ToProcess) | |
| invalidateAndEraseMI(GR, MI); | |
| } | |
| static void insertInlineAsm(MachineFunction &MF, SPIRVGlobalRegistry *GR, | |
| const SPIRVSubtarget &ST, | |
| MachineIRBuilder MIRBuilder) { | |
| SmallVector<MachineInstr *> ToProcess; | |
| for (MachineBasicBlock &MBB : MF) { | |
| for (MachineInstr &MI : MBB) { | |
| if (isSpvIntrinsic(MI, Intrinsic::spv_inline_asm) || | |
| MI.getOpcode() == TargetOpcode::INLINEASM) | |
| ToProcess.push_back(&MI); | |
| } | |
| } | |
| if (ToProcess.size() == 0) | |
| return; | |
| if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_inline_assembly)) | |
| report_fatal_error("Inline assembly instructions require the " | |
| "following SPIR-V extension: SPV_INTEL_inline_assembly", | |
| false); | |
| insertInlineAsmProcess(MF, GR, ST, MIRBuilder, ToProcess); | |
| } | |
| static uint32_t convertFloatToSPIRVWord(float F) { | |
| union { | |
| float F; | |
| uint32_t Spir; | |
| } FPMaxError; | |
| FPMaxError.F = F; | |
| return FPMaxError.Spir; | |
| } | |
| static void insertSpirvDecorations(MachineFunction &MF, SPIRVGlobalRegistry *GR, | |
| MachineIRBuilder MIB) { | |
| const SPIRVSubtarget &ST = cast<SPIRVSubtarget>(MIB.getMF().getSubtarget()); | |
| SmallVector<MachineInstr *, 10> ToErase; | |
| for (MachineBasicBlock &MBB : MF) { | |
| for (MachineInstr &MI : MBB) { | |
| if (!isSpvIntrinsic(MI, Intrinsic::spv_assign_decoration) && | |
| !isSpvIntrinsic(MI, Intrinsic::spv_assign_aliasing_decoration) && | |
| !isSpvIntrinsic(MI, Intrinsic::spv_assign_fpmaxerror_decoration)) | |
| continue; | |
| MIB.setInsertPt(*MI.getParent(), MI.getNextNode()); | |
| if (isSpvIntrinsic(MI, Intrinsic::spv_assign_decoration)) { | |
| buildOpSpirvDecorations(MI.getOperand(1).getReg(), MIB, | |
| MI.getOperand(2).getMetadata(), ST); | |
| } else if (isSpvIntrinsic(MI, | |
| Intrinsic::spv_assign_fpmaxerror_decoration)) { | |
| ConstantFP *OpV = mdconst::dyn_extract<ConstantFP>( | |
| MI.getOperand(2).getMetadata()->getOperand(0)); | |
| uint32_t OpValue = | |
| convertFloatToSPIRVWord(OpV->getValueAPF().convertToFloat()); | |
| buildOpDecorate(MI.getOperand(1).getReg(), MIB, | |
| SPIRV::Decoration::FPMaxErrorDecorationINTEL, | |
| {OpValue}); | |
| } else { | |
| GR->buildMemAliasingOpDecorate(MI.getOperand(1).getReg(), MIB, | |
| MI.getOperand(2).getImm(), | |
| MI.getOperand(3).getMetadata()); | |
| } | |
| ToErase.push_back(&MI); | |
| } | |
| } | |
| for (MachineInstr *MI : ToErase) | |
| invalidateAndEraseMI(GR, MI); | |
| } | |
| // LLVM allows the switches to use registers as cases, while SPIR-V required | |
| // those to be immediate values. This function replaces such operands with the | |
| // equivalent immediate constant. | |
| static void processSwitchesConstants(MachineFunction &MF, | |
| SPIRVGlobalRegistry *GR, | |
| MachineIRBuilder MIB) { | |
| MachineRegisterInfo &MRI = MF.getRegInfo(); | |
| for (MachineBasicBlock &MBB : MF) { | |
| for (MachineInstr &MI : MBB) { | |
| if (!isSpvIntrinsic(MI, Intrinsic::spv_switch)) | |
| continue; | |
| SmallVector<MachineOperand, 8> NewOperands; | |
| NewOperands.push_back(MI.getOperand(0)); // Opcode | |
| NewOperands.push_back(MI.getOperand(1)); // Condition | |
| NewOperands.push_back(MI.getOperand(2)); // Default | |
| for (unsigned i = 3; i < MI.getNumOperands(); i += 2) { | |
| Register Reg = MI.getOperand(i).getReg(); | |
| MachineInstr *ConstInstr = getDefInstrMaybeConstant(Reg, &MRI); | |
| NewOperands.push_back( | |
| MachineOperand::CreateCImm(ConstInstr->getOperand(1).getCImm())); | |
| NewOperands.push_back(MI.getOperand(i + 1)); | |
| } | |
| assert(MI.getNumOperands() == NewOperands.size()); | |
| while (MI.getNumOperands() > 0) | |
| MI.removeOperand(0); | |
| for (auto &MO : NewOperands) | |
| MI.addOperand(MO); | |
| } | |
| } | |
| } | |
| // Some instructions are used during CodeGen but should never be emitted. | |
| // Cleaning up those. | |
| static void cleanupHelperInstructions(MachineFunction &MF, | |
| SPIRVGlobalRegistry *GR) { | |
| SmallVector<MachineInstr *, 8> ToEraseMI; | |
| for (MachineBasicBlock &MBB : MF) { | |
| for (MachineInstr &MI : MBB) { | |
| if (isSpvIntrinsic(MI, Intrinsic::spv_track_constant) || | |
| MI.getOpcode() == TargetOpcode::G_BRINDIRECT) | |
| ToEraseMI.push_back(&MI); | |
| } | |
| } | |
| for (MachineInstr *MI : ToEraseMI) | |
| invalidateAndEraseMI(GR, MI); | |
| } | |
| // Find all usages of G_BLOCK_ADDR in our intrinsics and replace those | |
| // operands/registers by the actual MBB it references. | |
| static void processBlockAddr(MachineFunction &MF, SPIRVGlobalRegistry *GR, | |
| MachineIRBuilder MIB) { | |
| // Gather the reverse-mapping BB -> MBB. | |
| DenseMap<const BasicBlock *, MachineBasicBlock *> BB2MBB; | |
| for (MachineBasicBlock &MBB : MF) | |
| BB2MBB[MBB.getBasicBlock()] = &MBB; | |
| // Gather instructions requiring patching. For now, only those can use | |
| // G_BLOCK_ADDR. | |
| SmallVector<MachineInstr *, 8> InstructionsToPatch; | |
| for (MachineBasicBlock &MBB : MF) { | |
| for (MachineInstr &MI : MBB) { | |
| if (isSpvIntrinsic(MI, Intrinsic::spv_switch) || | |
| isSpvIntrinsic(MI, Intrinsic::spv_loop_merge) || | |
| isSpvIntrinsic(MI, Intrinsic::spv_selection_merge)) | |
| InstructionsToPatch.push_back(&MI); | |
| } | |
| } | |
| // For each instruction to fix, we replace all the G_BLOCK_ADDR operands by | |
| // the actual MBB it references. Once those references have been updated, we | |
| // can cleanup remaining G_BLOCK_ADDR references. | |
| SmallPtrSet<MachineBasicBlock *, 8> ClearAddressTaken; | |
| SmallPtrSet<MachineInstr *, 8> ToEraseMI; | |
| MachineRegisterInfo &MRI = MF.getRegInfo(); | |
| for (MachineInstr *MI : InstructionsToPatch) { | |
| SmallVector<MachineOperand, 8> NewOps; | |
| for (unsigned i = 0; i < MI->getNumOperands(); ++i) { | |
| // The operand is not a register, keep as-is. | |
| if (!MI->getOperand(i).isReg()) { | |
| NewOps.push_back(MI->getOperand(i)); | |
| continue; | |
| } | |
| Register Reg = MI->getOperand(i).getReg(); | |
| MachineInstr *BuildMBB = MRI.getVRegDef(Reg); | |
| // The register is not the result of G_BLOCK_ADDR, keep as-is. | |
| if (!BuildMBB || BuildMBB->getOpcode() != TargetOpcode::G_BLOCK_ADDR) { | |
| NewOps.push_back(MI->getOperand(i)); | |
| continue; | |
| } | |
| assert(BuildMBB && BuildMBB->getOpcode() == TargetOpcode::G_BLOCK_ADDR && | |
| BuildMBB->getOperand(1).isBlockAddress() && | |
| BuildMBB->getOperand(1).getBlockAddress()); | |
| BasicBlock *BB = | |
| BuildMBB->getOperand(1).getBlockAddress()->getBasicBlock(); | |
| auto It = BB2MBB.find(BB); | |
| if (It == BB2MBB.end()) | |
| report_fatal_error("cannot find a machine basic block by a basic block " | |
| "in a switch statement"); | |
| MachineBasicBlock *ReferencedBlock = It->second; | |
| NewOps.push_back(MachineOperand::CreateMBB(ReferencedBlock)); | |
| ClearAddressTaken.insert(ReferencedBlock); | |
| ToEraseMI.insert(BuildMBB); | |
| } | |
| // Replace the operands. | |
| assert(MI->getNumOperands() == NewOps.size()); | |
| while (MI->getNumOperands() > 0) | |
| MI->removeOperand(0); | |
| for (auto &MO : NewOps) | |
| MI->addOperand(MO); | |
| if (MachineInstr *Next = MI->getNextNode()) { | |
| if (isSpvIntrinsic(*Next, Intrinsic::spv_track_constant)) { | |
| ToEraseMI.insert(Next); | |
| Next = MI->getNextNode(); | |
| } | |
| if (Next && Next->getOpcode() == TargetOpcode::G_BRINDIRECT) | |
| ToEraseMI.insert(Next); | |
| } | |
| } | |
| // BlockAddress operands were used to keep information between passes, | |
| // let's undo the "address taken" status to reflect that Succ doesn't | |
| // actually correspond to an IR-level basic block. | |
| for (MachineBasicBlock *Succ : ClearAddressTaken) | |
| Succ->setAddressTakenIRBlock(nullptr); | |
| // If we just delete G_BLOCK_ADDR instructions with BlockAddress operands, | |
| // this leaves their BasicBlock counterparts in a "address taken" status. This | |
| // would make AsmPrinter to generate a series of unneeded labels of a "Address | |
| // of block that was removed by CodeGen" kind. Let's first ensure that we | |
| // don't have a dangling BlockAddress constants by zapping the BlockAddress | |
| // nodes, and only after that proceed with erasing G_BLOCK_ADDR instructions. | |
| Constant *Replacement = | |
| ConstantInt::get(Type::getInt32Ty(MF.getFunction().getContext()), 1); | |
| for (MachineInstr *BlockAddrI : ToEraseMI) { | |
| if (BlockAddrI->getOpcode() == TargetOpcode::G_BLOCK_ADDR) { | |
| BlockAddress *BA = const_cast<BlockAddress *>( | |
| BlockAddrI->getOperand(1).getBlockAddress()); | |
| BA->replaceAllUsesWith( | |
| ConstantExpr::getIntToPtr(Replacement, BA->getType())); | |
| BA->destroyConstant(); | |
| } | |
| invalidateAndEraseMI(GR, BlockAddrI); | |
| } | |
| } | |
| static bool isImplicitFallthrough(MachineBasicBlock &MBB) { | |
| if (MBB.empty()) | |
| return MBB.getNextNode() != nullptr; | |
| // Branching SPIR-V intrinsics are not detected by this generic method. | |
| // Thus, we can only trust negative result. | |
| if (!MBB.canFallThrough()) | |
| return false; | |
| // Otherwise, we must manually check if we have a SPIR-V intrinsic which | |
| // prevent an implicit fallthrough. | |
| for (MachineBasicBlock::reverse_iterator It = MBB.rbegin(), E = MBB.rend(); | |
| It != E; ++It) { | |
| if (isSpvIntrinsic(*It, Intrinsic::spv_switch)) | |
| return false; | |
| } | |
| return true; | |
| } | |
| static void removeImplicitFallthroughs(MachineFunction &MF, | |
| MachineIRBuilder MIB) { | |
| // It is valid for MachineBasicBlocks to not finish with a branch instruction. | |
| // In such cases, they will simply fallthrough their immediate successor. | |
| for (MachineBasicBlock &MBB : MF) { | |
| if (!isImplicitFallthrough(MBB)) | |
| continue; | |
| assert(MBB.succ_size() == 1); | |
| MIB.setInsertPt(MBB, MBB.end()); | |
| MIB.buildBr(**MBB.successors().begin()); | |
| } | |
| } | |
| bool SPIRVPreLegalizer::runOnMachineFunction(MachineFunction &MF) { | |
| // Initialize the type registry. | |
| const SPIRVSubtarget &ST = MF.getSubtarget<SPIRVSubtarget>(); | |
| SPIRVGlobalRegistry *GR = ST.getSPIRVGlobalRegistry(); | |
| GR->setCurrentFunc(MF); | |
| MachineIRBuilder MIB(MF); | |
| // a registry of target extension constants | |
| DenseMap<MachineInstr *, Type *> TargetExtConstTypes; | |
| // to keep record of tracked constants | |
| addConstantsToTrack(MF, GR, ST, TargetExtConstTypes); | |
| foldConstantsIntoIntrinsics(MF, GR, MIB); | |
| insertBitcasts(MF, GR, MIB); | |
| generateAssignInstrs(MF, GR, MIB, TargetExtConstTypes); | |
| processSwitchesConstants(MF, GR, MIB); | |
| processBlockAddr(MF, GR, MIB); | |
| cleanupHelperInstructions(MF, GR); | |
| processInstrsWithTypeFolding(MF, GR, MIB); | |
| removeImplicitFallthroughs(MF, MIB); | |
| insertSpirvDecorations(MF, GR, MIB); | |
| insertInlineAsm(MF, GR, ST, MIB); | |
| lowerBitcasts(MF, GR, MIB); | |
| return true; | |
| } | |
| INITIALIZE_PASS(SPIRVPreLegalizer, DEBUG_TYPE, "SPIRV pre legalizer", false, | |
| false) | |
| char SPIRVPreLegalizer::ID = 0; | |
| FunctionPass *llvm::createSPIRVPreLegalizerPass() { | |
| return new SPIRVPreLegalizer(); | |
| } |