| //===- SIInsertHardClauses.cpp - Insert Hard Clauses ----------------------===// | |
| // | |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |
| // See https://llvm.org/LICENSE.txt for license information. | |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |
| // | |
| //===----------------------------------------------------------------------===// | |
| // | |
| /// \file | |
| /// Insert s_clause instructions to form hard clauses. | |
| /// | |
| /// Clausing load instructions can give cache coherency benefits. Before gfx10, | |
| /// the hardware automatically detected "soft clauses", which were sequences of | |
| /// memory instructions of the same type. In gfx10 this detection was removed, | |
| /// and the s_clause instruction was introduced to explicitly mark "hard | |
| /// clauses". | |
| /// | |
| /// It's the scheduler's job to form the clauses by putting similar memory | |
| /// instructions next to each other. Our job is just to insert an s_clause | |
| /// instruction to mark the start of each clause. | |
| /// | |
| /// Note that hard clauses are very similar to, but logically distinct from, the | |
| /// groups of instructions that have to be restartable when XNACK is enabled. | |
| /// The rules are slightly different in each case. For example an s_nop | |
| /// instruction breaks a restartable group, but can appear in the middle of a | |
| /// hard clause. (Before gfx10 there wasn't a distinction, and both were called | |
| /// "soft clauses" or just "clauses".) | |
| /// | |
| /// The SIFormMemoryClauses pass and GCNHazardRecognizer deal with restartable | |
| /// groups, not hard clauses. | |
| // | |
| //===----------------------------------------------------------------------===// | |
| #include "AMDGPU.h" | |
| #include "GCNSubtarget.h" | |
| #include "llvm/ADT/SmallVector.h" | |
| #include "llvm/CodeGen/MachineFunctionPass.h" | |
| #include "llvm/CodeGen/MachinePassManager.h" | |
| using namespace llvm; | |
| #define DEBUG_TYPE "si-insert-hard-clauses" | |
| static cl::opt<unsigned> | |
| HardClauseLengthLimit("amdgpu-hard-clause-length-limit", | |
| cl::desc("Maximum number of memory instructions to " | |
| "place in the same hard clause"), | |
| cl::Hidden); | |
| namespace { | |
| enum HardClauseType { | |
| // For GFX10 and GFX1250: | |
| // Texture, buffer, global or scratch memory instructions. | |
| HARDCLAUSE_VMEM, | |
| // Flat (not global or scratch) memory instructions. | |
| HARDCLAUSE_FLAT, | |
| // For GFX11: | |
| // Texture memory instructions. | |
| HARDCLAUSE_MIMG_LOAD, | |
| HARDCLAUSE_MIMG_STORE, | |
| HARDCLAUSE_MIMG_ATOMIC, | |
| HARDCLAUSE_MIMG_SAMPLE, | |
| // Buffer, global or scratch memory instructions. | |
| HARDCLAUSE_VMEM_LOAD, | |
| HARDCLAUSE_VMEM_STORE, | |
| HARDCLAUSE_VMEM_ATOMIC, | |
| // Flat (not global or scratch) memory instructions. | |
| HARDCLAUSE_FLAT_LOAD, | |
| HARDCLAUSE_FLAT_STORE, | |
| HARDCLAUSE_FLAT_ATOMIC, | |
| // BVH instructions. | |
| HARDCLAUSE_BVH, | |
| // Common: | |
| // Instructions that access LDS. | |
| HARDCLAUSE_LDS, | |
| // Scalar memory instructions. | |
| HARDCLAUSE_SMEM, | |
| // VALU instructions. | |
| HARDCLAUSE_VALU, | |
| LAST_REAL_HARDCLAUSE_TYPE = HARDCLAUSE_VALU, | |
| // Internal instructions, which are allowed in the middle of a hard clause, | |
| // except for s_waitcnt. | |
| HARDCLAUSE_INTERNAL, | |
| // Meta instructions that do not result in any ISA like KILL. | |
| HARDCLAUSE_IGNORE, | |
| // Instructions that are not allowed in a hard clause: SALU, export, branch, | |
| // message, GDS, s_waitcnt and anything else not mentioned above. | |
| HARDCLAUSE_ILLEGAL, | |
| }; | |
| class SIInsertHardClauses { | |
| public: | |
| const GCNSubtarget *ST = nullptr; | |
| HardClauseType getHardClauseType(const MachineInstr &MI) { | |
| if (MI.mayLoad() || (MI.mayStore() && ST->shouldClusterStores())) { | |
| if (ST->getGeneration() == AMDGPUSubtarget::GFX10 || | |
| ST->hasGFX1250Insts()) { | |
| if ((SIInstrInfo::isVMEM(MI) && !SIInstrInfo::isFLAT(MI)) || | |
| SIInstrInfo::isSegmentSpecificFLAT(MI)) { | |
| if (ST->hasNSAClauseBug()) { | |
| const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(MI.getOpcode()); | |
| if (Info && Info->MIMGEncoding == AMDGPU::MIMGEncGfx10NSA) | |
| return HARDCLAUSE_ILLEGAL; | |
| } | |
| return HARDCLAUSE_VMEM; | |
| } | |
| if (SIInstrInfo::isFLAT(MI)) | |
| return HARDCLAUSE_FLAT; | |
| } else { | |
| if (SIInstrInfo::isMIMG(MI)) { | |
| const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(MI.getOpcode()); | |
| const AMDGPU::MIMGBaseOpcodeInfo *BaseInfo = | |
| AMDGPU::getMIMGBaseOpcodeInfo(Info->BaseOpcode); | |
| if (BaseInfo->BVH) | |
| return HARDCLAUSE_BVH; | |
| if (BaseInfo->Sampler || BaseInfo->MSAA) | |
| return HARDCLAUSE_MIMG_SAMPLE; | |
| return MI.mayLoad() ? MI.mayStore() ? HARDCLAUSE_MIMG_ATOMIC | |
| : HARDCLAUSE_MIMG_LOAD | |
| : HARDCLAUSE_MIMG_STORE; | |
| } | |
| if ((SIInstrInfo::isVMEM(MI) && !SIInstrInfo::isFLAT(MI)) || | |
| SIInstrInfo::isSegmentSpecificFLAT(MI)) { | |
| return MI.mayLoad() ? MI.mayStore() ? HARDCLAUSE_VMEM_ATOMIC | |
| : HARDCLAUSE_VMEM_LOAD | |
| : HARDCLAUSE_VMEM_STORE; | |
| } | |
| if (SIInstrInfo::isFLAT(MI)) { | |
| return MI.mayLoad() ? MI.mayStore() ? HARDCLAUSE_FLAT_ATOMIC | |
| : HARDCLAUSE_FLAT_LOAD | |
| : HARDCLAUSE_FLAT_STORE; | |
| } | |
| } | |
| // TODO: LDS | |
| if (SIInstrInfo::isSMRD(MI)) | |
| return HARDCLAUSE_SMEM; | |
| } | |
| // Don't form VALU clauses. It's not clear what benefit they give, if any. | |
| // In practice s_nop is the only internal instruction we're likely to see. | |
| // It's safe to treat the rest as illegal. | |
| if (MI.getOpcode() == AMDGPU::S_NOP) | |
| return HARDCLAUSE_INTERNAL; | |
| if (MI.isMetaInstruction()) | |
| return HARDCLAUSE_IGNORE; | |
| return HARDCLAUSE_ILLEGAL; | |
| } | |
| // Track information about a clause as we discover it. | |
| struct ClauseInfo { | |
| // The type of all (non-internal) instructions in the clause. | |
| HardClauseType Type = HARDCLAUSE_ILLEGAL; | |
| // The first (necessarily non-internal) instruction in the clause. | |
| MachineInstr *First = nullptr; | |
| // The last non-internal instruction in the clause. | |
| MachineInstr *Last = nullptr; | |
| // The length of the clause including any internal instructions in the | |
| // middle (but not at the end) of the clause. | |
| unsigned Length = 0; | |
| // Internal instructions at the and of a clause should not be included in | |
| // the clause. Count them in TrailingInternalLength until a new memory | |
| // instruction is added. | |
| unsigned TrailingInternalLength = 0; | |
| // The base operands of *Last. | |
| SmallVector<const MachineOperand *, 4> BaseOps; | |
| }; | |
| bool emitClause(const ClauseInfo &CI, const SIInstrInfo *SII) { | |
| if (CI.First == CI.Last) | |
| return false; | |
| assert(CI.Length <= ST->maxHardClauseLength() && | |
| "Hard clause is too long!"); | |
| auto &MBB = *CI.First->getParent(); | |
| auto ClauseMI = | |
| BuildMI(MBB, *CI.First, DebugLoc(), SII->get(AMDGPU::S_CLAUSE)) | |
| .addImm(CI.Length - 1); | |
| finalizeBundle(MBB, ClauseMI->getIterator(), | |
| std::next(CI.Last->getIterator())); | |
| return true; | |
| } | |
| // \return if scopes are different on gfx1250 and disallowed to be claused. | |
| bool isIncompatibleScope(const MachineInstr &MI1, const MachineInstr &MI2, | |
| const SIInstrInfo *SII) const { | |
| assert(ST->getGeneration() == AMDGPUSubtarget::GFX12 && | |
| ST->hasGFX1250Insts()); | |
| int CPol1 = 0, CPol2 = 0; | |
| if (const MachineOperand *Op = | |
| SII->getNamedOperand(MI1, AMDGPU::OpName::cpol)) { | |
| CPol1 = Op->getImm() & AMDGPU::CPol::SCOPE; | |
| } | |
| if (const MachineOperand *Op = | |
| SII->getNamedOperand(MI2, AMDGPU::OpName::cpol)) { | |
| CPol2 = Op->getImm() & AMDGPU::CPol::SCOPE; | |
| } | |
| return CPol1 != CPol2; | |
| } | |
| bool run(MachineFunction &MF) { | |
| ST = &MF.getSubtarget<GCNSubtarget>(); | |
| if (!ST->hasHardClauses()) | |
| return false; | |
| unsigned MaxClauseLength = MF.getFunction().getFnAttributeAsParsedInteger( | |
| "amdgpu-hard-clause-length-limit", 255); | |
| if (HardClauseLengthLimit.getNumOccurrences()) | |
| MaxClauseLength = HardClauseLengthLimit; | |
| MaxClauseLength = std::min(MaxClauseLength, ST->maxHardClauseLength()); | |
| if (MaxClauseLength <= 1) | |
| return false; | |
| const SIInstrInfo *SII = ST->getInstrInfo(); | |
| bool Changed = false; | |
| for (auto &MBB : MF) { | |
| ClauseInfo CI; | |
| unsigned ExistingClauseRemaining = 0; | |
| for (auto &MI : MBB) { | |
| HardClauseType Type; | |
| if (ExistingClauseRemaining) { | |
| if (!MI.isMetaInstruction()) | |
| ExistingClauseRemaining--; | |
| Type = HARDCLAUSE_ILLEGAL; | |
| } else if (MI.getOpcode() == AMDGPU::S_CLAUSE) { | |
| // Respect existing explicit clauses. Re-clausing instructions that | |
| // are already covered by an S_CLAUSE can create nested clauses. | |
| ExistingClauseRemaining = (MI.getOperand(0).getImm() & 63) + 1; | |
| Type = HARDCLAUSE_ILLEGAL; | |
| } else { | |
| Type = getHardClauseType(MI); | |
| } | |
| int64_t Dummy1; | |
| bool Dummy2; | |
| LocationSize Dummy3 = LocationSize::precise(0); | |
| SmallVector<const MachineOperand *, 4> BaseOps; | |
| if (Type <= LAST_REAL_HARDCLAUSE_TYPE) { | |
| if (!SII->getMemOperandsWithOffsetWidth(MI, BaseOps, Dummy1, Dummy2, | |
| Dummy3)) { | |
| // We failed to get the base operands, so we'll never clause this | |
| // instruction with any other, so pretend it's illegal. | |
| Type = HARDCLAUSE_ILLEGAL; | |
| } | |
| } | |
| if (CI.Length == MaxClauseLength || | |
| (CI.Length && Type != HARDCLAUSE_INTERNAL && | |
| Type != HARDCLAUSE_IGNORE && | |
| (Type != CI.Type || | |
| // shouldClusterMemOps() would also apply the pre-RA memory | |
| // cluster budget, which must not limit clauses formed after | |
| // register allocation. | |
| !SII->memOpsHaveSameBase(CI.BaseOps, BaseOps))) || | |
| (CI.Length && ST->hasGFX1250_STRICT() && | |
| isIncompatibleScope(MI, *CI.Last, SII))) { | |
| // Finish the current clause. | |
| Changed |= emitClause(CI, SII); | |
| CI = ClauseInfo(); | |
| } | |
| if (CI.Length) { | |
| // Extend the current clause. | |
| if (Type != HARDCLAUSE_IGNORE) { | |
| if (Type == HARDCLAUSE_INTERNAL) { | |
| ++CI.TrailingInternalLength; | |
| } else { | |
| ++CI.Length; | |
| CI.Length += CI.TrailingInternalLength; | |
| CI.TrailingInternalLength = 0; | |
| CI.Last = &MI; | |
| CI.BaseOps = std::move(BaseOps); | |
| } | |
| } | |
| } else if (Type <= LAST_REAL_HARDCLAUSE_TYPE) { | |
| // Start a new clause. | |
| CI = ClauseInfo{Type, &MI, &MI, 1, 0, std::move(BaseOps)}; | |
| } | |
| } | |
| // Finish the last clause in the basic block if any. | |
| if (CI.Length) | |
| Changed |= emitClause(CI, SII); | |
| } | |
| return Changed; | |
| } | |
| }; | |
| class SIInsertHardClausesLegacy : public MachineFunctionPass { | |
| public: | |
| static char ID; | |
| SIInsertHardClausesLegacy() : MachineFunctionPass(ID) {} | |
| bool runOnMachineFunction(MachineFunction &MF) override { | |
| if (skipFunction(MF.getFunction())) | |
| return false; | |
| return SIInsertHardClauses().run(MF); | |
| } | |
| void getAnalysisUsage(AnalysisUsage &AU) const override { | |
| AU.setPreservesCFG(); | |
| MachineFunctionPass::getAnalysisUsage(AU); | |
| } | |
| }; | |
| } // namespace | |
| PreservedAnalyses | |
| llvm::SIInsertHardClausesPass::run(MachineFunction &MF, | |
| MachineFunctionAnalysisManager &MFAM) { | |
| if (!SIInsertHardClauses().run(MF)) | |
| return PreservedAnalyses::all(); | |
| auto PA = getMachineFunctionPassPreservedAnalyses(); | |
| PA.preserveSet<CFGAnalyses>(); | |
| return PA; | |
| } | |
| char SIInsertHardClausesLegacy::ID = 0; | |
| char &llvm::SIInsertHardClausesID = SIInsertHardClausesLegacy::ID; | |
| INITIALIZE_PASS(SIInsertHardClausesLegacy, DEBUG_TYPE, "SI Insert Hard Clauses", | |
| false, false) |