llvm/lib/Target/AMDGPU/SIFormMemoryClauses.cpp - llvm-project - Git at Google

 //===-- SIFormMemoryClauses.cpp -------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 /// \file This pass extends the live ranges of registers used as pointers in
 /// sequences of adjacent SMEM and VMEM instructions if XNACK is enabled. A
 /// load that would overwrite a pointer would require breaking the soft clause.
 /// Artificially extend the live ranges of the pointer operands by adding
 /// implicit-def early-clobber operands throughout the soft clause.
 ///
 //===----------------------------------------------------------------------===//

 #include "AMDGPU.h"
 #include "GCNRegPressure.h"
 #include "SIMachineFunctionInfo.h"
 #include "llvm/InitializePasses.h"

 using namespace llvm;

 #define DEBUG_TYPE "si-form-memory-clauses"

 // Clauses longer then 15 instructions would overflow one of the counters
 // and stall. They can stall even earlier if there are outstanding counters.
 static cl::opt<unsigned>
 MaxClause("amdgpu-max-memory-clause", cl::Hidden, cl::init(15),
           cl::desc("Maximum length of a memory clause, instructions"));

 namespace {

 class SIFormMemoryClauses : public MachineFunctionPass {
   typedef DenseMap<unsigned, std::pair<unsigned, LaneBitmask>> RegUse;

 public:
   static char ID;

 public:
   SIFormMemoryClauses() : MachineFunctionPass(ID) {
     initializeSIFormMemoryClausesPass(*PassRegistry::getPassRegistry());
   }

   bool runOnMachineFunction(MachineFunction &MF) override;

   StringRef getPassName() const override {
     return "SI Form memory clauses";
   }

   void getAnalysisUsage(AnalysisUsage &AU) const override {
     AU.addRequired<LiveIntervals>();
     AU.setPreservesAll();
     MachineFunctionPass::getAnalysisUsage(AU);
   }

   MachineFunctionProperties getClearedProperties() const override {
     return MachineFunctionProperties().set(
         MachineFunctionProperties::Property::IsSSA);
   }

 private:
   bool canBundle(const MachineInstr &MI, const RegUse &Defs,
                  const RegUse &Uses) const;
   bool checkPressure(const MachineInstr &MI, GCNDownwardRPTracker &RPT);
   void collectRegUses(const MachineInstr &MI, RegUse &Defs, RegUse &Uses) const;
   bool processRegUses(const MachineInstr &MI, RegUse &Defs, RegUse &Uses,
                       GCNDownwardRPTracker &RPT);

   const GCNSubtarget *ST;
   const SIRegisterInfo *TRI;
   const MachineRegisterInfo *MRI;
   SIMachineFunctionInfo *MFI;

   unsigned LastRecordedOccupancy;
   unsigned MaxVGPRs;
   unsigned MaxSGPRs;
 };

 } // End anonymous namespace.

 INITIALIZE_PASS_BEGIN(SIFormMemoryClauses, DEBUG_TYPE,
                       "SI Form memory clauses", false, false)
 INITIALIZE_PASS_DEPENDENCY(LiveIntervals)
 INITIALIZE_PASS_END(SIFormMemoryClauses, DEBUG_TYPE,
                     "SI Form memory clauses", false, false)


 char SIFormMemoryClauses::ID = 0;

 char &llvm::SIFormMemoryClausesID = SIFormMemoryClauses::ID;

 FunctionPass *llvm::createSIFormMemoryClausesPass() {
   return new SIFormMemoryClauses();
 }

 static bool isVMEMClauseInst(const MachineInstr &MI) {
   return SIInstrInfo::isFLAT(MI) || SIInstrInfo::isVMEM(MI);
 }

 static bool isSMEMClauseInst(const MachineInstr &MI) {
   return SIInstrInfo::isSMRD(MI);
 }

 // There no sense to create store clauses, they do not define anything,
 // thus there is nothing to set early-clobber.
 static bool isValidClauseInst(const MachineInstr &MI, bool IsVMEMClause) {
   assert(!MI.isDebugInstr() && "debug instructions should not reach here");
   if (MI.isBundled())
     return false;
   if (!MI.mayLoad() || MI.mayStore())
     return false;
   if (SIInstrInfo::isAtomic(MI))
     return false;
   if (IsVMEMClause && !isVMEMClauseInst(MI))
     return false;
   if (!IsVMEMClause && !isSMEMClauseInst(MI))
     return false;
   // If this is a load instruction where the result has been coalesced with an operand, then we cannot clause it.
   for (const MachineOperand &ResMO : MI.defs()) {
     Register ResReg = ResMO.getReg();
     for (const MachineOperand &MO : MI.uses()) {
       if (!MO.isReg() || MO.isDef())
         continue;
       if (MO.getReg() == ResReg)
         return false;
     }
     break; // Only check the first def.
   }
   return true;
 }

 static unsigned getMopState(const MachineOperand &MO) {
   unsigned S = 0;
   if (MO.isImplicit())
     S |= RegState::Implicit;
   if (MO.isDead())
     S |= RegState::Dead;
   if (MO.isUndef())
     S |= RegState::Undef;
   if (MO.isKill())
     S |= RegState::Kill;
   if (MO.isEarlyClobber())
     S |= RegState::EarlyClobber;
   if (MO.getReg().isPhysical() && MO.isRenamable())
     S |= RegState::Renamable;
   return S;
 }

 // Returns false if there is a use of a def already in the map.
 // In this case we must break the clause.
 bool SIFormMemoryClauses::canBundle(const MachineInstr &MI, const RegUse &Defs,
                                     const RegUse &Uses) const {
   // Check interference with defs.
   for (const MachineOperand &MO : MI.operands()) {
     // TODO: Prologue/Epilogue Insertion pass does not process bundled
     //       instructions.
     if (MO.isFI())
       return false;

     if (!MO.isReg())
       continue;

     Register Reg = MO.getReg();

     // If it is tied we will need to write same register as we read.
     if (MO.isTied())
       return false;

     const RegUse &Map = MO.isDef() ? Uses : Defs;
     auto Conflict = Map.find(Reg);
     if (Conflict == Map.end())
       continue;

     if (Reg.isPhysical())
       return false;

     LaneBitmask Mask = TRI->getSubRegIndexLaneMask(MO.getSubReg());
     if ((Conflict->second.second & Mask).any())
       return false;
   }

   return true;
 }

 // Since all defs in the clause are early clobber we can run out of registers.
 // Function returns false if pressure would hit the limit if instruction is
 // bundled into a memory clause.
 bool SIFormMemoryClauses::checkPressure(const MachineInstr &MI,
                                         GCNDownwardRPTracker &RPT) {
   // NB: skip advanceBeforeNext() call. Since all defs will be marked
   // early-clobber they will all stay alive at least to the end of the
   // clause. Therefor we should not decrease pressure even if load
   // pointer becomes dead and could otherwise be reused for destination.
   RPT.advanceToNext();
   GCNRegPressure MaxPressure = RPT.moveMaxPressure();
   unsigned Occupancy = MaxPressure.getOccupancy(*ST);

   // Don't push over half the register budget. We don't want to introduce
   // spilling just to form a soft clause.
   //
   // FIXME: This pressure check is fundamentally broken. First, this is checking
   // the global pressure, not the pressure at this specific point in the
   // program. Second, it's not accounting for the increased liveness of the use
   // operands due to the early clobber we will introduce. Third, the pressure
   // tracking does not account for the alignment requirements for SGPRs, or the
   // fragmentation of registers the allocator will need to satisfy.
   if (Occupancy >= MFI->getMinAllowedOccupancy() &&
       MaxPressure.getVGPRNum(ST->hasGFX90AInsts()) <= MaxVGPRs / 2 &&
       MaxPressure.getSGPRNum() <= MaxSGPRs / 2) {
     LastRecordedOccupancy = Occupancy;
     return true;
   }
   return false;
 }

 // Collect register defs and uses along with their lane masks and states.
 void SIFormMemoryClauses::collectRegUses(const MachineInstr &MI,
                                          RegUse &Defs, RegUse &Uses) const {
   for (const MachineOperand &MO : MI.operands()) {
     if (!MO.isReg())
       continue;
     Register Reg = MO.getReg();
     if (!Reg)
       continue;

     LaneBitmask Mask = Reg.isVirtual()
                            ? TRI->getSubRegIndexLaneMask(MO.getSubReg())
                            : LaneBitmask::getAll();
     RegUse &Map = MO.isDef() ? Defs : Uses;

     auto Loc = Map.find(Reg);
     unsigned State = getMopState(MO);
     if (Loc == Map.end()) {
       Map[Reg] = std::make_pair(State, Mask);
     } else {
       Loc->second.first |= State;
       Loc->second.second |= Mask;
     }
   }
 }

 // Check register def/use conflicts, occupancy limits and collect def/use maps.
 // Return true if instruction can be bundled with previous. It it cannot
 // def/use maps are not updated.
 bool SIFormMemoryClauses::processRegUses(const MachineInstr &MI,
                                          RegUse &Defs, RegUse &Uses,
                                          GCNDownwardRPTracker &RPT) {
   if (!canBundle(MI, Defs, Uses))
     return false;

   if (!checkPressure(MI, RPT))
     return false;

   collectRegUses(MI, Defs, Uses);
   return true;
 }

 bool SIFormMemoryClauses::runOnMachineFunction(MachineFunction &MF) {
   if (skipFunction(MF.getFunction()))
     return false;

   ST = &MF.getSubtarget<GCNSubtarget>();
   if (!ST->isXNACKEnabled())
     return false;

   const SIInstrInfo *TII = ST->getInstrInfo();
   TRI = ST->getRegisterInfo();
   MRI = &MF.getRegInfo();
   MFI = MF.getInfo<SIMachineFunctionInfo>();
   LiveIntervals *LIS = &getAnalysis<LiveIntervals>();
   SlotIndexes *Ind = LIS->getSlotIndexes();
   bool Changed = false;

   MaxVGPRs = TRI->getAllocatableSet(MF, &AMDGPU::VGPR_32RegClass).count();
   MaxSGPRs = TRI->getAllocatableSet(MF, &AMDGPU::SGPR_32RegClass).count();
   unsigned FuncMaxClause = AMDGPU::getIntegerAttribute(
       MF.getFunction(), "amdgpu-max-memory-clause", MaxClause);

   for (MachineBasicBlock &MBB : MF) {
     GCNDownwardRPTracker RPT(*LIS);
     MachineBasicBlock::instr_iterator Next;
     for (auto I = MBB.instr_begin(), E = MBB.instr_end(); I != E; I = Next) {
       MachineInstr &MI = *I;
       Next = std::next(I);

       if (MI.isMetaInstruction())
         continue;

       bool IsVMEM = isVMEMClauseInst(MI);

       if (!isValidClauseInst(MI, IsVMEM))
         continue;

       if (!RPT.getNext().isValid())
         RPT.reset(MI);
       else { // Advance the state to the current MI.
         RPT.advance(MachineBasicBlock::const_iterator(MI));
         RPT.advanceBeforeNext();
       }

       const GCNRPTracker::LiveRegSet LiveRegsCopy(RPT.getLiveRegs());
       RegUse Defs, Uses;
       if (!processRegUses(MI, Defs, Uses, RPT)) {
         RPT.reset(MI, &LiveRegsCopy);
         continue;
       }

       MachineBasicBlock::iterator LastClauseInst = Next;
       unsigned Length = 1;
       for ( ; Next != E && Length < FuncMaxClause; ++Next) {
         // Debug instructions should not change the kill insertion.
         if (Next->isMetaInstruction())
           continue;

         if (!isValidClauseInst(*Next, IsVMEM))
           break;

         // A load from pointer which was loaded inside the same bundle is an
         // impossible clause because we will need to write and read the same
         // register inside. In this case processRegUses will return false.
         if (!processRegUses(*Next, Defs, Uses, RPT))
           break;

         LastClauseInst = Next;
         ++Length;
       }
       if (Length < 2) {
         RPT.reset(MI, &LiveRegsCopy);
         continue;
       }

       Changed = true;
       MFI->limitOccupancy(LastRecordedOccupancy);

       assert(!LastClauseInst->isMetaInstruction());

       SlotIndex ClauseLiveInIdx = LIS->getInstructionIndex(MI);
       SlotIndex ClauseLiveOutIdx =
           LIS->getInstructionIndex(*LastClauseInst).getNextIndex();

       // Track the last inserted kill.
       MachineInstrBuilder Kill;

       // Insert one kill per register, with operands covering all necessary
       // subregisters.
       for (auto &&R : Uses) {
         Register Reg = R.first;
         if (Reg.isPhysical())
           continue;

         // Collect the register operands we should extend the live ranges of.
         SmallVector<std::tuple<unsigned, unsigned>> KillOps;
         const LiveInterval &LI = LIS->getInterval(R.first);

         if (!LI.hasSubRanges()) {
           if (!LI.liveAt(ClauseLiveOutIdx)) {
             KillOps.emplace_back(R.second.first | RegState::Kill,
                                  AMDGPU::NoSubRegister);
           }
         } else {
           LaneBitmask KilledMask;
           for (const LiveInterval::SubRange &SR : LI.subranges()) {
             if (SR.liveAt(ClauseLiveInIdx) && !SR.liveAt(ClauseLiveOutIdx))
               KilledMask |= SR.LaneMask;
           }

           if (KilledMask.none())
             continue;

           SmallVector<unsigned> KilledIndexes;
           bool Success = TRI->getCoveringSubRegIndexes(
               *MRI, MRI->getRegClass(Reg), KilledMask, KilledIndexes);
           (void)Success;
           assert(Success && "Failed to find subregister mask to cover lanes");
           for (unsigned SubReg : KilledIndexes) {
             KillOps.emplace_back(R.second.first | RegState::Kill, SubReg);
           }
         }

         if (KillOps.empty())
           continue;

         // We only want to extend the live ranges of used registers. If they
         // already have existing uses beyond the bundle, we don't need the kill.
         //
         // It's possible all of the use registers were already live past the
         // bundle.
         Kill = BuildMI(*MI.getParent(), std::next(LastClauseInst),
                        DebugLoc(), TII->get(AMDGPU::KILL));
         for (auto &Op : KillOps)
           Kill.addUse(Reg, std::get<0>(Op), std::get<1>(Op));
         Ind->insertMachineInstrInMaps(*Kill);
       }

       if (!Kill) {
         RPT.reset(MI, &LiveRegsCopy);
         continue;
       }

       // Restore the state after processing the end of the bundle.
       RPT.reset(*Kill, &LiveRegsCopy);

       for (auto &&R : Defs) {
         Register Reg = R.first;
         Uses.erase(Reg);
         if (Reg.isPhysical())
           continue;
         LIS->removeInterval(Reg);
         LIS->createAndComputeVirtRegInterval(Reg);
       }

       for (auto &&R : Uses) {
         Register Reg = R.first;
         if (Reg.isPhysical())
           continue;
         LIS->removeInterval(Reg);
         LIS->createAndComputeVirtRegInterval(Reg);
       }
     }
   }

   return Changed;
 }
	//===-- SIFormMemoryClauses.cpp -------------------------------------------===//
	//
	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	// See https://llvm.org/LICENSE.txt for license information.
	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	//
	//===----------------------------------------------------------------------===//
	//
	/// \file This pass extends the live ranges of registers used as pointers in
	/// sequences of adjacent SMEM and VMEM instructions if XNACK is enabled. A
	/// load that would overwrite a pointer would require breaking the soft clause.
	/// Artificially extend the live ranges of the pointer operands by adding
	/// implicit-def early-clobber operands throughout the soft clause.
	///
	//===----------------------------------------------------------------------===//

	#include "AMDGPU.h"
	#include "GCNRegPressure.h"
	#include "SIMachineFunctionInfo.h"
	#include "llvm/InitializePasses.h"

	using namespace llvm;

	#define DEBUG_TYPE "si-form-memory-clauses"

	// Clauses longer then 15 instructions would overflow one of the counters
	// and stall. They can stall even earlier if there are outstanding counters.
	static cl::opt<unsigned>
	MaxClause("amdgpu-max-memory-clause", cl::Hidden, cl::init(15),
	cl::desc("Maximum length of a memory clause, instructions"));

	namespace {

	class SIFormMemoryClauses : public MachineFunctionPass {
	typedef DenseMap<unsigned, std::pair<unsigned, LaneBitmask>> RegUse;

	public:
	static char ID;

	public:
	SIFormMemoryClauses() : MachineFunctionPass(ID) {
	initializeSIFormMemoryClausesPass(*PassRegistry::getPassRegistry());
	}

	bool runOnMachineFunction(MachineFunction &MF) override;

	StringRef getPassName() const override {
	return "SI Form memory clauses";
	}

	void getAnalysisUsage(AnalysisUsage &AU) const override {
	AU.addRequired<LiveIntervals>();
	AU.setPreservesAll();
	MachineFunctionPass::getAnalysisUsage(AU);
	}

	MachineFunctionProperties getClearedProperties() const override {
	return MachineFunctionProperties().set(
	MachineFunctionProperties::Property::IsSSA);
	}

	private:
	bool canBundle(const MachineInstr &MI, const RegUse &Defs,
	const RegUse &Uses) const;
	bool checkPressure(const MachineInstr &MI, GCNDownwardRPTracker &RPT);
	void collectRegUses(const MachineInstr &MI, RegUse &Defs, RegUse &Uses) const;
	bool processRegUses(const MachineInstr &MI, RegUse &Defs, RegUse &Uses,
	GCNDownwardRPTracker &RPT);

	const GCNSubtarget *ST;
	const SIRegisterInfo *TRI;
	const MachineRegisterInfo *MRI;
	SIMachineFunctionInfo *MFI;

	unsigned LastRecordedOccupancy;
	unsigned MaxVGPRs;
	unsigned MaxSGPRs;
	};

	} // End anonymous namespace.

	INITIALIZE_PASS_BEGIN(SIFormMemoryClauses, DEBUG_TYPE,
	"SI Form memory clauses", false, false)
	INITIALIZE_PASS_DEPENDENCY(LiveIntervals)
	INITIALIZE_PASS_END(SIFormMemoryClauses, DEBUG_TYPE,
	"SI Form memory clauses", false, false)


	char SIFormMemoryClauses::ID = 0;

	char &llvm::SIFormMemoryClausesID = SIFormMemoryClauses::ID;

	FunctionPass *llvm::createSIFormMemoryClausesPass() {
	return new SIFormMemoryClauses();
	}

	static bool isVMEMClauseInst(const MachineInstr &MI) {
	return SIInstrInfo::isFLAT(MI) \|\| SIInstrInfo::isVMEM(MI);
	}

	static bool isSMEMClauseInst(const MachineInstr &MI) {
	return SIInstrInfo::isSMRD(MI);
	}

	// There no sense to create store clauses, they do not define anything,
	// thus there is nothing to set early-clobber.
	static bool isValidClauseInst(const MachineInstr &MI, bool IsVMEMClause) {
	assert(!MI.isDebugInstr() && "debug instructions should not reach here");
	if (MI.isBundled())
	return false;
	if (!MI.mayLoad() \|\| MI.mayStore())
	return false;
	if (SIInstrInfo::isAtomic(MI))
	return false;
	if (IsVMEMClause && !isVMEMClauseInst(MI))
	return false;
	if (!IsVMEMClause && !isSMEMClauseInst(MI))
	return false;
	// If this is a load instruction where the result has been coalesced with an operand, then we cannot clause it.
	for (const MachineOperand &ResMO : MI.defs()) {
	Register ResReg = ResMO.getReg();
	for (const MachineOperand &MO : MI.uses()) {
	if (!MO.isReg() \|\| MO.isDef())
	continue;
	if (MO.getReg() == ResReg)
	return false;
	}
	break; // Only check the first def.
	}
	return true;
	}

	static unsigned getMopState(const MachineOperand &MO) {
	unsigned S = 0;
	if (MO.isImplicit())
	S \|= RegState::Implicit;
	if (MO.isDead())
	S \|= RegState::Dead;
	if (MO.isUndef())
	S \|= RegState::Undef;
	if (MO.isKill())
	S \|= RegState::Kill;
	if (MO.isEarlyClobber())
	S \|= RegState::EarlyClobber;
	if (MO.getReg().isPhysical() && MO.isRenamable())
	S \|= RegState::Renamable;
	return S;
	}

	// Returns false if there is a use of a def already in the map.
	// In this case we must break the clause.
	bool SIFormMemoryClauses::canBundle(const MachineInstr &MI, const RegUse &Defs,
	const RegUse &Uses) const {
	// Check interference with defs.
	for (const MachineOperand &MO : MI.operands()) {
	// TODO: Prologue/Epilogue Insertion pass does not process bundled
	// instructions.
	if (MO.isFI())
	return false;

	if (!MO.isReg())
	continue;

	Register Reg = MO.getReg();

	// If it is tied we will need to write same register as we read.
	if (MO.isTied())
	return false;

	const RegUse &Map = MO.isDef() ? Uses : Defs;
	auto Conflict = Map.find(Reg);
	if (Conflict == Map.end())
	continue;

	if (Reg.isPhysical())
	return false;

	LaneBitmask Mask = TRI->getSubRegIndexLaneMask(MO.getSubReg());
	if ((Conflict->second.second & Mask).any())
	return false;
	}

	return true;
	}

	// Since all defs in the clause are early clobber we can run out of registers.
	// Function returns false if pressure would hit the limit if instruction is
	// bundled into a memory clause.
	bool SIFormMemoryClauses::checkPressure(const MachineInstr &MI,
	GCNDownwardRPTracker &RPT) {
	// NB: skip advanceBeforeNext() call. Since all defs will be marked
	// early-clobber they will all stay alive at least to the end of the
	// clause. Therefor we should not decrease pressure even if load
	// pointer becomes dead and could otherwise be reused for destination.
	RPT.advanceToNext();
	GCNRegPressure MaxPressure = RPT.moveMaxPressure();
	unsigned Occupancy = MaxPressure.getOccupancy(*ST);

	// Don't push over half the register budget. We don't want to introduce
	// spilling just to form a soft clause.
	//
	// FIXME: This pressure check is fundamentally broken. First, this is checking
	// the global pressure, not the pressure at this specific point in the
	// program. Second, it's not accounting for the increased liveness of the use
	// operands due to the early clobber we will introduce. Third, the pressure
	// tracking does not account for the alignment requirements for SGPRs, or the
	// fragmentation of registers the allocator will need to satisfy.
	if (Occupancy >= MFI->getMinAllowedOccupancy() &&
	MaxPressure.getVGPRNum(ST->hasGFX90AInsts()) <= MaxVGPRs / 2 &&
	MaxPressure.getSGPRNum() <= MaxSGPRs / 2) {
	LastRecordedOccupancy = Occupancy;
	return true;
	}
	return false;
	}

	// Collect register defs and uses along with their lane masks and states.
	void SIFormMemoryClauses::collectRegUses(const MachineInstr &MI,
	RegUse &Defs, RegUse &Uses) const {
	for (const MachineOperand &MO : MI.operands()) {
	if (!MO.isReg())
	continue;
	Register Reg = MO.getReg();
	if (!Reg)
	continue;

	LaneBitmask Mask = Reg.isVirtual()
	? TRI->getSubRegIndexLaneMask(MO.getSubReg())
	: LaneBitmask::getAll();
	RegUse &Map = MO.isDef() ? Defs : Uses;

	auto Loc = Map.find(Reg);
	unsigned State = getMopState(MO);
	if (Loc == Map.end()) {
	Map[Reg] = std::make_pair(State, Mask);
	} else {
	Loc->second.first \|= State;
	Loc->second.second \|= Mask;
	}
	}
	}

	// Check register def/use conflicts, occupancy limits and collect def/use maps.
	// Return true if instruction can be bundled with previous. It it cannot
	// def/use maps are not updated.
	bool SIFormMemoryClauses::processRegUses(const MachineInstr &MI,
	RegUse &Defs, RegUse &Uses,
	GCNDownwardRPTracker &RPT) {
	if (!canBundle(MI, Defs, Uses))
	return false;

	if (!checkPressure(MI, RPT))
	return false;

	collectRegUses(MI, Defs, Uses);
	return true;
	}

	bool SIFormMemoryClauses::runOnMachineFunction(MachineFunction &MF) {
	if (skipFunction(MF.getFunction()))
	return false;

	ST = &MF.getSubtarget<GCNSubtarget>();
	if (!ST->isXNACKEnabled())
	return false;

	const SIInstrInfo *TII = ST->getInstrInfo();
	TRI = ST->getRegisterInfo();
	MRI = &MF.getRegInfo();
	MFI = MF.getInfo<SIMachineFunctionInfo>();
	LiveIntervals *LIS = &getAnalysis<LiveIntervals>();
	SlotIndexes *Ind = LIS->getSlotIndexes();
	bool Changed = false;

	MaxVGPRs = TRI->getAllocatableSet(MF, &AMDGPU::VGPR_32RegClass).count();
	MaxSGPRs = TRI->getAllocatableSet(MF, &AMDGPU::SGPR_32RegClass).count();
	unsigned FuncMaxClause = AMDGPU::getIntegerAttribute(
	MF.getFunction(), "amdgpu-max-memory-clause", MaxClause);

	for (MachineBasicBlock &MBB : MF) {
	GCNDownwardRPTracker RPT(*LIS);
	MachineBasicBlock::instr_iterator Next;
	for (auto I = MBB.instr_begin(), E = MBB.instr_end(); I != E; I = Next) {
	MachineInstr &MI = *I;
	Next = std::next(I);

	if (MI.isMetaInstruction())
	continue;

	bool IsVMEM = isVMEMClauseInst(MI);

	if (!isValidClauseInst(MI, IsVMEM))
	continue;

	if (!RPT.getNext().isValid())
	RPT.reset(MI);
	else { // Advance the state to the current MI.
	RPT.advance(MachineBasicBlock::const_iterator(MI));
	RPT.advanceBeforeNext();
	}

	const GCNRPTracker::LiveRegSet LiveRegsCopy(RPT.getLiveRegs());
	RegUse Defs, Uses;
	if (!processRegUses(MI, Defs, Uses, RPT)) {
	RPT.reset(MI, &LiveRegsCopy);
	continue;
	}

	MachineBasicBlock::iterator LastClauseInst = Next;
	unsigned Length = 1;
	for ( ; Next != E && Length < FuncMaxClause; ++Next) {
	// Debug instructions should not change the kill insertion.
	if (Next->isMetaInstruction())
	continue;

	if (!isValidClauseInst(*Next, IsVMEM))
	break;

	// A load from pointer which was loaded inside the same bundle is an
	// impossible clause because we will need to write and read the same
	// register inside. In this case processRegUses will return false.
	if (!processRegUses(*Next, Defs, Uses, RPT))
	break;

	LastClauseInst = Next;
	++Length;
	}
	if (Length < 2) {
	RPT.reset(MI, &LiveRegsCopy);
	continue;
	}

	Changed = true;
	MFI->limitOccupancy(LastRecordedOccupancy);

	assert(!LastClauseInst->isMetaInstruction());

	SlotIndex ClauseLiveInIdx = LIS->getInstructionIndex(MI);
	SlotIndex ClauseLiveOutIdx =
	LIS->getInstructionIndex(*LastClauseInst).getNextIndex();

	// Track the last inserted kill.
	MachineInstrBuilder Kill;

	// Insert one kill per register, with operands covering all necessary
	// subregisters.
	for (auto &&R : Uses) {
	Register Reg = R.first;
	if (Reg.isPhysical())
	continue;

	// Collect the register operands we should extend the live ranges of.
	SmallVector<std::tuple<unsigned, unsigned>> KillOps;
	const LiveInterval &LI = LIS->getInterval(R.first);

	if (!LI.hasSubRanges()) {
	if (!LI.liveAt(ClauseLiveOutIdx)) {
	KillOps.emplace_back(R.second.first \| RegState::Kill,
	AMDGPU::NoSubRegister);
	}
	} else {
	LaneBitmask KilledMask;
	for (const LiveInterval::SubRange &SR : LI.subranges()) {
	if (SR.liveAt(ClauseLiveInIdx) && !SR.liveAt(ClauseLiveOutIdx))
	KilledMask \|= SR.LaneMask;
	}

	if (KilledMask.none())
	continue;

	SmallVector<unsigned> KilledIndexes;
	bool Success = TRI->getCoveringSubRegIndexes(
	*MRI, MRI->getRegClass(Reg), KilledMask, KilledIndexes);
	(void)Success;
	assert(Success && "Failed to find subregister mask to cover lanes");
	for (unsigned SubReg : KilledIndexes) {
	KillOps.emplace_back(R.second.first \| RegState::Kill, SubReg);
	}
	}

	if (KillOps.empty())
	continue;

	// We only want to extend the live ranges of used registers. If they
	// already have existing uses beyond the bundle, we don't need the kill.
	//
	// It's possible all of the use registers were already live past the
	// bundle.
	Kill = BuildMI(*MI.getParent(), std::next(LastClauseInst),
	DebugLoc(), TII->get(AMDGPU::KILL));
	for (auto &Op : KillOps)
	Kill.addUse(Reg, std::get<0>(Op), std::get<1>(Op));
	Ind->insertMachineInstrInMaps(*Kill);
	}

	if (!Kill) {
	RPT.reset(MI, &LiveRegsCopy);
	continue;
	}

	// Restore the state after processing the end of the bundle.
	RPT.reset(*Kill, &LiveRegsCopy);

	for (auto &&R : Defs) {
	Register Reg = R.first;
	Uses.erase(Reg);
	if (Reg.isPhysical())
	continue;
	LIS->removeInterval(Reg);
	LIS->createAndComputeVirtRegInterval(Reg);
	}

	for (auto &&R : Uses) {
	Register Reg = R.first;
	if (Reg.isPhysical())
	continue;
	LIS->removeInterval(Reg);
	LIS->createAndComputeVirtRegInterval(Reg);
	}
	}
	}

	return Changed;
	}