llvm/lib/Target/SPIRV/SPIRVMergeRegionExitTargets.cpp

//===-- SPIRVMergeRegionExitTargets.cpp ----------------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// Merge the multiple exit targets of a convergence region into a single block.
// Each exit target will be assigned a constant value, and a phi node + switch
// will allow the new exit target to re-route to the correct basic block.
//
//===----------------------------------------------------------------------===//

#include "SPIRVMergeRegionExitTargets.h"
#include "Analysis/SPIRVConvergenceRegionAnalysis.h"
#include "SPIRV.h"
#include "SPIRVSubtarget.h"
#include "SPIRVUtils.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/InitializePasses.h"
#include "llvm/Transforms/Utils/Cloning.h"
#include "llvm/Transforms/Utils/LoopSimplify.h"
#include "llvm/Transforms/Utils/LowerMemIntrinsics.h"

using namespace llvm;

namespace {

// Run the pass on the given convergence region, ignoring the sub-regions.
// Returns true if the CFG changed, false otherwise.
static bool runOnConvergenceRegionNoRecurse(LoopInfo &LI,
                                            SPIRV::ConvergenceRegion *CR) {
  // Gather all the exit targets for this region.
  SmallPtrSet<BasicBlock *, 4> ExitTargets;
  for (BasicBlock *Exit : CR->Exits) {
    for (BasicBlock *Target : successors(Exit)) {
      if (CR->Blocks.count(Target) == 0)
        ExitTargets.insert(Target);
    }
  }

  // If we have zero or one exit target, nothing do to.
  if (ExitTargets.size() <= 1)
    return false;

  // Create the new single exit target.
  auto F = CR->Entry->getParent();
  auto NewExitTarget = BasicBlock::Create(F->getContext(), "new.exit", F);
  IRBuilder<> Builder(NewExitTarget);

  AllocaInst *Variable = createVariable(*F, Builder.getInt32Ty());

  // CodeGen output needs to be stable. Using the set as-is would order
  // the targets differently depending on the allocation pattern.
  // Sorting per basic-block ordering in the function.
  std::vector<BasicBlock *> SortedExitTargets;
  std::vector<BasicBlock *> SortedExits;
  for (BasicBlock &BB : *F) {
    if (ExitTargets.count(&BB) != 0)
      SortedExitTargets.push_back(&BB);
    if (CR->Exits.count(&BB) != 0)
      SortedExits.push_back(&BB);
  }

  // Creating one constant per distinct exit target. This will be route to the
  // correct target.
  DenseMap<BasicBlock *, ConstantInt *> TargetToValue;
  for (BasicBlock *Target : SortedExitTargets)
    TargetToValue.insert(
        std::make_pair(Target, Builder.getInt32(TargetToValue.size())));

  // Creating one variable per exit node, set to the constant matching the
  // targeted external block.
  std::vector<std::pair<BasicBlock *, Value *>> ExitToVariable;
  for (auto Exit : SortedExits) {
    llvm::Value *Value = createExitVariable(Exit, TargetToValue);
    IRBuilder<> B2(Exit);
    B2.SetInsertPoint(Exit->getFirstInsertionPt());
    B2.CreateStore(Value, Variable);
    ExitToVariable.emplace_back(std::make_pair(Exit, Value));
  }

  llvm::Value *Load = Builder.CreateLoad(Builder.getInt32Ty(), Variable);

  // Creating the switch to jump to the correct exit target.
  llvm::SwitchInst *Sw = Builder.CreateSwitch(Load, SortedExitTargets[0],
                                              SortedExitTargets.size() - 1);
  for (size_t i = 1; i < SortedExitTargets.size(); i++) {
    BasicBlock *BB = SortedExitTargets[i];
    Sw->addCase(TargetToValue[BB], BB);
  }

  // Fix exit branches to redirect to the new exit.
  for (auto Exit : CR->Exits) {
    Instruction *T = Exit->getTerminator();
    for (auto I = succ_begin(T), E = succ_end(T); I != E; ++I)
      if (ExitTargets.contains(*I))
        I.getUse()->set(NewExitTarget);
  }

  CR = CR->Parent;
  while (CR) {
    CR->Blocks.insert(NewExitTarget);
    CR = CR->Parent;
  }

  return true;
}

/// Run the pass on the given convergence region and sub-regions (DFS).
/// Returns true if a region/sub-region was modified, false otherwise.
/// This returns as soon as one region/sub-region has been modified.
static bool runOnConvergenceRegion(LoopInfo &LI, SPIRV::ConvergenceRegion *CR) {
  for (auto *Child : CR->Children)
    if (runOnConvergenceRegion(LI, Child))
      return true;

  return runOnConvergenceRegionNoRecurse(LI, CR);
}

#if !NDEBUG
/// Validates each edge exiting the region has the same destination basic
/// block.
static void validateRegionExits(const SPIRV::ConvergenceRegion *CR) {
  for (auto *Child : CR->Children)
    validateRegionExits(Child);

  SmallPtrSet<BasicBlock *, 0> ExitTargets;
  for (auto *Exit : CR->Exits) {
    for (auto *BB : successors(Exit)) {
      if (CR->Blocks.count(BB) == 0)
        ExitTargets.insert(BB);
    }
  }

  assert(ExitTargets.size() <= 1);
}
#endif

static bool runImpl(Function &F, LoopInfo &LI,
                    SPIRV::ConvergenceRegionInfo &RegionInfo) {
  auto *TopLevelRegion = RegionInfo.getWritableTopLevelRegion();

  // FIXME: very inefficient method: each time a region is modified, we bubble
  // back up, and recompute the whole convergence region tree. Once the
  // algorithm is completed and test coverage good enough, rewrite this pass
  // to be efficient instead of simple.
  bool Modified = false;
  while (runOnConvergenceRegion(LI, TopLevelRegion)) {
    Modified = true;
  }

#if !defined(NDEBUG) || defined(EXPENSIVE_CHECKS)
  validateRegionExits(TopLevelRegion);
#endif
  return Modified;
}

class SPIRVMergeRegionExitTargetsLegacy : public FunctionPass {
public:
  static char ID;

  SPIRVMergeRegionExitTargetsLegacy() : FunctionPass(ID) {}

  bool runOnFunction(Function &F) override {
    LoopInfo &LI = getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
    auto &RegionInfo = getAnalysis<SPIRVConvergenceRegionAnalysisWrapperPass>()
                           .getRegionInfo();
    return runImpl(F, LI, RegionInfo);
  }

  void getAnalysisUsage(AnalysisUsage &AU) const override {
    AU.addRequired<DominatorTreeWrapperPass>();
    AU.addRequired<LoopInfoWrapperPass>();
    AU.addRequired<SPIRVConvergenceRegionAnalysisWrapperPass>();

    AU.addPreserved<SPIRVConvergenceRegionAnalysisWrapperPass>();
    FunctionPass::getAnalysisUsage(AU);
  }
};
} // namespace

PreservedAnalyses
SPIRVMergeRegionExitTargets::run(Function &F, FunctionAnalysisManager &AM) {
  auto &LI = AM.getResult<LoopAnalysis>(F);
  auto &RegionInfo = AM.getResult<SPIRVConvergenceRegionAnalysis>(F);
  return runImpl(F, LI, RegionInfo) ? PreservedAnalyses::none()
                                    : PreservedAnalyses::all();
}

char SPIRVMergeRegionExitTargetsLegacy::ID = 0;

INITIALIZE_PASS_BEGIN(SPIRVMergeRegionExitTargetsLegacy,
                      "split-region-exit-blocks",
                      "SPIRV split region exit blocks", false, false)
INITIALIZE_PASS_DEPENDENCY(LoopSimplify)
INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
INITIALIZE_PASS_DEPENDENCY(SPIRVConvergenceRegionAnalysisWrapperPass)

INITIALIZE_PASS_END(SPIRVMergeRegionExitTargetsLegacy,
                    "split-region-exit-blocks",
                    "SPIRV split region exit blocks", false, false)

FunctionPass *llvm::createSPIRVMergeRegionExitTargetsPass() {
  return new SPIRVMergeRegionExitTargetsLegacy();
}