mlir/test/lib/Dialect/SCF/TestSCFUtils.cpp

//===- TestSCFUtils.cpp --- Pass to test independent SCF dialect utils ----===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file implements a pass to test SCF dialect utils.
//
//===----------------------------------------------------------------------===//

#include "mlir/Dialect/Arith/IR/Arith.h"
#include "mlir/Dialect/Func/IR/FuncOps.h"
#include "mlir/Dialect/MemRef/IR/MemRef.h"
#include "mlir/Dialect/SCF/IR/SCF.h"
#include "mlir/Dialect/SCF/Transforms/Patterns.h"
#include "mlir/Dialect/SCF/Utils/Utils.h"
#include "mlir/IR/Builders.h"
#include "mlir/IR/PatternMatch.h"
#include "mlir/Pass/Pass.h"
#include "mlir/Transforms/GreedyPatternRewriteDriver.h"

#include <limits>

using namespace mlir;

namespace {
struct TestSCFForUtilsPass
    : public PassWrapper<TestSCFForUtilsPass, OperationPass<func::FuncOp>> {
  MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestSCFForUtilsPass)

  StringRef getArgument() const final { return "test-scf-for-utils"; }
  StringRef getDescription() const final { return "test scf.for utils"; }
  explicit TestSCFForUtilsPass() = default;
  TestSCFForUtilsPass(const TestSCFForUtilsPass &pass) : PassWrapper(pass) {}

  Option<bool> testReplaceWithNewYields{
      *this, "test-replace-with-new-yields",
      llvm::cl::desc("Test replacing a loop with a new loop that returns new "
                     "additional yield values"),
      llvm::cl::init(false)};

  void runOnOperation() override {
    func::FuncOp func = getOperation();

    // Annotate every loop-like operation with the static trip count.
    func.walk([&](LoopLikeOpInterface loopOp) {
      std::optional<APInt> tripCount = loopOp.getStaticTripCount();
      if (tripCount.has_value())
        loopOp->setDiscardableAttr(
            "test.trip-count",
            IntegerAttr::get(IntegerType::get(&getContext(),
                                              tripCount.value().getBitWidth()),
                             tripCount.value().getZExtValue()));
      else
        loopOp->setDiscardableAttr("test.trip-count",
                                   StringAttr::get(&getContext(), "none"));
    });

    if (testReplaceWithNewYields) {
      func.walk([&](scf::ForOp forOp) {
        if (forOp.getNumResults() == 0)
          return;
        auto newInitValues = forOp.getInitArgs();
        if (newInitValues.empty())
          return;
        SmallVector<Value> oldYieldValues =
            llvm::to_vector(forOp.getYieldedValues());
        NewYieldValuesFn fn = [&](OpBuilder &b, Location loc,
                                  ArrayRef<BlockArgument> newBBArgs) {
          SmallVector<Value> newYieldValues;
          for (auto yieldVal : oldYieldValues) {
            newYieldValues.push_back(
                arith::AddFOp::create(b, loc, yieldVal, yieldVal));
          }
          return newYieldValues;
        };
        IRRewriter rewriter(forOp.getContext());
        if (failed(forOp.replaceWithAdditionalYields(
                rewriter, newInitValues, /*replaceInitOperandUsesInLoop=*/true,
                fn)))
          signalPassFailure();
      });
    }
  }
};

struct TestSCFIfUtilsPass
    : public PassWrapper<TestSCFIfUtilsPass, OperationPass<ModuleOp>> {
  MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestSCFIfUtilsPass)

  StringRef getArgument() const final { return "test-scf-if-utils"; }
  StringRef getDescription() const final { return "test scf.if utils"; }
  explicit TestSCFIfUtilsPass() = default;

  void getDependentDialects(DialectRegistry &registry) const override {
    registry.insert<func::FuncDialect>();
  }

  void runOnOperation() override {
    int count = 0;
    getOperation().walk([&](scf::IfOp ifOp) {
      auto strCount = std::to_string(count++);
      func::FuncOp thenFn, elseFn;
      OpBuilder b(ifOp);
      IRRewriter rewriter(b);
      if (failed(outlineIfOp(rewriter, ifOp, &thenFn,
                             std::string("outlined_then") + strCount, &elseFn,
                             std::string("outlined_else") + strCount))) {
        this->signalPassFailure();
        return WalkResult::interrupt();
      }
      return WalkResult::advance();
    });
  }
};

static const StringLiteral kTestPipeliningLoopMarker =
    "__test_pipelining_loop__";
static const StringLiteral kTestPipeliningStageMarker =
    "__test_pipelining_stage__";
/// Marker to express the order in which operations should be after
/// pipelining.
static const StringLiteral kTestPipeliningOpOrderMarker =
    "__test_pipelining_op_order__";

static const StringLiteral kTestPipeliningAnnotationPart =
    "__test_pipelining_part";
static const StringLiteral kTestPipeliningAnnotationIteration =
    "__test_pipelining_iteration";

struct TestSCFPipeliningPass
    : public PassWrapper<TestSCFPipeliningPass, OperationPass<func::FuncOp>> {
  MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestSCFPipeliningPass)

  TestSCFPipeliningPass() = default;
  TestSCFPipeliningPass(const TestSCFPipeliningPass &) {}
  StringRef getArgument() const final { return "test-scf-pipelining"; }
  StringRef getDescription() const final { return "test scf.forOp pipelining"; }

  Option<bool> annotatePipeline{
      *this, "annotate",
      llvm::cl::desc("Annote operations during loop pipelining transformation"),
      llvm::cl::init(false)};

  Option<bool> noEpiloguePeeling{
      *this, "no-epilogue-peeling",
      llvm::cl::desc("Use predicates instead of peeling the epilogue."),
      llvm::cl::init(false)};

  static void
  getSchedule(scf::ForOp forOp,
              std::vector<std::pair<Operation *, unsigned>> &schedule) {
    if (!forOp->hasDiscardableAttr(kTestPipeliningLoopMarker))
      return;

    schedule.resize(forOp.getBody()->getOperations().size() - 1);
    WalkResult result = forOp.walk([&schedule](Operation *op) {
      auto attrStage =
          op->getDiscardableAttrOfType<IntegerAttr>(kTestPipeliningStageMarker);
      auto attrCycle = op->getDiscardableAttrOfType<IntegerAttr>(
          kTestPipeliningOpOrderMarker);
      if (attrCycle && attrStage) {
        const APInt &stage = attrStage.getValue();
        if (stage.isNegative() ||
            stage.getActiveBits() > std::numeric_limits<unsigned>::digits) {
          op->emitOpError("invalid pipeline stage");
          return WalkResult::interrupt();
        }
        const APInt &order = attrCycle.getValue();
        if (order.isNegative() || order.getActiveBits() > 64 ||
            order.getZExtValue() >= schedule.size()) {
          op->emitOpError("invalid pipeline op order");
          return WalkResult::interrupt();
        }
        size_t orderIndex = static_cast<size_t>(order.getZExtValue());
        if (schedule[orderIndex].first) {
          op->emitOpError("duplicate pipeline op order");
          return WalkResult::interrupt();
        }
        schedule[orderIndex] =
            std::make_pair(op, static_cast<unsigned>(stage.getZExtValue()));
      }
      return WalkResult::advance();
    });
    if (result.wasInterrupted())
      schedule.clear();
  }

  /// Helper to generate "predicated" version of `op`. For simplicity we just
  /// wrap the operation in a scf.ifOp operation.
  static Operation *predicateOp(RewriterBase &rewriter, Operation *op,
                                Value pred) {
    Location loc = op->getLoc();
    auto ifOp =
        scf::IfOp::create(rewriter, loc, op->getResultTypes(), pred, true);
    // True branch.
    rewriter.moveOpBefore(op, &ifOp.getThenRegion().front(),
                          ifOp.getThenRegion().front().begin());
    rewriter.setInsertionPointAfter(op);
    if (op->getNumResults() > 0)
      scf::YieldOp::create(rewriter, loc, op->getResults());
    // False branch.
    rewriter.setInsertionPointToStart(&ifOp.getElseRegion().front());
    SmallVector<Value> elseYieldOperands;
    elseYieldOperands.reserve(ifOp.getNumResults());
    if (auto viewOp = dyn_cast<memref::SubViewOp>(op)) {
      // For sub-views, just clone the op.
      // NOTE: This is okay in the test because we use dynamic memref sizes, so
      // the verifier will not complain. Otherwise, we may create a logically
      // out-of-bounds view and a different technique should be used.
      Operation *opClone = rewriter.clone(*op);
      elseYieldOperands.append(opClone->result_begin(), opClone->result_end());
    } else {
      // Default to assuming constant numeric values.
      for (Type type : op->getResultTypes()) {
        elseYieldOperands.push_back(arith::ConstantOp::create(
            rewriter, loc, rewriter.getZeroAttr(type)));
      }
    }
    if (op->getNumResults() > 0)
      scf::YieldOp::create(rewriter, loc, elseYieldOperands);
    return ifOp.getOperation();
  }

  static void annotate(Operation *op,
                       mlir::scf::PipeliningOption::PipelinerPart part,
                       unsigned iteration) {
    OpBuilder b(op);
    switch (part) {
    case mlir::scf::PipeliningOption::PipelinerPart::Prologue:
      op->setDiscardableAttr(kTestPipeliningAnnotationPart,
                             b.getStringAttr("prologue"));
      break;
    case mlir::scf::PipeliningOption::PipelinerPart::Kernel:
      op->setDiscardableAttr(kTestPipeliningAnnotationPart,
                             b.getStringAttr("kernel"));
      break;
    case mlir::scf::PipeliningOption::PipelinerPart::Epilogue:
      op->setDiscardableAttr(kTestPipeliningAnnotationPart,
                             b.getStringAttr("epilogue"));
      break;
    }
    op->setDiscardableAttr(kTestPipeliningAnnotationIteration,
                           b.getI32IntegerAttr(iteration));
  }

  void getDependentDialects(DialectRegistry &registry) const override {
    registry.insert<arith::ArithDialect, memref::MemRefDialect>();
  }

  void runOnOperation() override {
    RewritePatternSet patterns(&getContext());
    mlir::scf::PipeliningOption options;
    options.getScheduleFn = getSchedule;
    options.supportDynamicLoops = true;
    options.predicateFn = predicateOp;
    if (annotatePipeline)
      options.annotateFn = annotate;
    if (noEpiloguePeeling) {
      options.peelEpilogue = false;
    }
    scf::populateSCFLoopPipeliningPatterns(patterns, options);
    (void)applyPatternsGreedily(getOperation(), std::move(patterns));
    getOperation().walk([](Operation *op) {
      // Clean up the markers.
      op->removeDiscardableAttr(kTestPipeliningStageMarker);
      op->removeDiscardableAttr(kTestPipeliningOpOrderMarker);
    });
  }
};

static constexpr StringLiteral kSplitAtAttr = "test.split_at";
static constexpr StringLiteral kSplitArgAttr = "test.split_arg";

struct TestSplitForOpAtPointPass
    : public PassWrapper<TestSplitForOpAtPointPass,
                         OperationPass<func::FuncOp>> {
  MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestSplitForOpAtPointPass)

  StringRef getArgument() const final { return "test-split-for-op-at-point"; }

  StringRef getDescription() const final { return "test splitForOpAtPoint"; }

  TestSplitForOpAtPointPass() = default;
  TestSplitForOpAtPointPass(const TestSplitForOpAtPointPass &) {}

  void getDependentDialects(DialectRegistry &registry) const override {
    registry.insert<arith::ArithDialect>();
  }

  void runOnOperation() override {
    func::FuncOp func = getOperation();
    SmallVector<scf::ForOp> loopsToSplit;
    func.walk([&](scf::ForOp forOp) {
      if (forOp->hasDiscardableAttr(kSplitAtAttr) ||
          forOp->hasDiscardableAttr(kSplitArgAttr))
        loopsToSplit.push_back(forOp);
    });

    IRRewriter rewriter(func.getContext());
    for (scf::ForOp forOp : loopsToSplit) {
      Value splitPoint;
      if (auto splitAttr =
              forOp->getDiscardableAttrOfType<IntegerAttr>(kSplitAtAttr)) {
        rewriter.setInsertionPoint(forOp);
        splitPoint =
            arith::ConstantOp::create(rewriter, forOp.getLoc(), splitAttr);
        rewriter.modifyOpInPlace(
            forOp, [&] { forOp->removeDiscardableAttr(kSplitAtAttr); });
      } else if (auto argAttr = forOp->getDiscardableAttrOfType<IntegerAttr>(
                     kSplitArgAttr)) {
        int64_t argNo = argAttr.getInt();
        if (argNo < 0 ||
            static_cast<unsigned>(argNo) >= func.getNumArguments()) {
          emitError(forOp.getLoc(), "test.split_arg is out of range");
          return signalPassFailure();
        }
        splitPoint = func.getArgument(argNo);
        rewriter.modifyOpInPlace(
            forOp, [&] { forOp->removeDiscardableAttr(kSplitArgAttr); });
      } else {
        continue;
      }
      if (failed(splitForOpAtPoint(rewriter, forOp, splitPoint))) {
        emitError(forOp.getLoc(), "failed to split scf.for");
        return signalPassFailure();
      }
    }
  }
};
} // namespace

namespace mlir {
namespace test {
void registerTestSCFUtilsPass() {
  PassRegistration<TestSCFForUtilsPass>();
  PassRegistration<TestSCFIfUtilsPass>();
  PassRegistration<TestSCFPipeliningPass>();
  PassRegistration<TestSplitForOpAtPointPass>();
}
} // namespace test
} // namespace mlir