mlir/lib/Target/LLVMIR/LoopAnnotationTranslation.cpp

//===- LoopAnnotationTranslation.cpp - Loop annotation export -------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//

#include "LoopAnnotationTranslation.h"
#include "llvm/IR/DebugInfoMetadata.h"

using namespace mlir;
using namespace mlir::LLVM;
using namespace mlir::LLVM::detail;

namespace {
/// Helper class that keeps the state of one attribute to metadata conversion.
struct LoopAnnotationConversion {
  LoopAnnotationConversion(LoopAnnotationAttr attr, Operation *op,
                           LoopAnnotationTranslation &loopAnnotationTranslation,
                           llvm::LLVMContext &ctx)
      : attr(attr), op(op),
        loopAnnotationTranslation(loopAnnotationTranslation), ctx(ctx) {}

  /// Converts this struct's loop annotation into a corresponding LLVMIR
  /// metadata representation.
  llvm::MDNode *convert();

  /// Conversion functions for different payload attribute kinds.
  void addUnitNode(StringRef name);
  void addUnitNode(StringRef name, BoolAttr attr);
  void addI32NodeWithVal(StringRef name, uint32_t val);
  void convertBoolNode(StringRef name, BoolAttr attr, bool negated = false);
  void convertBooleanUnitNode(StringRef enableName, StringRef disableName,
                              BoolAttr attr, bool negated = false);
  void convertI32Node(StringRef name, IntegerAttr attr);
  void convertFollowupNode(StringRef name, LoopAnnotationAttr attr);
  void convertLocation(FusedLoc attr);

  /// Conversion functions for each for each loop annotation sub-attribute.
  void convertLoopOptions(LoopVectorizeAttr options);
  void convertLoopOptions(LoopInterleaveAttr options);
  void convertLoopOptions(LoopUnrollAttr options);
  void convertLoopOptions(LoopUnrollAndJamAttr options);
  void convertLoopOptions(LoopLICMAttr options);
  void convertLoopOptions(LoopDistributeAttr options);
  void convertLoopOptions(LoopPipelineAttr options);
  void convertLoopOptions(LoopPeeledAttr options);
  void convertLoopOptions(LoopUnswitchAttr options);

  LoopAnnotationAttr attr;
  Operation *op;
  LoopAnnotationTranslation &loopAnnotationTranslation;
  llvm::LLVMContext &ctx;
  llvm::SmallVector<llvm::Metadata *> metadataNodes;
};
} // namespace

void LoopAnnotationConversion::addUnitNode(StringRef name) {
  metadataNodes.push_back(
      llvm::MDNode::get(ctx, {llvm::MDString::get(ctx, name)}));
}

void LoopAnnotationConversion::addUnitNode(StringRef name, BoolAttr attr) {
  if (attr && attr.getValue())
    addUnitNode(name);
}

void LoopAnnotationConversion::addI32NodeWithVal(StringRef name, uint32_t val) {
  llvm::Constant *cstValue = llvm::ConstantInt::get(
      llvm::IntegerType::get(ctx, /*NumBits=*/32), val, /*isSigned=*/false);
  metadataNodes.push_back(
      llvm::MDNode::get(ctx, {llvm::MDString::get(ctx, name),
                              llvm::ConstantAsMetadata::get(cstValue)}));
}

void LoopAnnotationConversion::convertBoolNode(StringRef name, BoolAttr attr,
                                               bool negated) {
  if (!attr)
    return;
  bool val = negated ^ attr.getValue();
  llvm::Constant *cstValue = llvm::ConstantInt::getBool(ctx, val);
  metadataNodes.push_back(
      llvm::MDNode::get(ctx, {llvm::MDString::get(ctx, name),
                              llvm::ConstantAsMetadata::get(cstValue)}));
}

/// Emits the single-operand node of an enable/disable pair. As in
/// convertBoolNode, the attribute is tri-state: a null \p attr emits no node
/// at all, otherwise \p negated ^ the attribute value is the enable bit and
/// selects which of the two names is emitted.
void LoopAnnotationConversion::convertBooleanUnitNode(StringRef enableName,
                                                      StringRef disableName,
                                                      BoolAttr attr,
                                                      bool negated) {
  if (!attr)
    return;
  addUnitNode((negated ^ attr.getValue()) ? enableName : disableName);
}

void LoopAnnotationConversion::convertI32Node(StringRef name,
                                              IntegerAttr attr) {
  if (!attr)
    return;
  addI32NodeWithVal(name, attr.getInt());
}

void LoopAnnotationConversion::convertFollowupNode(StringRef name,
                                                   LoopAnnotationAttr attr) {
  if (!attr)
    return;

  llvm::MDNode *loopID =
      loopAnnotationTranslation.translateLoopAnnotation(attr, op);

  // Follow-ups contain properties directly, without the LoopID self-reference.
  SmallVector<llvm::Metadata *> operands;
  operands.push_back(llvm::MDString::get(ctx, name));
  for (const llvm::MDOperand &operand : llvm::drop_begin(loopID->operands()))
    operands.push_back(operand.get());
  metadataNodes.push_back(llvm::MDNode::get(ctx, operands));
}

void LoopAnnotationConversion::convertLoopOptions(LoopVectorizeAttr options) {
  convertBooleanUnitNode("llvm.loop.vectorize.enable",
                         "llvm.loop.vectorize.disable", options.getDisable(),
                         /*negated=*/true);
  convertBooleanUnitNode("llvm.loop.vectorize.predicate.enable",
                         "llvm.loop.vectorize.predicate.disable",
                         options.getPredicateEnable());
  convertBooleanUnitNode("llvm.loop.vectorize.scalable.enable",
                         "llvm.loop.vectorize.scalable.disable",
                         options.getScalableEnable());
  convertI32Node("llvm.loop.vectorize.width", options.getWidth());
  convertFollowupNode("llvm.loop.vectorize.followup_vectorized",
                      options.getFollowupVectorized());
  convertFollowupNode("llvm.loop.vectorize.followup_epilogue",
                      options.getFollowupEpilogue());
  convertFollowupNode("llvm.loop.vectorize.followup_all",
                      options.getFollowupAll());
}

void LoopAnnotationConversion::convertLoopOptions(LoopInterleaveAttr options) {
  convertI32Node("llvm.loop.interleave.count", options.getCount());
}

void LoopAnnotationConversion::convertLoopOptions(LoopUnrollAttr options) {
  if (auto disable = options.getDisable())
    addUnitNode(disable.getValue() ? "llvm.loop.unroll.disable"
                                   : "llvm.loop.unroll.enable");
  convertI32Node("llvm.loop.unroll.count", options.getCount());
  convertBoolNode("llvm.loop.unroll.runtime.disable",
                  options.getRuntimeDisable());
  addUnitNode("llvm.loop.unroll.full", options.getFull());
  convertFollowupNode("llvm.loop.unroll.followup_unrolled",
                      options.getFollowupUnrolled());
  convertFollowupNode("llvm.loop.unroll.followup_remainder",
                      options.getFollowupRemainder());
  convertFollowupNode("llvm.loop.unroll.followup_all",
                      options.getFollowupAll());
}

void LoopAnnotationConversion::convertLoopOptions(
    LoopUnrollAndJamAttr options) {
  if (auto disable = options.getDisable())
    addUnitNode(disable.getValue() ? "llvm.loop.unroll_and_jam.disable"
                                   : "llvm.loop.unroll_and_jam.enable");
  convertI32Node("llvm.loop.unroll_and_jam.count", options.getCount());
  convertFollowupNode("llvm.loop.unroll_and_jam.followup_outer",
                      options.getFollowupOuter());
  convertFollowupNode("llvm.loop.unroll_and_jam.followup_inner",
                      options.getFollowupInner());
  convertFollowupNode("llvm.loop.unroll_and_jam.followup_remainder_outer",
                      options.getFollowupRemainderOuter());
  convertFollowupNode("llvm.loop.unroll_and_jam.followup_remainder_inner",
                      options.getFollowupRemainderInner());
  convertFollowupNode("llvm.loop.unroll_and_jam.followup_all",
                      options.getFollowupAll());
}

void LoopAnnotationConversion::convertLoopOptions(LoopLICMAttr options) {
  addUnitNode("llvm.licm.disable", options.getDisable());
  addUnitNode("llvm.loop.licm_versioning.disable",
              options.getVersioningDisable());
}

void LoopAnnotationConversion::convertLoopOptions(LoopDistributeAttr options) {
  convertBooleanUnitNode("llvm.loop.distribute.enable",
                         "llvm.loop.distribute.disable", options.getDisable(),
                         /*negated=*/true);
  convertFollowupNode("llvm.loop.distribute.followup_coincident",
                      options.getFollowupCoincident());
  convertFollowupNode("llvm.loop.distribute.followup_sequential",
                      options.getFollowupSequential());
  convertFollowupNode("llvm.loop.distribute.followup_fallback",
                      options.getFollowupFallback());
  convertFollowupNode("llvm.loop.distribute.followup_all",
                      options.getFollowupAll());
}

void LoopAnnotationConversion::convertLoopOptions(LoopPipelineAttr options) {
  convertBoolNode("llvm.loop.pipeline.disable", options.getDisable());
  convertI32Node("llvm.loop.pipeline.initiationinterval",
                 options.getInitiationinterval());
}

void LoopAnnotationConversion::convertLoopOptions(LoopPeeledAttr options) {
  convertI32Node("llvm.loop.peeled.count", options.getCount());
}

void LoopAnnotationConversion::convertLoopOptions(LoopUnswitchAttr options) {
  addUnitNode("llvm.loop.unswitch.partial.disable",
              options.getPartialDisable());
}

void LoopAnnotationConversion::convertLocation(FusedLoc location) {
  auto localScopeAttr =
      dyn_cast_or_null<DILocalScopeAttr>(location.getMetadata());
  if (!localScopeAttr)
    return;
  auto *localScope = dyn_cast<llvm::DILocalScope>(
      loopAnnotationTranslation.moduleTranslation.translateDebugInfo(
          localScopeAttr));
  if (!localScope)
    return;
  llvm::Metadata *loc =
      loopAnnotationTranslation.moduleTranslation.translateLoc(location,
                                                               localScope);
  metadataNodes.push_back(loc);
}

llvm::MDNode *LoopAnnotationConversion::convert() {
  // Reserve operand 0 for loop id self reference.
  auto dummy = llvm::MDNode::getTemporary(ctx, {});
  metadataNodes.push_back(dummy.get());

  if (FusedLoc startLoc = attr.getStartLoc())
    convertLocation(startLoc);

  if (FusedLoc endLoc = attr.getEndLoc())
    convertLocation(endLoc);

  addUnitNode("llvm.loop.disable_nonforced", attr.getDisableNonforced());
  addUnitNode("llvm.loop.mustprogress", attr.getMustProgress());
  // "isvectorized" is encoded as an i32 value.
  if (BoolAttr isVectorized = attr.getIsVectorized())
    addI32NodeWithVal("llvm.loop.isvectorized", isVectorized.getValue());

  if (auto options = attr.getVectorize())
    convertLoopOptions(options);
  if (auto options = attr.getInterleave())
    convertLoopOptions(options);
  if (auto options = attr.getUnroll())
    convertLoopOptions(options);
  if (auto options = attr.getUnrollAndJam())
    convertLoopOptions(options);
  if (auto options = attr.getLicm())
    convertLoopOptions(options);
  if (auto options = attr.getDistribute())
    convertLoopOptions(options);
  if (auto options = attr.getPipeline())
    convertLoopOptions(options);
  if (auto options = attr.getPeeled())
    convertLoopOptions(options);
  if (auto options = attr.getUnswitch())
    convertLoopOptions(options);

  ArrayRef<AccessGroupAttr> parallelAccessGroups = attr.getParallelAccesses();
  if (!parallelAccessGroups.empty()) {
    SmallVector<llvm::Metadata *> parallelAccess;
    parallelAccess.push_back(
        llvm::MDString::get(ctx, "llvm.loop.parallel_accesses"));
    for (AccessGroupAttr accessGroupAttr : parallelAccessGroups)
      parallelAccess.push_back(
          loopAnnotationTranslation.getAccessGroup(accessGroupAttr));
    metadataNodes.push_back(llvm::MDNode::get(ctx, parallelAccess));
  }

  // Create loop options and set the first operand to itself.
  llvm::MDNode *loopMD = llvm::MDNode::get(ctx, metadataNodes);
  loopMD->replaceOperandWith(0, loopMD);

  return loopMD;
}

llvm::MDNode *
LoopAnnotationTranslation::translateLoopAnnotation(LoopAnnotationAttr attr,
                                                   Operation *op) {
  if (!attr)
    return nullptr;

  llvm::MDNode *loopMD = lookupLoopMetadata(attr);
  if (loopMD)
    return loopMD;

  loopMD =
      LoopAnnotationConversion(attr, op, *this, this->llvmModule.getContext())
          .convert();
  // Store a map from this Attribute to the LLVM metadata in case we
  // encounter it again.
  mapLoopMetadata(attr, loopMD);
  return loopMD;
}

llvm::MDNode *
LoopAnnotationTranslation::getAccessGroup(AccessGroupAttr accessGroupAttr) {
  auto [result, inserted] =
      accessGroupMetadataMapping.try_emplace(accessGroupAttr);
  if (inserted)
    result->second = llvm::MDNode::getDistinct(llvmModule.getContext(), {});
  return result->second;
}

llvm::MDNode *
LoopAnnotationTranslation::getAccessGroups(AccessGroupOpInterface op) {
  ArrayAttr accessGroups = op.getAccessGroupsOrNull();
  if (!accessGroups || accessGroups.empty())
    return nullptr;

  SmallVector<llvm::Metadata *> groupMDs;
  for (AccessGroupAttr group : accessGroups.getAsRange<AccessGroupAttr>())
    groupMDs.push_back(getAccessGroup(group));
  if (groupMDs.size() == 1)
    return llvm::cast<llvm::MDNode>(groupMDs.front());
  return llvm::MDNode::get(llvmModule.getContext(), groupMDs);
}