llvm/lib/Target/SPIRV/SPIRVCombinerHelper.cpp

//===-- SPIRVCombinerHelper.cpp -------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//

#include "SPIRVCombinerHelper.h"
#include "SPIRVGlobalRegistry.h"
#include "SPIRVUtils.h"
#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/IntrinsicsSPIRV.h"
#include "llvm/Target/TargetMachine.h"

using namespace llvm;
using namespace MIPatternMatch;

SPIRVCombinerHelper::SPIRVCombinerHelper(
    GISelChangeObserver &Observer, MachineIRBuilder &B, bool IsPreLegalize,
    GISelValueTracking *VT, MachineDominatorTree *MDT, const LegalizerInfo *LI,
    const SPIRVSubtarget &STI)
    : CombinerHelper(Observer, B, IsPreLegalize, VT, MDT, LI), STI(STI) {}

/// This match is part of a combine that
/// rewrites X / length(X) to normalize(X)
///   (vXf32 (g_fdiv
///             (vXf32 X)
///             (vXf32 splat
///                    (f32 (g_intrinsic length (vXf32 X))))))
/// ->
///   (vXf32 (g_intrinsic normalize (vXf32 X)))
///
bool SPIRVCombinerHelper::matchFDivToNormalize(MachineInstr &MI) const {
  Register NumeratorReg = MI.getOperand(1).getReg();
  Register DivisorReg = MI.getOperand(2).getReg();

  // Match the divisor as a splat of length, inserted into lane 0.
  MachineInstr *ShuffleInstr = MRI.getVRegDef(DivisorReg);
  if (ShuffleInstr->getOpcode() != TargetOpcode::G_SHUFFLE_VECTOR)
    return false;
  if (!all_of(cast<GShuffleVector>(ShuffleInstr)->getMask(),
              [](int M) { return M == 0; }))
    return false;

  MachineInstr *InsertInstr =
      MRI.getVRegDef(ShuffleInstr->getOperand(1).getReg());
  if (!isSpvIntrinsic(*InsertInstr, Intrinsic::spv_insertelt))
    return false;
  if (!mi_match(InsertInstr->getOperand(4).getReg(), MRI, m_ZeroInt()))
    return false;

  MachineInstr *LengthInstr =
      MRI.getVRegDef(InsertInstr->getOperand(3).getReg());
  if (!isSpvIntrinsic(*LengthInstr, Intrinsic::spv_length))
    return false;

  // Check that length's argument is the same as the numerator.
  return LengthInstr->getOperand(2).getReg() == NumeratorReg;
}

void SPIRVCombinerHelper::applySPIRVNormalize(MachineInstr &MI) const {
  // Extract the operand for X from the match criteria.
  Register NumeratorReg = MI.getOperand(1).getReg();
  Register ResultReg = MI.getOperand(0).getReg();

  Builder.setInstrAndDebugLoc(MI);
  Builder.buildIntrinsic(Intrinsic::spv_normalize, ResultReg)
      .addUse(NumeratorReg);

  MI.eraseFromParent();
}

/// This match is part of a combine that
/// rewrites select(fcmp(dot(I, Ng), 0), N, -N) to faceforward(N, I, Ng)
///   (vXf32 (g_select
///             (g_fcmp
///                (g_intrinsic dot(vXf32 I) (vXf32 Ng)
///                 0)
///             (vXf32 N)
///             (vXf32 g_fneg (vXf32 N))))
/// ->
///   (vXf32 (g_intrinsic faceforward
///             (vXf32 N) (vXf32 I) (vXf32 Ng)))
///
/// This only works for Vulkan shader targets.
///
bool SPIRVCombinerHelper::matchSelectToFaceForward(MachineInstr &MI) const {
  if (!STI.isShader())
    return false;

  // Match overall select pattern.
  Register CondReg, TrueReg, FalseReg;
  if (!mi_match(MI.getOperand(0).getReg(), MRI,
                m_GISelect(m_Reg(CondReg), m_Reg(TrueReg), m_Reg(FalseReg))))
    return false;

  // Match the FCMP condition.
  Register DotReg, CondZeroReg;
  CmpInst::Predicate Pred;
  if (!mi_match(CondReg, MRI,
                m_GFCmp(m_Pred(Pred), m_Reg(DotReg), m_Reg(CondZeroReg))))
    return false;
  if (Pred == CmpInst::FCMP_OGT || Pred == CmpInst::FCMP_UGT)
    std::swap(DotReg, CondZeroReg);
  else if (!(Pred == CmpInst::FCMP_OLT || Pred == CmpInst::FCMP_ULT))
    return false;

  // Check if FCMP is a comparison between a dot product and 0.
  if (!mi_match(DotReg, MRI, m_GIntrinsic<Intrinsic::spv_fdot>())) {
    Register DotOperand1, DotOperand2;
    // Check for scalar dot product.
    if (!mi_match(DotReg, MRI,
                  m_GFMul(m_Reg(DotOperand1), m_Reg(DotOperand2))) ||
        !MRI.getType(DotOperand1).isScalar() ||
        !MRI.getType(DotOperand2).isScalar())
      return false;
  }

  const ConstantFP *ZeroVal;
  if (!mi_match(CondZeroReg, MRI, m_GFCst(ZeroVal)) || !ZeroVal->isZero())
    return false;

  // Check if select's false operand is the negation of the true operand.
  auto AreNegatedConstantsOrSplats = [&](Register TrueReg, Register FalseReg) {
    std::optional<FPValueAndVReg> TrueVal, FalseVal;
    if (!mi_match(TrueReg, MRI, m_GFCstOrSplat(TrueVal)) ||
        !mi_match(FalseReg, MRI, m_GFCstOrSplat(FalseVal)))
      return false;
    APFloat TrueValNegated = TrueVal->Value;
    TrueValNegated.changeSign();
    return FalseVal->Value.compare(TrueValNegated) == APFloat::cmpEqual;
  };

  if (!mi_match(TrueReg, MRI, m_GFNeg(m_SpecificReg(FalseReg))) &&
      !mi_match(FalseReg, MRI, m_GFNeg(m_SpecificReg(TrueReg)))) {
    std::optional<FPValueAndVReg> MulConstant;
    GBuildVector *TrueInstr, *FalseInstr;
    if (mi_match(TrueReg, MRI, m_GBuildVector(TrueInstr)) &&
        mi_match(FalseReg, MRI, m_GBuildVector(FalseInstr)) &&
        TrueInstr->getNumOperands() == FalseInstr->getNumOperands()) {
      for (unsigned I = 1; I < TrueInstr->getNumOperands(); ++I)
        if (!AreNegatedConstantsOrSplats(TrueInstr->getOperand(I).getReg(),
                                         FalseInstr->getOperand(I).getReg()))
          return false;
    } else if (mi_match(TrueReg, MRI,
                        m_GFMul(m_SpecificReg(FalseReg),
                                m_GFCstOrSplat(MulConstant))) ||
               mi_match(FalseReg, MRI,
                        m_GFMul(m_SpecificReg(TrueReg),
                                m_GFCstOrSplat(MulConstant))) ||
               mi_match(TrueReg, MRI,
                        m_GFMul(m_GFCstOrSplat(MulConstant),
                                m_SpecificReg(FalseReg))) ||
               mi_match(FalseReg, MRI,
                        m_GFMul(m_GFCstOrSplat(MulConstant),
                                m_SpecificReg(TrueReg)))) {
      if (!MulConstant || !MulConstant->Value.isMinusOne())
        return false;
    } else if (!AreNegatedConstantsOrSplats(TrueReg, FalseReg))
      return false;
  }

  return true;
}

void SPIRVCombinerHelper::applySPIRVFaceForward(MachineInstr &MI) const {
  // Extract the operands for N, I, and Ng from the match criteria.
  Register CondReg = MI.getOperand(1).getReg();
  MachineInstr *CondInstr = MRI.getVRegDef(CondReg);
  Register DotReg = CondInstr->getOperand(2).getReg();
  CmpInst::Predicate Pred = cast<GFCmp>(CondInstr)->getCond();
  if (Pred == CmpInst::FCMP_OGT || Pred == CmpInst::FCMP_UGT)
    DotReg = CondInstr->getOperand(3).getReg();
  MachineInstr *DotInstr = MRI.getVRegDef(DotReg);
  Register DotOperand1, DotOperand2;
  if (DotInstr->getOpcode() == TargetOpcode::G_FMUL) {
    DotOperand1 = DotInstr->getOperand(1).getReg();
    DotOperand2 = DotInstr->getOperand(2).getReg();
  } else {
    DotOperand1 = DotInstr->getOperand(2).getReg();
    DotOperand2 = DotInstr->getOperand(3).getReg();
  }
  Register TrueReg = MI.getOperand(2).getReg();
  Register FalseReg = MI.getOperand(3).getReg();
  MachineInstr *TrueInstr = MRI.getVRegDef(TrueReg);
  if (TrueInstr->getOpcode() == TargetOpcode::G_FNEG ||
      TrueInstr->getOpcode() == TargetOpcode::G_FMUL)
    std::swap(TrueReg, FalseReg);

  Register ResultReg = MI.getOperand(0).getReg();
  Builder.setInstrAndDebugLoc(MI);
  Builder.buildIntrinsic(Intrinsic::spv_faceforward, ResultReg)
      .addUse(TrueReg)      // N
      .addUse(DotOperand1)  // I
      .addUse(DotOperand2); // Ng

  MI.eraseFromParent();
}

void SPIRVCombinerHelper::applyMatrixTranspose(MachineInstr &MI) const {
  Register ResReg = MI.getOperand(0).getReg();
  Register InReg = MI.getOperand(2).getReg();
  uint32_t Rows = MI.getOperand(3).getImm();
  uint32_t Cols = MI.getOperand(4).getImm();

  Builder.setInstrAndDebugLoc(MI);

  // A 1xN or Nx1 transpose is a pure reshape.
  if (Rows == 1 || Cols == 1) {
    Builder.buildCopy(ResReg, InReg);
    MI.eraseFromParent();
    return;
  }

  SmallVector<int, 16> Mask;
  for (uint32_t K = 0; K < Rows * Cols; ++K) {
    uint32_t R = K / Cols;
    uint32_t C = K % Cols;
    Mask.push_back(C * Rows + R);
  }

  Builder.buildShuffleVector(ResReg, InReg, InReg, Mask);
  MI.eraseFromParent();
}

SmallVector<Register, 4>
SPIRVCombinerHelper::extractColumns(Register MatrixReg, uint32_t NumberOfCols,
                                    SPIRVTypeInst SpvColType,
                                    SPIRVGlobalRegistry *GR) const {
  // If the matrix is a single colunm, return that single column.
  if (NumberOfCols == 1)
    return {MatrixReg};

  SmallVector<Register, 4> Cols;
  LLT ColTy = GR->getRegType(SpvColType);
  for (uint32_t J = 0; J < NumberOfCols; ++J)
    Cols.push_back(MRI.createGenericVirtualRegister(ColTy));
  Builder.buildUnmerge(Cols, MatrixReg);
  for (Register R : Cols) {
    setRegClassType(R, SpvColType, GR, &MRI, Builder.getMF());
  }
  return Cols;
}

SmallVector<Register, 4>
SPIRVCombinerHelper::extractRows(Register MatrixReg, uint32_t NumRows,
                                 uint32_t NumCols, SPIRVTypeInst SpvRowType,
                                 SPIRVGlobalRegistry *GR) const {
  SmallVector<Register, 4> Rows;
  LLT VecTy = GR->getRegType(SpvRowType);

  // If there is only one column, then each row is a scalar that needs
  // to be extracted.
  if (NumCols == 1) {
    assert(!isVectorType(SpvRowType));
    for (uint32_t I = 0; I < NumRows; ++I)
      Rows.push_back(MRI.createGenericVirtualRegister(VecTy));
    Builder.buildUnmerge(Rows, MatrixReg);
    for (Register R : Rows) {
      setRegClassType(R, SpvRowType, GR, &MRI, Builder.getMF());
    }
    return Rows;
  }

  // If the matrix is a single row return that row.
  if (NumRows == 1) {
    return {MatrixReg};
  }

  for (uint32_t I = 0; I < NumRows; ++I) {
    SmallVector<int, 4> Mask;
    for (uint32_t k = 0; k < NumCols; ++k)
      Mask.push_back(k * NumRows + I);
    Rows.push_back(Builder.buildShuffleVector(VecTy, MatrixReg, MatrixReg, Mask)
                       .getReg(0));
  }
  for (Register R : Rows) {
    setRegClassType(R, SpvRowType, GR, &MRI, Builder.getMF());
  }
  return Rows;
}

Register SPIRVCombinerHelper::computeDotProduct(Register RowA, Register ColB,
                                                SPIRVTypeInst SpvVecType,
                                                SPIRVGlobalRegistry *GR) const {
  SPIRVTypeInst SpvScalarType = GR->getScalarOrVectorComponentType(SpvVecType);
  bool IsFloatOp = SpvScalarType->getOpcode() == SPIRV::OpTypeFloat;
  LLT VecTy = GR->getRegType(SpvVecType);

  Register DotRes;
  if (isVectorType(SpvVecType)) {
    LLT ScalarTy = VecTy.getElementType();
    Intrinsic::SPVIntrinsics DotIntrinsic =
        (IsFloatOp ? Intrinsic::spv_fdot : Intrinsic::spv_udot);
    DotRes = Builder.buildIntrinsic(DotIntrinsic, {ScalarTy})
                 .addUse(RowA)
                 .addUse(ColB)
                 .getReg(0);
  } else {
    if (IsFloatOp)
      DotRes = Builder.buildFMul(VecTy, RowA, ColB).getReg(0);
    else
      DotRes = Builder.buildMul(VecTy, RowA, ColB).getReg(0);
  }
  setRegClassType(DotRes, SpvScalarType, GR, &MRI, Builder.getMF());
  return DotRes;
}

SmallVector<Register, 16> SPIRVCombinerHelper::computeDotProducts(
    ArrayRef<Register> RowsA, ArrayRef<Register> ColsB,
    SPIRVTypeInst SpvVecType, SPIRVGlobalRegistry *GR) const {
  SmallVector<Register, 16> ResultScalars;
  for (uint32_t J = 0; J < ColsB.size(); ++J) {
    for (uint32_t I = 0; I < RowsA.size(); ++I) {
      ResultScalars.push_back(
          computeDotProduct(RowsA[I], ColsB[J], SpvVecType, GR));
    }
  }
  return ResultScalars;
}

SPIRVTypeInst
SPIRVCombinerHelper::getDotProductVectorType(Register ResReg, uint32_t K,
                                             SPIRVGlobalRegistry *GR) const {
  // Loop over all non debug uses of ResReg
  Type *ScalarResType = nullptr;
  for (auto &UseMI : MRI.use_instructions(ResReg)) {
    if (UseMI.getOpcode() != TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS)
      continue;

    if (!isSpvIntrinsic(UseMI, Intrinsic::spv_assign_type))
      continue;

    Type *Ty = getMDOperandAsType(UseMI.getOperand(2).getMetadata(), 0);
    if (Ty->isVectorTy())
      ScalarResType = cast<VectorType>(Ty)->getElementType();
    else
      ScalarResType = Ty;
    assert(ScalarResType->isIntegerTy() || ScalarResType->isFloatingPointTy());
    break;
  }
  if (!ScalarResType)
    llvm_unreachable("Could not determine scalar result type");
  Type *VecType =
      (K > 1 ? FixedVectorType::get(ScalarResType, K) : ScalarResType);
  return GR->getOrCreateSPIRVType(VecType, Builder,
                                  SPIRV::AccessQualifier::None, false);
}

void SPIRVCombinerHelper::applyMatrixMultiply(MachineInstr &MI) const {
  Register ResReg = MI.getOperand(0).getReg();
  Register AReg = MI.getOperand(2).getReg();
  Register BReg = MI.getOperand(3).getReg();
  uint32_t NumRowsA = MI.getOperand(4).getImm();
  uint32_t NumColsA = MI.getOperand(5).getImm();
  uint32_t NumColsB = MI.getOperand(6).getImm();

  Builder.setInstrAndDebugLoc(MI);

  SPIRVGlobalRegistry *GR =
      MI.getMF()->getSubtarget<SPIRVSubtarget>().getSPIRVGlobalRegistry();

  SPIRVTypeInst SpvVecType = getDotProductVectorType(ResReg, NumColsA, GR);
  SmallVector<Register, 4> ColsB =
      extractColumns(BReg, NumColsB, SpvVecType, GR);
  SmallVector<Register, 4> RowsA =
      extractRows(AReg, NumRowsA, NumColsA, SpvVecType, GR);
  SmallVector<Register, 16> ResultScalars =
      computeDotProducts(RowsA, ColsB, SpvVecType, GR);

  if (ResultScalars.size() == 1)
    Builder.buildCopy(ResReg, ResultScalars[0]);
  else
    Builder.buildBuildVector(ResReg, ResultScalars);
  MI.eraseFromParent();
}