//===- AMDGPUBaseInfo.cpp - AMDGPU Base encoding information --------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//

#include "AMDGPUBaseInfo.h"
#include "AMDGPU.h"
#include "AMDGPUAsmUtils.h"
#include "AMDKernelCodeT.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "Utils/AMDKernelCodeTUtils.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/BinaryFormat/ELF.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/IR/IntrinsicsR600.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/Metadata.h"
#include "llvm/MC/MCInstrInfo.h"
#include "llvm/MC/MCRegisterInfo.h"
#include "llvm/MC/MCSubtargetInfo.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/TargetParser/AMDGPUTargetParser.h"
#include <optional>

#define GET_INSTRINFO_NAMED_OPS
#define GET_INSTRMAP_INFO
#include "AMDGPUGenInstrInfo.inc"

static llvm::cl::opt<unsigned> DefaultAMDHSACodeObjectVersion(
    "amdhsa-code-object-version", llvm::cl::Hidden,
    llvm::cl::init(llvm::AMDGPU::AMDHSA_COV6),
    llvm::cl::desc("Set default AMDHSA Code Object Version (module flag "
                   "or asm directive still take priority if present)"));

namespace {

/// \returns Bit mask for given bit \p Shift and bit \p Width.
unsigned getBitMask(unsigned Shift, unsigned Width) {
  return ((1 << Width) - 1) << Shift;
}

/// Packs \p Src into \p Dst for given bit \p Shift and bit \p Width.
///
/// \returns Packed \p Dst.
unsigned packBits(unsigned Src, unsigned Dst, unsigned Shift, unsigned Width) {
  unsigned Mask = getBitMask(Shift, Width);
  return ((Src << Shift) & Mask) | (Dst & ~Mask);
}

/// Unpacks bits from \p Src for given bit \p Shift and bit \p Width.
///
/// \returns Unpacked bits.
unsigned unpackBits(unsigned Src, unsigned Shift, unsigned Width) {
  return (Src & getBitMask(Shift, Width)) >> Shift;
}

/// \returns Vmcnt bit shift (lower bits).
unsigned getVmcntBitShiftLo(unsigned VersionMajor) {
  return VersionMajor >= 11 ? 10 : 0;
}

/// \returns Vmcnt bit width (lower bits).
unsigned getVmcntBitWidthLo(unsigned VersionMajor) {
  return VersionMajor >= 11 ? 6 : 4;
}

/// \returns Expcnt bit shift.
unsigned getExpcntBitShift(unsigned VersionMajor) {
  return VersionMajor >= 11 ? 0 : 4;
}

/// \returns Expcnt bit width.
unsigned getExpcntBitWidth(unsigned VersionMajor) { return 3; }

/// \returns Lgkmcnt bit shift.
unsigned getLgkmcntBitShift(unsigned VersionMajor) {
  return VersionMajor >= 11 ? 4 : 8;
}

/// \returns Lgkmcnt bit width.
unsigned getLgkmcntBitWidth(unsigned VersionMajor) {
  return VersionMajor >= 10 ? 6 : 4;
}

/// \returns Vmcnt bit shift (higher bits).
unsigned getVmcntBitShiftHi(unsigned VersionMajor) { return 14; }

/// \returns Vmcnt bit width (higher bits).
unsigned getVmcntBitWidthHi(unsigned VersionMajor) {
  return (VersionMajor == 9 || VersionMajor == 10) ? 2 : 0;
}

/// \returns Loadcnt bit width
unsigned getLoadcntBitWidth(unsigned VersionMajor) {
  return VersionMajor >= 12 ? 6 : 0;
}

/// \returns Samplecnt bit width.
unsigned getSamplecntBitWidth(unsigned VersionMajor) {
  return VersionMajor >= 12 ? 6 : 0;
}

/// \returns Bvhcnt bit width.
unsigned getBvhcntBitWidth(unsigned VersionMajor) {
  return VersionMajor >= 12 ? 3 : 0;
}

/// \returns Dscnt bit width.
unsigned getDscntBitWidth(unsigned VersionMajor) {
  return VersionMajor >= 12 ? 6 : 0;
}

/// \returns Dscnt bit shift in combined S_WAIT instructions.
unsigned getDscntBitShift(unsigned VersionMajor) { return 0; }

/// \returns Storecnt or Vscnt bit width, depending on VersionMajor.
unsigned getStorecntBitWidth(unsigned VersionMajor) {
  return VersionMajor >= 10 ? 6 : 0;
}

/// \returns Kmcnt bit width.
unsigned getKmcntBitWidth(unsigned VersionMajor) {
  return VersionMajor >= 12 ? 5 : 0;
}

/// \returns Xcnt bit width.
unsigned getXcntBitWidth(unsigned VersionMajor, unsigned VersionMinor) {
  return VersionMajor == 12 && VersionMinor == 5 ? 6 : 0;
}

/// \returns Asynccnt bit width.
unsigned getAsynccntBitWidth(unsigned VersionMajor, unsigned VersionMinor) {
  return VersionMajor == 12 && VersionMinor == 5 ? 6 : 0;
}

/// \returns shift for Loadcnt/Storecnt in combined S_WAIT instructions.
unsigned getLoadcntStorecntBitShift(unsigned VersionMajor) {
  return VersionMajor >= 12 ? 8 : 0;
}

/// \returns barrier member count bit width in M0.
unsigned getBarrierMemberCountBitWidth(unsigned VersionMajor) {
  return VersionMajor >= 13 ? 8 : 6;
}

/// \returns VaSdst bit width
inline unsigned getVaSdstBitWidth() { return 3; }

/// \returns VaSdst bit shift
inline unsigned getVaSdstBitShift() { return 9; }

/// \returns VmVsrc bit width
inline unsigned getVmVsrcBitWidth() { return 3; }

/// \returns VmVsrc bit shift
inline unsigned getVmVsrcBitShift() { return 2; }

/// \returns VaVdst bit width
inline unsigned getVaVdstBitWidth() { return 4; }

/// \returns VaVdst bit shift
inline unsigned getVaVdstBitShift() { return 12; }

/// \returns VaVcc bit width
inline unsigned getVaVccBitWidth() { return 1; }

/// \returns VaVcc bit shift
inline unsigned getVaVccBitShift() { return 1; }

/// \returns SaSdst bit width
inline unsigned getSaSdstBitWidth() { return 1; }

/// \returns SaSdst bit shift
inline unsigned getSaSdstBitShift() { return 0; }

/// \returns VaSsrc width
inline unsigned getVaSsrcBitWidth() { return 1; }

/// \returns VaSsrc bit shift
inline unsigned getVaSsrcBitShift() { return 8; }

/// \returns HoldCnt bit shift
inline unsigned getHoldCntWidth(unsigned VersionMajor, unsigned VersionMinor) {
  static constexpr const unsigned MinMajor = 10;
  static constexpr const unsigned MinMinor = 3;
  return std::tie(VersionMajor, VersionMinor) >= std::tie(MinMajor, MinMinor)
             ? 1
             : 0;
}

/// \returns HoldCnt bit shift
inline unsigned getHoldCntBitShift() { return 7; }

} // end anonymous namespace

namespace llvm {

namespace AMDGPU {

/// \returns true if the target supports signed immediate offset for SMRD
/// instructions.
bool hasSMRDSignedImmOffset(const MCSubtargetInfo &ST) {
  return isGFX9Plus(ST);
}

/// \returns True if \p STI is AMDHSA.
bool isHsaAbi(const MCSubtargetInfo &STI) {
  return STI.getTargetTriple().getOS() == Triple::AMDHSA;
}

unsigned getAMDHSACodeObjectVersion(const Module &M) {
  if (auto *Ver = mdconst::extract_or_null<ConstantInt>(
          M.getModuleFlag("amdhsa_code_object_version"))) {
    return (unsigned)Ver->getZExtValue() / 100;
  }

  return getDefaultAMDHSACodeObjectVersion();
}

unsigned getDefaultAMDHSACodeObjectVersion() {
  return DefaultAMDHSACodeObjectVersion;
}

unsigned getAMDHSACodeObjectVersion(unsigned ABIVersion) {
  switch (ABIVersion) {
  case ELF::ELFABIVERSION_AMDGPU_HSA_V4:
    return 4;
  case ELF::ELFABIVERSION_AMDGPU_HSA_V5:
    return 5;
  case ELF::ELFABIVERSION_AMDGPU_HSA_V6:
    return 6;
  default:
    return getDefaultAMDHSACodeObjectVersion();
  }
}

uint8_t getELFABIVersion(const Triple &T, unsigned CodeObjectVersion) {
  if (T.getOS() != Triple::AMDHSA)
    return 0;

  switch (CodeObjectVersion) {
  case 4:
    return ELF::ELFABIVERSION_AMDGPU_HSA_V4;
  case 5:
    return ELF::ELFABIVERSION_AMDGPU_HSA_V5;
  case 6:
    return ELF::ELFABIVERSION_AMDGPU_HSA_V6;
  default:
    report_fatal_error("Unsupported AMDHSA Code Object Version " +
                       Twine(CodeObjectVersion));
  }
}

unsigned getMultigridSyncArgImplicitArgPosition(unsigned CodeObjectVersion) {
  switch (CodeObjectVersion) {
  case AMDHSA_COV4:
    return 48;
  case AMDHSA_COV5:
  case AMDHSA_COV6:
  default:
    return AMDGPU::ImplicitArg::MULTIGRID_SYNC_ARG_OFFSET;
  }
}

// FIXME: All such magic numbers about the ABI should be in a
// central TD file.
unsigned getHostcallImplicitArgPosition(unsigned CodeObjectVersion) {
  switch (CodeObjectVersion) {
  case AMDHSA_COV4:
    return 24;
  case AMDHSA_COV5:
  case AMDHSA_COV6:
  default:
    return AMDGPU::ImplicitArg::HOSTCALL_PTR_OFFSET;
  }
}

unsigned getDefaultQueueImplicitArgPosition(unsigned CodeObjectVersion) {
  switch (CodeObjectVersion) {
  case AMDHSA_COV4:
    return 32;
  case AMDHSA_COV5:
  case AMDHSA_COV6:
  default:
    return AMDGPU::ImplicitArg::DEFAULT_QUEUE_OFFSET;
  }
}

unsigned getCompletionActionImplicitArgPosition(unsigned CodeObjectVersion) {
  switch (CodeObjectVersion) {
  case AMDHSA_COV4:
    return 40;
  case AMDHSA_COV5:
  case AMDHSA_COV6:
  default:
    return AMDGPU::ImplicitArg::COMPLETION_ACTION_OFFSET;
  }
}

#define GET_MIMGBaseOpcodesTable_IMPL
#define GET_MIMGDimInfoTable_IMPL
#define GET_MIMGInfoTable_IMPL
#define GET_MIMGLZMappingTable_IMPL
#define GET_MIMGMIPMappingTable_IMPL
#define GET_MIMGBiasMappingTable_IMPL
#define GET_MIMGOffsetMappingTable_IMPL
#define GET_MIMGG16MappingTable_IMPL
#define GET_MAIInstInfoTable_IMPL
#define GET_WMMAInstInfoTable_IMPL
#include "AMDGPUGenSearchableTables.inc"

int getMIMGOpcode(unsigned BaseOpcode, unsigned MIMGEncoding,
                  unsigned VDataDwords, unsigned VAddrDwords, bool IndexedRsrc,
                  bool IndexedSamp) {
  const MIMGInfo *Info =
      getMIMGOpcodeHelper(BaseOpcode, MIMGEncoding, VDataDwords, VAddrDwords,
                          IndexedRsrc, IndexedSamp);
  return Info ? Info->Opcode : -1;
}

const MIMGBaseOpcodeInfo *getMIMGBaseOpcode(unsigned Opc) {
  const MIMGInfo *Info = getMIMGInfo(Opc);
  return Info ? getMIMGBaseOpcodeInfo(Info->BaseOpcode) : nullptr;
}

int getMaskedMIMGOp(unsigned Opc, unsigned NewChannels) {
  const MIMGInfo *OrigInfo = getMIMGInfo(Opc);
  const MIMGInfo *NewInfo = getMIMGOpcodeHelper(
      OrigInfo->BaseOpcode, OrigInfo->MIMGEncoding, NewChannels,
      OrigInfo->VAddrDwords, OrigInfo->IndexedRsrc, OrigInfo->IndexedSamp);
  return NewInfo ? NewInfo->Opcode : -1;
}

unsigned getAddrSizeMIMGOp(const MIMGBaseOpcodeInfo *BaseOpcode,
                           const MIMGDimInfo *Dim, bool IsA16,
                           bool IsG16Supported) {
  unsigned AddrWords = BaseOpcode->NumExtraArgs;
  unsigned AddrComponents = (BaseOpcode->Coordinates ? Dim->NumCoords : 0) +
                            (BaseOpcode->LodOrClampOrMip ? 1 : 0);
  if (IsA16)
    AddrWords += divideCeil(AddrComponents, 2);
  else
    AddrWords += AddrComponents;

  // Note: For subtargets that support A16 but not G16, enabling A16 also
  // enables 16 bit gradients.
  // For subtargets that support A16 (operand) and G16 (done with a different
  // instruction encoding), they are independent.

  if (BaseOpcode->Gradients) {
    if ((IsA16 && !IsG16Supported) || BaseOpcode->G16)
      // There are two gradients per coordinate, we pack them separately.
      // For the 3d case,
      // we get (dy/du, dx/du) (-, dz/du) (dy/dv, dx/dv) (-, dz/dv)
      AddrWords += alignTo<2>(Dim->NumGradients / 2);
    else
      AddrWords += Dim->NumGradients;
  }
  return AddrWords;
}

struct MUBUFInfo {
  uint32_t Opcode;
  uint32_t BaseOpcode;
  uint8_t elements;
  bool has_vaddr;
  bool has_srsrc;
  bool has_soffset;
  bool IsBufferInv;
  bool tfe;
};

struct MTBUFInfo {
  uint32_t Opcode;
  uint32_t BaseOpcode;
  uint8_t elements;
  bool has_vaddr;
  bool has_srsrc;
  bool has_soffset;
};

struct SMInfo {
  uint32_t Opcode;
};

struct VOPInfo {
  uint32_t Opcode;
};

struct VOPC64DPPInfo {
  uint32_t Opcode;
};

struct VOPCDPPAsmOnlyInfo {
  uint32_t Opcode;
};

struct VOPDComponentInfo {
  uint16_t BaseVOP;
  uint16_t VOPDOp;
};

struct VOPDInfo {
  uint32_t Opcode;
  uint16_t OpX;
  uint16_t OpY;
  uint16_t Subtarget;
  bool VOPD3;
};

struct VOPTrue16Info {
  uint32_t Opcode;
};

struct VOPDXYInfo {
  uint16_t VOPDXYKey;
  bool IsX;
  bool IsY;
};

#define GET_FP4FP8DstByteSelTable_DECL
#define GET_FP4FP8DstByteSelTable_IMPL

struct DPMACCInstructionInfo {
  uint32_t Opcode;
};

struct FP4FP8DstByteSelInfo {
  uint32_t Opcode;
  bool HasFP8DstByteSel;
  bool HasFP4DstByteSel;
};

struct VOP3PHasNoOpSelHiInfo {
  uint32_t Opcode;
};

#define GET_DPMACCInstructionTable_DECL
#define GET_DPMACCInstructionTable_IMPL
#define GET_MTBUFInfoTable_DECL
#define GET_MTBUFInfoTable_IMPL
#define GET_MUBUFInfoTable_DECL
#define GET_MUBUFInfoTable_IMPL
#define GET_SMInfoTable_DECL
#define GET_SMInfoTable_IMPL
#define GET_VOP1InfoTable_DECL
#define GET_VOP1InfoTable_IMPL
#define GET_VOP2InfoTable_DECL
#define GET_VOP2InfoTable_IMPL
#define GET_VOP3InfoTable_DECL
#define GET_VOP3InfoTable_IMPL
#define GET_VOPC64DPPTable_DECL
#define GET_VOPC64DPPTable_IMPL
#define GET_VOPC64DPP8Table_DECL
#define GET_VOPC64DPP8Table_IMPL
#define GET_VOPCAsmOnlyInfoTable_DECL
#define GET_VOPCAsmOnlyInfoTable_IMPL
#define GET_VOP3CAsmOnlyInfoTable_DECL
#define GET_VOP3CAsmOnlyInfoTable_IMPL
#define GET_VOPDComponentTable_DECL
#define GET_VOPDComponentTable_IMPL
#define GET_VOPDPairs_DECL
#define GET_VOPDPairs_IMPL
#define GET_VOPDXYTable_DECL
#define GET_VOPDXYTable_IMPL
#define GET_VOPTrue16Table_DECL
#define GET_VOPTrue16Table_IMPL
#define GET_VOP3PHasNoOpSelHiTable_DECL
#define GET_VOP3PHasNoOpSelHiTable_IMPL
#define GET_True16D16Table_IMPL
#define GET_WMMAOpcode2AddrMappingTable_DECL
#define GET_WMMAOpcode2AddrMappingTable_IMPL
#define GET_WMMAOpcode3AddrMappingTable_DECL
#define GET_WMMAOpcode3AddrMappingTable_IMPL
#define GET_getMFMA_F8F6F4_WithSize_DECL
#define GET_getMFMA_F8F6F4_WithSize_IMPL
#define GET_isMFMA_F8F6F4Table_IMPL
#define GET_isCvtScaleF32_F32F16ToF8F4Table_IMPL

#include "AMDGPUGenSearchableTables.inc"

int getMTBUFBaseOpcode(unsigned Opc) {
  const MTBUFInfo *Info = getMTBUFInfoFromOpcode(Opc);
  return Info ? Info->BaseOpcode : -1;
}

int getMTBUFOpcode(unsigned BaseOpc, unsigned Elements) {
  const MTBUFInfo *Info =
      getMTBUFInfoFromBaseOpcodeAndElements(BaseOpc, Elements);
  return Info ? Info->Opcode : -1;
}

int getMTBUFElements(unsigned Opc) {
  const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc);
  return Info ? Info->elements : 0;
}

bool getMTBUFHasVAddr(unsigned Opc) {
  const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc);
  return Info && Info->has_vaddr;
}

bool getMTBUFHasSrsrc(unsigned Opc) {
  const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc);
  return Info && Info->has_srsrc;
}

bool getMTBUFHasSoffset(unsigned Opc) {
  const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc);
  return Info && Info->has_soffset;
}

int getMUBUFBaseOpcode(unsigned Opc) {
  const MUBUFInfo *Info = getMUBUFInfoFromOpcode(Opc);
  return Info ? Info->BaseOpcode : -1;
}

int getMUBUFOpcode(unsigned BaseOpc, unsigned Elements) {
  const MUBUFInfo *Info =
      getMUBUFInfoFromBaseOpcodeAndElements(BaseOpc, Elements);
  return Info ? Info->Opcode : -1;
}

int getMUBUFElements(unsigned Opc) {
  const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc);
  return Info ? Info->elements : 0;
}

bool getMUBUFHasVAddr(unsigned Opc) {
  const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc);
  return Info && Info->has_vaddr;
}

bool getMUBUFHasSrsrc(unsigned Opc) {
  const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc);
  return Info && Info->has_srsrc;
}

bool getMUBUFHasSoffset(unsigned Opc) {
  const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc);
  return Info && Info->has_soffset;
}

bool getMUBUFIsBufferInv(unsigned Opc) {
  const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc);
  return Info && Info->IsBufferInv;
}

bool getMUBUFTfe(unsigned Opc) {
  const MUBUFInfo *Info = getMUBUFOpcodeHelper(Opc);
  return Info && Info->tfe;
}

bool getSMEMIsBuffer(unsigned Opc) {
  return isSMEMOpcodeHelper(Opc) != nullptr;
}

bool getVOP1IsSingle(unsigned Opc) {
  return isVOP1SingleOpcodeHelper(Opc) != nullptr;
}

bool getVOP2IsSingle(unsigned Opc) {
  return isVOP2SingleOpcodeHelper(Opc) != nullptr;
}

bool getVOP3IsSingle(unsigned Opc) {
  return isVOP3SingleOpcodeHelper(Opc) != nullptr;
}

bool isVOPC64DPP(unsigned Opc) {
  return isVOPC64DPPOpcodeHelper(Opc) || isVOPC64DPP8OpcodeHelper(Opc);
}

bool isVOPCAsmOnly(unsigned Opc) { return isVOPCAsmOnlyOpcodeHelper(Opc); }

bool getMAIIsDGEMM(unsigned Opc) {
  const MAIInstInfo *Info = getMAIInstInfoHelper(Opc);
  return Info && Info->is_dgemm;
}

bool getMAIIsGFX940XDL(unsigned Opc) {
  const MAIInstInfo *Info = getMAIInstInfoHelper(Opc);
  return Info && Info->is_gfx940_xdl;
}

bool getWMMAIsXDL(unsigned Opc) {
  const WMMAInstInfo *Info = getWMMAInstInfoHelper(Opc);
  return Info ? Info->is_wmma_xdl : false;
}

bool getHasMatrixScale(unsigned Opc) {
  const WMMAInstInfo *Info = getWMMAInstInfoHelper(Opc);
  return Info && Info->HasMatrixScale;
}

uint8_t mfmaScaleF8F6F4FormatToNumRegs(unsigned EncodingVal) {
  switch (EncodingVal) {
  case MFMAScaleFormats::FP6_E2M3:
  case MFMAScaleFormats::FP6_E3M2:
    return 6;
  case MFMAScaleFormats::FP4_E2M1:
    return 4;
  case MFMAScaleFormats::FP8_E4M3:
  case MFMAScaleFormats::FP8_E5M2:
  default:
    return 8;
  }

  llvm_unreachable("covered switch over mfma scale formats");
}

const MFMA_F8F6F4_Info *getMFMA_F8F6F4_WithFormatArgs(unsigned CBSZ,
                                                      unsigned BLGP,
                                                      unsigned F8F8Opcode) {
  uint8_t SrcANumRegs = mfmaScaleF8F6F4FormatToNumRegs(CBSZ);
  uint8_t SrcBNumRegs = mfmaScaleF8F6F4FormatToNumRegs(BLGP);
  return getMFMA_F8F6F4_InstWithNumRegs(SrcANumRegs, SrcBNumRegs, F8F8Opcode);
}

uint8_t wmmaScaleF8F6F4FormatToNumRegs(unsigned Fmt) {
  switch (Fmt) {
  case WMMA::MATRIX_FMT_FP8:
  case WMMA::MATRIX_FMT_BF8:
    return 16;
  case WMMA::MATRIX_FMT_FP6:
  case WMMA::MATRIX_FMT_BF6:
    return 12;
  case WMMA::MATRIX_FMT_FP4:
    return 8;
  }

  llvm_unreachable("covered switch over wmma scale formats");
}

const MFMA_F8F6F4_Info *getWMMA_F8F6F4_WithFormatArgs(unsigned FmtA,
                                                      unsigned FmtB,
                                                      unsigned F8F8Opcode) {
  uint8_t SrcANumRegs = wmmaScaleF8F6F4FormatToNumRegs(FmtA);
  uint8_t SrcBNumRegs = wmmaScaleF8F6F4FormatToNumRegs(FmtB);
  return getMFMA_F8F6F4_InstWithNumRegs(SrcANumRegs, SrcBNumRegs, F8F8Opcode);
}

bool isValidWMMAScaleFmtCombination(unsigned AFmt, unsigned AScale,
                                    unsigned BFmt, unsigned BScale) {
  auto isValid = [](unsigned Fmt, unsigned Scale) -> bool {
    switch (Fmt) {
    case WMMA::MATRIX_FMT_FP8:
    case WMMA::MATRIX_FMT_BF8:
    case WMMA::MATRIX_FMT_FP6:
    case WMMA::MATRIX_FMT_BF6:
      if (Scale != WMMA::MATRIX_SCALE_FMT_E8)
        return false;
      break;
    case WMMA::MATRIX_FMT_FP4:
      if (Scale != WMMA::MATRIX_SCALE_FMT_E8 &&
          Scale != WMMA::MATRIX_SCALE_FMT_E5M3 &&
          Scale != WMMA::MATRIX_SCALE_FMT_E4M3)
        return false;
      break;
    }
    return true;
  };

  if (!isValid(AFmt, AScale) || !isValid(BFmt, BScale))
    return false;

  if (AFmt == WMMA::MATRIX_FMT_FP4 && BFmt == WMMA::MATRIX_FMT_FP4 &&
      AScale != BScale)
    return false;

  return true;
}

unsigned getVOPDEncodingFamily(const MCSubtargetInfo &ST) {
  if (ST.hasFeature(AMDGPU::FeatureGFX13Insts))
    return SIEncodingFamily::GFX13;
  if (ST.hasFeature(AMDGPU::FeatureGFX1250Insts))
    return SIEncodingFamily::GFX1250;
  if (ST.hasFeature(AMDGPU::FeatureGFX12Insts))
    return SIEncodingFamily::GFX12;
  if (ST.hasFeature(AMDGPU::FeatureGFX11_7Insts))
    return SIEncodingFamily::GFX1170;
  if (ST.hasFeature(AMDGPU::FeatureGFX11Insts))
    return SIEncodingFamily::GFX11;
  llvm_unreachable("Subtarget generation does not support VOPD!");
}

CanBeVOPD getCanBeVOPD(unsigned Opc, unsigned EncodingFamily, bool VOPD3) {
  bool IsConvertibleToBitOp = VOPD3 ? getBitOp2(Opc) : 0;
  Opc = IsConvertibleToBitOp ? (unsigned)AMDGPU::V_BITOP3_B32_e64 : Opc;
  // Normalize through VOPDComponentTable so that e32 and e64 variants
  // of the same logical opcode all share a single entry.
  const VOPDComponentInfo *Info = getVOPDComponentHelper(Opc);
  if (!Info)
    return {false, false};
  unsigned Key =
      (Info->VOPDOp << 5) | (EncodingFamily << 1) | (VOPD3 ? 1u : 0u);
  const VOPDXYInfo *XYInfo = getVOPDXYInfo(static_cast<uint16_t>(Key));
  if (!XYInfo)
    return {false, false};
  return {XYInfo->IsX, XYInfo->IsY};
}

unsigned getVOPDOpcode(unsigned Opc, bool VOPD3) {
  bool IsConvertibleToBitOp = VOPD3 ? getBitOp2(Opc) : 0;
  Opc = IsConvertibleToBitOp ? (unsigned)AMDGPU::V_BITOP3_B32_e64 : Opc;
  const VOPDComponentInfo *Info = getVOPDComponentHelper(Opc);
  return Info ? Info->VOPDOp : ~0u;
}

bool isVOPD(unsigned Opc) {
  return AMDGPU::hasNamedOperand(Opc, AMDGPU::OpName::src0X);
}

bool isMAC(unsigned Opc) {
  return Opc == AMDGPU::V_MAC_F32_e64_gfx6_gfx7 ||
         Opc == AMDGPU::V_MAC_F32_e64_gfx10 ||
         Opc == AMDGPU::V_MAC_F32_e64_vi ||
         Opc == AMDGPU::V_MAC_LEGACY_F32_e64_gfx6_gfx7 ||
         Opc == AMDGPU::V_MAC_LEGACY_F32_e64_gfx10 ||
         Opc == AMDGPU::V_MAC_F16_e64_vi ||
         Opc == AMDGPU::V_FMAC_F64_e64_gfx90a ||
         Opc == AMDGPU::V_FMAC_F64_e64_gfx12 ||
         Opc == AMDGPU::V_FMAC_F64_e64_gfx13 ||
         Opc == AMDGPU::V_FMAC_F32_e64_gfx10 ||
         Opc == AMDGPU::V_FMAC_F32_e64_gfx11 ||
         Opc == AMDGPU::V_FMAC_F32_e64_gfx12 ||
         Opc == AMDGPU::V_FMAC_F32_e64_gfx13 ||
         Opc == AMDGPU::V_FMAC_F32_e64_vi ||
         Opc == AMDGPU::V_FMAC_LEGACY_F32_e64_gfx10 ||
         Opc == AMDGPU::V_FMAC_DX9_ZERO_F32_e64_gfx11 ||
         Opc == AMDGPU::V_FMAC_F16_e64_gfx10 ||
         Opc == AMDGPU::V_FMAC_F16_t16_e64_gfx11 ||
         Opc == AMDGPU::V_FMAC_F16_fake16_e64_gfx11 ||
         Opc == AMDGPU::V_FMAC_F16_t16_e64_gfx12 ||
         Opc == AMDGPU::V_FMAC_F16_fake16_e64_gfx12 ||
         Opc == AMDGPU::V_FMAC_F16_t16_e64_gfx13 ||
         Opc == AMDGPU::V_FMAC_F16_fake16_e64_gfx13 ||
         Opc == AMDGPU::V_DOT2C_F32_F16_e64_vi ||
         Opc == AMDGPU::V_DOT2C_F32_BF16_e64_vi ||
         Opc == AMDGPU::V_DOT2C_I32_I16_e64_vi ||
         Opc == AMDGPU::V_DOT4C_I32_I8_e64_vi ||
         Opc == AMDGPU::V_DOT8C_I32_I4_e64_vi;
}

bool isPermlane16(unsigned Opc) {
  return Opc == AMDGPU::V_PERMLANE16_B32_gfx10 ||
         Opc == AMDGPU::V_PERMLANEX16_B32_gfx10 ||
         Opc == AMDGPU::V_PERMLANE16_B32_e64_gfx11 ||
         Opc == AMDGPU::V_PERMLANEX16_B32_e64_gfx11 ||
         Opc == AMDGPU::V_PERMLANE16_B32_e64_gfx12 ||
         Opc == AMDGPU::V_PERMLANE16_B32_e64_gfx13 ||
         Opc == AMDGPU::V_PERMLANEX16_B32_e64_gfx12 ||
         Opc == AMDGPU::V_PERMLANEX16_B32_e64_gfx13 ||
         Opc == AMDGPU::V_PERMLANE16_VAR_B32_e64_gfx12 ||
         Opc == AMDGPU::V_PERMLANE16_VAR_B32_e64_gfx13 ||
         Opc == AMDGPU::V_PERMLANEX16_VAR_B32_e64_gfx12 ||
         Opc == AMDGPU::V_PERMLANEX16_VAR_B32_e64_gfx13;
}

bool isCvt_F32_Fp8_Bf8_e64(unsigned Opc) {
  return Opc == AMDGPU::V_CVT_F32_BF8_e64_gfx12 ||
         Opc == AMDGPU::V_CVT_F32_FP8_e64_gfx12 ||
         Opc == AMDGPU::V_CVT_F32_BF8_e64_dpp_gfx12 ||
         Opc == AMDGPU::V_CVT_F32_FP8_e64_dpp_gfx12 ||
         Opc == AMDGPU::V_CVT_F32_BF8_e64_dpp8_gfx12 ||
         Opc == AMDGPU::V_CVT_F32_FP8_e64_dpp8_gfx12 ||
         Opc == AMDGPU::V_CVT_PK_F32_BF8_fake16_e64_gfx12 ||
         Opc == AMDGPU::V_CVT_PK_F32_FP8_fake16_e64_gfx12 ||
         Opc == AMDGPU::V_CVT_PK_F32_BF8_t16_e64_gfx12 ||
         Opc == AMDGPU::V_CVT_PK_F32_FP8_t16_e64_gfx12;
}

bool isGenericAtomic(unsigned Opc) {
  return Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SWAP ||
         Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_ADD ||
         Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SUB ||
         Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMIN ||
         Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMIN ||
         Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMAX ||
         Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMAX ||
         Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_AND ||
         Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_OR ||
         Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_XOR ||
         Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_INC ||
         Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_DEC ||
         Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FADD ||
         Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FMIN ||
         Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FMAX ||
         Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_CMPSWAP ||
         Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SUB_CLAMP_U32 ||
         Opc == AMDGPU::G_AMDGPU_BUFFER_ATOMIC_COND_SUB_U32 ||
         Opc == AMDGPU::G_AMDGPU_ATOMIC_CMPXCHG;
}

bool isAsyncStore(unsigned Opc) {
  return Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B8_gfx1250 ||
         Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B32_gfx1250 ||
         Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B64_gfx1250 ||
         Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B128_gfx1250 ||
         Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B8_SADDR_gfx1250 ||
         Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B32_SADDR_gfx1250 ||
         Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B64_SADDR_gfx1250 ||
         Opc == GLOBAL_STORE_ASYNC_FROM_LDS_B128_SADDR_gfx1250;
}

bool isTensorStore(unsigned Opc) {
  return Opc == TENSOR_STORE_FROM_LDS_d2_gfx1250 ||
         Opc == TENSOR_STORE_FROM_LDS_d4_gfx1250;
}

unsigned getTemporalHintType(const MCInstrDesc TID) {
  if (SIInstrFlags::isAtomic(TID))
    return CPol::TH_TYPE_ATOMIC;
  unsigned Opc = TID.getOpcode();
  // Async and Tensor store should have the temporal hint type of TH_TYPE_STORE
  if (TID.mayStore() &&
      (isAsyncStore(Opc) || isTensorStore(Opc) || !TID.mayLoad()))
    return CPol::TH_TYPE_STORE;

  // This will default to returning TH_TYPE_LOAD when neither MayStore nor
  // MayLoad flag is present which is the case with instructions like
  // image_get_resinfo.
  return CPol::TH_TYPE_LOAD;
}

bool isTrue16Inst(unsigned Opc) { return isTrue16Opcode(Opc) != nullptr; }

bool hasNoOpSelHi(unsigned Opc) {
  return getVOP3PHasNoOpSelHiHelper(Opc) != nullptr;
}

FPType getFPDstSelType(unsigned Opc) {
  const FP4FP8DstByteSelInfo *Info = getFP4FP8DstByteSelHelper(Opc);
  if (!Info)
    return FPType::None;
  if (Info->HasFP8DstByteSel)
    return FPType::FP8;
  if (Info->HasFP4DstByteSel)
    return FPType::FP4;

  return FPType::None;
}

bool isDPMACCInstruction(unsigned Opc) {
  return isDPMACCInstructionHelper(Opc) != nullptr;
}

unsigned mapWMMA2AddrTo3AddrOpcode(unsigned Opc) {
  const WMMAOpcodeMappingInfo *Info = getWMMAMappingInfoFrom2AddrOpcode(Opc);
  return Info ? Info->Opcode3Addr : ~0u;
}

// Wrapper for Tablegen'd function.  enum Subtarget is not defined in any
// header files, so we need to wrap it in a function that takes unsigned
// instead.
int32_t getMCOpcode(uint32_t Opcode, unsigned Gen) {
  return getMCOpcodeGen(Opcode, static_cast<Subtarget>(Gen));
}

unsigned getBitOp2(unsigned Opc) {
  switch (Opc) {
  default:
    return 0;
  case AMDGPU::V_AND_B32_e32:
    return 0x40;
  case AMDGPU::V_OR_B32_e32:
    return 0x54;
  case AMDGPU::V_XOR_B32_e32:
    return 0x14;
  case AMDGPU::V_XNOR_B32_e32:
    return 0x41;
  }
}

int getVOPDFull(unsigned OpX, unsigned OpY, unsigned EncodingFamily,
                bool VOPD3) {
  bool IsConvertibleToBitOp = VOPD3 ? getBitOp2(OpY) : 0;
  OpY = IsConvertibleToBitOp ? (unsigned)AMDGPU::V_BITOP3_B32_e64 : OpY;
  const VOPDInfo *Info = getVOPDInfoFromComponentOpcodes(
      static_cast<uint8_t>(OpX), static_cast<uint8_t>(OpY),
      static_cast<uint8_t>(EncodingFamily), VOPD3);
  return Info ? Info->Opcode : -1;
}

std::pair<unsigned, unsigned> getVOPDComponents(unsigned VOPDOpcode) {
  const VOPDInfo *Info = getVOPDOpcodeHelper(VOPDOpcode);
  assert(Info);
  const auto *OpX = getVOPDBaseFromComponent(Info->OpX);
  const auto *OpY = getVOPDBaseFromComponent(Info->OpY);
  assert(OpX && OpY);
  return {OpX->BaseVOP, OpY->BaseVOP};
}

namespace VOPD {

ComponentProps::ComponentProps(const MCInstrDesc &OpDesc, bool VOP3Layout) {
  assert(OpDesc.getNumDefs() == Component::DST_NUM);

  assert(OpDesc.getOperandConstraint(Component::SRC0, MCOI::TIED_TO) == -1);
  assert(OpDesc.getOperandConstraint(Component::SRC1, MCOI::TIED_TO) == -1);
  auto TiedIdx = OpDesc.getOperandConstraint(Component::SRC2, MCOI::TIED_TO);
  assert(TiedIdx == -1 || TiedIdx == Component::DST);
  HasSrc2Acc = TiedIdx != -1;
  Opcode = OpDesc.getOpcode();

  IsVOP3 = VOP3Layout || SIInstrFlags::isVOP3(OpDesc);
  SrcOperandsNum = AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::src2)   ? 3
                   : AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::imm)  ? 3
                   : AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::src1) ? 2
                                                                           : 1;
  assert(SrcOperandsNum <= Component::MAX_SRC_NUM);

  if (Opcode == AMDGPU::V_CNDMASK_B32_e32 ||
      Opcode == AMDGPU::V_CNDMASK_B32_e64) {
    // CNDMASK is an awkward exception, it has FP modifiers, but not FP
    // operands.
    NumVOPD3Mods = 2;
    if (IsVOP3)
      SrcOperandsNum = 3;
  } else if (Opcode == AMDGPU::V_DOT2_F32_F16 ||
             Opcode == AMDGPU::V_DOT2_F32_BF16) {
    // VOP3P opcodes that have VOPD but don't have VOP2 version. Using VOPD3
    // path in getIndexOfSrcInMCOperands to get correct src operand indexes,
    // but generating VOPD, not VOPD3.
    NumVOPD3Mods = SrcOperandsNum;
  } else if (isSISrcFPOperand(OpDesc,
                              getNamedOperandIdx(Opcode, OpName::src0))) {
    // All FP VOPD instructions have Neg modifiers for all operands except
    // for tied src2.
    NumVOPD3Mods = SrcOperandsNum;
    if (HasSrc2Acc)
      --NumVOPD3Mods;
  }

  if (SIInstrFlags::isVOP3(OpDesc))
    return;

  auto OperandsNum = OpDesc.getNumOperands();
  unsigned CompOprIdx;
  for (CompOprIdx = Component::SRC1; CompOprIdx < OperandsNum; ++CompOprIdx) {
    if (OpDesc.operands()[CompOprIdx].OperandType == AMDGPU::OPERAND_KIMM32) {
      MandatoryLiteralIdx = CompOprIdx;
      break;
    }
  }
}

int ComponentProps::getBitOp3OperandIdx() const {
  return getNamedOperandIdx(Opcode, OpName::bitop3);
}

unsigned ComponentInfo::getIndexInParsedOperands(unsigned CompOprIdx) const {
  assert(CompOprIdx < Component::MAX_OPR_NUM);

  if (CompOprIdx == Component::DST)
    return getIndexOfDstInParsedOperands();

  auto CompSrcIdx = CompOprIdx - Component::DST_NUM;
  if (CompSrcIdx < getCompParsedSrcOperandsNum())
    return getIndexOfSrcInParsedOperands(CompSrcIdx);

  // The specified operand does not exist.
  return 0;
}

std::optional<unsigned> InstInfo::getInvalidCompOperandIndex(
    std::function<MCRegister(unsigned, unsigned)> GetRegIdx,
    const MCRegisterInfo &MRI, bool SkipSrc, bool AllowSameVGPR, bool VOPD3,
    bool HasGFX11InterlockHazard) const {

  auto OpXRegs = getRegIndices(ComponentIndex::X, GetRegIdx,
                               CompInfo[ComponentIndex::X].isVOP3());
  auto OpYRegs = getRegIndices(ComponentIndex::Y, GetRegIdx,
                               CompInfo[ComponentIndex::Y].isVOP3());

  const auto banksOverlap = [&MRI](MCRegister X, MCRegister Y,
                                   unsigned BanksMask) -> bool {
    MCRegister BaseX = MRI.getSubReg(X, AMDGPU::sub0);
    MCRegister BaseY = MRI.getSubReg(Y, AMDGPU::sub0);
    if (!BaseX)
      BaseX = X;
    if (!BaseY)
      BaseY = Y;
    if ((BaseX.id() & BanksMask) == (BaseY.id() & BanksMask))
      return true;
    if (BaseX != X /* This is 64-bit register */ &&
        ((BaseX.id() + 1) & BanksMask) == (BaseY.id() & BanksMask))
      return true;
    if (BaseY != Y &&
        (BaseX.id() & BanksMask) == ((BaseY.id() + 1) & BanksMask))
      return true;

    // If both are 64-bit bank conflict will be detected yet while checking
    // the first subreg.
    return false;
  };

  unsigned CompOprIdx;
  for (CompOprIdx = 0; CompOprIdx < Component::MAX_OPR_NUM; ++CompOprIdx) {
    unsigned BanksMasks = VOPD3 ? VOPD3_VGPR_BANK_MASKS[CompOprIdx]
                          : HasGFX11InterlockHazard
                              ? VOPD_GFX11_VGPR_BANK_MASKS[CompOprIdx]
                              : VOPD_VGPR_BANK_MASKS[CompOprIdx];
    if (!OpXRegs[CompOprIdx] || !OpYRegs[CompOprIdx])
      continue;

    if (getVGPREncodingMSBs(OpXRegs[CompOprIdx], MRI) !=
        getVGPREncodingMSBs(OpYRegs[CompOprIdx], MRI))
      return CompOprIdx;

    if (SkipSrc && CompOprIdx >= Component::DST_NUM)
      continue;

    if (CompOprIdx < Component::DST_NUM) {
      // Even if we do not check vdst parity, vdst operands still shall not
      // overlap.
      if (MRI.regsOverlap(OpXRegs[CompOprIdx], OpYRegs[CompOprIdx]))
        return CompOprIdx;
      if (VOPD3) // No need to check dst parity.
        continue;
    }

    if (banksOverlap(OpXRegs[CompOprIdx], OpYRegs[CompOprIdx], BanksMasks) &&
        (!AllowSameVGPR || CompOprIdx < Component::DST_NUM ||
         OpXRegs[CompOprIdx] != OpYRegs[CompOprIdx]))
      return CompOprIdx;
  }

  return {};
}

// Return an array of VGPR registers [DST,SRC0,SRC1,SRC2] used
// by the specified component. If an operand is unused
// or is not a VGPR, the corresponding value is 0.
//
// GetRegIdx(Component, MCOperandIdx) must return a VGPR register index
// for the specified component and MC operand. The callback must return 0
// if the operand is not a register or not a VGPR.
InstInfo::RegIndices
InstInfo::getRegIndices(unsigned CompIdx,
                        std::function<MCRegister(unsigned, unsigned)> GetRegIdx,
                        bool VOPD3) const {
  assert(CompIdx < COMPONENTS_NUM);

  const auto &Comp = CompInfo[CompIdx];
  InstInfo::RegIndices RegIndices;

  RegIndices[DST] = GetRegIdx(CompIdx, Comp.getIndexOfDstInMCOperands());

  for (unsigned CompOprIdx : {SRC0, SRC1, SRC2}) {
    unsigned CompSrcIdx = CompOprIdx - DST_NUM;
    RegIndices[CompOprIdx] =
        Comp.hasRegSrcOperand(CompSrcIdx)
            ? GetRegIdx(CompIdx,
                        Comp.getIndexOfSrcInMCOperands(CompSrcIdx, VOPD3))
            : MCRegister();
  }
  return RegIndices;
}

} // namespace VOPD

VOPD::InstInfo getVOPDInstInfo(const MCInstrDesc &OpX, const MCInstrDesc &OpY) {
  return VOPD::InstInfo(OpX, OpY);
}

VOPD::InstInfo getVOPDInstInfo(unsigned VOPDOpcode,
                               const MCInstrInfo *InstrInfo) {
  auto [OpX, OpY] = getVOPDComponents(VOPDOpcode);
  const auto &OpXDesc = InstrInfo->get(OpX);
  const auto &OpYDesc = InstrInfo->get(OpY);
  bool VOPD3 = SIInstrFlags::isVOPD3(*InstrInfo, VOPDOpcode);
  VOPD::ComponentInfo OpXInfo(OpXDesc, VOPD::ComponentKind::COMPONENT_X, VOPD3);
  VOPD::ComponentInfo OpYInfo(OpYDesc, OpXInfo, VOPD3);
  return VOPD::InstInfo(OpXInfo, OpYInfo);
}

TargetID createAMDGPUTargetID(const MCSubtargetInfo &STI,
                              StringRef FeatureString) {
  // In codegen the mode comes from module flags and FeatureString is empty, so
  // the processor defaults apply. The assembler has no target directive, so it
  // pins the mode via the +xnack/-xnack/+sramecc/-sramecc feature string.
  return TargetID::createFromSubtargetFeatures(STI.getTargetTriple(),
                                               STI.getCPU(), FeatureString);
}

namespace IsaInfo {

unsigned getInstCacheLineSize(const MCSubtargetInfo &STI) {
  if (STI.getFeatureBits().test(FeatureInstCacheLineSize128))
    return 128;
  if (STI.getFeatureBits().test(FeatureInstCacheLineSize64))
    return 64;
  return 64;
}

unsigned getWavefrontSize(const MCSubtargetInfo &STI) {
  if (STI.getFeatureBits().test(FeatureWavefrontSize16))
    return 16;
  if (STI.getFeatureBits().test(FeatureWavefrontSize32))
    return 32;

  return 64;
}

// Maximum LDS a single work-group can address. This is a fixed HW cap. It does
// not depend on how many SIMDs a work-group runs on.
static unsigned getMaxHWAddressableLocalMemorySize(const MCSubtargetInfo &STI) {
  if (STI.getFeatureBits().test(FeatureAddressableLocalMemorySize32768))
    return 32768;
  if (STI.getFeatureBits().test(FeatureAddressableLocalMemorySize65536))
    return 65536;
  if (STI.getFeatureBits().test(FeatureAddressableLocalMemorySize163840))
    return 163840;
  if (STI.getFeatureBits().test(FeatureAddressableLocalMemorySize196608))
    return 196608;
  if (STI.getFeatureBits().test(FeatureAddressableLocalMemorySize327680))
    return 327680;
  return 32768;
}

// Total physical size of LDS on the block, in bytes. On targets with
// FeatureHalfAddressablePhysicalLocalMemory the physical block is twice the
// addressable size (gfx6: 64 KiB physical and 32 KiB addressable;
// gfx10/11/12: 128 KiB physical and 64 KiB addressable). On other targets it is
// equal to the addressable size.
static unsigned getPhysicalLocalMemorySize(const MCSubtargetInfo &STI) {
  unsigned Addressable = getMaxHWAddressableLocalMemorySize(STI);
  if (STI.getFeatureBits().test(FeatureHalfAddressablePhysicalLocalMemory))
    return 2 * Addressable;
  return Addressable;
}

// Sizes in use, by generation (addressable / physical block):
//   gfx6              :  32 KiB addressable, 64 KiB physical block
//   gfx7 / gfx8 / gfx9:  64 KiB
//   gfx9.5 (gfx950)   : 160 KiB
//   gfx10 / 11 / 12   :  64 KiB addressable, 128 KiB physical block
//   gfx12.5 (gfx1250) : 320 KiB (always runs on four SIMDs)
//   gfx13             : 192 KiB on four SIMDs, 96 KiB on two
// Total available in the current mode. The physical size is halved when a
// work-group runs on two SIMDs.
unsigned getLocalMemorySize(const MCSubtargetInfo &STI) {
  unsigned Size = getPhysicalLocalMemorySize(STI);
  if (!isFullSIMDMode(STI))
    Size /= 2;
  return Size;
}

// What one work-group can allocate in the current mode. This is the HW
// addressable cap, but never more than the total available in the current mode.
unsigned getAddressableLocalMemorySize(const MCSubtargetInfo &STI) {
  return std::min(getMaxHWAddressableLocalMemorySize(STI),
                  getLocalMemorySize(STI));
}

unsigned getMaxWorkGroupsPerCU(const MCSubtargetInfo &STI,
                               unsigned FlatWorkGroupSize) {
  assert(FlatWorkGroupSize != 0);
  if (!STI.getTargetTriple().isAMDGCN())
    return 8;
  GPUKind Kind = parseArchAMDGCN(STI.getCPU());
  unsigned MaxWaves =
      getMaxWavesPerEU(Kind) * getNumWorkGroupSIMDs(isFullSIMDMode(STI));
  unsigned N = getWavesPerWorkGroup(STI, FlatWorkGroupSize);
  if (N == 1) {
    // Single-wave workgroups don't consume barrier resources.
    return MaxWaves;
  }

  unsigned MaxBarriers = 16;
  if (isGFX10Plus(STI) && !STI.getFeatureBits().test(FeatureCuMode))
    MaxBarriers = 32;

  return std::min(MaxWaves / N, MaxBarriers);
}

unsigned getWavesPerEUForWorkGroup(const MCSubtargetInfo &STI,
                                   unsigned FlatWorkGroupSize) {
  return divideCeil(getWavesPerWorkGroup(STI, FlatWorkGroupSize),
                    getNumWorkGroupSIMDs(isFullSIMDMode(STI)));
}

unsigned getWavesPerWorkGroup(const MCSubtargetInfo &STI,
                              unsigned FlatWorkGroupSize) {
  return divideCeil(FlatWorkGroupSize, getWavefrontSize(STI));
}

unsigned getSGPREncodingGranule(const MCSubtargetInfo &STI) { return 8; }

// Per-wave SGPRs reserved for the trap handler when enabled.
static unsigned getSGPRTrapHandlerReserve(const MCSubtargetInfo &STI) {
  return STI.getFeatureBits().test(FeatureTrapHandler) ? TRAP_NUM_SGPRS : 0;
}

// Per-wave SGPR budget (before the addressable clamp): take off the trap
// reserve, round down to \p Granule. Shared by getMinNumSGPRs() and
// getMaxNumSGPRs(); getOccupancyWithNumSGPRs() is the closed-form algebraic
// inverse of this same budget (it does not call this helper), so the two encode
// one model.
static unsigned getSGPRBudgetPerWave(unsigned TotalNumSGPRs,
                                     unsigned WavesPerEU, unsigned TrapReserve,
                                     unsigned Granule) {
  assert(WavesPerEU != 0 && Granule != 0);
  unsigned Budget = TotalNumSGPRs / WavesPerEU;
  Budget -= std::min(Budget, TrapReserve);
  return alignDown(Budget, Granule);
}

unsigned getMinNumSGPRs(const MCSubtargetInfo &STI, unsigned WavesPerEU) {
  assert(WavesPerEU != 0);

  IsaVersion Version = getIsaVersion(STI.getCPU());
  if (Version.Major >= 10)
    return 0;

  GPUKind Kind = parseArchAMDGCN(STI.getCPU());
  if (WavesPerEU >= getMaxWavesPerEU(Kind))
    return 0;

  unsigned MinNumSGPRs =
      getSGPRBudgetPerWave(getTotalNumSGPRs(Kind), WavesPerEU + 1,
                           getSGPRTrapHandlerReserve(STI),
                           getSGPRAllocGranule(Kind)) +
      1;
  return std::min(MinNumSGPRs, getAddressableNumSGPRs(Kind));
}

unsigned getMaxNumSGPRs(const MCSubtargetInfo &STI, unsigned WavesPerEU,
                        bool Addressable) {
  assert(WavesPerEU != 0);

  GPUKind Kind = parseArchAMDGCN(STI.getCPU());
  unsigned AddressableNumSGPRs = getAddressableNumSGPRs(Kind);
  IsaVersion Version = getIsaVersion(STI.getCPU());
  if (Version.Major >= 10)
    return Addressable ? AddressableNumSGPRs : 108;
  if (Version.Major >= 8 && !Addressable)
    AddressableNumSGPRs = 112;
  unsigned MaxNumSGPRs = getSGPRBudgetPerWave(
      getTotalNumSGPRs(Kind), WavesPerEU, getSGPRTrapHandlerReserve(STI),
      getSGPRAllocGranule(Kind));
  return std::min(MaxNumSGPRs, AddressableNumSGPRs);
}

bool isSGPROccupancyLimited(const MCSubtargetInfo &STI) {
  // From GFX10 on the SGPR file is large enough that SGPRs never limit
  // occupancy. Kept as one capability so callers don't each test the version.
  return getIsaVersion(STI.getCPU()).Major < 10;
}

unsigned getNumExtraSGPRs(const MCSubtargetInfo &STI, bool VCCUsed,
                          bool FlatScrUsed, bool XNACKUsed) {
  unsigned ExtraSGPRs = 0;
  if (VCCUsed)
    ExtraSGPRs = 2;

  IsaVersion Version = getIsaVersion(STI.getCPU());
  if (Version.Major >= 10)
    return ExtraSGPRs;

  if (Version.Major < 8) {
    if (FlatScrUsed)
      ExtraSGPRs = 4;
  } else {
    if (XNACKUsed)
      ExtraSGPRs = 4;

    if (FlatScrUsed ||
        STI.getFeatureBits().test(AMDGPU::FeatureArchitectedFlatScratch))
      ExtraSGPRs = 6;
  }

  return ExtraSGPRs;
}

static unsigned getGranulatedNumRegisterBlocks(unsigned NumRegs,
                                               unsigned Granule) {
  return divideCeil(std::max(1u, NumRegs), Granule);
}

unsigned getNumSGPRBlocks(const MCSubtargetInfo &STI, unsigned NumSGPRs) {
  // SGPRBlocks is actual number of SGPR blocks minus 1.
  return getGranulatedNumRegisterBlocks(NumSGPRs, getSGPREncodingGranule(STI)) -
         1;
}

unsigned getVGPRAllocGranule(const MCSubtargetInfo &STI,
                             unsigned DynamicVGPRBlockSize,
                             std::optional<bool> EnableWavefrontSize32) {
  if (STI.getFeatureBits().test(FeatureGFX90AInsts))
    return 8;

  if (DynamicVGPRBlockSize != 0)
    return DynamicVGPRBlockSize;

  bool IsWave32 = EnableWavefrontSize32
                      ? *EnableWavefrontSize32
                      : STI.getFeatureBits().test(FeatureWavefrontSize32);

  if (STI.getFeatureBits().test(Feature1536VGPRs))
    return IsWave32 ? 24 : 12;

  if (hasGFX10_3Insts(STI))
    return IsWave32 ? 16 : 8;

  return IsWave32 ? 8 : 4;
}

unsigned getVGPREncodingGranule(const MCSubtargetInfo &STI,
                                std::optional<bool> EnableWavefrontSize32) {
  if (STI.getFeatureBits().test(FeatureGFX90AInsts))
    return 8;

  bool IsWave32 = EnableWavefrontSize32
                      ? *EnableWavefrontSize32
                      : STI.getFeatureBits().test(FeatureWavefrontSize32);

  if (STI.getFeatureBits().test(Feature1024AddressableVGPRs))
    return IsWave32 ? 16 : 8;

  return IsWave32 ? 8 : 4;
}

unsigned getArchVGPRAllocGranule() { return 4; }

unsigned getAddressableNumArchVGPRs(const MCSubtargetInfo &STI) {
  const auto &Features = STI.getFeatureBits();
  if (Features.test(Feature1024AddressableVGPRs))
    return Features.test(FeatureWavefrontSize32) ? 1024 : 512;
  return 256;
}

unsigned getAddressableNumVGPRs(const MCSubtargetInfo &STI,
                                unsigned DynamicVGPRBlockSize) {
  const auto &Features = STI.getFeatureBits();
  if (Features.test(FeatureGFX90AInsts))
    return 512;

  if (DynamicVGPRBlockSize != 0) {
    // On GFX12 we can allocate at most MaxDynamicVGPRBlocks blocks of VGPRs.
    return MaxDynamicVGPRBlocks *
           getVGPRAllocGranule(STI, DynamicVGPRBlockSize);
  }
  return getAddressableNumArchVGPRs(STI);
}

unsigned getNumWavesPerEUWithNumVGPRs(const MCSubtargetInfo &STI,
                                      unsigned NumVGPRs,
                                      unsigned DynamicVGPRBlockSize) {
  GPUKind Kind = parseArchAMDGCN(STI.getCPU());
  bool IsWave32 = STI.getFeatureBits().test(FeatureWavefrontSize32);
  return getNumWavesPerEUWithNumVGPRs(
      NumVGPRs, getVGPRAllocGranule(STI, DynamicVGPRBlockSize),
      getMaxWavesPerEU(Kind), AMDGPU::getTotalNumVGPRs(Kind, IsWave32));
}

unsigned getNumWavesPerEUWithNumVGPRs(unsigned NumVGPRs, unsigned Granule,
                                      unsigned MaxWaves,
                                      unsigned TotalNumVGPRs) {
  if (NumVGPRs < Granule)
    return MaxWaves;
  unsigned RoundedRegs = alignTo(NumVGPRs, Granule);
  return std::min(std::max(TotalNumVGPRs / RoundedRegs, 1u), MaxWaves);
}

unsigned getOccupancyWithNumSGPRs(unsigned SGPRs, unsigned MaxWaves,
                                  unsigned TotalNumSGPRs, unsigned Granule,
                                  unsigned TrapReserve) {
  // Closed-form inverse of getMaxNumSGPRs(): the budget condition
  //   SGPRs <= alignDown(TotalNumSGPRs / W - TrapReserve, Granule)
  // solves to W <= TotalNumSGPRs / (alignTo(SGPRs, Granule) + TrapReserve).
  unsigned PerWave = alignTo(SGPRs, Granule) + TrapReserve;
  return PerWave ? std::clamp(TotalNumSGPRs / PerWave, 1u, MaxWaves) : MaxWaves;
}

unsigned getOccupancyWithNumSGPRs(const MCSubtargetInfo &STI, unsigned SGPRs) {
  GPUKind Kind = parseArchAMDGCN(STI.getCPU());
  unsigned MaxWaves = getMaxWavesPerEU(Kind);

  if (!isSGPROccupancyLimited(STI))
    return MaxWaves;

  return getOccupancyWithNumSGPRs(SGPRs, MaxWaves, getTotalNumSGPRs(Kind),
                                  getSGPRAllocGranule(Kind),
                                  getSGPRTrapHandlerReserve(STI));
}

unsigned getMinNumVGPRs(const MCSubtargetInfo &STI, unsigned WavesPerEU,
                        unsigned DynamicVGPRBlockSize) {
  assert(WavesPerEU != 0);

  // In dynamic VGPR mode, (static) occupancy does not depend on VGPR usage,
  // so getMaxNumVGPRs does not depend on WavesPerEU, and thus we need to return
  // zero because there is no nonzero VGPR usage N where going below N
  // achieves higher (static) occupancy.
  bool DynamicVGPREnabled = (DynamicVGPRBlockSize != 0);
  if (DynamicVGPREnabled)
    return 0;

  GPUKind Kind = parseArchAMDGCN(STI.getCPU());
  unsigned MaxWavesPerEU = getMaxWavesPerEU(Kind);
  if (WavesPerEU >= MaxWavesPerEU)
    return 0;

  unsigned TotNumVGPRs = AMDGPU::getTotalNumVGPRs(
      Kind, STI.getFeatureBits().test(FeatureWavefrontSize32));
  unsigned AddrsableNumVGPRs =
      getAddressableNumVGPRs(STI, DynamicVGPRBlockSize);
  unsigned Granule = getVGPRAllocGranule(STI, DynamicVGPRBlockSize);
  unsigned MaxNumVGPRs = alignDown(TotNumVGPRs / WavesPerEU, Granule);

  if (MaxNumVGPRs == alignDown(TotNumVGPRs / MaxWavesPerEU, Granule))
    return 0;

  unsigned MinWavesPerEU = getNumWavesPerEUWithNumVGPRs(STI, AddrsableNumVGPRs,
                                                        DynamicVGPRBlockSize);
  if (WavesPerEU < MinWavesPerEU)
    return getMinNumVGPRs(STI, MinWavesPerEU, DynamicVGPRBlockSize);

  unsigned MaxNumVGPRsNext = alignDown(TotNumVGPRs / (WavesPerEU + 1), Granule);
  unsigned MinNumVGPRs = 1 + std::min(MaxNumVGPRs - Granule, MaxNumVGPRsNext);
  return std::min(MinNumVGPRs, AddrsableNumVGPRs);
}

unsigned getMaxNumVGPRs(const MCSubtargetInfo &STI, unsigned WavesPerEU,
                        unsigned DynamicVGPRBlockSize) {
  assert(WavesPerEU != 0);

  unsigned TotNumVGPRs = AMDGPU::getTotalNumVGPRs(
      parseArchAMDGCN(STI.getCPU()),
      STI.getFeatureBits().test(FeatureWavefrontSize32));

  // In dynamic VGPR mode, WavesPerEU does not imply a VGPR limit.
  bool DynamicVGPREnabled = (DynamicVGPRBlockSize != 0);
  unsigned MaxNumVGPRs =
      DynamicVGPREnabled
          ? TotNumVGPRs
          : alignDown(TotNumVGPRs / WavesPerEU,
                      getVGPRAllocGranule(STI, DynamicVGPRBlockSize));
  unsigned AddressableNumVGPRs =
      getAddressableNumVGPRs(STI, DynamicVGPRBlockSize);
  return std::min(MaxNumVGPRs, AddressableNumVGPRs);
}

unsigned getAllocatedNumVGPRBlocks(const MCSubtargetInfo &STI,
                                   unsigned NumVGPRs,
                                   unsigned DynamicVGPRBlockSize,
                                   std::optional<bool> EnableWavefrontSize32) {
  return getGranulatedNumRegisterBlocks(
      NumVGPRs,
      getVGPRAllocGranule(STI, DynamicVGPRBlockSize, EnableWavefrontSize32));
}
} // end namespace IsaInfo

void initDefaultAMDKernelCodeT(AMDGPUMCKernelCodeT &KernelCode,
                               const MCSubtargetInfo &STI) {
  IsaVersion Version = getIsaVersion(STI.getCPU());
  KernelCode.amd_kernel_code_version_major = 1;
  KernelCode.amd_kernel_code_version_minor = 2;
  KernelCode.amd_machine_kind = 1; // AMD_MACHINE_KIND_AMDGPU
  KernelCode.amd_machine_version_major = Version.Major;
  KernelCode.amd_machine_version_minor = Version.Minor;
  KernelCode.amd_machine_version_stepping = Version.Stepping;
  KernelCode.kernel_code_entry_byte_offset = sizeof(amd_kernel_code_t);
  if (STI.getFeatureBits().test(FeatureWavefrontSize32)) {
    KernelCode.wavefront_size = 5;
    KernelCode.code_properties |= AMD_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32;
  } else {
    KernelCode.wavefront_size = 6;
  }

  // If the code object does not support indirect functions, then the value must
  // be 0xffffffff.
  KernelCode.call_convention = -1;

  // These alignment values are specified in powers of two, so alignment =
  // 2^n.  The minimum alignment is 2^4 = 16.
  KernelCode.kernarg_segment_alignment = 4;
  KernelCode.group_segment_alignment = 4;
  KernelCode.private_segment_alignment = 4;

  if (Version.Major >= 10) {
    KernelCode.compute_pgm_resource_registers |=
        S_00B848_WGP_MODE(STI.getFeatureBits().test(FeatureCuMode) ? 0 : 1) |
        S_00B848_MEM_ORDERED(1) | S_00B848_FWD_PROGRESS(1);
  }
}

bool isReadOnlySegment(const GlobalValue *GV) {
  unsigned AS = GV->getAddressSpace();
  return AS == AMDGPUAS::CONSTANT_ADDRESS ||
         AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT;
}

bool shouldEmitConstantsToTextSection(const Triple &TT) {
  return TT.getArch() == Triple::r600;
}

static bool isValidRegPrefix(char C) {
  return C == 'v' || C == 's' || C == 'a';
}

std::tuple<char, unsigned, unsigned> parseAsmPhysRegName(StringRef RegName) {
  if (RegName.empty())
    return {};

  char Kind = RegName.front();
  if (!isValidRegPrefix(Kind))
    return {};

  RegName = RegName.drop_front();
  if (RegName.consume_front("[")) {
    unsigned Idx, End;
    bool Failed = RegName.consumeInteger(10, Idx);
    Failed |= !RegName.consume_front(":");
    Failed |= RegName.consumeInteger(10, End);
    Failed |= !RegName.consume_back("]");
    if (!Failed) {
      unsigned NumRegs = End - Idx + 1;
      if (NumRegs > 1)
        return {Kind, Idx, NumRegs};
    }
  } else {
    unsigned Idx;
    bool Failed = RegName.getAsInteger(10, Idx);
    if (!Failed)
      return {Kind, Idx, 1};
  }

  return {};
}

std::tuple<char, unsigned, unsigned>
parseAsmConstraintPhysReg(StringRef Constraint) {
  StringRef RegName = Constraint;
  if (!RegName.consume_front("{") || !RegName.consume_back("}"))
    return {};
  return parseAsmPhysRegName(RegName);
}

std::pair<unsigned, unsigned>
getIntegerPairAttribute(const Function &F, StringRef Name,
                        std::pair<unsigned, unsigned> Default,
                        bool OnlyFirstRequired) {
  if (auto Attr = getIntegerPairAttribute(F, Name, OnlyFirstRequired))
    return {Attr->first, Attr->second.value_or(Default.second)};
  return Default;
}

std::optional<std::pair<unsigned, std::optional<unsigned>>>
getIntegerPairAttribute(const Function &F, StringRef Name,
                        bool OnlyFirstRequired) {
  Attribute A = F.getFnAttribute(Name);
  if (!A.isStringAttribute())
    return std::nullopt;

  LLVMContext &Ctx = F.getContext();
  std::pair<unsigned, std::optional<unsigned>> Ints;
  std::pair<StringRef, StringRef> Strs = A.getValueAsString().split(',');
  if (Strs.first.trim().getAsInteger(0, Ints.first)) {
    Ctx.emitError("can't parse first integer attribute " + Name);
    return std::nullopt;
  }
  unsigned Second = 0;
  if (Strs.second.trim().getAsInteger(0, Second)) {
    if (!OnlyFirstRequired || !Strs.second.trim().empty()) {
      Ctx.emitError("can't parse second integer attribute " + Name);
      return std::nullopt;
    }
  } else {
    Ints.second = Second;
  }

  return Ints;
}

SmallVector<unsigned> getIntegerVecAttribute(const Function &F, StringRef Name,
                                             unsigned Size,
                                             unsigned DefaultVal) {
  std::optional<SmallVector<unsigned>> R =
      getIntegerVecAttribute(F, Name, Size);
  return R.has_value() ? *R : SmallVector<unsigned>(Size, DefaultVal);
}

std::optional<SmallVector<unsigned>>
getIntegerVecAttribute(const Function &F, StringRef Name, unsigned Size) {
  assert(Size > 2);
  LLVMContext &Ctx = F.getContext();

  Attribute A = F.getFnAttribute(Name);
  if (!A.isValid())
    return std::nullopt;
  if (!A.isStringAttribute()) {
    Ctx.emitError(Name + " is not a string attribute");
    return std::nullopt;
  }

  SmallVector<unsigned> Vals(Size);

  StringRef S = A.getValueAsString();
  unsigned i = 0;
  for (; !S.empty() && i < Size; i++) {
    std::pair<StringRef, StringRef> Strs = S.split(',');
    unsigned IntVal;
    if (Strs.first.trim().getAsInteger(0, IntVal)) {
      Ctx.emitError("can't parse integer attribute " + Strs.first + " in " +
                    Name);
      return std::nullopt;
    }
    Vals[i] = IntVal;
    S = Strs.second;
  }

  if (!S.empty() || i < Size) {
    Ctx.emitError("attribute " + Name +
                  " has incorrect number of integers; expected " +
                  llvm::utostr(Size));
    return std::nullopt;
  }
  return Vals;
}

SmallVector<unsigned> getMaxNumWorkGroups(const Function &F) {
  return getIntegerVecAttribute(F, "amdgpu-max-num-workgroups", 3,
                                std::numeric_limits<uint32_t>::max());
}

bool hasValueInRangeLikeMetadata(const MDNode &MD, int64_t Val) {
  assert((MD.getNumOperands() % 2 == 0) && "invalid number of operands!");
  for (unsigned I = 0, E = MD.getNumOperands() / 2; I != E; ++I) {
    auto Low =
        mdconst::extract<ConstantInt>(MD.getOperand(2 * I + 0))->getValue();
    auto High =
        mdconst::extract<ConstantInt>(MD.getOperand(2 * I + 1))->getValue();
    // There are two types of [A; B) ranges:
    //  A < B, e.g. [4; 5) which is a range that only includes 4.
    //  A > B, e.g. [5; 4) which is a range that wraps around and includes
    //         everything except 4.
    if (Low.ult(High)) {
      if (Low.ule(Val) && High.ugt(Val))
        return true;
    } else {
      if (Low.uge(Val) && High.ult(Val))
        return true;
    }
  }

  return false;
}

unsigned getVmcntBitMask(const IsaVersion &Version) {
  return (1 << (getVmcntBitWidthLo(Version.Major) +
                getVmcntBitWidthHi(Version.Major))) -
         1;
}

unsigned getLoadcntBitMask(const IsaVersion &Version) {
  return (1 << getLoadcntBitWidth(Version.Major)) - 1;
}

unsigned getSamplecntBitMask(const IsaVersion &Version) {
  return (1 << getSamplecntBitWidth(Version.Major)) - 1;
}

unsigned getBvhcntBitMask(const IsaVersion &Version) {
  return (1 << getBvhcntBitWidth(Version.Major)) - 1;
}

unsigned getExpcntBitMask(const IsaVersion &Version) {
  return (1 << getExpcntBitWidth(Version.Major)) - 1;
}

unsigned getLgkmcntBitMask(const IsaVersion &Version) {
  return (1 << getLgkmcntBitWidth(Version.Major)) - 1;
}

unsigned getDscntBitMask(const IsaVersion &Version) {
  return (1 << getDscntBitWidth(Version.Major)) - 1;
}

unsigned getKmcntBitMask(const IsaVersion &Version) {
  return (1 << getKmcntBitWidth(Version.Major)) - 1;
}

unsigned getXcntBitMask(const IsaVersion &Version) {
  return (1 << getXcntBitWidth(Version.Major, Version.Minor)) - 1;
}

unsigned getAsynccntBitMask(const IsaVersion &Version) {
  return (1 << getAsynccntBitWidth(Version.Major, Version.Minor)) - 1;
}

unsigned getStorecntBitMask(const IsaVersion &Version) {
  return (1 << getStorecntBitWidth(Version.Major)) - 1;
}

unsigned getBarrierMemberCountBitMask(const IsaVersion &Version) {
  return (1 << getBarrierMemberCountBitWidth(Version.Major)) - 1;
}

unsigned getWaitcntBitMask(const IsaVersion &Version) {
  unsigned VmcntLo = getBitMask(getVmcntBitShiftLo(Version.Major),
                                getVmcntBitWidthLo(Version.Major));
  unsigned Expcnt = getBitMask(getExpcntBitShift(Version.Major),
                               getExpcntBitWidth(Version.Major));
  unsigned Lgkmcnt = getBitMask(getLgkmcntBitShift(Version.Major),
                                getLgkmcntBitWidth(Version.Major));
  unsigned VmcntHi = getBitMask(getVmcntBitShiftHi(Version.Major),
                                getVmcntBitWidthHi(Version.Major));
  return VmcntLo | Expcnt | Lgkmcnt | VmcntHi;
}

unsigned decodeVmcnt(const IsaVersion &Version, unsigned Waitcnt) {
  unsigned VmcntLo = unpackBits(Waitcnt, getVmcntBitShiftLo(Version.Major),
                                getVmcntBitWidthLo(Version.Major));
  unsigned VmcntHi = unpackBits(Waitcnt, getVmcntBitShiftHi(Version.Major),
                                getVmcntBitWidthHi(Version.Major));
  return VmcntLo | VmcntHi << getVmcntBitWidthLo(Version.Major);
}

unsigned decodeExpcnt(const IsaVersion &Version, unsigned Waitcnt) {
  return unpackBits(Waitcnt, getExpcntBitShift(Version.Major),
                    getExpcntBitWidth(Version.Major));
}

unsigned decodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt) {
  return unpackBits(Waitcnt, getLgkmcntBitShift(Version.Major),
                    getLgkmcntBitWidth(Version.Major));
}

unsigned decodeLoadcnt(const IsaVersion &Version, unsigned Waitcnt) {
  return unpackBits(Waitcnt, getLoadcntStorecntBitShift(Version.Major),
                    getLoadcntBitWidth(Version.Major));
}

unsigned decodeStorecnt(const IsaVersion &Version, unsigned Waitcnt) {
  return unpackBits(Waitcnt, getLoadcntStorecntBitShift(Version.Major),
                    getStorecntBitWidth(Version.Major));
}

unsigned decodeDscnt(const IsaVersion &Version, unsigned Waitcnt) {
  return unpackBits(Waitcnt, getDscntBitShift(Version.Major),
                    getDscntBitWidth(Version.Major));
}

void decodeWaitcnt(const IsaVersion &Version, unsigned Waitcnt, unsigned &Vmcnt,
                   unsigned &Expcnt, unsigned &Lgkmcnt) {
  Vmcnt = decodeVmcnt(Version, Waitcnt);
  Expcnt = decodeExpcnt(Version, Waitcnt);
  Lgkmcnt = decodeLgkmcnt(Version, Waitcnt);
}

unsigned encodeVmcnt(const IsaVersion &Version, unsigned Waitcnt,
                     unsigned Vmcnt) {
  Waitcnt = packBits(Vmcnt, Waitcnt, getVmcntBitShiftLo(Version.Major),
                     getVmcntBitWidthLo(Version.Major));
  return packBits(Vmcnt >> getVmcntBitWidthLo(Version.Major), Waitcnt,
                  getVmcntBitShiftHi(Version.Major),
                  getVmcntBitWidthHi(Version.Major));
}

unsigned encodeExpcnt(const IsaVersion &Version, unsigned Waitcnt,
                      unsigned Expcnt) {
  return packBits(Expcnt, Waitcnt, getExpcntBitShift(Version.Major),
                  getExpcntBitWidth(Version.Major));
}

unsigned encodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt,
                       unsigned Lgkmcnt) {
  return packBits(Lgkmcnt, Waitcnt, getLgkmcntBitShift(Version.Major),
                  getLgkmcntBitWidth(Version.Major));
}

unsigned encodeWaitcnt(const IsaVersion &Version, unsigned Vmcnt,
                       unsigned Expcnt, unsigned Lgkmcnt) {
  unsigned Waitcnt = getWaitcntBitMask(Version);
  Waitcnt = encodeVmcnt(Version, Waitcnt, Vmcnt);
  Waitcnt = encodeExpcnt(Version, Waitcnt, Expcnt);
  Waitcnt = encodeLgkmcnt(Version, Waitcnt, Lgkmcnt);
  return Waitcnt;
}

static unsigned getCombinedCountBitMask(const IsaVersion &Version,
                                        bool IsStore) {
  unsigned Dscnt = getBitMask(getDscntBitShift(Version.Major),
                              getDscntBitWidth(Version.Major));
  if (IsStore) {
    unsigned Storecnt = getBitMask(getLoadcntStorecntBitShift(Version.Major),
                                   getStorecntBitWidth(Version.Major));
    return Dscnt | Storecnt;
  }
  unsigned Loadcnt = getBitMask(getLoadcntStorecntBitShift(Version.Major),
                                getLoadcntBitWidth(Version.Major));
  return Dscnt | Loadcnt;
}

static unsigned encodeLoadcnt(const IsaVersion &Version, unsigned Waitcnt,
                              unsigned Loadcnt) {
  return packBits(Loadcnt, Waitcnt, getLoadcntStorecntBitShift(Version.Major),
                  getLoadcntBitWidth(Version.Major));
}

static unsigned encodeStorecnt(const IsaVersion &Version, unsigned Waitcnt,
                               unsigned Storecnt) {
  return packBits(Storecnt, Waitcnt, getLoadcntStorecntBitShift(Version.Major),
                  getStorecntBitWidth(Version.Major));
}

static unsigned encodeDscnt(const IsaVersion &Version, unsigned Waitcnt,
                            unsigned Dscnt) {
  return packBits(Dscnt, Waitcnt, getDscntBitShift(Version.Major),
                  getDscntBitWidth(Version.Major));
}

unsigned encodeLoadcntDscnt(const IsaVersion &Version, unsigned Loadcnt,
                            unsigned Dscnt) {
  unsigned Waitcnt = getCombinedCountBitMask(Version, false);
  Waitcnt = encodeLoadcnt(Version, Waitcnt, Loadcnt);
  Waitcnt = encodeDscnt(Version, Waitcnt, Dscnt);
  return Waitcnt;
}

unsigned encodeStorecntDscnt(const IsaVersion &Version, unsigned Storecnt,
                             unsigned Dscnt) {
  unsigned Waitcnt = getCombinedCountBitMask(Version, true);
  Waitcnt = encodeStorecnt(Version, Waitcnt, Storecnt);
  Waitcnt = encodeDscnt(Version, Waitcnt, Dscnt);
  return Waitcnt;
}

//===----------------------------------------------------------------------===//
// Custom Operand Values
//===----------------------------------------------------------------------===//

static bool isSymbolicCustomOperandEncoding(const CustomOperandVal *Opr,
                                            int Size, unsigned Code,
                                            bool &HasNonDefaultVal,
                                            const MCSubtargetInfo &STI) {
  unsigned UsedOprMask = 0;
  HasNonDefaultVal = false;
  for (int Idx = 0; Idx < Size; ++Idx) {
    const auto &Op = Opr[Idx];
    if (!Op.isSupported(STI))
      continue;
    UsedOprMask |= Op.getMask();
    unsigned Val = Op.decode(Code);
    if (!Op.isValid(Val))
      return false;
    HasNonDefaultVal |= (Val != Op.Default);
  }
  return (Code & ~UsedOprMask) == 0;
}

static bool decodeCustomOperand(const CustomOperandVal *Opr, int Size,
                                unsigned Code, int &Idx, StringRef &Name,
                                unsigned &Val, bool &IsDefault,
                                const MCSubtargetInfo &STI) {
  while (Idx < Size) {
    const auto &Op = Opr[Idx++];
    if (Op.isSupported(STI)) {
      Name = Op.Name;
      Val = Op.decode(Code);
      IsDefault = (Val == Op.Default);
      return true;
    }
  }

  return false;
}

static int encodeCustomOperandVal(const CustomOperandVal &Op,
                                  int64_t InputVal) {
  if (InputVal < 0 || InputVal > Op.Max)
    return OPR_VAL_INVALID;
  return Op.encode(static_cast<unsigned>(InputVal));
}

static int encodeCustomOperand(const CustomOperandVal *Opr, int Size,
                               const StringRef Name, int64_t InputVal,
                               unsigned &UsedOprMask,
                               const MCSubtargetInfo &STI) {
  int InvalidId = OPR_ID_UNKNOWN;
  for (int Idx = 0; Idx < Size; ++Idx) {
    const auto &Op = Opr[Idx];
    if (Op.Name == Name) {
      if (!Op.isSupported(STI)) {
        InvalidId = OPR_ID_UNSUPPORTED;
        continue;
      }
      auto OprMask = Op.getMask();
      if (OprMask & UsedOprMask)
        return OPR_ID_DUPLICATE;
      UsedOprMask |= OprMask;
      return encodeCustomOperandVal(Op, InputVal);
    }
  }
  return InvalidId;
}

//===----------------------------------------------------------------------===//
// DepCtr
//===----------------------------------------------------------------------===//

namespace DepCtr {

int getDefaultDepCtrEncoding(const MCSubtargetInfo &STI) {
  unsigned Enc = 0;
  for (int Idx = 0; Idx < DEP_CTR_SIZE; ++Idx) {
    const auto &Op = DepCtrInfo[Idx];
    if (Op.isSupported(STI))
      Enc |= Op.encode(Op.Default);
  }
  return Enc;
}

bool isSymbolicDepCtrEncoding(unsigned Code, bool &HasNonDefaultVal,
                              const MCSubtargetInfo &STI) {
  return isSymbolicCustomOperandEncoding(DepCtrInfo, DEP_CTR_SIZE, Code,
                                         HasNonDefaultVal, STI);
}

bool decodeDepCtr(unsigned Code, int &Id, StringRef &Name, unsigned &Val,
                  bool &IsDefault, const MCSubtargetInfo &STI) {
  return decodeCustomOperand(DepCtrInfo, DEP_CTR_SIZE, Code, Id, Name, Val,
                             IsDefault, STI);
}

int encodeDepCtr(const StringRef Name, int64_t Val, unsigned &UsedOprMask,
                 const MCSubtargetInfo &STI) {
  return encodeCustomOperand(DepCtrInfo, DEP_CTR_SIZE, Name, Val, UsedOprMask,
                             STI);
}

unsigned getVaVdstBitMask() { return (1 << getVaVdstBitWidth()) - 1; }

unsigned getVaSdstBitMask() { return (1 << getVaSdstBitWidth()) - 1; }

unsigned getVaSsrcBitMask() { return (1 << getVaSsrcBitWidth()) - 1; }

unsigned getHoldCntBitMask(const IsaVersion &Version) {
  return (1 << getHoldCntWidth(Version.Major, Version.Minor)) - 1;
}

unsigned getVmVsrcBitMask() { return (1 << getVmVsrcBitWidth()) - 1; }

unsigned getVaVccBitMask() { return (1 << getVaVccBitWidth()) - 1; }

unsigned getSaSdstBitMask() { return (1 << getSaSdstBitWidth()) - 1; }

unsigned decodeFieldVmVsrc(unsigned Encoded) {
  return unpackBits(Encoded, getVmVsrcBitShift(), getVmVsrcBitWidth());
}

unsigned decodeFieldVaVdst(unsigned Encoded) {
  return unpackBits(Encoded, getVaVdstBitShift(), getVaVdstBitWidth());
}

unsigned decodeFieldSaSdst(unsigned Encoded) {
  return unpackBits(Encoded, getSaSdstBitShift(), getSaSdstBitWidth());
}

unsigned decodeFieldVaSdst(unsigned Encoded) {
  return unpackBits(Encoded, getVaSdstBitShift(), getVaSdstBitWidth());
}

unsigned decodeFieldVaVcc(unsigned Encoded) {
  return unpackBits(Encoded, getVaVccBitShift(), getVaVccBitWidth());
}

unsigned decodeFieldVaSsrc(unsigned Encoded) {
  return unpackBits(Encoded, getVaSsrcBitShift(), getVaSsrcBitWidth());
}

unsigned decodeFieldHoldCnt(unsigned Encoded, const IsaVersion &Version) {
  return unpackBits(Encoded, getHoldCntBitShift(),
                    getHoldCntWidth(Version.Major, Version.Minor));
}

unsigned encodeFieldVmVsrc(unsigned Encoded, unsigned VmVsrc) {
  return packBits(VmVsrc, Encoded, getVmVsrcBitShift(), getVmVsrcBitWidth());
}

unsigned encodeFieldVmVsrc(unsigned VmVsrc, const MCSubtargetInfo &STI) {
  unsigned Encoded = getDefaultDepCtrEncoding(STI);
  return encodeFieldVmVsrc(Encoded, VmVsrc);
}

unsigned encodeFieldVaVdst(unsigned Encoded, unsigned VaVdst) {
  return packBits(VaVdst, Encoded, getVaVdstBitShift(), getVaVdstBitWidth());
}

unsigned encodeFieldVaVdst(unsigned VaVdst, const MCSubtargetInfo &STI) {
  unsigned Encoded = getDefaultDepCtrEncoding(STI);
  return encodeFieldVaVdst(Encoded, VaVdst);
}

unsigned encodeFieldSaSdst(unsigned Encoded, unsigned SaSdst) {
  return packBits(SaSdst, Encoded, getSaSdstBitShift(), getSaSdstBitWidth());
}

unsigned encodeFieldSaSdst(unsigned SaSdst, const MCSubtargetInfo &STI) {
  unsigned Encoded = getDefaultDepCtrEncoding(STI);
  return encodeFieldSaSdst(Encoded, SaSdst);
}

unsigned encodeFieldVaSdst(unsigned Encoded, unsigned VaSdst) {
  return packBits(VaSdst, Encoded, getVaSdstBitShift(), getVaSdstBitWidth());
}

unsigned encodeFieldVaSdst(unsigned VaSdst, const MCSubtargetInfo &STI) {
  unsigned Encoded = getDefaultDepCtrEncoding(STI);
  return encodeFieldVaSdst(Encoded, VaSdst);
}

unsigned encodeFieldVaVcc(unsigned Encoded, unsigned VaVcc) {
  return packBits(VaVcc, Encoded, getVaVccBitShift(), getVaVccBitWidth());
}

unsigned encodeFieldVaVcc(unsigned VaVcc, const MCSubtargetInfo &STI) {
  unsigned Encoded = getDefaultDepCtrEncoding(STI);
  return encodeFieldVaVcc(Encoded, VaVcc);
}

unsigned encodeFieldVaSsrc(unsigned Encoded, unsigned VaSsrc) {
  return packBits(VaSsrc, Encoded, getVaSsrcBitShift(), getVaSsrcBitWidth());
}

unsigned encodeFieldVaSsrc(unsigned VaSsrc, const MCSubtargetInfo &STI) {
  unsigned Encoded = getDefaultDepCtrEncoding(STI);
  return encodeFieldVaSsrc(Encoded, VaSsrc);
}

unsigned encodeFieldHoldCnt(unsigned Encoded, unsigned HoldCnt,
                            const IsaVersion &Version) {
  return packBits(HoldCnt, Encoded, getHoldCntBitShift(),
                  getHoldCntWidth(Version.Major, Version.Minor));
}

unsigned encodeFieldHoldCnt(unsigned HoldCnt, const MCSubtargetInfo &STI) {
  unsigned Encoded = getDefaultDepCtrEncoding(STI);
  return encodeFieldHoldCnt(Encoded, HoldCnt, getIsaVersion(STI.getCPU()));
}

} // namespace DepCtr

//===----------------------------------------------------------------------===//
// exp tgt
//===----------------------------------------------------------------------===//

namespace Exp {

struct ExpTgt {
  StringLiteral Name;
  unsigned Tgt;
  unsigned MaxIndex;
};

// clang-format off
static constexpr ExpTgt ExpTgtInfo[] = {
    {{"null"},          ET_NULL,            ET_NULL_MAX_IDX},
    {{"mrtz"},          ET_MRTZ,            ET_MRTZ_MAX_IDX},
    {{"prim"},          ET_PRIM,            ET_PRIM_MAX_IDX},
    {{"mrt"},           ET_MRT0,            ET_MRT_MAX_IDX},
    {{"pos"},           ET_POS0,            ET_POS_MAX_IDX},
    {{"dual_src_blend"},ET_DUAL_SRC_BLEND0, ET_DUAL_SRC_BLEND_MAX_IDX},
    {{"param"},         ET_PARAM0,          ET_PARAM_MAX_IDX},
};
// clang-format on

bool getTgtName(unsigned Id, StringRef &Name, int &Index) {
  for (const ExpTgt &Val : ExpTgtInfo) {
    if (Val.Tgt <= Id && Id <= Val.Tgt + Val.MaxIndex) {
      Index = (Val.MaxIndex == 0) ? -1 : (Id - Val.Tgt);
      Name = Val.Name;
      return true;
    }
  }
  return false;
}

unsigned getTgtId(const StringRef Name) {

  for (const ExpTgt &Val : ExpTgtInfo) {
    if (Val.MaxIndex == 0 && Name == Val.Name)
      return Val.Tgt;

    if (Val.MaxIndex > 0 && Name.starts_with(Val.Name)) {
      StringRef Suffix = Name.drop_front(Val.Name.size());

      unsigned Id;
      if (Suffix.getAsInteger(10, Id) || Id > Val.MaxIndex)
        return ET_INVALID;

      // Disable leading zeroes
      if (Suffix.size() > 1 && Suffix[0] == '0')
        return ET_INVALID;

      return Val.Tgt + Id;
    }
  }
  return ET_INVALID;
}

bool isSupportedTgtId(unsigned Id, const MCSubtargetInfo &STI) {
  switch (Id) {
  case ET_NULL:
    return !isGFX11Plus(STI);
  case ET_POS4:
  case ET_PRIM:
    return isGFX10Plus(STI);
  case ET_DUAL_SRC_BLEND0:
  case ET_DUAL_SRC_BLEND1:
    return isGFX11Plus(STI);
  default:
    if (Id >= ET_PARAM0 && Id <= ET_PARAM31)
      return !isGFX11Plus(STI) || isGFX13Plus(STI);
    return true;
  }
}

} // namespace Exp

//===----------------------------------------------------------------------===//
// MTBUF Format
//===----------------------------------------------------------------------===//

namespace MTBUFFormat {

int64_t getDfmt(const StringRef Name) {
  for (int Id = DFMT_MIN; Id <= DFMT_MAX; ++Id) {
    if (Name == DfmtSymbolic[Id])
      return Id;
  }
  return DFMT_UNDEF;
}

StringRef getDfmtName(unsigned Id) {
  assert(Id <= DFMT_MAX);
  return DfmtSymbolic[Id];
}

static StringLiteral const *getNfmtLookupTable(const MCSubtargetInfo &STI) {
  if (isSI(STI) || isCI(STI))
    return NfmtSymbolicSICI;
  if (isVI(STI) || isGFX9(STI))
    return NfmtSymbolicVI;
  return NfmtSymbolicGFX10;
}

int64_t getNfmt(const StringRef Name, const MCSubtargetInfo &STI) {
  const auto *lookupTable = getNfmtLookupTable(STI);
  for (int Id = NFMT_MIN; Id <= NFMT_MAX; ++Id) {
    if (Name == lookupTable[Id])
      return Id;
  }
  return NFMT_UNDEF;
}

StringRef getNfmtName(unsigned Id, const MCSubtargetInfo &STI) {
  assert(Id <= NFMT_MAX);
  return getNfmtLookupTable(STI)[Id];
}

bool isValidDfmtNfmt(unsigned Id, const MCSubtargetInfo &STI) {
  unsigned Dfmt;
  unsigned Nfmt;
  decodeDfmtNfmt(Id, Dfmt, Nfmt);
  return isValidNfmt(Nfmt, STI);
}

bool isValidNfmt(unsigned Id, const MCSubtargetInfo &STI) {
  return !getNfmtName(Id, STI).empty();
}

int64_t encodeDfmtNfmt(unsigned Dfmt, unsigned Nfmt) {
  return (Dfmt << DFMT_SHIFT) | (Nfmt << NFMT_SHIFT);
}

void decodeDfmtNfmt(unsigned Format, unsigned &Dfmt, unsigned &Nfmt) {
  Dfmt = (Format >> DFMT_SHIFT) & DFMT_MASK;
  Nfmt = (Format >> NFMT_SHIFT) & NFMT_MASK;
}

int64_t getUnifiedFormat(const StringRef Name, const MCSubtargetInfo &STI) {
  if (isGFX11Plus(STI)) {
    for (int Id = UfmtGFX11::UFMT_FIRST; Id <= UfmtGFX11::UFMT_LAST; ++Id) {
      if (Name == UfmtSymbolicGFX11[Id])
        return Id;
    }
  } else {
    for (int Id = UfmtGFX10::UFMT_FIRST; Id <= UfmtGFX10::UFMT_LAST; ++Id) {
      if (Name == UfmtSymbolicGFX10[Id])
        return Id;
    }
  }
  return UFMT_UNDEF;
}

StringRef getUnifiedFormatName(unsigned Id, const MCSubtargetInfo &STI) {
  if (isValidUnifiedFormat(Id, STI))
    return isGFX10(STI) ? UfmtSymbolicGFX10[Id] : UfmtSymbolicGFX11[Id];
  return "";
}

bool isValidUnifiedFormat(unsigned Id, const MCSubtargetInfo &STI) {
  return isGFX10(STI) ? Id <= UfmtGFX10::UFMT_LAST : Id <= UfmtGFX11::UFMT_LAST;
}

int64_t convertDfmtNfmt2Ufmt(unsigned Dfmt, unsigned Nfmt,
                             const MCSubtargetInfo &STI) {
  int64_t Fmt = encodeDfmtNfmt(Dfmt, Nfmt);
  if (isGFX11Plus(STI)) {
    for (int Id = UfmtGFX11::UFMT_FIRST; Id <= UfmtGFX11::UFMT_LAST; ++Id) {
      if (Fmt == DfmtNfmt2UFmtGFX11[Id])
        return Id;
    }
  } else {
    for (int Id = UfmtGFX10::UFMT_FIRST; Id <= UfmtGFX10::UFMT_LAST; ++Id) {
      if (Fmt == DfmtNfmt2UFmtGFX10[Id])
        return Id;
    }
  }
  return UFMT_UNDEF;
}

bool isValidFormatEncoding(unsigned Val, const MCSubtargetInfo &STI) {
  return isGFX10Plus(STI) ? (Val <= UFMT_MAX) : (Val <= DFMT_NFMT_MAX);
}

unsigned getDefaultFormatEncoding(const MCSubtargetInfo &STI) {
  if (isGFX10Plus(STI))
    return UFMT_DEFAULT;
  return DFMT_NFMT_DEFAULT;
}

} // namespace MTBUFFormat

//===----------------------------------------------------------------------===//
// SendMsg
//===----------------------------------------------------------------------===//

namespace SendMsg {

static uint64_t getMsgIdMask(const MCSubtargetInfo &STI) {
  return isGFX11Plus(STI) ? ID_MASK_GFX11Plus_ : ID_MASK_PreGFX11_;
}

bool isValidMsgId(int64_t MsgId, const MCSubtargetInfo &STI) {
  return (MsgId & ~(getMsgIdMask(STI))) == 0;
}

bool isValidMsgOp(int64_t MsgId, int64_t OpId, const MCSubtargetInfo &STI,
                  bool Strict) {
  assert(isValidMsgId(MsgId, STI));

  if (!Strict)
    return 0 <= OpId && isUInt<OP_WIDTH_>(OpId);

  if (msgRequiresOp(MsgId, STI)) {
    if (MsgId == ID_GS_PreGFX11 && OpId == OP_GS_NOP)
      return false;

    return !getMsgOpName(MsgId, OpId, STI).empty();
  }

  return OpId == OP_NONE_;
}

bool isValidMsgStream(int64_t MsgId, int64_t OpId, int64_t StreamId,
                      const MCSubtargetInfo &STI, bool Strict) {
  assert(isValidMsgOp(MsgId, OpId, STI, Strict));

  if (!Strict)
    return 0 <= StreamId && isUInt<STREAM_ID_WIDTH_>(StreamId);

  if (!isGFX11Plus(STI)) {
    switch (MsgId) {
    case ID_GS_PreGFX11:
      return STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_;
    case ID_GS_DONE_PreGFX11:
      return (OpId == OP_GS_NOP)
                 ? (StreamId == STREAM_ID_NONE_)
                 : (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_);
    }
  }
  return StreamId == STREAM_ID_NONE_;
}

bool msgRequiresOp(int64_t MsgId, const MCSubtargetInfo &STI) {
  return MsgId == ID_SYSMSG ||
         (!isGFX11Plus(STI) &&
          (MsgId == ID_GS_PreGFX11 || MsgId == ID_GS_DONE_PreGFX11));
}

bool msgSupportsStream(int64_t MsgId, int64_t OpId,
                       const MCSubtargetInfo &STI) {
  return !isGFX11Plus(STI) &&
         (MsgId == ID_GS_PreGFX11 || MsgId == ID_GS_DONE_PreGFX11) &&
         OpId != OP_GS_NOP;
}

void decodeMsg(unsigned Val, uint16_t &MsgId, uint16_t &OpId,
               uint16_t &StreamId, const MCSubtargetInfo &STI) {
  MsgId = static_cast<uint16_t>(Val & getMsgIdMask(STI));
  if (isGFX11Plus(STI)) {
    OpId = 0;
    StreamId = 0;
  } else {
    OpId = (Val & OP_MASK_) >> OP_SHIFT_;
    StreamId = (Val & STREAM_ID_MASK_) >> STREAM_ID_SHIFT_;
  }
}

uint64_t encodeMsg(uint64_t MsgId, uint64_t OpId, uint64_t StreamId) {
  return MsgId | (OpId << OP_SHIFT_) | (StreamId << STREAM_ID_SHIFT_);
}

bool msgDoesNotUseM0(int64_t MsgId, const MCSubtargetInfo &STI) {
  // Explicitly list message types that are known to not use m0.
  // This is safer than excluding only GS_ALLOC_REQ, in case new message
  // types are added in the future that do use m0.
  if (isGFX11Plus(STI)) {
    switch (MsgId) {
    case ID_DEALLOC_VGPRS_GFX11Plus:
      return true;
    default:
      break;
    }
  }
  switch (MsgId) {
  case ID_SAVEWAVE:
  case ID_STALL_WAVE_GEN:
  case ID_HALT_WAVES:
  case ID_ORDERED_PS_DONE:
  case ID_EARLY_PRIM_DEALLOC:
  case ID_GET_DOORBELL:
  case ID_GET_DDID:
  case ID_SYSMSG:
    return true;
  default:
    return false;
  }
}

} // namespace SendMsg

//===----------------------------------------------------------------------===//
//
//===----------------------------------------------------------------------===//

unsigned getInitialPSInputAddr(const Function &F) {
  return static_cast<unsigned>(
      F.getFnAttributeAsParsedInteger("InitialPSInputAddr", 0));
}

bool getHasColorExport(const Function &F) {
  // As a safe default always respond as if PS has color exports.
  return F.getFnAttributeAsParsedInteger(
             "amdgpu-color-export",
             F.getCallingConv() == CallingConv::AMDGPU_PS ? 1 : 0) != 0;
}

bool getHasDepthExport(const Function &F) {
  return F.getFnAttributeAsParsedInteger("amdgpu-depth-export", 0) != 0;
}

unsigned getDynamicVGPRBlockSize(const Function &F) {
  unsigned BlockSize = static_cast<unsigned>(
      F.getFnAttributeAsParsedInteger("amdgpu-dynamic-vgpr-block-size", 0));

  if (BlockSize == 16 || BlockSize == 32)
    return BlockSize;

  return 0;
}

bool hasMIMG_R128(const MCSubtargetInfo &STI) {
  return STI.hasFeature(AMDGPU::FeatureMIMG_R128) &&
         !STI.hasFeature(AMDGPU::FeatureR128A16);
}

bool hasA16(const MCSubtargetInfo &STI) {
  return STI.hasFeature(AMDGPU::FeatureA16);
}

bool hasG16(const MCSubtargetInfo &STI) {
  return STI.hasFeature(AMDGPU::FeatureG16);
}

bool hasPackedD16(const MCSubtargetInfo &STI) {
  return !STI.hasFeature(AMDGPU::FeatureUnpackedD16VMem) && !isCI(STI) &&
         !isSI(STI);
}

bool hasGDS(const MCSubtargetInfo &STI) {
  return STI.hasFeature(AMDGPU::FeatureGDS);
}

unsigned getNSAMaxSize(const MCSubtargetInfo &STI, bool HasSampler) {
  auto Version = getIsaVersion(STI.getCPU());
  if (Version.Major == 10)
    return Version.Minor >= 3 ? 13 : 5;
  if (Version.Major == 11)
    return 5;
  if (Version.Major >= 12)
    return HasSampler ? 4 : 5;
  return 0;
}

unsigned getMaxNumUserSGPRs(const MCSubtargetInfo &STI) {
  if (isGFX1250Plus(STI))
    return 32;
  return 16;
}

bool isSI(const MCSubtargetInfo &STI) {
  return STI.hasFeature(AMDGPU::FeatureSouthernIslands);
}

bool isCI(const MCSubtargetInfo &STI) {
  return STI.hasFeature(AMDGPU::FeatureSeaIslands);
}

bool isVI(const MCSubtargetInfo &STI) {
  return STI.hasFeature(AMDGPU::FeatureVolcanicIslands);
}

bool isGFX9(const MCSubtargetInfo &STI) {
  return STI.hasFeature(AMDGPU::FeatureGFX9);
}

bool isGFX9_GFX10(const MCSubtargetInfo &STI) {
  return isGFX9(STI) || isGFX10(STI);
}

bool isGFX9_GFX10_GFX11(const MCSubtargetInfo &STI) {
  return isGFX9(STI) || isGFX10(STI) || isGFX11(STI);
}

bool isGFX8_GFX9_GFX10(const MCSubtargetInfo &STI) {
  return isVI(STI) || isGFX9(STI) || isGFX10(STI);
}

bool isGFX8Plus(const MCSubtargetInfo &STI) {
  return isVI(STI) || isGFX9Plus(STI);
}

bool isGFX9Plus(const MCSubtargetInfo &STI) {
  return isGFX9(STI) || isGFX10Plus(STI);
}

bool isNotGFX9Plus(const MCSubtargetInfo &STI) { return !isGFX9Plus(STI); }

bool hasPopsExitingWaveID(const MCSubtargetInfo &STI) {
  return STI.hasFeature(AMDGPU::FeaturePopsExitingWaveID);
}

bool hasPrivateApertureRegs(const MCSubtargetInfo &STI) {
  return STI.hasFeature(AMDGPU::FeatureApertureRegs) &&
         !STI.hasFeature(AMDGPU::FeatureGloballyAddressableScratch);
}

bool isGFX10(const MCSubtargetInfo &STI) {
  return STI.hasFeature(AMDGPU::FeatureGFX10);
}

bool isGFX10_GFX11(const MCSubtargetInfo &STI) {
  return isGFX10(STI) || isGFX11(STI);
}

bool isGFX10Plus(const MCSubtargetInfo &STI) {
  return isGFX10(STI) || isGFX11Plus(STI);
}

bool isGFX11(const MCSubtargetInfo &STI) {
  return STI.hasFeature(AMDGPU::FeatureGFX11);
}

bool isGFX11Plus(const MCSubtargetInfo &STI) {
  return isGFX11(STI) || isGFX12Plus(STI);
}

bool isGFX12(const MCSubtargetInfo &STI) {
  return STI.getFeatureBits()[AMDGPU::FeatureGFX12];
}

bool isGFX12Plus(const MCSubtargetInfo &STI) {
  return isGFX12(STI) || isGFX13Plus(STI);
}

bool isNotGFX12Plus(const MCSubtargetInfo &STI) { return !isGFX12Plus(STI); }

bool isGFX1250(const MCSubtargetInfo &STI) {
  return STI.getFeatureBits()[AMDGPU::FeatureGFX1250Insts] && !isGFX13(STI);
}

bool isFullSIMDMode(const MCSubtargetInfo &STI) {
  return isGFX1250(STI) || !STI.getFeatureBits().test(FeatureCuMode);
}

bool isGFX1250Plus(const MCSubtargetInfo &STI) {
  return STI.getFeatureBits()[AMDGPU::FeatureGFX1250Insts];
}

bool isGFX13(const MCSubtargetInfo &STI) {
  return STI.getFeatureBits()[AMDGPU::FeatureGFX13];
}

bool isGFX13Plus(const MCSubtargetInfo &STI) { return isGFX13(STI); }

bool supportsWGP(const MCSubtargetInfo &STI) {
  if (isGFX1250(STI))
    return false;
  return isGFX10Plus(STI);
}

bool isNotGFX11Plus(const MCSubtargetInfo &STI) { return !isGFX11Plus(STI); }

bool isNotGFX10Plus(const MCSubtargetInfo &STI) {
  return isSI(STI) || isCI(STI) || isVI(STI) || isGFX9(STI);
}

bool isGFX10Before1030(const MCSubtargetInfo &STI) {
  return isGFX10(STI) && !AMDGPU::isGFX10_BEncoding(STI);
}

bool isGCN3Encoding(const MCSubtargetInfo &STI) {
  return STI.hasFeature(AMDGPU::FeatureGCN3Encoding);
}

bool isGFX10_BEncoding(const MCSubtargetInfo &STI) {
  return STI.hasFeature(AMDGPU::FeatureGFX10_BEncoding);
}

bool hasGFX10_3Insts(const MCSubtargetInfo &STI) {
  return STI.hasFeature(AMDGPU::FeatureGFX10_3Insts);
}

bool isGFX10_3_GFX11(const MCSubtargetInfo &STI) {
  return isGFX10_BEncoding(STI) && !isGFX12Plus(STI);
}

bool isGFX90A(const MCSubtargetInfo &STI) {
  return STI.hasFeature(AMDGPU::FeatureGFX90AInsts);
}

bool isGFX940(const MCSubtargetInfo &STI) {
  return STI.hasFeature(AMDGPU::FeatureGFX940Insts);
}

bool hasArchitectedFlatScratch(const MCSubtargetInfo &STI) {
  return STI.hasFeature(AMDGPU::FeatureArchitectedFlatScratch);
}

bool hasMAIInsts(const MCSubtargetInfo &STI) {
  return STI.hasFeature(AMDGPU::FeatureMAIInsts);
}

bool hasVOPD(const MCSubtargetInfo &STI) {
  return STI.hasFeature(AMDGPU::FeatureVOPDInsts);
}

bool hasDPPSrc1SGPR(const MCSubtargetInfo &STI) {
  return STI.hasFeature(AMDGPU::FeatureDPPSrc1SGPR);
}

unsigned hasKernargPreload(const MCSubtargetInfo &STI) {
  return STI.hasFeature(AMDGPU::FeatureKernargPreload);
}

int32_t getTotalNumVGPRs(bool has90AInsts, int32_t ArgNumAGPR,
                         int32_t ArgNumVGPR) {
  if (has90AInsts && ArgNumAGPR)
    return alignTo(ArgNumVGPR, 4) + ArgNumAGPR;
  return std::max(ArgNumVGPR, ArgNumAGPR);
}

bool isSGPR(MCRegister Reg, const MCRegisterInfo *TRI) {
  const MCRegisterClass &SGPRClass =
      TRI->getRegClass(AMDGPU::SReg_32RegClassID);
  const MCRegister FirstSubReg = TRI->getSubReg(Reg, AMDGPU::sub0);
  return SGPRClass.contains(FirstSubReg != 0 ? FirstSubReg : Reg) ||
         Reg == AMDGPU::SCC;
}

bool isRsrcIndexReg(MCRegister Reg, const MCRegisterInfo &MRI) {
  return MRI.getRegClass(AMDGPU::RsrcReg32RegClassID).contains(Reg);
}

bool isHi16Reg(MCRegister Reg, const MCRegisterInfo &MRI) {
  return MRI.getEncodingValue(Reg) & AMDGPU::HWEncoding::IS_HI16;
}

#define MAP_REG2REG                                                                                            \
  using namespace AMDGPU;                                                                                      \
  switch (Reg.id()) {                                                                                          \
  default:                                                                                                     \
    return Reg;                                                                                                \
    CASE_CI_VI(FLAT_SCR)                                                                                       \
    CASE_CI_VI(FLAT_SCR_LO)                                                                                    \
    CASE_CI_VI(FLAT_SCR_HI)                                                                                    \
    CASE_VI_GFX9PLUS(TTMP0)                                                                                    \
    CASE_VI_GFX9PLUS(TTMP1)                                                                                    \
    CASE_VI_GFX9PLUS(TTMP2)                                                                                    \
    CASE_VI_GFX9PLUS(TTMP3)                                                                                    \
    CASE_VI_GFX9PLUS(TTMP4)                                                                                    \
    CASE_VI_GFX9PLUS(TTMP5)                                                                                    \
    CASE_VI_GFX9PLUS(TTMP6)                                                                                    \
    CASE_VI_GFX9PLUS(TTMP7)                                                                                    \
    CASE_VI_GFX9PLUS(TTMP8)                                                                                    \
    CASE_VI_GFX9PLUS(TTMP9)                                                                                    \
    CASE_VI_GFX9PLUS(TTMP10)                                                                                   \
    CASE_VI_GFX9PLUS(TTMP11)                                                                                   \
    CASE_VI_GFX9PLUS(TTMP12)                                                                                   \
    CASE_VI_GFX9PLUS(TTMP13)                                                                                   \
    CASE_VI_GFX9PLUS(TTMP14)                                                                                   \
    CASE_VI_GFX9PLUS(TTMP15)                                                                                   \
    CASE_VI_GFX9PLUS(TTMP0_TTMP1)                                                                              \
    CASE_VI_GFX9PLUS(TTMP2_TTMP3)                                                                              \
    CASE_VI_GFX9PLUS(TTMP4_TTMP5)                                                                              \
    CASE_VI_GFX9PLUS(TTMP6_TTMP7)                                                                              \
    CASE_VI_GFX9PLUS(TTMP8_TTMP9)                                                                              \
    CASE_VI_GFX9PLUS(TTMP10_TTMP11)                                                                            \
    CASE_VI_GFX9PLUS(TTMP12_TTMP13)                                                                            \
    CASE_VI_GFX9PLUS(TTMP14_TTMP15)                                                                            \
    CASE_VI_GFX9PLUS(TTMP0_TTMP1_TTMP2_TTMP3)                                                                  \
    CASE_VI_GFX9PLUS(TTMP4_TTMP5_TTMP6_TTMP7)                                                                  \
    CASE_VI_GFX9PLUS(TTMP8_TTMP9_TTMP10_TTMP11)                                                                \
    CASE_VI_GFX9PLUS(TTMP12_TTMP13_TTMP14_TTMP15)                                                              \
    CASE_VI_GFX9PLUS(TTMP0_TTMP1_TTMP2_TTMP3_TTMP4_TTMP5_TTMP6_TTMP7)                                          \
    CASE_VI_GFX9PLUS(TTMP4_TTMP5_TTMP6_TTMP7_TTMP8_TTMP9_TTMP10_TTMP11)                                        \
    CASE_VI_GFX9PLUS(TTMP8_TTMP9_TTMP10_TTMP11_TTMP12_TTMP13_TTMP14_TTMP15)                                    \
    CASE_VI_GFX9PLUS(                                                                                          \
        TTMP0_TTMP1_TTMP2_TTMP3_TTMP4_TTMP5_TTMP6_TTMP7_TTMP8_TTMP9_TTMP10_TTMP11_TTMP12_TTMP13_TTMP14_TTMP15) \
    CASE_GFXPRE11_GFX11PLUS(M0)                                                                                \
    CASE_GFXPRE11_GFX11PLUS(SGPR_NULL)                                                                         \
    CASE_GFXPRE11_GFX11PLUS_TO(SGPR_NULL64, SGPR_NULL)                                                         \
  }

#define CASE_CI_VI(node)                                                       \
  assert(!isSI(STI));                                                          \
  case node:                                                                   \
    return isCI(STI) ? node##_ci : node##_vi;

#define CASE_VI_GFX9PLUS(node)                                                 \
  case node:                                                                   \
    return isGFX9Plus(STI) ? node##_gfx9plus : node##_vi;

#define CASE_GFXPRE11_GFX11PLUS(node)                                          \
  case node:                                                                   \
    return isGFX11Plus(STI) ? node##_gfx11plus : node##_gfxpre11;

#define CASE_GFXPRE11_GFX11PLUS_TO(node, result)                               \
  case node:                                                                   \
    return isGFX11Plus(STI) ? result##_gfx11plus : result##_gfxpre11;

MCRegister getMCReg(MCRegister Reg, const MCSubtargetInfo &STI) {
  if (STI.getTargetTriple().getArch() == Triple::r600)
    return Reg;
  MAP_REG2REG
}

#undef CASE_CI_VI
#undef CASE_VI_GFX9PLUS
#undef CASE_GFXPRE11_GFX11PLUS
#undef CASE_GFXPRE11_GFX11PLUS_TO

#define CASE_CI_VI(node)                                                       \
  case node##_ci:                                                              \
  case node##_vi:                                                              \
    return node;
#define CASE_VI_GFX9PLUS(node)                                                 \
  case node##_vi:                                                              \
  case node##_gfx9plus:                                                        \
    return node;
#define CASE_GFXPRE11_GFX11PLUS(node)                                          \
  case node##_gfx11plus:                                                       \
  case node##_gfxpre11:                                                        \
    return node;
#define CASE_GFXPRE11_GFX11PLUS_TO(node, result)

MCRegister mc2PseudoReg(MCRegister Reg) { MAP_REG2REG }

bool isInlineValue(MCRegister Reg) {
  switch (Reg.id()) {
  case AMDGPU::SRC_SHARED_BASE_LO:
  case AMDGPU::SRC_SHARED_BASE:
  case AMDGPU::SRC_SHARED_LIMIT_LO:
  case AMDGPU::SRC_SHARED_LIMIT:
  case AMDGPU::SRC_PRIVATE_BASE_LO:
  case AMDGPU::SRC_PRIVATE_BASE:
  case AMDGPU::SRC_PRIVATE_LIMIT_LO:
  case AMDGPU::SRC_PRIVATE_LIMIT:
  case AMDGPU::SRC_FLAT_SCRATCH_BASE_LO:
  case AMDGPU::SRC_FLAT_SCRATCH_BASE_HI:
  case AMDGPU::SRC_POPS_EXITING_WAVE_ID:
    return true;
  case AMDGPU::SRC_VCCZ:
  case AMDGPU::SRC_EXECZ:
  case AMDGPU::SRC_SCC:
    return true;
  case AMDGPU::SGPR_NULL:
    return true;
  default:
    return false;
  }
}

#undef CASE_CI_VI
#undef CASE_VI_GFX9PLUS
#undef CASE_GFXPRE11_GFX11PLUS
#undef CASE_GFXPRE11_GFX11PLUS_TO
#undef MAP_REG2REG

bool isKImmOperand(const MCInstrDesc &Desc, unsigned OpNo) {
  assert(OpNo < Desc.NumOperands);
  unsigned OpType = Desc.operands()[OpNo].OperandType;
  return OpType >= AMDGPU::OPERAND_KIMM_FIRST &&
         OpType <= AMDGPU::OPERAND_KIMM_LAST;
}

bool isSISrcFPOperand(const MCInstrDesc &Desc, unsigned OpNo) {
  assert(OpNo < Desc.NumOperands);
  unsigned OpType = Desc.operands()[OpNo].OperandType;
  switch (OpType) {
  case AMDGPU::OPERAND_REG_IMM_FP32:
  case AMDGPU::OPERAND_REG_IMM_FP64:
  case AMDGPU::OPERAND_REG_IMM_FP16:
  case AMDGPU::OPERAND_REG_IMM_NOINLINE_FP16:
  case AMDGPU::OPERAND_REG_IMM_V2FP16:
  case AMDGPU::OPERAND_REG_IMM_V2FP16_SPLAT:
  case AMDGPU::OPERAND_REG_IMM_NOINLINE_V2FP16:
  case AMDGPU::OPERAND_REG_INLINE_C_FP32:
  case AMDGPU::OPERAND_REG_INLINE_C_FP64:
  case AMDGPU::OPERAND_REG_INLINE_C_FP16:
  case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
  case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
  case AMDGPU::OPERAND_REG_IMM_V2FP32:
  case AMDGPU::OPERAND_REG_INLINE_AC_FP64:
  case AMDGPU::OPERAND_REG_IMM_V2FP64:
    return true;
  default:
    return false;
  }
}

// Avoid using MCRegisterClass::getSize, since that function will go away
// (move from MC* level to Target* level). Return size in bits.
unsigned getRegBitWidth(unsigned RCID) {
  switch (RCID) {
  case AMDGPU::VGPR_16RegClassID:
  case AMDGPU::VGPR_16_Lo128RegClassID:
  case AMDGPU::SGPR_LO16RegClassID:
  case AMDGPU::AGPR_LO16RegClassID:
    return 16;
  case AMDGPU::SGPR_32RegClassID:
  case AMDGPU::VGPR_32RegClassID:
  case AMDGPU::VGPR_32_Lo256RegClassID:
  case AMDGPU::VRegOrLds_32RegClassID:
  case AMDGPU::AGPR_32RegClassID:
  case AMDGPU::VS_32RegClassID:
  case AMDGPU::AV_32RegClassID:
  case AMDGPU::SReg_32RegClassID:
  case AMDGPU::SReg_32_XM0RegClassID:
  case AMDGPU::SRegOrLds_32RegClassID:
    return 32;
  case AMDGPU::SGPR_64RegClassID:
  case AMDGPU::VS_64RegClassID:
  case AMDGPU::SReg_64RegClassID:
  case AMDGPU::VReg_64RegClassID:
  case AMDGPU::AReg_64RegClassID:
  case AMDGPU::SReg_64_XEXECRegClassID:
  case AMDGPU::VReg_64_Align2RegClassID:
  case AMDGPU::AReg_64_Align2RegClassID:
  case AMDGPU::AV_64RegClassID:
  case AMDGPU::AV_64_Align2RegClassID:
  case AMDGPU::VReg_64_Lo256_Align2RegClassID:
  case AMDGPU::VS_64_Lo256RegClassID:
    return 64;
  case AMDGPU::SGPR_96RegClassID:
  case AMDGPU::SReg_96RegClassID:
  case AMDGPU::VReg_96RegClassID:
  case AMDGPU::AReg_96RegClassID:
  case AMDGPU::VReg_96_Align2RegClassID:
  case AMDGPU::AReg_96_Align2RegClassID:
  case AMDGPU::AV_96RegClassID:
  case AMDGPU::AV_96_Align2RegClassID:
  case AMDGPU::VReg_96_Lo256_Align2RegClassID:
    return 96;
  case AMDGPU::SGPR_128RegClassID:
  case AMDGPU::SReg_128RegClassID:
  case AMDGPU::VReg_128RegClassID:
  case AMDGPU::AReg_128RegClassID:
  case AMDGPU::VReg_128_Align2RegClassID:
  case AMDGPU::AReg_128_Align2RegClassID:
  case AMDGPU::AV_128RegClassID:
  case AMDGPU::AV_128_Align2RegClassID:
  case AMDGPU::SReg_128_XNULLRegClassID:
  case AMDGPU::VReg_128_Lo256_Align2RegClassID:
    return 128;
  case AMDGPU::SGPR_160RegClassID:
  case AMDGPU::SReg_160RegClassID:
  case AMDGPU::VReg_160RegClassID:
  case AMDGPU::AReg_160RegClassID:
  case AMDGPU::VReg_160_Align2RegClassID:
  case AMDGPU::AReg_160_Align2RegClassID:
  case AMDGPU::AV_160RegClassID:
  case AMDGPU::AV_160_Align2RegClassID:
  case AMDGPU::VReg_160_Lo256_Align2RegClassID:
    return 160;
  case AMDGPU::SGPR_192RegClassID:
  case AMDGPU::SReg_192RegClassID:
  case AMDGPU::VReg_192RegClassID:
  case AMDGPU::AReg_192RegClassID:
  case AMDGPU::VReg_192_Align2RegClassID:
  case AMDGPU::AReg_192_Align2RegClassID:
  case AMDGPU::AV_192RegClassID:
  case AMDGPU::AV_192_Align2RegClassID:
  case AMDGPU::VReg_192_Lo256_Align2RegClassID:
    return 192;
  case AMDGPU::SGPR_224RegClassID:
  case AMDGPU::SReg_224RegClassID:
  case AMDGPU::VReg_224RegClassID:
  case AMDGPU::AReg_224RegClassID:
  case AMDGPU::VReg_224_Align2RegClassID:
  case AMDGPU::AReg_224_Align2RegClassID:
  case AMDGPU::AV_224RegClassID:
  case AMDGPU::AV_224_Align2RegClassID:
  case AMDGPU::VReg_224_Lo256_Align2RegClassID:
    return 224;
  case AMDGPU::SGPR_256RegClassID:
  case AMDGPU::SReg_256RegClassID:
  case AMDGPU::VReg_256RegClassID:
  case AMDGPU::AReg_256RegClassID:
  case AMDGPU::VReg_256_Align2RegClassID:
  case AMDGPU::AReg_256_Align2RegClassID:
  case AMDGPU::AV_256RegClassID:
  case AMDGPU::AV_256_Align2RegClassID:
  case AMDGPU::SReg_256_XNULLRegClassID:
  case AMDGPU::VReg_256_Lo256_Align2RegClassID:
    return 256;
  case AMDGPU::SGPR_288RegClassID:
  case AMDGPU::SReg_288RegClassID:
  case AMDGPU::VReg_288RegClassID:
  case AMDGPU::AReg_288RegClassID:
  case AMDGPU::VReg_288_Align2RegClassID:
  case AMDGPU::AReg_288_Align2RegClassID:
  case AMDGPU::AV_288RegClassID:
  case AMDGPU::AV_288_Align2RegClassID:
  case AMDGPU::VReg_288_Lo256_Align2RegClassID:
    return 288;
  case AMDGPU::SGPR_320RegClassID:
  case AMDGPU::SReg_320RegClassID:
  case AMDGPU::VReg_320RegClassID:
  case AMDGPU::AReg_320RegClassID:
  case AMDGPU::VReg_320_Align2RegClassID:
  case AMDGPU::AReg_320_Align2RegClassID:
  case AMDGPU::AV_320RegClassID:
  case AMDGPU::AV_320_Align2RegClassID:
  case AMDGPU::VReg_320_Lo256_Align2RegClassID:
    return 320;
  case AMDGPU::SGPR_352RegClassID:
  case AMDGPU::SReg_352RegClassID:
  case AMDGPU::VReg_352RegClassID:
  case AMDGPU::AReg_352RegClassID:
  case AMDGPU::VReg_352_Align2RegClassID:
  case AMDGPU::AReg_352_Align2RegClassID:
  case AMDGPU::AV_352RegClassID:
  case AMDGPU::AV_352_Align2RegClassID:
  case AMDGPU::VReg_352_Lo256_Align2RegClassID:
    return 352;
  case AMDGPU::SGPR_384RegClassID:
  case AMDGPU::SReg_384RegClassID:
  case AMDGPU::VReg_384RegClassID:
  case AMDGPU::AReg_384RegClassID:
  case AMDGPU::VReg_384_Align2RegClassID:
  case AMDGPU::AReg_384_Align2RegClassID:
  case AMDGPU::AV_384RegClassID:
  case AMDGPU::AV_384_Align2RegClassID:
  case AMDGPU::VReg_384_Lo256_Align2RegClassID:
    return 384;
  case AMDGPU::SGPR_512RegClassID:
  case AMDGPU::SReg_512RegClassID:
  case AMDGPU::VReg_512RegClassID:
  case AMDGPU::AReg_512RegClassID:
  case AMDGPU::VReg_512_Align2RegClassID:
  case AMDGPU::AReg_512_Align2RegClassID:
  case AMDGPU::AV_512RegClassID:
  case AMDGPU::AV_512_Align2RegClassID:
  case AMDGPU::VReg_512_Lo256_Align2RegClassID:
    return 512;
  case AMDGPU::SGPR_1024RegClassID:
  case AMDGPU::SReg_1024RegClassID:
  case AMDGPU::VReg_1024RegClassID:
  case AMDGPU::AReg_1024RegClassID:
  case AMDGPU::VReg_1024_Align2RegClassID:
  case AMDGPU::AReg_1024_Align2RegClassID:
  case AMDGPU::AV_1024RegClassID:
  case AMDGPU::AV_1024_Align2RegClassID:
  case AMDGPU::VReg_1024_Lo256_Align2RegClassID:
    return 1024;
  default:
    llvm_unreachable("Unexpected register class");
  }
}

unsigned getRegBitWidth(const MCRegisterClass &RC) {
  return getRegBitWidth(RC.getID());
}

bool isInlinableLiteral64(int64_t Literal, bool HasInv2Pi) {
  if (isInlinableIntLiteral(Literal))
    return true;

  uint64_t Val = static_cast<uint64_t>(Literal);
  return (Val == llvm::bit_cast<uint64_t>(0.0)) ||
         (Val == llvm::bit_cast<uint64_t>(1.0)) ||
         (Val == llvm::bit_cast<uint64_t>(-1.0)) ||
         (Val == llvm::bit_cast<uint64_t>(0.5)) ||
         (Val == llvm::bit_cast<uint64_t>(-0.5)) ||
         (Val == llvm::bit_cast<uint64_t>(2.0)) ||
         (Val == llvm::bit_cast<uint64_t>(-2.0)) ||
         (Val == llvm::bit_cast<uint64_t>(4.0)) ||
         (Val == llvm::bit_cast<uint64_t>(-4.0)) ||
         (Val == 0x3fc45f306dc9c882 && HasInv2Pi);
}

bool isInlinableLiteral32(int32_t Literal, bool HasInv2Pi) {
  if (isInlinableIntLiteral(Literal))
    return true;

  // The actual type of the operand does not seem to matter as long
  // as the bits match one of the inline immediate values.  For example:
  //
  // -nan has the hexadecimal encoding of 0xfffffffe which is -2 in decimal,
  // so it is a legal inline immediate.
  //
  // 1065353216 has the hexadecimal encoding 0x3f800000 which is 1.0f in
  // floating-point, so it is a legal inline immediate.

  uint32_t Val = static_cast<uint32_t>(Literal);
  return (Val == llvm::bit_cast<uint32_t>(0.0f)) ||
         (Val == llvm::bit_cast<uint32_t>(1.0f)) ||
         (Val == llvm::bit_cast<uint32_t>(-1.0f)) ||
         (Val == llvm::bit_cast<uint32_t>(0.5f)) ||
         (Val == llvm::bit_cast<uint32_t>(-0.5f)) ||
         (Val == llvm::bit_cast<uint32_t>(2.0f)) ||
         (Val == llvm::bit_cast<uint32_t>(-2.0f)) ||
         (Val == llvm::bit_cast<uint32_t>(4.0f)) ||
         (Val == llvm::bit_cast<uint32_t>(-4.0f)) ||
         (Val == 0x3e22f983 && HasInv2Pi);
}

bool isInlinableLiteralBF16(int16_t Literal, bool HasInv2Pi) {
  if (!HasInv2Pi)
    return false;
  if (isInlinableIntLiteral(Literal))
    return true;
  uint16_t Val = static_cast<uint16_t>(Literal);
  return Val == 0x3F00 || // 0.5
         Val == 0xBF00 || // -0.5
         Val == 0x3F80 || // 1.0
         Val == 0xBF80 || // -1.0
         Val == 0x4000 || // 2.0
         Val == 0xC000 || // -2.0
         Val == 0x4080 || // 4.0
         Val == 0xC080 || // -4.0
         Val == 0x3E22;   // 1.0 / (2.0 * pi)
}

bool isInlinableLiteralI16(int32_t Literal, bool HasInv2Pi) {
  return isInlinableLiteral32(Literal, HasInv2Pi);
}

bool isInlinableLiteralFP16(int16_t Literal, bool HasInv2Pi) {
  if (!HasInv2Pi)
    return false;
  if (isInlinableIntLiteral(Literal))
    return true;
  uint16_t Val = static_cast<uint16_t>(Literal);
  return Val == 0x3C00 || // 1.0
         Val == 0xBC00 || // -1.0
         Val == 0x3800 || // 0.5
         Val == 0xB800 || // -0.5
         Val == 0x4000 || // 2.0
         Val == 0xC000 || // -2.0
         Val == 0x4400 || // 4.0
         Val == 0xC400 || // -4.0
         Val == 0x3118;   // 1/2pi
}

std::optional<unsigned> getInlineEncodingV216(bool IsFloat, uint32_t Literal) {
  // Unfortunately, the Instruction Set Architecture Reference Guide is
  // misleading about how the inline operands work for (packed) 16-bit
  // instructions. In a nutshell, the actual HW behavior is:
  //
  //  - integer encodings (-16 .. 64) are always produced as sign-extended
  //    32-bit values
  //  - float encodings are produced as:
  //    - for F16 instructions: corresponding half-precision float values in
  //      the LSBs, 0 in the MSBs
  //    - for UI16 instructions: corresponding single-precision float value
  int32_t Signed = static_cast<int32_t>(Literal);
  if (Signed >= 0 && Signed <= 64)
    return 128 + Signed;

  if (Signed >= -16 && Signed <= -1)
    return 192 + std::abs(Signed);

  if (IsFloat) {
    // clang-format off
    switch (Literal) {
    case 0x3800: return 240; // 0.5
    case 0xB800: return 241; // -0.5
    case 0x3C00: return 242; // 1.0
    case 0xBC00: return 243; // -1.0
    case 0x4000: return 244; // 2.0
    case 0xC000: return 245; // -2.0
    case 0x4400: return 246; // 4.0
    case 0xC400: return 247; // -4.0
    case 0x3118: return 248; // 1.0 / (2.0 * pi)
    default: break;
    }
    // clang-format on
  } else {
    // clang-format off
    switch (Literal) {
    case 0x3F000000: return 240; // 0.5
    case 0xBF000000: return 241; // -0.5
    case 0x3F800000: return 242; // 1.0
    case 0xBF800000: return 243; // -1.0
    case 0x40000000: return 244; // 2.0
    case 0xC0000000: return 245; // -2.0
    case 0x40800000: return 246; // 4.0
    case 0xC0800000: return 247; // -4.0
    case 0x3E22F983: return 248; // 1.0 / (2.0 * pi)
    default: break;
    }
    // clang-format on
  }

  return {};
}

// Encoding of the literal as an inline constant for a V_PK_*_IU16 instruction
// or nullopt.
std::optional<unsigned> getInlineEncodingV2I16(uint32_t Literal) {
  return getInlineEncodingV216(false, Literal);
}

// Encoding of the literal as an inline constant for a V_PK_*_BF16 instruction
// or nullopt.
std::optional<unsigned> getInlineEncodingV2BF16(uint32_t Literal) {
  int32_t Signed = static_cast<int32_t>(Literal);
  if (Signed >= 0 && Signed <= 64)
    return 128 + Signed;

  if (Signed >= -16 && Signed <= -1)
    return 192 + std::abs(Signed);

  // clang-format off
  switch (Literal) {
  case 0x3F00: return 240; // 0.5
  case 0xBF00: return 241; // -0.5
  case 0x3F80: return 242; // 1.0
  case 0xBF80: return 243; // -1.0
  case 0x4000: return 244; // 2.0
  case 0xC000: return 245; // -2.0
  case 0x4080: return 246; // 4.0
  case 0xC080: return 247; // -4.0
  case 0x3E22: return 248; // 1.0 / (2.0 * pi)
  default: break;
  }
  // clang-format on

  return std::nullopt;
}

// Encoding of the literal as an inline constant for a V_PK_*_F16 instruction
// or nullopt.
std::optional<unsigned> getInlineEncodingV2F16(uint32_t Literal) {
  return getInlineEncodingV216(true, Literal);
}

// Encoding of the literal as an inline constant for V_PK_FMAC_F16 instruction
// or nullopt. This accounts for different inline constant behavior:
// - Pre-GFX11: fp16 inline constants have the value in low 16 bits, 0 in high
// - GFX11+: fp16 inline constants are duplicated into both halves
std::optional<unsigned> getPKFMACF16InlineEncoding(uint32_t Literal,
                                                   bool IsGFX11Plus) {
  // Pre-GFX11 behavior: f16 in low bits, 0 in high bits
  if (!IsGFX11Plus)
    return getInlineEncodingV216(/*IsFloat=*/true, Literal);

  // GFX11+ behavior: f16 duplicated in both halves
  // First, check for sign-extended integer inline constants (-16 to 64)
  // These work the same across all generations
  int32_t Signed = static_cast<int32_t>(Literal);
  if (Signed >= 0 && Signed <= 64)
    return 128 + Signed;

  if (Signed >= -16 && Signed <= -1)
    return 192 + std::abs(Signed);

  // For float inline constants on GFX11+, both halves must be equal
  uint16_t Lo = static_cast<uint16_t>(Literal);
  uint16_t Hi = static_cast<uint16_t>(Literal >> 16);
  if (Lo != Hi)
    return std::nullopt;
  return getInlineEncodingV216(/*IsFloat=*/true, Lo);
}

// Whether the given literal can be inlined for a V_PK_* instruction.
bool isInlinableLiteralV216(uint32_t Literal, uint8_t OpType) {
  switch (OpType) {
  case AMDGPU::OPERAND_REG_IMM_V2INT16:
  case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
    return getInlineEncodingV216(false, Literal).has_value();
  case AMDGPU::OPERAND_REG_IMM_V2FP16:
  case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
    return getInlineEncodingV216(true, Literal).has_value();
  case AMDGPU::OPERAND_REG_IMM_V2FP16_SPLAT:
    llvm_unreachable("OPERAND_REG_IMM_V2FP16_SPLAT is not supported");
  case AMDGPU::OPERAND_REG_IMM_V2BF16:
  case AMDGPU::OPERAND_REG_INLINE_C_V2BF16:
    return isInlinableLiteralV2BF16(Literal);
  case AMDGPU::OPERAND_REG_IMM_NOINLINE_V2FP16:
    return false;
  default:
    llvm_unreachable("bad packed operand type");
  }
}

// Whether the given literal can be inlined for a V_PK_*_IU16 instruction.
bool isInlinableLiteralV2I16(uint32_t Literal) {
  return getInlineEncodingV2I16(Literal).has_value();
}

// Whether the given literal can be inlined for a V_PK_*_BF16 instruction.
bool isInlinableLiteralV2BF16(uint32_t Literal) {
  return getInlineEncodingV2BF16(Literal).has_value();
}

// Whether the given literal can be inlined for a V_PK_*_F16 instruction.
bool isInlinableLiteralV2F16(uint32_t Literal) {
  return getInlineEncodingV2F16(Literal).has_value();
}

// Whether the given literal can be inlined for V_PK_FMAC_F16 instruction.
bool isPKFMACF16InlineConstant(uint32_t Literal, bool IsGFX11Plus) {
  return getPKFMACF16InlineEncoding(Literal, IsGFX11Plus).has_value();
}

bool isValid32BitLiteral(uint64_t Val, bool IsFP64) {
  if (IsFP64)
    return !Lo_32(Val);

  return isUInt<32>(Val) || isInt<32>(Val);
}

int64_t encode32BitLiteral(int64_t Imm, OperandType Type, bool IsLit) {
  switch (Type) {
  default:
    break;
  case OPERAND_REG_IMM_BF16:
  case OPERAND_REG_IMM_FP16:
  case OPERAND_REG_IMM_NOINLINE_FP16:
  case OPERAND_REG_INLINE_C_BF16:
  case OPERAND_REG_INLINE_C_FP16:
    return Imm & 0xffff;
  case OPERAND_INLINE_SPLIT_BARRIER_INT32:
  case OPERAND_REG_IMM_FP32:
  case OPERAND_REG_IMM_INT32:
  case OPERAND_REG_IMM_V2BF16:
  case OPERAND_REG_IMM_V2FP16:
  case OPERAND_REG_IMM_V2FP16_SPLAT:
  case OPERAND_REG_IMM_V2FP32:
  case OPERAND_REG_IMM_V2INT16:
  case OPERAND_REG_IMM_V2INT32:
  case OPERAND_REG_INLINE_AC_FP32:
  case OPERAND_REG_INLINE_AC_INT32:
  case OPERAND_REG_INLINE_C_FP32:
  case OPERAND_REG_INLINE_C_INT32:
    return Lo_32(Imm);
  case OPERAND_REG_IMM_FP64:
  case AMDGPU::OPERAND_REG_IMM_V2FP64:
    return IsLit ? Imm : Hi_32(Imm);
  }
  return Imm;
}

bool isArgPassedInSGPR(const Argument *A) {
  const Function *F = A->getParent();

  // Arguments to compute shaders are never a source of divergence.
  CallingConv::ID CC = F->getCallingConv();
  switch (CC) {
  case CallingConv::AMDGPU_KERNEL:
  case CallingConv::SPIR_KERNEL:
    return true;
  case CallingConv::AMDGPU_VS:
  case CallingConv::AMDGPU_LS:
  case CallingConv::AMDGPU_HS:
  case CallingConv::AMDGPU_ES:
  case CallingConv::AMDGPU_GS:
  case CallingConv::AMDGPU_PS:
  case CallingConv::AMDGPU_CS:
  case CallingConv::AMDGPU_Gfx:
  case CallingConv::AMDGPU_CS_Chain:
  case CallingConv::AMDGPU_CS_ChainPreserve:
    // For non-compute shaders, SGPR inputs are marked with either inreg or
    // byval. Everything else is in VGPRs.
    return A->hasAttribute(Attribute::InReg) ||
           A->hasAttribute(Attribute::ByVal);
  default:
    // TODO: treat i1 as divergent?
    return A->hasAttribute(Attribute::InReg);
  }
}

bool isArgPassedInSGPR(const CallBase *CB, unsigned ArgNo) {
  // Arguments to compute shaders are never a source of divergence.
  CallingConv::ID CC = CB->getCallingConv();
  switch (CC) {
  case CallingConv::AMDGPU_KERNEL:
  case CallingConv::SPIR_KERNEL:
    return true;
  case CallingConv::AMDGPU_VS:
  case CallingConv::AMDGPU_LS:
  case CallingConv::AMDGPU_HS:
  case CallingConv::AMDGPU_ES:
  case CallingConv::AMDGPU_GS:
  case CallingConv::AMDGPU_PS:
  case CallingConv::AMDGPU_CS:
  case CallingConv::AMDGPU_Gfx:
  case CallingConv::AMDGPU_CS_Chain:
  case CallingConv::AMDGPU_CS_ChainPreserve:
    // For non-compute shaders, SGPR inputs are marked with either inreg or
    // byval. Everything else is in VGPRs.
    return CB->paramHasAttr(ArgNo, Attribute::InReg) ||
           CB->isByValArgument(ArgNo);
  default:
    return CB->paramHasAttr(ArgNo, Attribute::InReg);
  }
}

static bool hasSMEMByteOffset(const MCSubtargetInfo &ST) {
  return isGCN3Encoding(ST) || isGFX10Plus(ST);
}

bool isLegalSMRDEncodedUnsignedOffset(const MCSubtargetInfo &ST,
                                      int64_t EncodedOffset) {
  if (isGFX12Plus(ST))
    return isUInt<23>(EncodedOffset);

  return hasSMEMByteOffset(ST) ? isUInt<20>(EncodedOffset)
                               : isUInt<8>(EncodedOffset);
}

bool isLegalSMRDEncodedSignedOffset(const MCSubtargetInfo &ST,
                                    int64_t EncodedOffset, bool IsBuffer) {
  if (isGFX12Plus(ST)) {
    if (IsBuffer && EncodedOffset < 0)
      return false;
    return isInt<24>(EncodedOffset);
  }

  return !IsBuffer && hasSMRDSignedImmOffset(ST) && isInt<21>(EncodedOffset);
}

static bool isDwordAligned(uint64_t ByteOffset) {
  return (ByteOffset & 3) == 0;
}

uint64_t convertSMRDOffsetUnits(const MCSubtargetInfo &ST,
                                uint64_t ByteOffset) {
  if (hasSMEMByteOffset(ST))
    return ByteOffset;

  assert(isDwordAligned(ByteOffset));
  return ByteOffset >> 2;
}

std::optional<int64_t> getSMRDEncodedOffset(const MCSubtargetInfo &ST,
                                            int64_t ByteOffset, bool IsBuffer,
                                            bool HasSOffset) {
  // For unbuffered smem loads, it is illegal for the Immediate Offset to be
  // negative if the resulting (Offset + (M0 or SOffset or zero) is negative.
  // Handle case where SOffset is not present.
  if (!IsBuffer && !HasSOffset && ByteOffset < 0 && hasSMRDSignedImmOffset(ST))
    return std::nullopt;

  if (isGFX12Plus(ST)) // 24 bit signed offsets
    return isInt<24>(ByteOffset) ? std::optional<int64_t>(ByteOffset)
                                 : std::nullopt;

  // The signed version is always a byte offset.
  if (!IsBuffer && hasSMRDSignedImmOffset(ST)) {
    assert(hasSMEMByteOffset(ST));
    return isInt<20>(ByteOffset) ? std::optional<int64_t>(ByteOffset)
                                 : std::nullopt;
  }

  if (!isDwordAligned(ByteOffset) && !hasSMEMByteOffset(ST))
    return std::nullopt;

  int64_t EncodedOffset = convertSMRDOffsetUnits(ST, ByteOffset);
  return isLegalSMRDEncodedUnsignedOffset(ST, EncodedOffset)
             ? std::optional<int64_t>(EncodedOffset)
             : std::nullopt;
}

std::optional<int64_t> getSMRDEncodedLiteralOffset32(const MCSubtargetInfo &ST,
                                                     int64_t ByteOffset) {
  if (!isCI(ST) || !isDwordAligned(ByteOffset))
    return std::nullopt;

  int64_t EncodedOffset = convertSMRDOffsetUnits(ST, ByteOffset);
  return isUInt<32>(EncodedOffset) ? std::optional<int64_t>(EncodedOffset)
                                   : std::nullopt;
}

unsigned getNumFlatOffsetBits(const MCSubtargetInfo &ST) {
  if (ST.getFeatureBits().test(FeatureFlatOffsetBits12))
    return 12;
  if (ST.getFeatureBits().test(FeatureFlatOffsetBits24))
    return 24;
  return 13;
}

namespace {

struct SourceOfDivergence {
  unsigned Intr;
};
const SourceOfDivergence *lookupSourceOfDivergence(unsigned Intr);

struct AlwaysUniform {
  unsigned Intr;
};
const AlwaysUniform *lookupAlwaysUniform(unsigned Intr);

struct LDSDMAIntrinsic {
  unsigned Intr;
};
const LDSDMAIntrinsic *lookupLDSDMAIntrinsic(unsigned Intr);

#define GET_SourcesOfDivergence_IMPL
#define GET_UniformIntrinsics_IMPL
#define GET_LDSDMAIntrinsicTable_IMPL
#define GET_Gfx9BufferFormat_IMPL
#define GET_Gfx10BufferFormat_IMPL
#define GET_Gfx11PlusBufferFormat_IMPL

#include "AMDGPUGenSearchableTables.inc"

} // end anonymous namespace

bool isIntrinsicSourceOfDivergence(unsigned IntrID) {
  return lookupSourceOfDivergence(IntrID);
}

bool isIntrinsicAlwaysUniform(unsigned IntrID) {
  return lookupAlwaysUniform(IntrID);
}

bool isLDSDMAIntrinsic(unsigned IntrID) {
  return lookupLDSDMAIntrinsic(IntrID);
}

const GcnBufferFormatInfo *getGcnBufferFormatInfo(uint8_t BitsPerComp,
                                                  uint8_t NumComponents,
                                                  uint8_t NumFormat,
                                                  const MCSubtargetInfo &STI) {
  return isGFX11Plus(STI) ? getGfx11PlusBufferFormatInfo(
                                BitsPerComp, NumComponents, NumFormat)
         : isGFX10(STI)
             ? getGfx10BufferFormatInfo(BitsPerComp, NumComponents, NumFormat)
             : getGfx9BufferFormatInfo(BitsPerComp, NumComponents, NumFormat);
}

const GcnBufferFormatInfo *getGcnBufferFormatInfo(uint8_t Format,
                                                  const MCSubtargetInfo &STI) {
  return isGFX11Plus(STI) ? getGfx11PlusBufferFormatInfo(Format)
         : isGFX10(STI)   ? getGfx10BufferFormatInfo(Format)
                          : getGfx9BufferFormatInfo(Format);
}

const MCRegisterClass *getVGPRPhysRegClass(MCRegister Reg,
                                           const MCRegisterInfo &MRI) {
  const unsigned VGPRClasses[] = {
      AMDGPU::VGPR_16RegClassID,  AMDGPU::VGPR_32RegClassID,
      AMDGPU::VReg_64RegClassID,  AMDGPU::VReg_96RegClassID,
      AMDGPU::VReg_128RegClassID, AMDGPU::VReg_160RegClassID,
      AMDGPU::VReg_192RegClassID, AMDGPU::VReg_224RegClassID,
      AMDGPU::VReg_256RegClassID, AMDGPU::VReg_288RegClassID,
      AMDGPU::VReg_320RegClassID, AMDGPU::VReg_352RegClassID,
      AMDGPU::VReg_384RegClassID, AMDGPU::VReg_512RegClassID,
      AMDGPU::VReg_1024RegClassID};

  for (unsigned RCID : VGPRClasses) {
    const MCRegisterClass &RC = MRI.getRegClass(RCID);
    if (RC.contains(Reg))
      return &RC;
  }

  return nullptr;
}

unsigned getVGPREncodingMSBs(MCRegister Reg, const MCRegisterInfo &MRI) {
  unsigned Enc = MRI.getEncodingValue(Reg);
  unsigned Idx = Enc & AMDGPU::HWEncoding::REG_IDX_MASK;
  return Idx >> 8;
}

MCRegister getVGPRWithMSBs(MCRegister Reg, unsigned MSBs,
                           const MCRegisterInfo &MRI) {
  unsigned Enc = MRI.getEncodingValue(Reg);
  unsigned Idx = Enc & AMDGPU::HWEncoding::REG_IDX_MASK;
  if (Idx >= 0x100)
    return MCRegister();

  const MCRegisterClass *RC = getVGPRPhysRegClass(Reg, MRI);
  if (!RC)
    return MCRegister();

  Idx |= MSBs << 8;
  if (RC->getID() == AMDGPU::VGPR_16RegClassID) {
    // This class has 2048 registers with interleaved lo16 and hi16.
    Idx *= 2;
    if (Enc & AMDGPU::HWEncoding::IS_HI16)
      ++Idx;
  }

  return RC->getRegister(Idx);
}

static std::optional<unsigned>
convertSetRegImmToVgprMSBs(uint64_t Imm, uint64_t Simm16,
                           bool HasSetregVGPRMSBFixup) {
  constexpr unsigned VGPRMSBShift =
      llvm::countr_zero_constexpr<unsigned>(AMDGPU::Hwreg::DST_VGPR_MSB);

  auto [HwRegId, Offset, Size] = Hwreg::HwregEncoding::decode(Simm16);
  if (HwRegId != Hwreg::ID_MODE ||
      (!HasSetregVGPRMSBFixup && (Offset + Size) < VGPRMSBShift))
    return {};
  // If there is SetregVGPRMSBFixup then Offset is ignored.
  if (!HasSetregVGPRMSBFixup)
    Imm <<= Offset;
  Imm = (Imm & Hwreg::VGPR_MSB_MASK) >> VGPRMSBShift;
  if (!HasSetregVGPRMSBFixup)
    Imm &= llvm::maskTrailingOnes<unsigned>(Size);
  return llvm::rotr<uint8_t>(static_cast<uint8_t>(Imm), /*R=*/2);
}

std::optional<unsigned> convertSetRegImmToVgprMSBs(const MachineInstr &MI,
                                                   bool HasSetregVGPRMSBFixup) {
  assert(MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32);
  return convertSetRegImmToVgprMSBs(MI.getOperand(0).getImm(),
                                    MI.getOperand(1).getImm(),
                                    HasSetregVGPRMSBFixup);
}

std::optional<unsigned> convertSetRegImmToVgprMSBs(const MCInst &MI,
                                                   bool HasSetregVGPRMSBFixup) {
  assert(MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32_gfx12);
  return convertSetRegImmToVgprMSBs(MI.getOperand(0).getImm(),
                                    MI.getOperand(1).getImm(),
                                    HasSetregVGPRMSBFixup);
}

std::pair<const AMDGPU::OpName *, const AMDGPU::OpName *>
getVGPRLoweringOperandTables(const MCInstrDesc &Desc) {
  static const AMDGPU::OpName VOPOps[4] = {
      AMDGPU::OpName::src0, AMDGPU::OpName::src1, AMDGPU::OpName::src2,
      AMDGPU::OpName::vdst};
  static const AMDGPU::OpName VDSOps[4] = {
      AMDGPU::OpName::addr, AMDGPU::OpName::data0, AMDGPU::OpName::data1,
      AMDGPU::OpName::vdst};
  static const AMDGPU::OpName FLATOps[4] = {
      AMDGPU::OpName::vaddr, AMDGPU::OpName::vdata,
      AMDGPU::OpName::NUM_OPERAND_NAMES, AMDGPU::OpName::vdst};
  static const AMDGPU::OpName BUFOps[4] = {
      AMDGPU::OpName::vaddr, AMDGPU::OpName::NUM_OPERAND_NAMES,
      AMDGPU::OpName::NUM_OPERAND_NAMES, AMDGPU::OpName::vdata};
  static const AMDGPU::OpName VIMGOps[4] = {
      AMDGPU::OpName::vaddr0, AMDGPU::OpName::vaddr1, AMDGPU::OpName::vaddr2,
      AMDGPU::OpName::vdata};

  // For VOPD instructions MSB of a corresponding Y component operand VGPR
  // address is supposed to match X operand, otherwise VOPD shall not be
  // combined.
  static const AMDGPU::OpName VOPDOpsX[4] = {
      AMDGPU::OpName::src0X, AMDGPU::OpName::vsrc1X, AMDGPU::OpName::vsrc2X,
      AMDGPU::OpName::vdstX};
  static const AMDGPU::OpName VOPDOpsY[4] = {
      AMDGPU::OpName::src0Y, AMDGPU::OpName::vsrc1Y, AMDGPU::OpName::vsrc2Y,
      AMDGPU::OpName::vdstY};

  // VOP2 MADMK instructions use src0, imm, src1 scheme.
  static const AMDGPU::OpName VOP2MADMKOps[4] = {
      AMDGPU::OpName::src0, AMDGPU::OpName::NUM_OPERAND_NAMES,
      AMDGPU::OpName::src1, AMDGPU::OpName::vdst};
  static const AMDGPU::OpName VOPDFMAMKOpsX[4] = {
      AMDGPU::OpName::src0X, AMDGPU::OpName::NUM_OPERAND_NAMES,
      AMDGPU::OpName::vsrc1X, AMDGPU::OpName::vdstX};
  static const AMDGPU::OpName VOPDFMAMKOpsY[4] = {
      AMDGPU::OpName::src0Y, AMDGPU::OpName::NUM_OPERAND_NAMES,
      AMDGPU::OpName::vsrc1Y, AMDGPU::OpName::vdstY};

  if (SIInstrFlags::isVOP1(Desc) || SIInstrFlags::isVOP2(Desc) ||
      SIInstrFlags::isVOP3Like(Desc) || SIInstrFlags::isVOPC(Desc) ||
      SIInstrFlags::isDPP(Desc)) {
    switch (Desc.getOpcode()) {
    // LD_SCALE operands ignore MSB.
    case AMDGPU::V_WMMA_LD_SCALE_PAIRED_B32:
    case AMDGPU::V_WMMA_LD_SCALE_PAIRED_B32_gfx1250:
    case AMDGPU::V_WMMA_LD_SCALE16_PAIRED_B64:
    case AMDGPU::V_WMMA_LD_SCALE16_PAIRED_B64_gfx1250:
      return {};
    case AMDGPU::V_FMAMK_F16:
    case AMDGPU::V_FMAMK_F16_t16:
    case AMDGPU::V_FMAMK_F16_t16_gfx12:
    case AMDGPU::V_FMAMK_F16_fake16:
    case AMDGPU::V_FMAMK_F16_fake16_gfx12:
    case AMDGPU::V_FMAMK_F32:
    case AMDGPU::V_FMAMK_F32_gfx12:
    case AMDGPU::V_FMAMK_F64:
    case AMDGPU::V_FMAMK_F64_gfx1250:
      return {VOP2MADMKOps, nullptr};
    default:
      break;
    }
    return {VOPOps, nullptr};
  }

  if (SIInstrFlags::isDS(Desc))
    return {VDSOps, nullptr};

  if (SIInstrFlags::isFLAT(Desc))
    return {FLATOps, nullptr};

  if (SIInstrFlags::isBuffer(Desc))
    return {BUFOps, nullptr};

  if (SIInstrFlags::isVIMAGE(Desc))
    return {VIMGOps, nullptr};

  if (AMDGPU::isVOPD(Desc.getOpcode())) {
    auto [OpX, OpY] = getVOPDComponents(Desc.getOpcode());
    return {(OpX == AMDGPU::V_FMAMK_F32) ? VOPDFMAMKOpsX : VOPDOpsX,
            (OpY == AMDGPU::V_FMAMK_F32) ? VOPDFMAMKOpsY : VOPDOpsY};
  }

  assert(!SIInstrFlags::isMIMG(Desc));

  if (SIInstrFlags::isVSAMPLE(Desc) || SIInstrFlags::isEXP(Desc))
    llvm_unreachable("Sample and export VGPR lowering is not implemented and"
                     " these instructions are not expected on gfx1250");

  return {};
}

bool supportsScaleOffset(const MCInstrInfo &MII, unsigned Opcode) {
  const MCInstrDesc &Desc = MII.get(Opcode);
  if (SIInstrFlags::isSMRD(Desc))
    return Desc.mayLoad() && !Desc.mayStore() && !getSMEMIsBuffer(Opcode);
  if (!SIInstrFlags::isFLAT(Desc))
    return false;

  // Only SV and SVS modes are supported.
  if (SIInstrFlags::isFlatScratch(MII, Opcode))
    return hasNamedOperand(Opcode, OpName::vaddr);

  // Only GVS mode is supported.
  return hasNamedOperand(Opcode, OpName::vaddr) &&
         hasNamedOperand(Opcode, OpName::saddr);

  return false;
}

static bool hasAny64BitVGPROperands(const MCInstrDesc &OpDesc,
                                    const MCInstrInfo &MII,
                                    const MCSubtargetInfo &ST) {
  for (auto OpName : {OpName::vdst, OpName::src0, OpName::src1, OpName::src2}) {
    int Idx = getNamedOperandIdx(OpDesc.getOpcode(), OpName);
    if (Idx == -1)
      continue;

    const MCOperandInfo &OpInfo = OpDesc.operands()[Idx];
    int16_t RegClass = MII.getOpRegClassID(
        OpInfo, ST.getHwMode(MCSubtargetInfo::HwMode_RegInfo));
    if (RegClass == AMDGPU::VReg_64RegClassID ||
        RegClass == AMDGPU::VReg_64_Align2RegClassID)
      return true;
  }

  return false;
}

bool isDPALU_DPP32BitOpc(unsigned Opc) {
  switch (Opc) {
  case AMDGPU::V_MUL_LO_U32_e64:
  case AMDGPU::V_MUL_LO_U32_e64_dpp:
  case AMDGPU::V_MUL_LO_U32_e64_dpp_gfx1250:
  case AMDGPU::V_MUL_HI_U32_e64:
  case AMDGPU::V_MUL_HI_U32_e64_dpp:
  case AMDGPU::V_MUL_HI_U32_e64_dpp_gfx1250:
  case AMDGPU::V_MUL_HI_I32_e64:
  case AMDGPU::V_MUL_HI_I32_e64_dpp:
  case AMDGPU::V_MUL_HI_I32_e64_dpp_gfx1250:
  case AMDGPU::V_MAD_U32_e64:
  case AMDGPU::V_MAD_U32_e64_dpp:
  case AMDGPU::V_MAD_U32_e64_dpp_gfx1250:
    return true;
  default:
    return false;
  }
}

bool isDPALU_DPP(const MCInstrDesc &OpDesc, const MCInstrInfo &MII,
                 const MCSubtargetInfo &ST) {
  if (isDPALU_DPP32BitOpc(OpDesc.getOpcode()))
    return true;

  return hasAny64BitVGPROperands(OpDesc, MII, ST);
}

bool isPackedSingleSGPRFP32Inst(unsigned Opc) {
  switch (Opc) {
  case AMDGPU::V_PK_ADD_F32_gfx1250:
  case AMDGPU::V_PK_ADD_F32_gfx1250_gfx12:
  case AMDGPU::V_PK_MUL_F32_gfx1250:
  case AMDGPU::V_PK_MUL_F32_gfx1250_gfx12:
  case AMDGPU::V_PK_FMA_F32_gfx1250:
  case AMDGPU::V_PK_FMA_F32_gfx1250_gfx12:
    return true;
  default:
    return false;
  }
}

// NOTE: This function is currently only used before pseudo-expansion.
bool isPackedSingleSGPR64BitInst(unsigned Opc) {
  switch (Opc) {
  case AMDGPU::V_PK_ADD_F64:
  case AMDGPU::V_PK_MUL_F64:
  case AMDGPU::V_PK_FMA_F64:
  case AMDGPU::V_PK_MAX_NUM_F64:
  case AMDGPU::V_PK_MIN_NUM_F64:
  case AMDGPU::V_PK_ADD_NC_U64:
  case AMDGPU::V_PK_SUB_NC_U64:
  case AMDGPU::V_PK_LSHL_ADD_U64:
    return true;
  default:
    return false;
  }
}

bool isSingleSGPRReadInst(unsigned Opc) {
  return isPackedSingleSGPRFP32Inst(Opc) || isPackedSingleSGPR64BitInst(Opc);
}

const std::array<unsigned, 3> &ClusterDimsAttr::getDims() const {
  assert(isFixedDims() && "expect kind to be FixedDims");
  return Dims;
}

std::string ClusterDimsAttr::to_string() const {
  SmallString<10> Buffer;
  raw_svector_ostream OS(Buffer);

  switch (getKind()) {
  case Kind::Unknown:
    return "";
  case Kind::NoCluster: {
    OS << EncoNoCluster << ',' << EncoNoCluster << ',' << EncoNoCluster;
    return Buffer.c_str();
  }
  case Kind::VariableDims: {
    OS << EncoVariableDims << ',' << EncoVariableDims << ','
       << EncoVariableDims;
    return Buffer.c_str();
  }
  case Kind::FixedDims: {
    OS << Dims[0] << ',' << Dims[1] << ',' << Dims[2];
    return Buffer.c_str();
  }
  }
  llvm_unreachable("Unknown ClusterDimsAttr kind");
}

ClusterDimsAttr ClusterDimsAttr::get(const Function &F) {
  std::optional<SmallVector<unsigned>> Attr =
      getIntegerVecAttribute(F, "amdgpu-cluster-dims", /*Size=*/3);
  ClusterDimsAttr::Kind AttrKind = Kind::FixedDims;

  if (!Attr.has_value())
    AttrKind = Kind::Unknown;
  else if (all_of(*Attr, equal_to(EncoNoCluster)))
    AttrKind = Kind::NoCluster;
  else if (all_of(*Attr, equal_to(EncoVariableDims)))
    AttrKind = Kind::VariableDims;

  ClusterDimsAttr A(AttrKind);
  if (AttrKind == Kind::FixedDims)
    A.Dims = {(*Attr)[0], (*Attr)[1], (*Attr)[2]};

  return A;
}

std::optional<APFloat> evaluateRcp(const APFloat &Val) {
  const fltSemantics &Sem = Val.getSemantics();

  // v_rcp_f16/bf16 are correctly rounded.
  if (&Sem == &APFloat::IEEEhalf() || &Sem == &APFloat::BFloat())
    return APFloat::getOne(Sem) / Val;

  // v_rcp_f32/f64 always flush a denormal input to zero (preserving sign)
  // before reciprocating.
  APFloat Arg = Val;
  if (Arg.isDenormal())
    Arg = APFloat::getZero(Sem, Arg.isNegative());

  APFloat Result = APFloat::getOne(Sem) / Arg;

  // v_rcp_f32/f64 always flush a denormal result to zero (preserving sign).
  if (Result.isDenormal())
    Result = APFloat::getZero(Sem, Result.isNegative());

  // v_rcp_f32/f64 only approximate the reciprocal, except for these special
  // cases where the result is exact.
  if (!Result.isZero() && !Result.isInfinity() && !Result.isNaN() &&
      !Result.isOne() && !Result.isMinusOne())
    return std::nullopt;

  return Result;
}

} // namespace AMDGPU

raw_ostream &operator<<(raw_ostream &OS, const AMDGPU::TargetIDSetting S) {
  switch (S) {
  case (AMDGPU::TargetIDSetting::Unsupported):
    OS << "Unsupported";
    break;
  case (AMDGPU::TargetIDSetting::Any):
    OS << "Any";
    break;
  case (AMDGPU::TargetIDSetting::Off):
    OS << "Off";
    break;
  case (AMDGPU::TargetIDSetting::On):
    OS << "On";
    break;
  }
  return OS;
}

} // namespace llvm