//===- AMDGPUTargetTransformInfo.cpp - AMDGPU specific TTI pass -----------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// \file
// This file implements a TargetTransformInfo analysis pass specific to the
// AMDGPU target machine. It uses the target's detailed information to provide
// more precise answers to certain TTI queries, while letting the target
// independent and default TTI implementations handle the rest.
//
//===----------------------------------------------------------------------===//

#include "AMDGPUTargetTransformInfo.h"
#include "AMDGPUSubtarget.h"
#include "AMDGPUTargetMachine.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "SIModeRegisterDefaults.h"
#include "llvm/ADT/SmallBitVector.h"
#include "llvm/Analysis/InlineCost.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/CodeGen/Analysis.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/IR/PatternMatch.h"
#include "llvm/Support/KnownBits.h"
#include <optional>

using namespace llvm;

#define DEBUG_TYPE "AMDGPUtti"

static cl::opt<unsigned> UnrollThresholdPrivate(
  "amdgpu-unroll-threshold-private",
  cl::desc("Unroll threshold for AMDGPU if private memory used in a loop"),
  cl::init(2700), cl::Hidden);

static cl::opt<unsigned> UnrollThresholdLocal(
  "amdgpu-unroll-threshold-local",
  cl::desc("Unroll threshold for AMDGPU if local memory used in a loop"),
  cl::init(1000), cl::Hidden);

static cl::opt<unsigned> UnrollThresholdIf(
  "amdgpu-unroll-threshold-if",
  cl::desc("Unroll threshold increment for AMDGPU for each if statement inside loop"),
  cl::init(200), cl::Hidden);

static cl::opt<bool> UnrollRuntimeLocal(
  "amdgpu-unroll-runtime-local",
  cl::desc("Allow runtime unroll for AMDGPU if local memory used in a loop"),
  cl::init(true), cl::Hidden);

static cl::opt<unsigned> UnrollMaxBlockToAnalyze(
    "amdgpu-unroll-max-block-to-analyze",
    cl::desc("Inner loop block size threshold to analyze in unroll for AMDGPU"),
    cl::init(32), cl::Hidden);

static cl::opt<unsigned> ArgAllocaCost("amdgpu-inline-arg-alloca-cost",
                                       cl::Hidden, cl::init(4000),
                                       cl::desc("Cost of alloca argument"));

// If the amount of scratch memory to eliminate exceeds our ability to allocate
// it into registers we gain nothing by aggressively inlining functions for that
// heuristic.
static cl::opt<unsigned>
    ArgAllocaCutoff("amdgpu-inline-arg-alloca-cutoff", cl::Hidden,
                    cl::init(256),
                    cl::desc("Maximum alloca size to use for inline cost"));

// Inliner constraint to achieve reasonable compilation time.
static cl::opt<size_t> InlineMaxBB(
    "amdgpu-inline-max-bb", cl::Hidden, cl::init(1100),
    cl::desc("Maximum number of BBs allowed in a function after inlining"
             " (compile time constraint)"));

// This default unroll factor is based on microbenchmarks on gfx1030.
static cl::opt<unsigned> MemcpyLoopUnroll(
    "amdgpu-memcpy-loop-unroll",
    cl::desc("Unroll factor (affecting 4x32-bit operations) to use for memory "
             "operations when lowering statically-sized memcpy, memmove, or"
             "memset as a loop"),
    cl::init(16), cl::Hidden);

static bool dependsOnLocalPhi(const Loop *L, const Value *Cond,
                              unsigned Depth = 0) {
  const Instruction *I = dyn_cast<Instruction>(Cond);
  if (!I)
    return false;

  if (!L->contains(I))
    return false;
  for (const Value *V : I->operand_values()) {
    if (const PHINode *PHI = dyn_cast<PHINode>(V)) {
      if (llvm::none_of(L->getSubLoops(), [PHI](const Loop* SubLoop) {
                  return SubLoop->contains(PHI); }))
        return true;
    } else if (Depth < 10 && dependsOnLocalPhi(L, V, Depth+1))
      return true;
  }
  return false;
}

AMDGPUTTIImpl::AMDGPUTTIImpl(const AMDGPUTargetMachine *TM, const Function &F)
    : BaseT(TM, F.getDataLayout()),
      TargetTriple(F.getParent()->getTargetTriple()),
      ST(static_cast<const GCNSubtarget *>(TM->getSubtargetImpl(F))),
      TLI(ST->getTargetLowering()) {}

void AMDGPUTTIImpl::getUnrollingPreferences(
    Loop *L, ScalarEvolution &SE, TTI::UnrollingPreferences &UP,
    OptimizationRemarkEmitter *ORE) const {
  const Function &F = *L->getHeader()->getParent();
  UP.Threshold =
      F.getFnAttributeAsParsedInteger("amdgpu-unroll-threshold", 300);
  UP.PartialThreshold =
      F.getFnAttributeAsParsedInteger("amdgpu-partial-unroll-threshold", 150);
  UP.MaxCount = std::numeric_limits<unsigned>::max();
  UP.Partial = true;

  // Conditional branch in a loop back edge needs 3 additional exec
  // manipulations in average.
  UP.BEInsns += 3;

  // We want to run unroll even for the loops which have been vectorized.
  UP.UnrollVectorizedLoop = true;

  // Enable runtime unrolling for loops whose trip count is not known at
  // compile time.
  UP.Runtime = true;

  // Maximum alloca size than can fit registers. Reserve 16 registers.
  const unsigned MaxAlloca = (256 - 16) * 4;
  unsigned ThresholdPrivate = UnrollThresholdPrivate;
  unsigned ThresholdLocal = UnrollThresholdLocal;

  // If this loop has the amdgpu.loop.unroll.threshold metadata we will use the
  // provided threshold value as the default for Threshold
  if (MDNode *LoopUnrollThreshold =
          findOptionMDForLoop(L, "amdgpu.loop.unroll.threshold")) {
    if (LoopUnrollThreshold->getNumOperands() == 2) {
      ConstantInt *MetaThresholdValue = mdconst::extract_or_null<ConstantInt>(
          LoopUnrollThreshold->getOperand(1));
      if (MetaThresholdValue) {
        // We will also use the supplied value for PartialThreshold for now.
        // We may introduce additional metadata if it becomes necessary in the
        // future.
        UP.Threshold = MetaThresholdValue->getSExtValue();
        UP.PartialThreshold = UP.Threshold;
        ThresholdPrivate = std::min(ThresholdPrivate, UP.Threshold);
        ThresholdLocal = std::min(ThresholdLocal, UP.Threshold);
      }
    }
  }

  unsigned MaxBoost = std::max(ThresholdPrivate, ThresholdLocal);
  for (const BasicBlock *BB : L->getBlocks()) {
    const DataLayout &DL = BB->getDataLayout();
    unsigned LocalGEPsSeen = 0;

    if (llvm::any_of(L->getSubLoops(), [BB](const Loop* SubLoop) {
               return SubLoop->contains(BB); }))
        continue; // Block belongs to an inner loop.

    for (const Instruction &I : *BB) {
      // Unroll a loop which contains an "if" statement whose condition
      // defined by a PHI belonging to the loop. This may help to eliminate
      // if region and potentially even PHI itself, saving on both divergence
      // and registers used for the PHI.
      // Add a small bonus for each of such "if" statements.
      if (const CondBrInst *Br = dyn_cast<CondBrInst>(&I)) {
        if (UP.Threshold < MaxBoost) {
          BasicBlock *Succ0 = Br->getSuccessor(0);
          BasicBlock *Succ1 = Br->getSuccessor(1);
          if ((L->contains(Succ0) && L->isLoopExiting(Succ0)) ||
              (L->contains(Succ1) && L->isLoopExiting(Succ1)))
            continue;
          if (dependsOnLocalPhi(L, Br->getCondition())) {
            UP.Threshold += UnrollThresholdIf;
            LLVM_DEBUG(dbgs() << "Set unroll threshold " << UP.Threshold
                              << " for loop:\n"
                              << *L << " due to " << *Br << '\n');
            if (UP.Threshold >= MaxBoost)
              return;
          }
        }
        continue;
      }

      const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(&I);
      if (!GEP)
        continue;

      unsigned AS = GEP->getAddressSpace();
      unsigned Threshold = 0;
      if (AS == AMDGPUAS::PRIVATE_ADDRESS)
        Threshold = ThresholdPrivate;
      else if (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS)
        Threshold = ThresholdLocal;
      else
        continue;

      if (UP.Threshold >= Threshold)
        continue;

      if (AS == AMDGPUAS::PRIVATE_ADDRESS) {
        const Value *Ptr = GEP->getPointerOperand();
        const AllocaInst *Alloca =
            dyn_cast<AllocaInst>(getUnderlyingObject(Ptr));
        if (!Alloca || !Alloca->isStaticAlloca())
          continue;
        auto AllocaSize = Alloca->getAllocationSize(DL);
        if (!AllocaSize || AllocaSize->getFixedValue() > MaxAlloca)
          continue;
      } else if (AS == AMDGPUAS::LOCAL_ADDRESS ||
                 AS == AMDGPUAS::REGION_ADDRESS) {
        LocalGEPsSeen++;
        // Inhibit unroll for local memory if we have seen addressing not to
        // a variable, most likely we will be unable to combine it.
        // Do not unroll too deep inner loops for local memory to give a chance
        // to unroll an outer loop for a more important reason.
        if (LocalGEPsSeen > 1 || L->getLoopDepth() > 2 ||
            (!isa<GlobalVariable>(GEP->getPointerOperand()) &&
             !isa<Argument>(GEP->getPointerOperand())))
          continue;
        LLVM_DEBUG(dbgs() << "Allow unroll runtime for loop:\n"
                          << *L << " due to LDS use.\n");
        UP.Runtime = UnrollRuntimeLocal;
      }

      // Check if GEP depends on a value defined by this loop itself.
      bool HasLoopDef = false;
      for (const Value *Op : GEP->operands()) {
        const Instruction *Inst = dyn_cast<Instruction>(Op);
        if (!Inst || L->isLoopInvariant(Op))
          continue;

        if (llvm::any_of(L->getSubLoops(), [Inst](const Loop* SubLoop) {
             return SubLoop->contains(Inst); }))
          continue;
        HasLoopDef = true;
        break;
      }
      if (!HasLoopDef)
        continue;

      // We want to do whatever we can to limit the number of alloca
      // instructions that make it through to the code generator.  allocas
      // require us to use indirect addressing, which is slow and prone to
      // compiler bugs.  If this loop does an address calculation on an
      // alloca ptr, then we want to use a higher than normal loop unroll
      // threshold. This will give SROA a better chance to eliminate these
      // allocas.
      //
      // We also want to have more unrolling for local memory to let ds
      // instructions with different offsets combine.
      //
      // Don't use the maximum allowed value here as it will make some
      // programs way too big.
      UP.Threshold = Threshold;
      LLVM_DEBUG(dbgs() << "Set unroll threshold " << Threshold
                        << " for loop:\n"
                        << *L << " due to " << *GEP << '\n');
      if (UP.Threshold >= MaxBoost)
        return;
    }

    // If we got a GEP in a small BB from inner loop then increase max trip
    // count to analyze for better estimation cost in unroll
    if (L->isInnermost() && BB->size() < UnrollMaxBlockToAnalyze)
      UP.MaxIterationsCountToAnalyze = 32;
  }
}

void AMDGPUTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
                                          TTI::PeelingPreferences &PP) const {
  BaseT::getPeelingPreferences(L, SE, PP);
}

uint64_t AMDGPUTTIImpl::getMaxMemIntrinsicInlineSizeThreshold() const {
  return 1024;
}

GCNTTIImpl::GCNTTIImpl(const AMDGPUTargetMachine *TM, const Function &F)
    : BaseT(TM, F.getDataLayout()),
      ST(static_cast<const GCNSubtarget *>(TM->getSubtargetImpl(F))),
      TLI(ST->getTargetLowering()), CommonTTI(TM, F),
      IsGraphics(AMDGPU::isGraphics(F.getCallingConv())) {
  SIModeRegisterDefaults Mode(F, *ST);
  HasFP32Denormals = Mode.FP32Denormals != DenormalMode::getPreserveSign();
}

bool GCNTTIImpl::hasBranchDivergence(const Function *F) const {
  return !F || !ST->isSingleLaneExecution(*F);
}

unsigned GCNTTIImpl::getNumberOfRegisters(unsigned RCID) const {
  // NB: RCID is not an RCID. In fact it is 0 or 1 for scalar or vector
  // registers. See getRegisterClassForType for the implementation.
  // In this case vector registers are not vector in terms of
  // VGPRs, but those which can hold multiple values.

  // This is really the number of registers to fill when vectorizing /
  // interleaving loops, so we lie to avoid trying to use all registers.
  return 4;
}

TypeSize
GCNTTIImpl::getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const {
  switch (K) {
  case TargetTransformInfo::RGK_Scalar:
    return TypeSize::getFixed(32);
  case TargetTransformInfo::RGK_FixedWidthVector:
    return TypeSize::getFixed(
        (ST->hasAnyPackedFP64Ops() || ST->hasAnyPackedU64Ops()) ? 128
        : ST->hasAnyPackedFP32Ops()                             ? 64
                                                                : 32);
  case TargetTransformInfo::RGK_ScalableVector:
    return TypeSize::getScalable(0);
  }
  llvm_unreachable("Unsupported register kind");
}

unsigned GCNTTIImpl::getMinVectorRegisterBitWidth() const {
  return 32;
}

unsigned GCNTTIImpl::getMaximumVF(unsigned ElemWidth, unsigned Opcode) const {
  if (Opcode == Instruction::Load || Opcode == Instruction::Store)
    return 32 * 4 / ElemWidth;
  // For a given width return the max 0number of elements that can be combined
  // into a wider bit value:
  return (ElemWidth == 8 && ST->has16BitInsts())          ? 4
         : (ElemWidth == 16 && ST->has16BitInsts())       ? 2
         : (ElemWidth == 32 && ST->hasAnyPackedFP32Ops()) ? 2
         : (ElemWidth == 64 &&
            (ST->hasAnyPackedFP64Ops() || ST->hasAnyPackedU64Ops()))
             ? 2
             : 1;
}

bool GCNTTIImpl::preferSLPInstCountCheck() const {
  // The integer inst-count heuristic causes regressions on gfx94x and gfx950
  // because 2-element vector trees that pass the scalar/vector instruction
  // count comparison still widen scalar moves (e.g. v_mov_b32 to v_mov_b64)
  // after codegen, increasing register pressure and throughput cost without
  // reducing the total instruction count.
  return !ST->hasGFX940Insts() && !ST->hasGFX950Insts();
}

unsigned GCNTTIImpl::getLoadVectorFactor(unsigned VF, unsigned LoadSize,
                                         unsigned ChainSizeInBytes,
                                         VectorType *VecTy) const {
  unsigned VecRegBitWidth = VF * LoadSize;
  if (VecRegBitWidth > 128 && VecTy->getScalarSizeInBits() < 32)
    // TODO: Support element-size less than 32bit?
    return 128 / LoadSize;

  return VF;
}

unsigned GCNTTIImpl::getStoreVectorFactor(unsigned VF, unsigned StoreSize,
                                             unsigned ChainSizeInBytes,
                                             VectorType *VecTy) const {
  unsigned VecRegBitWidth = VF * StoreSize;
  if (VecRegBitWidth > 128)
    return 128 / StoreSize;

  return VF;
}

unsigned GCNTTIImpl::getLoadStoreVecRegBitWidth(unsigned AddrSpace) const {
  if (AddrSpace == AMDGPUAS::GLOBAL_ADDRESS ||
      AddrSpace == AMDGPUAS::CONSTANT_ADDRESS ||
      AddrSpace == AMDGPUAS::CONSTANT_ADDRESS_32BIT ||
      AddrSpace == AMDGPUAS::BUFFER_FAT_POINTER ||
      AddrSpace == AMDGPUAS::BUFFER_RESOURCE ||
      AddrSpace == AMDGPUAS::BUFFER_STRIDED_POINTER) {
    return 512;
  }

  if (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS)
    return 8 * ST->getMaxPrivateElementSize();

  // Common to flat, global, local and region. Assume for unknown addrspace.
  return 128;
}

bool GCNTTIImpl::isLegalToVectorizeMemChain(unsigned ChainSizeInBytes,
                                            Align Alignment,
                                            unsigned AddrSpace) const {
  // We allow vectorization of flat stores, even though we may need to decompose
  // them later if they may access private memory. We don't have enough context
  // here, and legalization can handle it.
  if (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS) {
    return (Alignment >= 4 || ST->hasUnalignedScratchAccessEnabled()) &&
           ChainSizeInBytes <= ST->getMaxPrivateElementSize();
  }
  return true;
}

bool GCNTTIImpl::isLegalToVectorizeLoadChain(unsigned ChainSizeInBytes,
                                             Align Alignment,
                                             unsigned AddrSpace) const {
  return isLegalToVectorizeMemChain(ChainSizeInBytes, Alignment, AddrSpace);
}

bool GCNTTIImpl::isLegalToVectorizeStoreChain(unsigned ChainSizeInBytes,
                                              Align Alignment,
                                              unsigned AddrSpace) const {
  return isLegalToVectorizeMemChain(ChainSizeInBytes, Alignment, AddrSpace);
}

uint64_t GCNTTIImpl::getMaxMemIntrinsicInlineSizeThreshold() const {
  return 1024;
}

Type *GCNTTIImpl::getMemcpyLoopLoweringType(
    LLVMContext &Context, Value *Length, unsigned SrcAddrSpace,
    unsigned DestAddrSpace, Align SrcAlign, Align DestAlign,
    std::optional<uint32_t> AtomicElementSize) const {

  if (AtomicElementSize)
    return Type::getIntNTy(Context, *AtomicElementSize * 8);

  // 16-byte accesses achieve the highest copy throughput.
  // If the operation has a fixed known length that is large enough, it is
  // worthwhile to return an even wider type and let legalization lower it into
  // multiple accesses, effectively unrolling the memcpy loop.
  // We also rely on legalization to decompose into smaller accesses for
  // subtargets and address spaces where it is necessary.
  //
  // Don't unroll if Length is not a constant, since unrolling leads to worse
  // performance for length values that are smaller or slightly larger than the
  // total size of the type returned here. Mitigating that would require a more
  // complex lowering for variable-length memcpy and memmove.
  unsigned I32EltsInVector = 4;
  if (MemcpyLoopUnroll > 0 && isa<ConstantInt>(Length))
    return FixedVectorType::get(Type::getInt32Ty(Context),
                                MemcpyLoopUnroll * I32EltsInVector);

  return FixedVectorType::get(Type::getInt32Ty(Context), I32EltsInVector);
}

void GCNTTIImpl::getMemcpyLoopResidualLoweringType(
    SmallVectorImpl<Type *> &OpsOut, LLVMContext &Context,
    unsigned RemainingBytes, unsigned SrcAddrSpace, unsigned DestAddrSpace,
    Align SrcAlign, Align DestAlign,
    std::optional<uint32_t> AtomicCpySize) const {

  if (AtomicCpySize)
    BaseT::getMemcpyLoopResidualLoweringType(
        OpsOut, Context, RemainingBytes, SrcAddrSpace, DestAddrSpace, SrcAlign,
        DestAlign, AtomicCpySize);

  Type *I32x4Ty = FixedVectorType::get(Type::getInt32Ty(Context), 4);
  while (RemainingBytes >= 16) {
    OpsOut.push_back(I32x4Ty);
    RemainingBytes -= 16;
  }

  Type *I64Ty = Type::getInt64Ty(Context);
  while (RemainingBytes >= 8) {
    OpsOut.push_back(I64Ty);
    RemainingBytes -= 8;
  }

  Type *I32Ty = Type::getInt32Ty(Context);
  while (RemainingBytes >= 4) {
    OpsOut.push_back(I32Ty);
    RemainingBytes -= 4;
  }

  Type *I16Ty = Type::getInt16Ty(Context);
  while (RemainingBytes >= 2) {
    OpsOut.push_back(I16Ty);
    RemainingBytes -= 2;
  }

  Type *I8Ty = Type::getInt8Ty(Context);
  while (RemainingBytes) {
    OpsOut.push_back(I8Ty);
    --RemainingBytes;
  }
}

unsigned GCNTTIImpl::getMaxInterleaveFactor(ElementCount VF,
                                            bool HasUnorderedReductions) const {
  // Disable unrolling if the loop is not vectorized.
  // TODO: Enable this again.
  if (VF.isScalar())
    return 1;

  return 8;
}

bool GCNTTIImpl::getTgtMemIntrinsic(IntrinsicInst *Inst,
                                       MemIntrinsicInfo &Info) const {
  switch (Inst->getIntrinsicID()) {
  case Intrinsic::amdgcn_ds_ordered_add:
  case Intrinsic::amdgcn_ds_ordered_swap: {
    auto *Ordering = dyn_cast<ConstantInt>(Inst->getArgOperand(2));
    auto *Volatile = dyn_cast<ConstantInt>(Inst->getArgOperand(4));
    if (!Ordering || !Volatile)
      return false; // Invalid.

    unsigned OrderingVal = Ordering->getZExtValue();
    if (OrderingVal > static_cast<unsigned>(AtomicOrdering::SequentiallyConsistent))
      return false;

    Info.PtrVal = Inst->getArgOperand(0);
    Info.Ordering = static_cast<AtomicOrdering>(OrderingVal);
    Info.ReadMem = true;
    Info.WriteMem = true;
    Info.IsVolatile = !Volatile->isZero();
    return true;
  }
  default:
    return false;
  }
}

/// \returns true if \p FMul and its single fadd/fsub user \p FAddSub are
/// expected to fuse during instruction selection. \p Ty is the type the fused
/// operation runs on.
static bool canFuseFMulWithFAddSub(const SITargetLowering &TLI, Type *Ty,
                                   const Instruction *FMul,
                                   const Instruction *FAddSub) {
  assert((FAddSub->getOpcode() == Instruction::FAdd ||
          FAddSub->getOpcode() == Instruction::FSub) &&
         "Expected an fadd or an fsub");

  // The mad forms fuse exactly without fast-math flags but flush denormals.
  // An fma forms only when it is not slower than the separate operations.
  const Function &F = *FAddSub->getFunction();
  const bool HasFMAD = TLI.isFMADLegal(F, Ty);
  const bool HasFMA = TLI.isFMAFasterThanFMulAndFAdd(F, Ty);
  if (!HasFMAD && !HasFMA)
    return false;

  // Without a mad the pair fuses only when both carry contract.
  return HasFMAD || (FAddSub->hasAllowContract() && FMul->hasAllowContract());
}

/// An fma holds one multiply, so only one fmul operand fuses with \p FAddSub.
static const Instruction *getFusedFMul(const SITargetLowering &TLI, Type *Ty,
                                       const Instruction *FAddSub) {
  for (const Value *Op : FAddSub->operands()) {
    const auto *FMul = dyn_cast<Instruction>(Op);
    if (FMul && FMul->getOpcode() == Instruction::FMul && FMul->hasOneUse() &&
        canFuseFMulWithFAddSub(TLI, Ty, FMul, FAddSub))
      return FMul;
  }
  return nullptr;
}

static bool isFusedFMul(const SITargetLowering &TLI, Type *Ty,
                        const Instruction *FMul, const Instruction *FAddSub) {
  const Instruction *Fused = getFusedFMul(TLI, Ty, FAddSub);
  if (Fused == FMul)
    return true;
  // (a * b + c * d) + e becomes fma(a, b, fma(c, d, e)) if the outer fadd has
  // reassoc.
  if (!Fused || FAddSub->getOpcode() != Instruction::FAdd ||
      !FAddSub->hasOneUse())
    return false;
  const auto *Outer = dyn_cast<BinaryOperator>(*FAddSub->user_begin());
  return Outer && Outer->getOpcode() == Instruction::FAdd &&
         Outer->hasAllowReassoc() &&
         canFuseFMulWithFAddSub(TLI, Ty, FMul, Outer);
}

InstructionCost GCNTTIImpl::getArithmeticInstrCost(
    unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
    TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
    ArrayRef<const Value *> Args, const Instruction *CtxI) const {

  // Legalize the type.
  std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
  int ISD = TLI->InstructionOpcodeToISD(Opcode);

  // Because we don't have any legal vector operations, but the legal types, we
  // need to account for split vectors.
  unsigned NElts = LT.second.isVector() ?
    LT.second.getVectorNumElements() : 1;

  MVT::SimpleValueType SLT = LT.second.getScalarType().SimpleTy;

  switch (ISD) {
  case ISD::SHL:
  case ISD::SRL:
  case ISD::SRA:
    if (SLT == MVT::i64)
      return get64BitInstrCost(CostKind) * LT.first * NElts;

    if (ST->has16BitInsts() && SLT == MVT::i16)
      NElts = (NElts + 1) / 2;

    // i32
    return getFullRateInstrCost() * LT.first * NElts;
  case ISD::ADD:
  case ISD::SUB:
    if (SLT == MVT::i64 && ST->hasAnyPackedU64Ops())
      NElts = (NElts + 1) / 2;
    [[fallthrough]];
  case ISD::AND:
  case ISD::OR:
  case ISD::XOR:
    if (SLT == MVT::i64) {
      // and, or and xor are typically split into 2 VALU instructions.
      return 2 * getFullRateInstrCost() * LT.first * NElts;
    }

    if (ST->has16BitInsts() && SLT == MVT::i16)
      NElts = (NElts + 1) / 2;

    return LT.first * NElts * getFullRateInstrCost();
  case ISD::MUL: {
    const int QuarterRateCost = getQuarterRateInstrCost(CostKind);
    if (SLT == MVT::i64) {
      const int FullRateCost = getFullRateInstrCost();
      return (4 * QuarterRateCost + (2 * 2) * FullRateCost) * LT.first * NElts;
    }

    if (ST->has16BitInsts() && SLT == MVT::i16)
      NElts = (NElts + 1) / 2;

    // i32
    return QuarterRateCost * NElts * LT.first;
  }
  case ISD::FMUL:
    // Check possible fuse {fadd|fsub}(a,fmul(b,c)) and return zero cost for
    // fmul(b,c) supposing the fadd|fsub will get estimated cost for the whole
    // fused operation.
    if (CtxI && CtxI->hasOneUse()) {
      const auto *FAddSub = dyn_cast<BinaryOperator>(*CtxI->user_begin());
      if (FAddSub &&
          (FAddSub->getOpcode() == Instruction::FAdd ||
           FAddSub->getOpcode() == Instruction::FSub) &&
          isFusedFMul(*TLI, Ty, CtxI, FAddSub))
        return TargetTransformInfo::TCC_Free;
    }
    [[fallthrough]];
  case ISD::FADD:
  case ISD::FSUB:
    if (ST->hasAnyPackedFP32Ops() && SLT == MVT::f32)
      NElts = (NElts + 1) / 2;
    if (ST->hasBF16PackedInsts() && SLT == MVT::bf16)
      NElts = (NElts + 1) / 2;
    if (SLT == MVT::f64) {
      if (ST->hasAnyPackedFP64Ops())
        NElts = (NElts + 1) / 2;
      return LT.first * NElts * get64BitInstrCost(CostKind);
    }

    if (ST->has16BitInsts() && SLT == MVT::f16)
      NElts = (NElts + 1) / 2;

    if (SLT == MVT::f32 || SLT == MVT::f16 || SLT == MVT::bf16)
      return LT.first * NElts * getFullRateInstrCost();
    break;
  case ISD::FDIV:
  case ISD::FREM:
    // FIXME: frem should be handled separately. The fdiv in it is most of it,
    // but the current lowering is also not entirely correct.
    if (SLT == MVT::f64) {
      int Cost = 7 * get64BitInstrCost(CostKind) +
                 getQuarterRateInstrCost(CostKind) +
                 3 * getHalfRateInstrCost(CostKind);
      // Add cost of workaround.
      if (!ST->hasUsableDivScaleConditionOutput())
        Cost += 3 * getFullRateInstrCost();

      return LT.first * Cost * NElts;
    }

    if (!Args.empty() && match(Args[0], PatternMatch::m_FPOne())) {
      // TODO: This is more complicated, unsafe flags etc.
      if ((SLT == MVT::f32 && !HasFP32Denormals) ||
          (SLT == MVT::f16 && ST->has16BitInsts())) {
        return LT.first * getTransInstrCost(CostKind) * NElts;
      }
    }

    if (SLT == MVT::f16 && ST->has16BitInsts()) {
      // 2 x v_cvt_f32_f16
      // f32 rcp
      // f32 fmul
      // v_cvt_f16_f32
      // f16 div_fixup
      int Cost = 4 * getFullRateInstrCost() + 2 * getTransInstrCost(CostKind);
      return LT.first * Cost * NElts;
    }

    if (SLT == MVT::f32 && (CtxI && CtxI->hasApproxFunc())) {
      // Fast unsafe fdiv lowering:
      // f32 rcp
      // f32 fmul
      int Cost = getTransInstrCost(CostKind) + getFullRateInstrCost();
      return LT.first * Cost * NElts;
    }

    if (SLT == MVT::f32 || SLT == MVT::f16) {
      // 4 more v_cvt_* insts without f16 insts support
      int Cost = (SLT == MVT::f16 ? 14 : 10) * getFullRateInstrCost() +
                 1 * getTransInstrCost(CostKind);

      if (!HasFP32Denormals) {
        // FP mode switches.
        Cost += 2 * getFullRateInstrCost();
      }

      return LT.first * NElts * Cost;
    }
    break;
  case ISD::FNEG:
    // Use the backend' estimation. If fneg is not free each element will cost
    // one additional instruction.
    return TLI->isFNegFree(SLT) ? 0 : NElts;
  default:
    break;
  }

  return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,
                                       Args, CtxI);
}

// Return true if there's a potential benefit from using v2f16/v2i16
// instructions for an intrinsic, even if it requires nontrivial legalization.
static bool intrinsicHasPackedVectorBenefit(Intrinsic::ID ID) {
  switch (ID) {
  case Intrinsic::fma:
  case Intrinsic::fmuladd:
  case Intrinsic::copysign:
  case Intrinsic::minimumnum:
  case Intrinsic::maximumnum:
  case Intrinsic::canonicalize:
  // There's a small benefit to using vector ops in the legalized code.
  case Intrinsic::round:
  case Intrinsic::uadd_sat:
  case Intrinsic::usub_sat:
  case Intrinsic::sadd_sat:
  case Intrinsic::ssub_sat:
  case Intrinsic::abs:
    return true;
  default:
    return false;
  }
}

InstructionCost
GCNTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
                                  TTI::TargetCostKind CostKind) const {
  switch (ICA.getID()) {
  case Intrinsic::fabs:
    // Free source modifier in the common case.
    return 0;
  case Intrinsic::amdgcn_workitem_id_x:
  case Intrinsic::amdgcn_workitem_id_y:
  case Intrinsic::amdgcn_workitem_id_z:
    // TODO: If hasPackedTID, or if the calling context is not an entry point
    // there may be a bit instruction.
    return 0;
  case Intrinsic::amdgcn_workgroup_id_x:
  case Intrinsic::amdgcn_workgroup_id_y:
  case Intrinsic::amdgcn_workgroup_id_z:
  case Intrinsic::amdgcn_lds_kernel_id:
  case Intrinsic::amdgcn_dispatch_ptr:
  case Intrinsic::amdgcn_dispatch_id:
  case Intrinsic::amdgcn_implicitarg_ptr:
  case Intrinsic::amdgcn_queue_ptr:
    // Read from an argument register.
    return 0;
  default:
    break;
  }

  Type *RetTy = ICA.getReturnType();

  Intrinsic::ID IID = ICA.getID();
  switch (IID) {
  case Intrinsic::exp:
  case Intrinsic::exp2:
  case Intrinsic::exp10: {
    // Legalize the type.
    std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(RetTy);
    MVT::SimpleValueType SLT = LT.second.getScalarType().SimpleTy;
    unsigned NElts =
        LT.second.isVector() ? LT.second.getVectorNumElements() : 1;

    if (SLT == MVT::f64) {
      unsigned NumOps = 20;
      if (IID == Intrinsic::exp)
        ++NumOps;
      else if (IID == Intrinsic::exp10)
        NumOps += 3;

      return LT.first * NElts * NumOps * get64BitInstrCost(CostKind);
    }

    if (SLT == MVT::f32) {
      unsigned NumFullRateOps = 0;
      // v_exp_f32 (transcendental).
      unsigned NumTransOps = 1;

      if (!ICA.getFlags().approxFunc() && IID != Intrinsic::exp2) {
        // Non-AFN exp/exp10: range reduction + v_exp_f32 + ldexp +
        // overflow/underflow checks (lowerFEXP). Denorm is also handled.
        // FMA preamble: ~13 full-rate ops; non-FMA: ~17.
        NumFullRateOps = ST->hasFastFMAF32() ? 13 : 17;
      } else {
        if (IID == Intrinsic::exp) {
          // lowerFEXPUnsafe: fmul (base conversion) + v_exp_f32.
          NumFullRateOps = 1;
        } else if (IID == Intrinsic::exp10) {
          // lowerFEXP10Unsafe: 3 fmul + 2 v_exp_f32 (double-exp2).
          NumFullRateOps = 3;
          NumTransOps = 2;
        }
        // Denorm scaling adds setcc + select + fadd + select + fmul.
        if (HasFP32Denormals)
          NumFullRateOps += 5;
      }

      InstructionCost Cost = NumFullRateOps * getFullRateInstrCost() +
                             NumTransOps * getTransInstrCost(CostKind);
      return LT.first * NElts * Cost;
    }

    break;
  }
  case Intrinsic::log:
  case Intrinsic::log2:
  case Intrinsic::log10: {
    std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(RetTy);
    MVT::SimpleValueType SLT = LT.second.getScalarType().SimpleTy;
    unsigned NElts =
        LT.second.isVector() ? LT.second.getVectorNumElements() : 1;

    if (SLT == MVT::f32) {
      unsigned NumFullRateOps = 0;

      if (IID == Intrinsic::log2) {
        // LowerFLOG2: just v_log_f32.
      } else if (ICA.getFlags().approxFunc()) {
        // LowerFLOGUnsafe: v_log_f32 + fmul (base conversion).
        NumFullRateOps = 1;
      } else {
        // LowerFLOGCommon non-AFN: v_log_f32 + extended-precision
        // multiply + finite check.
        NumFullRateOps = ST->hasFastFMAF32() ? 8 : 11;
      }

      if (HasFP32Denormals)
        NumFullRateOps += 5;

      InstructionCost Cost =
          NumFullRateOps * getFullRateInstrCost() + getTransInstrCost(CostKind);
      return LT.first * NElts * Cost;
    }

    break;
  }
  case Intrinsic::sin:
  case Intrinsic::cos: {
    std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(RetTy);
    MVT::SimpleValueType SLT = LT.second.getScalarType().SimpleTy;
    unsigned NElts =
        LT.second.isVector() ? LT.second.getVectorNumElements() : 1;

    if (SLT == MVT::f32) {
      // LowerTrig: fmul(1/2pi) + v_sin/v_cos.
      unsigned NumFullRateOps = ST->hasTrigReducedRange() ? 2 : 1;

      InstructionCost Cost =
          NumFullRateOps * getFullRateInstrCost() + getTransInstrCost(CostKind);
      return LT.first * NElts * Cost;
    }

    break;
  }
  case Intrinsic::sqrt: {
    std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(RetTy);
    MVT::SimpleValueType SLT = LT.second.getScalarType().SimpleTy;
    unsigned NElts =
        LT.second.isVector() ? LT.second.getVectorNumElements() : 1;

    if (SLT == MVT::f32) {
      unsigned NumFullRateOps = 0;

      if (!ICA.getFlags().approxFunc()) {
        // lowerFSQRTF32 non-AFN: v_sqrt_f32 + refinement + scale fixup.
        NumFullRateOps = HasFP32Denormals ? 17 : 16;
      }

      InstructionCost Cost =
          NumFullRateOps * getFullRateInstrCost() + getTransInstrCost(CostKind);
      return LT.first * NElts * Cost;
    }

    break;
  }
  default:
    break;
  }

  if (!intrinsicHasPackedVectorBenefit(ICA.getID()))
    return BaseT::getIntrinsicInstrCost(ICA, CostKind);

  std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(RetTy);
  MVT::SimpleValueType SLT = LT.second.getScalarType().SimpleTy;
  unsigned NElts = LT.second.isVector() ? LT.second.getVectorNumElements() : 1;

  if ((ST->hasVOP3PInsts() &&
       (SLT == MVT::f16 || SLT == MVT::i16 ||
        (SLT == MVT::bf16 && ST->hasBF16PackedInsts()))) ||
      (ST->hasAnyPackedFP64Ops() && SLT == MVT::f64) ||
      (ST->hasAnyPackedU64Ops() && SLT == MVT::i64)) {
    NElts = (NElts + 1) / 2;
  } else if (SLT == MVT::f32) {
    bool HasPk2FP32Op = ST->hasAnyPackedFP32Ops() &&
                        IID != Intrinsic::minimumnum &&
                        IID != Intrinsic::maximumnum;
    NElts = HasPk2FP32Op ? (NElts + 1) / 2 : NElts;
  }

  // TODO: Get more refined intrinsic costs?
  unsigned InstRate = getQuarterRateInstrCost(CostKind);

  switch (ICA.getID()) {
  case Intrinsic::fma:
  case Intrinsic::fmuladd:
    if (SLT == MVT::f64) {
      InstRate = get64BitInstrCost(CostKind);
      break;
    }

    if ((SLT == MVT::f32 && ST->hasFastFMAF32()) || SLT == MVT::f16)
      InstRate = getFullRateInstrCost();
    else {
      InstRate = ST->hasFastFMAF32() ? getHalfRateInstrCost(CostKind)
                                     : getQuarterRateInstrCost(CostKind);
    }
    break;
  case Intrinsic::copysign:
    return NElts * getFullRateInstrCost();
  case Intrinsic::minimumnum:
  case Intrinsic::maximumnum: {
    // Instruction + 2 canonicalizes. For cases that need type promotion, we the
    // promotion takes the place of the canonicalize.
    unsigned NumOps = 3;
    if (const IntrinsicInst *II = ICA.getInst()) {
      // Directly legal with ieee=0
      // TODO: Not directly legal with strictfp
      if (fpenvIEEEMode(*II) == KnownIEEEMode::Off)
        NumOps = 1;
    }

    unsigned BaseRate =
        SLT == MVT::f64 ? get64BitInstrCost(CostKind) : getFullRateInstrCost();
    InstRate = BaseRate * NumOps;
    break;
  }
  case Intrinsic::canonicalize: {
    InstRate =
        SLT == MVT::f64 ? get64BitInstrCost(CostKind) : getFullRateInstrCost();
    break;
  }
  case Intrinsic::uadd_sat:
  case Intrinsic::usub_sat:
  case Intrinsic::sadd_sat:
  case Intrinsic::ssub_sat: {
    if (SLT == MVT::i16 || SLT == MVT::i32)
      InstRate = getFullRateInstrCost();

    static const auto ValidSatTys = {MVT::v2i16, MVT::v4i16};
    if (any_of(ValidSatTys, equal_to(LT.second)))
      NElts = 1;
    break;
  }
  case Intrinsic::abs:
    // Expansion takes 2 instructions for VALU
    if (SLT == MVT::i16 || SLT == MVT::i32)
      InstRate = 2 * getFullRateInstrCost();
    break;
  default:
    break;
  }

  return LT.first * NElts * InstRate;
}

InstructionCost GCNTTIImpl::getCFInstrCost(unsigned Opcode,
                                           TTI::TargetCostKind CostKind,
                                           const Instruction *I) const {
  assert((I == nullptr || I->getOpcode() == Opcode) &&
         "Opcode should reflect passed instruction.");
  const bool SCost =
      (CostKind == TTI::TCK_CodeSize || CostKind == TTI::TCK_SizeAndLatency);
  const int CBrCost = SCost ? 5 : 7;
  switch (Opcode) {
  case Instruction::UncondBr:
    // Branch instruction takes about 4 slots on gfx900.
    return SCost ? 1 : 4;
  case Instruction::CondBr:
    // Suppose conditional branch takes additional 3 exec manipulations
    // instructions in average.
    return CBrCost;
  case Instruction::Switch: {
    const auto *SI = dyn_cast_or_null<SwitchInst>(I);
    // Each case (including default) takes 1 cmp + 1 cbr instructions in
    // average.
    return (SI ? (SI->getNumCases() + 1) : 4) * (CBrCost + 1);
  }
  case Instruction::Ret:
    return SCost ? 1 : 10;
  }
  return BaseT::getCFInstrCost(Opcode, CostKind, I);
}

InstructionCost GCNTTIImpl::getCmpSelInstrCost(
    unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
    TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info,
    TTI::OperandValueInfo Op2Info, const Instruction *I) const {
  // For size and latency cost kinds, return a low cost independent of vector
  // width to enable SimplifyCFG's speculativelyExecuteBB optimization.
  if (CostKind == TTI::TCK_SizeAndLatency)
    return 1;

  return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
                                   Op1Info, Op2Info, I);
}

// Measured packing cost of i1 for gfx9-12 is 4.0 to 4.8, up to 5.4 with
// true16; unpacking is 2.6 to 2.9.
static constexpr unsigned MaskPackCostPerElt = 4;
static constexpr unsigned MaskUnpackCostPerElt = 3;

static std::optional<unsigned> getNumberOfPackedMaskElts(Type *Ty) {
  auto *FVT = dyn_cast<FixedVectorType>(Ty);
  if (FVT && FVT->getElementType()->isIntegerTy(1) && FVT->getNumElements() > 1)
    return FVT->getNumElements();
  return std::nullopt;
}

InstructionCost GCNTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
                                             Type *Src,
                                             TTI::CastContextHint CCH,
                                             TTI::TargetCostKind CostKind,
                                             const Instruction *I) const {
  // A bitcast between a vector of i1 and an integer packs or unpacks a mask.
  if (Opcode == Instruction::BitCast) {
    if (std::optional<unsigned> Elts = getNumberOfPackedMaskElts(Src);
        Elts && Dst->isIntegerTy(*Elts))
      return InstructionCost(MaskPackCostPerElt) * *Elts *
             getFullRateInstrCost();
    if (std::optional<unsigned> Elts = getNumberOfPackedMaskElts(Dst);
        Elts && Src->isIntegerTy(*Elts))
      return InstructionCost(MaskUnpackCostPerElt) * *Elts *
             getFullRateInstrCost();
  }

  const int ISD = TLI->InstructionOpcodeToISD(Opcode);
  switch (ISD) {
  case ISD::SINT_TO_FP:
  case ISD::UINT_TO_FP:
  case ISD::FP_TO_SINT:
  case ISD::FP_TO_UINT:
    break;
  default:
    return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
  }

  const bool IsIntToFP = ISD == ISD::SINT_TO_FP || ISD == ISD::UINT_TO_FP;
  Type *FPTy = (IsIntToFP ? Dst : Src)->getScalarType();
  if (!FPTy->isHalfTy() && !FPTy->isBFloatTy() && !FPTy->isFloatTy() &&
      !FPTy->isDoubleTy())
    return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);

  unsigned NElts = 1;
  if (auto *VT = dyn_cast<FixedVectorType>(Src))
    NElts = VT->getNumElements();

  const unsigned SrcBits = Src->getScalarSizeInBits();
  const unsigned DstBits = Dst->getScalarSizeInBits();
  const bool IsSigned = ISD == ISD::SINT_TO_FP || ISD == ISD::FP_TO_SINT;
  const unsigned IntBits = IsIntToFP ? SrcBits : DstBits;
  const bool UsesInt64 = IntBits > 32 && IntBits <= 64;

  auto Scale = [&](unsigned FullRateOps,
                   unsigned FP64Ops = 0) -> InstructionCost {
    return NElts * (InstructionCost(FullRateOps) * getFullRateInstrCost() +
                    InstructionCost(FP64Ops) * get64BitInstrCost(CostKind));
  };

  if (IsIntToFP) {
    // A scalar load of a whole number of bytes that is not a power of two is
    // split and its high part load extends the source. A constant address
    // space or invariant global load aligned to 4 bytes may be widened instead
    // and then still needs the extension. So does a buffer fat pointer load.
    const auto *Load =
        I && Src->isIntegerTy() && I->getOperand(0)->getType() == Src
            ? dyn_cast<LoadInst>(I->getOperand(0))
            : nullptr;
    const unsigned AS = Load ? Load->getPointerAddressSpace() : 0;
    const bool LoadMayWiden =
        Load && Load->getAlign() >= Align(4) &&
        (AS == AMDGPUAS::CONSTANT_ADDRESS ||
         AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT ||
         (AS == AMDGPUAS::GLOBAL_ADDRESS &&
          Load->hasMetadata(LLVMContext::MD_invariant_load)));
    const bool LoadExtends = Load && Load->isSimple() && Load->hasOneUse() &&
                             !LoadMayWiden &&
                             AS != AMDGPUAS::BUFFER_FAT_POINTER &&
                             AS != AMDGPUAS::BUFFER_STRIDED_POINTER &&
                             SrcBits % 8 == 0 && !isPowerOf2_32(SrcBits);
    const unsigned ExtOps =
        UsesInt64 && SrcBits < 64 && !LoadExtends ? (IsSigned ? 2 : 1) : 0;
    if (FPTy->isBFloatTy()) {
      if (SrcBits < 8 || (SrcBits > 32 && !UsesInt64))
        return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);

      // Each integer is converted to f32 first.
      InstructionCost FloatCost =
          Scale(UsesInt64 ? ExtOps + (IsSigned ? 12 : 8) : 1);
      if (SrcBits < 32)
        FloatCost = getCastInstrCost(
            Opcode, Dst->getWithNewType(Type::getFloatTy(Dst->getContext())),
            Src, CCH, CostKind, I);

      // Native rounding can convert a pair. With 16 bit instructions the
      // expansion extracts the low significand bit, adds the rounding bias,
      // preserves NaNs and shifts the result. Without gfx9 instructions the two
      // additions cannot use v_add3_u32.
      InstructionCost RoundCost =
          ST->hasBF16ConversionInsts()
              ? InstructionCost(divideCeil(NElts, 2)) * getFullRateInstrCost()
              : Scale(!ST->has16BitInsts() ? 1
                      : ST->hasGFX9Insts() ? 6
                                           : 7);
      return FloatCost + RoundCost;
    }

    // No instruction converts from a 64 bit integer.
    if (UsesInt64) {
      if (FPTy->isDoubleTy()) {
        // Two conversions, ldexp and add, all using the FP64 rate.
        return Scale(ExtOps, 4);
      }
      if (FPTy->isFloatTy())
        return Scale(ExtOps + (IsSigned ? 12 : 8));
      return Scale(ExtOps + (IsSigned ? 13 : 9));
    }

    // A narrow vector source is converted lane by lane.
    if (SrcBits >= 8 && SrcBits < 32 && isa<FixedVectorType>(Src)) {
      if (FPTy->isDoubleTy())
        return Scale(1 + (SrcBits < 16 && IsSigned && ST->has16BitInsts()), 1);
      if (FPTy->isHalfTy()) {
        const InstructionCost PairCost =
            InstructionCost(NElts / 2) * getFullRateInstrCost();
        // Lanes wider than 16 bits, and every lane without 16 bit instructions,
        // are converted to f32 first. With the packed conversion each pair is
        // then converted at once. Otherwise each lane is converted, and without
        // real true16 each pair of halves is packed.
        if (!ST->has16BitInsts() || SrcBits > 16) {
          auto *FloatTy =
              FixedVectorType::get(Type::getFloatTy(Dst->getContext()), NElts);
          const InstructionCost FloatCost =
              getCastInstrCost(Opcode, FloatTy, Src, CCH, CostKind);
          if (ST->has16BitInsts() && ST->hasCvtPkF16F32Inst())
            return FloatCost + InstructionCost(divideCeil(NElts, 2)) *
                                   getFullRateInstrCost();
          const unsigned PackOps =
              !ST->has16BitInsts() ? 2 : !ST->useRealTrue16Insts();
          return FloatCost + Scale(1) + PackOps * PairCost;
        }
        // Each lane is converted from a 16 bit subword. Lanes narrower than 16
        // bits are extended lane by lane, except bytes with SDWA. Signed bytes
        // are sign extended a pair at a time. Real true16 reads the high
        // subword in place, so only signed lanes narrower than 16 bits pay per
        // pair. Otherwise wider lanes shift the high subword of each pair down
        // without SDWA, and each pair of halves is packed.
        const unsigned PerElt =
            1 + (SrcBits == 8 ? !ST->hasSDWA() : SrcBits < 16);
        const bool RealTrue16 = ST->useRealTrue16Insts();
        const unsigned PerPair =
            !RealTrue16 + (SrcBits == 8 || RealTrue16 ? SrcBits < 16 && IsSigned
                                                      : !ST->hasSDWA());
        return Scale(PerElt) + PerPair * PairCost;
      }
      // An unsigned byte is converted straight out of its register.
      if (SrcBits == 8 && !IsSigned)
        return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
      if (SrcBits < 16 && !IsSigned)
        return Scale(2);
      unsigned PerElt = ST->hasSDWA() && SrcBits <= 16 ? 1 : 2;
      if (SrcBits < 16 && ST->has16BitInsts())
        ++PerElt;
      return Scale(PerElt);
    }

    // A narrow scalar source is extended before the conversion. An unsigned
    // byte is converted straight out of its register, and a signed one with
    // SDWA. A source wider than a byte but narrower than 16 bits is masked
    // or sign extended first. Without 16 bit instructions a half result is
    // rounded from f32.
    if (SrcBits >= 8 && SrcBits < 32) {
      if (LoadExtends) {
        if (FPTy->isDoubleTy())
          return Scale(0, 1);
        return Scale(FPTy->isHalfTy() ? 2 : 1);
      }
      const bool Narrow = SrcBits > 8 && SrcBits < 16;
      const bool SignExtend16 = IsSigned && Narrow && ST->has16BitInsts();
      if (FPTy->isDoubleTy())
        return Scale(1 + 2 * SignExtend16, 1);
      if (SrcBits > 16)
        return Scale(FPTy->isHalfTy() ? 3 : 2);
      if (FPTy->isHalfTy() && ST->has16BitInsts())
        return Scale(Narrow                           ? (IsSigned ? 3 : 2)
                     : SrcBits == 8 && !ST->hasSDWA() ? 2
                                                      : 1);
      const InstructionCost FloatCost =
          SignExtend16                ? Scale(ST->hasSDWA() ? 3 : 4)
          : Narrow                    ? Scale(2)
          : SrcBits == 8 && !IsSigned ? Scale(1)
                                      : Scale(ST->hasSDWA() ? 1 : 2);
      return FPTy->isHalfTy() ? FloatCost + Scale(1) : FloatCost;
    }

    return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
  }

  // No instruction converts to a 64 bit integer.
  if (UsesInt64) {
    const bool IsSigned64 = IsSigned || (DstBits < 64 && NElts == 1);
    if (FPTy->isDoubleTy()) {
      // With native trunc/floor there are six FP64 operations. The expanded
      // rounding sequence has seven, plus integer operations and constants.
      return ST->haveRoundOpsF64() ? Scale(1, 6) : Scale(22, 7);
    }
    // The f32 expansion has one fma, which is quarter rate without fast FMA.
    const InstructionCost SlowFMACost =
        ST->hasFastFMAF32() ? 0
                            : NElts * (getQuarterRateInstrCost(CostKind) -
                                       getFullRateInstrCost());
    if (FPTy->isFloatTy())
      return Scale(IsSigned64 ? 13 : 6) + SlowFMACost;
    if (FPTy->isBFloatTy()) {
      // Unlike half, bf16 does not fit in i32. Extend to f32 and use the full
      // i64 expansion.
      return Scale(1 + (IsSigned64 ? 13 : 6)) + SlowFMACost;
    }
    return Scale(3);
  }

  return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
}

InstructionCost
GCNTTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
                                       std::optional<FastMathFlags> FMF,
                                       TTI::TargetCostKind CostKind) const {
  if (TTI::requiresOrderedReduction(FMF))
    return BaseT::getArithmeticReductionCost(Opcode, Ty, FMF, CostKind);

  // An add or xor reduction over a vector of i1 becomes a bit count over the
  // packed mask; the generic model prices a shuffle tree and misses that.
  if (Opcode == Instruction::Add || Opcode == Instruction::Xor) {
    if (std::optional<unsigned> Elts = getNumberOfPackedMaskElts(Ty))
      return InstructionCost(MaskPackCostPerElt) * *Elts *
             getFullRateInstrCost();
  }

  EVT OrigTy = TLI->getValueType(DL, Ty);

  // Computes cost on targets that have packed math instructions(which support
  // 16-bit types only).
  if (!ST->hasVOP3PInsts() || OrigTy.getScalarSizeInBits() != 16)
    return BaseT::getArithmeticReductionCost(Opcode, Ty, FMF, CostKind);

  std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
  return LT.first * getFullRateInstrCost();
}

InstructionCost
GCNTTIImpl::getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty,
                                   FastMathFlags FMF,
                                   TTI::TargetCostKind CostKind) const {
  EVT OrigTy = TLI->getValueType(DL, Ty);

  // Computes cost on targets that have packed math instructions(which support
  // 16-bit types only).
  if (!ST->hasVOP3PInsts() || OrigTy.getScalarSizeInBits() != 16)
    return BaseT::getMinMaxReductionCost(IID, Ty, FMF, CostKind);

  std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
  return LT.first * getHalfRateInstrCost(CostKind);
}

InstructionCost GCNTTIImpl::getVectorInstrCost(
    unsigned Opcode, Type *ValTy, TTI::TargetCostKind CostKind, unsigned Index,
    const Value *Op0, const Value *Op1, TTI::VectorInstrContext VIC) const {
  switch (Opcode) {
  case Instruction::ExtractElement:
  case Instruction::InsertElement: {
    unsigned EltSize
      = DL.getTypeSizeInBits(cast<VectorType>(ValTy)->getElementType());
    // Dynamic indexing isn't free and is best avoided.
    if (Index == ~0u)
      return 2;
    if (EltSize < 32) {
      if (EltSize == 16 && Index == 0 && ST->has16BitInsts())
        return 0;
      // Inserts of booleans are free.
      // TODO: Extracts are free too.
      if (EltSize == 1 && Opcode == Instruction::InsertElement)
        return TargetTransformInfo::TCC_Free;
      // Extract element sequences of consecutive i8 values that match a
      // register size are free most likely. It is not possible to know
      // if this extract is part of a consecutive sequence so this may
      // apply more generally.
      if (Opcode == Instruction::ExtractElement && EltSize == 8) {
        if (auto *FVTy = dyn_cast<FixedVectorType>(ValTy)) {
          unsigned NumElts = FVTy->getNumElements();
          if (NumElts >= 4 && isPowerOf2_32(NumElts))
            return 0;
        }
      }
      return BaseT::getVectorInstrCost(Opcode, ValTy, CostKind, Index, Op0, Op1,
                                       VIC);
    }

    // Extracts are just reads of a subregister, so are free. Inserts are
    // considered free because we don't want to have any cost for scalarizing
    // operations, and we don't have to copy into a different register class.
    return 0;
  }
  default:
    return BaseT::getVectorInstrCost(Opcode, ValTy, CostKind, Index, Op0, Op1,
                                     VIC);
  }
}

/// Analyze if the results of inline asm are divergent. If \p Indices is empty,
/// this is analyzing the collective result of all output registers. Otherwise,
/// this is only querying a specific result index if this returns multiple
/// registers in a struct.
bool GCNTTIImpl::isInlineAsmSourceOfDivergence(
  const CallInst *CI, ArrayRef<unsigned> Indices) const {
  // TODO: Handle complex extract indices
  if (Indices.size() > 1)
    return true;

  const DataLayout &DL = CI->getDataLayout();
  const SIRegisterInfo *TRI = ST->getRegisterInfo();
  TargetLowering::AsmOperandInfoVector TargetConstraints =
      TLI->ParseConstraints(DL, ST->getRegisterInfo(), *CI);

  const int TargetOutputIdx = Indices.empty() ? -1 : Indices[0];

  int OutputIdx = 0;
  for (auto &TC : TargetConstraints) {
    if (TC.Type != InlineAsm::isOutput)
      continue;

    // Skip outputs we don't care about.
    if (TargetOutputIdx != -1 && TargetOutputIdx != OutputIdx++)
      continue;

    TLI->ComputeConstraintToUse(TC, SDValue());

    const TargetRegisterClass *RC = TLI->getRegForInlineAsmConstraint(
        TRI, TC.ConstraintCode, TC.ConstraintVT).second;

    // For AGPR constraints null is returned on subtargets without AGPRs, so
    // assume divergent for null.
    if (!RC || !TRI->isSGPRClass(RC))
      return true;
  }

  return false;
}

bool GCNTTIImpl::isReadRegisterSourceOfDivergence(
    const IntrinsicInst *ReadReg) const {
  Metadata *MD =
      cast<MetadataAsValue>(ReadReg->getArgOperand(0))->getMetadata();
  StringRef RegName =
      cast<MDString>(cast<MDNode>(MD)->getOperand(0))->getString();

  // Special case registers that look like VCC.
  MVT VT = MVT::getVT(ReadReg->getType());
  if (VT == MVT::i1)
    return true;

  // Special case scalar registers that start with 'v'.
  if (RegName.starts_with("vcc") || RegName.empty())
    return false;

  // VGPR or AGPR is divergent. There aren't any specially named vector
  // registers.
  return RegName[0] == 'v' || RegName[0] == 'a';
}

/// \returns true if the result of the value could potentially be
/// different across workitems in a wavefront.
bool GCNTTIImpl::isSourceOfDivergence(const Value *V) const {
  if (const Argument *A = dyn_cast<Argument>(V))
    return !AMDGPU::isArgPassedInSGPR(A);

  // Loads from the private and flat address spaces are divergent, because
  // threads can execute the load instruction with the same inputs and get
  // different results.
  //
  // All other loads are not divergent, because if threads issue loads with the
  // same arguments, they will always get the same result.
  if (const LoadInst *Load = dyn_cast<LoadInst>(V))
    return Load->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS ||
           Load->getPointerAddressSpace() == AMDGPUAS::FLAT_ADDRESS;

  // Atomics are divergent because they are executed sequentially: when an
  // atomic operation refers to the same address in each thread, then each
  // thread after the first sees the value written by the previous thread as
  // original value.
  if (isa<AtomicRMWInst, AtomicCmpXchgInst>(V))
    return true;

  if (const IntrinsicInst *Intrinsic = dyn_cast<IntrinsicInst>(V)) {
    Intrinsic::ID IID = Intrinsic->getIntrinsicID();
    switch (IID) {
    case Intrinsic::read_register:
      return isReadRegisterSourceOfDivergence(Intrinsic);
    case Intrinsic::amdgcn_workitem_id_y:
    case Intrinsic::amdgcn_workitem_id_z: {
      const Function *F = Intrinsic->getFunction();
      bool HasUniformYZ =
          ST->hasWavefrontsEvenlySplittingXDim(*F, /*RequitezUniformYZ=*/true);
      std::optional<unsigned> ThisDimSize = ST->getReqdWorkGroupSize(
          *F, IID == Intrinsic::amdgcn_workitem_id_y ? 1 : 2);
      return !HasUniformYZ && (!ThisDimSize || *ThisDimSize != 1);
    }
    default:
      return AMDGPU::isIntrinsicSourceOfDivergence(IID);
    }
  }

  // Assume all function calls are a source of divergence.
  if (const CallInst *CI = dyn_cast<CallInst>(V)) {
    if (CI->isInlineAsm())
      return isInlineAsmSourceOfDivergence(CI);
    return true;
  }

  // Assume all function calls are a source of divergence.
  if (isa<InvokeInst>(V))
    return true;

  // If the target supports globally addressable scratch, the mapping from
  // scratch memory to the flat aperture changes therefore an address space cast
  // is no longer uniform.
  if (auto *CastI = dyn_cast<AddrSpaceCastInst>(V)) {
    return CastI->getSrcAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS &&
           CastI->getDestAddressSpace() == AMDGPUAS::FLAT_ADDRESS &&
           ST->hasGloballyAddressableScratch();
  }

  return false;
}

bool GCNTTIImpl::isAlwaysUniform(const Value *V) const {
  if (const IntrinsicInst *Intrinsic = dyn_cast<IntrinsicInst>(V))
    return AMDGPU::isIntrinsicAlwaysUniform(Intrinsic->getIntrinsicID());

  if (const CallInst *CI = dyn_cast<CallInst>(V)) {
    if (CI->isInlineAsm())
      return !isInlineAsmSourceOfDivergence(CI);
    return false;
  }

  // In most cases TID / wavefrontsize is uniform.
  //
  // However, if a kernel has uneven dimesions we can have a value of
  // workitem-id-x divided by the wavefrontsize non-uniform. For example
  // dimensions (65, 2) will have workitems with address (64, 0) and (0, 1)
  // packed into a same wave which gives 1 and 0 after the division by 64
  // respectively.
  //
  // The X dimension doesn't reset within a wave if either both the Y
  // and Z dimensions are of length 1, or if the X dimension's required
  // size is a power of 2. Note, however, if the X dimension's maximum
  // size is a power of 2 < the wavefront size, division by the wavefront
  // size is guaranteed to yield 0, so this is also a no-reset case.
  bool XDimDoesntResetWithinWaves = false;
  if (auto *I = dyn_cast<Instruction>(V)) {
    const Function *F = I->getFunction();
    XDimDoesntResetWithinWaves = ST->hasWavefrontsEvenlySplittingXDim(*F);
  }
  using namespace llvm::PatternMatch;
  uint64_t C;
  if (match(V, m_LShr(m_Intrinsic<Intrinsic::amdgcn_workitem_id_x>(),
                      m_ConstantInt(C))) ||
      match(V, m_AShr(m_Intrinsic<Intrinsic::amdgcn_workitem_id_x>(),
                      m_ConstantInt(C)))) {
    return C >= ST->getWavefrontSizeLog2() && XDimDoesntResetWithinWaves;
  }

  Value *Mask;
  if (match(V, m_c_And(m_Intrinsic<Intrinsic::amdgcn_workitem_id_x>(),
                       m_Value(Mask)))) {
    return computeKnownBits(Mask, DL).countMinTrailingZeros() >=
               ST->getWavefrontSizeLog2() &&
           XDimDoesntResetWithinWaves;
  }

  const ExtractValueInst *ExtValue = dyn_cast<ExtractValueInst>(V);
  if (!ExtValue)
    return false;

  const CallInst *CI = dyn_cast<CallInst>(ExtValue->getOperand(0));
  if (!CI)
    return false;

  if (const IntrinsicInst *Intrinsic = dyn_cast<IntrinsicInst>(CI)) {
    switch (Intrinsic->getIntrinsicID()) {
    default:
      return false;
    case Intrinsic::amdgcn_if:
    case Intrinsic::amdgcn_else: {
      ArrayRef<unsigned> Indices = ExtValue->getIndices();
      return Indices.size() == 1 && Indices[0] == 1;
    }
    }
  }

  // If we have inline asm returning mixed SGPR and VGPR results, we inferred
  // divergent for the overall struct return. We need to override it in the
  // case we're extracting an SGPR component here.
  if (CI->isInlineAsm())
    return !isInlineAsmSourceOfDivergence(CI, ExtValue->getIndices());

  return false;
}

bool GCNTTIImpl::collectFlatAddressOperands(SmallVectorImpl<int> &OpIndexes,
                                            Intrinsic::ID IID) const {
  switch (IID) {
  case Intrinsic::amdgcn_is_shared:
  case Intrinsic::amdgcn_is_private:
  case Intrinsic::amdgcn_flat_atomic_fmax_num:
  case Intrinsic::amdgcn_flat_atomic_fmin_num:
  case Intrinsic::amdgcn_load_to_lds:
  case Intrinsic::amdgcn_make_buffer_rsrc:
    OpIndexes.push_back(0);
    return true;
  default:
    return false;
  }
}

Value *GCNTTIImpl::rewriteIntrinsicWithAddressSpace(IntrinsicInst *II,
                                                    Value *OldV,
                                                    Value *NewV) const {
  auto IntrID = II->getIntrinsicID();
  switch (IntrID) {
  case Intrinsic::amdgcn_is_shared:
  case Intrinsic::amdgcn_is_private: {
    unsigned TrueAS = IntrID == Intrinsic::amdgcn_is_shared ?
      AMDGPUAS::LOCAL_ADDRESS : AMDGPUAS::PRIVATE_ADDRESS;
    unsigned NewAS = NewV->getType()->getPointerAddressSpace();
    LLVMContext &Ctx = NewV->getType()->getContext();
    ConstantInt *NewVal = (TrueAS == NewAS) ?
      ConstantInt::getTrue(Ctx) : ConstantInt::getFalse(Ctx);
    return NewVal;
  }
  case Intrinsic::amdgcn_flat_atomic_fmax_num:
  case Intrinsic::amdgcn_flat_atomic_fmin_num: {
    Type *DestTy = II->getType();
    Type *SrcTy = NewV->getType();
    unsigned NewAS = SrcTy->getPointerAddressSpace();
    if (!AMDGPU::isExtendedGlobalAddrSpace(NewAS))
      return nullptr;
    Module *M = II->getModule();
    Function *NewDecl = Intrinsic::getOrInsertDeclaration(
        M, II->getIntrinsicID(), {DestTy, SrcTy, DestTy});
    II->setArgOperand(0, NewV);
    II->setCalledFunction(NewDecl);
    return II;
  }
  case Intrinsic::amdgcn_load_to_lds: {
    Type *SrcTy = NewV->getType();
    Module *M = II->getModule();
    Function *NewDecl =
        Intrinsic::getOrInsertDeclaration(M, II->getIntrinsicID(), {SrcTy});
    II->setArgOperand(0, NewV);
    II->setCalledFunction(NewDecl);
    return II;
  }
  case Intrinsic::amdgcn_make_buffer_rsrc: {
    Type *SrcTy = NewV->getType();
    Type *DstTy = II->getType();
    Type *NumRecordsTy = II->getArgOperand(2)->getType();
    Module *M = II->getModule();
    Function *NewDecl = Intrinsic::getOrInsertDeclaration(
        M, II->getIntrinsicID(), {DstTy, SrcTy, NumRecordsTy});
    II->setArgOperand(0, NewV);
    II->setCalledFunction(NewDecl);
    return II;
  }
  default:
    return nullptr;
  }
}

InstructionCost GCNTTIImpl::getShuffleCost(
    TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
    TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index,
    VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CtxI,
    TTI::VectorInstrContext VIC) const {
  if (!isa<FixedVectorType>(SrcTy))
    return BaseT::getShuffleCost(Kind, DstTy, SrcTy, CostKind, Mask, Index,
                                 SubTp);

  Kind = improveShuffleKindFromMask(Kind, Mask, SrcTy, Index, SubTp);

  unsigned ScalarSize = DL.getTypeSizeInBits(SrcTy->getElementType());
  if (ST->getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS &&
      (ScalarSize == 16 || ScalarSize == 8)) {
    // Larger vector widths may require additional instructions, but are
    // typically cheaper than scalarized versions.
    //
    // We assume that shuffling at a register granularity can be done for free.
    // This is not true for vectors fed into memory instructions, but it is
    // effectively true for all other shuffling. The emphasis of the logic here
    // is to assist generic transform in cleaning up / canonicalizing those
    // shuffles.

    // With op_sel VOP3P instructions freely can access the low half or high
    // half of a register, so any swizzle of two elements is free.
    if (auto *SrcVecTy = dyn_cast<FixedVectorType>(SrcTy)) {
      unsigned NumSrcElts = SrcVecTy->getNumElements();
      if (ST->hasVOP3PInsts() && ScalarSize == 16 && NumSrcElts == 2 &&
          (Kind == TTI::SK_Broadcast || Kind == TTI::SK_Reverse ||
           Kind == TTI::SK_PermuteSingleSrc))
        return 0;
    }

    unsigned EltsPerReg = 32 / ScalarSize;
    switch (Kind) {
    case TTI::SK_Broadcast:
      // A single v_perm_b32 can be re-used for all destination registers.
      return 1;
    case TTI::SK_Reverse:
      // One instruction per register.
      if (auto *DstVecTy = dyn_cast<FixedVectorType>(DstTy))
        return divideCeil(DstVecTy->getNumElements(), EltsPerReg);
      return InstructionCost::getInvalid();
    case TTI::SK_ExtractSubvector:
      if (Index % EltsPerReg == 0)
        return 0; // Shuffling at register granularity
      if (auto *DstVecTy = dyn_cast<FixedVectorType>(DstTy))
        return divideCeil(DstVecTy->getNumElements(), EltsPerReg);
      return InstructionCost::getInvalid();
    case TTI::SK_InsertSubvector: {
      auto *DstVecTy = dyn_cast<FixedVectorType>(DstTy);
      if (!DstVecTy)
        return InstructionCost::getInvalid();
      unsigned NumDstElts = DstVecTy->getNumElements();
      unsigned NumInsertElts = cast<FixedVectorType>(SubTp)->getNumElements();
      unsigned EndIndex = Index + NumInsertElts;
      unsigned BeginSubIdx = Index % EltsPerReg;
      unsigned EndSubIdx = EndIndex % EltsPerReg;
      unsigned Cost = 0;

      if (BeginSubIdx != 0) {
        // Need to shift the inserted vector into place. The cost is the number
        // of destination registers overlapped by the inserted vector.
        Cost = divideCeil(EndIndex, EltsPerReg) - (Index / EltsPerReg);
      }

      // If the last register overlap is partial, there may be three source
      // registers feeding into it; that takes an extra instruction.
      if (EndIndex < NumDstElts && BeginSubIdx < EndSubIdx)
        Cost += 1;

      return Cost;
    }
    case TTI::SK_Splice: {
      auto *DstVecTy = dyn_cast<FixedVectorType>(DstTy);
      if (!DstVecTy)
        return InstructionCost::getInvalid();
      unsigned NumElts = DstVecTy->getNumElements();
      assert(NumElts == cast<FixedVectorType>(SrcTy)->getNumElements());
      // Determine the sub-region of the result vector that requires
      // sub-register shuffles / mixing.
      unsigned EltsFromLHS = NumElts - Index;
      bool LHSIsAligned = (Index % EltsPerReg) == 0;
      bool RHSIsAligned = (EltsFromLHS % EltsPerReg) == 0;
      if (LHSIsAligned && RHSIsAligned)
        return 0;
      if (LHSIsAligned && !RHSIsAligned)
        return divideCeil(NumElts, EltsPerReg) - (EltsFromLHS / EltsPerReg);
      if (!LHSIsAligned && RHSIsAligned)
        return divideCeil(EltsFromLHS, EltsPerReg);
      return divideCeil(NumElts, EltsPerReg);
    }
    default:
      break;
    }

    if (!Mask.empty()) {
      unsigned NumSrcElts = cast<FixedVectorType>(SrcTy)->getNumElements();

      // Generically estimate the cost by assuming that each destination
      // register is derived from sources via v_perm_b32 instructions if it
      // can't be copied as-is.
      //
      // For each destination register, derive the cost of obtaining it based
      // on the number of source registers that feed into it.
      unsigned Cost = 0;
      for (unsigned DstIdx = 0; DstIdx < Mask.size(); DstIdx += EltsPerReg) {
        SmallVector<int, 4> Regs;
        bool Aligned = true;
        for (unsigned I = 0; I < EltsPerReg && DstIdx + I < Mask.size(); ++I) {
          int SrcIdx = Mask[DstIdx + I];
          if (SrcIdx == -1)
            continue;
          int Reg;
          if (SrcIdx < (int)NumSrcElts) {
            Reg = SrcIdx / EltsPerReg;
            if (SrcIdx % EltsPerReg != I)
              Aligned = false;
          } else {
            Reg = NumSrcElts + (SrcIdx - NumSrcElts) / EltsPerReg;
            if ((SrcIdx - NumSrcElts) % EltsPerReg != I)
              Aligned = false;
          }
          if (!llvm::is_contained(Regs, Reg))
            Regs.push_back(Reg);
        }
        if (Regs.size() >= 2)
          Cost += Regs.size() - 1;
        else if (!Aligned)
          Cost += 1;
      }
      return Cost;
    }
  }

  return BaseT::getShuffleCost(Kind, DstTy, SrcTy, CostKind, Mask, Index,
                               SubTp);
}

/// Whether it is profitable to sink the operands of an
/// Instruction I to the basic block of I.
/// This helps using several modifiers (like abs and neg) more often.
bool GCNTTIImpl::isProfitableToSinkOperands(Instruction *I,
                                            SmallVectorImpl<Use *> &Ops) const {
  using namespace PatternMatch;

  // The cost model prices this fmul as free assuming it fuses with its
  // fadd/fsub user, which needs them in one block. Sink a stranded
  // loop-invariant fmul back to the user when they would fuse. Single use only,
  // so this stays a move.
  if (I->getOpcode() == Instruction::FAdd ||
      I->getOpcode() == Instruction::FSub) {
    const Instruction *FMul = getFusedFMul(*TLI, I->getType(), I);
    if (FMul && FMul->getParent() != I->getParent())
      Ops.push_back(&I->getOperandUse(I->getOperand(0) == FMul ? 0 : 1));
  }

  for (auto &Op : I->operands()) {
    // Ensure we are not already sinking this operand.
    if (any_of(Ops, [&](Use *U) { return U->get() == Op.get(); }))
      continue;

    if (match(&Op, m_FAbs(m_Value())) || match(&Op, m_FNeg(m_Value()))) {
      Ops.push_back(&Op);
      continue;
    }

    // Check for zero-cost multiple use InsertElement/ExtractElement
    // instructions
    if (Instruction *OpInst = dyn_cast<Instruction>(Op.get())) {
      if (OpInst->getType()->isVectorTy() && OpInst->getNumOperands() > 1) {
        Instruction *VecOpInst = dyn_cast<Instruction>(OpInst->getOperand(0));
        if (VecOpInst && VecOpInst->hasOneUse())
          continue;

        if (getVectorInstrCost(OpInst->getOpcode(), OpInst->getType(),
                               TTI::TCK_RecipThroughput, 0,
                               OpInst->getOperand(0),
                               OpInst->getOperand(1)) == 0) {
          Ops.push_back(&Op);
          continue;
        }
      }
    }

    if (auto *Shuffle = dyn_cast<ShuffleVectorInst>(Op.get())) {

      unsigned EltSize = DL.getTypeSizeInBits(
          cast<VectorType>(Shuffle->getType())->getElementType());

      // For i32 (or greater) shufflevectors, these will be lowered into a
      // series of insert / extract elements, which will be coalesced away.
      if (EltSize < 16 || !ST->has16BitInsts())
        continue;

      int NumSubElts, SubIndex;
      if (Shuffle->changesLength()) {
        if (Shuffle->increasesLength() && Shuffle->isIdentityWithPadding()) {
          Ops.push_back(&Op);
          continue;
        }

        if ((Shuffle->isExtractSubvectorMask(SubIndex) ||
             Shuffle->isInsertSubvectorMask(NumSubElts, SubIndex)) &&
            !(SubIndex & 0x1)) {
          Ops.push_back(&Op);
          continue;
        }
      }

      if (Shuffle->isReverse() || Shuffle->isZeroEltSplat() ||
          Shuffle->isSingleSource()) {
        Ops.push_back(&Op);
        continue;
      }
    }
  }

  return !Ops.empty();
}

bool GCNTTIImpl::areInlineCompatible(const Function *Caller,
                                     const Function *Callee) const {
  const TargetMachine &TM = getTLI()->getTargetMachine();
  const GCNSubtarget *CallerST
    = static_cast<const GCNSubtarget *>(TM.getSubtargetImpl(*Caller));
  const GCNSubtarget *CalleeST
    = static_cast<const GCNSubtarget *>(TM.getSubtargetImpl(*Callee));

  if (!BaseT::areInlineCompatible(Caller, Callee))
    return false;

  // FIXME: dx10_clamp can just take the caller setting, but there seems to be
  // no way to support merge for backend defined attributes.
  SIModeRegisterDefaults CallerMode(*Caller, *CallerST);
  SIModeRegisterDefaults CalleeMode(*Callee, *CalleeST);
  if (!CallerMode.isInlineCompatible(CalleeMode))
    return false;

  if (Callee->hasFnAttribute(Attribute::AlwaysInline) ||
      Callee->hasFnAttribute(Attribute::InlineHint))
    return true;

  // Hack to make compile times reasonable.
  if (InlineMaxBB) {
    // Single BB does not increase total BB amount.
    if (Callee->size() == 1)
      return true;
    size_t BBSize = Caller->size() + Callee->size() - 1;
    if (BBSize > InlineMaxBB) {
      LLVM_DEBUG(dbgs() << "AMDGPU inline max-BB rejected inlining "
                        << Callee->getName() << " into " << Caller->getName()
                        << ": caller BBs=" << Caller->size() << ", callee BBs="
                        << Callee->size() << ", combined BBs=" << BBSize
                        << ", max BBs=" << InlineMaxBB << '\n');
      return false;
    }
  }

  return true;
}

static unsigned adjustInliningThresholdUsingCallee(const CallBase *CB,
                                                   const SITargetLowering *TLI,
                                                   const GCNTTIImpl *TTIImpl) {
  const int NrOfSGPRUntilSpill = 26;
  const int NrOfVGPRUntilSpill = 32;

  const DataLayout &DL = TTIImpl->getDataLayout();

  unsigned adjustThreshold = 0;
  int SGPRsInUse = 0;
  int VGPRsInUse = 0;
  for (const Use &A : CB->args()) {
    SmallVector<EVT, 4> ValueVTs;
    ComputeValueVTs(*TLI, DL, A.get()->getType(), ValueVTs);
    for (auto ArgVT : ValueVTs) {
      unsigned CCRegNum = TLI->getNumRegistersForCallingConv(
          CB->getContext(), CB->getCallingConv(), ArgVT);
      if (AMDGPU::isArgPassedInSGPR(CB, CB->getArgOperandNo(&A)))
        SGPRsInUse += CCRegNum;
      else
        VGPRsInUse += CCRegNum;
    }
  }

  // The cost of passing function arguments through the stack:
  //  1 instruction to put a function argument on the stack in the caller.
  //  1 instruction to take a function argument from the stack in callee.
  //  1 instruction is explicitly take care of data dependencies in callee
  //  function.
  InstructionCost ArgStackCost(1);
  ArgStackCost += const_cast<GCNTTIImpl *>(TTIImpl)->getMemoryOpCost(
      Instruction::Store, Type::getInt32Ty(CB->getContext()), Align(4),
      AMDGPUAS::PRIVATE_ADDRESS, TTI::TCK_SizeAndLatency);
  ArgStackCost += const_cast<GCNTTIImpl *>(TTIImpl)->getMemoryOpCost(
      Instruction::Load, Type::getInt32Ty(CB->getContext()), Align(4),
      AMDGPUAS::PRIVATE_ADDRESS, TTI::TCK_SizeAndLatency);

  // The penalty cost is computed relative to the cost of instructions and does
  // not model any storage costs.
  adjustThreshold += std::max(0, SGPRsInUse - NrOfSGPRUntilSpill) *
                     ArgStackCost.getValue() * InlineConstants::getInstrCost();
  adjustThreshold += std::max(0, VGPRsInUse - NrOfVGPRUntilSpill) *
                     ArgStackCost.getValue() * InlineConstants::getInstrCost();
  return adjustThreshold;
}

static unsigned getCallArgsTotalAllocaSize(const CallBase *CB,
                                           const DataLayout &DL) {
  // If we have a pointer to a private array passed into a function
  // it will not be optimized out, leaving scratch usage.
  // This function calculates the total size in bytes of the memory that would
  // end in scratch if the call was not inlined.
  unsigned AllocaSize = 0;
  SmallPtrSet<const AllocaInst *, 8> AIVisited;
  for (Value *PtrArg : CB->args()) {
    PointerType *Ty = dyn_cast<PointerType>(PtrArg->getType());
    if (!Ty)
      continue;

    unsigned AddrSpace = Ty->getAddressSpace();
    if (AddrSpace != AMDGPUAS::FLAT_ADDRESS &&
        AddrSpace != AMDGPUAS::PRIVATE_ADDRESS)
      continue;

    const AllocaInst *AI = dyn_cast<AllocaInst>(getUnderlyingObject(PtrArg));
    if (!AI || !AI->isStaticAlloca() || !AIVisited.insert(AI).second)
      continue;

    if (auto Size = AI->getAllocationSize(DL))
      AllocaSize += Size->getFixedValue();
  }
  return AllocaSize;
}

int GCNTTIImpl::getInliningLastCallToStaticBonus() const {
  return BaseT::getInliningLastCallToStaticBonus() *
         getInliningThresholdMultiplier();
}

unsigned GCNTTIImpl::adjustInliningThreshold(const CallBase *CB) const {
  unsigned Threshold = adjustInliningThresholdUsingCallee(CB, TLI, this);

  // Private object passed as arguments may end up in scratch usage if the call
  // is not inlined. Increase the inline threshold to promote inlining.
  unsigned AllocaSize = getCallArgsTotalAllocaSize(CB, DL);
  if (AllocaSize > 0)
    Threshold += ArgAllocaCost;
  return Threshold;
}

unsigned GCNTTIImpl::getCallerAllocaCost(const CallBase *CB,
                                         const AllocaInst *AI) const {

  // Below the cutoff, assume that the private memory objects would be
  // optimized
  auto AllocaSize = getCallArgsTotalAllocaSize(CB, DL);
  if (AllocaSize <= ArgAllocaCutoff)
    return 0;

  // Above the cutoff, we give a cost to each private memory object
  // depending its size. If the array can be optimized by SROA this cost is not
  // added to the total-cost in the inliner cost analysis.
  //
  // We choose the total cost of the alloca such that their sum cancels the
  // bonus given in the threshold (ArgAllocaCost).
  //
  //   Cost_Alloca_0 + ... + Cost_Alloca_N == ArgAllocaCost
  //
  // Awkwardly, the ArgAllocaCost bonus is multiplied by threshold-multiplier,
  // the single-bb bonus and the vector-bonus.
  //
  // We compensate the first two multipliers, by repeating logic from the
  // inliner-cost in here. The vector-bonus is 0 on AMDGPU.
  static_assert(InlinerVectorBonusPercent == 0, "vector bonus assumed to be 0");
  unsigned Threshold = ArgAllocaCost * getInliningThresholdMultiplier();

  bool SingleBB = none_of(*CB->getCalledFunction(), [](const BasicBlock &BB) {
    return BB.getTerminator()->getNumSuccessors() > 1;
  });
  if (SingleBB) {
    Threshold += Threshold / 2;
  }

  auto ArgAllocaSize = AI->getAllocationSize(DL);
  if (!ArgAllocaSize)
    return 0;

  // Attribute the bonus proportionally to the alloca size
  unsigned AllocaThresholdBonus =
      (Threshold * ArgAllocaSize->getFixedValue()) / AllocaSize;

  return AllocaThresholdBonus;
}

void GCNTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
                                         TTI::UnrollingPreferences &UP,
                                         OptimizationRemarkEmitter *ORE) const {
  CommonTTI.getUnrollingPreferences(L, SE, UP, ORE);
}

void GCNTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
                                       TTI::PeelingPreferences &PP) const {
  CommonTTI.getPeelingPreferences(L, SE, PP);
}

int GCNTTIImpl::getTransInstrCost(TTI::TargetCostKind CostKind) const {
  return getQuarterRateInstrCost(CostKind);
}

int GCNTTIImpl::get64BitInstrCost(TTI::TargetCostKind CostKind) const {
  return ST->hasFullRate64Ops()
             ? getFullRateInstrCost()
             : ST->hasHalfRate64Ops() ? getHalfRateInstrCost(CostKind)
                                      : getQuarterRateInstrCost(CostKind);
}

std::pair<InstructionCost, MVT>
GCNTTIImpl::getTypeLegalizationCost(Type *Ty) const {
  std::pair<InstructionCost, MVT> Cost = BaseT::getTypeLegalizationCost(Ty);
  auto Size = DL.getTypeSizeInBits(Ty);
  // Maximum load or store can handle 8 dwords for scalar and 4 for
  // vector ALU. Let's assume anything above 8 dwords is expensive
  // even if legal.
  if (Size <= 256)
    return Cost;

  Cost.first += (Size + 255) / 256;
  return Cost;
}

unsigned GCNTTIImpl::getCacheLineSize() const {
  if (ST->hasVmemPrefInsts() || ST->hasSmemPrefetchInsts())
    return ST->getDataCacheLineSize();
  return 0;
}

unsigned GCNTTIImpl::getPrefetchDistance() const {
  return ST->hasPrefetch() ? 128 : 0;
}

bool GCNTTIImpl::shouldPrefetchAddressSpace(unsigned AS) const {
  return AMDGPU::isFlatGlobalAddrSpace(AS);
}

void GCNTTIImpl::collectKernelLaunchBounds(
    const Function &F,
    SmallVectorImpl<std::pair<StringRef, int64_t>> &LB) const {
  SmallVector<unsigned> MaxNumWorkgroups = AMDGPU::getMaxNumWorkGroups(F);
  LB.push_back({"amdgpu-max-num-workgroups[0]", MaxNumWorkgroups[0]});
  LB.push_back({"amdgpu-max-num-workgroups[1]", MaxNumWorkgroups[1]});
  LB.push_back({"amdgpu-max-num-workgroups[2]", MaxNumWorkgroups[2]});
  std::pair<unsigned, unsigned> FlatWorkGroupSize =
      ST->getFlatWorkGroupSizes(F);
  LB.push_back({"amdgpu-flat-work-group-size[0]", FlatWorkGroupSize.first});
  LB.push_back({"amdgpu-flat-work-group-size[1]", FlatWorkGroupSize.second});
  std::pair<unsigned, unsigned> WavesPerEU = ST->getWavesPerEU(F);
  LB.push_back({"amdgpu-waves-per-eu[0]", WavesPerEU.first});
  LB.push_back({"amdgpu-waves-per-eu[1]", WavesPerEU.second});
}

GCNTTIImpl::KnownIEEEMode
GCNTTIImpl::fpenvIEEEMode(const Instruction &I) const {
  if (!ST->hasFeature(AMDGPU::FeatureDX10ClampAndIEEEMode))
    return KnownIEEEMode::On; // Only mode on gfx1170+

  const Function *F = I.getFunction();
  if (!F)
    return KnownIEEEMode::Unknown;

  Attribute IEEEAttr = F->getFnAttribute("amdgpu-ieee");
  if (IEEEAttr.isValid())
    return IEEEAttr.getValueAsBool() ? KnownIEEEMode::On : KnownIEEEMode::Off;

  return AMDGPU::isShader(F->getCallingConv()) ? KnownIEEEMode::Off
                                               : KnownIEEEMode::On;
}

InstructionCost GCNTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
                                            Align Alignment,
                                            unsigned AddressSpace,
                                            TTI::TargetCostKind CostKind,
                                            TTI::OperandValueInfo OpInfo,
                                            const Instruction *I) const {
  if (VectorType *VecTy = dyn_cast<VectorType>(Src)) {
    if ((Opcode == Instruction::Load || Opcode == Instruction::Store) &&
        CostKind != TTI::TCK_Latency &&
        VecTy->getElementType()->isIntegerTy(8)) {
      return divideCeil(DL.getTypeSizeInBits(VecTy) - 1,
                        getLoadStoreVecRegBitWidth(AddressSpace));
    }
  }
  return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, CostKind,
                                OpInfo, I);
}

unsigned GCNTTIImpl::getNumberOfParts(Type *Tp) const {
  if (VectorType *VecTy = dyn_cast<VectorType>(Tp)) {
    if (VecTy->getElementType()->isIntegerTy(8)) {
      unsigned ElementCount = VecTy->getElementCount().getFixedValue();
      return divideCeil(ElementCount - 1, 4);
    }
  }
  return BaseT::getNumberOfParts(Tp);
}

ValueUniformity GCNTTIImpl::getValueUniformity(const Value *V) const {
  if (const IntrinsicInst *Intrinsic = dyn_cast<IntrinsicInst>(V)) {
    switch (Intrinsic->getIntrinsicID()) {
    case Intrinsic::amdgcn_wave_shuffle:
      return ValueUniformity::Custom;
    default:
      break;
    }
  }

  if (isAlwaysUniform(V))
    return ValueUniformity::AlwaysUniform;

  if (isSourceOfDivergence(V))
    return ValueUniformity::NeverUniform;

  return ValueUniformity::Default;
}

InstructionCost GCNTTIImpl::getScalingFactorCost(Type *Ty, GlobalValue *BaseGV,
                                                 StackOffset BaseOffset,
                                                 bool HasBaseReg, int64_t Scale,
                                                 unsigned AddrSpace) const {
  if (HasBaseReg && Scale != 0) {
    // gfx1250+ can fold base+scale*index when scale matches the memory access
    // size (scale_offset bit). Supported for flat/global/constant/scratch
    // (VMEM, max 128 bits) and constant_32bit (SMRD, capped to 128 bits here).
    if (getST()->hasScaleOffset() && Ty && Ty->isSized() &&
        (AMDGPU::isExtendedGlobalAddrSpace(AddrSpace) ||
         AddrSpace == AMDGPUAS::FLAT_ADDRESS ||
         AddrSpace == AMDGPUAS::PRIVATE_ADDRESS)) {
      TypeSize StoreSize = getDataLayout().getTypeStoreSize(Ty);
      if (TypeSize::isKnownLE(StoreSize, TypeSize::getFixed(16)) &&
          static_cast<int64_t>(StoreSize.getFixedValue()) == Scale)
        return 0;
    }
    return 1;
  }
  return BaseT::getScalingFactorCost(Ty, BaseGV, BaseOffset, HasBaseReg, Scale,
                                     AddrSpace);
}

bool GCNTTIImpl::isLSRCostLess(const TTI::LSRCost &A,
                               const TTI::LSRCost &B) const {
  // Favor lower per-iteration work over preheader/setup costs.
  // AMDGPU lacks rich addressing modes, so ScaleCost is folded into the
  // effective instruction count (base+scale*index requires a separate ADD).
  unsigned EffInsnsA = A.Insns + A.ScaleCost;
  unsigned EffInsnsB = B.Insns + B.ScaleCost;

  return std::tie(EffInsnsA, A.NumIVMuls, A.AddRecCost, A.NumBaseAdds,
                  A.SetupCost, A.ImmCost, A.NumRegs) <
         std::tie(EffInsnsB, B.NumIVMuls, B.AddRecCost, B.NumBaseAdds,
                  B.SetupCost, B.ImmCost, B.NumRegs);
}

bool GCNTTIImpl::isNumRegsMajorCostOfLSR() const {
  // isLSRCostLess de-prioritizes register count; keep consistent.
  return false;
}

bool GCNTTIImpl::shouldDropLSRSolutionIfLessProfitable() const {
  // Prefer the baseline when LSR cannot clearly reduce per-iteration work.
  return true;
}

bool GCNTTIImpl::isUniform(const Instruction *I,
                           const SmallBitVector &UniformArgs) const {
  const IntrinsicInst *Intrinsic = cast<IntrinsicInst>(I);
  switch (Intrinsic->getIntrinsicID()) {
  case Intrinsic::amdgcn_wave_shuffle:
    // wave_shuffle(Value, Index): result is uniform when either Value or Index
    // is uniform.
    return UniformArgs[0] || UniformArgs[1];
  default:
    llvm_unreachable("unexpected intrinsic in isUniform");
  }
}