//===- AMDGPULegalizerInfo.cpp -----------------------------------*- C++ -*-==//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
/// \file
/// This file implements the targeting of the Machinelegalizer class for
/// AMDGPU.
/// \todo This should be generated by TableGen.
//===----------------------------------------------------------------------===//

#include "AMDGPULegalizerInfo.h"

#include "AMDGPU.h"
#include "AMDGPUGlobalISelUtils.h"
#include "AMDGPUInstrInfo.h"
#include "AMDGPUMemoryUtils.h"
#include "AMDGPUTargetMachine.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "SIInstrInfo.h"
#include "SIMachineFunctionInfo.h"
#include "SIRegisterInfo.h"
#include "Utils/AMDGPUBaseInfo.h"
#include "llvm/ADT/ScopeExit.h"
#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
#include "llvm/CodeGen/GlobalISel/Utils.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/PseudoSourceValueManager.h"
#include "llvm/CodeGen/TargetOpcodes.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/IR/IntrinsicsR600.h"

#define DEBUG_TYPE "amdgpu-legalinfo"

using namespace llvm;
using namespace LegalizeActions;
using namespace LegalizeMutations;
using namespace LegalityPredicates;
using namespace MIPatternMatch;

// Hack until load/store selection patterns support any tuple of legal types.
static cl::opt<bool> EnableNewLegality(
  "amdgpu-global-isel-new-legality",
  cl::desc("Use GlobalISel desired legality, rather than try to use"
           "rules compatible with selection patterns"),
  cl::init(false),
  cl::ReallyHidden);

static constexpr unsigned MaxRegisterSize = 1024;

// Round the number of elements to the next power of two elements
static LLT getPow2VectorType(LLT Ty) {
  unsigned NElts = Ty.getNumElements();
  unsigned Pow2NElts = 1 <<  Log2_32_Ceil(NElts);
  return Ty.changeElementCount(ElementCount::getFixed(Pow2NElts));
}

// Round the number of bits to the next power of two bits
static LLT getPow2ScalarType(LLT Ty) {
  unsigned Bits = Ty.getSizeInBits();
  unsigned Pow2Bits = 1 <<  Log2_32_Ceil(Bits);
  return LLT::scalar(Pow2Bits);
}

/// \returns true if this is an odd sized vector which should widen by adding an
/// additional element. This is mostly to handle <3 x s16> -> <4 x s16>. This
/// excludes s1 vectors, which should always be scalarized.
static LegalityPredicate isSmallOddVector(unsigned TypeIdx) {
  return [=](const LegalityQuery &Query) {
    const LLT Ty = Query.Types[TypeIdx];
    if (!Ty.isVector())
      return false;

    const LLT EltTy = Ty.getElementType();
    const unsigned EltSize = EltTy.getSizeInBits();
    return Ty.getNumElements() % 2 != 0 &&
           EltSize > 1 && EltSize < 32 &&
           Ty.getSizeInBits() % 32 != 0;
  };
}

static LegalityPredicate sizeIsMultipleOf32(unsigned TypeIdx) {
  return [=](const LegalityQuery &Query) {
    const LLT Ty = Query.Types[TypeIdx];
    return Ty.getSizeInBits() % 32 == 0;
  };
}

static LegalityPredicate isWideVec16(unsigned TypeIdx) {
  return [=](const LegalityQuery &Query) {
    const LLT Ty = Query.Types[TypeIdx];
    const LLT EltTy = Ty.getScalarType();
    return EltTy.getSizeInBits() == 16 && Ty.getNumElements() > 2;
  };
}

static LegalizeMutation oneMoreElement(unsigned TypeIdx) {
  return [=](const LegalityQuery &Query) {
    const LLT Ty = Query.Types[TypeIdx];
    const LLT EltTy = Ty.getElementType();
    return std::pair(TypeIdx,
                     LLT::fixed_vector(Ty.getNumElements() + 1, EltTy));
  };
}

static LegalizeMutation fewerEltsToSize64Vector(unsigned TypeIdx) {
  return [=](const LegalityQuery &Query) {
    const LLT Ty = Query.Types[TypeIdx];
    const LLT EltTy = Ty.getElementType();
    unsigned Size = Ty.getSizeInBits();
    unsigned Pieces = (Size + 63) / 64;
    unsigned NewNumElts = (Ty.getNumElements() + 1) / Pieces;
    return std::pair(TypeIdx, LLT::scalarOrVector(
                                  ElementCount::getFixed(NewNumElts), EltTy));
  };
}

// Increase the number of vector elements to reach the next multiple of 32-bit
// type.
static LegalizeMutation moreEltsToNext32Bit(unsigned TypeIdx) {
  return [=](const LegalityQuery &Query) {
    const LLT Ty = Query.Types[TypeIdx];

    const LLT EltTy = Ty.getElementType();
    const int Size = Ty.getSizeInBits();
    const int EltSize = EltTy.getSizeInBits();
    const int NextMul32 = (Size + 31) / 32;

    assert(EltSize < 32);

    const int NewNumElts = (32 * NextMul32 + EltSize - 1) / EltSize;
    return std::pair(TypeIdx, LLT::fixed_vector(NewNumElts, EltTy));
  };
}

// Retrieves the scalar type that's the same size as the mem desc
static LegalizeMutation getScalarTypeFromMemDesc(unsigned TypeIdx) {
  return [=](const LegalityQuery &Query) {
    unsigned MemSize = Query.MMODescrs[0].MemoryTy.getSizeInBits();
    return std::make_pair(TypeIdx, LLT::integer(MemSize));
  };
}

// Increase the number of vector elements to reach the next legal RegClass.
static LegalizeMutation moreElementsToNextExistingRegClass(unsigned TypeIdx) {
  return [=](const LegalityQuery &Query) {
    const LLT Ty = Query.Types[TypeIdx];
    const unsigned NumElts = Ty.getNumElements();
    const unsigned EltSize = Ty.getElementType().getSizeInBits();
    const unsigned MaxNumElts = MaxRegisterSize / EltSize;

    assert(EltSize == 32 || EltSize == 64);
    assert(Ty.getSizeInBits() < MaxRegisterSize);

    unsigned NewNumElts;
    // Find the nearest legal RegClass that is larger than the current type.
    for (NewNumElts = NumElts; NewNumElts < MaxNumElts; ++NewNumElts) {
      if (SIRegisterInfo::getSGPRClassForBitWidth(NewNumElts * EltSize))
        break;
    }
    return std::pair(TypeIdx,
                     LLT::fixed_vector(NewNumElts, Ty.getElementType()));
  };
}

static LLT getBufferRsrcScalarType(const LLT Ty) {
  if (!Ty.isVector())
    return LLT::scalar(128);
  const ElementCount NumElems = Ty.getElementCount();
  return LLT::vector(NumElems, LLT::scalar(128));
}

static LLT getBufferRsrcRegisterType(const LLT Ty) {
  if (!Ty.isVector())
    return LLT::fixed_vector(4, LLT::integer(32));
  const unsigned NumElems = Ty.getElementCount().getFixedValue();
  return LLT::fixed_vector(NumElems * 4, LLT::integer(32));
}

static LLT getBitcastRegisterType(const LLT Ty) {
  const unsigned Size = Ty.getSizeInBits();

  if (Size <= 32) {
    // <2 x i8> -> i16
    // <4 x i8> -> i32
    return LLT::integer(Size);
  }

  return LLT::fixed_vector(Size / 32, LLT::integer(32));
}

static LegalizeMutation bitcastToRegisterType(unsigned TypeIdx) {
  return [=](const LegalityQuery &Query) {
    const LLT Ty = Query.Types[TypeIdx];
    return std::pair(TypeIdx, getBitcastRegisterType(Ty));
  };
}

static LegalizeMutation bitcastToVectorElement32(unsigned TypeIdx) {
  return [=](const LegalityQuery &Query) {
    const LLT Ty = Query.Types[TypeIdx];
    unsigned Size = Ty.getSizeInBits();
    assert(Size % 32 == 0);
    return std::pair(TypeIdx,
                     LLT::scalarOrVector(ElementCount::getFixed(Size / 32),
                                         LLT::integer(32)));
  };
}

static LegalityPredicate vectorSmallerThan(unsigned TypeIdx, unsigned Size) {
  return [=](const LegalityQuery &Query) {
    const LLT QueryTy = Query.Types[TypeIdx];
    return QueryTy.isVector() && QueryTy.getSizeInBits() < Size;
  };
}

static LegalityPredicate vectorWiderThan(unsigned TypeIdx, unsigned Size) {
  return [=](const LegalityQuery &Query) {
    const LLT QueryTy = Query.Types[TypeIdx];
    return QueryTy.isVector() && QueryTy.getSizeInBits() > Size;
  };
}

static LegalityPredicate numElementsNotEven(unsigned TypeIdx) {
  return [=](const LegalityQuery &Query) {
    const LLT QueryTy = Query.Types[TypeIdx];
    return QueryTy.isVector() && QueryTy.getNumElements() % 2 != 0;
  };
}

static bool isRegisterSize(const GCNSubtarget &ST, unsigned Size) {
  return ((ST.useRealTrue16Insts() && Size == 16) || Size % 32 == 0) &&
         Size <= MaxRegisterSize;
}

static bool isRegisterVectorElementType(LLT EltTy) {
  const int EltSize = EltTy.getSizeInBits();
  return EltSize == 16 || EltSize % 32 == 0;
}

static bool isRegisterVectorType(LLT Ty) {
  const int EltSize = Ty.getElementType().getSizeInBits();
  return EltSize == 32 || EltSize == 64 ||
         (EltSize == 16 && Ty.getNumElements() % 2 == 0) ||
         EltSize == 128 || EltSize == 256;
}

// TODO: replace all uses of isRegisterType with isRegisterClassType
static bool isRegisterType(const GCNSubtarget &ST, LLT Ty) {
  if (!isRegisterSize(ST, Ty.getSizeInBits()))
    return false;

  if (Ty.isVector())
    return isRegisterVectorType(Ty);

  return true;
}

// Any combination of 32 or 64-bit elements up the maximum register size, and
// multiples of v2s16.
static LegalityPredicate isRegisterType(const GCNSubtarget &ST,
                                        unsigned TypeIdx) {
  return [=, &ST](const LegalityQuery &Query) {
    return isRegisterType(ST, Query.Types[TypeIdx]);
  };
}

// RegisterType that doesn't have a corresponding RegClass.
// TODO: Once `isRegisterType` is replaced with `isRegisterClassType` this
// should be removed.
static LegalityPredicate isIllegalRegisterType(const GCNSubtarget &ST,
                                               unsigned TypeIdx) {
  return [=, &ST](const LegalityQuery &Query) {
    LLT Ty = Query.Types[TypeIdx];
    return isRegisterType(ST, Ty) &&
           !SIRegisterInfo::getSGPRClassForBitWidth(Ty.getSizeInBits());
  };
}

static LegalityPredicate elementTypeIsLegal(unsigned TypeIdx) {
  return [=](const LegalityQuery &Query) {
    const LLT QueryTy = Query.Types[TypeIdx];
    if (!QueryTy.isVector())
      return false;
    const LLT EltTy = QueryTy.getElementType();
    return EltTy == LLT::scalar(16) || EltTy.getSizeInBits() >= 32;
  };
}

const LLT I16 = LLT::integer(16);
constexpr LLT F16 = LLT::float16();
constexpr LLT BF16 = LLT::bfloat16();
constexpr LLT F32 = LLT::float32();
constexpr LLT F64 = LLT::float64();
constexpr LLT V2F16 = LLT::fixed_vector(2, F16);
constexpr LLT V2BF16 = LLT::fixed_vector(2, BF16);

constexpr LLT S1 = LLT::scalar(1);
constexpr LLT S8 = LLT::scalar(8);
constexpr LLT S16 = LLT::scalar(16);
constexpr LLT S32 = LLT::scalar(32);
constexpr LLT S64 = LLT::scalar(64);
constexpr LLT S96 = LLT::scalar(96);
constexpr LLT S128 = LLT::scalar(128);
constexpr LLT S160 = LLT::scalar(160);
constexpr LLT S192 = LLT::scalar(192);
constexpr LLT S224 = LLT::scalar(224);
constexpr LLT S256 = LLT::scalar(256);
constexpr LLT S512 = LLT::scalar(512);
constexpr LLT S1024 = LLT::scalar(1024);
constexpr LLT MaxScalar = LLT::scalar(MaxRegisterSize);

constexpr LLT V2S8 = LLT::fixed_vector(2, 8);
constexpr LLT V2S16 = LLT::fixed_vector(2, 16);
constexpr LLT V4S16 = LLT::fixed_vector(4, 16);
constexpr LLT V6S16 = LLT::fixed_vector(6, 16);
constexpr LLT V8S16 = LLT::fixed_vector(8, 16);
constexpr LLT V10S16 = LLT::fixed_vector(10, 16);
constexpr LLT V12S16 = LLT::fixed_vector(12, 16);
constexpr LLT V16S16 = LLT::fixed_vector(16, 16);

constexpr LLT V2S32 = LLT::fixed_vector(2, 32);
constexpr LLT V3S32 = LLT::fixed_vector(3, 32);
constexpr LLT V4S32 = LLT::fixed_vector(4, 32);
constexpr LLT V5S32 = LLT::fixed_vector(5, 32);
constexpr LLT V6S32 = LLT::fixed_vector(6, 32);
constexpr LLT V7S32 = LLT::fixed_vector(7, 32);
constexpr LLT V8S32 = LLT::fixed_vector(8, 32);
constexpr LLT V9S32 = LLT::fixed_vector(9, 32);
constexpr LLT V10S32 = LLT::fixed_vector(10, 32);
constexpr LLT V11S32 = LLT::fixed_vector(11, 32);
constexpr LLT V12S32 = LLT::fixed_vector(12, 32);
constexpr LLT V16S32 = LLT::fixed_vector(16, 32);
constexpr LLT V32S32 = LLT::fixed_vector(32, 32);

constexpr LLT V2S64 = LLT::fixed_vector(2, 64);
constexpr LLT V3S64 = LLT::fixed_vector(3, 64);
constexpr LLT V4S64 = LLT::fixed_vector(4, 64);
constexpr LLT V5S64 = LLT::fixed_vector(5, 64);
constexpr LLT V6S64 = LLT::fixed_vector(6, 64);
constexpr LLT V7S64 = LLT::fixed_vector(7, 64);
constexpr LLT V8S64 = LLT::fixed_vector(8, 64);
constexpr LLT V16S64 = LLT::fixed_vector(16, 64);

constexpr LLT V2S128 = LLT::fixed_vector(2, 128);
constexpr LLT V4S128 = LLT::fixed_vector(4, 128);

constexpr std::initializer_list<LLT> AllScalarTypes = {
    S32, S64, S96, S128, S160, S192, S224, S256, S512, S1024};

constexpr std::initializer_list<LLT> AllS16Vectors{
    V2S16, V4S16, V6S16, V8S16, V10S16, V12S16, V16S16, V2S128, V4S128};

constexpr std::initializer_list<LLT> AllS32Vectors = {
    V2S32, V3S32,  V4S32,  V5S32,  V6S32,  V7S32, V8S32,
    V9S32, V10S32, V11S32, V12S32, V16S32, V32S32};

constexpr std::initializer_list<LLT> AllS64Vectors = {
    V2S64, V3S64, V4S64, V5S64, V6S64, V7S64, V8S64, V16S64};

constexpr std::initializer_list<LLT> AllVectors{
    V2S16,  V4S16,  V6S16,  V8S16,  V10S16, V12S16, V16S16, V2S128,
    V4S128, V2S32,  V3S32,  V4S32,  V5S32,  V6S32,  V7S32,  V8S32,
    V9S32,  V10S32, V11S32, V12S32, V16S32, V32S32, V2S64,  V3S64,
    V4S64,  V5S64,  V6S64,  V7S64,  V8S64,  V16S64};

// Checks whether a type is in the list of legal register types.
static bool isRegisterClassType(const GCNSubtarget &ST, LLT Ty) {
  if (Ty.isPointerOrPointerVector())
    Ty = Ty.changeElementType(LLT::scalar(Ty.getScalarSizeInBits()));

  return is_contained(AllS32Vectors, Ty) || is_contained(AllS64Vectors, Ty) ||
         is_contained(AllScalarTypes, Ty) ||
         (ST.useRealTrue16Insts() && Ty == S16) ||
         is_contained(AllS16Vectors, Ty);
}

static LegalityPredicate isRegisterClassType(const GCNSubtarget &ST,
                                             unsigned TypeIdx) {
  return [&ST, TypeIdx](const LegalityQuery &Query) {
    return isRegisterClassType(ST, Query.Types[TypeIdx]);
  };
}

// If we have a truncating store or an extending load with a data size larger
// than 32-bits, we need to reduce to a 32-bit type.
static LegalityPredicate isWideScalarExtLoadTruncStore(unsigned TypeIdx) {
  return [=](const LegalityQuery &Query) {
    const LLT Ty = Query.Types[TypeIdx];
    return !Ty.isVector() && Ty.getSizeInBits() > 32 &&
           Query.MMODescrs[0].MemoryTy.getSizeInBits() < Ty.getSizeInBits();
  };
}

// If we have a truncating store or an extending load with a data size larger
// than 32-bits and mem location is a power of 2
static LegalityPredicate isTruncStoreToSizePowerOf2(unsigned TypeIdx) {
  return [=](const LegalityQuery &Query) {
    unsigned MemSize = Query.MMODescrs[0].MemoryTy.getSizeInBits();
    return isWideScalarExtLoadTruncStore(TypeIdx)(Query) &&
           isPowerOf2_64(MemSize);
  };
}

// TODO: Should load to s16 be legal? Most loads extend to 32-bits, but we
// handle some operations by just promoting the register during
// selection. There are also d16 loads on GFX9+ which preserve the high bits.
static unsigned maxSizeForAddrSpace(const GCNSubtarget &ST, unsigned AS,
                                    bool IsLoad, bool IsAtomic) {
  switch (AS) {
  case AMDGPUAS::PRIVATE_ADDRESS:
    // FIXME: Private element size.
    return ST.hasFlatScratchEnabled() ? 128 : 32;
  case AMDGPUAS::LOCAL_ADDRESS:
    return ST.useDS128() ? 128 : 64;
  case AMDGPUAS::GLOBAL_ADDRESS:
  case AMDGPUAS::CONSTANT_ADDRESS:
  case AMDGPUAS::CONSTANT_ADDRESS_32BIT:
  case AMDGPUAS::BUFFER_RESOURCE:
    // Treat constant and global as identical. SMRD loads are sometimes usable for
    // global loads (ideally constant address space should be eliminated)
    // depending on the context. Legality cannot be context dependent, but
    // RegBankSelect can split the load as necessary depending on the pointer
    // register bank/uniformity and if the memory is invariant or not written in a
    // kernel.
    return IsLoad ? 512 : 128;
  default:
    // FIXME: Flat addresses may contextually need to be split to 32-bit parts
    // if they may alias scratch depending on the subtarget.  This needs to be
    // moved to custom handling to use addressMayBeAccessedAsPrivate
    return ST.hasMultiDwordFlatScratchAddressing() || IsAtomic ? 128 : 32;
  }
}

static bool isLoadStoreSizeLegal(const GCNSubtarget &ST,
                                 const LegalityQuery &Query) {
  const LLT Ty = Query.Types[0];

  // Handle G_LOAD, G_ZEXTLOAD, G_SEXTLOAD
  const bool IsLoad = Query.Opcode != AMDGPU::G_STORE;

  unsigned RegSize = Ty.getSizeInBits();
  uint64_t MemSize = Query.MMODescrs[0].MemoryTy.getSizeInBits();
  uint64_t AlignBits = Query.MMODescrs[0].AlignInBits;
  unsigned AS = Query.Types[1].getAddressSpace();

  // All of these need to be custom lowered to cast the pointer operand.
  if (AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT)
    return false;

  // Do not handle extending vector loads.
  if (Ty.isVector() && MemSize != RegSize)
    return false;

  // TODO: We should be able to widen loads if the alignment is high enough, but
  // we also need to modify the memory access size.
#if 0
  // Accept widening loads based on alignment.
  if (IsLoad && MemSize < Size)
    MemSize = std::max(MemSize, Align);
#endif

  // Only 1-byte and 2-byte to 32-bit extloads are valid.
  if (MemSize != RegSize && RegSize != 32)
    return false;

  if (MemSize > maxSizeForAddrSpace(ST, AS, IsLoad,
                                    Query.MMODescrs[0].Ordering !=
                                        AtomicOrdering::NotAtomic))
    return false;

  switch (MemSize) {
  case 8:
  case 16:
  case 32:
  case 64:
  case 128:
    break;
  case 96:
    if (!ST.hasDwordx3LoadStores())
      return false;
    break;
  case 256:
  case 512:
    // These may contextually need to be broken down.
    break;
  default:
    return false;
  }

  assert(RegSize >= MemSize);

  if (AlignBits < MemSize) {
    const SITargetLowering *TLI = ST.getTargetLowering();
    if (!TLI->allowsMisalignedMemoryAccessesImpl(MemSize, AS,
                                                 Align(AlignBits / 8)))
      return false;
  }

  return true;
}

// The newer buffer intrinsic forms take their resource arguments as
// pointers in address space 8, aka s128 values. However, in order to not break
// SelectionDAG, the underlying operations have to continue to take v4i32
// arguments. Therefore, we convert resource pointers - or vectors of them
// to integer values here.
static bool hasBufferRsrcWorkaround(const LLT Ty) {
  if (Ty.isPointer() && Ty.getAddressSpace() == AMDGPUAS::BUFFER_RESOURCE)
    return true;
  if (Ty.isVector()) {
    const LLT ElemTy = Ty.getElementType();
    return hasBufferRsrcWorkaround(ElemTy);
  }
  return false;
}

// The current selector can't handle <6 x s16>, <8 x s16>, s96, s128 etc, so
// workaround this. Eventually it should ignore the type for loads and only care
// about the size. Return true in cases where we will workaround this for now by
// bitcasting.
static bool loadStoreBitcastWorkaround(const LLT Ty) {
  if (EnableNewLegality)
    return false;

  const unsigned Size = Ty.getSizeInBits();
  if (Ty.isPointerVector())
    return true;
  if (Size <= 64)
    return false;
  // Address space 8 pointers get their own workaround.
  if (hasBufferRsrcWorkaround(Ty))
    return false;
  if (!Ty.isVector())
    return true;

  unsigned EltSize = Ty.getScalarSizeInBits();
  return EltSize != 32 && EltSize != 64;
}

static bool isLoadStoreLegal(const GCNSubtarget &ST, const LegalityQuery &Query) {
  const LLT Ty = Query.Types[0];
  return isRegisterType(ST, Ty) && isLoadStoreSizeLegal(ST, Query) &&
         !hasBufferRsrcWorkaround(Ty) && !loadStoreBitcastWorkaround(Ty);
}

/// Return true if a load or store of the type should be lowered with a bitcast
/// to a different type.
static bool shouldBitcastLoadStoreType(const GCNSubtarget &ST, const LLT Ty,
                                       const LLT MemTy) {
  const unsigned MemSizeInBits = MemTy.getSizeInBits();
  const unsigned Size = Ty.getSizeInBits();
  if (Size != MemSizeInBits)
    return Size <= 32 && Ty.isVector();

  if (loadStoreBitcastWorkaround(Ty) && isRegisterType(ST, Ty))
    return true;

  // Don't try to handle bitcasting vector ext loads for now.
  return Ty.isVector() && (!MemTy.isVector() || MemTy == Ty) &&
         (Size <= 32 || isRegisterSize(ST, Size)) &&
         !isRegisterVectorElementType(Ty.getElementType());
}

/// Return true if we should legalize a load by widening an odd sized memory
/// access up to the alignment. Note this case when the memory access itself
/// changes, not the size of the result register.
static bool shouldWidenLoad(const GCNSubtarget &ST, LLT MemoryTy,
                            uint64_t AlignInBits, unsigned AddrSpace,
                            unsigned Opcode) {
  unsigned SizeInBits = MemoryTy.getSizeInBits();
  // We don't want to widen cases that are naturally legal.
  if (isPowerOf2_32(SizeInBits))
    return false;

  // If we have 96-bit memory operations, we shouldn't touch them. Note we may
  // end up widening these for a scalar load during RegBankSelect, if we don't
  // have 96-bit scalar loads.
  if (SizeInBits == 96 && ST.hasDwordx3LoadStores())
    return false;

  if (SizeInBits >= maxSizeForAddrSpace(ST, AddrSpace, Opcode, false))
    return false;

  // A load is known dereferenceable up to the alignment, so it's legal to widen
  // to it.
  //
  // TODO: Could check dereferenceable for less aligned cases.
  unsigned RoundedSize = NextPowerOf2(SizeInBits);
  if (AlignInBits < RoundedSize)
    return false;

  // Do not widen if it would introduce a slow unaligned load.
  const SITargetLowering *TLI = ST.getTargetLowering();
  unsigned Fast = 0;
  return TLI->allowsMisalignedMemoryAccessesImpl(
             RoundedSize, AddrSpace, Align(AlignInBits / 8),
             MachineMemOperand::MOLoad, &Fast) &&
         Fast;
}

static bool shouldWidenLoad(const GCNSubtarget &ST, const LegalityQuery &Query,
                            unsigned Opcode) {
  if (Query.MMODescrs[0].Ordering != AtomicOrdering::NotAtomic)
    return false;

  return shouldWidenLoad(ST, Query.MMODescrs[0].MemoryTy,
                         Query.MMODescrs[0].AlignInBits,
                         Query.Types[1].getAddressSpace(), Opcode);
}

/// Mutates IR (typicaly a load instruction) to use a <4 x s32> as the initial
/// type of the operand `idx` and then to transform it to a `p8` via bitcasts
/// and inttoptr. In addition, handle vectors of p8. Returns the new type.
static LLT castBufferRsrcFromV4I32(MachineInstr &MI, MachineIRBuilder &B,
                                   MachineRegisterInfo &MRI, unsigned Idx) {
  MachineOperand &MO = MI.getOperand(Idx);

  const LLT PointerTy = MRI.getType(MO.getReg());

  // Paranoidly prevent us from doing this multiple times.
  if (!hasBufferRsrcWorkaround(PointerTy))
    return PointerTy;

  const LLT ScalarTy = getBufferRsrcScalarType(PointerTy);
  const LLT VectorTy = getBufferRsrcRegisterType(PointerTy);
  if (!PointerTy.isVector()) {
    // Happy path: (4 x s32) -> (s32, s32, s32, s32) -> (p8)
    const unsigned NumParts = PointerTy.getSizeInBits() / 32;
    const LLT I32 = LLT::integer(32);

    Register VectorReg = MRI.createGenericVirtualRegister(VectorTy);
    std::array<Register, 4> VectorElems;
    B.setInsertPt(B.getMBB(), ++B.getInsertPt());
    for (unsigned I = 0; I < NumParts; ++I)
      VectorElems[I] =
          B.buildExtractVectorElementConstant(I32, VectorReg, I).getReg(0);
    B.buildMergeValues(MO, VectorElems);
    MO.setReg(VectorReg);
    return VectorTy;
  }
  Register BitcastReg = MRI.createGenericVirtualRegister(VectorTy);
  B.setInsertPt(B.getMBB(), ++B.getInsertPt());
  auto Scalar = B.buildBitcast(ScalarTy, BitcastReg);
  B.buildIntToPtr(MO, Scalar);
  MO.setReg(BitcastReg);

  return VectorTy;
}

/// Cast a buffer resource (an address space 8 pointer) into a 4xi32, which is
/// the form in which the value must be in order to be passed to the low-level
/// representations used for MUBUF/MTBUF intrinsics. This is a hack, which is
/// needed in order to account for the fact that we can't define a register
/// class for s128 without breaking SelectionDAG.
static Register castBufferRsrcToV4I32(Register Pointer, MachineIRBuilder &B) {
  MachineRegisterInfo &MRI = *B.getMRI();
  const LLT PointerTy = MRI.getType(Pointer);
  const LLT ScalarTy = getBufferRsrcScalarType(PointerTy);
  const LLT VectorTy = getBufferRsrcRegisterType(PointerTy);

  if (!PointerTy.isVector()) {
    // Special case: p8 -> (s32, s32, s32, s32) -> (4xs32)
    SmallVector<Register, 4> PointerParts;
    const unsigned NumParts = PointerTy.getSizeInBits() / 32;
    auto Unmerged = B.buildUnmerge(LLT::integer(32), Pointer);
    for (unsigned I = 0; I < NumParts; ++I)
      PointerParts.push_back(Unmerged.getReg(I));
    return B.buildBuildVector(VectorTy, PointerParts).getReg(0);
  }
  Register Scalar = B.buildPtrToInt(ScalarTy, Pointer).getReg(0);
  return B.buildBitcast(VectorTy, Scalar).getReg(0);
}

static void castBufferRsrcArgToV4I32(MachineInstr &MI, MachineIRBuilder &B,
                                     unsigned Idx) {
  MachineOperand &MO = MI.getOperand(Idx);

  const LLT PointerTy = B.getMRI()->getType(MO.getReg());
  // Paranoidly prevent us from doing this multiple times.
  if (!hasBufferRsrcWorkaround(PointerTy))
    return;
  MO.setReg(castBufferRsrcToV4I32(MO.getReg(), B));
}

AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
                                         const GCNTargetMachine &TM)
  :  ST(ST_) {
  using namespace TargetOpcode;

  auto GetAddrSpacePtr = [&TM](unsigned AS) {
    return LLT::pointer(AS, TM.getPointerSizeInBits(AS));
  };

  const LLT GlobalPtr = GetAddrSpacePtr(AMDGPUAS::GLOBAL_ADDRESS);
  const LLT ConstantPtr = GetAddrSpacePtr(AMDGPUAS::CONSTANT_ADDRESS);
  const LLT Constant32Ptr = GetAddrSpacePtr(AMDGPUAS::CONSTANT_ADDRESS_32BIT);
  const LLT LocalPtr = GetAddrSpacePtr(AMDGPUAS::LOCAL_ADDRESS);
  const LLT RegionPtr = GetAddrSpacePtr(AMDGPUAS::REGION_ADDRESS);
  const LLT FlatPtr = GetAddrSpacePtr(AMDGPUAS::FLAT_ADDRESS);
  const LLT PrivatePtr = GetAddrSpacePtr(AMDGPUAS::PRIVATE_ADDRESS);
  const LLT BufferFatPtr = GetAddrSpacePtr(AMDGPUAS::BUFFER_FAT_POINTER);
  const LLT RsrcPtr = GetAddrSpacePtr(AMDGPUAS::BUFFER_RESOURCE);
  const LLT BufferStridedPtr =
      GetAddrSpacePtr(AMDGPUAS::BUFFER_STRIDED_POINTER);

  const LLT CodePtr = FlatPtr;

  const std::initializer_list<LLT> AddrSpaces64 = {
    GlobalPtr, ConstantPtr, FlatPtr
  };

  const std::initializer_list<LLT> AddrSpaces32 = {
    LocalPtr, PrivatePtr, Constant32Ptr, RegionPtr
  };

  const std::initializer_list<LLT> AddrSpaces128 = {RsrcPtr};

  const std::initializer_list<LLT> FPTypesBase = {
    S32, S64
  };

  const std::initializer_list<LLT> FPTypes16 = {
    S32, S64, S16
  };

  const std::initializer_list<LLT> FPTypesPK16 = {
    S32, S64, S16, V2S16
  };

  const std::initializer_list<LLT> FPTypesPK16_64 = {S32, S64, S16, V2S16,
                                                     V2S64};

  const LLT MinScalarFPTy = ST.has16BitInsts() ? S16 : S32;

  getActionDefinitionsBuilder(G_BR).alwaysLegal();

  // s1 for VCC branches, s32 for SCC branches.
  getActionDefinitionsBuilder(G_BRCOND).legalFor({S1, S32});

  // TODO: All multiples of 32, vectors of pointers, all v2s16 pairs, more
  // elements for v3s16
  getActionDefinitionsBuilder(G_PHI)
      .legalFor({S32, S64, V2S16, S16, V4S16, S1, S128, S256})
      .legalFor(AllS32Vectors)
      .legalFor(AllS64Vectors)
      .legalFor(AddrSpaces64)
      .legalFor(AddrSpaces32)
      .legalFor(AddrSpaces128)
      .legalIf(isPointer(0))
      .clampScalar(0, S16, S256)
      .widenScalarToNextPow2(0, 32)
      .clampMaxNumElements(0, S32, 16)
      .moreElementsIf(isSmallOddVector(0), oneMoreElement(0))
      .scalarize(0);

  if (ST.hasVOP3PInsts() && ST.hasAddNoCarryInsts() && ST.hasIntClamp()) {
    // Full set of gfx9 features.
    if (ST.hasPackedU64Ops()) {
      getActionDefinitionsBuilder({G_ADD, G_SUB})
          .legalFor({S64, S32, S16, V2S16, V2S64})
          .clampMaxNumElementsStrict(0, S16, 2)
          .clampMaxNumElementsStrict(0, S64, 2)
          .scalarize(0)
          .minScalar(0, S16)
          .widenScalarToNextMultipleOf(0, 32)
          .maxScalar(0, S32);
    } else if (ST.hasScalarAddSub64()) {
      getActionDefinitionsBuilder({G_ADD, G_SUB})
          .legalFor({S64, S32, S16, V2S16})
          .clampMaxNumElementsStrict(0, S16, 2)
          .scalarize(0)
          .minScalar(0, S16)
          .widenScalarToNextMultipleOf(0, 32)
          .maxScalar(0, S32);
    } else {
      getActionDefinitionsBuilder({G_ADD, G_SUB})
          .legalFor({S32, S16, V2S16})
          .clampMaxNumElementsStrict(0, S16, 2)
          .scalarize(0)
          .minScalar(0, S16)
          .widenScalarToNextMultipleOf(0, 32)
          .maxScalar(0, S32);
    }

    if (ST.hasScalarSMulU64()) {
      getActionDefinitionsBuilder(G_MUL)
          .legalFor({S64, S32, S16, V2S16})
          .clampMaxNumElementsStrict(0, S16, 2)
          .scalarize(0)
          .minScalar(0, S16)
          .widenScalarToNextMultipleOf(0, 32)
          .custom();
    } else {
      getActionDefinitionsBuilder(G_MUL)
          .legalFor({S32, S16, V2S16})
          .clampMaxNumElementsStrict(0, S16, 2)
          .scalarize(0)
          .minScalar(0, S16)
          .widenScalarToNextMultipleOf(0, 32)
          .custom();
    }
    assert(ST.hasMad64_32());

    getActionDefinitionsBuilder({G_UADDSAT, G_USUBSAT, G_SADDSAT, G_SSUBSAT})
      .legalFor({S32, S16, V2S16}) // Clamp modifier
      .minScalarOrElt(0, S16)
      .clampMaxNumElementsStrict(0, S16, 2)
      .scalarize(0)
      .widenScalarToNextPow2(0, 32)
      .lower();
  } else if (ST.has16BitInsts()) {
    getActionDefinitionsBuilder({G_ADD, G_SUB})
      .legalFor({S32, S16})
      .minScalar(0, S16)
      .widenScalarToNextMultipleOf(0, 32)
      .maxScalar(0, S32)
      .scalarize(0);

    getActionDefinitionsBuilder(G_MUL)
      .legalFor({S32, S16})
      .scalarize(0)
      .minScalar(0, S16)
      .widenScalarToNextMultipleOf(0, 32)
      .custom();
    assert(ST.hasMad64_32());

    // Technically the saturating operations require clamp bit support, but this
    // was introduced at the same time as 16-bit operations.
    getActionDefinitionsBuilder({G_UADDSAT, G_USUBSAT})
      .legalFor({S32, S16}) // Clamp modifier
      .minScalar(0, S16)
      .scalarize(0)
      .widenScalarToNextPow2(0, 16)
      .lower();

    // We're just lowering this, but it helps get a better result to try to
    // coerce to the desired type first.
    getActionDefinitionsBuilder({G_SADDSAT, G_SSUBSAT})
      .minScalar(0, S16)
      .scalarize(0)
      .lower();
  } else {
    getActionDefinitionsBuilder({G_ADD, G_SUB})
      .legalFor({S32})
      .widenScalarToNextMultipleOf(0, 32)
      .clampScalar(0, S32, S32)
      .scalarize(0);

    auto &Mul = getActionDefinitionsBuilder(G_MUL)
      .legalFor({S32})
      .scalarize(0)
      .minScalar(0, S32)
      .widenScalarToNextMultipleOf(0, 32);

    if (ST.hasMad64_32())
      Mul.custom();
    else
      Mul.maxScalar(0, S32);

    if (ST.hasIntClamp()) {
      getActionDefinitionsBuilder({G_UADDSAT, G_USUBSAT})
        .legalFor({S32}) // Clamp modifier.
        .scalarize(0)
        .minScalarOrElt(0, S32)
        .lower();
    } else {
      // Clamp bit support was added in VI, along with 16-bit operations.
      getActionDefinitionsBuilder({G_UADDSAT, G_USUBSAT})
        .minScalar(0, S32)
        .scalarize(0)
        .lower();
    }

    // FIXME: DAG expansion gets better results. The widening uses the smaller
    // range values and goes for the min/max lowering directly.
    getActionDefinitionsBuilder({G_SADDSAT, G_SSUBSAT})
      .minScalar(0, S32)
      .scalarize(0)
      .lower();
  }

  getActionDefinitionsBuilder(
      {G_SDIV, G_UDIV, G_SREM, G_UREM, G_SDIVREM, G_UDIVREM})
      .customFor({S32, S64})
      .clampScalar(0, S32, S64)
      .widenScalarToNextPow2(0, 32)
      .scalarize(0);

  auto &Mulh = getActionDefinitionsBuilder({G_UMULH, G_SMULH})
                   .legalFor({S32})
                   .maxScalar(0, S32);

  if (ST.hasVOP3PInsts()) {
    Mulh
      .clampMaxNumElements(0, S8, 2)
      .lowerFor({V2S8});
  }

  Mulh
    .scalarize(0)
    .lower();

  // Report legal for any types we can handle anywhere. For the cases only legal
  // on the SALU, RegBankSelect will be able to re-legalize.
  getActionDefinitionsBuilder({G_AND, G_OR, G_XOR})
      .legalFor({S32, S1, S64, V2S32, S16, V2S16, V4S16})
      .clampScalar(0, S32, S64)
      .moreElementsIf(isSmallOddVector(0), oneMoreElement(0))
      .fewerElementsIf(
          all(vectorWiderThan(0, 64), scalarOrEltNarrowerThan(0, 64)),
          fewerEltsToSize64Vector(0))
      .widenScalarToNextPow2(0)
      .scalarize(0);

  getActionDefinitionsBuilder(
      {G_UADDO, G_USUBO, G_UADDE, G_SADDE, G_USUBE, G_SSUBE})
      .legalFor({{S32, S1}, {S32, S32}})
      .clampScalar(0, S32, S32)
      .scalarize(0);

  getActionDefinitionsBuilder(G_BITCAST)
      // Don't worry about the size constraint.
      .legalIf(all(isRegisterClassType(ST, 0), isRegisterClassType(ST, 1)))
      .widenScalarIf(all(typeInSet(0, {I16, F16, BF16}), isScalar(1)),
                     changeTo(0, LLT::integer(32)))
      .widenScalarIf(all(isScalar(0), typeInSet(1, {I16, F16, BF16})),
                     changeTo(1, LLT::integer(32)))
      .lower();

  getActionDefinitionsBuilder(G_CONSTANT)
    .legalFor({S1, S32, S64, S16, GlobalPtr,
               LocalPtr, ConstantPtr, PrivatePtr, FlatPtr })
    .legalIf(isPointer(0))
    .clampScalar(0, S32, S64)
    .widenScalarToNextPow2(0);

  getActionDefinitionsBuilder(G_FCONSTANT)
    .legalFor({S32, S64, S16})
    .clampScalar(0, S16, S64);

  getActionDefinitionsBuilder({G_IMPLICIT_DEF, G_FREEZE})
      .legalIf(isRegisterClassType(ST, 0))
      // s1 and s16 are special cases because they have legal operations on
      // them, but don't really occupy registers in the normal way.
      .legalFor({S1, S16})
      .clampNumElements(0, V16S32, V32S32)
      .moreElementsIf(isSmallOddVector(0), oneMoreElement(0))
      .clampScalarOrElt(0, S32, MaxScalar)
      .widenScalarToNextPow2(0, 32)
      .clampMaxNumElements(0, S32, 16);

  getActionDefinitionsBuilder(G_FRAME_INDEX).legalFor({PrivatePtr});

  // If the amount is divergent, we have to do a wave reduction to get the
  // maximum value, so this is expanded during RegBankSelect.
  getActionDefinitionsBuilder(G_DYN_STACKALLOC)
    .legalFor({{PrivatePtr, S32}});

  getActionDefinitionsBuilder(G_STACKSAVE)
    .customFor({PrivatePtr});
  getActionDefinitionsBuilder(G_STACKRESTORE)
    .legalFor({PrivatePtr});

  getActionDefinitionsBuilder({G_GET_FPENV, G_SET_FPENV}).customFor({S64});

  getActionDefinitionsBuilder({G_GET_ROUNDING, G_SET_ROUNDING}).legalFor({S32});

  getActionDefinitionsBuilder(G_GLOBAL_VALUE)
    .customIf(typeIsNot(0, PrivatePtr));

  getActionDefinitionsBuilder(G_BLOCK_ADDR).legalFor({CodePtr});

  auto &FPOpActions = getActionDefinitionsBuilder(
    { G_FADD, G_FMUL, G_FMA, G_FCANONICALIZE,
      G_STRICT_FADD, G_STRICT_FMUL, G_STRICT_FMA})
    .legalFor({S32, S64});
  auto &TrigActions = getActionDefinitionsBuilder({G_FSIN, G_FCOS})
    .customFor({S32, S64});
  auto &FDIVActions = getActionDefinitionsBuilder(G_FDIV)
    .customFor({S32, S64});

  if (ST.has16BitInsts()) {
    if (ST.hasVOP3PInsts())
      FPOpActions.legalFor({S16, V2S16});
    else
      FPOpActions.legalFor({S16});

    TrigActions.customFor({S16});
    FDIVActions.customFor({S16});
  }

  if (ST.hasPackedFP32Ops()) {
    FPOpActions.legalFor({V2S32});
    FPOpActions.clampMaxNumElementsStrict(0, S32, 2);
  }

  if (ST.hasPackedFP64Ops()) {
    FPOpActions.legalFor({V2S64});
    FPOpActions.clampMaxNumElementsStrict(0, S64, 2);
  }

  if (ST.hasPackedFP64Ops()) {
    FPOpActions.legalFor({V2S64});
    FPOpActions.clampMaxNumElementsStrict(0, S64, 2);
  }

  auto &MinNumMaxNumIeee =
      getActionDefinitionsBuilder({G_FMINNUM_IEEE, G_FMAXNUM_IEEE});

  if (ST.hasVOP3PInsts()) {
    MinNumMaxNumIeee.legalFor(FPTypesPK16)
        .moreElementsIf(isSmallOddVector(0), oneMoreElement(0))
        .clampMaxNumElements(0, S16, 2)
        .clampScalar(0, S16, S64)
        .scalarize(0);
  } else if (ST.has16BitInsts()) {
    MinNumMaxNumIeee.legalFor(FPTypes16).clampScalar(0, S16, S64).scalarize(0);
  } else {
    MinNumMaxNumIeee.legalFor(FPTypesBase)
        .clampScalar(0, S32, S64)
        .scalarize(0);
  }

  auto &MinNumMaxNum = getActionDefinitionsBuilder(
      {G_FMINNUM, G_FMAXNUM, G_FMINIMUMNUM, G_FMAXIMUMNUM});

  if (ST.hasPackedFP64Ops()) {
    MinNumMaxNum.customFor(FPTypesPK16_64)
        .moreElementsIf(isSmallOddVector(0), oneMoreElement(0))
        .clampMaxNumElements(0, S16, 2)
        .clampMaxNumElements(0, S64, 2)
        .clampScalar(0, S16, S64)
        .scalarize(0);
  } else if (ST.hasVOP3PInsts()) {
    MinNumMaxNum.customFor(FPTypesPK16)
      .moreElementsIf(isSmallOddVector(0), oneMoreElement(0))
      .clampMaxNumElements(0, S16, 2)
      .clampScalar(0, S16, S64)
      .scalarize(0);
  } else if (ST.has16BitInsts()) {
    MinNumMaxNum.customFor(FPTypes16)
      .clampScalar(0, S16, S64)
      .scalarize(0);
  } else {
    MinNumMaxNum.customFor(FPTypesBase)
      .clampScalar(0, S32, S64)
      .scalarize(0);
  }

  if (ST.hasVOP3PInsts())
    FPOpActions.clampMaxNumElementsStrict(0, S16, 2);

  FPOpActions
    .scalarize(0)
    .clampScalar(0, ST.has16BitInsts() ? S16 : S32, S64);

  TrigActions
    .scalarize(0)
    .clampScalar(0, ST.has16BitInsts() ? S16 : S32, S64);

  FDIVActions
    .scalarize(0)
    .clampScalar(0, ST.has16BitInsts() ? S16 : S32, S64);

  auto &FNegAbs = getActionDefinitionsBuilder({G_FNEG, G_FABS});
  FNegAbs.legalFor(FPTypesPK16)
      .legalFor(ST.hasPackedFP32Ops(), {V2S32})
      .clampMaxNumElementsStrict(0, S16, 2);
  if (ST.hasPackedFP32Ops())
    FNegAbs.clampMaxNumElementsStrict(0, S32, 2);
  FNegAbs.scalarize(0).clampScalar(0, S16, S64);

  if (ST.has16BitInsts()) {
    getActionDefinitionsBuilder(G_FSQRT)
      .legalFor({S16})
      .customFor({S32, S64})
      .scalarize(0)
      .unsupported();
    getActionDefinitionsBuilder(G_FFLOOR)
      .legalFor({S32, S64, S16})
      .scalarize(0)
      .clampScalar(0, S16, S64);

    getActionDefinitionsBuilder({G_FLDEXP, G_STRICT_FLDEXP})
      .legalFor({{S32, S32}, {S64, S32}, {S16, S16}})
      .scalarize(0)
      .maxScalarIf(typeIs(0, S16), 1, S16)
      .clampScalar(1, S32, S32)
      .lower();

    getActionDefinitionsBuilder(G_FFREXP)
      .customFor({{S32, S32}, {S64, S32}, {S16, S16}, {S16, S32}})
      .scalarize(0)
      .lower();

    getActionDefinitionsBuilder(G_FMODF)
        .lowerFor({S16, S32, S64})
        .scalarize(0)
        .lower();
  } else {
    getActionDefinitionsBuilder(G_FSQRT)
      .customFor({S32, S64, S16})
      .scalarize(0)
      .unsupported();


    if (ST.hasFractBug()) {
      getActionDefinitionsBuilder(G_FFLOOR)
        .customFor({S64})
        .legalFor({S32, S64})
        .scalarize(0)
        .clampScalar(0, S32, S64);
    } else {
      getActionDefinitionsBuilder(G_FFLOOR)
        .legalFor({S32, S64})
        .scalarize(0)
        .clampScalar(0, S32, S64);
    }

    getActionDefinitionsBuilder({G_FLDEXP, G_STRICT_FLDEXP})
      .legalFor({{S32, S32}, {S64, S32}})
      .scalarize(0)
      .clampScalar(0, S32, S64)
      .clampScalar(1, S32, S32)
      .lower();

    getActionDefinitionsBuilder(G_FFREXP)
      .customFor({{S32, S32}, {S64, S32}})
      .scalarize(0)
      .minScalar(0, S32)
      .clampScalar(1, S32, S32)
      .lower();

    getActionDefinitionsBuilder(G_FMODF)
        .lowerFor({S32, S64})
        .scalarize(0)
        .lower();
  }

  auto &FPTruncActions = getActionDefinitionsBuilder(G_FPTRUNC);
  if (ST.hasCvtPkF16F32Inst()) {
    FPTruncActions.legalFor({{S32, S64}, {S16, S32}, {V2S16, V2S32}})
        .clampMaxNumElements(0, S16, 2);
  } else {
    FPTruncActions.legalFor({{S32, S64}, {S16, S32}});
  }
  FPTruncActions.scalarize(0).lower();

  getActionDefinitionsBuilder(G_FPEXT)
      .legalFor({{S64, S32}, {S32, S16}})
      .narrowScalarFor({{S64, S16}}, changeElementSizeTo(0, S32))
      .scalarize(0);

  auto &FSubActions = getActionDefinitionsBuilder({G_FSUB, G_STRICT_FSUB});
  if (ST.has16BitInsts()) {
    FSubActions
      // Use actual fsub instruction
      .legalFor({S32, S16})
      // Must use fadd + fneg
      .lowerFor({S64, V2S16});
  } else {
    FSubActions
      // Use actual fsub instruction
      .legalFor({S32})
      // Must use fadd + fneg
      .lowerFor({S64, S16, V2S16});
  }

  if (ST.hasPackedFP32Ops())
    FSubActions.lowerFor({V2S32}).clampMaxNumElements(0, S32, 2);

  FSubActions
    .clampMaxNumElements(0, S16, 2)
    .scalarize(0)
    .clampScalar(0, S32, S64);

  // Whether this is legal depends on the floating point mode for the function.
  auto &FMad = getActionDefinitionsBuilder(G_FMAD);
  if (ST.hasMadF16() && ST.hasMadMacF32Insts())
    FMad.customFor({S32, S16});
  else if (ST.hasMadMacF32Insts())
    FMad.customFor({S32});
  else if (ST.hasMadF16())
    FMad.customFor({S16});
  FMad.scalarize(0)
      .lower();

  auto &FRem = getActionDefinitionsBuilder(G_FREM);
  if (ST.has16BitInsts()) {
    FRem.customFor({S16, S32, S64});
  } else {
    FRem.minScalar(0, S32)
        .customFor({S32, S64});
  }
  FRem.scalarize(0);

  // TODO: Do we need to clamp maximum bitwidth?
  getActionDefinitionsBuilder(G_TRUNC)
    .legalIf(isScalar(0))
    .legalFor({{V2S16, V2S32}})
    .clampMaxNumElements(0, S16, 2)
    // Avoid scalarizing in cases that should be truly illegal. In unresolvable
    // situations (like an invalid implicit use), we don't want to infinite loop
    // in the legalizer.
    .fewerElementsIf(elementTypeIsLegal(0), LegalizeMutations::scalarize(0))
    .alwaysLegal();

  getActionDefinitionsBuilder({G_SEXT, G_ZEXT, G_ANYEXT})
    .legalFor({{S64, S32}, {S32, S16}, {S64, S16},
               {S32, S1}, {S64, S1}, {S16, S1}})
    .scalarize(0)
    .clampScalar(0, S32, S64)
    .widenScalarToNextPow2(1, 32);

  // TODO: Split s1->s64 during regbankselect for VALU.
  auto &IToFP = getActionDefinitionsBuilder({G_SITOFP, G_UITOFP})
                    .legalFor({{S32, S32}, {S64, S32}})
                    .widenScalarFor({{S16, S32}}, changeElementSizeTo(0, S32))
                    .lowerIf(typeIs(1, S1))
                    .customFor({{S32, S64}, {S64, S64}});
  if (ST.has16BitInsts())
    IToFP.legalFor({{S16, S16}});
  IToFP.clampScalar(1, S32, S64)
       .minScalar(0, S32)
       .scalarize(0)
       .widenScalarToNextPow2(1);

  auto &FPToI = getActionDefinitionsBuilder({G_FPTOSI, G_FPTOUI})
                    .legalFor({{S32, S32}, {S32, S64}})
                    .customFor({{S64, S32}, {S64, S64}})
                    .widenScalarFor({{S32, S16}}, changeElementSizeTo(1, S32))
                    .narrowScalarFor({{S64, S16}}, changeElementSizeTo(0, S32));
  if (ST.has16BitInsts())
    FPToI.legalFor({{S16, S16}});
  else
    FPToI.minScalar(1, S32);

  FPToI.minScalar(0, S32)
       .widenScalarToNextPow2(0, 32)
       .scalarize(0)
       .lower();

  // clang-format off
  auto &FPToISat = getActionDefinitionsBuilder({G_FPTOSI_SAT, G_FPTOUI_SAT})
    .legalFor({{S32, S32}, {S32, S64}, {S16, S32}})
    .legalFor(ST.has16BitInsts(), {{S16, S16}})
    .legalFor(ST.hasVCvtPkIU16F32(), {{V2S16, V2S32}})
    .narrowScalarFor({{S64, S16}}, changeElementSizeTo(0, S32));

  // If available, widen width <16 to i16, intead of i32 so v_cvt_i16/u16_f16 can be used.
  if (ST.has16BitInsts())
    FPToISat.minScalarIf(typeIs(1, S16), 0, S16);

  if (ST.hasVCvtPkIU16F32())
    FPToISat.clampMaxNumElements(0, S16, 2);

  FPToISat.minScalar(1, S32);
  FPToISat.minScalar(0, S32)
       .widenScalarToNextPow2(0, 32)
       .scalarize(0)
       .lower();
  // clang-format on

  getActionDefinitionsBuilder({G_LROUND, G_LLROUND})
      .clampScalar(0, S16, S64)
      .scalarize(0)
      .lower();

  getActionDefinitionsBuilder(G_INTRINSIC_FPTRUNC_ROUND)
      .legalFor({S16, S32})
      .scalarize(0)
      .lower();

  // Lower G_FNEARBYINT and G_FRINT into G_INTRINSIC_ROUNDEVEN
  getActionDefinitionsBuilder({G_INTRINSIC_ROUND, G_FRINT, G_FNEARBYINT})
      .scalarize(0)
      .lower();

  getActionDefinitionsBuilder({G_INTRINSIC_LRINT, G_INTRINSIC_LLRINT})
      .clampScalar(0, S16, S64)
      .scalarize(0)
      .lower();

  if (ST.has16BitInsts()) {
    getActionDefinitionsBuilder(
        {G_INTRINSIC_TRUNC, G_FCEIL, G_INTRINSIC_ROUNDEVEN})
        .legalFor({S16, S32, S64})
        .clampScalar(0, S16, S64)
        .scalarize(0);
  } else if (ST.getGeneration() >= AMDGPUSubtarget::SEA_ISLANDS) {
    getActionDefinitionsBuilder(
        {G_INTRINSIC_TRUNC, G_FCEIL, G_INTRINSIC_ROUNDEVEN})
        .legalFor({S32, S64})
        .clampScalar(0, S32, S64)
        .scalarize(0);
  } else {
    getActionDefinitionsBuilder(
        {G_INTRINSIC_TRUNC, G_FCEIL, G_INTRINSIC_ROUNDEVEN})
        .legalFor({S32})
        .customFor({S64})
        .clampScalar(0, S32, S64)
        .scalarize(0);
  }

  getActionDefinitionsBuilder(G_PTR_ADD)
      .unsupportedFor({BufferFatPtr, BufferStridedPtr, RsrcPtr})
      .legalIf(all(isPointer(0), sameSize(0, 1)))
      .scalarize(0)
      .scalarSameSizeAs(1, 0);

  getActionDefinitionsBuilder(G_PTRMASK)
    .legalIf(all(sameSize(0, 1), typeInSet(1, {S64, S32})))
    .scalarSameSizeAs(1, 0)
    .scalarize(0);

  auto &CmpBuilder =
    getActionDefinitionsBuilder(G_ICMP)
    // The compare output type differs based on the register bank of the output,
    // so make both s1 and s32 legal.
    //
    // Scalar compares producing output in scc will be promoted to s32, as that
    // is the allocatable register type that will be needed for the copy from
    // scc. This will be promoted during RegBankSelect, and we assume something
    // before that won't try to use s32 result types.
    //
    // Vector compares producing an output in vcc/SGPR will use s1 in VCC reg
    // bank.
    .legalForCartesianProduct(
      {S1}, {S32, S64, GlobalPtr, LocalPtr, ConstantPtr, PrivatePtr, FlatPtr})
    .legalForCartesianProduct(
      {S32}, {S32, S64, GlobalPtr, LocalPtr, ConstantPtr, PrivatePtr, FlatPtr});
  if (ST.has16BitInsts()) {
    CmpBuilder.legalFor({{S1, S16}});
  }

  CmpBuilder
    .widenScalarToNextPow2(1)
    .clampScalar(1, S32, S64)
    .scalarize(0)
    .legalIf(all(typeInSet(0, {S1, S32}), isPointer(1)));

  auto &FCmpBuilder =
      getActionDefinitionsBuilder(G_FCMP).legalForCartesianProduct(
          {S1}, ST.has16BitInsts() ? FPTypes16 : FPTypesBase);

  if (ST.hasSALUFloatInsts())
    FCmpBuilder.legalForCartesianProduct({S32}, {S16, S32});

  FCmpBuilder
    .widenScalarToNextPow2(1)
    .clampScalar(1, S32, S64)
    .scalarize(0);

  // FIXME: fpow has a selection pattern that should move to custom lowering.
  auto &ExpOps = getActionDefinitionsBuilder(G_FPOW);
  if (ST.has16BitInsts())
    ExpOps.customFor({{S32}, {S16}});
  else
    ExpOps.customFor({S32});
  ExpOps.clampScalar(0, MinScalarFPTy, S32)
        .scalarize(0);

  getActionDefinitionsBuilder(G_FPOWI)
    .clampScalar(0, MinScalarFPTy, S32)
    .lower();

  getActionDefinitionsBuilder(G_FLOG2)
      .legalFor(ST.has16BitInsts(), {S16})
      .customFor({S32, S16})
      .scalarize(0)
      .lower();

  getActionDefinitionsBuilder(G_FEXP2)
      .legalFor(ST.has16BitInsts(), {S16})
      .customFor({S32, S64, S16})
      .scalarize(0)
      .lower();

  auto &LogOps =
      getActionDefinitionsBuilder({G_FLOG, G_FLOG10, G_FEXP, G_FEXP10});
  LogOps.customFor({S32, S16, S64});
  LogOps.clampScalar(0, MinScalarFPTy, S32)
        .scalarize(0);

  // The 64-bit versions produce 32-bit results, but only on the SALU.
  getActionDefinitionsBuilder(G_CTPOP)
    .legalFor({{S32, S32}, {S32, S64}})
    .clampScalar(0, S32, S32)
    .widenScalarToNextPow2(1, 32)
    .clampScalar(1, S32, S64)
    .scalarize(0)
    .widenScalarToNextPow2(0, 32);

  // If no 16 bit instr is available, lower into different instructions.
  if (ST.has16BitInsts())
    getActionDefinitionsBuilder(G_IS_FPCLASS)
        .legalForCartesianProduct({S1}, FPTypes16)
        .widenScalarToNextPow2(1)
        .scalarize(0)
        .lower();
  else
    getActionDefinitionsBuilder(G_IS_FPCLASS)
        .legalForCartesianProduct({S1}, FPTypesBase)
        .lowerFor({S1, S16})
        .widenScalarToNextPow2(1)
        .scalarize(0)
        .lower();

  // The hardware instructions return a different result on 0 than the generic
  // instructions expect. The hardware produces -1, but these produce the
  // bitwidth.
  getActionDefinitionsBuilder({G_CTLZ, G_CTTZ})
    .scalarize(0)
    .clampScalar(0, S32, S32)
    .clampScalar(1, S32, S64)
    .widenScalarToNextPow2(0, 32)
    .widenScalarToNextPow2(1, 32)
    .custom();

  // The 64-bit versions produce 32-bit results, but only on the SALU.
  getActionDefinitionsBuilder(G_CTLZ_ZERO_POISON)
      .legalFor({{S32, S32}, {S32, S64}})
      .customIf(scalarNarrowerThan(1, 32))
      .clampScalar(0, S32, S32)
      .clampScalar(1, S32, S64)
      .scalarize(0)
      .widenScalarToNextPow2(0, 32)
      .widenScalarToNextPow2(1, 32);

  getActionDefinitionsBuilder(G_CTTZ_ZERO_POISON)
      .legalFor({{S32, S32}, {S32, S64}})
      .clampScalar(0, S32, S32)
      .clampScalar(1, S32, S64)
      .scalarize(0)
      .widenScalarToNextPow2(0, 32)
      .widenScalarToNextPow2(1, 32);

  getActionDefinitionsBuilder(G_CTLS)
      .customFor({{S32, S32}})
      .scalarize(0)
      .clampScalar(0, S32, S32)
      .clampScalar(1, S32, S32);

  // S64 is only legal on SALU, and needs to be broken into 32-bit elements in
  // RegBankSelect.
  getActionDefinitionsBuilder(G_BITREVERSE)
    .legalFor({S32, S64})
    .clampScalar(0, S32, S64)
    .scalarize(0)
    .widenScalarToNextPow2(0);

  if (ST.has16BitInsts()) {
    getActionDefinitionsBuilder(G_BSWAP)
      .legalFor({S16, S32, V2S16})
      .clampMaxNumElementsStrict(0, S16, 2)
      // FIXME: Fixing non-power-of-2 before clamp is workaround for
      // narrowScalar limitation.
      .widenScalarToNextPow2(0)
      .clampScalar(0, S16, S32)
      .scalarize(0);

    if (ST.hasVOP3PInsts()) {
      getActionDefinitionsBuilder(G_ABS)
          .legalFor({S32, S16, V2S16})
          .clampMaxNumElements(0, S16, 2)
          .minScalar(0, S16)
          .widenScalarToNextPow2(0)
          .scalarize(0)
          .lower();
      if (ST.hasMinMaxI64Insts()) {
        getActionDefinitionsBuilder({G_SMIN, G_SMAX, G_UMIN, G_UMAX})
            .legalFor({S32, S16, S64, V2S16})
            .clampMaxNumElements(0, S16, 2)
            .minScalar(0, S16)
            .widenScalarToNextPow2(0)
            .scalarize(0)
            .lower();
      } else {
        getActionDefinitionsBuilder({G_SMIN, G_SMAX, G_UMIN, G_UMAX})
            .legalFor({S32, S16, V2S16})
            .clampMaxNumElements(0, S16, 2)
            .minScalar(0, S16)
            .widenScalarToNextPow2(0)
            .scalarize(0)
            .lower();
      }
    } else {
      getActionDefinitionsBuilder({G_SMIN, G_SMAX, G_UMIN, G_UMAX, G_ABS})
        .legalFor({S32, S16})
        .widenScalarToNextPow2(0)
        .minScalar(0, S16)
        .scalarize(0)
        .lower();
    }
  } else {
    // TODO: Should have same legality without v_perm_b32
    getActionDefinitionsBuilder(G_BSWAP)
      .legalFor({S32})
      .lowerIf(scalarNarrowerThan(0, 32))
      // FIXME: Fixing non-power-of-2 before clamp is workaround for
      // narrowScalar limitation.
      .widenScalarToNextPow2(0)
      .maxScalar(0, S32)
      .scalarize(0)
      .lower();

    getActionDefinitionsBuilder({G_SMIN, G_SMAX, G_UMIN, G_UMAX, G_ABS})
      .legalFor({S32})
      .minScalar(0, S32)
      .widenScalarToNextPow2(0)
      .scalarize(0)
      .lower();
  }

  getActionDefinitionsBuilder(G_INTTOPTR)
      // List the common cases
      .legalForCartesianProduct(AddrSpaces64, {S64})
      .legalForCartesianProduct(AddrSpaces32, {S32})
      .scalarize(0)
      // Accept any address space as long as the size matches
      .legalIf(sameSize(0, 1))
      .widenScalarIf(smallerThan(1, 0),
                     [](const LegalityQuery &Query) {
                       return std::pair(
                           1, LLT::scalar(Query.Types[0].getSizeInBits()));
                     })
      .narrowScalarIf(largerThan(1, 0), [](const LegalityQuery &Query) {
        return std::pair(1, LLT::scalar(Query.Types[0].getSizeInBits()));
      });

  getActionDefinitionsBuilder(G_PTRTOINT)
      // List the common cases
      .legalForCartesianProduct(AddrSpaces64, {S64})
      .legalForCartesianProduct(AddrSpaces32, {S32})
      .scalarize(0)
      // Accept any address space as long as the size matches
      .legalIf(sameSize(0, 1))
      .widenScalarIf(smallerThan(0, 1),
                     [](const LegalityQuery &Query) {
                       return std::pair(
                           0, LLT::scalar(Query.Types[1].getSizeInBits()));
                     })
      .narrowScalarIf(largerThan(0, 1), [](const LegalityQuery &Query) {
        return std::pair(0, LLT::scalar(Query.Types[1].getSizeInBits()));
      });

  getActionDefinitionsBuilder(G_ADDRSPACE_CAST)
    .scalarize(0)
    .custom();

  const auto needToSplitMemOp = [=](const LegalityQuery &Query,
                                    bool IsLoad) -> bool {
    const LLT DstTy = Query.Types[0];

    // Split vector extloads.
    unsigned MemSize = Query.MMODescrs[0].MemoryTy.getSizeInBits();

    if (DstTy.isVector() && DstTy.getSizeInBits() > MemSize)
      return true;

    const LLT PtrTy = Query.Types[1];
    unsigned AS = PtrTy.getAddressSpace();
    if (MemSize > maxSizeForAddrSpace(ST, AS, IsLoad,
                                      Query.MMODescrs[0].Ordering !=
                                          AtomicOrdering::NotAtomic))
      return true;

    // Catch weird sized loads that don't evenly divide into the access sizes
    // TODO: May be able to widen depending on alignment etc.
    unsigned NumRegs = (MemSize + 31) / 32;
    if (NumRegs == 3) {
      if (!ST.hasDwordx3LoadStores())
        return true;
    } else {
      // If the alignment allows, these should have been widened.
      if (!isPowerOf2_32(NumRegs))
        return true;
    }

    return false;
  };

  unsigned GlobalAlign32 = ST.hasUnalignedBufferAccessEnabled() ? 0 : 32;
  unsigned GlobalAlign16 = ST.hasUnalignedBufferAccessEnabled() ? 0 : 16;
  unsigned GlobalAlign8 = ST.hasUnalignedBufferAccessEnabled() ? 0 : 8;

  // TODO: Refine based on subtargets which support unaligned access or 128-bit
  // LDS
  // TODO: Unsupported flat for SI.

  for (unsigned Op : {G_LOAD, G_STORE}) {
    const bool IsStore = Op == G_STORE;

    auto &Actions = getActionDefinitionsBuilder(Op);
    // Explicitly list some common cases.
    // TODO: Does this help compile time at all?
    Actions.legalForTypesWithMemDesc({{S32, GlobalPtr, S32, GlobalAlign32},
                                      {V2S32, GlobalPtr, V2S32, GlobalAlign32},
                                      {V4S32, GlobalPtr, V4S32, GlobalAlign32},
                                      {S64, GlobalPtr, S64, GlobalAlign32},
                                      {V2S64, GlobalPtr, V2S64, GlobalAlign32},
                                      {V2S16, GlobalPtr, V2S16, GlobalAlign32},
                                      {S32, GlobalPtr, S8, GlobalAlign8},
                                      {S32, GlobalPtr, S16, GlobalAlign16},

                                      {S32, LocalPtr, S32, 32},
                                      {S64, LocalPtr, S64, 32},
                                      {V2S32, LocalPtr, V2S32, 32},
                                      {S32, LocalPtr, S8, 8},
                                      {S32, LocalPtr, S16, 16},
                                      {V2S16, LocalPtr, S32, 32},

                                      {S32, PrivatePtr, S32, 32},
                                      {S32, PrivatePtr, S8, 8},
                                      {S32, PrivatePtr, S16, 16},
                                      {V2S16, PrivatePtr, S32, 32},

                                      {S32, ConstantPtr, S32, GlobalAlign32},
                                      {V2S32, ConstantPtr, V2S32, GlobalAlign32},
                                      {V4S32, ConstantPtr, V4S32, GlobalAlign32},
                                      {S64, ConstantPtr, S64, GlobalAlign32},
                                      {V2S32, ConstantPtr, V2S32, GlobalAlign32}});

    Actions.legalForTypesWithMemDesc(ST.useRealTrue16Insts(), /* Pred */
                                     {{S16, GlobalPtr, S8, GlobalAlign8},
                                      {S16, GlobalPtr, S16, GlobalAlign16},
                                      {S16, LocalPtr, S8, 8},
                                      {S16, LocalPtr, S16, 16},
                                      {S16, PrivatePtr, S8, 8},
                                      {S16, PrivatePtr, S16, 16}});

    Actions.legalIf(
      [=](const LegalityQuery &Query) -> bool {
        return isLoadStoreLegal(ST, Query);
      });

    // The custom pointers (fat pointers, buffer resources) don't work with load
    // and store at this level. Fat pointers should have been lowered to
    // intrinsics before the translation to MIR.
    Actions.unsupportedIf(
        typeInSet(1, {BufferFatPtr, BufferStridedPtr, RsrcPtr}));

    // Address space 8 pointers are handled by a 4xs32 load, bitcast, and
    // ptrtoint. This is needed to account for the fact that we can't have i128
    // as a register class for SelectionDAG reasons.
    Actions.customIf([=](const LegalityQuery &Query) -> bool {
      return hasBufferRsrcWorkaround(Query.Types[0]);
    });

    // Constant 32-bit is handled by addrspacecasting the 32-bit pointer to
    // 64-bits.
    //
    // TODO: Should generalize bitcast action into coerce, which will also cover
    // inserting addrspacecasts.
    Actions.customIf(typeIs(1, Constant32Ptr));

    // Turn any illegal element vectors into something easier to deal
    // with. These will ultimately produce 32-bit scalar shifts to extract the
    // parts anyway.
    //
    // For odd 16-bit element vectors, prefer to split those into pieces with
    // 16-bit vector parts.
    Actions.bitcastIf(
      [=](const LegalityQuery &Query) -> bool {
        return shouldBitcastLoadStoreType(ST, Query.Types[0],
                                          Query.MMODescrs[0].MemoryTy);
      }, bitcastToRegisterType(0));

    if (!IsStore) {
      // Widen suitably aligned loads by loading extra bytes. The standard
      // legalization actions can't properly express widening memory operands.
      Actions.customIf([=](const LegalityQuery &Query) -> bool {
        return shouldWidenLoad(ST, Query, G_LOAD);
      });
    }

    // FIXME: load/store narrowing should be moved to lower action
    Actions
        .narrowScalarIf(
            [=](const LegalityQuery &Query) -> bool {
              return !Query.Types[0].isVector() &&
                     needToSplitMemOp(Query, Op == G_LOAD);
            },
            [=](const LegalityQuery &Query) -> std::pair<unsigned, LLT> {
              const LLT DstTy = Query.Types[0];
              const LLT PtrTy = Query.Types[1];

              const unsigned DstSize = DstTy.getSizeInBits();
              unsigned MemSize = Query.MMODescrs[0].MemoryTy.getSizeInBits();

              // Split extloads.
              if (DstSize > MemSize)
                return std::pair(0, LLT::scalar(MemSize));

              unsigned MaxSize = maxSizeForAddrSpace(
                  ST, PtrTy.getAddressSpace(), Op == G_LOAD,
                  Query.MMODescrs[0].Ordering != AtomicOrdering::NotAtomic);
              if (MemSize > MaxSize)
                return std::pair(0, LLT::scalar(MaxSize));

              uint64_t Align = Query.MMODescrs[0].AlignInBits;
              return std::pair(0, LLT::scalar(Align));
            })
        .fewerElementsIf(
            [=](const LegalityQuery &Query) -> bool {
              return Query.Types[0].isVector() &&
                     needToSplitMemOp(Query, Op == G_LOAD);
            },
            [=](const LegalityQuery &Query) -> std::pair<unsigned, LLT> {
              const LLT DstTy = Query.Types[0];
              const LLT PtrTy = Query.Types[1];

              LLT EltTy = DstTy.getElementType();
              unsigned MaxSize = maxSizeForAddrSpace(
                  ST, PtrTy.getAddressSpace(), Op == G_LOAD,
                  Query.MMODescrs[0].Ordering != AtomicOrdering::NotAtomic);

              // FIXME: Handle widened to power of 2 results better. This ends
              // up scalarizing.
              // FIXME: 3 element stores scalarized on SI

              // Split if it's too large for the address space.
              unsigned MemSize = Query.MMODescrs[0].MemoryTy.getSizeInBits();
              if (MemSize > MaxSize) {
                unsigned NumElts = DstTy.getNumElements();
                unsigned EltSize = EltTy.getSizeInBits();

                if (MaxSize % EltSize == 0) {
                  return std::pair(
                      0, LLT::scalarOrVector(
                             ElementCount::getFixed(MaxSize / EltSize), EltTy));
                }

                unsigned NumPieces = MemSize / MaxSize;

                // FIXME: Refine when odd breakdowns handled
                // The scalars will need to be re-legalized.
                if (NumPieces == 1 || NumPieces >= NumElts ||
                    NumElts % NumPieces != 0)
                  return std::pair(0, EltTy);

                return std::pair(0,
                                 LLT::fixed_vector(NumElts / NumPieces, EltTy));
              }

              // FIXME: We could probably handle weird extending loads better.
              if (DstTy.getSizeInBits() > MemSize)
                return std::pair(0, EltTy);

              unsigned EltSize = EltTy.getSizeInBits();
              unsigned DstSize = DstTy.getSizeInBits();
              if (!isPowerOf2_32(DstSize)) {
                // We're probably decomposing an odd sized store. Try to split
                // to the widest type. TODO: Account for alignment. As-is it
                // should be OK, since the new parts will be further legalized.
                unsigned FloorSize = llvm::bit_floor(DstSize);
                return std::pair(
                    0, LLT::scalarOrVector(
                           ElementCount::getFixed(FloorSize / EltSize), EltTy));
              }

              // May need relegalization for the scalars.
              return std::pair(0, EltTy);
            })
        .widenScalarIf(scalarNarrowerThan(0, 32), changeTo(0, LLT::integer(32)))
        .narrowScalarIf(isTruncStoreToSizePowerOf2(0),
                        getScalarTypeFromMemDesc(0))
        .widenScalarToNextPow2(0)
        .moreElementsIf(vectorSmallerThan(0, 32), moreEltsToNext32Bit(0))
        .lower();
  }

  // FIXME: Unaligned accesses not lowered.
  auto &ExtLoads =
      getActionDefinitionsBuilder({G_SEXTLOAD, G_ZEXTLOAD})
          .legalForTypesWithMemDesc({{S32, GlobalPtr, S8, 8},
                                     {S32, GlobalPtr, S16, 2 * 8},
                                     {S32, LocalPtr, S8, 8},
                                     {S32, LocalPtr, S16, 16},
                                     {S32, PrivatePtr, S8, 8},
                                     {S32, PrivatePtr, S16, 16},
                                     {S32, ConstantPtr, S8, 8},
                                     {S32, ConstantPtr, S16, 2 * 8}})
          .legalForTypesWithMemDesc(ST.useRealTrue16Insts(),
                                    {{S16, GlobalPtr, S8, GlobalAlign8},
                                     {S16, LocalPtr, S8, GlobalAlign8},
                                     {S16, PrivatePtr, S8, GlobalAlign8},
                                     {S16, ConstantPtr, S8, GlobalAlign8}})
          .legalIf([=](const LegalityQuery &Query) -> bool {
            return isLoadStoreLegal(ST, Query);
          });

  if (ST.hasFlatAddressSpace()) {
    ExtLoads.legalForTypesWithMemDesc(
        {{S32, FlatPtr, S8, 8}, {S32, FlatPtr, S16, 16}});

    ExtLoads.legalForTypesWithMemDesc(ST.useRealTrue16Insts(),
                                      {{S16, FlatPtr, S8, GlobalAlign8}});
  }

  // Constant 32-bit is handled by addrspacecasting the 32-bit pointer to
  // 64-bits.
  //
  // TODO: Should generalize bitcast action into coerce, which will also cover
  // inserting addrspacecasts.
  ExtLoads.customIf(typeIs(1, Constant32Ptr));

  ExtLoads.narrowScalarIf(
      [](const LegalityQuery &Query) {
        LLT MemTy = Query.MMODescrs[0].MemoryTy;
        return MemTy.isScalar() && MemTy.getSizeInBits() > 32 &&
               Query.Types[0].getSizeInBits() > MemTy.getSizeInBits();
      }, // For large MemSize, narrowscalar to MemSize (load MemSize + ext)
      getScalarTypeFromMemDesc(0));
  ExtLoads.clampScalar(0, S32, S32)
          .widenScalarToNextPow2(0)
          .lower();

  auto &Atomics = getActionDefinitionsBuilder(
    {G_ATOMICRMW_XCHG, G_ATOMICRMW_ADD, G_ATOMICRMW_SUB,
     G_ATOMICRMW_AND, G_ATOMICRMW_OR, G_ATOMICRMW_XOR,
     G_ATOMICRMW_MAX, G_ATOMICRMW_MIN, G_ATOMICRMW_UMAX,
     G_ATOMICRMW_UMIN, G_ATOMICRMW_UINC_WRAP, G_ATOMICRMW_UDEC_WRAP})
    .legalFor({{S32, GlobalPtr}, {S32, LocalPtr},
               {S64, GlobalPtr}, {S64, LocalPtr},
               {S32, RegionPtr}, {S64, RegionPtr}});
  if (ST.hasFlatAddressSpace()) {
    Atomics.legalFor({{S32, FlatPtr}, {S64, FlatPtr}});
  }

  auto &Atomics32 =
      getActionDefinitionsBuilder({G_ATOMICRMW_USUB_COND, G_ATOMICRMW_USUB_SAT})
          .legalFor({{S32, GlobalPtr}, {S32, LocalPtr}, {S32, RegionPtr}});
  if (ST.hasFlatAddressSpace()) {
    Atomics32.legalFor({{S32, FlatPtr}});
  }

  // TODO: v2bf16 operations, and fat buffer pointer support.
  auto &Atomic = getActionDefinitionsBuilder(G_ATOMICRMW_FADD);
  if (ST.hasLDSFPAtomicAddF32()) {
    Atomic.legalFor({{S32, LocalPtr}, {S32, RegionPtr}});
    if (ST.hasLdsAtomicAddF64())
      Atomic.legalFor({{S64, LocalPtr}});
    if (ST.hasAtomicDsPkAdd16Insts())
      Atomic.legalFor({{V2F16, LocalPtr}, {V2BF16, LocalPtr}});
  }
  if (ST.hasAtomicFaddInsts())
    Atomic.legalFor({{S32, GlobalPtr}});
  if (ST.hasFlatAtomicFaddF32Inst())
    Atomic.legalFor({{S32, FlatPtr}});

  if (ST.hasGFX90AInsts() || ST.hasGFX1250Insts()) {
    // These are legal with some caveats, and should have undergone expansion in
    // the IR in most situations
    // TODO: Move atomic expansion into legalizer
    Atomic.legalFor({
        {S32, GlobalPtr},
        {S64, GlobalPtr},
        {S64, FlatPtr}
      });
  }

  if (ST.hasAtomicBufferGlobalPkAddF16NoRtnInsts() ||
      ST.hasAtomicBufferGlobalPkAddF16Insts())
    Atomic.legalFor({{V2F16, GlobalPtr}, {V2F16, BufferFatPtr}});
  if (ST.hasAtomicGlobalPkAddBF16Inst())
    Atomic.legalFor({{V2BF16, GlobalPtr}});
  if (ST.hasAtomicFlatPkAdd16Insts())
    Atomic.legalFor({{V2F16, FlatPtr}, {V2BF16, FlatPtr}});


  // Most of the legalization work here is done by AtomicExpand. We could
  // probably use a simpler legality rule that just assumes anything is OK.
  auto &AtomicFMinFMax =
    getActionDefinitionsBuilder({G_ATOMICRMW_FMIN, G_ATOMICRMW_FMAX})
    .legalFor({{F32, LocalPtr}, {F64, LocalPtr}});

  if (ST.hasAtomicFMinFMaxF32GlobalInsts())
    AtomicFMinFMax.legalFor({{F32, GlobalPtr},{F32, BufferFatPtr}});
  if (ST.hasAtomicFMinFMaxF64GlobalInsts())
    AtomicFMinFMax.legalFor({{F64, GlobalPtr}, {F64, BufferFatPtr}});
  if (ST.hasAtomicFMinFMaxF32FlatInsts())
    AtomicFMinFMax.legalFor({F32, FlatPtr});
  if (ST.hasAtomicFMinFMaxF64FlatInsts())
    AtomicFMinFMax.legalFor({F64, FlatPtr});

  // BUFFER/FLAT_ATOMIC_CMP_SWAP on GCN GPUs needs input marshalling, and output
  // demarshalling
  getActionDefinitionsBuilder(G_ATOMIC_CMPXCHG)
    .customFor({{S32, GlobalPtr}, {S64, GlobalPtr},
                {S32, FlatPtr}, {S64, FlatPtr}})
    .legalFor({{S32, LocalPtr}, {S64, LocalPtr},
               {S32, RegionPtr}, {S64, RegionPtr}});
  // TODO: Pointer types, any 32-bit or 64-bit vector

  // Condition should be s32 for scalar, s1 for vector.
  getActionDefinitionsBuilder(G_SELECT)
      .legalForCartesianProduct({S32, S64, S16, V2S32, V2S16, V4S16, GlobalPtr,
                                 LocalPtr, FlatPtr, PrivatePtr,
                                 LLT::fixed_vector(2, LocalPtr),
                                 LLT::fixed_vector(2, PrivatePtr)},
                                {S1, S32})
      .clampScalar(0, S16, S64)
      .scalarize(1)
      .moreElementsIf(isSmallOddVector(0), oneMoreElement(0))
      .fewerElementsIf(numElementsNotEven(0), scalarize(0))
      .clampMaxNumElements(0, S32, 2)
      .clampMaxNumElements(0, LocalPtr, 2)
      .clampMaxNumElements(0, PrivatePtr, 2)
      .scalarize(0)
      .widenScalarToNextPow2(0)
      .legalIf(all(isPointer(0), typeInSet(1, {S1, S32})));

  // TODO: Only the low 4/5/6 bits of the shift amount are observed, so we can
  // be more flexible with the shift amount type.
  auto &Shifts = getActionDefinitionsBuilder({G_SHL, G_LSHR, G_ASHR})
    .legalFor({{S32, S32}, {S64, S32}});
  if (ST.has16BitInsts()) {
    if (ST.hasVOP3PInsts()) {
      Shifts.legalFor({{S16, S16}, {V2S16, V2S16}})
            .clampMaxNumElements(0, S16, 2);
    } else
      Shifts.legalFor({{S16, S16}});

    // TODO: Support 16-bit shift amounts for all types
    Shifts.widenScalarIf(
        [=](const LegalityQuery &Query) {
          // Use 16-bit shift amounts for any 16-bit shift. Otherwise we want a
          // 32-bit amount.
          const LLT ValTy = Query.Types[0];
          const LLT AmountTy = Query.Types[1];
          return ValTy.isScalar() && ValTy.getSizeInBits() <= 16 &&
                 AmountTy.getSizeInBits() < 16;
        },
        changeElementSizeTo(1, S16));
    Shifts.maxScalarIf(typeIs(0, S16), 1, S16);
    Shifts.clampScalar(1, S32, S32);
    Shifts.widenScalarToNextPow2(0, 16);
    Shifts.clampScalar(0, S16, S64);

    getActionDefinitionsBuilder({G_SSHLSAT, G_USHLSAT})
      .minScalar(0, S16)
      .scalarize(0)
      .lower();
  } else {
    // Make sure we legalize the shift amount type first, as the general
    // expansion for the shifted type will produce much worse code if it hasn't
    // been truncated already.
    Shifts.clampScalar(1, S32, S32);
    Shifts.widenScalarToNextPow2(0, 32);
    Shifts.clampScalar(0, S32, S64);

    getActionDefinitionsBuilder({G_SSHLSAT, G_USHLSAT})
      .minScalar(0, S32)
      .scalarize(0)
      .lower();
  }
  Shifts.scalarize(0);

  for (unsigned Op : {G_EXTRACT_VECTOR_ELT, G_INSERT_VECTOR_ELT}) {
    unsigned VecTypeIdx = Op == G_EXTRACT_VECTOR_ELT ? 1 : 0;
    unsigned EltTypeIdx = Op == G_EXTRACT_VECTOR_ELT ? 0 : 1;
    unsigned IdxTypeIdx = 2;

    getActionDefinitionsBuilder(Op)
        .customIf([=](const LegalityQuery &Query) {
          const LLT EltTy = Query.Types[EltTypeIdx];
          const LLT VecTy = Query.Types[VecTypeIdx];
          const LLT IdxTy = Query.Types[IdxTypeIdx];
          const unsigned EltSize = EltTy.getSizeInBits();
          const bool isLegalVecType =
              !!SIRegisterInfo::getSGPRClassForBitWidth(VecTy.getSizeInBits());
          // Address space 8 pointers are 128-bit wide values, but the logic
          // below will try to bitcast them to 2N x s64, which will fail.
          // Therefore, as an intermediate step, wrap extracts/insertions from a
          // ptrtoint-ing the vector and scalar arguments (or inttoptring the
          // extraction result) in order to produce a vector operation that can
          // be handled by the logic below.
          if (EltTy.isPointer() && EltSize > 64)
            return true;
          return (EltSize == 32 || EltSize == 64) &&
                  VecTy.getSizeInBits() % 32 == 0 &&
                  VecTy.getSizeInBits() <= MaxRegisterSize &&
                  IdxTy.getSizeInBits() == 32 &&
                  isLegalVecType;
        })
        .bitcastIf(all(sizeIsMultipleOf32(VecTypeIdx),
                       scalarOrEltNarrowerThan(VecTypeIdx, 32)),
                   bitcastToVectorElement32(VecTypeIdx))
        //.bitcastIf(vectorSmallerThan(1, 32), bitcastToScalar(1))
        .bitcastIf(all(sizeIsMultipleOf32(VecTypeIdx),
                       scalarOrEltWiderThan(VecTypeIdx, 64)),
                   [=](const LegalityQuery &Query) {
                     // For > 64-bit element types, try to turn this into a
                     // 64-bit element vector since we may be able to do better
                     // indexing if this is scalar. If not, fall back to 32.
                     const LLT EltTy = Query.Types[EltTypeIdx];
                     const LLT VecTy = Query.Types[VecTypeIdx];
                     const unsigned DstEltSize = EltTy.getSizeInBits();
                     const unsigned VecSize = VecTy.getSizeInBits();

                     const unsigned TargetEltSize =
                         DstEltSize % 64 == 0 ? 64 : 32;
                     return std::pair(VecTypeIdx,
                                      LLT::fixed_vector(VecSize / TargetEltSize,
                                                        TargetEltSize));
                   })
        .clampScalar(EltTypeIdx, S32, S64)
        .clampScalar(VecTypeIdx, S32, S64)
        .clampScalar(IdxTypeIdx, S32, S32)
        .clampMaxNumElements(VecTypeIdx, S32, 32)
        // TODO: Clamp elements for 64-bit vectors?
        .moreElementsIf(isIllegalRegisterType(ST, VecTypeIdx),
                        moreElementsToNextExistingRegClass(VecTypeIdx))
        // It should only be necessary with variable indexes.
        // As a last resort, lower to the stack
        .lower();
  }

  getActionDefinitionsBuilder(G_EXTRACT_VECTOR_ELT)
    .unsupportedIf([=](const LegalityQuery &Query) {
        const LLT &EltTy = Query.Types[1].getElementType();
        return Query.Types[0] != EltTy;
      });

  for (unsigned Op : {G_EXTRACT, G_INSERT}) {
    unsigned BigTyIdx = Op == G_EXTRACT ? 1 : 0;
    unsigned LitTyIdx = Op == G_EXTRACT ? 0 : 1;
    getActionDefinitionsBuilder(Op)
        .widenScalarIf(
            [=](const LegalityQuery &Query) {
              const LLT BigTy = Query.Types[BigTyIdx];
              return (BigTy.getScalarSizeInBits() < 16);
            },
            LegalizeMutations::widenScalarOrEltToNextPow2(BigTyIdx, 16))
        .widenScalarIf(
            [=](const LegalityQuery &Query) {
              const LLT LitTy = Query.Types[LitTyIdx];
              return (LitTy.getScalarSizeInBits() < 16);
            },
            LegalizeMutations::widenScalarOrEltToNextPow2(LitTyIdx, 16))
        .moreElementsIf(isSmallOddVector(BigTyIdx), oneMoreElement(BigTyIdx))
        .widenScalarToNextPow2(BigTyIdx, 32)
        .customIf([=](const LegalityQuery &Query) {
          // Generic lower operates on the full-width value, producing
          // shift+trunc/mask sequences. For simple cases where extract/insert
          // values are 32-bit aligned, we can instead unmerge/merge and work on
          // the 32-bit components. However, we can't check the offset here so
          // custom lower function will have to call generic lowering if offset
          // is not 32-bit aligned.
          const LLT BigTy = Query.Types[BigTyIdx];
          const LLT LitTy = Query.Types[LitTyIdx];
          return !BigTy.isVector() && BigTy.getSizeInBits() % 32 == 0 &&
                 LitTy.getSizeInBits() % 32 == 0;
        })
        .lower();
  }

  auto &BuildVector =
      getActionDefinitionsBuilder(G_BUILD_VECTOR)
          .legalForCartesianProduct(AllS32Vectors, {S32})
          .legalForCartesianProduct(AllS64Vectors, {S64})
          .clampNumElements(0, V16S32, V32S32)
          .clampNumElements(0, V2S64, V16S64)
          .fewerElementsIf(isWideVec16(0),
                           changeElementCountTo(0, ElementCount::getFixed(2)))
          .moreElementsIf(isIllegalRegisterType(ST, 0),
                          moreElementsToNextExistingRegClass(0));

  if (ST.hasScalarPackInsts()) {
    BuildVector
      // FIXME: Should probably widen s1 vectors straight to s32
      .minScalarOrElt(0, S16)
      .minScalar(1, S16);

    getActionDefinitionsBuilder(G_BUILD_VECTOR_TRUNC)
      .legalFor({V2S16, S32})
      .lower();
  } else {
    BuildVector.customFor({V2S16, S16});
    BuildVector.minScalarOrElt(0, S32);

    getActionDefinitionsBuilder(G_BUILD_VECTOR_TRUNC)
      .customFor({V2S16, S32})
      .lower();
  }

  BuildVector.legalIf(isRegisterType(ST, 0));

  // FIXME: Clamp maximum size
  getActionDefinitionsBuilder(G_CONCAT_VECTORS)
      .legalIf(all(isRegisterType(ST, 0), isRegisterType(ST, 1)))
      .clampMaxNumElements(0, S32, 32)
      .clampMaxNumElements(1, S16, 2) // TODO: Make 4?
      .clampMaxNumElements(0, S16, 64);

  getActionDefinitionsBuilder(G_SHUFFLE_VECTOR).lower();

  // Merge/Unmerge
  for (unsigned Op : {G_MERGE_VALUES, G_UNMERGE_VALUES}) {
    unsigned BigTyIdx = Op == G_MERGE_VALUES ? 0 : 1;
    unsigned LitTyIdx = Op == G_MERGE_VALUES ? 1 : 0;

    auto notValidElt = [=](const LegalityQuery &Query, unsigned TypeIdx) {
      const LLT Ty = Query.Types[TypeIdx];
      if (Ty.isVector()) {
        const LLT &EltTy = Ty.getElementType();
        if (EltTy.getSizeInBits() < 8 || EltTy.getSizeInBits() > 512)
          return true;
        if (!llvm::has_single_bit<uint32_t>(EltTy.getSizeInBits()))
          return true;
      }
      return false;
    };

    auto &Builder =
        getActionDefinitionsBuilder(Op)
            .legalIf(all(isRegisterType(ST, 0), isRegisterType(ST, 1)))
            .lowerFor({{S16, V2S16}})
            .lowerIf([=](const LegalityQuery &Query) {
              const LLT BigTy = Query.Types[BigTyIdx];
              return BigTy.getSizeInBits() == 32;
            })
            // Try to widen to s16 first for small types.
            // TODO: Only do this on targets with legal s16 shifts
            .minScalarOrEltIf(scalarNarrowerThan(LitTyIdx, 16), LitTyIdx, S16)
            .widenScalarToNextPow2(LitTyIdx, /*Min*/ 16)
            .moreElementsIf(isSmallOddVector(BigTyIdx),
                            oneMoreElement(BigTyIdx))
            .fewerElementsIf(all(typeIs(0, S16), vectorWiderThan(1, 32),
                                 elementTypeIs(1, S16)),
                             changeElementCountTo(1, ElementCount::getFixed(2)))
            // Clamp the little scalar to s8-s256 and make it a power of 2. It's
            // not worth considering the multiples of 64 since 2*192 and 2*384
            // are not valid.
            .clampScalar(LitTyIdx, S32, S512)
            .widenScalarToNextPow2(LitTyIdx, /*Min*/ 32)
            // Break up vectors with weird elements into scalars
            .fewerElementsIf(
                [=](const LegalityQuery &Query) {
                  return notValidElt(Query, LitTyIdx);
                },
                scalarize(0))
            .fewerElementsIf(
                [=](const LegalityQuery &Query) {
                  return notValidElt(Query, BigTyIdx);
                },
                scalarize(1))
            .clampScalar(BigTyIdx, S32, MaxScalar);

    if (Op == G_MERGE_VALUES) {
      Builder.widenScalarIf(
          // TODO: Use 16-bit shifts if legal for 8-bit values?
          [=](const LegalityQuery &Query) {
            const LLT Ty = Query.Types[LitTyIdx];
            return Ty.getSizeInBits() < 32;
          },
          changeElementSizeTo(LitTyIdx, S32));
    }

    Builder.widenScalarIf(
      [=](const LegalityQuery &Query) {
        const LLT Ty = Query.Types[BigTyIdx];
        return Ty.getSizeInBits() % 16 != 0;
      },
      [=](const LegalityQuery &Query) {
        // Pick the next power of 2, or a multiple of 64 over 128.
        // Whichever is smaller.
        const LLT &Ty = Query.Types[BigTyIdx];
        unsigned NewSizeInBits = 1 << Log2_32_Ceil(Ty.getSizeInBits() + 1);
        if (NewSizeInBits >= 256) {
          unsigned RoundedTo = alignTo<64>(Ty.getSizeInBits() + 1);
          if (RoundedTo < NewSizeInBits)
            NewSizeInBits = RoundedTo;
        }
        return std::pair(BigTyIdx, LLT::scalar(NewSizeInBits));
      })
      // Any vectors left are the wrong size. Scalarize them.
      .scalarize(0)
      .scalarize(1);
  }

  // S64 is only legal on SALU, and needs to be broken into 32-bit elements in
  // RegBankSelect.
  auto &SextInReg = getActionDefinitionsBuilder(G_SEXT_INREG)
                        .legalFor({{S32}, {S64}})
                        .clampScalar(0, S32, S64);

  if (ST.hasVOP3PInsts()) {
    SextInReg.lowerFor({{V2S16}})
      // Prefer to reduce vector widths for 16-bit vectors before lowering, to
      // get more vector shift opportunities, since we'll get those when
      // expanded.
      .clampMaxNumElementsStrict(0, S16, 2);
  } else if (ST.has16BitInsts()) {
    SextInReg.lowerFor({{S32}, {S64}, {S16}});
  } else {
    // Prefer to promote to s32 before lowering if we don't have 16-bit
    // shifts. This avoid a lot of intermediate truncate and extend operations.
    SextInReg.lowerFor({{S32}, {S64}});
  }

  SextInReg
    .scalarize(0)
    .clampScalar(0, S32, S64)
    .lower();

  getActionDefinitionsBuilder({G_ROTR, G_ROTL})
    .scalarize(0)
    .lower();

  auto &FSHRActionDefs = getActionDefinitionsBuilder(G_FSHR);
  FSHRActionDefs.legalFor({{S32, S32}})
                              .clampMaxNumElementsStrict(0, S16, 2);
  if (ST.hasVOP3PInsts())
    FSHRActionDefs.lowerFor({{V2S16, V2S16}});
  FSHRActionDefs.scalarize(0).lower();

  if (ST.hasVOP3PInsts()) {
    getActionDefinitionsBuilder(G_FSHL)
      .lowerFor({{V2S16, V2S16}})
      .clampMaxNumElementsStrict(0, S16, 2)
      .scalarize(0)
      .lower();
  } else {
    getActionDefinitionsBuilder(G_FSHL)
      .scalarize(0)
      .lower();
  }

  getActionDefinitionsBuilder(G_READCYCLECOUNTER)
    .legalFor({S64});

  getActionDefinitionsBuilder(G_READSTEADYCOUNTER).legalFor({S64});

  getActionDefinitionsBuilder(G_FENCE)
    .alwaysLegal();

  getActionDefinitionsBuilder({G_SMULO, G_UMULO})
      .scalarize(0)
      .minScalar(0, S32)
      .lower();

  getActionDefinitionsBuilder({G_SBFX, G_UBFX})
      .legalFor({{S32, S32}, {S64, S32}})
      .clampScalar(1, S32, S32)
      .clampScalar(0, S32, S64)
      .widenScalarToNextPow2(0)
      .scalarize(0);

  getActionDefinitionsBuilder(
      {// TODO: Verify V_BFI_B32 is generated from expanded bit ops
       G_FCOPYSIGN,

       G_ATOMIC_CMPXCHG_WITH_SUCCESS, G_ATOMICRMW_NAND, G_ATOMICRMW_FSUB,
       G_READ_REGISTER, G_WRITE_REGISTER,

       G_SADDO, G_SSUBO})
      .lower();

  if (ST.hasIEEEMinimumMaximumInsts()) {
    getActionDefinitionsBuilder({G_FMINIMUM, G_FMAXIMUM})
        .legalFor(FPTypesPK16)
        .clampMaxNumElements(0, S16, 2)
        .scalarize(0);
  } else if (ST.hasVOP3PInsts()) {
    getActionDefinitionsBuilder({G_FMINIMUM, G_FMAXIMUM})
        .lowerFor({V2S16})
        .clampMaxNumElementsStrict(0, S16, 2)
        .scalarize(0)
        .lower();
  } else {
    getActionDefinitionsBuilder({G_FMINIMUM, G_FMAXIMUM})
        .scalarize(0)
        .clampScalar(0, S32, S64)
        .lower();
  }

  getActionDefinitionsBuilder(
      {G_MEMCPY, G_MEMCPY_INLINE, G_MEMMOVE, G_MEMSET, G_MEMSET_INLINE})
      .lower();

  getActionDefinitionsBuilder({G_TRAP, G_DEBUGTRAP}).custom();

  getActionDefinitionsBuilder({G_VASTART, G_VAARG, G_BRJT, G_JUMP_TABLE,
        G_INDEXED_LOAD, G_INDEXED_SEXTLOAD,
        G_INDEXED_ZEXTLOAD, G_INDEXED_STORE})
    .unsupported();

  getActionDefinitionsBuilder(G_PREFETCH).alwaysLegal();

  getActionDefinitionsBuilder(
      {G_VECREDUCE_SMIN, G_VECREDUCE_SMAX, G_VECREDUCE_UMIN, G_VECREDUCE_UMAX,
       G_VECREDUCE_ADD, G_VECREDUCE_MUL, G_VECREDUCE_FMUL, G_VECREDUCE_FMIN,
       G_VECREDUCE_FMAX, G_VECREDUCE_FMINIMUM, G_VECREDUCE_FMAXIMUM,
       G_VECREDUCE_OR, G_VECREDUCE_AND, G_VECREDUCE_XOR})
      .legalFor(AllVectors)
      .scalarize(1)
      .lower();

  getActionDefinitionsBuilder({G_INTRINSIC, G_INTRINSIC_W_SIDE_EFFECTS,
                               G_INTRINSIC_CONVERGENT,
                               G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS})
      .alwaysLegal();

  verify(*ST.getInstrInfo());
}

bool AMDGPULegalizerInfo::legalizeCustom(
    LegalizerHelper &Helper, MachineInstr &MI,
    LostDebugLocObserver &LocObserver) const {
  MachineIRBuilder &B = Helper.MIRBuilder;
  MachineRegisterInfo &MRI = *B.getMRI();

  switch (MI.getOpcode()) {
  case TargetOpcode::G_ADDRSPACE_CAST:
    return legalizeAddrSpaceCast(MI, MRI, B);
  case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
    return legalizeFroundeven(MI, MRI, B);
  case TargetOpcode::G_FCEIL:
    return legalizeFceil(MI, MRI, B);
  case TargetOpcode::G_FREM:
    return legalizeFrem(MI, MRI, B);
  case TargetOpcode::G_INTRINSIC_TRUNC:
    return legalizeIntrinsicTrunc(MI, MRI, B);
  case TargetOpcode::G_SITOFP:
    return legalizeITOFP(MI, MRI, B, true);
  case TargetOpcode::G_UITOFP:
    return legalizeITOFP(MI, MRI, B, false);
  case TargetOpcode::G_FPTOSI:
    return legalizeFPTOI(MI, MRI, B, true);
  case TargetOpcode::G_FPTOUI:
    return legalizeFPTOI(MI, MRI, B, false);
  case TargetOpcode::G_FMINNUM:
  case TargetOpcode::G_FMAXNUM:
  case TargetOpcode::G_FMINIMUMNUM:
  case TargetOpcode::G_FMAXIMUMNUM:
    return legalizeMinNumMaxNum(Helper, MI);
  case TargetOpcode::G_EXTRACT:
    return legalizeExtract(Helper, MI);
  case TargetOpcode::G_INSERT:
    return legalizeInsert(Helper, MI);
  case TargetOpcode::G_EXTRACT_VECTOR_ELT:
    return legalizeExtractVectorElt(MI, MRI, B);
  case TargetOpcode::G_INSERT_VECTOR_ELT:
    return legalizeInsertVectorElt(MI, MRI, B);
  case TargetOpcode::G_FSIN:
  case TargetOpcode::G_FCOS:
    return legalizeSinCos(MI, MRI, B);
  case TargetOpcode::G_GLOBAL_VALUE:
    return legalizeGlobalValue(MI, MRI, B);
  case TargetOpcode::G_LOAD:
  case TargetOpcode::G_SEXTLOAD:
  case TargetOpcode::G_ZEXTLOAD:
    return legalizeLoad(Helper, MI);
  case TargetOpcode::G_STORE:
    return legalizeStore(Helper, MI);
  case TargetOpcode::G_FMAD:
    return legalizeFMad(MI, MRI, B);
  case TargetOpcode::G_FDIV:
    return legalizeFDIV(MI, MRI, B);
  case TargetOpcode::G_FFREXP:
    return legalizeFFREXP(MI, MRI, B);
  case TargetOpcode::G_FSQRT:
    return legalizeFSQRT(MI, MRI, B);
  case TargetOpcode::G_UDIV:
  case TargetOpcode::G_UREM:
  case TargetOpcode::G_UDIVREM:
    return legalizeUnsignedDIV_REM(MI, MRI, B);
  case TargetOpcode::G_SDIV:
  case TargetOpcode::G_SREM:
  case TargetOpcode::G_SDIVREM:
    return legalizeSignedDIV_REM(MI, MRI, B);
  case TargetOpcode::G_ATOMIC_CMPXCHG:
    return legalizeAtomicCmpXChg(MI, MRI, B);
  case TargetOpcode::G_FLOG2:
    return legalizeFlog2(MI, B);
  case TargetOpcode::G_FLOG:
  case TargetOpcode::G_FLOG10:
    return legalizeFlogCommon(MI, B);
  case TargetOpcode::G_FEXP2:
    return legalizeFExp2(MI, B);
  case TargetOpcode::G_FEXP:
  case TargetOpcode::G_FEXP10:
    return legalizeFExp(MI, B);
  case TargetOpcode::G_FPOW:
    return legalizeFPow(MI, B);
  case TargetOpcode::G_FFLOOR:
    return legalizeFFloor(MI, MRI, B);
  case TargetOpcode::G_BUILD_VECTOR:
  case TargetOpcode::G_BUILD_VECTOR_TRUNC:
    return legalizeBuildVector(MI, MRI, B);
  case TargetOpcode::G_MUL:
    return legalizeMul(Helper, MI);
  case TargetOpcode::G_CTLZ:
  case TargetOpcode::G_CTTZ:
    return legalizeCTLZ_CTTZ(MI, MRI, B);
  case TargetOpcode::G_CTLS:
    return legalizeCTLS(MI, MRI, B);
  case TargetOpcode::G_CTLZ_ZERO_POISON:
    return legalizeCTLZ_ZERO_POISON(MI, MRI, B);
  case TargetOpcode::G_STACKSAVE:
    return legalizeStackSave(MI, B);
  case TargetOpcode::G_GET_FPENV:
    return legalizeGetFPEnv(MI, MRI, B);
  case TargetOpcode::G_SET_FPENV:
    return legalizeSetFPEnv(MI, MRI, B);
  case TargetOpcode::G_TRAP:
    return legalizeTrap(MI, MRI, B);
  case TargetOpcode::G_DEBUGTRAP:
    return legalizeDebugTrap(MI, MRI, B);
  default:
    return false;
  }

  llvm_unreachable("expected switch to return");
}

Register AMDGPULegalizerInfo::getSegmentAperture(
  unsigned AS,
  MachineRegisterInfo &MRI,
  MachineIRBuilder &B) const {
  MachineFunction &MF = B.getMF();
  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
  const LLT I32 = LLT::integer(32);
  const LLT I64 = LLT::integer(64);

  assert(AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::PRIVATE_ADDRESS);

  if (ST.hasApertureRegs()) {
    // Note: this register is somewhat broken. When used as a 32-bit operand,
    // it only returns zeroes. The real value is in the upper 32 bits.
    // Thus, we must emit extract the high 32 bits.
    const unsigned ApertureRegNo = (AS == AMDGPUAS::LOCAL_ADDRESS)
                                       ? AMDGPU::SRC_SHARED_BASE
                                       : AMDGPU::SRC_PRIVATE_BASE;
    assert((ApertureRegNo != AMDGPU::SRC_PRIVATE_BASE ||
            !ST.hasGloballyAddressableScratch()) &&
           "Cannot use src_private_base with globally addressable scratch!");
    Register Dst = MRI.createGenericVirtualRegister(I64);
    MRI.setRegClass(Dst, &AMDGPU::SReg_64RegClass);
    B.buildCopy({Dst}, {Register(ApertureRegNo)});
    return B.buildUnmerge(I32, Dst).getReg(1);
  }

  Register LoadAddr = MRI.createGenericVirtualRegister(
    LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64));
  // For code object version 5, private_base and shared_base are passed through
  // implicit kernargs.
  if (AMDGPU::getAMDHSACodeObjectVersion(*MF.getFunction().getParent()) >=
      AMDGPU::AMDHSA_COV5) {
    MachinePointerInfo PtrInfo = getKernargSegmentPtrInfo(B.getMF());

    AMDGPUTargetLowering::ImplicitParameter Param =
        AS == AMDGPUAS::LOCAL_ADDRESS ? AMDGPUTargetLowering::SHARED_BASE
                                      : AMDGPUTargetLowering::PRIVATE_BASE;
    uint64_t Offset =
        ST.getTargetLowering()->getImplicitParameterOffset(B.getMF(), Param);

    Register KernargPtrReg = MRI.createGenericVirtualRegister(
        LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64));

    if (!loadInputValue(KernargPtrReg, B,
                        AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR))
      return Register();

    MachineMemOperand *MMO = MF.getMachineMemOperand(
        PtrInfo.getWithOffset(Offset),
        MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
            MachineMemOperand::MOInvariant,
        LLT::integer(32), commonAlignment(Align(64), Offset));

    // Pointer address
    B.buildObjectPtrOffset(LoadAddr, KernargPtrReg,
                           B.buildConstant(LLT::integer(64), Offset).getReg(0));
    // Load address
    return B.buildLoad(I32, LoadAddr, *MMO).getReg(0);
  }

  Register QueuePtr = MRI.createGenericVirtualRegister(
    LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64));

  if (!loadInputValue(QueuePtr, B, AMDGPUFunctionArgInfo::QUEUE_PTR))
    return Register();

  // TODO: Use custom PseudoSourceValue
  MachinePointerInfo PtrInfo(AMDGPUAS::CONSTANT_ADDRESS);

  // Offset into amd_queue_t for group_segment_aperture_base_hi /
  // private_segment_aperture_base_hi.
  uint32_t StructOffset = (AS == AMDGPUAS::LOCAL_ADDRESS) ? 0x40 : 0x44;

  MachineMemOperand *MMO = MF.getMachineMemOperand(
      PtrInfo,
      MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
          MachineMemOperand::MOInvariant,
      LLT::integer(32), commonAlignment(Align(64), StructOffset));

  B.buildObjectPtrOffset(
      LoadAddr, QueuePtr,
      B.buildConstant(LLT::integer(64), StructOffset).getReg(0));
  return B.buildLoad(I32, LoadAddr, *MMO).getReg(0);
}

/// Return true if the value is a known valid address, such that a null check is
/// not necessary.
static bool isKnownNonNull(Register Val, MachineRegisterInfo &MRI,
                           const AMDGPUTargetMachine &TM, unsigned AddrSpace) {
  MachineInstr *Def = MRI.getVRegDef(Val);
  switch (Def->getOpcode()) {
  case AMDGPU::G_FRAME_INDEX:
  case AMDGPU::G_GLOBAL_VALUE:
  case AMDGPU::G_BLOCK_ADDR:
    return true;
  case AMDGPU::G_CONSTANT: {
    const ConstantInt *CI = Def->getOperand(1).getCImm();
    return CI->getSExtValue() != AMDGPU::getNullPointerValue(AddrSpace);
  }
  default:
    return false;
  }

  return false;
}

bool AMDGPULegalizerInfo::legalizeAddrSpaceCast(
  MachineInstr &MI, MachineRegisterInfo &MRI,
  MachineIRBuilder &B) const {
  MachineFunction &MF = B.getMF();

  // MI can either be a G_ADDRSPACE_CAST or a
  // G_INTRINSIC @llvm.amdgcn.addrspacecast.nonnull
  assert(MI.getOpcode() == TargetOpcode::G_ADDRSPACE_CAST ||
         (isa<GIntrinsic>(MI) && cast<GIntrinsic>(MI).getIntrinsicID() ==
                                     Intrinsic::amdgcn_addrspacecast_nonnull));

  const LLT I32 = LLT::integer(32);
  const LLT I64 = LLT::integer(64);
  Register Dst = MI.getOperand(0).getReg();
  Register Src = isa<GIntrinsic>(MI) ? MI.getOperand(2).getReg()
                                     : MI.getOperand(1).getReg();
  LLT DstTy = MRI.getType(Dst);
  LLT SrcTy = MRI.getType(Src);
  unsigned DestAS = DstTy.getAddressSpace();
  unsigned SrcAS = SrcTy.getAddressSpace();

  // TODO: Avoid reloading from the queue ptr for each cast, or at least each
  // vector element.
  assert(!DstTy.isVector());

  const AMDGPUTargetMachine &TM
    = static_cast<const AMDGPUTargetMachine &>(MF.getTarget());

  if (TM.isNoopAddrSpaceCast(SrcAS, DestAS)) {
    MI.setDesc(B.getTII().get(TargetOpcode::G_BITCAST));
    return true;
  }

  if (SrcAS == AMDGPUAS::FLAT_ADDRESS &&
      (DestAS == AMDGPUAS::LOCAL_ADDRESS ||
       DestAS == AMDGPUAS::PRIVATE_ADDRESS)) {
    auto castFlatToLocalOrPrivate = [&](const DstOp &Dst) -> Register {
      if (DestAS == AMDGPUAS::PRIVATE_ADDRESS &&
          ST.hasGloballyAddressableScratch()) {
        // flat -> private with globally addressable scratch: subtract
        // src_flat_scratch_base_lo.
        Register SrcLo = B.buildExtract(I32, Src, 0).getReg(0);
        Register FlatScratchBaseLo =
            B.buildInstr(AMDGPU::S_MOV_B32, {I32},
                         {Register(AMDGPU::SRC_FLAT_SCRATCH_BASE_LO)})
                .getReg(0);
        MRI.setRegClass(FlatScratchBaseLo, &AMDGPU::SReg_32RegClass);
        Register Sub = B.buildSub(I32, SrcLo, FlatScratchBaseLo).getReg(0);
        return B.buildIntToPtr(Dst, Sub).getReg(0);
      }

      // Extract low 32-bits of the pointer.
      return B.buildExtract(Dst, Src, 0).getReg(0);
    };

    // For llvm.amdgcn.addrspacecast.nonnull we can always assume non-null, for
    // G_ADDRSPACE_CAST we need to guess.
    if (isa<GIntrinsic>(MI) || isKnownNonNull(Src, MRI, TM, SrcAS)) {
      castFlatToLocalOrPrivate(Dst);
      MI.eraseFromParent();
      return true;
    }

    unsigned NullVal = AMDGPU::getNullPointerValue(DestAS);

    auto SegmentNull = B.buildConstant(DstTy, NullVal);
    auto FlatNull = B.buildConstant(SrcTy, 0);

    // Extract low 32-bits of the pointer.
    auto PtrLo32 = castFlatToLocalOrPrivate(DstTy);

    auto CmpRes =
        B.buildICmp(CmpInst::ICMP_NE, LLT::scalar(1), Src, FlatNull.getReg(0));
    B.buildSelect(Dst, CmpRes, PtrLo32, SegmentNull.getReg(0));

    MI.eraseFromParent();
    return true;
  }

  if (DestAS == AMDGPUAS::FLAT_ADDRESS &&
      (SrcAS == AMDGPUAS::LOCAL_ADDRESS ||
       SrcAS == AMDGPUAS::PRIVATE_ADDRESS)) {
    auto castLocalOrPrivateToFlat = [&](const DstOp &Dst) -> Register {
      // Coerce the type of the low half of the result so we can use
      // merge_values.
      Register SrcAsInt = B.buildPtrToInt(I32, Src).getReg(0);

      if (SrcAS == AMDGPUAS::PRIVATE_ADDRESS &&
          ST.hasGloballyAddressableScratch()) {
        // For wave32: Addr = (TID[4:0] << 52) + FLAT_SCRATCH_BASE + privateAddr
        // For wave64: Addr = (TID[5:0] << 51) + FLAT_SCRATCH_BASE + privateAddr
        Register AllOnes = B.buildConstant(I32, -1).getReg(0);
        Register ThreadID = B.buildConstant(I32, 0).getReg(0);
        ThreadID = B.buildIntrinsic(Intrinsic::amdgcn_mbcnt_lo, {I32})
                       .addUse(AllOnes)
                       .addUse(ThreadID)
                       .getReg(0);
        if (ST.isWave64()) {
          ThreadID = B.buildIntrinsic(Intrinsic::amdgcn_mbcnt_hi, {I32})
                         .addUse(AllOnes)
                         .addUse(ThreadID)
                         .getReg(0);
        }
        Register ShAmt =
            B.buildConstant(I32, 57 - 32 - ST.getWavefrontSizeLog2()).getReg(0);
        Register SrcHi = B.buildShl(I32, ThreadID, ShAmt).getReg(0);
        Register CvtPtr =
            B.buildMergeLikeInstr(DstTy, {SrcAsInt, SrcHi}).getReg(0);
        // Accessing src_flat_scratch_base_lo as a 64-bit operand gives the full
        // 64-bit hi:lo value.
        Register FlatScratchBase =
            B.buildInstr(AMDGPU::S_MOV_B64, {I64},
                         {Register(AMDGPU::SRC_FLAT_SCRATCH_BASE)})
                .getReg(0);
        MRI.setRegClass(FlatScratchBase, &AMDGPU::SReg_64RegClass);
        return B.buildPtrAdd(Dst, CvtPtr, FlatScratchBase).getReg(0);
      }

      Register ApertureReg = getSegmentAperture(SrcAS, MRI, B);
      if (!ApertureReg.isValid())
        return false;

      // TODO: Should we allow mismatched types but matching sizes in merges to
      // avoid the ptrtoint?
      return B.buildMergeLikeInstr(Dst, {SrcAsInt, ApertureReg}).getReg(0);
    };

    // For llvm.amdgcn.addrspacecast.nonnull we can always assume non-null, for
    // G_ADDRSPACE_CAST we need to guess.
    if (isa<GIntrinsic>(MI) || isKnownNonNull(Src, MRI, TM, SrcAS)) {
      castLocalOrPrivateToFlat(Dst);
      MI.eraseFromParent();
      return true;
    }

    Register BuildPtr = castLocalOrPrivateToFlat(DstTy);

    auto SegmentNull =
        B.buildConstant(SrcTy, AMDGPU::getNullPointerValue(SrcAS));
    auto FlatNull = B.buildConstant(DstTy, AMDGPU::getNullPointerValue(DestAS));

    auto CmpRes = B.buildICmp(CmpInst::ICMP_NE, LLT::scalar(1), Src,
                              SegmentNull.getReg(0));

    B.buildSelect(Dst, CmpRes, BuildPtr, FlatNull);

    MI.eraseFromParent();
    return true;
  }

  if (DestAS == AMDGPUAS::CONSTANT_ADDRESS_32BIT &&
      SrcTy.getSizeInBits() == 64) {
    // Truncate.
    B.buildExtract(Dst, Src, 0);
    MI.eraseFromParent();
    return true;
  }

  if (SrcAS == AMDGPUAS::CONSTANT_ADDRESS_32BIT &&
      DstTy.getSizeInBits() == 64) {
    const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
    uint32_t AddrHiVal = Info->get32BitAddressHighBits();
    auto PtrLo = B.buildPtrToInt(I32, Src);
    if (AddrHiVal == 0) {
      auto Zext = B.buildZExt(I64, PtrLo);
      B.buildIntToPtr(Dst, Zext);
    } else {
      auto HighAddr = B.buildConstant(I32, AddrHiVal);
      B.buildMergeLikeInstr(Dst, {PtrLo, HighAddr});
    }

    MI.eraseFromParent();
    return true;
  }

  // Invalid casts are poison.
  // TODO: Should return poison
  B.buildUndef(Dst);
  MI.eraseFromParent();
  return true;
}

bool AMDGPULegalizerInfo::legalizeFroundeven(MachineInstr &MI,
                                             MachineRegisterInfo &MRI,
                                             MachineIRBuilder &B) const {
  Register Src = MI.getOperand(1).getReg();
  LLT Ty = MRI.getType(Src);
  assert(Ty.isScalar() && Ty.getSizeInBits() == 64);

  APFloat C1Val(APFloat::IEEEdouble(), "0x1.0p+52");
  APFloat C2Val(APFloat::IEEEdouble(), "0x1.fffffffffffffp+51");

  auto C1 = B.buildFConstant(Ty, C1Val);
  auto CopySign = B.buildFCopysign(Ty, C1, Src);

  // TODO: Should this propagate fast-math-flags?
  auto Tmp1 = B.buildFAdd(Ty, Src, CopySign);
  auto Tmp2 = B.buildFSub(Ty, Tmp1, CopySign);

  auto C2 = B.buildFConstant(Ty, C2Val);
  auto Fabs = B.buildFAbs(Ty, Src);

  auto Cond = B.buildFCmp(CmpInst::FCMP_OGT, LLT::scalar(1), Fabs, C2);
  B.buildSelect(MI.getOperand(0).getReg(), Cond, Src, Tmp2);
  MI.eraseFromParent();
  return true;
}

bool AMDGPULegalizerInfo::legalizeFceil(
  MachineInstr &MI, MachineRegisterInfo &MRI,
  MachineIRBuilder &B) const {

  const LLT S1 = LLT::scalar(1);

  Register Src = MI.getOperand(1).getReg();
  assert(MRI.getType(Src) == F64);

  // result = trunc(src)
  // if (src > 0.0 && src != result)
  //   result += 1.0

  auto Trunc = B.buildIntrinsicTrunc(F64, Src);

  const auto Zero = B.buildFConstant(F64, 0.0);
  const auto One = B.buildFConstant(F64, 1.0);
  auto Lt0 = B.buildFCmp(CmpInst::FCMP_OGT, S1, Src, Zero);
  auto NeTrunc = B.buildFCmp(CmpInst::FCMP_ONE, S1, Src, Trunc);
  auto And = B.buildAnd(S1, Lt0, NeTrunc);
  auto Add = B.buildSelect(F64, And, One, Zero);

  // TODO: Should this propagate fast-math-flags?
  B.buildFAdd(MI.getOperand(0).getReg(), Trunc, Add);
  MI.eraseFromParent();
  return true;
}

bool AMDGPULegalizerInfo::legalizeFrem(
  MachineInstr &MI, MachineRegisterInfo &MRI,
  MachineIRBuilder &B) const {
    Register DstReg = MI.getOperand(0).getReg();
    Register Src0Reg = MI.getOperand(1).getReg();
    Register Src1Reg = MI.getOperand(2).getReg();
    auto Flags = MI.getFlags();
    LLT Ty = MRI.getType(DstReg);

    auto Div = B.buildFDiv(Ty, Src0Reg, Src1Reg, Flags);
    auto Trunc = B.buildIntrinsicTrunc(Ty, Div, Flags);
    auto Neg = B.buildFNeg(Ty, Trunc, Flags);
    B.buildFMA(DstReg, Neg, Src1Reg, Src0Reg, Flags);
    MI.eraseFromParent();
    return true;
}

static MachineInstrBuilder extractF64Exponent(Register Hi,
                                              MachineIRBuilder &B) {
  const unsigned FractBits = 52;
  const unsigned ExpBits = 11;
  LLT I32 = LLT::integer(32);

  auto Const0 = B.buildConstant(I32, FractBits - 32);
  auto Const1 = B.buildConstant(I32, ExpBits);

  auto ExpPart = B.buildIntrinsic(Intrinsic::amdgcn_ubfe, {I32})
                     .addUse(Hi)
                     .addUse(Const0.getReg(0))
                     .addUse(Const1.getReg(0));

  return B.buildSub(I32, ExpPart, B.buildConstant(I32, 1023));
}

bool AMDGPULegalizerInfo::legalizeIntrinsicTrunc(
  MachineInstr &MI, MachineRegisterInfo &MRI,
  MachineIRBuilder &B) const {
  const LLT S1 = LLT::scalar(1);
  const LLT I32 = LLT::integer(32);
  const LLT I64 = LLT::integer(64);

  Register Src = MI.getOperand(1).getReg();
  assert(MRI.getType(Src) == F64);

  auto SrcInt = B.buildBitcast(I64, Src);

  // TODO: Should this use extract since the low half is unused?
  auto Unmerge = B.buildUnmerge({I32, I32}, SrcInt);
  Register Hi = Unmerge.getReg(1);

  // Extract the upper half, since this is where we will find the sign and
  // exponent.
  auto Exp = extractF64Exponent(Hi, B);

  const unsigned FractBits = 52;

  // Extract the sign bit.
  const auto SignBitMask = B.buildConstant(I32, UINT32_C(1) << 31);
  auto SignBit = B.buildAnd(I32, Hi, SignBitMask);

  const auto FractMask = B.buildConstant(I64, (UINT64_C(1) << FractBits) - 1);

  const auto Zero32 = B.buildConstant(I32, 0);

  // Extend back to 64-bits.
  auto SignBit64 = B.buildMergeLikeInstr(I64, {Zero32, SignBit});

  auto Shr = B.buildAShr(I64, FractMask, Exp);
  auto Not = B.buildNot(I64, Shr);
  auto Tmp0 = B.buildAnd(I64, SrcInt, Not);
  auto FiftyOne = B.buildConstant(I32, FractBits - 1);

  auto ExpLt0 = B.buildICmp(CmpInst::ICMP_SLT, S1, Exp, Zero32);
  auto ExpGt51 = B.buildICmp(CmpInst::ICMP_SGT, S1, Exp, FiftyOne);

  auto Tmp1 = B.buildSelect(I64, ExpLt0, SignBit64, Tmp0);
  auto Res = B.buildSelect(I64, ExpGt51, SrcInt, Tmp1);
  B.buildBitcast(MI.getOperand(0).getReg(), Res);
  MI.eraseFromParent();
  return true;
}

bool AMDGPULegalizerInfo::legalizeITOFP(
  MachineInstr &MI, MachineRegisterInfo &MRI,
  MachineIRBuilder &B, bool Signed) const {

  Register Dst = MI.getOperand(0).getReg();
  Register Src = MI.getOperand(1).getReg();

  const LLT I64 = LLT::integer(64);
  const LLT I32 = LLT::integer(32);

  assert(MRI.getType(Src) == I64);

  auto Unmerge = B.buildUnmerge({I32, I32}, Src);
  auto ThirtyTwo = B.buildConstant(I32, 32);

  if (MRI.getType(Dst) == F64) {
    auto CvtHi = Signed ? B.buildSITOFP(F64, Unmerge.getReg(1))
                        : B.buildUITOFP(F64, Unmerge.getReg(1));

    auto CvtLo = B.buildUITOFP(F64, Unmerge.getReg(0));
    auto LdExp = B.buildFLdexp(F64, CvtHi, ThirtyTwo);

    // TODO: Should this propagate fast-math-flags?
    B.buildFAdd(Dst, LdExp, CvtLo);
    MI.eraseFromParent();
    return true;
  }

  assert(MRI.getType(Dst) == F32);

  auto One = B.buildConstant(I32, 1);

  MachineInstrBuilder ShAmt;
  if (Signed) {
    auto ThirtyOne = B.buildConstant(I32, 31);
    auto X = B.buildXor(I32, Unmerge.getReg(0), Unmerge.getReg(1));
    auto OppositeSign = B.buildAShr(I32, X, ThirtyOne);
    auto MaxShAmt = B.buildAdd(I32, ThirtyTwo, OppositeSign);
    auto LS = B.buildIntrinsic(Intrinsic::amdgcn_sffbh, {I32})
                  .addUse(Unmerge.getReg(1));
    auto LS2 = B.buildSub(I32, LS, One);
    ShAmt = B.buildUMin(I32, LS2, MaxShAmt);
  } else
    ShAmt = B.buildCTLZ(I32, Unmerge.getReg(1));
  auto Norm = B.buildShl(I64, Src, ShAmt);
  auto Unmerge2 = B.buildUnmerge({I32, I32}, Norm);
  auto Adjust = B.buildUMin(I32, One, Unmerge2.getReg(0));
  auto Norm2 = B.buildOr(I32, Unmerge2.getReg(1), Adjust);
  auto FVal = Signed ? B.buildSITOFP(F32, Norm2) : B.buildUITOFP(F32, Norm2);
  auto Scale = B.buildSub(I32, ThirtyTwo, ShAmt);
  B.buildFLdexp(Dst, FVal, Scale);
  MI.eraseFromParent();
  return true;
}

// TODO: Copied from DAG implementation. Verify logic and document how this
// actually works.
bool AMDGPULegalizerInfo::legalizeFPTOI(MachineInstr &MI,
                                        MachineRegisterInfo &MRI,
                                        MachineIRBuilder &B,
                                        bool Signed) const {

  Register Dst = MI.getOperand(0).getReg();
  Register Src = MI.getOperand(1).getReg();

  const LLT I64 = LLT::integer(64);
  const LLT I32 = LLT::integer(32);

  const LLT SrcLT = MRI.getType(Src);
  assert((SrcLT == F32 || SrcLT == F64) && MRI.getType(Dst) == I64);

  unsigned Flags = MI.getFlags();

  // The basic idea of converting a floating point number into a pair of 32-bit
  // integers is illustrated as follows:
  //
  //     tf := trunc(val);
  //    hif := floor(tf * 2^-32);
  //    lof := tf - hif * 2^32; // lof is always positive due to floor.
  //     hi := fptoi(hif);
  //     lo := fptoi(lof);
  //
  auto Trunc = B.buildIntrinsicTrunc(SrcLT, Src, Flags);
  MachineInstrBuilder Sign;
  if (Signed && SrcLT == F32) {
    // However, a 32-bit floating point number has only 23 bits mantissa and
    // it's not enough to hold all the significant bits of `lof` if val is
    // negative. To avoid the loss of precision, We need to take the absolute
    // value after truncating and flip the result back based on the original
    // signedness.
    auto SrcInt = B.buildBitcast(I32, Src);
    Sign = B.buildAShr(I32, SrcInt, B.buildConstant(I32, 31));
    Trunc = B.buildFAbs(F32, Trunc, Flags);
  }
  MachineInstrBuilder K0, K1;
  if (SrcLT == F64) {
    K0 = B.buildFConstant(
        F64, llvm::bit_cast<double>(UINT64_C(/*2^-32*/ 0x3df0000000000000)));
    K1 = B.buildFConstant(
        F64, llvm::bit_cast<double>(UINT64_C(/*-2^32*/ 0xc1f0000000000000)));
  } else {
    K0 = B.buildFConstant(
        F32, llvm::bit_cast<float>(UINT32_C(/*2^-32*/ 0x2f800000)));
    K1 = B.buildFConstant(
        F32, llvm::bit_cast<float>(UINT32_C(/*-2^32*/ 0xcf800000)));
  }

  auto Mul = B.buildFMul(SrcLT, Trunc, K0, Flags);
  auto FloorMul = B.buildFFloor(SrcLT, Mul, Flags);
  auto Fma = B.buildFMA(SrcLT, FloorMul, K1, Trunc, Flags);

  auto Hi = (Signed && SrcLT == F64) ? B.buildFPTOSI(I32, FloorMul)
                                     : B.buildFPTOUI(I32, FloorMul);
  auto Lo = B.buildFPTOUI(I32, Fma);

  if (Signed && SrcLT == F32) {
    // Flip the result based on the signedness, which is either all 0s or 1s.
    Sign = B.buildMergeLikeInstr(I64, {Sign, Sign});
    // r := xor({lo, hi}, sign) - sign;
    B.buildSub(Dst, B.buildXor(I64, B.buildMergeLikeInstr(I64, {Lo, Hi}), Sign),
               Sign);
  } else
    B.buildMergeLikeInstr(Dst, {Lo, Hi});
  MI.eraseFromParent();

  return true;
}

bool AMDGPULegalizerInfo::legalizeMinNumMaxNum(LegalizerHelper &Helper,
                                               MachineInstr &MI) const {
  MachineFunction &MF = Helper.MIRBuilder.getMF();
  const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();

  // With ieee_mode disabled, the instructions have the correct behavior.
  if (!MFI->getMode().IEEE)
    return true;

  return Helper.lowerFMinNumMaxNum(MI) == LegalizerHelper::Legalized;
}

bool AMDGPULegalizerInfo::legalizeExtract(LegalizerHelper &Helper,
                                          MachineInstr &MI) const {
  MachineIRBuilder &B = Helper.MIRBuilder;
  MachineRegisterInfo &MRI = *B.getMRI();
  Register DstReg = MI.getOperand(0).getReg();
  Register SrcReg = MI.getOperand(1).getReg();
  uint64_t Offset = MI.getOperand(2).getImm();

  // Fall back to generic lowering for offset 0 (trivial trunc) and
  // non-32-bit-aligned cases which require shift+trunc sequences
  // that generic code handles correctly.
  if (Offset == 0 || Offset % 32 != 0)
    return Helper.lowerExtract(MI) == LegalizerHelper::Legalized;

  const LLT DstTy = MRI.getType(DstReg);
  unsigned StartIdx = Offset / 32;
  unsigned DstCount = DstTy.getSizeInBits() / 32;
  auto Unmerge = B.buildUnmerge(LLT::integer(32), SrcReg);

  if (DstCount == 1) {
    if (DstTy.isPointer())
      B.buildIntToPtr(DstReg, Unmerge.getReg(StartIdx));
    else
      MRI.replaceRegWith(DstReg, Unmerge.getReg(StartIdx));
  } else {
    SmallVector<Register, 8> MergeVec;
    for (unsigned I = 0; I < DstCount; ++I)
      MergeVec.push_back(Unmerge.getReg(StartIdx + I));
    B.buildMergeLikeInstr(DstReg, MergeVec);
  }

  MI.eraseFromParent();
  return true;
}

bool AMDGPULegalizerInfo::legalizeInsert(LegalizerHelper &Helper,
                                         MachineInstr &MI) const {
  MachineIRBuilder &B = Helper.MIRBuilder;
  MachineRegisterInfo &MRI = *B.getMRI();
  Register DstReg = MI.getOperand(0).getReg();
  Register SrcReg = MI.getOperand(1).getReg();
  Register InsertSrc = MI.getOperand(2).getReg();
  uint64_t Offset = MI.getOperand(3).getImm();

  unsigned DstSize = MRI.getType(DstReg).getSizeInBits();
  const LLT InsertTy = MRI.getType(InsertSrc);
  unsigned InsertSize = InsertTy.getSizeInBits();

  // Fall back to generic lowering for non-32-bit-aligned cases which
  // require shift+mask sequences that generic code handles correctly.
  if (Offset % 32 != 0 || DstSize % 32 != 0 || InsertSize % 32 != 0)
    return Helper.lowerInsert(MI) == LegalizerHelper::Legalized;

  const LLT I32 = LLT::integer(32);
  unsigned DstCount = DstSize / 32;
  unsigned InsertCount = InsertSize / 32;
  unsigned StartIdx = Offset / 32;

  auto SrcUnmerge = B.buildUnmerge(I32, SrcReg);

  SmallVector<Register, 8> MergeVec;
  for (unsigned I = 0; I < StartIdx; ++I)
    MergeVec.push_back(SrcUnmerge.getReg(I));

  if (InsertCount == 1) {
    // Merge-like instructions require same source types. Convert pointer
    // to scalar when inserting a pointer value into a scalar.
    if (InsertTy.isPointer())
      InsertSrc = B.buildPtrToInt(I32, InsertSrc).getReg(0);
    MergeVec.push_back(InsertSrc);
  } else {
    auto InsertUnmerge = B.buildUnmerge(I32, InsertSrc);
    for (unsigned I = 0; I < InsertCount; ++I)
      MergeVec.push_back(InsertUnmerge.getReg(I));
  }

  for (unsigned I = StartIdx + InsertCount; I < DstCount; ++I)
    MergeVec.push_back(SrcUnmerge.getReg(I));

  B.buildMergeLikeInstr(DstReg, MergeVec);

  MI.eraseFromParent();
  return true;
}

bool AMDGPULegalizerInfo::legalizeExtractVectorElt(
  MachineInstr &MI, MachineRegisterInfo &MRI,
  MachineIRBuilder &B) const {
  // TODO: Should move some of this into LegalizerHelper.

  // TODO: Promote dynamic indexing of i16/f16 to i32/f32

  Register Dst = MI.getOperand(0).getReg();
  Register Vec = MI.getOperand(1).getReg();

  LLT VecTy = MRI.getType(Vec);
  LLT EltTy = VecTy.getElementType();
  assert(EltTy == MRI.getType(Dst));

  // Other legalization maps vector<? x [type bigger than 64 bits]> via bitcasts
  // but we can't go directly to that logic becasue you can't bitcast a vector
  // of pointers to a vector of integers. Therefore, introduce an intermediate
  // vector of integers using ptrtoint (and inttoptr on the output) in order to
  // drive the legalization forward.
  if (EltTy.isPointer() && EltTy.getSizeInBits() > 64) {
    LLT IntTy = LLT::integer(EltTy.getSizeInBits());
    LLT IntVecTy = VecTy.changeElementType(IntTy);

    auto IntVec = B.buildPtrToInt(IntVecTy, Vec);
    auto IntElt = B.buildExtractVectorElement(IntTy, IntVec, MI.getOperand(2));
    B.buildIntToPtr(Dst, IntElt);

    MI.eraseFromParent();
    return true;
  }

  // FIXME: Artifact combiner probably should have replaced the truncated
  // constant before this, so we shouldn't need
  // getIConstantVRegValWithLookThrough.
  std::optional<ValueAndVReg> MaybeIdxVal =
      getIConstantVRegValWithLookThrough(MI.getOperand(2).getReg(), MRI);
  if (!MaybeIdxVal) // Dynamic case will be selected to register indexing.
    return true;
  const uint64_t IdxVal = MaybeIdxVal->Value.getZExtValue();

  if (IdxVal < VecTy.getNumElements()) {
    auto Unmerge = B.buildUnmerge(EltTy, Vec);
    B.buildCopy(Dst, Unmerge.getReg(IdxVal));
  } else {
    B.buildUndef(Dst);
  }

  MI.eraseFromParent();
  return true;
}

bool AMDGPULegalizerInfo::legalizeInsertVectorElt(
  MachineInstr &MI, MachineRegisterInfo &MRI,
  MachineIRBuilder &B) const {
  // TODO: Should move some of this into LegalizerHelper.

  // TODO: Promote dynamic indexing of i16/f16 to i32/f32

  Register Dst = MI.getOperand(0).getReg();
  Register Vec = MI.getOperand(1).getReg();
  Register Ins = MI.getOperand(2).getReg();

  LLT VecTy = MRI.getType(Vec);
  LLT EltTy = VecTy.getElementType();
  assert(EltTy == MRI.getType(Ins));

  // Other legalization maps vector<? x [type bigger than 64 bits]> via bitcasts
  // but we can't go directly to that logic becasue you can't bitcast a vector
  // of pointers to a vector of integers. Therefore, make the pointer vector
  // into an equivalent vector of integers with ptrtoint, insert the ptrtoint'd
  // new value, and then inttoptr the result vector back. This will then allow
  // the rest of legalization to take over.
  if (EltTy.isPointer() && EltTy.getSizeInBits() > 64) {
    LLT IntTy = LLT::integer(EltTy.getSizeInBits());
    LLT IntVecTy = VecTy.changeElementType(IntTy);

    auto IntVecSource = B.buildPtrToInt(IntVecTy, Vec);
    auto IntIns = B.buildPtrToInt(IntTy, Ins);
    auto IntVecDest = B.buildInsertVectorElement(IntVecTy, IntVecSource, IntIns,
                                                 MI.getOperand(3));
    B.buildIntToPtr(Dst, IntVecDest);
    MI.eraseFromParent();
    return true;
  }

  // FIXME: Artifact combiner probably should have replaced the truncated
  // constant before this, so we shouldn't need
  // getIConstantVRegValWithLookThrough.
  std::optional<ValueAndVReg> MaybeIdxVal =
      getIConstantVRegValWithLookThrough(MI.getOperand(3).getReg(), MRI);
  if (!MaybeIdxVal) // Dynamic case will be selected to register indexing.
    return true;

  const uint64_t IdxVal = MaybeIdxVal->Value.getZExtValue();

  unsigned NumElts = VecTy.getNumElements();
  if (IdxVal < NumElts) {
    SmallVector<Register, 8> SrcRegs;
    for (unsigned i = 0; i < NumElts; ++i)
      SrcRegs.push_back(MRI.createGenericVirtualRegister(EltTy));
    B.buildUnmerge(SrcRegs, Vec);

    SrcRegs[IdxVal] = MI.getOperand(2).getReg();
    B.buildMergeLikeInstr(Dst, SrcRegs);
  } else {
    B.buildUndef(Dst);
  }

  MI.eraseFromParent();
  return true;
}

bool AMDGPULegalizerInfo::legalizeSinCos(
  MachineInstr &MI, MachineRegisterInfo &MRI,
  MachineIRBuilder &B) const {

  Register DstReg = MI.getOperand(0).getReg();
  Register SrcReg = MI.getOperand(1).getReg();
  LLT Ty = MRI.getType(DstReg);
  unsigned Flags = MI.getFlags();

  Register TrigVal;
  auto OneOver2Pi = B.buildFConstant(Ty, 0.5 * numbers::inv_pi);
  if (ST.hasTrigReducedRange()) {
    auto MulVal = B.buildFMul(Ty, SrcReg, OneOver2Pi, Flags);
    TrigVal = B.buildIntrinsic(Intrinsic::amdgcn_fract, {Ty})
                  .addUse(MulVal.getReg(0))
                  .setMIFlags(Flags)
                  .getReg(0);
  } else
    TrigVal = B.buildFMul(Ty, SrcReg, OneOver2Pi, Flags).getReg(0);

  Intrinsic::ID TrigIntrin = MI.getOpcode() == AMDGPU::G_FSIN ?
    Intrinsic::amdgcn_sin : Intrinsic::amdgcn_cos;
  B.buildIntrinsic(TrigIntrin, ArrayRef<Register>(DstReg))
      .addUse(TrigVal)
      .setMIFlags(Flags);
  MI.eraseFromParent();
  return true;
}

bool AMDGPULegalizerInfo::buildPCRelGlobalAddress(Register DstReg, LLT PtrTy,
                                                  MachineIRBuilder &B,
                                                  const GlobalValue *GV,
                                                  int64_t Offset,
                                                  unsigned GAFlags) const {
  assert(isInt<32>(Offset + 4) && "32-bit offset is expected!");
  // In order to support pc-relative addressing, SI_PC_ADD_REL_OFFSET is lowered
  // to the following code sequence:
  //
  // For constant address space:
  //   s_getpc_b64 s[0:1]
  //   s_add_u32 s0, s0, $symbol
  //   s_addc_u32 s1, s1, 0
  //
  //   s_getpc_b64 returns the address of the s_add_u32 instruction and then
  //   a fixup or relocation is emitted to replace $symbol with a literal
  //   constant, which is a pc-relative offset from the encoding of the $symbol
  //   operand to the global variable.
  //
  // For global address space:
  //   s_getpc_b64 s[0:1]
  //   s_add_u32 s0, s0, $symbol@{gotpc}rel32@lo
  //   s_addc_u32 s1, s1, $symbol@{gotpc}rel32@hi
  //
  //   s_getpc_b64 returns the address of the s_add_u32 instruction and then
  //   fixups or relocations are emitted to replace $symbol@*@lo and
  //   $symbol@*@hi with lower 32 bits and higher 32 bits of a literal constant,
  //   which is a 64-bit pc-relative offset from the encoding of the $symbol
  //   operand to the global variable.

  LLT ConstPtrTy = LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64);

  Register PCReg = PtrTy.getSizeInBits() != 32 ? DstReg :
    B.getMRI()->createGenericVirtualRegister(ConstPtrTy);

  if (ST.has64BitLiterals()) {
    assert(GAFlags != SIInstrInfo::MO_NONE);

    MachineInstrBuilder MIB =
        B.buildInstr(AMDGPU::SI_PC_ADD_REL_OFFSET64).addDef(PCReg);
    MIB.addGlobalAddress(GV, Offset, GAFlags + 2);
  } else {
    MachineInstrBuilder MIB =
        B.buildInstr(AMDGPU::SI_PC_ADD_REL_OFFSET).addDef(PCReg);

    MIB.addGlobalAddress(GV, Offset, GAFlags);
    if (GAFlags == SIInstrInfo::MO_NONE)
      MIB.addImm(0);
    else
      MIB.addGlobalAddress(GV, Offset, GAFlags + 1);
  }

  if (!B.getMRI()->getRegClassOrNull(PCReg))
    B.getMRI()->setRegClass(PCReg, &AMDGPU::SReg_64RegClass);

  if (PtrTy.getSizeInBits() == 32)
    B.buildExtract(DstReg, PCReg, 0);
  return true;
}

// Emit a ABS32_LO / ABS32_HI relocation stub.
void AMDGPULegalizerInfo::buildAbsGlobalAddress(
    Register DstReg, LLT PtrTy, MachineIRBuilder &B, const GlobalValue *GV,
    MachineRegisterInfo &MRI) const {
  bool RequiresHighHalf = PtrTy.getSizeInBits() != 32;

  if (RequiresHighHalf && ST.has64BitLiterals()) {
    if (!MRI.getRegClassOrNull(DstReg))
      MRI.setRegClass(DstReg, &AMDGPU::SReg_64RegClass);
    B.buildInstr(AMDGPU::S_MOV_B64)
        .addDef(DstReg)
        .addGlobalAddress(GV, 0, SIInstrInfo::MO_ABS64);
    return;
  }

  LLT I32 = LLT::integer(32);

  // Use the destination directly, if and only if we store the lower address
  // part only and we don't have a register class being set.
  Register AddrLo = !RequiresHighHalf && !MRI.getRegClassOrNull(DstReg)
                        ? DstReg
                        : MRI.createGenericVirtualRegister(I32);

  if (!MRI.getRegClassOrNull(AddrLo))
    MRI.setRegClass(AddrLo, &AMDGPU::SReg_32RegClass);

  // Write the lower half.
  B.buildInstr(AMDGPU::S_MOV_B32)
      .addDef(AddrLo)
      .addGlobalAddress(GV, 0, SIInstrInfo::MO_ABS32_LO);

  // If required, write the upper half as well.
  if (RequiresHighHalf) {
    assert(PtrTy.getSizeInBits() == 64 &&
           "Must provide a 64-bit pointer type!");

    Register AddrHi = MRI.createGenericVirtualRegister(I32);
    MRI.setRegClass(AddrHi, &AMDGPU::SReg_32RegClass);

    B.buildInstr(AMDGPU::S_MOV_B32)
        .addDef(AddrHi)
        .addGlobalAddress(GV, 0, SIInstrInfo::MO_ABS32_HI);

    // Use the destination directly, if and only if we don't have a register
    // class being set.
    Register AddrDst = !MRI.getRegClassOrNull(DstReg)
                           ? DstReg
                           : MRI.createGenericVirtualRegister(LLT::integer(64));

    if (!MRI.getRegClassOrNull(AddrDst))
      MRI.setRegClass(AddrDst, &AMDGPU::SReg_64RegClass);

    B.buildMergeValues(AddrDst, {AddrLo, AddrHi});

    // If we created a new register for the destination, cast the result into
    // the final output.
    if (AddrDst != DstReg)
      B.buildCast(DstReg, AddrDst);
  } else if (AddrLo != DstReg) {
    // If we created a new register for the destination, cast the result into
    // the final output.
    B.buildCast(DstReg, AddrLo);
  }
}

bool AMDGPULegalizerInfo::legalizeGlobalValue(
  MachineInstr &MI, MachineRegisterInfo &MRI,
  MachineIRBuilder &B) const {
  Register DstReg = MI.getOperand(0).getReg();
  LLT Ty = MRI.getType(DstReg);
  unsigned AS = Ty.getAddressSpace();

  const GlobalValue *GV = MI.getOperand(1).getGlobal();
  MachineFunction &MF = B.getMF();
  SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();

  if (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS) {
    if (!MFI->isModuleEntryFunction() &&
        GV->getName() != "llvm.amdgcn.module.lds" &&
        !AMDGPU::isNamedBarrier(*cast<GlobalVariable>(GV))) {
      const Function &Fn = MF.getFunction();
      Fn.getContext().diagnose(DiagnosticInfoUnsupported(
          Fn, "local memory global used by non-kernel function",
          MI.getDebugLoc(), DS_Warning));

      // We currently don't have a way to correctly allocate LDS objects that
      // aren't directly associated with a kernel. We do force inlining of
      // functions that use local objects. However, if these dead functions are
      // not eliminated, we don't want a compile time error. Just emit a warning
      // and a trap, since there should be no callable path here.
      B.buildTrap();
      B.buildUndef(DstReg);
      MI.eraseFromParent();
      return true;
    }

    // TODO: We could emit code to handle the initialization somewhere.
    // We ignore the initializer for now and legalize it to allow selection.
    // The initializer will anyway get errored out during assembly emission.
    const SITargetLowering *TLI = ST.getTargetLowering();
    if (!TLI->shouldUseLDSConstAddress(GV)) {
      MI.getOperand(1).setTargetFlags(SIInstrInfo::MO_ABS32_LO);
      return true; // Leave in place;
    }

    const GlobalVariable &GVar = *cast<GlobalVariable>(GV);
    if (AS == AMDGPUAS::LOCAL_ADDRESS && GV->hasExternalLinkage()) {
      // HIP uses an unsized array `extern __shared__ T s[]` or similar
      // zero-sized type in other languages to declare the dynamic shared
      // memory which size is not known at the compile time. They will be
      // allocated by the runtime and placed directly after the static
      // allocated ones. They all share the same offset.
      if (GVar.getGlobalSize(GVar.getDataLayout()) == 0) {
        // Adjust alignment for that dynamic shared memory array.
        MFI->setDynLDSAlign(MF.getFunction(), GVar);
        LLT I32 = LLT::integer(32);
        auto Sz = B.buildIntrinsic(Intrinsic::amdgcn_groupstaticsize, {I32});
        B.buildIntToPtr(DstReg, Sz);
        MI.eraseFromParent();
        return true;
      }
    }

    B.buildConstant(DstReg, MFI->allocateLDSGlobal(B.getDataLayout(), GVar));
    MI.eraseFromParent();
    return true;
  }

  if (ST.isAmdPalOS() || ST.isMesa3DOS()) {
    buildAbsGlobalAddress(DstReg, Ty, B, GV, MRI);
    MI.eraseFromParent();
    return true;
  }

  const SITargetLowering *TLI = ST.getTargetLowering();

  if (TLI->shouldEmitFixup(GV)) {
    buildPCRelGlobalAddress(DstReg, Ty, B, GV, 0);
    MI.eraseFromParent();
    return true;
  }

  if (TLI->shouldEmitPCReloc(GV)) {
    buildPCRelGlobalAddress(DstReg, Ty, B, GV, 0, SIInstrInfo::MO_REL32);
    MI.eraseFromParent();
    return true;
  }

  LLT PtrTy = LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64);
  Register GOTAddr = MRI.createGenericVirtualRegister(PtrTy);

  LLT LoadTy = Ty.getSizeInBits() == 32 ? PtrTy : Ty;
  MachineMemOperand *GOTMMO = MF.getMachineMemOperand(
      MachinePointerInfo::getGOT(MF),
      MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
          MachineMemOperand::MOInvariant,
      LoadTy, Align(8));

  buildPCRelGlobalAddress(GOTAddr, PtrTy, B, GV, 0, SIInstrInfo::MO_GOTPCREL32);

  if (Ty.getSizeInBits() == 32) {
    // Truncate if this is a 32-bit constant address.
    auto Load = B.buildLoad(PtrTy, GOTAddr, *GOTMMO);
    B.buildExtract(DstReg, Load, 0);
  } else
    B.buildLoad(DstReg, GOTAddr, *GOTMMO);

  MI.eraseFromParent();
  return true;
}

static LLT widenToNextPowerOf2(LLT Ty) {
  if (Ty.isVector())
    return Ty.changeElementCount(
        ElementCount::getFixed(PowerOf2Ceil(Ty.getNumElements())));
  return Ty.changeElementSize(PowerOf2Ceil(Ty.getSizeInBits()));
}

bool AMDGPULegalizerInfo::legalizeLoad(LegalizerHelper &Helper,
                                       MachineInstr &MI) const {
  MachineIRBuilder &B = Helper.MIRBuilder;
  MachineRegisterInfo &MRI = *B.getMRI();
  GISelChangeObserver &Observer = Helper.Observer;

  Register PtrReg = MI.getOperand(1).getReg();
  LLT PtrTy = MRI.getType(PtrReg);
  unsigned AddrSpace = PtrTy.getAddressSpace();

  if (AddrSpace == AMDGPUAS::CONSTANT_ADDRESS_32BIT) {
    LLT ConstPtr = LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64);
    auto Cast = B.buildAddrSpaceCast(ConstPtr, PtrReg);
    Observer.changingInstr(MI);
    MI.getOperand(1).setReg(Cast.getReg(0));
    Observer.changedInstr(MI);
    return true;
  }

  if (MI.getOpcode() != AMDGPU::G_LOAD)
    return false;

  Register ValReg = MI.getOperand(0).getReg();
  LLT ValTy = MRI.getType(ValReg);

  if (hasBufferRsrcWorkaround(ValTy)) {
    Observer.changingInstr(MI);
    castBufferRsrcFromV4I32(MI, B, MRI, 0);
    Observer.changedInstr(MI);
    return true;
  }

  MachineMemOperand *MMO = *MI.memoperands_begin();
  const unsigned ValSize = ValTy.getSizeInBits();
  const LLT MemTy = MMO->getMemoryType();
  const Align MemAlign = MMO->getAlign();
  const unsigned MemSize = MemTy.getSizeInBits();
  const uint64_t AlignInBits = 8 * MemAlign.value();

  // Widen non-power-of-2 loads to the alignment if needed
  if (shouldWidenLoad(ST, MemTy, AlignInBits, AddrSpace, MI.getOpcode())) {
    const unsigned WideMemSize = PowerOf2Ceil(MemSize);

    // This was already the correct extending load result type, so just adjust
    // the memory type.
    if (WideMemSize == ValSize) {
      MachineFunction &MF = B.getMF();

      MachineMemOperand *WideMMO =
          MF.getMachineMemOperand(MMO, 0, WideMemSize / 8);
      Observer.changingInstr(MI);
      MI.setMemRefs(MF, {WideMMO});
      Observer.changedInstr(MI);
      return true;
    }

    // Don't bother handling edge case that should probably never be produced.
    if (ValSize > WideMemSize)
      return false;

    LLT WideTy = widenToNextPowerOf2(ValTy);

    Register WideLoad;
    if (!WideTy.isVector()) {
      WideLoad = B.buildLoadFromOffset(WideTy, PtrReg, *MMO, 0).getReg(0);
      B.buildTrunc(ValReg, WideLoad).getReg(0);
    } else {
      // Extract the subvector.

      if (isRegisterType(ST, ValTy)) {
        // If this a case where G_EXTRACT is legal, use it.
        // (e.g. <3 x i32> -> <4 x i32>)
        WideLoad = B.buildLoadFromOffset(WideTy, PtrReg, *MMO, 0).getReg(0);
        B.buildExtract(ValReg, WideLoad, 0);
      } else {
        // For cases where the widened type isn't a nice register value, unmerge
        // from a widened register (e.g. <3 x i16> -> <4 x i16>)
        WideLoad = B.buildLoadFromOffset(WideTy, PtrReg, *MMO, 0).getReg(0);
        B.buildDeleteTrailingVectorElements(ValReg, WideLoad);
      }
    }

    MI.eraseFromParent();
    return true;
  }

  return false;
}

bool AMDGPULegalizerInfo::legalizeStore(LegalizerHelper &Helper,
                                        MachineInstr &MI) const {
  MachineIRBuilder &B = Helper.MIRBuilder;
  MachineRegisterInfo &MRI = *B.getMRI();
  GISelChangeObserver &Observer = Helper.Observer;

  Register DataReg = MI.getOperand(0).getReg();
  LLT DataTy = MRI.getType(DataReg);

  if (hasBufferRsrcWorkaround(DataTy)) {
    Observer.changingInstr(MI);
    castBufferRsrcArgToV4I32(MI, B, 0);
    Observer.changedInstr(MI);
    return true;
  }
  return false;
}

bool AMDGPULegalizerInfo::legalizeFMad(
  MachineInstr &MI, MachineRegisterInfo &MRI,
  MachineIRBuilder &B) const {
  LLT Ty = MRI.getType(MI.getOperand(0).getReg());
  assert(Ty.isScalar());

  MachineFunction &MF = B.getMF();
  const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();

  // TODO: Always legal with future ftz flag.
  // TODO: Type is expected to be LLT::float32()/LLT::float16()
  // FIXME: Do we need just output?
  if (Ty == F32 &&
      MFI->getMode().FP32Denormals == DenormalMode::getPreserveSign())
    return true;
  if (Ty == F16 &&
      MFI->getMode().FP64FP16Denormals == DenormalMode::getPreserveSign())
    return true;

  MachineIRBuilder HelperBuilder(MI);
  GISelObserverWrapper DummyObserver;
  LegalizerHelper Helper(MF, DummyObserver, HelperBuilder);
  return Helper.lowerFMad(MI) == LegalizerHelper::Legalized;
}

bool AMDGPULegalizerInfo::legalizeAtomicCmpXChg(
  MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const {
  Register DstReg = MI.getOperand(0).getReg();
  Register PtrReg = MI.getOperand(1).getReg();
  Register CmpVal = MI.getOperand(2).getReg();
  Register NewVal = MI.getOperand(3).getReg();

  assert(AMDGPU::isFlatGlobalAddrSpace(MRI.getType(PtrReg).getAddressSpace()) &&
         "this should not have been custom lowered");

  LLT ValTy = MRI.getType(CmpVal);
  LLT VecTy = LLT::fixed_vector(2, ValTy);

  Register PackedVal = B.buildBuildVector(VecTy, { NewVal, CmpVal }).getReg(0);

  B.buildInstr(AMDGPU::G_AMDGPU_ATOMIC_CMPXCHG)
    .addDef(DstReg)
    .addUse(PtrReg)
    .addUse(PackedVal)
    .setMemRefs(MI.memoperands());

  MI.eraseFromParent();
  return true;
}

/// Return true if it's known that \p Src can never be an f32 denormal value.
static bool valueIsKnownNeverF32Denorm(const MachineRegisterInfo &MRI,
                                       Register Src) {
  const MachineInstr *DefMI = MRI.getVRegDef(Src);
  switch (DefMI->getOpcode()) {
  case TargetOpcode::G_INTRINSIC: {
    switch (cast<GIntrinsic>(DefMI)->getIntrinsicID()) {
    case Intrinsic::amdgcn_frexp_mant:
    case Intrinsic::amdgcn_log:
    case Intrinsic::amdgcn_log_clamp:
    case Intrinsic::amdgcn_exp2:
    case Intrinsic::amdgcn_sqrt:
      return true;
    default:
      break;
    }

    break;
  }
  case TargetOpcode::G_FSQRT:
    return true;
  case TargetOpcode::G_FFREXP: {
    if (DefMI->getOperand(0).getReg() == Src)
      return true;
    break;
  }
  case TargetOpcode::G_FPEXT: {
    return MRI.getType(DefMI->getOperand(1).getReg()) == F16;
  }
  default:
    return false;
  }

  return false;
}

static bool allowApproxFunc(const MachineFunction &MF, unsigned Flags) {
  return Flags & MachineInstr::FmAfn;
}

static bool needsDenormHandlingF32(const MachineFunction &MF, Register Src,
                                   unsigned Flags) {
  return !valueIsKnownNeverF32Denorm(MF.getRegInfo(), Src) &&
         MF.getDenormalMode(APFloat::IEEEsingle()).Input !=
             DenormalMode::PreserveSign;
}

std::pair<Register, Register>
AMDGPULegalizerInfo::getScaledLogInput(MachineIRBuilder &B, Register Src,
                                       unsigned Flags) const {
  if (!needsDenormHandlingF32(B.getMF(), Src, Flags))
    return {};

  auto SmallestNormal = B.buildFConstant(
      F32, APFloat::getSmallestNormalized(APFloat::IEEEsingle()));
  auto IsLtSmallestNormal =
      B.buildFCmp(CmpInst::FCMP_OLT, LLT::scalar(1), Src, SmallestNormal);

  auto Scale32 = B.buildFConstant(F32, 0x1.0p+32);
  auto One = B.buildFConstant(F32, 1.0);
  auto ScaleFactor =
      B.buildSelect(F32, IsLtSmallestNormal, Scale32, One, Flags);
  auto ScaledInput = B.buildFMul(F32, Src, ScaleFactor, Flags);

  return {ScaledInput.getReg(0), IsLtSmallestNormal.getReg(0)};
}

bool AMDGPULegalizerInfo::legalizeFlog2(MachineInstr &MI,
                                        MachineIRBuilder &B) const {
  // v_log_f32 is good enough for OpenCL, except it doesn't handle denormals.
  // If we have to handle denormals, scale up the input and adjust the result.

  // scaled = x * (is_denormal ? 0x1.0p+32 : 1.0)
  // log2 = amdgpu_log2 - (is_denormal ? 32.0 : 0.0)

  Register Dst = MI.getOperand(0).getReg();
  Register Src = MI.getOperand(1).getReg();
  LLT Ty = B.getMRI()->getType(Dst);
  unsigned Flags = MI.getFlags();

  if (Ty == F16) {
    // Nothing in half is a denormal when promoted to f32.
    auto Ext = B.buildFPExt(F32, Src, Flags);
    auto Log2 = B.buildIntrinsic(Intrinsic::amdgcn_log, {F32})
                    .addUse(Ext.getReg(0))
                    .setMIFlags(Flags);
    B.buildFPTrunc(Dst, Log2, Flags);
    MI.eraseFromParent();
    return true;
  }

  assert(Ty == F32);

  auto [ScaledInput, IsLtSmallestNormal] = getScaledLogInput(B, Src, Flags);
  if (!ScaledInput) {
    B.buildIntrinsic(Intrinsic::amdgcn_log, {MI.getOperand(0)})
        .addUse(Src)
        .setMIFlags(Flags);
    MI.eraseFromParent();
    return true;
  }

  auto Log2 = B.buildIntrinsic(Intrinsic::amdgcn_log, {Ty})
                  .addUse(ScaledInput)
                  .setMIFlags(Flags);

  auto ThirtyTwo = B.buildFConstant(Ty, 32.0);
  auto Zero = B.buildFConstant(Ty, 0.0);
  auto ResultOffset =
      B.buildSelect(Ty, IsLtSmallestNormal, ThirtyTwo, Zero, Flags);
  B.buildFSub(Dst, Log2, ResultOffset, Flags);

  MI.eraseFromParent();
  return true;
}

static Register getMad(MachineIRBuilder &B, LLT Ty, Register X, Register Y,
                       Register Z, unsigned Flags) {
  auto FMul = B.buildFMul(Ty, X, Y, Flags);
  return B.buildFAdd(Ty, FMul, Z, Flags).getReg(0);
}

bool AMDGPULegalizerInfo::legalizeFlogCommon(MachineInstr &MI,
                                             MachineIRBuilder &B) const {
  const bool IsLog10 = MI.getOpcode() == TargetOpcode::G_FLOG10;
  assert(IsLog10 || MI.getOpcode() == TargetOpcode::G_FLOG);

  MachineRegisterInfo &MRI = *B.getMRI();
  Register Dst = MI.getOperand(0).getReg();
  Register X = MI.getOperand(1).getReg();
  unsigned Flags = MI.getFlags();
  const LLT Ty = MRI.getType(X);

  if (Ty == F16 || MI.getFlag(MachineInstr::FmAfn)) {
    // TODO: The direct f16 path is 1.79 ulp for f16. This should be used
    // depending on !fpmath metadata.
    bool PromoteToF32 =
        Ty == F16 && (!MI.getFlag(MachineInstr::FmAfn) || !ST.has16BitInsts());
    if (PromoteToF32) {
      Register LogVal = MRI.createGenericVirtualRegister(F32);
      auto PromoteSrc = B.buildFPExt(F32, X);
      legalizeFlogUnsafe(B, LogVal, PromoteSrc.getReg(0), IsLog10, Flags);
      B.buildFPTrunc(Dst, LogVal);
    } else {
      legalizeFlogUnsafe(B, Dst, X, IsLog10, Flags);
    }

    MI.eraseFromParent();
    return true;
  }

  auto [ScaledInput, IsScaled] = getScaledLogInput(B, X, Flags);
  if (ScaledInput)
    X = ScaledInput;

  auto Y =
      B.buildIntrinsic(Intrinsic::amdgcn_log, {Ty}).addUse(X).setMIFlags(Flags);

  Register R;
  if (ST.hasFastFMAF32()) {
    // c+cc are ln(2)/ln(10) to more than 49 bits
    const float c_log10 = 0x1.344134p-2f;
    const float cc_log10 = 0x1.09f79ep-26f;

    // c + cc is ln(2) to more than 49 bits
    const float c_log = 0x1.62e42ep-1f;
    const float cc_log = 0x1.efa39ep-25f;

    auto C = B.buildFConstant(Ty, IsLog10 ? c_log10 : c_log);
    auto CC = B.buildFConstant(Ty, IsLog10 ? cc_log10 : cc_log);
    // This adds correction terms for which contraction may lead to an increase
    // in the error of the approximation, so disable it.
    auto NewFlags = Flags & ~(MachineInstr::FmContract);
    R = B.buildFMul(Ty, Y, C, NewFlags).getReg(0);
    auto NegR = B.buildFNeg(Ty, R, NewFlags);
    auto FMA0 = B.buildFMA(Ty, Y, C, NegR, NewFlags);
    auto FMA1 = B.buildFMA(Ty, Y, CC, FMA0, NewFlags);
    R = B.buildFAdd(Ty, R, FMA1, NewFlags).getReg(0);
  } else {
    // ch+ct is ln(2)/ln(10) to more than 36 bits
    const float ch_log10 = 0x1.344000p-2f;
    const float ct_log10 = 0x1.3509f6p-18f;

    // ch + ct is ln(2) to more than 36 bits
    const float ch_log = 0x1.62e000p-1f;
    const float ct_log = 0x1.0bfbe8p-15f;

    auto CH = B.buildFConstant(Ty, IsLog10 ? ch_log10 : ch_log);
    auto CT = B.buildFConstant(Ty, IsLog10 ? ct_log10 : ct_log);

    auto MaskConst = B.buildConstant(Ty, 0xfffff000);
    auto YH = B.buildAnd(Ty, Y, MaskConst);
    auto YT = B.buildFSub(Ty, Y, YH, Flags);
    // This adds correction terms for which contraction may lead to an increase
    // in the error of the approximation, so disable it.
    auto NewFlags = Flags & ~(MachineInstr::FmContract);
    auto YTCT = B.buildFMul(Ty, YT, CT, NewFlags);

    Register Mad0 =
        getMad(B, Ty, YH.getReg(0), CT.getReg(0), YTCT.getReg(0), NewFlags);
    Register Mad1 = getMad(B, Ty, YT.getReg(0), CH.getReg(0), Mad0, NewFlags);
    R = getMad(B, Ty, YH.getReg(0), CH.getReg(0), Mad1, NewFlags);
  }

  const bool IsFiniteOnly =
      MI.getFlag(MachineInstr::FmNoNans) && MI.getFlag(MachineInstr::FmNoInfs);

  if (!IsFiniteOnly) {
    // Expand isfinite(x) => fabs(x) < inf
    auto Inf = B.buildFConstant(Ty, APFloat::getInf(APFloat::IEEEsingle()));
    auto Fabs = B.buildFAbs(Ty, Y);
    auto IsFinite =
        B.buildFCmp(CmpInst::FCMP_OLT, LLT::scalar(1), Fabs, Inf, Flags);
    R = B.buildSelect(Ty, IsFinite, R, Y, Flags).getReg(0);
  }

  if (ScaledInput) {
    auto Zero = B.buildFConstant(Ty, 0.0);
    auto ShiftK =
        B.buildFConstant(Ty, IsLog10 ? 0x1.344136p+3f : 0x1.62e430p+4f);
    auto Shift = B.buildSelect(Ty, IsScaled, ShiftK, Zero, Flags);
    B.buildFSub(Dst, R, Shift, Flags);
  } else {
    B.buildCopy(Dst, R);
  }

  MI.eraseFromParent();
  return true;
}

bool AMDGPULegalizerInfo::legalizeFlogUnsafe(MachineIRBuilder &B, Register Dst,
                                             Register Src, bool IsLog10,
                                             unsigned Flags) const {
  const double Log2BaseInverted =
      IsLog10 ? numbers::ln2 / numbers::ln10 : numbers::ln2;

  LLT Ty = B.getMRI()->getType(Dst);

  if (Ty == F32) {
    auto [ScaledInput, IsScaled] = getScaledLogInput(B, Src, Flags);
    if (ScaledInput) {
      auto LogSrc = B.buildIntrinsic(Intrinsic::amdgcn_log, {Ty})
                        .addUse(Src)
                        .setMIFlags(Flags);
      auto ScaledResultOffset = B.buildFConstant(Ty, -32.0 * Log2BaseInverted);
      auto Zero = B.buildFConstant(Ty, 0.0);
      auto ResultOffset =
          B.buildSelect(Ty, IsScaled, ScaledResultOffset, Zero, Flags);
      auto Log2Inv = B.buildFConstant(Ty, Log2BaseInverted);

      if (ST.hasFastFMAF32())
        B.buildFMA(Dst, LogSrc, Log2Inv, ResultOffset, Flags);
      else {
        auto Mul = B.buildFMul(Ty, LogSrc, Log2Inv, Flags);
        B.buildFAdd(Dst, Mul, ResultOffset, Flags);
      }

      return true;
    }
  }

  auto Log2Operand = Ty == F16 ? B.buildFLog2(Ty, Src, Flags)
                               : B.buildIntrinsic(Intrinsic::amdgcn_log, {Ty})
                                     .addUse(Src)
                                     .setMIFlags(Flags);
  auto Log2BaseInvertedOperand = B.buildFConstant(Ty, Log2BaseInverted);
  B.buildFMul(Dst, Log2Operand, Log2BaseInvertedOperand, Flags);
  return true;
}

bool AMDGPULegalizerInfo::legalizeFExp2(MachineInstr &MI,
                                        MachineIRBuilder &B) const {
  // v_exp_f32 is good enough for OpenCL, except it doesn't handle denormals.
  // If we have to handle denormals, scale up the input and adjust the result.

  Register Dst = MI.getOperand(0).getReg();
  Register Src = MI.getOperand(1).getReg();
  unsigned Flags = MI.getFlags();
  LLT Ty = B.getMRI()->getType(Dst);

  if (Ty == F64)
    return legalizeFEXPF64(MI, B);

  if (Ty == F16) {
    // Nothing in half is a denormal when promoted to f32.
    auto Ext = B.buildFPExt(F32, Src, Flags);
    auto Log2 = B.buildIntrinsic(Intrinsic::amdgcn_exp2, {F32})
                    .addUse(Ext.getReg(0))
                    .setMIFlags(Flags);
    B.buildFPTrunc(Dst, Log2, Flags);
    MI.eraseFromParent();
    return true;
  }

  assert(Ty == F32);

  if (!needsDenormHandlingF32(B.getMF(), Src, Flags)) {
    B.buildIntrinsic(Intrinsic::amdgcn_exp2, ArrayRef<Register>{Dst})
        .addUse(Src)
        .setMIFlags(Flags);
    MI.eraseFromParent();
    return true;
  }

  // bool needs_scaling = x < -0x1.f80000p+6f;
  // v_exp_f32(x + (s ? 0x1.0p+6f : 0.0f)) * (s ? 0x1.0p-64f : 1.0f);

  // -nextafter(128.0, -1)
  auto RangeCheckConst = B.buildFConstant(Ty, -0x1.f80000p+6f);
  auto NeedsScaling = B.buildFCmp(CmpInst::FCMP_OLT, LLT::scalar(1), Src,
                                  RangeCheckConst, Flags);

  auto SixtyFour = B.buildFConstant(Ty, 0x1.0p+6f);
  auto Zero = B.buildFConstant(Ty, 0.0);
  auto AddOffset = B.buildSelect(F32, NeedsScaling, SixtyFour, Zero, Flags);
  auto AddInput = B.buildFAdd(F32, Src, AddOffset, Flags);

  auto Exp2 = B.buildIntrinsic(Intrinsic::amdgcn_exp2, {Ty})
                  .addUse(AddInput.getReg(0))
                  .setMIFlags(Flags);

  auto TwoExpNeg64 = B.buildFConstant(Ty, 0x1.0p-64f);
  auto One = B.buildFConstant(Ty, 1.0);
  auto ResultScale = B.buildSelect(F32, NeedsScaling, TwoExpNeg64, One, Flags);
  B.buildFMul(Dst, Exp2, ResultScale, Flags);
  MI.eraseFromParent();
  return true;
}

static MachineInstrBuilder buildExp(MachineIRBuilder &B, const DstOp &Dst,
                                    const SrcOp &Src, unsigned Flags) {
  LLT Ty = Dst.getLLTTy(*B.getMRI());

  if (Ty == F32) {
    return B.buildIntrinsic(Intrinsic::amdgcn_exp2, {Dst})
        .addUse(Src.getReg())
        .setMIFlags(Flags);
  }
  return B.buildFExp2(Dst, Src, Flags);
}

bool AMDGPULegalizerInfo::legalizeFExpUnsafeImpl(MachineIRBuilder &B,
                                                 Register Dst, Register X,
                                                 unsigned Flags,
                                                 bool IsExp10) const {
  LLT Ty = B.getMRI()->getType(X);

  // exp(x) -> exp2(M_LOG2E_F * x);
  // exp10(x) -> exp2(log2(10) * x);
  auto Const = B.buildFConstant(Ty, IsExp10 ? 0x1.a934f0p+1f : numbers::log2e);
  auto Mul = B.buildFMul(Ty, X, Const, Flags);
  buildExp(B, Dst, Mul, Flags);
  return true;
}

bool AMDGPULegalizerInfo::legalizeFExpUnsafe(MachineIRBuilder &B, Register Dst,
                                             Register X, unsigned Flags) const {
  LLT Ty = B.getMRI()->getType(Dst);

  if (Ty != F32 || !needsDenormHandlingF32(B.getMF(), X, Flags)) {
    return legalizeFExpUnsafeImpl(B, Dst, X, Flags, /*IsExp10=*/false);
  }

  auto Threshold = B.buildFConstant(Ty, -0x1.5d58a0p+6f);
  auto NeedsScaling =
      B.buildFCmp(CmpInst::FCMP_OLT, LLT::scalar(1), X, Threshold, Flags);
  auto ScaleOffset = B.buildFConstant(Ty, 0x1.0p+6f);
  auto ScaledX = B.buildFAdd(Ty, X, ScaleOffset, Flags);
  auto AdjustedX = B.buildSelect(Ty, NeedsScaling, ScaledX, X, Flags);

  auto Log2E = B.buildFConstant(Ty, numbers::log2e);
  auto ExpInput = B.buildFMul(Ty, AdjustedX, Log2E, Flags);

  auto Exp2 = B.buildIntrinsic(Intrinsic::amdgcn_exp2, {Ty})
    .addUse(ExpInput.getReg(0))
    .setMIFlags(Flags);

  auto ResultScaleFactor = B.buildFConstant(Ty, 0x1.969d48p-93f);
  auto AdjustedResult = B.buildFMul(Ty, Exp2, ResultScaleFactor, Flags);
  B.buildSelect(Dst, NeedsScaling, AdjustedResult, Exp2, Flags);
  return true;
}

bool AMDGPULegalizerInfo::legalizeFExp10Unsafe(MachineIRBuilder &B,
                                               Register Dst, Register X,
                                               unsigned Flags) const {
  LLT Ty = B.getMRI()->getType(Dst);

  if (Ty != F32 || !needsDenormHandlingF32(B.getMF(), X, Flags)) {
    // exp2(x * 0x1.a92000p+1f) * exp2(x * 0x1.4f0978p-11f);
    auto K0 = B.buildFConstant(Ty, 0x1.a92000p+1f);
    auto K1 = B.buildFConstant(Ty, 0x1.4f0978p-11f);

    auto Mul1 = B.buildFMul(Ty, X, K1, Flags);
    auto Exp2_1 = buildExp(B, Ty, Mul1, Flags);
    auto Mul0 = B.buildFMul(Ty, X, K0, Flags);
    auto Exp2_0 = buildExp(B, Ty, Mul0, Flags);
    B.buildFMul(Dst, Exp2_0, Exp2_1, Flags);
    return true;
  }

  // bool s = x < -0x1.2f7030p+5f;
  // x += s ? 0x1.0p+5f : 0.0f;
  // exp10 = exp2(x * 0x1.a92000p+1f) *
  //        exp2(x * 0x1.4f0978p-11f) *
  //        (s ? 0x1.9f623ep-107f : 1.0f);

  auto Threshold = B.buildFConstant(Ty, -0x1.2f7030p+5f);
  auto NeedsScaling =
      B.buildFCmp(CmpInst::FCMP_OLT, LLT::scalar(1), X, Threshold);

  auto ScaleOffset = B.buildFConstant(Ty, 0x1.0p+5f);
  auto ScaledX = B.buildFAdd(Ty, X, ScaleOffset, Flags);
  auto AdjustedX = B.buildSelect(Ty, NeedsScaling, ScaledX, X);

  auto K0 = B.buildFConstant(Ty, 0x1.a92000p+1f);
  auto K1 = B.buildFConstant(Ty, 0x1.4f0978p-11f);

  auto Mul1 = B.buildFMul(Ty, AdjustedX, K1, Flags);
  auto Exp2_1 = buildExp(B, Ty, Mul1, Flags);
  auto Mul0 = B.buildFMul(Ty, AdjustedX, K0, Flags);
  auto Exp2_0 = buildExp(B, Ty, Mul0, Flags);

  auto MulExps = B.buildFMul(Ty, Exp2_0, Exp2_1, Flags);
  auto ResultScaleFactor = B.buildFConstant(Ty, 0x1.9f623ep-107f);
  auto AdjustedResult = B.buildFMul(Ty, MulExps, ResultScaleFactor, Flags);

  B.buildSelect(Dst, NeedsScaling, AdjustedResult, MulExps);
  return true;
}

// This expansion gives a result slightly better than 1ulp.
bool AMDGPULegalizerInfo::legalizeFEXPF64(MachineInstr &MI,
                                          MachineIRBuilder &B) const {

  Register X = MI.getOperand(1).getReg();
  LLT I32 = LLT::integer(32);
  LLT S1 = LLT::scalar(1);

  // TODO: Check if reassoc is safe. There is an output change in exp2 and
  // exp10, which slightly increases ulp.
  unsigned Flags = MI.getFlags() & ~MachineInstr::FmReassoc;

  Register Dn, F, T;

  if (MI.getOpcode() == TargetOpcode::G_FEXP2) {
    // Dn = rint(X)
    Dn = B.buildFRint(F64, X, Flags).getReg(0);
    // F = X - Dn
    F = B.buildFSub(F64, X, Dn, Flags).getReg(0);
    // T = F*C1 + F*C2
    auto C1 = B.buildFConstant(F64, APFloat(0x1.62e42fefa39efp-1));
    auto C2 = B.buildFConstant(F64, APFloat(0x1.abc9e3b39803fp-56));
    auto Mul2 = B.buildFMul(F64, F, C2, Flags).getReg(0);
    T = B.buildFMA(F64, F, C1, Mul2, Flags).getReg(0);

  } else if (MI.getOpcode() == TargetOpcode::G_FEXP10) {
    auto C1 = B.buildFConstant(F64, APFloat(0x1.a934f0979a371p+1));
    auto Mul = B.buildFMul(F64, X, C1, Flags).getReg(0);
    Dn = B.buildFRint(F64, Mul, Flags).getReg(0);

    auto NegDn = B.buildFNeg(F64, Dn, Flags).getReg(0);
    auto C2 = B.buildFConstant(F64, APFloat(-0x1.9dc1da994fd21p-59));
    auto C3 = B.buildFConstant(F64, APFloat(0x1.34413509f79ffp-2));
    auto Inner = B.buildFMA(F64, NegDn, C3, X, Flags).getReg(0);
    F = B.buildFMA(F64, NegDn, C2, Inner, Flags).getReg(0);

    auto C4 = B.buildFConstant(F64, APFloat(0x1.26bb1bbb55516p+1));
    auto C5 = B.buildFConstant(F64, APFloat(-0x1.f48ad494ea3e9p-53));
    auto MulF = B.buildFMul(F64, F, C5, Flags).getReg(0);
    T = B.buildFMA(F64, F, C4, MulF, Flags).getReg(0);

  } else { // G_FEXP
    auto C1 = B.buildFConstant(F64, APFloat(0x1.71547652b82fep+0));
    auto Mul = B.buildFMul(F64, X, C1, Flags).getReg(0);
    Dn = B.buildFRint(F64, Mul, Flags).getReg(0);

    auto NegDn = B.buildFNeg(F64, Dn, Flags).getReg(0);
    auto C2 = B.buildFConstant(F64, APFloat(0x1.abc9e3b39803fp-56));
    auto C3 = B.buildFConstant(F64, APFloat(0x1.62e42fefa39efp-1));
    auto Inner = B.buildFMA(F64, NegDn, C3, X, Flags).getReg(0);
    T = B.buildFMA(F64, NegDn, C2, Inner, Flags).getReg(0);
  }

  // Polynomial chain for P
  auto P = B.buildFConstant(F64, 0x1.ade156a5dcb37p-26);
  P = B.buildFMA(F64, T, P, B.buildFConstant(F64, 0x1.28af3fca7ab0cp-22),
                 Flags);
  P = B.buildFMA(F64, T, P, B.buildFConstant(F64, 0x1.71dee623fde64p-19),
                 Flags);
  P = B.buildFMA(F64, T, P, B.buildFConstant(F64, 0x1.a01997c89e6b0p-16),
                 Flags);
  P = B.buildFMA(F64, T, P, B.buildFConstant(F64, 0x1.a01a014761f6ep-13),
                 Flags);
  P = B.buildFMA(F64, T, P, B.buildFConstant(F64, 0x1.6c16c1852b7b0p-10),
                 Flags);
  P = B.buildFMA(F64, T, P, B.buildFConstant(F64, 0x1.1111111122322p-7), Flags);
  P = B.buildFMA(F64, T, P, B.buildFConstant(F64, 0x1.55555555502a1p-5), Flags);
  P = B.buildFMA(F64, T, P, B.buildFConstant(F64, 0x1.5555555555511p-3), Flags);
  P = B.buildFMA(F64, T, P, B.buildFConstant(F64, 0x1.000000000000bp-1), Flags);

  auto One = B.buildFConstant(F64, 1.0);
  P = B.buildFMA(F64, T, P, One, Flags);
  P = B.buildFMA(F64, T, P, One, Flags);

  // Z = FLDEXP(P, (int)Dn)
  auto DnInt = B.buildFPTOSI(I32, Dn);
  auto Z = B.buildFLdexp(F64, P, DnInt, Flags);

  if (!(Flags & MachineInstr::FmNoInfs)) {
    // Overflow guard: if X <= 1024.0 then Z else +inf
    auto CondHi = B.buildFCmp(CmpInst::FCMP_ULE, S1, X,
                              B.buildFConstant(F64, APFloat(1024.0)));
    auto PInf = B.buildFConstant(F64, APFloat::getInf(APFloat::IEEEdouble()));
    Z = B.buildSelect(F64, CondHi, Z, PInf, Flags);
  }

  // Underflow guard: if X >= -1075.0 then Z else 0.0
  auto CondLo = B.buildFCmp(CmpInst::FCMP_UGE, S1, X,
                            B.buildFConstant(F64, APFloat(-1075.0)));
  auto Zero = B.buildFConstant(F64, APFloat(0.0));
  B.buildSelect(MI.getOperand(0).getReg(), CondLo, Z, Zero, Flags);

  MI.eraseFromParent();
  return true;
}

bool AMDGPULegalizerInfo::legalizeFExp(MachineInstr &MI,
                                       MachineIRBuilder &B) const {
  Register Dst = MI.getOperand(0).getReg();
  Register X = MI.getOperand(1).getReg();
  const unsigned Flags = MI.getFlags();
  MachineFunction &MF = B.getMF();
  MachineRegisterInfo &MRI = *B.getMRI();
  LLT Ty = MRI.getType(Dst);

  if (Ty == F64)
    return legalizeFEXPF64(MI, B);

  const bool IsExp10 = MI.getOpcode() == TargetOpcode::G_FEXP10;

  if (Ty == F16) {
    // v_exp_f16 (fmul x, log2e)
    if (allowApproxFunc(MF, Flags)) {
      // TODO: Does this really require fast?
      IsExp10 ? legalizeFExp10Unsafe(B, Dst, X, Flags)
              : legalizeFExpUnsafe(B, Dst, X, Flags);
      MI.eraseFromParent();
      return true;
    }

    // Nothing in half is a denormal when promoted to f32.
    //
    // exp(f16 x) ->
    //   fptrunc (v_exp_f32 (fmul (fpext x), log2e))
    //
    // exp10(f16 x) ->
    //   fptrunc (v_exp_f32 (fmul (fpext x), log2(10)))
    auto Ext = B.buildFPExt(F32, X, Flags);
    Register Lowered = MRI.createGenericVirtualRegister(F32);
    legalizeFExpUnsafeImpl(B, Lowered, Ext.getReg(0), Flags, IsExp10);
    B.buildFPTrunc(Dst, Lowered, Flags);
    MI.eraseFromParent();
    return true;
  }

  assert(Ty == F32);

  // TODO: Interpret allowApproxFunc as ignoring DAZ. This is currently copying
  // library behavior. Also, is known-not-daz source sufficient?
  if (allowApproxFunc(MF, Flags)) {
    IsExp10 ? legalizeFExp10Unsafe(B, Dst, X, Flags)
            : legalizeFExpUnsafe(B, Dst, X, Flags);
    MI.eraseFromParent();
    return true;
  }

  //    Algorithm:
  //
  //    e^x = 2^(x/ln(2)) = 2^(x*(64/ln(2))/64)
  //
  //    x*(64/ln(2)) = n + f, |f| <= 0.5, n is integer
  //    n = 64*m + j,   0 <= j < 64
  //
  //    e^x = 2^((64*m + j + f)/64)
  //        = (2^m) * (2^(j/64)) * 2^(f/64)
  //        = (2^m) * (2^(j/64)) * e^(f*(ln(2)/64))
  //
  //    f = x*(64/ln(2)) - n
  //    r = f*(ln(2)/64) = x - n*(ln(2)/64)
  //
  //    e^x = (2^m) * (2^(j/64)) * e^r
  //
  //    (2^(j/64)) is precomputed
  //
  //    e^r = 1 + r + (r^2)/2! + (r^3)/3! + (r^4)/4! + (r^5)/5!
  //    e^r = 1 + q
  //
  //    q = r + (r^2)/2! + (r^3)/3! + (r^4)/4! + (r^5)/5!
  //
  //    e^x = (2^m) * ( (2^(j/64)) + q*(2^(j/64)) )
  const unsigned FlagsNoContract = Flags & ~MachineInstr::FmContract;
  Register PH, PL;

  if (ST.hasFastFMAF32()) {
    const float c_exp = numbers::log2ef;
    const float cc_exp = 0x1.4ae0bep-26f; // c+cc are 49 bits
    const float c_exp10 = 0x1.a934f0p+1f;
    const float cc_exp10 = 0x1.2f346ep-24f;

    auto C = B.buildFConstant(Ty, IsExp10 ? c_exp10 : c_exp);
    PH = B.buildFMul(Ty, X, C, Flags).getReg(0);
    auto NegPH = B.buildFNeg(Ty, PH, Flags);
    auto FMA0 = B.buildFMA(Ty, X, C, NegPH, Flags);

    auto CC = B.buildFConstant(Ty, IsExp10 ? cc_exp10 : cc_exp);
    PL = B.buildFMA(Ty, X, CC, FMA0, Flags).getReg(0);
  } else {
    const float ch_exp = 0x1.714000p+0f;
    const float cl_exp = 0x1.47652ap-12f; // ch + cl are 36 bits

    const float ch_exp10 = 0x1.a92000p+1f;
    const float cl_exp10 = 0x1.4f0978p-11f;

    auto MaskConst = B.buildConstant(Ty, 0xfffff000);
    auto XH = B.buildAnd(Ty, X, MaskConst);
    auto XL = B.buildFSub(Ty, X, XH, Flags);

    auto CH = B.buildFConstant(Ty, IsExp10 ? ch_exp10 : ch_exp);
    PH = B.buildFMul(Ty, XH, CH, Flags).getReg(0);

    auto CL = B.buildFConstant(Ty, IsExp10 ? cl_exp10 : cl_exp);
    auto XLCL = B.buildFMul(Ty, XL, CL, Flags);

    Register Mad0 =
        getMad(B, Ty, XL.getReg(0), CH.getReg(0), XLCL.getReg(0), Flags);
    PL = getMad(B, Ty, XH.getReg(0), CL.getReg(0), Mad0, Flags);
  }

  auto E = B.buildIntrinsicRoundeven(Ty, PH, Flags);

  // It is unsafe to contract this fsub into the PH multiply.
  auto PHSubE = B.buildFSub(Ty, PH, E, FlagsNoContract);
  auto A = B.buildFAdd(Ty, PHSubE, PL, Flags);
  const LLT I32 = LLT::integer(32);
  auto IntE = B.buildFPTOSI(I32, E);

  auto Exp2 = B.buildIntrinsic(Intrinsic::amdgcn_exp2, {Ty})
                  .addUse(A.getReg(0))
                  .setMIFlags(Flags);
  auto R = B.buildFLdexp(Ty, Exp2, IntE, Flags);

  auto UnderflowCheckConst =
      B.buildFConstant(Ty, IsExp10 ? -0x1.66d3e8p+5f : -0x1.9d1da0p+6f);
  auto Zero = B.buildFConstant(Ty, 0.0);
  auto Underflow =
      B.buildFCmp(CmpInst::FCMP_OLT, LLT::scalar(1), X, UnderflowCheckConst);

  R = B.buildSelect(Ty, Underflow, Zero, R);

  if (!(Flags & MachineInstr::FmNoInfs)) {
    auto OverflowCheckConst =
        B.buildFConstant(Ty, IsExp10 ? 0x1.344136p+5f : 0x1.62e430p+6f);

    auto Overflow =
        B.buildFCmp(CmpInst::FCMP_OGT, LLT::scalar(1), X, OverflowCheckConst);
    auto Inf = B.buildFConstant(Ty, APFloat::getInf(APFloat::IEEEsingle()));
    R = B.buildSelect(Ty, Overflow, Inf, R, Flags);
  }

  B.buildCopy(Dst, R);
  MI.eraseFromParent();
  return true;
}

bool AMDGPULegalizerInfo::legalizeFPow(MachineInstr &MI,
                                       MachineIRBuilder &B) const {
  Register Dst = MI.getOperand(0).getReg();
  Register Src0 = MI.getOperand(1).getReg();
  Register Src1 = MI.getOperand(2).getReg();
  unsigned Flags = MI.getFlags();
  LLT Ty = B.getMRI()->getType(Dst);

  if (Ty == F32) {
    auto Log = B.buildFLog2(F32, Src0, Flags);
    auto Mul = B.buildIntrinsic(Intrinsic::amdgcn_fmul_legacy, {F32})
                   .addUse(Log.getReg(0))
                   .addUse(Src1)
                   .setMIFlags(Flags);
    B.buildFExp2(Dst, Mul, Flags);
  } else if (Ty == F16) {
    // There's no f16 fmul_legacy, so we need to convert for it.
    auto Log = B.buildFLog2(F16, Src0, Flags);
    auto Ext0 = B.buildFPExt(F32, Log, Flags);
    auto Ext1 = B.buildFPExt(F32, Src1, Flags);
    auto Mul = B.buildIntrinsic(Intrinsic::amdgcn_fmul_legacy, {F32})
                   .addUse(Ext0.getReg(0))
                   .addUse(Ext1.getReg(0))
                   .setMIFlags(Flags);
    B.buildFExp2(Dst, B.buildFPTrunc(F16, Mul), Flags);
  } else
    return false;

  MI.eraseFromParent();
  return true;
}

// Find a source register, ignoring any possible source modifiers.
static Register stripAnySourceMods(Register OrigSrc, MachineRegisterInfo &MRI) {
  Register ModSrc = OrigSrc;
  if (MachineInstr *SrcFNeg = getOpcodeDef(AMDGPU::G_FNEG, ModSrc, MRI)) {
    ModSrc = SrcFNeg->getOperand(1).getReg();
    if (MachineInstr *SrcFAbs = getOpcodeDef(AMDGPU::G_FABS, ModSrc, MRI))
      ModSrc = SrcFAbs->getOperand(1).getReg();
  } else if (MachineInstr *SrcFAbs = getOpcodeDef(AMDGPU::G_FABS, ModSrc, MRI))
    ModSrc = SrcFAbs->getOperand(1).getReg();
  return ModSrc;
}

bool AMDGPULegalizerInfo::legalizeFFloor(MachineInstr &MI,
                                         MachineRegisterInfo &MRI,
                                         MachineIRBuilder &B) const {

  const LLT S1 = LLT::scalar(1);
  Register Dst = MI.getOperand(0).getReg();
  Register OrigSrc = MI.getOperand(1).getReg();
  unsigned Flags = MI.getFlags();
  assert(ST.hasFractBug() && MRI.getType(Dst) == F64 &&
         "this should not have been custom lowered");

  // V_FRACT is buggy on SI, so the F32 version is never used and (x-floor(x))
  // is used instead. However, SI doesn't have V_FLOOR_F64, so the most
  // efficient way to implement it is using V_FRACT_F64. The workaround for the
  // V_FRACT bug is:
  //    fract(x) = isnan(x) ? x : min(V_FRACT(x), 0.99999999999999999)
  //
  // Convert floor(x) to (x - fract(x))

  auto Fract = B.buildIntrinsic(Intrinsic::amdgcn_fract, {F64})
                   .addUse(OrigSrc)
                   .setMIFlags(Flags);

  // Give source modifier matching some assistance before obscuring a foldable
  // pattern.

  // TODO: We can avoid the neg on the fract? The input sign to fract
  // shouldn't matter?
  Register ModSrc = stripAnySourceMods(OrigSrc, MRI);

  auto Const =
      B.buildFConstant(F64, llvm::bit_cast<double>(0x3fefffffffffffff));

  Register Min = MRI.createGenericVirtualRegister(F64);

  // We don't need to concern ourselves with the snan handling difference, so
  // use the one which will directly select.
  const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
  if (MFI->getMode().IEEE)
    B.buildFMinNumIEEE(Min, Fract, Const, Flags);
  else
    B.buildFMinNum(Min, Fract, Const, Flags);

  Register CorrectedFract = Min;
  if (!MI.getFlag(MachineInstr::FmNoNans)) {
    auto IsNan = B.buildFCmp(CmpInst::FCMP_ORD, S1, ModSrc, ModSrc, Flags);
    CorrectedFract = B.buildSelect(F64, IsNan, ModSrc, Min, Flags).getReg(0);
  }

  auto NegFract = B.buildFNeg(F64, CorrectedFract, Flags);
  B.buildFAdd(Dst, OrigSrc, NegFract, Flags);

  MI.eraseFromParent();
  return true;
}

// Turn an illegal packed v2i16/v2f16 build vector into bit operations.
// TODO: This should probably be a bitcast action in LegalizerHelper.
bool AMDGPULegalizerInfo::legalizeBuildVector(
  MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const {
  Register Dst = MI.getOperand(0).getReg();
  const LLT I32 = LLT::integer(32);
  const LLT I16 = LLT::integer(16);
  assert(MRI.getType(Dst).isVector() &&
         MRI.getType(Dst).getNumElements() == 2 &&
         MRI.getType(Dst).getScalarSizeInBits() == 16);

  Register Src0 = MI.getOperand(1).getReg();
  Register Src1 = MI.getOperand(2).getReg();

  if (MI.getOpcode() == AMDGPU::G_BUILD_VECTOR_TRUNC) {
    assert(MRI.getType(Src0) == I32);
    Src0 = B.buildTrunc(I16, MI.getOperand(1).getReg()).getReg(0);
    Src1 = B.buildTrunc(I16, MI.getOperand(2).getReg()).getReg(0);
  }

  auto Merge = B.buildMergeLikeInstr(I32, {Src0, Src1});
  B.buildBitcast(Dst, Merge);

  MI.eraseFromParent();
  return true;
}

// Build a big integer multiply or multiply-add using MAD_64_32 instructions.
//
// Source and accumulation registers must all be 32-bits.
//
// TODO: When the multiply is uniform, we should produce a code sequence
// that is better suited to instruction selection on the SALU. Instead of
// the outer loop going over parts of the result, the outer loop should go
// over parts of one of the factors. This should result in instruction
// selection that makes full use of S_ADDC_U32 instructions.
void AMDGPULegalizerInfo::buildMultiply(LegalizerHelper &Helper,
                                        MutableArrayRef<Register> Accum,
                                        ArrayRef<Register> Src0,
                                        ArrayRef<Register> Src1,
                                        bool UsePartialMad64_32,
                                        bool SeparateOddAlignedProducts) const {
  // Use (possibly empty) vectors of S1 registers to represent the set of
  // carries from one pair of positions to the next.
  using Carry = SmallVector<Register, 2>;

  MachineIRBuilder &B = Helper.MIRBuilder;
  GISelValueTracking &VT = *Helper.getValueTracking();

  const LLT S1 = LLT::scalar(1);
  const LLT I32 = LLT::integer(32);
  const LLT I64 = LLT::integer(64);

  Register Zero32;
  Register Zero64;

  auto getZero32 = [&]() -> Register {
    if (!Zero32)
      Zero32 = B.buildConstant(I32, 0).getReg(0);
    return Zero32;
  };
  auto getZero64 = [&]() -> Register {
    if (!Zero64)
      Zero64 = B.buildConstant(I64, 0).getReg(0);
    return Zero64;
  };

  SmallVector<bool, 2> Src0KnownZeros, Src1KnownZeros;
  for (unsigned i = 0; i < Src0.size(); ++i) {
    Src0KnownZeros.push_back(VT.getKnownBits(Src0[i]).isZero());
    Src1KnownZeros.push_back(VT.getKnownBits(Src1[i]).isZero());
  }

  // Merge the given carries into the 32-bit LocalAccum, which is modified
  // in-place.
  //
  // Returns the carry-out, which is a single S1 register or null.
  auto mergeCarry =
      [&](Register &LocalAccum, const Carry &CarryIn) -> Register {
        if (CarryIn.empty())
          return Register();

        bool HaveCarryOut = true;
        Register CarryAccum;
        if (CarryIn.size() == 1) {
          if (!LocalAccum) {
            LocalAccum = B.buildZExt(I32, CarryIn[0]).getReg(0);
            return Register();
          }

          CarryAccum = getZero32();
        } else {
          CarryAccum = B.buildZExt(I32, CarryIn[0]).getReg(0);
          for (unsigned i = 1; i + 1 < CarryIn.size(); ++i) {
            CarryAccum =
                B.buildUAdde(I32, S1, CarryAccum, getZero32(), CarryIn[i])
                    .getReg(0);
          }

          if (!LocalAccum) {
            LocalAccum = getZero32();
            HaveCarryOut = false;
          }
        }

        auto Add =
            B.buildUAdde(I32, S1, CarryAccum, LocalAccum, CarryIn.back());
        LocalAccum = Add.getReg(0);
        return HaveCarryOut ? Add.getReg(1) : Register();
      };

  // Build a multiply-add chain to compute
  //
  //   LocalAccum + (partial products at DstIndex)
  //       + (opportunistic subset of CarryIn)
  //
  // LocalAccum is an array of one or two 32-bit registers that are updated
  // in-place. The incoming registers may be null.
  //
  // In some edge cases, carry-ins can be consumed "for free". In that case,
  // the consumed carry bits are removed from CarryIn in-place.
  auto buildMadChain =
      [&](MutableArrayRef<Register> LocalAccum, unsigned DstIndex, Carry &CarryIn)
          -> Carry {
        assert((DstIndex + 1 < Accum.size() && LocalAccum.size() == 2) ||
               (DstIndex + 1 >= Accum.size() && LocalAccum.size() == 1));

        Carry CarryOut;
        unsigned j0 = 0;

        // Use plain 32-bit multiplication for the most significant part of the
        // result by default.
        if (LocalAccum.size() == 1 &&
            (!UsePartialMad64_32 || !CarryIn.empty())) {
          do {
            // Skip multiplication if one of the operands is 0
            unsigned j1 = DstIndex - j0;
            if (Src0KnownZeros[j0] || Src1KnownZeros[j1]) {
              ++j0;
              continue;
            }
            auto Mul = B.buildMul(I32, Src0[j0], Src1[j1]);
            if (!LocalAccum[0] || VT.getKnownBits(LocalAccum[0]).isZero()) {
              LocalAccum[0] = Mul.getReg(0);
            } else {
              if (CarryIn.empty()) {
                LocalAccum[0] = B.buildAdd(I32, LocalAccum[0], Mul).getReg(0);
              } else {
                LocalAccum[0] =
                    B.buildUAdde(I32, S1, LocalAccum[0], Mul, CarryIn.back())
                        .getReg(0);
                CarryIn.pop_back();
              }
            }
            ++j0;
          } while (j0 <= DstIndex && (!UsePartialMad64_32 || !CarryIn.empty()));
        }

        // Build full 64-bit multiplies.
        if (j0 <= DstIndex) {
          bool HaveSmallAccum = false;
          Register Tmp;

          if (LocalAccum[0]) {
            if (LocalAccum.size() == 1) {
              Tmp = B.buildAnyExt(I64, LocalAccum[0]).getReg(0);
              HaveSmallAccum = true;
            } else if (LocalAccum[1]) {
              Tmp = B.buildMergeLikeInstr(I64, LocalAccum).getReg(0);
              HaveSmallAccum = false;
            } else {
              Tmp = B.buildZExt(I64, LocalAccum[0]).getReg(0);
              HaveSmallAccum = true;
            }
          } else {
            assert(LocalAccum.size() == 1 || !LocalAccum[1]);
            Tmp = getZero64();
            HaveSmallAccum = true;
          }

          do {
            unsigned j1 = DstIndex - j0;
            if (Src0KnownZeros[j0] || Src1KnownZeros[j1]) {
              ++j0;
              continue;
            }
            auto Mad = B.buildInstr(AMDGPU::G_AMDGPU_MAD_U64_U32, {I64, S1},
                                    {Src0[j0], Src1[j1], Tmp});
            Tmp = Mad.getReg(0);
            if (!HaveSmallAccum)
              CarryOut.push_back(Mad.getReg(1));
            HaveSmallAccum = false;

            ++j0;
          } while (j0 <= DstIndex);

          auto Unmerge = B.buildUnmerge(I32, Tmp);
          LocalAccum[0] = Unmerge.getReg(0);
          if (LocalAccum.size() > 1)
            LocalAccum[1] = Unmerge.getReg(1);
        }

        return CarryOut;
      };

  // Outer multiply loop, iterating over destination parts from least
  // significant to most significant parts.
  //
  // The columns of the following diagram correspond to the destination parts
  // affected by one iteration of the outer loop (ignoring boundary
  // conditions).
  //
  //   Dest index relative to 2 * i:      1 0 -1
  //                                      ------
  //   Carries from previous iteration:     e o
  //   Even-aligned partial product sum:  E E .
  //   Odd-aligned partial product sum:     O O
  //
  // 'o' is OddCarry, 'e' is EvenCarry.
  // EE and OO are computed from partial products via buildMadChain and use
  // accumulation where possible and appropriate.
  //
  Register SeparateOddCarry;
  Carry EvenCarry;
  Carry OddCarry;

  for (unsigned i = 0; i <= Accum.size() / 2; ++i) {
    Carry OddCarryIn = std::move(OddCarry);
    Carry EvenCarryIn = std::move(EvenCarry);
    OddCarry.clear();
    EvenCarry.clear();

    // Partial products at offset 2 * i.
    if (2 * i < Accum.size()) {
      auto LocalAccum = Accum.drop_front(2 * i).take_front(2);
      EvenCarry = buildMadChain(LocalAccum, 2 * i, EvenCarryIn);
    }

    // Partial products at offset 2 * i - 1.
    if (i > 0) {
      if (!SeparateOddAlignedProducts) {
        auto LocalAccum = Accum.drop_front(2 * i - 1).take_front(2);
        OddCarry = buildMadChain(LocalAccum, 2 * i - 1, OddCarryIn);
      } else {
        bool IsHighest = 2 * i >= Accum.size();
        Register SeparateOddOut[2];
        auto LocalAccum = MutableArrayRef(SeparateOddOut)
                              .take_front(IsHighest ? 1 : 2);
        OddCarry = buildMadChain(LocalAccum, 2 * i - 1, OddCarryIn);

        MachineInstr *Lo;

        if (i == 1) {
          if (!IsHighest)
            Lo = B.buildUAddo(I32, S1, Accum[2 * i - 1], SeparateOddOut[0]);
          else
            Lo = B.buildAdd(I32, Accum[2 * i - 1], SeparateOddOut[0]);
        } else {
          Lo = B.buildUAdde(I32, S1, Accum[2 * i - 1], SeparateOddOut[0],
                            SeparateOddCarry);
        }
        Accum[2 * i - 1] = Lo->getOperand(0).getReg();

        if (!IsHighest) {
          auto Hi = B.buildUAdde(I32, S1, Accum[2 * i], SeparateOddOut[1],
                                 Lo->getOperand(1).getReg());
          Accum[2 * i] = Hi.getReg(0);
          SeparateOddCarry = Hi.getReg(1);
        }
      }
    }

    // Add in the carries from the previous iteration
    if (i > 0) {
      if (Register CarryOut = mergeCarry(Accum[2 * i - 1], OddCarryIn))
        EvenCarryIn.push_back(CarryOut);

      if (2 * i < Accum.size()) {
        if (Register CarryOut = mergeCarry(Accum[2 * i], EvenCarryIn))
          OddCarry.push_back(CarryOut);
      }
    }
  }
}

// Custom narrowing of wide multiplies using wide multiply-add instructions.
//
// TODO: If the multiply is followed by an addition, we should attempt to
// integrate it to make better use of V_MAD_U64_U32's multiply-add capabilities.
bool AMDGPULegalizerInfo::legalizeMul(LegalizerHelper &Helper,
                                      MachineInstr &MI) const {
  assert(ST.hasMad64_32());
  assert(MI.getOpcode() == TargetOpcode::G_MUL);

  MachineIRBuilder &B = Helper.MIRBuilder;
  MachineRegisterInfo &MRI = *B.getMRI();

  Register DstReg = MI.getOperand(0).getReg();
  Register Src0 = MI.getOperand(1).getReg();
  Register Src1 = MI.getOperand(2).getReg();

  LLT Ty = MRI.getType(DstReg);
  assert(Ty.isScalar());

  unsigned Size = Ty.getSizeInBits();
  if (ST.hasVMulU64Inst() && Size == 64)
    return true;

  unsigned NumParts = Size / 32;
  assert((Size % 32) == 0);
  assert(NumParts >= 2);

  // Whether to use MAD_64_32 for partial products whose high half is
  // discarded. This avoids some ADD instructions but risks false dependency
  // stalls on some subtargets in some cases.
  const bool UsePartialMad64_32 = ST.getGeneration() < AMDGPUSubtarget::GFX10;

  // Whether to compute odd-aligned partial products separately. This is
  // advisable on subtargets where the accumulator of MAD_64_32 must be placed
  // in an even-aligned VGPR.
  const bool SeparateOddAlignedProducts = ST.hasFullRate64Ops();

  LLT I32 = LLT::integer(32);
  SmallVector<Register, 2> Src0Parts, Src1Parts;
  for (unsigned i = 0; i < NumParts; ++i) {
    Src0Parts.push_back(MRI.createGenericVirtualRegister(I32));
    Src1Parts.push_back(MRI.createGenericVirtualRegister(I32));
  }
  B.buildUnmerge(Src0Parts, Src0);
  B.buildUnmerge(Src1Parts, Src1);

  SmallVector<Register, 2> AccumRegs(NumParts);
  buildMultiply(Helper, AccumRegs, Src0Parts, Src1Parts, UsePartialMad64_32,
                SeparateOddAlignedProducts);

  B.buildMergeLikeInstr(DstReg, AccumRegs);
  MI.eraseFromParent();
  return true;
}

// Legalize ctlz/cttz to ffbh/ffbl instead of the default legalization to
// ctlz/cttz_zero_poison. This allows us to fix up the result for the zero input
// case with a single min instruction instead of a compare+select.
bool AMDGPULegalizerInfo::legalizeCTLZ_CTTZ(MachineInstr &MI,
                                            MachineRegisterInfo &MRI,
                                            MachineIRBuilder &B) const {
  Register Dst = MI.getOperand(0).getReg();
  Register Src = MI.getOperand(1).getReg();
  LLT DstTy = MRI.getType(Dst);
  LLT SrcTy = MRI.getType(Src);

  unsigned NewOpc = MI.getOpcode() == AMDGPU::G_CTLZ
                        ? AMDGPU::G_AMDGPU_FFBH_U32
                        : AMDGPU::G_AMDGPU_FFBL_B32;
  auto Tmp = B.buildInstr(NewOpc, {DstTy}, {Src});
  B.buildUMin(Dst, Tmp, B.buildConstant(DstTy, SrcTy.getSizeInBits()));

  MI.eraseFromParent();
  return true;
}

bool AMDGPULegalizerInfo::legalizeCTLZ_ZERO_POISON(MachineInstr &MI,
                                                   MachineRegisterInfo &MRI,
                                                   MachineIRBuilder &B) const {
  Register Dst = MI.getOperand(0).getReg();
  Register Src = MI.getOperand(1).getReg();
  LLT SrcTy = MRI.getType(Src);
  TypeSize NumBits = SrcTy.getSizeInBits();

  assert(NumBits < 32u);

  const LLT I32 = LLT::integer(32);
  auto ShiftAmt = B.buildConstant(I32, 32u - NumBits);
  auto Extend = B.buildAnyExt(I32, {Src}).getReg(0u);
  auto Shift = B.buildShl(I32, Extend, ShiftAmt);
  auto Ctlz = B.buildInstr(AMDGPU::G_AMDGPU_FFBH_U32, {I32}, {Shift});
  B.buildTrunc(Dst, Ctlz);
  MI.eraseFromParent();
  return true;
}

bool AMDGPULegalizerInfo::legalizeCTLS(MachineInstr &MI,
                                       MachineRegisterInfo &MRI,
                                       MachineIRBuilder &B) const {
  Register Dst = MI.getOperand(0).getReg();
  Register Src = MI.getOperand(1).getReg();
  LLT SrcTy = MRI.getType(Src);
  const LLT I32 = LLT::integer(32);
  assert(SrcTy == I32 && "legalizeCTLS only supports i32");
  unsigned BitWidth = SrcTy.getSizeInBits();

  auto Sffbh = B.buildIntrinsic(Intrinsic::amdgcn_sffbh, {I32}).addUse(Src);
  auto Clamped = B.buildUMin(I32, Sffbh, B.buildConstant(I32, BitWidth));
  B.buildSub(Dst, Clamped, B.buildConstant(I32, 1));
  MI.eraseFromParent();
  return true;
}

// Check that this is a G_XOR x, -1
static bool isNot(const MachineRegisterInfo &MRI, const MachineInstr &MI) {
  if (MI.getOpcode() != TargetOpcode::G_XOR)
    return false;
  auto ConstVal = getIConstantVRegSExtVal(MI.getOperand(2).getReg(), MRI);
  return ConstVal == -1;
}

// Return the use branch instruction, otherwise null if the usage is invalid.
static MachineInstr *
verifyCFIntrinsic(MachineInstr &MI, MachineRegisterInfo &MRI, MachineInstr *&Br,
                  MachineBasicBlock *&UncondBrTarget, bool &Negated) {
  Register CondDef = MI.getOperand(0).getReg();
  if (!MRI.hasOneNonDBGUse(CondDef))
    return nullptr;

  MachineBasicBlock *Parent = MI.getParent();
  MachineInstr *UseMI = &*MRI.use_instr_nodbg_begin(CondDef);

  if (isNot(MRI, *UseMI)) {
    Register NegatedCond = UseMI->getOperand(0).getReg();
    if (!MRI.hasOneNonDBGUse(NegatedCond))
      return nullptr;

    // We're deleting the def of this value, so we need to remove it.
    eraseInstr(*UseMI, MRI);

    UseMI = &*MRI.use_instr_nodbg_begin(NegatedCond);
    Negated = true;
  }

  if (UseMI->getParent() != Parent || UseMI->getOpcode() != AMDGPU::G_BRCOND)
    return nullptr;

  // Make sure the cond br is followed by a G_BR, or is the last instruction.
  MachineBasicBlock::iterator Next = std::next(UseMI->getIterator());
  if (Next == Parent->end()) {
    MachineFunction::iterator NextMBB = std::next(Parent->getIterator());
    if (NextMBB == Parent->getParent()->end()) // Illegal intrinsic use.
      return nullptr;
    UncondBrTarget = &*NextMBB;
  } else {
    if (Next->getOpcode() != AMDGPU::G_BR)
      return nullptr;
    Br = &*Next;
    UncondBrTarget = Br->getOperand(0).getMBB();
  }

  return UseMI;
}

void AMDGPULegalizerInfo::buildLoadInputValue(Register DstReg,
                                              MachineIRBuilder &B,
                                              const ArgDescriptor *Arg,
                                              const TargetRegisterClass *ArgRC,
                                              LLT ArgTy) const {
  MCRegister SrcReg = Arg->getRegister();
  assert(SrcReg.isPhysical() && "Physical register expected");
  assert(DstReg.isVirtual() && "Virtual register expected");

  Register LiveIn = getFunctionLiveInPhysReg(B.getMF(), B.getTII(), SrcReg,
                                             *ArgRC, B.getDebugLoc(), ArgTy);
  if (Arg->isMasked()) {
    // TODO: Should we try to emit this once in the entry block?
    const LLT I32 = LLT::integer(32);
    const unsigned Mask = Arg->getMask();
    const unsigned Shift = llvm::countr_zero<unsigned>(Mask);

    Register AndMaskSrc = LiveIn;

    // TODO: Avoid clearing the high bits if we know workitem id y/z are always
    // 0.
    if (Shift != 0) {
      auto ShiftAmt = B.buildConstant(I32, Shift);
      AndMaskSrc = B.buildLShr(I32, LiveIn, ShiftAmt).getReg(0);
    }

    B.buildAnd(DstReg, AndMaskSrc, B.buildConstant(I32, Mask >> Shift));
  } else {
    B.buildCopy(DstReg, LiveIn);
  }
}

bool AMDGPULegalizerInfo::legalizeWorkGroupId(
    MachineInstr &MI, MachineIRBuilder &B,
    AMDGPUFunctionArgInfo::PreloadedValue WorkGroupIdPV,
    AMDGPUFunctionArgInfo::PreloadedValue ClusterMaxIdPV,
    AMDGPUFunctionArgInfo::PreloadedValue ClusterWorkGroupIdPV) const {
  Register DstReg = MI.getOperand(0).getReg();
  if (!ST.hasClusters()) {
    if (!loadInputValue(DstReg, B, WorkGroupIdPV))
      return false;
    MI.eraseFromParent();
    return true;
  }

  // Clusters are supported. Return the global position in the grid. If clusters
  // are enabled, WorkGroupIdPV returns the cluster ID not the workgroup ID.

  // WorkGroupIdXYZ = ClusterId == 0 ?
  //   ClusterIdXYZ :
  //   ClusterIdXYZ * (ClusterMaxIdXYZ + 1) + ClusterWorkGroupIdXYZ
  MachineRegisterInfo &MRI = *B.getMRI();
  const LLT I32 = LLT::integer(32);
  Register ClusterIdXYZ = MRI.createGenericVirtualRegister(I32);
  Register ClusterMaxIdXYZ = MRI.createGenericVirtualRegister(I32);
  Register ClusterWorkGroupIdXYZ = MRI.createGenericVirtualRegister(I32);
  if (!loadInputValue(ClusterIdXYZ, B, WorkGroupIdPV) ||
      !loadInputValue(ClusterWorkGroupIdXYZ, B, ClusterWorkGroupIdPV) ||
      !loadInputValue(ClusterMaxIdXYZ, B, ClusterMaxIdPV))
    return false;

  auto One = B.buildConstant(I32, 1);
  auto ClusterSizeXYZ = B.buildAdd(I32, ClusterMaxIdXYZ, One);
  auto GlobalIdXYZ = B.buildAdd(I32, ClusterWorkGroupIdXYZ,
                                B.buildMul(I32, ClusterIdXYZ, ClusterSizeXYZ));

  const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();

  switch (MFI->getClusterDims().getKind()) {
  case AMDGPU::ClusterDimsAttr::Kind::FixedDims:
  case AMDGPU::ClusterDimsAttr::Kind::VariableDims: {
    B.buildCopy(DstReg, GlobalIdXYZ);
    MI.eraseFromParent();
    return true;
  }
  case AMDGPU::ClusterDimsAttr::Kind::NoCluster: {
    B.buildCopy(DstReg, ClusterIdXYZ);
    MI.eraseFromParent();
    return true;
  }
  case AMDGPU::ClusterDimsAttr::Kind::Unknown: {
    using namespace AMDGPU::Hwreg;
    unsigned ClusterIdField = HwregEncoding::encode(ID_IB_STS2, 6, 4);
    Register ClusterId = MRI.createGenericVirtualRegister(I32);
    MRI.setRegClass(ClusterId, &AMDGPU::SReg_32RegClass);
    B.buildInstr(AMDGPU::S_GETREG_B32_const)
        .addDef(ClusterId)
        .addImm(ClusterIdField);
    auto Zero = B.buildConstant(I32, 0);
    auto NoClusters =
        B.buildICmp(CmpInst::ICMP_EQ, LLT::scalar(1), ClusterId, Zero);
    B.buildSelect(DstReg, NoClusters, ClusterIdXYZ, GlobalIdXYZ);
    MI.eraseFromParent();
    return true;
  }
  }

  llvm_unreachable("nothing should reach here");
}

bool AMDGPULegalizerInfo::loadInputValue(
    Register DstReg, MachineIRBuilder &B,
    AMDGPUFunctionArgInfo::PreloadedValue ArgType) const {
  const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
  const ArgDescriptor *Arg = nullptr;
  const TargetRegisterClass *ArgRC = nullptr;
  LLT ArgTy;

  CallingConv::ID CC = B.getMF().getFunction().getCallingConv();
  const ArgDescriptor WorkGroupIDX =
      ArgDescriptor::createRegister(AMDGPU::TTMP9);
  // If GridZ is not programmed in an entry function then the hardware will set
  // it to all zeros, so there is no need to mask the GridY value in the low
  // order bits.
  const ArgDescriptor WorkGroupIDY = ArgDescriptor::createRegister(
      AMDGPU::TTMP7,
      AMDGPU::isEntryFunctionCC(CC) && !MFI->hasWorkGroupIDZ() ? ~0u : 0xFFFFu);
  const ArgDescriptor WorkGroupIDZ =
      ArgDescriptor::createRegister(AMDGPU::TTMP7, 0xFFFF0000u);
  const ArgDescriptor ClusterWorkGroupIDX =
      ArgDescriptor::createRegister(AMDGPU::TTMP6, 0x0000000Fu);
  const ArgDescriptor ClusterWorkGroupIDY =
      ArgDescriptor::createRegister(AMDGPU::TTMP6, 0x000000F0u);
  const ArgDescriptor ClusterWorkGroupIDZ =
      ArgDescriptor::createRegister(AMDGPU::TTMP6, 0x00000F00u);
  const ArgDescriptor ClusterWorkGroupMaxIDX =
      ArgDescriptor::createRegister(AMDGPU::TTMP6, 0x0000F000u);
  const ArgDescriptor ClusterWorkGroupMaxIDY =
      ArgDescriptor::createRegister(AMDGPU::TTMP6, 0x000F0000u);
  const ArgDescriptor ClusterWorkGroupMaxIDZ =
      ArgDescriptor::createRegister(AMDGPU::TTMP6, 0x00F00000u);
  const ArgDescriptor ClusterWorkGroupMaxFlatID =
      ArgDescriptor::createRegister(AMDGPU::TTMP6, 0x0F000000u);

  auto LoadConstant = [&](unsigned N) {
    B.buildConstant(DstReg, N);
    return true;
  };

  if (ST.hasArchitectedSGPRs() &&
      (AMDGPU::isCompute(CC) || CC == CallingConv::AMDGPU_Gfx)) {
    AMDGPU::ClusterDimsAttr ClusterDims = MFI->getClusterDims();
    bool HasFixedDims = ClusterDims.isFixedDims();

    switch (ArgType) {
    case AMDGPUFunctionArgInfo::WORKGROUP_ID_X:
      Arg = &WorkGroupIDX;
      ArgRC = &AMDGPU::SReg_32RegClass;
      ArgTy = LLT::integer(32);
      break;
    case AMDGPUFunctionArgInfo::WORKGROUP_ID_Y:
      Arg = &WorkGroupIDY;
      ArgRC = &AMDGPU::SReg_32RegClass;
      ArgTy = LLT::integer(32);
      break;
    case AMDGPUFunctionArgInfo::WORKGROUP_ID_Z:
      Arg = &WorkGroupIDZ;
      ArgRC = &AMDGPU::SReg_32RegClass;
      ArgTy = LLT::integer(32);
      break;
    case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_X:
      if (HasFixedDims && ClusterDims.getDims()[0] == 1)
        return LoadConstant(0);
      Arg = &ClusterWorkGroupIDX;
      ArgRC = &AMDGPU::SReg_32RegClass;
      ArgTy = LLT::integer(32);
      break;
    case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_Y:
      if (HasFixedDims && ClusterDims.getDims()[1] == 1)
        return LoadConstant(0);
      Arg = &ClusterWorkGroupIDY;
      ArgRC = &AMDGPU::SReg_32RegClass;
      ArgTy = LLT::integer(32);
      break;
    case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_Z:
      if (HasFixedDims && ClusterDims.getDims()[2] == 1)
        return LoadConstant(0);
      Arg = &ClusterWorkGroupIDZ;
      ArgRC = &AMDGPU::SReg_32RegClass;
      ArgTy = LLT::integer(32);
      break;
    case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_X:
      if (HasFixedDims)
        return LoadConstant(ClusterDims.getDims()[0] - 1);
      Arg = &ClusterWorkGroupMaxIDX;
      ArgRC = &AMDGPU::SReg_32RegClass;
      ArgTy = LLT::integer(32);
      break;
    case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_Y:
      if (HasFixedDims)
        return LoadConstant(ClusterDims.getDims()[1] - 1);
      Arg = &ClusterWorkGroupMaxIDY;
      ArgRC = &AMDGPU::SReg_32RegClass;
      ArgTy = LLT::integer(32);
      break;
    case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_Z:
      if (HasFixedDims)
        return LoadConstant(ClusterDims.getDims()[2] - 1);
      Arg = &ClusterWorkGroupMaxIDZ;
      ArgRC = &AMDGPU::SReg_32RegClass;
      ArgTy = LLT::integer(32);
      break;
    case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_FLAT_ID:
      Arg = &ClusterWorkGroupMaxFlatID;
      ArgRC = &AMDGPU::SReg_32RegClass;
      ArgTy = LLT::integer(32);
      break;
    default:
      break;
    }
  }

  if (!Arg)
    std::tie(Arg, ArgRC, ArgTy) = MFI->getPreloadedValue(ArgType);

  if (!Arg) {
    if (ArgType == AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR) {
      // The intrinsic may appear when we have a 0 sized kernarg segment, in
      // which case the pointer argument may be missing and we use null.
      return LoadConstant(0);
    }

    // It's undefined behavior if a function marked with the amdgpu-no-*
    // attributes uses the corresponding intrinsic.
    B.buildUndef(DstReg);
    return true;
  }

  if (!Arg->isRegister() || !Arg->getRegister().isValid())
    return false; // TODO: Handle these
  buildLoadInputValue(DstReg, B, Arg, ArgRC, ArgTy);
  return true;
}

bool AMDGPULegalizerInfo::legalizePreloadedArgIntrin(
    MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B,
    AMDGPUFunctionArgInfo::PreloadedValue ArgType) const {
  if (!loadInputValue(MI.getOperand(0).getReg(), B, ArgType))
    return false;

  MI.eraseFromParent();
  return true;
}

static bool replaceWithConstant(MachineIRBuilder &B, MachineInstr &MI,
                                int64_t C) {
  B.buildConstant(MI.getOperand(0).getReg(), C);
  MI.eraseFromParent();
  return true;
}

bool AMDGPULegalizerInfo::legalizeWorkitemIDIntrinsic(
    MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B,
    unsigned Dim, AMDGPUFunctionArgInfo::PreloadedValue ArgType) const {
  unsigned MaxID = ST.getMaxWorkitemID(B.getMF().getFunction(), Dim);
  if (MaxID == 0)
    return replaceWithConstant(B, MI, 0);

  const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
  const ArgDescriptor *Arg;
  const TargetRegisterClass *ArgRC;
  LLT ArgTy;
  std::tie(Arg, ArgRC, ArgTy) = MFI->getPreloadedValue(ArgType);

  Register DstReg = MI.getOperand(0).getReg();
  if (!Arg) {
    // It's undefined behavior if a function marked with the amdgpu-no-*
    // attributes uses the corresponding intrinsic.
    B.buildUndef(DstReg);
    MI.eraseFromParent();
    return true;
  }

  if (Arg->isMasked()) {
    // Don't bother inserting AssertZext for packed IDs since we're emitting the
    // masking operations anyway.
    //
    // TODO: We could assert the top bit is 0 for the source copy.
    if (!loadInputValue(DstReg, B, ArgType))
      return false;
  } else {
    Register TmpReg = MRI.createGenericVirtualRegister(LLT::integer(32));
    if (!loadInputValue(TmpReg, B, ArgType))
      return false;
    B.buildAssertZExt(DstReg, TmpReg, llvm::bit_width(MaxID));
  }

  MI.eraseFromParent();
  return true;
}

MachinePointerInfo
AMDGPULegalizerInfo::getKernargSegmentPtrInfo(MachineFunction &MF) const {
  // This isn't really a constant pool but close enough.
  MachinePointerInfo PtrInfo(MF.getPSVManager().getConstantPool());
  PtrInfo.AddrSpace = AMDGPUAS::CONSTANT_ADDRESS;
  return PtrInfo;
}

Register AMDGPULegalizerInfo::getKernargParameterPtr(MachineIRBuilder &B,
                                                     int64_t Offset) const {
  LLT PtrTy = LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64);
  Register KernArgReg = B.getMRI()->createGenericVirtualRegister(PtrTy);

  // TODO: If we passed in the base kernel offset we could have a better
  // alignment than 4, but we don't really need it.
  if (!loadInputValue(KernArgReg, B,
                      AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR))
    llvm_unreachable("failed to find kernarg segment ptr");

  auto COffset = B.buildConstant(LLT::integer(64), Offset);
  return B.buildObjectPtrOffset(PtrTy, KernArgReg, COffset).getReg(0);
}

/// Legalize a value that's loaded from kernel arguments. This is only used by
/// legacy intrinsics.
bool AMDGPULegalizerInfo::legalizeKernargMemParameter(MachineInstr &MI,
                                                      MachineIRBuilder &B,
                                                      uint64_t Offset,
                                                      Align Alignment) const {
  Register DstReg = MI.getOperand(0).getReg();

  assert(B.getMRI()->getType(DstReg) == LLT::integer(32) &&
         "unexpected kernarg parameter type");

  Register Ptr = getKernargParameterPtr(B, Offset);
  MachinePointerInfo PtrInfo = getKernargSegmentPtrInfo(B.getMF());
  B.buildLoad(DstReg, Ptr, PtrInfo.getWithOffset(Offset), Align(4),
              MachineMemOperand::MODereferenceable |
                  MachineMemOperand::MOInvariant);
  MI.eraseFromParent();
  return true;
}

bool AMDGPULegalizerInfo::legalizeFDIV(MachineInstr &MI,
                                       MachineRegisterInfo &MRI,
                                       MachineIRBuilder &B) const {
  Register Dst = MI.getOperand(0).getReg();
  LLT DstTy = MRI.getType(Dst);

  if (DstTy == F16)
    return legalizeFDIV16(MI, MRI, B);
  if (DstTy == F32)
    return legalizeFDIV32(MI, MRI, B);
  if (DstTy == F64)
    return legalizeFDIV64(MI, MRI, B);

  return false;
}

void AMDGPULegalizerInfo::legalizeUnsignedDIV_REM32Impl(MachineIRBuilder &B,
                                                        Register DstDivReg,
                                                        Register DstRemReg,
                                                        Register X,
                                                        Register Y) const {
  const LLT S1 = LLT::scalar(1);
  const LLT I32 = LLT::integer(32);

  // See AMDGPUCodeGenPrepare::expandDivRem32 for a description of the
  // algorithm used here.

  // Initial estimate of inv(y).
  auto FloatY = B.buildUITOFP(F32, Y);
  auto RcpIFlag = B.buildInstr(AMDGPU::G_AMDGPU_RCP_IFLAG, {F32}, {FloatY});
  auto Scale = B.buildFConstant(F32, llvm::bit_cast<float>(0x4f7ffffe));
  auto ScaledY = B.buildFMul(F32, RcpIFlag, Scale);
  auto Z = B.buildFPTOUI(I32, ScaledY);

  // One round of UNR.
  auto NegY = B.buildSub(I32, B.buildConstant(I32, 0), Y);
  auto NegYZ = B.buildMul(I32, NegY, Z);
  Z = B.buildAdd(I32, Z, B.buildUMulH(I32, Z, NegYZ));

  // Quotient/remainder estimate.
  auto Q = B.buildUMulH(I32, X, Z);
  auto R = B.buildSub(I32, X, B.buildMul(I32, Q, Y));

  // First quotient/remainder refinement.
  auto One = B.buildConstant(I32, 1);
  auto Cond = B.buildICmp(CmpInst::ICMP_UGE, S1, R, Y);
  if (DstDivReg)
    Q = B.buildSelect(I32, Cond, B.buildAdd(I32, Q, One), Q);
  R = B.buildSelect(I32, Cond, B.buildSub(I32, R, Y), R);

  // Second quotient/remainder refinement.
  Cond = B.buildICmp(CmpInst::ICMP_UGE, S1, R, Y);
  if (DstDivReg)
    B.buildSelect(DstDivReg, Cond, B.buildAdd(I32, Q, One), Q);

  if (DstRemReg)
    B.buildSelect(DstRemReg, Cond, B.buildSub(I32, R, Y), R);
}

// Build integer reciprocal sequence around V_RCP_IFLAG_F32
//
// Return lo, hi of result
//
// %cvt.lo = G_UITOFP Val.lo
// %cvt.hi = G_UITOFP Val.hi
// %mad = G_FMAD %cvt.hi, 2**32, %cvt.lo
// %rcp = G_AMDGPU_RCP_IFLAG %mad
// %mul1 = G_FMUL %rcp, 0x5f7ffffc
// %mul2 = G_FMUL %mul1, 2**(-32)
// %trunc = G_INTRINSIC_TRUNC %mul2
// %mad2 = G_FMAD %trunc, -(2**32), %mul1
// return {G_FPTOUI %mad2, G_FPTOUI %trunc}
static std::pair<Register, Register> emitReciprocalU64(MachineIRBuilder &B,
                                                       Register Val) {
  const LLT I32 = LLT::integer(32);
  auto Unmerge = B.buildUnmerge(I32, Val);

  auto CvtLo = B.buildUITOFP(F32, Unmerge.getReg(0));
  auto CvtHi = B.buildUITOFP(F32, Unmerge.getReg(1));

  auto Mad = B.buildFMAD(
      F32, CvtHi, // 2**32
      B.buildFConstant(F32, llvm::bit_cast<float>(0x4f800000)), CvtLo);

  auto Rcp = B.buildInstr(AMDGPU::G_AMDGPU_RCP_IFLAG, {F32}, {Mad});
  auto Mul1 = B.buildFMul(
      F32, Rcp, B.buildFConstant(F32, llvm::bit_cast<float>(0x5f7ffffc)));

  // 2**(-32)
  auto Mul2 = B.buildFMul(
      F32, Mul1, B.buildFConstant(F32, llvm::bit_cast<float>(0x2f800000)));
  auto Trunc = B.buildIntrinsicTrunc(F32, Mul2);

  // -(2**32)
  auto Mad2 = B.buildFMAD(
      F32, Trunc, B.buildFConstant(F32, llvm::bit_cast<float>(0xcf800000)),
      Mul1);

  auto ResultLo = B.buildFPTOUI(I32, Mad2);
  auto ResultHi = B.buildFPTOUI(I32, Trunc);

  return {ResultLo.getReg(0), ResultHi.getReg(0)};
}

void AMDGPULegalizerInfo::legalizeUnsignedDIV_REM64Impl(MachineIRBuilder &B,
                                                        Register DstDivReg,
                                                        Register DstRemReg,
                                                        Register Numer,
                                                        Register Denom) const {
  const LLT I32 = LLT::integer(32);
  const LLT I64 = LLT::integer(64);
  const LLT S1 = LLT::scalar(1);
  Register RcpLo, RcpHi;

  std::tie(RcpLo, RcpHi) = emitReciprocalU64(B, Denom);

  auto Rcp = B.buildMergeLikeInstr(I64, {RcpLo, RcpHi});

  auto Zero64 = B.buildConstant(I64, 0);
  auto NegDenom = B.buildSub(I64, Zero64, Denom);

  auto MulLo1 = B.buildMul(I64, NegDenom, Rcp);
  auto MulHi1 = B.buildUMulH(I64, Rcp, MulLo1);

  auto UnmergeMulHi1 = B.buildUnmerge(I32, MulHi1);
  Register MulHi1_Lo = UnmergeMulHi1.getReg(0);
  Register MulHi1_Hi = UnmergeMulHi1.getReg(1);

  auto Add1_Lo = B.buildUAddo(I32, S1, RcpLo, MulHi1_Lo);
  auto Add1_Hi = B.buildUAdde(I32, S1, RcpHi, MulHi1_Hi, Add1_Lo.getReg(1));
  auto Add1 = B.buildMergeLikeInstr(I64, {Add1_Lo, Add1_Hi});

  auto MulLo2 = B.buildMul(I64, NegDenom, Add1);
  auto MulHi2 = B.buildUMulH(I64, Add1, MulLo2);
  auto UnmergeMulHi2 = B.buildUnmerge(I32, MulHi2);
  Register MulHi2_Lo = UnmergeMulHi2.getReg(0);
  Register MulHi2_Hi = UnmergeMulHi2.getReg(1);

  auto Zero32 = B.buildConstant(I32, 0);
  auto Add2_Lo = B.buildUAddo(I32, S1, Add1_Lo, MulHi2_Lo);
  auto Add2_Hi = B.buildUAdde(I32, S1, Add1_Hi, MulHi2_Hi, Add2_Lo.getReg(1));
  auto Add2 = B.buildMergeLikeInstr(I64, {Add2_Lo, Add2_Hi});

  auto UnmergeNumer = B.buildUnmerge(I32, Numer);
  Register NumerLo = UnmergeNumer.getReg(0);
  Register NumerHi = UnmergeNumer.getReg(1);

  auto MulHi3 = B.buildUMulH(I64, Numer, Add2);
  auto Mul3 = B.buildMul(I64, Denom, MulHi3);
  auto UnmergeMul3 = B.buildUnmerge(I32, Mul3);
  Register Mul3_Lo = UnmergeMul3.getReg(0);
  Register Mul3_Hi = UnmergeMul3.getReg(1);
  auto Sub1_Lo = B.buildUSubo(I32, S1, NumerLo, Mul3_Lo);
  auto Sub1_Hi = B.buildUSube(I32, S1, NumerHi, Mul3_Hi, Sub1_Lo.getReg(1));
  auto Sub1_Mi = B.buildSub(I32, NumerHi, Mul3_Hi);
  auto Sub1 = B.buildMergeLikeInstr(I64, {Sub1_Lo, Sub1_Hi});

  auto UnmergeDenom = B.buildUnmerge(I32, Denom);
  Register DenomLo = UnmergeDenom.getReg(0);
  Register DenomHi = UnmergeDenom.getReg(1);

  auto CmpHi = B.buildICmp(CmpInst::ICMP_UGE, S1, Sub1_Hi, DenomHi);
  auto C1 = B.buildSExt(I32, CmpHi);

  auto CmpLo = B.buildICmp(CmpInst::ICMP_UGE, S1, Sub1_Lo, DenomLo);
  auto C2 = B.buildSExt(I32, CmpLo);

  auto CmpEq = B.buildICmp(CmpInst::ICMP_EQ, S1, Sub1_Hi, DenomHi);
  auto C3 = B.buildSelect(I32, CmpEq, C2, C1);

  // TODO: Here and below portions of the code can be enclosed into if/endif.
  // Currently control flow is unconditional and we have 4 selects after
  // potential endif to substitute PHIs.

  // if C3 != 0 ...
  auto Sub2_Lo = B.buildUSubo(I32, S1, Sub1_Lo, DenomLo);
  auto Sub2_Mi = B.buildUSube(I32, S1, Sub1_Mi, DenomHi, Sub1_Lo.getReg(1));
  auto Sub2_Hi = B.buildUSube(I32, S1, Sub2_Mi, Zero32, Sub2_Lo.getReg(1));
  auto Sub2 = B.buildMergeLikeInstr(I64, {Sub2_Lo, Sub2_Hi});

  auto One64 = B.buildConstant(I64, 1);
  auto Add3 = B.buildAdd(I64, MulHi3, One64);

  auto C4 =
      B.buildSExt(I32, B.buildICmp(CmpInst::ICMP_UGE, S1, Sub2_Hi, DenomHi));
  auto C5 =
      B.buildSExt(I32, B.buildICmp(CmpInst::ICMP_UGE, S1, Sub2_Lo, DenomLo));
  auto C6 = B.buildSelect(
      I32, B.buildICmp(CmpInst::ICMP_EQ, S1, Sub2_Hi, DenomHi), C5, C4);

  // if (C6 != 0)
  auto Add4 = B.buildAdd(I64, Add3, One64);
  auto Sub3_Lo = B.buildUSubo(I32, S1, Sub2_Lo, DenomLo);

  auto Sub3_Mi = B.buildUSube(I32, S1, Sub2_Mi, DenomHi, Sub2_Lo.getReg(1));
  auto Sub3_Hi = B.buildUSube(I32, S1, Sub3_Mi, Zero32, Sub3_Lo.getReg(1));
  auto Sub3 = B.buildMergeLikeInstr(I64, {Sub3_Lo, Sub3_Hi});

  // endif C6
  // endif C3

  if (DstDivReg) {
    auto Sel1 = B.buildSelect(
        I64, B.buildICmp(CmpInst::ICMP_NE, S1, C6, Zero32), Add4, Add3);
    B.buildSelect(DstDivReg, B.buildICmp(CmpInst::ICMP_NE, S1, C3, Zero32),
                  Sel1, MulHi3);
  }

  if (DstRemReg) {
    auto Sel2 = B.buildSelect(
        I64, B.buildICmp(CmpInst::ICMP_NE, S1, C6, Zero32), Sub3, Sub2);
    B.buildSelect(DstRemReg, B.buildICmp(CmpInst::ICMP_NE, S1, C3, Zero32),
                  Sel2, Sub1);
  }
}

bool AMDGPULegalizerInfo::legalizeUnsignedDIV_REM(MachineInstr &MI,
                                                  MachineRegisterInfo &MRI,
                                                  MachineIRBuilder &B) const {
  Register DstDivReg, DstRemReg;
  switch (MI.getOpcode()) {
  default:
    llvm_unreachable("Unexpected opcode!");
  case AMDGPU::G_UDIV: {
    DstDivReg = MI.getOperand(0).getReg();
    break;
  }
  case AMDGPU::G_UREM: {
    DstRemReg = MI.getOperand(0).getReg();
    break;
  }
  case AMDGPU::G_UDIVREM: {
    DstDivReg = MI.getOperand(0).getReg();
    DstRemReg = MI.getOperand(1).getReg();
    break;
  }
  }

  const LLT I64 = LLT::integer(64);
  const LLT I32 = LLT::integer(32);
  const unsigned FirstSrcOpIdx = MI.getNumExplicitDefs();
  Register Num = MI.getOperand(FirstSrcOpIdx).getReg();
  Register Den = MI.getOperand(FirstSrcOpIdx + 1).getReg();
  LLT Ty = MRI.getType(MI.getOperand(0).getReg());

  if (Ty == I32)
    legalizeUnsignedDIV_REM32Impl(B, DstDivReg, DstRemReg, Num, Den);
  else if (Ty == I64)
    legalizeUnsignedDIV_REM64Impl(B, DstDivReg, DstRemReg, Num, Den);
  else
    return false;

  MI.eraseFromParent();
  return true;
}

bool AMDGPULegalizerInfo::legalizeSignedDIV_REM(MachineInstr &MI,
                                                MachineRegisterInfo &MRI,
                                                MachineIRBuilder &B) const {
  const LLT I64 = LLT::integer(64);
  const LLT I32 = LLT::integer(32);

  LLT Ty = MRI.getType(MI.getOperand(0).getReg());
  if (Ty != I32 && Ty != I64)
    return false;

  const unsigned FirstSrcOpIdx = MI.getNumExplicitDefs();
  Register LHS = MI.getOperand(FirstSrcOpIdx).getReg();
  Register RHS = MI.getOperand(FirstSrcOpIdx + 1).getReg();

  auto SignBitOffset = B.buildConstant(I32, Ty.getSizeInBits() - 1);
  auto LHSign = B.buildAShr(Ty, LHS, SignBitOffset);
  auto RHSign = B.buildAShr(Ty, RHS, SignBitOffset);

  LHS = B.buildAdd(Ty, LHS, LHSign).getReg(0);
  RHS = B.buildAdd(Ty, RHS, RHSign).getReg(0);

  LHS = B.buildXor(Ty, LHS, LHSign).getReg(0);
  RHS = B.buildXor(Ty, RHS, RHSign).getReg(0);

  Register DstDivReg, DstRemReg, TmpDivReg, TmpRemReg;
  switch (MI.getOpcode()) {
  default:
    llvm_unreachable("Unexpected opcode!");
  case AMDGPU::G_SDIV: {
    DstDivReg = MI.getOperand(0).getReg();
    TmpDivReg = MRI.createGenericVirtualRegister(Ty);
    break;
  }
  case AMDGPU::G_SREM: {
    DstRemReg = MI.getOperand(0).getReg();
    TmpRemReg = MRI.createGenericVirtualRegister(Ty);
    break;
  }
  case AMDGPU::G_SDIVREM: {
    DstDivReg = MI.getOperand(0).getReg();
    DstRemReg = MI.getOperand(1).getReg();
    TmpDivReg = MRI.createGenericVirtualRegister(Ty);
    TmpRemReg = MRI.createGenericVirtualRegister(Ty);
    break;
  }
  }

  if (Ty == I32)
    legalizeUnsignedDIV_REM32Impl(B, TmpDivReg, TmpRemReg, LHS, RHS);
  else
    legalizeUnsignedDIV_REM64Impl(B, TmpDivReg, TmpRemReg, LHS, RHS);

  if (DstDivReg) {
    auto Sign = B.buildXor(Ty, LHSign, RHSign).getReg(0);
    auto SignXor = B.buildXor(Ty, TmpDivReg, Sign).getReg(0);
    B.buildSub(DstDivReg, SignXor, Sign);
  }

  if (DstRemReg) {
    auto Sign = LHSign.getReg(0); // Remainder sign is the same as LHS
    auto SignXor = B.buildXor(Ty, TmpRemReg, Sign).getReg(0);
    B.buildSub(DstRemReg, SignXor, Sign);
  }

  MI.eraseFromParent();
  return true;
}

bool AMDGPULegalizerInfo::legalizeFastUnsafeFDIV(MachineInstr &MI,
                                                 MachineRegisterInfo &MRI,
                                                 MachineIRBuilder &B) const {
  Register Res = MI.getOperand(0).getReg();
  Register LHS = MI.getOperand(1).getReg();
  Register RHS = MI.getOperand(2).getReg();
  uint16_t Flags = MI.getFlags();
  LLT ResTy = MRI.getType(Res);

  bool AllowInaccurateRcp = MI.getFlag(MachineInstr::FmAfn);

  if (const auto *CLHS = getConstantFPVRegVal(LHS, MRI)) {
    if (!AllowInaccurateRcp && ResTy != F16)
      return false;

    // v_rcp_f32 and v_rsq_f32 do not support denormals, and according to
    // the CI documentation has a worst case error of 1 ulp.
    // OpenCL requires <= 2.5 ulp for 1.0 / x, so it should always be OK to
    // use it as long as we aren't trying to use denormals.
    //
    // v_rcp_f16 and v_rsq_f16 DO support denormals and 0.51ulp.

    // 1 / x -> RCP(x)
    if (CLHS->isOne()) {
      B.buildIntrinsic(Intrinsic::amdgcn_rcp, Res)
          .addUse(RHS)
          .setMIFlags(Flags);

      MI.eraseFromParent();
      return true;
    }

    // -1 / x -> RCP( FNEG(x) )
    if (CLHS->isMinusOne()) {
      auto FNeg = B.buildFNeg(ResTy, RHS, Flags);
      B.buildIntrinsic(Intrinsic::amdgcn_rcp, Res)
          .addUse(FNeg.getReg(0))
          .setMIFlags(Flags);

      MI.eraseFromParent();
      return true;
    }
  }

  // For f16 require afn or arcp.
  // For f32 require afn.
  if (!AllowInaccurateRcp &&
      (ResTy != F16 || !MI.getFlag(MachineInstr::FmArcp)))
    return false;

  // x / y -> x * (1.0 / y)
  auto RCP = B.buildIntrinsic(Intrinsic::amdgcn_rcp, {ResTy})
                 .addUse(RHS)
                 .setMIFlags(Flags);
  B.buildFMul(Res, LHS, RCP, Flags);

  MI.eraseFromParent();
  return true;
}

bool AMDGPULegalizerInfo::legalizeFastUnsafeFDIV64(MachineInstr &MI,
                                                   MachineRegisterInfo &MRI,
                                                   MachineIRBuilder &B) const {
  Register Res = MI.getOperand(0).getReg();
  Register X = MI.getOperand(1).getReg();
  Register Y = MI.getOperand(2).getReg();
  uint16_t Flags = MI.getFlags();
  LLT ResTy = MRI.getType(Res);

  bool AllowInaccurateRcp = MI.getFlag(MachineInstr::FmAfn);

  if (!AllowInaccurateRcp)
    return false;

  const ConstantFP *CLHS = getConstantFPVRegVal(X, MRI);
  bool IsNegRcp = CLHS && CLHS->isMinusOne();

  // Pull out the negation so it folds for free into the source modifiers.
  if (IsNegRcp)
    X = B.buildFConstant(ResTy, 1.0).getReg(0);

  Register NegY = IsNegRcp ? Y : B.buildFNeg(ResTy, Y).getReg(0);
  auto One = B.buildFConstant(ResTy, 1.0);

  auto R = B.buildIntrinsic(Intrinsic::amdgcn_rcp, {ResTy})
               .addUse(Y)
               .setMIFlags(Flags);
  if (IsNegRcp)
    R = B.buildFNeg(ResTy, R);

  auto Tmp0 = B.buildFMA(ResTy, NegY, R, One);
  R = B.buildFMA(ResTy, Tmp0, R, R);

  auto Tmp1 = B.buildFMA(ResTy, NegY, R, One);
  R = B.buildFMA(ResTy, Tmp1, R, R);

  // Skip the last 2 correction terms for reciprocal.
  if (IsNegRcp || (CLHS && CLHS->isOne())) {
    B.buildCopy(Res, R);
    MI.eraseFromParent();
    return true;
  }

  auto Ret = B.buildFMul(ResTy, X, R);
  auto Tmp2 = B.buildFMA(ResTy, NegY, Ret, X);

  B.buildFMA(Res, Tmp2, R, Ret);
  MI.eraseFromParent();
  return true;
}

bool AMDGPULegalizerInfo::legalizeFDIV16(MachineInstr &MI,
                                         MachineRegisterInfo &MRI,
                                         MachineIRBuilder &B) const {
  if (legalizeFastUnsafeFDIV(MI, MRI, B))
    return true;

  Register Res = MI.getOperand(0).getReg();
  Register LHS = MI.getOperand(1).getReg();
  Register RHS = MI.getOperand(2).getReg();

  uint16_t Flags = MI.getFlags();

  LLT I32 = LLT::integer(32);

  // a32.u = opx(V_CVT_F32_F16, a.u); // CVT to F32
  // b32.u = opx(V_CVT_F32_F16, b.u); // CVT to F32
  // r32.u = opx(V_RCP_F32, b32.u); // rcp = 1 / d
  // q32.u = opx(V_MUL_F32, a32.u, r32.u); // q = n * rcp
  // e32.u = opx(V_MAD_F32, (b32.u^_neg32), q32.u, a32.u); // err = -d * q + n
  // q32.u = opx(V_MAD_F32, e32.u, r32.u, q32.u); // q = n * rcp
  // e32.u = opx(V_MAD_F32, (b32.u^_neg32), q32.u, a32.u); // err = -d * q + n
  // tmp.u = opx(V_MUL_F32, e32.u, r32.u);
  // tmp.u = opx(V_AND_B32, tmp.u, 0xff800000)
  // q32.u = opx(V_ADD_F32, tmp.u, q32.u);
  // q16.u = opx(V_CVT_F16_F32, q32.u);
  // q16.u = opx(V_DIV_FIXUP_F16, q16.u, b.u, a.u); // q = touchup(q, d, n)

  auto LHSExt = B.buildFPExt(F32, LHS, Flags);
  auto RHSExt = B.buildFPExt(F32, RHS, Flags);
  auto NegRHSExt = B.buildFNeg(F32, RHSExt);
  auto Rcp = B.buildIntrinsic(Intrinsic::amdgcn_rcp, {F32})
                 .addUse(RHSExt.getReg(0))
                 .setMIFlags(Flags);
  auto Quot = B.buildFMul(F32, LHSExt, Rcp, Flags);
  MachineInstrBuilder Err;
  if (ST.hasMadMacF32Insts()) {
    Err = B.buildFMAD(F32, NegRHSExt, Quot, LHSExt, Flags);
    Quot = B.buildFMAD(F32, Err, Rcp, Quot, Flags);
    Err = B.buildFMAD(F32, NegRHSExt, Quot, LHSExt, Flags);
  } else {
    Err = B.buildFMA(F32, NegRHSExt, Quot, LHSExt, Flags);
    Quot = B.buildFMA(F32, Err, Rcp, Quot, Flags);
    Err = B.buildFMA(F32, NegRHSExt, Quot, LHSExt, Flags);
  }
  auto Tmp = B.buildFMul(F32, Err, Rcp, Flags);
  auto TmpInt = B.buildBitcast(I32, Tmp);
  auto MaskedInt = B.buildAnd(I32, TmpInt, B.buildConstant(I32, 0xff800000));
  auto Masked = B.buildBitcast(F32, MaskedInt);
  Quot = B.buildFAdd(F32, Masked, Quot, Flags);
  auto RDst = B.buildFPTrunc(F16, Quot, Flags);
  B.buildIntrinsic(Intrinsic::amdgcn_div_fixup, Res)
      .addUse(RDst.getReg(0))
      .addUse(RHS)
      .addUse(LHS)
      .setMIFlags(Flags);

  MI.eraseFromParent();
  return true;
}

static constexpr unsigned SPDenormModeBitField =
    AMDGPU::Hwreg::HwregEncoding::encode(AMDGPU::Hwreg::ID_MODE, 4, 2);

// Enable or disable FP32 denorm mode. When 'Enable' is true, emit instructions
// to enable denorm mode. When 'Enable' is false, disable denorm mode.
static void toggleSPDenormMode(bool Enable, MachineIRBuilder &B,
                               const GCNSubtarget &ST,
                               SIModeRegisterDefaults Mode) {
  // Set SP denorm mode to this value.
  unsigned SPDenormMode =
    Enable ? FP_DENORM_FLUSH_NONE : Mode.fpDenormModeSPValue();

  if (ST.hasDenormModeInst()) {
    // Preserve default FP64FP16 denorm mode while updating FP32 mode.
    uint32_t DPDenormModeDefault = Mode.fpDenormModeDPValue();

    uint32_t NewDenormModeValue = SPDenormMode | (DPDenormModeDefault << 2);
    B.buildInstr(AMDGPU::S_DENORM_MODE)
      .addImm(NewDenormModeValue);

  } else {
    B.buildInstr(AMDGPU::S_SETREG_IMM32_B32)
      .addImm(SPDenormMode)
      .addImm(SPDenormModeBitField);
  }
}

bool AMDGPULegalizerInfo::legalizeFDIV32(MachineInstr &MI,
                                         MachineRegisterInfo &MRI,
                                         MachineIRBuilder &B) const {
  if (legalizeFastUnsafeFDIV(MI, MRI, B))
    return true;

  Register Res = MI.getOperand(0).getReg();
  Register LHS = MI.getOperand(1).getReg();
  Register RHS = MI.getOperand(2).getReg();
  const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
  SIModeRegisterDefaults Mode = MFI->getMode();

  uint16_t Flags = MI.getFlags();

  LLT S1 = LLT::scalar(1);

  auto One = B.buildFConstant(F32, 1.0f);

  auto DenominatorScaled =
      B.buildIntrinsic(Intrinsic::amdgcn_div_scale, {F32, S1})
          .addUse(LHS)
          .addUse(RHS)
          .addImm(0)
          .setMIFlags(Flags);
  auto NumeratorScaled =
      B.buildIntrinsic(Intrinsic::amdgcn_div_scale, {F32, S1})
          .addUse(LHS)
          .addUse(RHS)
          .addImm(1)
          .setMIFlags(Flags);

  auto ApproxRcp = B.buildIntrinsic(Intrinsic::amdgcn_rcp, {F32})
                       .addUse(DenominatorScaled.getReg(0))
                       .setMIFlags(Flags);
  auto NegDivScale0 = B.buildFNeg(F32, DenominatorScaled, Flags);

  const bool PreservesDenormals = Mode.FP32Denormals == DenormalMode::getIEEE();
  const bool HasDynamicDenormals =
      (Mode.FP32Denormals.Input == DenormalMode::Dynamic) ||
      (Mode.FP32Denormals.Output == DenormalMode::Dynamic);

  Register SavedSPDenormMode;
  if (!PreservesDenormals) {
    if (HasDynamicDenormals) {
      SavedSPDenormMode = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
      B.buildInstr(AMDGPU::S_GETREG_B32)
          .addDef(SavedSPDenormMode)
          .addImm(SPDenormModeBitField);
    }
    toggleSPDenormMode(true, B, ST, Mode);
  }

  auto Fma0 = B.buildFMA(F32, NegDivScale0, ApproxRcp, One, Flags);
  auto Fma1 = B.buildFMA(F32, Fma0, ApproxRcp, ApproxRcp, Flags);
  auto Mul = B.buildFMul(F32, NumeratorScaled, Fma1, Flags);
  auto Fma2 = B.buildFMA(F32, NegDivScale0, Mul, NumeratorScaled, Flags);
  auto Fma3 = B.buildFMA(F32, Fma2, Fma1, Mul, Flags);
  auto Fma4 = B.buildFMA(F32, NegDivScale0, Fma3, NumeratorScaled, Flags);

  if (!PreservesDenormals) {
    if (HasDynamicDenormals) {
      assert(SavedSPDenormMode);
      B.buildInstr(AMDGPU::S_SETREG_B32)
          .addReg(SavedSPDenormMode)
          .addImm(SPDenormModeBitField);
    } else
      toggleSPDenormMode(false, B, ST, Mode);
  }

  auto Fmas = B.buildIntrinsic(Intrinsic::amdgcn_div_fmas, {F32})
                  .addUse(Fma4.getReg(0))
                  .addUse(Fma1.getReg(0))
                  .addUse(Fma3.getReg(0))
                  .addUse(NumeratorScaled.getReg(1))
                  .setMIFlags(Flags);

  B.buildIntrinsic(Intrinsic::amdgcn_div_fixup, Res)
      .addUse(Fmas.getReg(0))
      .addUse(RHS)
      .addUse(LHS)
      .setMIFlags(Flags);

  MI.eraseFromParent();
  return true;
}

bool AMDGPULegalizerInfo::legalizeFDIV64(MachineInstr &MI,
                                         MachineRegisterInfo &MRI,
                                         MachineIRBuilder &B) const {
  if (legalizeFastUnsafeFDIV64(MI, MRI, B))
    return true;

  Register Res = MI.getOperand(0).getReg();
  Register LHS = MI.getOperand(1).getReg();
  Register RHS = MI.getOperand(2).getReg();

  uint16_t Flags = MI.getFlags();

  LLT S1 = LLT::scalar(1);

  auto One = B.buildFConstant(F64, 1.0);

  auto DivScale0 = B.buildIntrinsic(Intrinsic::amdgcn_div_scale, {F64, S1})
                       .addUse(LHS)
                       .addUse(RHS)
                       .addImm(0)
                       .setMIFlags(Flags);

  auto NegDivScale0 = B.buildFNeg(F64, DivScale0.getReg(0), Flags);

  auto Rcp = B.buildIntrinsic(Intrinsic::amdgcn_rcp, {F64})
                 .addUse(DivScale0.getReg(0))
                 .setMIFlags(Flags);

  auto Fma0 = B.buildFMA(F64, NegDivScale0, Rcp, One, Flags);
  auto Fma1 = B.buildFMA(F64, Rcp, Fma0, Rcp, Flags);
  auto Fma2 = B.buildFMA(F64, NegDivScale0, Fma1, One, Flags);

  auto DivScale1 = B.buildIntrinsic(Intrinsic::amdgcn_div_scale, {F64, S1})
                       .addUse(LHS)
                       .addUse(RHS)
                       .addImm(1)
                       .setMIFlags(Flags);

  auto Fma3 = B.buildFMA(F64, Fma1, Fma2, Fma1, Flags);
  auto Mul = B.buildFMul(F64, DivScale1.getReg(0), Fma3, Flags);
  auto Fma4 = B.buildFMA(F64, NegDivScale0, Mul, DivScale1.getReg(0), Flags);

  Register Scale;
  if (!ST.hasUsableDivScaleConditionOutput()) {
    // Workaround a hardware bug on SI where the condition output from div_scale
    // is not usable.

    LLT I32 = LLT::integer(32);
    LLT I64 = LLT::integer(64);

    auto NumUnmerge = B.buildUnmerge(I32, B.buildBitcast(I64, LHS));
    auto DenUnmerge = B.buildUnmerge(I32, B.buildBitcast(I64, RHS));
    auto Scale0Unmerge = B.buildUnmerge(I32, B.buildBitcast(I64, DivScale0));
    auto Scale1Unmerge = B.buildUnmerge(I32, B.buildBitcast(I64, DivScale1));

    auto CmpNum = B.buildICmp(ICmpInst::ICMP_EQ, S1, NumUnmerge.getReg(1),
                              Scale1Unmerge.getReg(1));
    auto CmpDen = B.buildICmp(ICmpInst::ICMP_EQ, S1, DenUnmerge.getReg(1),
                              Scale0Unmerge.getReg(1));
    Scale = B.buildXor(S1, CmpNum, CmpDen).getReg(0);
  } else {
    Scale = DivScale1.getReg(1);
  }

  auto Fmas = B.buildIntrinsic(Intrinsic::amdgcn_div_fmas, {F64})
                  .addUse(Fma4.getReg(0))
                  .addUse(Fma3.getReg(0))
                  .addUse(Mul.getReg(0))
                  .addUse(Scale)
                  .setMIFlags(Flags);

  B.buildIntrinsic(Intrinsic::amdgcn_div_fixup, ArrayRef(Res))
      .addUse(Fmas.getReg(0))
      .addUse(RHS)
      .addUse(LHS)
      .setMIFlags(Flags);

  MI.eraseFromParent();
  return true;
}

bool AMDGPULegalizerInfo::legalizeFFREXP(MachineInstr &MI,
                                         MachineRegisterInfo &MRI,
                                         MachineIRBuilder &B) const {
  Register Res0 = MI.getOperand(0).getReg();
  Register Res1 = MI.getOperand(1).getReg();
  Register Val = MI.getOperand(2).getReg();
  uint16_t Flags = MI.getFlags();

  LLT Ty = MRI.getType(Res0);
  LLT InstrExpTy = Ty == F16 ? LLT::integer(16) : LLT::integer(32);

  auto Mant = B.buildIntrinsic(Intrinsic::amdgcn_frexp_mant, {Ty})
                  .addUse(Val)
                  .setMIFlags(Flags);
  auto Exp = B.buildIntrinsic(Intrinsic::amdgcn_frexp_exp, {InstrExpTy})
                 .addUse(Val)
                 .setMIFlags(Flags);

  if (ST.hasFractBug()) {
    auto Fabs = B.buildFAbs(Ty, Val);
    auto Inf = B.buildFConstant(Ty, APFloat::getInf(getFltSemanticForLLT(Ty)));
    auto IsFinite =
        B.buildFCmp(CmpInst::FCMP_OLT, LLT::scalar(1), Fabs, Inf, Flags);
    auto Zero = B.buildConstant(InstrExpTy, 0);
    Exp = B.buildSelect(InstrExpTy, IsFinite, Exp, Zero);
    Mant = B.buildSelect(Ty, IsFinite, Mant, Val);
  }

  B.buildCopy(Res0, Mant);
  B.buildSExtOrTrunc(Res1, Exp);

  MI.eraseFromParent();
  return true;
}

bool AMDGPULegalizerInfo::legalizeFDIVFastIntrin(MachineInstr &MI,
                                                 MachineRegisterInfo &MRI,
                                                 MachineIRBuilder &B) const {
  Register Res = MI.getOperand(0).getReg();
  Register LHS = MI.getOperand(2).getReg();
  Register RHS = MI.getOperand(3).getReg();
  uint16_t Flags = MI.getFlags();

  LLT S1 = LLT::scalar(1);

  auto Abs = B.buildFAbs(F32, RHS, Flags);
  const APFloat C0Val(1.0f);

  auto C0 = B.buildFConstant(F32, 0x1p+96f);
  auto C1 = B.buildFConstant(F32, 0x1p-32f);
  auto C2 = B.buildFConstant(F32, 1.0f);

  auto CmpRes = B.buildFCmp(CmpInst::FCMP_OGT, S1, Abs, C0, Flags);
  auto Sel = B.buildSelect(F32, CmpRes, C1, C2, Flags);

  auto Mul0 = B.buildFMul(F32, RHS, Sel, Flags);

  auto RCP = B.buildIntrinsic(Intrinsic::amdgcn_rcp, {F32})
                 .addUse(Mul0.getReg(0))
                 .setMIFlags(Flags);

  auto Mul1 = B.buildFMul(F32, LHS, RCP, Flags);

  B.buildFMul(Res, Sel, Mul1, Flags);

  MI.eraseFromParent();
  return true;
}

bool AMDGPULegalizerInfo::legalizeFSQRTF16(MachineInstr &MI,
                                           MachineRegisterInfo &MRI,
                                           MachineIRBuilder &B) const {
  // Bypass the correct expansion a standard promotion through G_FSQRT would
  // get. The f32 op is accurate enough for the f16 cas.
  unsigned Flags = MI.getFlags();
  assert(!ST.has16BitInsts());
  auto Ext = B.buildFPExt(F32, MI.getOperand(1), Flags);
  auto Log2 = B.buildIntrinsic(Intrinsic::amdgcn_sqrt, {F32})
    .addUse(Ext.getReg(0))
    .setMIFlags(Flags);
  B.buildFPTrunc(MI.getOperand(0), Log2, Flags);
  MI.eraseFromParent();
  return true;
}

bool AMDGPULegalizerInfo::legalizeFSQRTF32(MachineInstr &MI,
                                           MachineRegisterInfo &MRI,
                                           MachineIRBuilder &B) const {
  MachineFunction &MF = B.getMF();
  Register Dst = MI.getOperand(0).getReg();
  Register X = MI.getOperand(1).getReg();
  const unsigned Flags = MI.getFlags();
  const LLT S1 = LLT::scalar(1);
  const LLT I32 = LLT::integer(32);

  if (allowApproxFunc(MF, Flags)) {
    B.buildIntrinsic(Intrinsic::amdgcn_sqrt, ArrayRef<Register>({Dst}))
      .addUse(X)
      .setMIFlags(Flags);
    MI.eraseFromParent();
    return true;
  }

  auto ScaleThreshold = B.buildFConstant(F32, 0x1.0p-96f);
  auto NeedScale = B.buildFCmp(CmpInst::FCMP_OGT, S1, ScaleThreshold, X, Flags);
  auto ScaleUpFactor = B.buildFConstant(F32, 0x1.0p+32f);
  auto ScaledX = B.buildFMul(F32, X, ScaleUpFactor, Flags);
  auto SqrtX = B.buildSelect(F32, NeedScale, ScaledX, X, Flags);

  Register SqrtS = MRI.createGenericVirtualRegister(F32);
  if (needsDenormHandlingF32(MF, X, Flags)) {
    B.buildIntrinsic(Intrinsic::amdgcn_sqrt, ArrayRef<Register>({SqrtS}))
      .addUse(SqrtX.getReg(0))
      .setMIFlags(Flags);

    auto SqrtSInt = B.buildBitcast(I32, SqrtS);
    auto NegOne = B.buildConstant(I32, -1);
    auto SqrtSNextDown = B.buildBitcast(F32, B.buildAdd(I32, SqrtSInt, NegOne));

    auto NegSqrtSNextDown = B.buildFNeg(F32, SqrtSNextDown, Flags);
    auto SqrtVP = B.buildFMA(F32, NegSqrtSNextDown, SqrtS, SqrtX, Flags);

    auto PosOne = B.buildConstant(I32, 1);
    auto SqrtSNextUp = B.buildBitcast(F32, B.buildAdd(I32, SqrtSInt, PosOne));

    auto NegSqrtSNextUp = B.buildFNeg(F32, SqrtSNextUp, Flags);
    auto SqrtVS = B.buildFMA(F32, NegSqrtSNextUp, SqrtS, SqrtX, Flags);

    auto Zero = B.buildFConstant(F32, 0.0f);
    auto SqrtVPLE0 = B.buildFCmp(CmpInst::FCMP_OLE, S1, SqrtVP, Zero, Flags);

    SqrtS =
        B.buildSelect(F32, SqrtVPLE0, SqrtSNextDown, SqrtS, Flags).getReg(0);

    auto SqrtVPVSGT0 = B.buildFCmp(CmpInst::FCMP_OGT, S1, SqrtVS, Zero, Flags);
    SqrtS =
        B.buildSelect(F32, SqrtVPVSGT0, SqrtSNextUp, SqrtS, Flags).getReg(0);
  } else {
    auto SqrtR =
        B.buildIntrinsic(Intrinsic::amdgcn_rsq, {F32}).addReg(SqrtX.getReg(0));
    B.buildFMul(SqrtS, SqrtX, SqrtR, Flags);

    auto Half = B.buildFConstant(F32, 0.5f);
    auto SqrtH = B.buildFMul(F32, SqrtR, Half, Flags);
    auto NegSqrtH = B.buildFNeg(F32, SqrtH, Flags);
    auto SqrtE = B.buildFMA(F32, NegSqrtH, SqrtS, Half, Flags);
    SqrtH = B.buildFMA(F32, SqrtH, SqrtE, SqrtH, Flags);
    SqrtS = B.buildFMA(F32, SqrtS, SqrtE, SqrtS, Flags).getReg(0);
    auto NegSqrtS = B.buildFNeg(F32, SqrtS, Flags);
    auto SqrtD = B.buildFMA(F32, NegSqrtS, SqrtS, SqrtX, Flags);
    SqrtS = B.buildFMA(F32, SqrtD, SqrtH, SqrtS, Flags).getReg(0);
  }

  auto ScaleDownFactor = B.buildFConstant(F32, 0x1.0p-16f);

  auto ScaledDown = B.buildFMul(F32, SqrtS, ScaleDownFactor, Flags);

  SqrtS = B.buildSelect(F32, NeedScale, ScaledDown, SqrtS, Flags).getReg(0);

  auto IsZeroOrInf = B.buildIsFPClass(LLT::scalar(1), SqrtX, fcZero | fcPosInf);
  B.buildSelect(Dst, IsZeroOrInf, SqrtX, SqrtS, Flags);

  MI.eraseFromParent();
  return true;
}

bool AMDGPULegalizerInfo::legalizeFSQRTF64(MachineInstr &MI,
                                           MachineRegisterInfo &MRI,
                                           MachineIRBuilder &B) const {
  // For double type, the SQRT and RSQ instructions don't have required
  // precision, we apply Goldschmidt's algorithm to improve the result:
  //
  //   y0 = rsq(x)
  //   g0 = x * y0
  //   h0 = 0.5 * y0
  //
  //   r0 = 0.5 - h0 * g0
  //   g1 = g0 * r0 + g0
  //   h1 = h0 * r0 + h0
  //
  //   r1 = 0.5 - h1 * g1 => d0 = x - g1 * g1
  //   g2 = g1 * r1 + g1     g2 = d0 * h1 + g1
  //   h2 = h1 * r1 + h1
  //
  //   r2 = 0.5 - h2 * g2 => d1 = x - g2 * g2
  //   g3 = g2 * r2 + g2     g3 = d1 * h1 + g2
  //
  //   sqrt(x) = g3

  const LLT S1 = LLT::scalar(1);
  const LLT I32 = LLT::integer(32);

  Register Dst = MI.getOperand(0).getReg();
  assert(MRI.getType(Dst) == F64 && "only expect to lower f64 sqrt");

  Register X = MI.getOperand(1).getReg();
  unsigned Flags = MI.getFlags();

  Register SqrtX = X;
  Register Scaling, ZeroInt;
  if (!MI.getFlag(MachineInstr::FmAfn)) {
    auto ScaleConstant = B.buildFConstant(F64, 0x1.0p-767);

    ZeroInt = B.buildConstant(I32, 0).getReg(0);
    Scaling = B.buildFCmp(FCmpInst::FCMP_OLT, S1, X, ScaleConstant).getReg(0);

    // Scale up input if it is too small.
    auto ScaleUpFactor = B.buildConstant(I32, 256);
    auto ScaleUp = B.buildSelect(I32, Scaling, ScaleUpFactor, ZeroInt);
    SqrtX = B.buildFLdexp(F64, X, ScaleUp, Flags).getReg(0);
  }

  auto SqrtY = B.buildIntrinsic(Intrinsic::amdgcn_rsq, {F64}).addReg(SqrtX);

  auto Half = B.buildFConstant(F64, 0.5);
  auto SqrtH0 = B.buildFMul(F64, SqrtY, Half);
  auto SqrtS0 = B.buildFMul(F64, SqrtX, SqrtY);

  auto NegSqrtH0 = B.buildFNeg(F64, SqrtH0);
  auto SqrtR0 = B.buildFMA(F64, NegSqrtH0, SqrtS0, Half);

  auto SqrtS1 = B.buildFMA(F64, SqrtS0, SqrtR0, SqrtS0);
  auto SqrtH1 = B.buildFMA(F64, SqrtH0, SqrtR0, SqrtH0);

  auto NegSqrtS1 = B.buildFNeg(F64, SqrtS1);
  auto SqrtD0 = B.buildFMA(F64, NegSqrtS1, SqrtS1, SqrtX);

  auto SqrtS2 = B.buildFMA(F64, SqrtD0, SqrtH1, SqrtS1);

  Register SqrtRet = SqrtS2.getReg(0);
  if (!MI.getFlag(MachineInstr::FmAfn)) {
    auto NegSqrtS2 = B.buildFNeg(F64, SqrtS2);
    auto SqrtD1 = B.buildFMA(F64, NegSqrtS2, SqrtS2, SqrtX);
    auto SqrtD2 = B.buildFMA(F64, SqrtD1, SqrtH1, SqrtS2);

    // Scale down the result.
    auto ScaleDownFactor = B.buildConstant(I32, -128);
    auto ScaleDown = B.buildSelect(I32, Scaling, ScaleDownFactor, ZeroInt);
    SqrtRet = B.buildFLdexp(F64, SqrtD2, ScaleDown, Flags).getReg(0);
  }

  Register IsZeroOrInf;
  if (MI.getFlag(MachineInstr::FmNoInfs)) {
    auto ZeroFP = B.buildFConstant(F64, 0.0);
    IsZeroOrInf = B.buildFCmp(FCmpInst::FCMP_OEQ, S1, SqrtX, ZeroFP).getReg(0);
  } else {
    IsZeroOrInf = B.buildIsFPClass(S1, SqrtX, fcZero | fcPosInf).getReg(0);
  }

  // TODO: Check for DAZ and expand to subnormals

  // If x is +INF, +0, or -0, use its original value
  B.buildSelect(Dst, IsZeroOrInf, SqrtX, SqrtRet, Flags);

  MI.eraseFromParent();
  return true;
}

bool AMDGPULegalizerInfo::legalizeFSQRT(MachineInstr &MI,
                                        MachineRegisterInfo &MRI,
                                        MachineIRBuilder &B) const {
  LLT Ty = MRI.getType(MI.getOperand(0).getReg());
  if (Ty == F32)
    return legalizeFSQRTF32(MI, MRI, B);
  if (Ty == F64)
    return legalizeFSQRTF64(MI, MRI, B);
  if (Ty == F16)
    return legalizeFSQRTF16(MI, MRI, B);
  return false;
}

// Expand llvm.amdgcn.rsq.clamp on targets that don't support the instruction.
// FIXME: Why do we handle this one but not other removed instructions?
//
// Reciprocal square root.  The clamp prevents infinite results, clamping
// infinities to max_float.  D.f = 1.0 / sqrt(S0.f), result clamped to
// +-max_float.
bool AMDGPULegalizerInfo::legalizeRsqClampIntrinsic(MachineInstr &MI,
                                                    MachineRegisterInfo &MRI,
                                                    MachineIRBuilder &B) const {
  if (ST.getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS)
    return true;

  Register Dst = MI.getOperand(0).getReg();
  Register Src = MI.getOperand(2).getReg();
  auto Flags = MI.getFlags();

  LLT Ty = MRI.getType(Dst);

  const fltSemantics *FltSemantics;
  if (Ty == F32)
    FltSemantics = &APFloat::IEEEsingle();
  else if (Ty == F64)
    FltSemantics = &APFloat::IEEEdouble();
  else
    return false;

  auto Rsq = B.buildIntrinsic(Intrinsic::amdgcn_rsq, {Ty})
                 .addUse(Src)
                 .setMIFlags(Flags);

  // We don't need to concern ourselves with the snan handling difference, since
  // the rsq quieted (or not) so use the one which will directly select.
  const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
  const bool UseIEEE = MFI->getMode().IEEE;

  auto MaxFlt = B.buildFConstant(Ty, APFloat::getLargest(*FltSemantics));
  auto ClampMax = UseIEEE ? B.buildFMinNumIEEE(Ty, Rsq, MaxFlt, Flags) :
                            B.buildFMinNum(Ty, Rsq, MaxFlt, Flags);

  auto MinFlt = B.buildFConstant(Ty, APFloat::getLargest(*FltSemantics, true));

  if (UseIEEE)
    B.buildFMaxNumIEEE(Dst, ClampMax, MinFlt, Flags);
  else
    B.buildFMaxNum(Dst, ClampMax, MinFlt, Flags);
  MI.eraseFromParent();
  return true;
}

// TODO: Fix pointer type handling
bool AMDGPULegalizerInfo::legalizeLaneOp(LegalizerHelper &Helper,
                                         MachineInstr &MI,
                                         Intrinsic::ID IID) const {

  MachineIRBuilder &B = Helper.MIRBuilder;
  MachineRegisterInfo &MRI = *B.getMRI();

  bool IsPermLane16 = IID == Intrinsic::amdgcn_permlane16 ||
                      IID == Intrinsic::amdgcn_permlanex16;
  bool IsSetInactive = IID == Intrinsic::amdgcn_set_inactive ||
                       IID == Intrinsic::amdgcn_set_inactive_chain_arg;
  bool IsPermlaneShuffle = IID == Intrinsic::amdgcn_permlane_bcast ||
                           IID == Intrinsic::amdgcn_permlane_up ||
                           IID == Intrinsic::amdgcn_permlane_down ||
                           IID == Intrinsic::amdgcn_permlane_xor;

  auto createLaneOp = [&IID, &B, &MI](Register Src0, Register Src1,
                                      Register Src2, LLT VT) -> Register {
    auto LaneOp = B.buildIntrinsic(IID, {VT}).addUse(Src0);
    switch (IID) {
    case Intrinsic::amdgcn_readfirstlane:
    case Intrinsic::amdgcn_permlane64:
      return LaneOp.getReg(0);
    case Intrinsic::amdgcn_readlane:
    case Intrinsic::amdgcn_set_inactive:
    case Intrinsic::amdgcn_set_inactive_chain_arg:
      return LaneOp.addUse(Src1).getReg(0);
    case Intrinsic::amdgcn_writelane:
    case Intrinsic::amdgcn_permlane_bcast:
    case Intrinsic::amdgcn_permlane_up:
    case Intrinsic::amdgcn_permlane_down:
    case Intrinsic::amdgcn_permlane_xor:
      return LaneOp.addUse(Src1).addUse(Src2).getReg(0);
    case Intrinsic::amdgcn_permlane16:
    case Intrinsic::amdgcn_permlanex16: {
      Register Src3 = MI.getOperand(5).getReg();
      int64_t Src4 = MI.getOperand(6).getImm();
      int64_t Src5 = MI.getOperand(7).getImm();
      return LaneOp.addUse(Src1)
          .addUse(Src2)
          .addUse(Src3)
          .addImm(Src4)
          .addImm(Src5)
          .getReg(0);
    }
    case Intrinsic::amdgcn_mov_dpp8:
      return LaneOp.addImm(MI.getOperand(3).getImm()).getReg(0);
    case Intrinsic::amdgcn_update_dpp:
      return LaneOp.addUse(Src1)
          .addImm(MI.getOperand(4).getImm())
          .addImm(MI.getOperand(5).getImm())
          .addImm(MI.getOperand(6).getImm())
          .addImm(MI.getOperand(7).getImm())
          .getReg(0);
    default:
      llvm_unreachable("unhandled lane op");
    }
  };

  Register DstReg = MI.getOperand(0).getReg();
  Register Src0 = MI.getOperand(2).getReg();
  Register Src1, Src2;
  if (IID == Intrinsic::amdgcn_readlane || IID == Intrinsic::amdgcn_writelane ||
      IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16 ||
      IsPermlaneShuffle) {
    Src1 = MI.getOperand(3).getReg();
    if (IID == Intrinsic::amdgcn_writelane || IsPermLane16 ||
        IsPermlaneShuffle) {
      Src2 = MI.getOperand(4).getReg();
    }
  }

  LLT Ty = MRI.getType(DstReg);
  unsigned Size = Ty.getSizeInBits();

  unsigned SplitSize = 32;
  if (IID == Intrinsic::amdgcn_update_dpp && (Size % 64 == 0) &&
      ST.hasDPALU_DPP() &&
      AMDGPU::isLegalDPALU_DPPControl(ST, MI.getOperand(4).getImm()))
    SplitSize = 64;

  if (Size == SplitSize) {
    // Already legal
    return true;
  }

  const LLT I32 = LLT::integer(32);

  bool IsFloat = Ty.getScalarType().isFloat();

  LLT IntTy = IsFloat ? LLT::integer(Size) : Ty;
  if (IsFloat) {
    Src0 = B.buildBitcast(IntTy, Src0).getReg(0);
    if (Src1 && MRI.getType(Src1).getScalarType().isFloat())
      Src1 = B.buildBitcast(IntTy, Src1).getReg(0);
    if (Src2 && MRI.getType(Src2).getScalarType().isFloat())
      Src2 = B.buildBitcast(IntTy, Src2).getReg(0);
  }

  if (Size < 32) {
    Src0 = B.buildAnyExt(I32, Src0).getReg(0);

    if (IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16)
      Src1 = B.buildAnyExt(I32, Src1).getReg(0);

    if (IID == Intrinsic::amdgcn_writelane)
      Src2 = B.buildAnyExt(I32, Src2).getReg(0);

    Register LaneOpDst = createLaneOp(Src0, Src1, Src2, I32);
    if (IsFloat)
      B.buildBitcast(DstReg, B.buildTrunc(IntTy, LaneOpDst));
    else
      B.buildTrunc(DstReg, LaneOpDst);
    MI.eraseFromParent();
    return true;
  }

  if (Size % SplitSize != 0)
    return false;

  LLT PartialResTy = LLT::integer(SplitSize);
  bool NeedsBitcast = false;
  if (IntTy.isVector()) {
    LLT EltTy = IntTy.getElementType();
    unsigned EltSize = EltTy.getSizeInBits();
    if (EltSize == SplitSize) {
      PartialResTy = EltTy;
    } else if (EltSize == 16 || EltSize == 32) {
      unsigned NElem = SplitSize / EltSize;
      PartialResTy = IntTy.changeElementCount(ElementCount::getFixed(NElem));
    } else {
      NeedsBitcast = true;
    }
  }

  SmallVector<Register, 4> PartialRes;
  unsigned NumParts = Size / SplitSize;
  MachineInstrBuilder Src0Parts = B.buildUnmerge(PartialResTy, Src0);
  MachineInstrBuilder Src1Parts, Src2Parts;

  if (IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16)
    Src1Parts = B.buildUnmerge(PartialResTy, Src1);

  if (IID == Intrinsic::amdgcn_writelane)
    Src2Parts = B.buildUnmerge(PartialResTy, Src2);

  for (unsigned i = 0; i < NumParts; ++i) {
    Src0 = Src0Parts.getReg(i);

    if (IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16)
      Src1 = Src1Parts.getReg(i);

    if (IID == Intrinsic::amdgcn_writelane)
      Src2 = Src2Parts.getReg(i);

    PartialRes.push_back(createLaneOp(Src0, Src1, Src2, PartialResTy));
  }

  if (NeedsBitcast || IsFloat)
    B.buildBitcast(
        DstReg,
        B.buildMergeLikeInstr(LLT::integer(IntTy.getSizeInBits()), PartialRes));
  else
    B.buildMergeLikeInstr(DstReg, PartialRes);

  MI.eraseFromParent();
  return true;
}

bool AMDGPULegalizerInfo::getImplicitArgPtr(Register DstReg,
                                            MachineRegisterInfo &MRI,
                                            MachineIRBuilder &B) const {
  uint64_t Offset =
    ST.getTargetLowering()->getImplicitParameterOffset(
      B.getMF(), AMDGPUTargetLowering::FIRST_IMPLICIT);
  LLT DstTy = MRI.getType(DstReg);
  LLT IdxTy = LLT::integer(DstTy.getSizeInBits());

  Register KernargPtrReg = MRI.createGenericVirtualRegister(DstTy);
  if (!loadInputValue(KernargPtrReg, B,
                      AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR))
    return false;

  B.buildObjectPtrOffset(DstReg, KernargPtrReg,
                         B.buildConstant(IdxTy, Offset).getReg(0));
  return true;
}

/// To create a buffer resource from a 64-bit pointer, mask off the upper 32
/// bits of the pointer and replace them with the stride argument, then
/// merge_values everything together. In the common case of a raw buffer (the
/// stride component is 0), we can just AND off the upper half.
bool AMDGPULegalizerInfo::legalizePointerAsRsrcIntrin(
    MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const {
  Register Result = MI.getOperand(0).getReg();
  Register Pointer = MI.getOperand(2).getReg();
  Register Stride = MI.getOperand(3).getReg();
  Register NumRecords = MI.getOperand(4).getReg();
  Register Flags = MI.getOperand(5).getReg();

  LLT I32 = LLT::integer(32);
  LLT I64 = LLT::integer(64);

  B.setInsertPt(B.getMBB(), ++B.getInsertPt());

  auto ExtStride = B.buildAnyExt(I32, Stride);

  if (ST.has45BitNumRecordsBufferResource()) {
    Register Zero = B.buildConstant(I32, 0).getReg(0);
    // Build the lower 64-bit value, which has a 57-bit base and the lower 7-bit
    // num_records.
    LLT PtrIntTy = LLT::integer(MRI.getType(Pointer).getSizeInBits());
    auto PointerInt = B.buildPtrToInt(PtrIntTy, Pointer);
    auto ExtPointer = B.buildAnyExtOrTrunc(I64, PointerInt);
    auto NumRecordsLHS = B.buildShl(I64, NumRecords, B.buildConstant(I32, 57));
    Register LowHalf = B.buildOr(I64, ExtPointer, NumRecordsLHS).getReg(0);

    // Build the higher 64-bit value, which has the higher 38-bit num_records,
    // 6-bit zero (omit), 16-bit stride and scale and 4-bit flag.
    auto NumRecordsRHS = B.buildLShr(I64, NumRecords, B.buildConstant(I32, 7));
    auto ShiftedStride = B.buildShl(I32, ExtStride, B.buildConstant(I32, 12));
    auto ExtShiftedStride =
        B.buildMergeValues(I64, {Zero, ShiftedStride.getReg(0)});
    auto ShiftedFlags = B.buildShl(I32, Flags, B.buildConstant(I32, 28));
    auto ExtShiftedFlags =
        B.buildMergeValues(I64, {Zero, ShiftedFlags.getReg(0)});
    auto CombinedFields = B.buildOr(I64, NumRecordsRHS, ExtShiftedStride);
    Register HighHalf =
        B.buildOr(I64, CombinedFields, ExtShiftedFlags).getReg(0);
    B.buildMergeValues(Result, {LowHalf, HighHalf});
  } else {
    NumRecords = B.buildTrunc(I32, NumRecords).getReg(0);
    auto Unmerge = B.buildUnmerge(I32, Pointer);
    auto LowHalf = Unmerge.getReg(0);
    auto HighHalf = Unmerge.getReg(1);

    auto AndMask = B.buildConstant(I32, 0x0000ffff);
    auto Masked = B.buildAnd(I32, HighHalf, AndMask);
    auto ShiftConst = B.buildConstant(I32, 16);
    auto ShiftedStride = B.buildShl(I32, ExtStride, ShiftConst);
    auto NewHighHalf = B.buildOr(I32, Masked, ShiftedStride);
    Register NewHighHalfReg = NewHighHalf.getReg(0);
    B.buildMergeValues(Result, {LowHalf, NewHighHalfReg, NumRecords, Flags});
  }

  MI.eraseFromParent();
  return true;
}

bool AMDGPULegalizerInfo::legalizeImplicitArgPtr(MachineInstr &MI,
                                                 MachineRegisterInfo &MRI,
                                                 MachineIRBuilder &B) const {
  const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
  if (!MFI->isEntryFunction()) {
    return legalizePreloadedArgIntrin(MI, MRI, B,
                                      AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR);
  }

  Register DstReg = MI.getOperand(0).getReg();
  if (!getImplicitArgPtr(DstReg, MRI, B))
    return false;

  MI.eraseFromParent();
  return true;
}

bool AMDGPULegalizerInfo::getLDSKernelId(Register DstReg,
                                         MachineRegisterInfo &MRI,
                                         MachineIRBuilder &B) const {
  Function &F = B.getMF().getFunction();
  std::optional<uint32_t> KnownSize =
      AMDGPUMachineFunctionInfo::getLDSKernelIdMetadata(F);
  if (KnownSize.has_value())
    B.buildConstant(DstReg, *KnownSize);
  return false;
}

bool AMDGPULegalizerInfo::legalizeLDSKernelId(MachineInstr &MI,
                                              MachineRegisterInfo &MRI,
                                              MachineIRBuilder &B) const {

  const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
  if (!MFI->isEntryFunction()) {
    return legalizePreloadedArgIntrin(MI, MRI, B,
                                      AMDGPUFunctionArgInfo::LDS_KERNEL_ID);
  }

  Register DstReg = MI.getOperand(0).getReg();
  if (!getLDSKernelId(DstReg, MRI, B))
    return false;

  MI.eraseFromParent();
  return true;
}

bool AMDGPULegalizerInfo::legalizeIsAddrSpace(MachineInstr &MI,
                                              MachineRegisterInfo &MRI,
                                              MachineIRBuilder &B,
                                              unsigned AddrSpace) const {
  const LLT I32 = LLT::integer(32);
  auto Unmerge = B.buildUnmerge(I32, MI.getOperand(2).getReg());
  Register Hi32 = Unmerge.getReg(1);

  if (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS &&
      ST.hasGloballyAddressableScratch()) {
    Register FlatScratchBaseHi =
        B.buildInstr(AMDGPU::S_MOV_B32, {I32},
                     {Register(AMDGPU::SRC_FLAT_SCRATCH_BASE_HI)})
            .getReg(0);
    MRI.setRegClass(FlatScratchBaseHi, &AMDGPU::SReg_32RegClass);
    // Test bits 63..58 against the aperture address.
    Register XOR = B.buildXor(I32, Hi32, FlatScratchBaseHi).getReg(0);
    B.buildICmp(ICmpInst::ICMP_ULT, MI.getOperand(0), XOR,
                B.buildConstant(I32, 1u << 26));
  } else {
    Register ApertureReg = getSegmentAperture(AddrSpace, MRI, B);
    B.buildICmp(ICmpInst::ICMP_EQ, MI.getOperand(0), Hi32, ApertureReg);
  }
  MI.eraseFromParent();
  return true;
}

// The raw.(t)buffer and struct.(t)buffer intrinsics have two offset args:
// offset (the offset that is included in bounds checking and swizzling, to be
// split between the instruction's voffset and immoffset fields) and soffset
// (the offset that is excluded from bounds checking and swizzling, to go in
// the instruction's soffset field).  This function takes the first kind of
// offset and figures out how to split it between voffset and immoffset.
std::pair<Register, unsigned>
AMDGPULegalizerInfo::splitBufferOffsets(MachineIRBuilder &B,
                                        Register OrigOffset) const {
  const unsigned MaxImm = SIInstrInfo::getMaxMUBUFImmOffset(ST);
  Register BaseReg;
  unsigned ImmOffset;
  const LLT I32 = LLT::integer(32);
  MachineRegisterInfo &MRI = *B.getMRI();

  // On GFX1250+, voffset and immoffset are zero-extended from 32 bits before
  // being added, so we can only safely match a 32-bit addition with no unsigned
  // overflow.
  bool CheckNUW = ST.hasGFX1250Insts();
  std::tie(BaseReg, ImmOffset) = AMDGPU::getBaseWithConstantOffset(
      MRI, OrigOffset, /*KnownBits=*/nullptr, CheckNUW);

  // If BaseReg is a pointer, convert it to int.
  if (MRI.getType(BaseReg).isPointer())
    BaseReg = B.buildPtrToInt(MRI.getType(OrigOffset), BaseReg).getReg(0);

  // If the immediate value is too big for the immoffset field, put only bits
  // that would normally fit in the immoffset field. The remaining value that
  // is copied/added for the voffset field is a large power of 2, and it
  // stands more chance of being CSEd with the copy/add for another similar
  // load/store.
  // However, do not do that rounding down if that is a negative
  // number, as it appears to be illegal to have a negative offset in the
  // vgpr, even if adding the immediate offset makes it positive.
  unsigned Overflow = ImmOffset & ~MaxImm;
  ImmOffset -= Overflow;
  if ((int32_t)Overflow < 0) {
    Overflow += ImmOffset;
    ImmOffset = 0;
  }

  if (Overflow != 0) {
    if (!BaseReg) {
      BaseReg = B.buildConstant(I32, Overflow).getReg(0);
    } else {
      auto OverflowVal = B.buildConstant(I32, Overflow);
      BaseReg = B.buildAdd(I32, BaseReg, OverflowVal).getReg(0);
    }
  }

  if (!BaseReg)
    BaseReg = B.buildConstant(I32, 0).getReg(0);

  return std::pair(BaseReg, ImmOffset);
}

/// Handle register layout difference for f16 images for some subtargets.
Register AMDGPULegalizerInfo::handleD16VData(MachineIRBuilder &B,
                                             MachineRegisterInfo &MRI,
                                             Register Reg,
                                             bool ImageStore) const {
  const LLT I16 = LLT::integer(16);
  const LLT I32 = LLT::integer(32);
  LLT StoreVT = MRI.getType(Reg);
  assert(StoreVT.isVector() && StoreVT.getElementType().getSizeInBits() == 16);

  LLT I16Vec = StoreVT.changeElementType(I16);
  Register RegI16 =
      StoreVT == I16Vec ? Reg : B.buildBitcast(I16Vec, Reg).getReg(0);

  if (ST.hasUnpackedD16VMem()) {
    auto Unmerge = B.buildUnmerge(I16, RegI16);

    SmallVector<Register, 4> WideRegs;
    for (int I = 0, E = Unmerge->getNumOperands() - 1; I != E; ++I)
      WideRegs.push_back(B.buildAnyExt(I32, Unmerge.getReg(I)).getReg(0));

    int NumElts = StoreVT.getNumElements();

    return B.buildBuildVector(LLT::fixed_vector(NumElts, I32), WideRegs)
        .getReg(0);
  }

  if (ImageStore && ST.hasImageStoreD16Bug()) {
    if (StoreVT.getNumElements() == 2) {
      SmallVector<Register, 4> PackedRegs;
      Reg = B.buildBitcast(I32, RegI16).getReg(0);
      PackedRegs.push_back(Reg);
      PackedRegs.resize(2, B.buildUndef(I32).getReg(0));
      return B.buildBuildVector(LLT::fixed_vector(2, I32), PackedRegs)
          .getReg(0);
    }

    if (StoreVT.getNumElements() == 3) {
      SmallVector<Register, 4> PackedRegs;
      auto Unmerge = B.buildUnmerge(I16, RegI16);
      for (int I = 0, E = Unmerge->getNumOperands() - 1; I != E; ++I)
        PackedRegs.push_back(Unmerge.getReg(I));
      PackedRegs.resize(6, B.buildUndef(I16).getReg(0));
      Reg = B.buildBuildVector(LLT::fixed_vector(6, I16), PackedRegs).getReg(0);
      return B.buildBitcast(LLT::fixed_vector(3, I32), Reg).getReg(0);
    }

    if (StoreVT.getNumElements() == 4) {
      SmallVector<Register, 4> PackedRegs;
      Reg = B.buildBitcast(LLT::fixed_vector(2, I32), RegI16).getReg(0);
      auto Unmerge = B.buildUnmerge(I32, Reg);
      for (int I = 0, E = Unmerge->getNumOperands() - 1; I != E; ++I)
        PackedRegs.push_back(Unmerge.getReg(I));
      PackedRegs.resize(4, B.buildUndef(I32).getReg(0));
      return B.buildBuildVector(LLT::fixed_vector(4, I32), PackedRegs)
          .getReg(0);
    }

    llvm_unreachable("invalid data type");
  }

  if (StoreVT.isVector() && StoreVT.getNumElements() == 3 &&
      StoreVT.getElementType().getSizeInBits() == 16) {
    Reg = B.buildPadVectorWithUndefElements(
               LLT::fixed_vector(4, StoreVT.getElementType()), Reg)
              .getReg(0);
  }
  return Reg;
}

Register AMDGPULegalizerInfo::fixStoreSourceType(MachineIRBuilder &B,
                                                 Register VData, LLT MemTy,
                                                 bool IsFormat) const {
  MachineRegisterInfo *MRI = B.getMRI();
  LLT Ty = MRI->getType(VData);

  // Fixup buffer resources themselves needing to be v4i128.
  if (hasBufferRsrcWorkaround(Ty))
    return castBufferRsrcToV4I32(VData, B);

  if (shouldBitcastLoadStoreType(ST, Ty, MemTy)) {
    Ty = getBitcastRegisterType(Ty);
    VData = B.buildBitcast(Ty, VData).getReg(0);
  }
  // Fixup illegal register types for i8 stores.
  if (Ty == LLT::integer(8) || Ty == LLT::integer(16) || Ty == F16) {
    Register AnyExt = B.buildAnyExt(LLT::integer(32), VData).getReg(0);
    return AnyExt;
  }

  if (Ty.isVector()) {
    if (Ty.getElementType().getSizeInBits() == 16 && Ty.getNumElements() <= 4) {
      if (IsFormat)
        return handleD16VData(B, *MRI, VData);
    }
  }

  return VData;
}

bool AMDGPULegalizerInfo::legalizeBufferStore(MachineInstr &MI,
                                              LegalizerHelper &Helper,
                                              bool IsTyped,
                                              bool IsFormat) const {
  MachineIRBuilder &B = Helper.MIRBuilder;
  MachineRegisterInfo &MRI = *B.getMRI();

  Register VData = MI.getOperand(1).getReg();
  LLT Ty = MRI.getType(VData);
  LLT EltTy = Ty.getScalarType();
  const bool IsD16 = IsFormat && (EltTy.getSizeInBits() == 16);
  const LLT I32 = LLT::integer(32);

  MachineMemOperand *MMO = *MI.memoperands_begin();
  const int MemSize = MMO->getSize().getValue();
  LLT MemTy = MMO->getMemoryType();

  if (IsFormat && !IsTyped && !IsD16 && MemTy.getSizeInBits() < 32) {
    const Function &Fn = B.getMF().getFunction();
    Fn.getContext().diagnose(DiagnosticInfoUnsupported(
        Fn, "unsupported sub-dword format buffer store", MI.getDebugLoc()));
    MI.eraseFromParent();
    return true;
  }

  VData = fixStoreSourceType(B, VData, MemTy, IsFormat);

  castBufferRsrcArgToV4I32(MI, B, 2);
  Register RSrc = MI.getOperand(2).getReg();

  unsigned ImmOffset;

  // The typed intrinsics add an immediate after the registers.
  const unsigned NumVIndexOps = IsTyped ? 8 : 7;

  // The struct intrinsic variants add one additional operand over raw.
  const bool HasVIndex = MI.getNumOperands() == NumVIndexOps;
  Register VIndex;
  int OpOffset = 0;
  if (HasVIndex) {
    VIndex = MI.getOperand(3).getReg();
    OpOffset = 1;
  } else {
    VIndex = B.buildConstant(I32, 0).getReg(0);
  }

  Register VOffset = MI.getOperand(3 + OpOffset).getReg();
  Register SOffset = MI.getOperand(4 + OpOffset).getReg();

  unsigned Format = 0;
  if (IsTyped) {
    Format = MI.getOperand(5 + OpOffset).getImm();
    ++OpOffset;
  }

  unsigned AuxiliaryData = MI.getOperand(5 + OpOffset).getImm();

  std::tie(VOffset, ImmOffset) = splitBufferOffsets(B, VOffset);

  unsigned Opc;
  if (IsTyped) {
    Opc = IsD16 ? AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT_D16 :
                  AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT;
  } else if (IsFormat) {
    Opc = IsD16 ? AMDGPU::G_AMDGPU_BUFFER_STORE_FORMAT_D16 :
                  AMDGPU::G_AMDGPU_BUFFER_STORE_FORMAT;
  } else {
    switch (MemSize) {
    case 1:
      Opc = AMDGPU::G_AMDGPU_BUFFER_STORE_BYTE;
      break;
    case 2:
      Opc = AMDGPU::G_AMDGPU_BUFFER_STORE_SHORT;
      break;
    default:
      Opc = AMDGPU::G_AMDGPU_BUFFER_STORE;
      break;
    }
  }

  auto MIB = B.buildInstr(Opc)
    .addUse(VData)              // vdata
    .addUse(RSrc)               // rsrc
    .addUse(VIndex)             // vindex
    .addUse(VOffset)            // voffset
    .addUse(SOffset)            // soffset
    .addImm(ImmOffset);         // offset(imm)

  if (IsTyped)
    MIB.addImm(Format);

  MIB.addImm(AuxiliaryData)      // cachepolicy, swizzled buffer(imm)
     .addImm(HasVIndex ? -1 : 0) // idxen(imm)
     .addMemOperand(MMO);

  MI.eraseFromParent();
  return true;
}

static void buildBufferLoad(unsigned Opc, Register LoadDstReg, Register RSrc,
                            Register VIndex, Register VOffset, Register SOffset,
                            unsigned ImmOffset, unsigned Format,
                            unsigned AuxiliaryData, MachineMemOperand *MMO,
                            bool IsTyped, bool HasVIndex, MachineIRBuilder &B) {
  auto MIB = B.buildInstr(Opc)
                 .addDef(LoadDstReg) // vdata
                 .addUse(RSrc)       // rsrc
                 .addUse(VIndex)     // vindex
                 .addUse(VOffset)    // voffset
                 .addUse(SOffset)    // soffset
                 .addImm(ImmOffset); // offset(imm)

  if (IsTyped)
    MIB.addImm(Format);

  MIB.addImm(AuxiliaryData)       // cachepolicy, swizzled buffer(imm)
      .addImm(HasVIndex ? -1 : 0) // idxen(imm)
      .addMemOperand(MMO);
}

bool AMDGPULegalizerInfo::legalizeBufferLoad(MachineInstr &MI,
                                             LegalizerHelper &Helper,
                                             bool IsFormat,
                                             bool IsTyped) const {
  MachineIRBuilder &B = Helper.MIRBuilder;
  MachineRegisterInfo &MRI = *B.getMRI();
  GISelChangeObserver &Observer = Helper.Observer;

  // FIXME: Verifier should enforce 1 MMO for these intrinsics.
  MachineMemOperand *MMO = *MI.memoperands_begin();
  const LLT MemTy = MMO->getMemoryType();
  const LLT I32 = LLT::integer(32);

  Register Dst = MI.getOperand(0).getReg();

  Register StatusDst;
  int OpOffset = 0;
  assert(MI.getNumExplicitDefs() == 1 || MI.getNumExplicitDefs() == 2);
  bool IsTFE = MI.getNumExplicitDefs() == 2;
  if (IsTFE) {
    StatusDst = MI.getOperand(1).getReg();
    ++OpOffset;
  }

  castBufferRsrcArgToV4I32(MI, B, 2 + OpOffset);
  Register RSrc = MI.getOperand(2 + OpOffset).getReg();

  // The typed intrinsics add an immediate after the registers.
  const unsigned NumVIndexOps = IsTyped ? 8 : 7;

  // The struct intrinsic variants add one additional operand over raw.
  const bool HasVIndex = MI.getNumOperands() == NumVIndexOps + OpOffset;
  Register VIndex;
  if (HasVIndex) {
    VIndex = MI.getOperand(3 + OpOffset).getReg();
    ++OpOffset;
  } else {
    VIndex = B.buildConstant(I32, 0).getReg(0);
  }

  Register VOffset = MI.getOperand(3 + OpOffset).getReg();
  Register SOffset = MI.getOperand(4 + OpOffset).getReg();

  unsigned Format = 0;
  if (IsTyped) {
    Format = MI.getOperand(5 + OpOffset).getImm();
    ++OpOffset;
  }

  unsigned AuxiliaryData = MI.getOperand(5 + OpOffset).getImm();
  unsigned ImmOffset;

  LLT Ty = MRI.getType(Dst);
  // Make addrspace 8 pointers loads into 4xi32 loads here, so the rest of the
  // logic doesn't have to handle that case.
  if (hasBufferRsrcWorkaround(Ty)) {
    Observer.changingInstr(MI);
    Ty = castBufferRsrcFromV4I32(MI, B, MRI, 0);
    Observer.changedInstr(MI);
    Dst = MI.getOperand(0).getReg();
    B.setInsertPt(B.getMBB(), MI);
  }
  if (shouldBitcastLoadStoreType(ST, Ty, MemTy)) {
    Ty = getBitcastRegisterType(Ty);
    Observer.changingInstr(MI);
    Helper.bitcastDst(MI, Ty, 0);
    Observer.changedInstr(MI);
    Dst = MI.getOperand(0).getReg();
    B.setInsertPt(B.getMBB(), MI);
  }

  LLT EltTy = Ty.getScalarType();
  const bool IsD16 = IsFormat && (EltTy.getSizeInBits() == 16);
  const bool Unpacked = ST.hasUnpackedD16VMem();

  if (IsFormat && !IsTyped && !IsD16 && MemTy.getSizeInBits() < 32) {
    const Function &Fn = B.getMF().getFunction();
    Fn.getContext().diagnose(DiagnosticInfoUnsupported(
        Fn, "unsupported sub-dword format buffer load", MI.getDebugLoc()));
    B.buildUndef(Dst);
    if (IsTFE)
      B.buildUndef(StatusDst);
    MI.eraseFromParent();
    return true;
  }

  std::tie(VOffset, ImmOffset) = splitBufferOffsets(B, VOffset);

  unsigned Opc;

  // TODO: Support TFE for typed and narrow loads.
  if (IsTyped) {
    if (IsTFE)
      return false;
    Opc = IsD16 ? AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT_D16 :
                  AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT;
  } else if (IsFormat) {
    if (IsD16) {
      if (IsTFE)
        return false;
      Opc = AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_D16;
    } else {
      Opc = IsTFE ? AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_TFE
                  : AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT;
    }
  } else {
    switch (MemTy.getSizeInBits()) {
    case 8:
      Opc = IsTFE ? AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE_TFE
                  : AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE;
      break;
    case 16:
      Opc = IsTFE ? AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT_TFE
                  : AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT;
      break;
    default:
      Opc = IsTFE ? AMDGPU::G_AMDGPU_BUFFER_LOAD_TFE
                  : AMDGPU::G_AMDGPU_BUFFER_LOAD;
      break;
    }
  }

  if (IsTFE) {
    unsigned NumValueDWords = divideCeil(Ty.getSizeInBits(), 32);
    unsigned NumLoadDWords = NumValueDWords + 1;
    LLT LoadTy = LLT::fixed_vector(NumLoadDWords, I32);
    Register LoadDstReg = B.getMRI()->createGenericVirtualRegister(LoadTy);
    buildBufferLoad(Opc, LoadDstReg, RSrc, VIndex, VOffset, SOffset, ImmOffset,
                    Format, AuxiliaryData, MMO, IsTyped, HasVIndex, B);
    bool IsFloat = Ty.getScalarType().isFloat();
    LLT DstIntTy =
        IsFloat ? Ty.changeElementType(LLT::integer(EltTy.getSizeInBits()))
                : Ty;
    Register DstInt =
        IsFloat ? B.getMRI()->createGenericVirtualRegister(DstIntTy) : Dst;
    if (MemTy.getSizeInBits() < 32) {
      Register ExtDst = B.getMRI()->createGenericVirtualRegister(I32);
      B.buildUnmerge({ExtDst, StatusDst}, LoadDstReg);
      B.buildTrunc(DstInt, ExtDst);
    } else if (NumValueDWords == 1) {
      B.buildUnmerge({DstInt, StatusDst}, LoadDstReg);
    } else {
      SmallVector<Register, 5> LoadElts;
      for (unsigned I = 0; I != NumValueDWords; ++I)
        LoadElts.push_back(B.getMRI()->createGenericVirtualRegister(I32));
      LoadElts.push_back(StatusDst);
      B.buildUnmerge(LoadElts, LoadDstReg);
      LoadElts.truncate(NumValueDWords);
      B.buildMergeLikeInstr(DstInt, LoadElts);
    }
    if (DstInt != Dst)
      B.buildBitcast(Dst, DstInt);
  } else if ((!IsD16 && MemTy.getSizeInBits() < 32) ||
             (IsD16 && !Ty.isVector())) {
    Register LoadDstReg = B.getMRI()->createGenericVirtualRegister(I32);
    buildBufferLoad(Opc, LoadDstReg, RSrc, VIndex, VOffset, SOffset, ImmOffset,
                    Format, AuxiliaryData, MMO, IsTyped, HasVIndex, B);
    B.setInsertPt(B.getMBB(), ++B.getInsertPt());
    B.buildTrunc(Dst, LoadDstReg);
  } else if (Unpacked && IsD16 && Ty.isVector()) {
    LLT UnpackedTy = LLT::fixed_vector(Ty.getNumElements(), LLT::integer(32));
    Register LoadDstReg = B.getMRI()->createGenericVirtualRegister(UnpackedTy);
    buildBufferLoad(Opc, LoadDstReg, RSrc, VIndex, VOffset, SOffset, ImmOffset,
                    Format, AuxiliaryData, MMO, IsTyped, HasVIndex, B);
    B.setInsertPt(B.getMBB(), ++B.getInsertPt());
    // FIXME: G_TRUNC should work, but legalization currently fails
    auto Unmerge = B.buildUnmerge(I32, LoadDstReg);
    SmallVector<Register, 4> Repack;
    for (unsigned I = 0, N = Unmerge->getNumOperands() - 1; I != N; ++I)
      Repack.push_back(B.buildTrunc(EltTy, Unmerge.getReg(I)).getReg(0));
    B.buildMergeLikeInstr(Dst, Repack);
  } else {
    buildBufferLoad(Opc, Dst, RSrc, VIndex, VOffset, SOffset, ImmOffset, Format,
                    AuxiliaryData, MMO, IsTyped, HasVIndex, B);
  }

  MI.eraseFromParent();
  return true;
}

static unsigned getBufferAtomicPseudo(Intrinsic::ID IntrID) {
  switch (IntrID) {
  case Intrinsic::amdgcn_raw_buffer_atomic_swap:
  case Intrinsic::amdgcn_raw_ptr_buffer_atomic_swap:
  case Intrinsic::amdgcn_struct_buffer_atomic_swap:
  case Intrinsic::amdgcn_struct_ptr_buffer_atomic_swap:
    return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SWAP;
  case Intrinsic::amdgcn_raw_buffer_atomic_add:
  case Intrinsic::amdgcn_raw_ptr_buffer_atomic_add:
  case Intrinsic::amdgcn_struct_buffer_atomic_add:
  case Intrinsic::amdgcn_struct_ptr_buffer_atomic_add:
    return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_ADD;
  case Intrinsic::amdgcn_raw_buffer_atomic_sub:
  case Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub:
  case Intrinsic::amdgcn_struct_buffer_atomic_sub:
  case Intrinsic::amdgcn_struct_ptr_buffer_atomic_sub:
    return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SUB;
  case Intrinsic::amdgcn_raw_buffer_atomic_smin:
  case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smin:
  case Intrinsic::amdgcn_struct_buffer_atomic_smin:
  case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smin:
    return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMIN;
  case Intrinsic::amdgcn_raw_buffer_atomic_umin:
  case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umin:
  case Intrinsic::amdgcn_struct_buffer_atomic_umin:
  case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umin:
    return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMIN;
  case Intrinsic::amdgcn_raw_buffer_atomic_smax:
  case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smax:
  case Intrinsic::amdgcn_struct_buffer_atomic_smax:
  case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smax:
    return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMAX;
  case Intrinsic::amdgcn_raw_buffer_atomic_umax:
  case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umax:
  case Intrinsic::amdgcn_struct_buffer_atomic_umax:
  case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umax:
    return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMAX;
  case Intrinsic::amdgcn_raw_buffer_atomic_and:
  case Intrinsic::amdgcn_raw_ptr_buffer_atomic_and:
  case Intrinsic::amdgcn_struct_buffer_atomic_and:
  case Intrinsic::amdgcn_struct_ptr_buffer_atomic_and:
    return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_AND;
  case Intrinsic::amdgcn_raw_buffer_atomic_or:
  case Intrinsic::amdgcn_raw_ptr_buffer_atomic_or:
  case Intrinsic::amdgcn_struct_buffer_atomic_or:
  case Intrinsic::amdgcn_struct_ptr_buffer_atomic_or:
    return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_OR;
  case Intrinsic::amdgcn_raw_buffer_atomic_xor:
  case Intrinsic::amdgcn_raw_ptr_buffer_atomic_xor:
  case Intrinsic::amdgcn_struct_buffer_atomic_xor:
  case Intrinsic::amdgcn_struct_ptr_buffer_atomic_xor:
    return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_XOR;
  case Intrinsic::amdgcn_raw_buffer_atomic_inc:
  case Intrinsic::amdgcn_raw_ptr_buffer_atomic_inc:
  case Intrinsic::amdgcn_struct_buffer_atomic_inc:
  case Intrinsic::amdgcn_struct_ptr_buffer_atomic_inc:
    return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_INC;
  case Intrinsic::amdgcn_raw_buffer_atomic_dec:
  case Intrinsic::amdgcn_raw_ptr_buffer_atomic_dec:
  case Intrinsic::amdgcn_struct_buffer_atomic_dec:
  case Intrinsic::amdgcn_struct_ptr_buffer_atomic_dec:
    return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_DEC;
  case Intrinsic::amdgcn_raw_buffer_atomic_cmpswap:
  case Intrinsic::amdgcn_raw_ptr_buffer_atomic_cmpswap:
  case Intrinsic::amdgcn_struct_buffer_atomic_cmpswap:
  case Intrinsic::amdgcn_struct_ptr_buffer_atomic_cmpswap:
    return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_CMPSWAP;
  case Intrinsic::amdgcn_raw_buffer_atomic_fadd:
  case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fadd:
  case Intrinsic::amdgcn_struct_buffer_atomic_fadd:
  case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fadd:
    return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FADD;
  case Intrinsic::amdgcn_raw_buffer_atomic_fmin:
  case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmin:
  case Intrinsic::amdgcn_struct_buffer_atomic_fmin:
  case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fmin:
    return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FMIN;
  case Intrinsic::amdgcn_raw_buffer_atomic_fmax:
  case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmax:
  case Intrinsic::amdgcn_struct_buffer_atomic_fmax:
  case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fmax:
    return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FMAX;
  case Intrinsic::amdgcn_raw_buffer_atomic_sub_clamp_u32:
  case Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub_clamp_u32:
  case Intrinsic::amdgcn_struct_buffer_atomic_sub_clamp_u32:
  case Intrinsic::amdgcn_struct_ptr_buffer_atomic_sub_clamp_u32:
    return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SUB_CLAMP_U32;
  case Intrinsic::amdgcn_raw_buffer_atomic_cond_sub_u32:
  case Intrinsic::amdgcn_raw_ptr_buffer_atomic_cond_sub_u32:
  case Intrinsic::amdgcn_struct_buffer_atomic_cond_sub_u32:
  case Intrinsic::amdgcn_struct_ptr_buffer_atomic_cond_sub_u32:
    return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_COND_SUB_U32;
  default:
    llvm_unreachable("unhandled atomic opcode");
  }
}

bool AMDGPULegalizerInfo::legalizeBufferAtomic(MachineInstr &MI,
                                               MachineIRBuilder &B,
                                               Intrinsic::ID IID) const {
  const bool IsCmpSwap =
      IID == Intrinsic::amdgcn_raw_buffer_atomic_cmpswap ||
      IID == Intrinsic::amdgcn_struct_buffer_atomic_cmpswap ||
      IID == Intrinsic::amdgcn_raw_ptr_buffer_atomic_cmpswap ||
      IID == Intrinsic::amdgcn_struct_ptr_buffer_atomic_cmpswap;

  Register Dst = MI.getOperand(0).getReg();
  // Since we don't have 128-bit atomics, we don't need to handle the case of
  // p8 argmunents to the atomic itself
  Register VData = MI.getOperand(2).getReg();

  Register CmpVal;
  int OpOffset = 0;

  if (IsCmpSwap) {
    CmpVal = MI.getOperand(3).getReg();
    ++OpOffset;
  }

  castBufferRsrcArgToV4I32(MI, B, 3 + OpOffset);
  Register RSrc = MI.getOperand(3 + OpOffset).getReg();
  const unsigned NumVIndexOps = IsCmpSwap ? 9 : 8;

  // The struct intrinsic variants add one additional operand over raw.
  const bool HasVIndex = MI.getNumOperands() == NumVIndexOps;
  Register VIndex;
  if (HasVIndex) {
    VIndex = MI.getOperand(4 + OpOffset).getReg();
    ++OpOffset;
  } else {
    VIndex = B.buildConstant(LLT::integer(32), 0).getReg(0);
  }

  Register VOffset = MI.getOperand(4 + OpOffset).getReg();
  Register SOffset = MI.getOperand(5 + OpOffset).getReg();
  unsigned AuxiliaryData = MI.getOperand(6 + OpOffset).getImm();

  MachineMemOperand *MMO = *MI.memoperands_begin();

  unsigned ImmOffset;
  std::tie(VOffset, ImmOffset) = splitBufferOffsets(B, VOffset);

  auto MIB = B.buildInstr(getBufferAtomicPseudo(IID))
      .addDef(Dst)
      .addUse(VData); // vdata

  if (IsCmpSwap)
    MIB.addReg(CmpVal);

  MIB.addUse(RSrc)               // rsrc
     .addUse(VIndex)             // vindex
     .addUse(VOffset)            // voffset
     .addUse(SOffset)            // soffset
     .addImm(ImmOffset)          // offset(imm)
     .addImm(AuxiliaryData)      // cachepolicy, swizzled buffer(imm)
     .addImm(HasVIndex ? -1 : 0) // idxen(imm)
     .addMemOperand(MMO);

  MI.eraseFromParent();
  return true;
}

/// Turn a set of f16 typed registers in \p AddrRegs into a dword sized
/// vector with f16 typed elements.
static void packImage16bitOpsToDwords(MachineIRBuilder &B, MachineInstr &MI,
                                      SmallVectorImpl<Register> &PackedAddrs,
                                      unsigned ArgOffset,
                                      const AMDGPU::ImageDimIntrinsicInfo *Intr,
                                      bool IsA16, bool IsG16) {
  auto EndIdx = Intr->VAddrEnd;

  for (unsigned I = Intr->VAddrStart; I < EndIdx; I++) {
    MachineOperand &SrcOp = MI.getOperand(ArgOffset + I);
    if (!SrcOp.isReg())
      continue; // _L to _LZ may have eliminated this.

    Register AddrReg = SrcOp.getReg();

    if ((I < Intr->GradientStart) ||
        (I >= Intr->GradientStart && I < Intr->CoordStart && !IsG16) ||
        (I >= Intr->CoordStart && !IsA16)) {
      if ((I < Intr->GradientStart) && IsA16 &&
          (B.getMRI()->getType(AddrReg) == F16)) {
        assert(I == Intr->BiasIndex && "Got unexpected 16-bit extra argument");
        // Special handling of bias when A16 is on. Bias is of type half but
        // occupies full 32-bit.
        PackedAddrs.push_back(
            B.buildBuildVector(V2F16, {AddrReg, B.buildUndef(F16).getReg(0)})
                .getReg(0));
      } else {
        assert((!IsA16 || Intr->NumBiasArgs == 0 || I != Intr->BiasIndex) &&
               "Bias needs to be converted to 16 bit in A16 mode");
        // Handle any gradient or coordinate operands that should not be packed
        AddrReg = B.buildBitcast(V2F16, AddrReg).getReg(0);
        PackedAddrs.push_back(AddrReg);
      }
    } else {
      const LLT EltTy = B.getMRI()->getType(AddrReg);
      const LLT V2EltTy = LLT::fixed_vector(2, EltTy);
      // Dz/dh, dz/dv and the last odd coord are packed with undef. Also, in 1D,
      // derivatives dx/dh and dx/dv are packed with undef.
      if (((I + 1) >= EndIdx) ||
          ((Intr->NumGradients / 2) % 2 == 1 &&
           (I == static_cast<unsigned>(Intr->GradientStart +
                                       (Intr->NumGradients / 2) - 1) ||
            I == static_cast<unsigned>(Intr->GradientStart +
                                       Intr->NumGradients - 1))) ||
          // Check for _L to _LZ optimization
          !MI.getOperand(ArgOffset + I + 1).isReg()) {
        PackedAddrs.push_back(
            B.buildBuildVector(V2EltTy,
                               {AddrReg, B.buildUndef(EltTy).getReg(0)})
                .getReg(0));
      } else {
        PackedAddrs.push_back(
            B.buildBuildVector(
                 V2EltTy, {AddrReg, MI.getOperand(ArgOffset + I + 1).getReg()})
                .getReg(0));
        ++I;
      }
    }
  }
}

/// Convert from separate vaddr components to a single vector address register,
/// and replace the remaining operands with $noreg.
static void convertImageAddrToPacked(MachineIRBuilder &B, MachineInstr &MI,
                                     int DimIdx, int NumVAddrs) {
  SmallVector<Register, 8> AddrRegs;
  for (int I = 0; I != NumVAddrs; ++I) {
    MachineOperand &SrcOp = MI.getOperand(DimIdx + I);
    if (SrcOp.isReg()) {
      Register Reg = SrcOp.getReg();
      LLT I32 = LLT::integer(32);
      assert(B.getMRI()->getType(Reg).getSizeInBits() == 32);
      if (B.getMRI()->getType(Reg) != I32)
        Reg = B.buildBitcast(I32, Reg).getReg(0);
      AddrRegs.push_back(Reg);
    }
  }

  int NumAddrRegs = AddrRegs.size();
  if (NumAddrRegs != 1) {
    LLT EltTy = B.getMRI()->getType(AddrRegs[0]);
    auto VAddr =
        B.buildBuildVector(LLT::fixed_vector(NumAddrRegs, EltTy), AddrRegs);
    MI.getOperand(DimIdx).setReg(VAddr.getReg(0));
  }

  for (int I = 1; I != NumVAddrs; ++I) {
    MachineOperand &SrcOp = MI.getOperand(DimIdx + I);
    if (SrcOp.isReg())
      MI.getOperand(DimIdx + I).setReg(AMDGPU::NoRegister);
  }
}

/// Rewrite image intrinsics to use register layouts expected by the subtarget.
///
/// Depending on the subtarget, load/store with 16-bit element data need to be
/// rewritten to use the low half of 32-bit registers, or directly use a packed
/// layout. 16-bit addresses should also sometimes be packed into 32-bit
/// registers.
///
/// We don't want to directly select image instructions just yet, but also want
/// to exposes all register repacking to the legalizer/combiners. We also don't
/// want a selected instruction entering RegBankSelect. In order to avoid
/// defining a multitude of intermediate image instructions, directly hack on
/// the intrinsic's arguments. In cases like a16 addresses, this requires
/// padding now unnecessary arguments with $noreg.
bool AMDGPULegalizerInfo::legalizeImageIntrinsic(
    MachineInstr &MI, MachineIRBuilder &B, GISelChangeObserver &Observer,
    const AMDGPU::ImageDimIntrinsicInfo *Intr) const {

  const MachineFunction &MF = *MI.getMF();
  const unsigned NumDefs = MI.getNumExplicitDefs();
  const unsigned ArgOffset = NumDefs + 1;
  bool IsTFE = NumDefs == 2;
  // We are only processing the operands of d16 image operations on subtargets
  // that use the unpacked register layout, or need to repack the TFE result.

  // TODO: Do we need to guard against already legalized intrinsics?
  const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
      AMDGPU::getMIMGBaseOpcodeInfo(Intr->BaseOpcode);

  MachineRegisterInfo *MRI = B.getMRI();
  const LLT I32 = LLT::integer(32);
  const LLT I16 = LLT::integer(16);
  const LLT V2I16 = LLT::fixed_vector(2, I16);

  unsigned DMask = 0;
  Register VData;
  LLT Ty;

  if (!BaseOpcode->NoReturn || BaseOpcode->Store) {
    VData = MI.getOperand(NumDefs == 0 ? 1 : 0).getReg();
    Ty = MRI->getType(VData);
  }

  const bool IsAtomicPacked16Bit =
      (BaseOpcode->BaseOpcode == AMDGPU::IMAGE_ATOMIC_PK_ADD_F16 ||
       BaseOpcode->BaseOpcode == AMDGPU::IMAGE_ATOMIC_PK_ADD_BF16);

  // Check for 16 bit addresses and pack if true.
  LLT GradTy =
      MRI->getType(MI.getOperand(ArgOffset + Intr->GradientStart).getReg());
  LLT AddrTy =
      MRI->getType(MI.getOperand(ArgOffset + Intr->CoordStart).getReg());
  const bool GradTyIs16 = GradTy == I16 || GradTy == F16;
  const bool AddrTyIs16 = AddrTy == I16 || AddrTy == F16;
  const bool DataTyIs16 =
      Ty.getScalarType() == I16 || Ty.getScalarType() == F16;
  const bool IsG16 =
      ST.hasG16() ? (BaseOpcode->Gradients && GradTyIs16) : GradTyIs16;
  const bool IsA16 = AddrTyIs16;
  const bool IsD16 = !IsAtomicPacked16Bit && DataTyIs16;

  int DMaskLanes = 0;
  if (!BaseOpcode->Atomic) {
    DMask = MI.getOperand(ArgOffset + Intr->DMaskIndex).getImm();
    if (BaseOpcode->Gather4) {
      DMaskLanes = 4;
    } else if (DMask != 0) {
      DMaskLanes = llvm::popcount(DMask);
    } else if (!IsTFE && !BaseOpcode->Store) {
      // If dmask is 0, this is a no-op load. This can be eliminated.
      B.buildUndef(MI.getOperand(0));
      MI.eraseFromParent();
      return true;
    }
  }

  Observer.changingInstr(MI);
  scope_exit ChangedInstr([&] { Observer.changedInstr(MI); });

  const unsigned StoreOpcode = IsD16 ? AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE_D16
                                     : AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE;
  const unsigned LoadOpcode = IsD16 ? AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_D16
                                    : AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD;
  unsigned NewOpcode = LoadOpcode;
  if (BaseOpcode->Store)
    NewOpcode = StoreOpcode;
  else if (BaseOpcode->NoReturn)
    NewOpcode = AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_NORET;

  // Track that we legalized this
  MI.setDesc(B.getTII().get(NewOpcode));

  // Expecting to get an error flag since TFC is on - and dmask is 0 Force
  // dmask to be at least 1 otherwise the instruction will fail
  if (IsTFE && DMask == 0) {
    DMask = 0x1;
    DMaskLanes = 1;
    MI.getOperand(ArgOffset + Intr->DMaskIndex).setImm(DMask);
  }

  if (BaseOpcode->Atomic) {
    Register VData0 = MI.getOperand(2).getReg();
    LLT Ty = MRI->getType(VData0);

    // TODO: Allow atomic swap and bit ops for v2f16/v4f16
    if (Ty.isVector() && !IsAtomicPacked16Bit)
      return false;

    if (BaseOpcode->AtomicX2) {
      Register VData1 = MI.getOperand(3).getReg();
      // The two values are packed in one register.
      LLT PackedTy = LLT::fixed_vector(2, Ty);
      auto Concat = B.buildBuildVector(PackedTy, {VData0, VData1});
      MI.getOperand(2).setReg(Concat.getReg(0));
      MI.getOperand(3).setReg(AMDGPU::NoRegister);
    }
  }

  unsigned CorrectedNumVAddrs = Intr->NumVAddrs;

  // Rewrite the addressing register layout before doing anything else.
  if (BaseOpcode->Gradients && !ST.hasG16() && (IsA16 != IsG16)) {
    // 16 bit gradients are supported, but are tied to the A16 control
    // so both gradients and addresses must be 16 bit
    return false;
  }

  if (IsA16 && !ST.hasA16()) {
    // A16 not supported
    return false;
  }

  const unsigned NSAMaxSize = ST.getNSAMaxSize(BaseOpcode->Sampler);
  const unsigned HasPartialNSA = ST.hasPartialNSAEncoding();

  if (IsA16 || IsG16) {
    // Even if NumVAddrs == 1 we should pack it into a 32-bit value, because the
    // instructions expect VGPR_32
    SmallVector<Register, 4> PackedRegs;

    packImage16bitOpsToDwords(B, MI, PackedRegs, ArgOffset, Intr, IsA16, IsG16);

    // See also below in the non-a16 branch
    const bool UseNSA = ST.hasNSAEncoding() &&
                        PackedRegs.size() >= ST.getNSAThreshold(MF) &&
                        (PackedRegs.size() <= NSAMaxSize || HasPartialNSA);
    const bool UsePartialNSA =
        UseNSA && HasPartialNSA && PackedRegs.size() > NSAMaxSize;

    if (UsePartialNSA) {
      // Pack registers that would go over NSAMaxSize into last VAddr register
      LLT PackedAddrTy =
          LLT::fixed_vector(2 * (PackedRegs.size() - NSAMaxSize + 1), F16);
      auto Concat = B.buildConcatVectors(
          PackedAddrTy, ArrayRef(PackedRegs).slice(NSAMaxSize - 1));
      PackedRegs[NSAMaxSize - 1] = Concat.getReg(0);
      PackedRegs.resize(NSAMaxSize);
    } else if (!UseNSA && PackedRegs.size() > 1) {
      LLT PackedAddrTy = LLT::fixed_vector(2 * PackedRegs.size(), F16);
      auto Concat = B.buildConcatVectors(PackedAddrTy, PackedRegs);
      PackedRegs[0] = Concat.getReg(0);
      PackedRegs.resize(1);
    }

    const unsigned NumPacked = PackedRegs.size();
    for (unsigned I = Intr->VAddrStart; I < Intr->VAddrEnd; I++) {
      MachineOperand &SrcOp = MI.getOperand(ArgOffset + I);
      if (!SrcOp.isReg()) {
        assert(SrcOp.isImm() && SrcOp.getImm() == 0);
        continue;
      }

      assert(SrcOp.getReg() != AMDGPU::NoRegister);

      if (I - Intr->VAddrStart < NumPacked)
        SrcOp.setReg(PackedRegs[I - Intr->VAddrStart]);
      else
        SrcOp.setReg(AMDGPU::NoRegister);
    }
  } else {
    // If the register allocator cannot place the address registers contiguously
    // without introducing moves, then using the non-sequential address encoding
    // is always preferable, since it saves VALU instructions and is usually a
    // wash in terms of code size or even better.
    //
    // However, we currently have no way of hinting to the register allocator
    // that MIMG addresses should be placed contiguously when it is possible to
    // do so, so force non-NSA for the common 2-address case as a heuristic.
    //
    // SIShrinkInstructions will convert NSA encodings to non-NSA after register
    // allocation when possible.
    //
    // Partial NSA is allowed on GFX11+ where the final register is a contiguous
    // set of the remaining addresses.
    const bool UseNSA = ST.hasNSAEncoding() &&
                        CorrectedNumVAddrs >= ST.getNSAThreshold(MF) &&
                        (CorrectedNumVAddrs <= NSAMaxSize || HasPartialNSA);
    const bool UsePartialNSA =
        UseNSA && HasPartialNSA && CorrectedNumVAddrs > NSAMaxSize;

    if (UsePartialNSA) {
      convertImageAddrToPacked(B, MI,
                               ArgOffset + Intr->VAddrStart + NSAMaxSize - 1,
                               Intr->NumVAddrs - NSAMaxSize + 1);
    } else if (!UseNSA && Intr->NumVAddrs > 1) {
      convertImageAddrToPacked(B, MI, ArgOffset + Intr->VAddrStart,
                               Intr->NumVAddrs);
    }
  }

  int Flags = 0;
  if (IsA16)
    Flags |= 1;
  if (IsG16)
    Flags |= 2;
  MI.addOperand(MachineOperand::CreateImm(Flags));

  if (BaseOpcode->NoReturn) { // No TFE for stores?
    // TODO: Handle dmask trim
    if (!Ty.isVector() || !IsD16)
      return true;

    Register RepackedReg = handleD16VData(B, *MRI, VData, true);
    if (RepackedReg != VData) {
      MI.getOperand(1).setReg(RepackedReg);
    }

    return true;
  }

  Register DstReg = MI.getOperand(0).getReg();
  const LLT EltTy = Ty.getScalarType();
  const int NumElts = Ty.isVector() ? Ty.getNumElements() : 1;

  // Confirm that the return type is large enough for the dmask specified
  if (NumElts < DMaskLanes)
    return false;

  if (NumElts > 4 || DMaskLanes > 4)
    return false;

  // Image atomic instructions are using DMask to specify how many bits
  // input/output data will have. 32-bits (i32, f32, v2f16) or 64-bits (i64,
  // f64, v4f16).
  // DMaskLanes for image atomic has default value '0'.
  // We must be sure that atomic variants (especially packed) will not be
  // truncated from v2f16 or v4f16 to f16 type.
  //
  // ChangeElementCount will be needed for image load where Ty is always scalar.
  const unsigned AdjustedNumElts = DMaskLanes == 0 ? 1 : DMaskLanes;
  const LLT AdjustedTy =
      DMaskLanes == 0
          ? Ty
          : Ty.changeElementCount(ElementCount::getFixed(AdjustedNumElts));

  // The raw dword aligned data component of the load. The only legal cases
  // where this matters should be when using the packed D16 format, for
  // f16 -> <2 x f16>, and <3 x f16> -> <4 x f16>,
  LLT RoundedTy;

  // I32 vector to cover all data, plus TFE result element.
  LLT TFETy;

  // Register type to use for each loaded component. Will be I32 or V2I16.
  LLT RegTy;

  if (IsD16 && ST.hasUnpackedD16VMem()) {
    RoundedTy =
        LLT::scalarOrVector(ElementCount::getFixed(AdjustedNumElts), I32);
    TFETy = LLT::fixed_vector(AdjustedNumElts + 1, I32);
    RegTy = I32;
  } else {
    unsigned EltSize = EltTy.getSizeInBits();
    unsigned RoundedElts = (AdjustedTy.getSizeInBits() + 31) / 32;
    unsigned RoundedSize = 32 * RoundedElts;
    RoundedTy = LLT::scalarOrVector(
        ElementCount::getFixed(RoundedSize / EltSize), EltTy);
    TFETy = LLT::fixed_vector(RoundedSize / 32 + 1, I32);
    RegTy = !IsTFE && EltSize == 16 ? V2I16 : I32;
  }

  // The return type does not need adjustment.
  // TODO: Should we change f16 case to i32 or <2 x f16>?
  if (!IsTFE && (RoundedTy == Ty || !Ty.isVector()))
    return true;

  Register Dst1Reg;

  // Insert after the instruction.
  B.setInsertPt(*MI.getParent(), ++MI.getIterator());

  // TODO: For TFE with d16, if we used a TFE type that was a multiple of <2 x
  // f16> instead of i32, we would only need 1 bitcast instead of multiple.
  const LLT LoadResultTy = IsTFE ? TFETy : RoundedTy;
  const int ResultNumRegs = LoadResultTy.getSizeInBits() / 32;

  Register NewResultReg = MRI->createGenericVirtualRegister(LoadResultTy);

  MI.getOperand(0).setReg(NewResultReg);

  // In the IR, TFE is supposed to be used with a 2 element struct return
  // type. The instruction really returns these two values in one contiguous
  // register, with one additional dword beyond the loaded data. Rewrite the
  // return type to use a single register result.

  if (IsTFE) {
    Dst1Reg = MI.getOperand(1).getReg();
    if (MRI->getType(Dst1Reg) != I32)
      return false;

    // TODO: Make sure the TFE operand bit is set.
    MI.removeOperand(1);

    // Handle the easy case that requires no repack instructions.
    if (!Ty.isVector() && Ty.getSizeInBits() == 32) {
      auto Unmerge = B.buildUnmerge({I32, I32}, NewResultReg);
      B.buildBitcast(DstReg, Unmerge.getReg(0));
      B.buildCopy(Dst1Reg, Unmerge.getReg(1));
      return true;
    }
  }

  // Now figure out how to copy the new result register back into the old
  // result.
  SmallVector<Register, 5> ResultRegs(ResultNumRegs, Dst1Reg);

  const int NumDataRegs = IsTFE ? ResultNumRegs - 1  : ResultNumRegs;

  if (ResultNumRegs == 1) {
    assert(!IsTFE);
    ResultRegs[0] = NewResultReg;
  } else {
    // We have to repack into a new vector of some kind.
    for (int I = 0; I != NumDataRegs; ++I)
      ResultRegs[I] = MRI->createGenericVirtualRegister(RegTy);
    B.buildUnmerge(ResultRegs, NewResultReg);

    // Drop the final TFE element to get the data part. The TFE result is
    // directly written to the right place already.
    if (IsTFE)
      ResultRegs.resize(NumDataRegs);
  }

  // For an f16 scalar result, we form an i32 result with a truncate regardless
  // of packed vs. unpacked.
  if (IsD16 && !Ty.isVector()) {
    B.buildTrunc(DstReg, ResultRegs[0]);
    return true;
  }

  // Avoid a build/concat_vector of 1 entry.
  if ((Ty == V2I16 || Ty == V2F16) && NumDataRegs == 1 &&
      !ST.hasUnpackedD16VMem()) {
    B.buildBitcast(DstReg, ResultRegs[0]);
    return true;
  }

  assert(Ty.isVector());

  if (IsD16) {
    // For packed D16 results with TFE enabled, all the data components are
    // I32. Cast back to the expected type.
    //
    // TODO: We don't really need to use load i32 elements. We would only need
    // one cast for the TFE result if a multiple of v2f16 was used.
    if (RegTy != V2I16 && !ST.hasUnpackedD16VMem()) {
      for (Register &Reg : ResultRegs)
        Reg = B.buildBitcast(V2I16, Reg).getReg(0);
    } else if (ST.hasUnpackedD16VMem()) {
      for (Register &Reg : ResultRegs)
        Reg = B.buildTrunc(I16, Reg).getReg(0);
    }
  }

  auto padWithUndef = [&](LLT Ty, int NumElts) {
    if (NumElts == 0)
      return;
    Register Undef = B.buildUndef(Ty).getReg(0);
    for (int I = 0; I != NumElts; ++I)
      ResultRegs.push_back(Undef);
  };

  // Pad out any elements eliminated due to the dmask.
  LLT ResTy = MRI->getType(ResultRegs[0]);
  if (!ResTy.isVector()) {
    padWithUndef(ResTy, NumElts - ResultRegs.size());
    B.buildBuildVector(DstReg, ResultRegs);
    return true;
  }

  assert(!ST.hasUnpackedD16VMem() && (ResTy == V2I16 || ResTy == V2F16));
  const int RegsToCover = (Ty.getSizeInBits() + 31) / 32;

  // Deal with the one annoying legal case.
  const LLT V3I16 = LLT::fixed_vector(3, I16);
  const LLT V3F16 = LLT::fixed_vector(3, F16);
  if (Ty == V3I16 || Ty == V3F16) {
    if (IsTFE) {
      if (ResultRegs.size() == 1) {
        NewResultReg = ResultRegs[0];
      } else if (ResultRegs.size() == 2) {
        LLT V4I16 = LLT::fixed_vector(4, I16);
        NewResultReg = B.buildConcatVectors(V4I16, ResultRegs).getReg(0);
      } else {
        return false;
      }
    }

    LLT DstTy = MRI->getType(DstReg);
    LLT NewResTy = MRI->getType(NewResultReg);
    LLT ResEltTy = NewResTy.getElementType();
    Register ResizeDst = DstTy.getElementType() == ResEltTy
                             ? DstReg
                             : MRI->createGenericVirtualRegister(
                                   DstTy.changeElementType(ResEltTy));

    if (DstTy.getNumElements() < NewResTy.getNumElements()) {
      B.buildDeleteTrailingVectorElements(ResizeDst, NewResultReg);
    } else {
      B.buildPadVectorWithUndefElements(ResizeDst, NewResultReg);
    }
    if (ResizeDst != DstReg)
      B.buildBitcast(DstReg, ResizeDst);
    return true;
  }

  padWithUndef(ResTy, RegsToCover - ResultRegs.size());
  B.buildConcatVectors(DstReg, ResultRegs);
  return true;
}

bool AMDGPULegalizerInfo::legalizeSBufferLoad(LegalizerHelper &Helper,
                                              MachineInstr &MI) const {
  MachineIRBuilder &B = Helper.MIRBuilder;
  GISelChangeObserver &Observer = Helper.Observer;

  Register OrigDst = MI.getOperand(0).getReg();
  Register Dst;
  LLT Ty = B.getMRI()->getType(OrigDst);
  unsigned Size = Ty.getSizeInBits();
  MachineFunction &MF = B.getMF();
  unsigned Opc = 0;
  if (Size < 32 && ST.hasScalarSubwordLoads()) {
    assert(Size == 8 || Size == 16);
    Opc = Size == 8 ? AMDGPU::G_AMDGPU_S_BUFFER_LOAD_UBYTE
                    : AMDGPU::G_AMDGPU_S_BUFFER_LOAD_USHORT;
    // The 8-bit and 16-bit scalar buffer load instructions have 32-bit
    // destination register.
    Dst = B.getMRI()->createGenericVirtualRegister(LLT::integer(32));
  } else {
    Opc = AMDGPU::G_AMDGPU_S_BUFFER_LOAD;
    Dst = OrigDst;
  }

  Observer.changingInstr(MI);

  // Handle needing to s.buffer.load() a p8 value.
  if (hasBufferRsrcWorkaround(Ty)) {
    Ty = castBufferRsrcFromV4I32(MI, B, *B.getMRI(), 0);
    B.setInsertPt(B.getMBB(), MI);
  }
  if (shouldBitcastLoadStoreType(ST, Ty, LLT::integer(Size))) {
    Ty = getBitcastRegisterType(Ty);
    Helper.bitcastDst(MI, Ty, 0);
    B.setInsertPt(B.getMBB(), MI);
  }

  // FIXME: We don't really need this intermediate instruction. The intrinsic
  // should be fixed to have a memory operand. Since it's readnone, we're not
  // allowed to add one.
  MI.setDesc(B.getTII().get(Opc));
  MI.removeOperand(1); // Remove intrinsic ID

  // FIXME: When intrinsic definition is fixed, this should have an MMO already.
  const unsigned MemSize = (Size + 7) / 8;
  const Align MemAlign = B.getDataLayout().getABITypeAlign(
      getTypeForLLT(Ty, MF.getFunction().getContext()));
  MachineMemOperand *MMO = MF.getMachineMemOperand(
      MachinePointerInfo(),
      MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
          MachineMemOperand::MOInvariant,
      MemSize, MemAlign);
  MI.addMemOperand(MF, MMO);
  if (Dst != OrigDst) {
    MI.getOperand(0).setReg(Dst);
    B.setInsertPt(B.getMBB(), ++B.getInsertPt());
    B.buildTrunc(OrigDst, Dst);
  }

  // If we don't have 96-bit result scalar loads, widening to 128-bit should
  // always be legal. We may need to restore this to a 96-bit result if it turns
  // out this needs to be converted to a vector load during RegBankSelect.
  if (!isPowerOf2_32(Size) && (Size != 96 || !ST.hasScalarDwordx3Loads())) {
    if (Ty.isVector())
      Helper.moreElementsVectorDst(MI, getPow2VectorType(Ty), 0);
    else
      Helper.widenScalarDst(MI, getPow2ScalarType(Ty), 0);
  }

  Observer.changedInstr(MI);
  return true;
}

bool AMDGPULegalizerInfo::legalizeSBufferPrefetch(LegalizerHelper &Helper,
                                                  MachineInstr &MI) const {
  MachineIRBuilder &B = Helper.MIRBuilder;
  GISelChangeObserver &Observer = Helper.Observer;
  Observer.changingInstr(MI);
  MI.setDesc(B.getTII().get(AMDGPU::G_AMDGPU_S_BUFFER_PREFETCH));
  MI.removeOperand(0); // Remove intrinsic ID
  castBufferRsrcArgToV4I32(MI, B, 0);
  Observer.changedInstr(MI);
  return true;
}

// TODO: Move to selection
bool AMDGPULegalizerInfo::legalizeTrap(MachineInstr &MI,
                                       MachineRegisterInfo &MRI,
                                       MachineIRBuilder &B) const {
  if (!ST.hasTrapHandler() ||
      ST.getTrapHandlerAbi() != GCNSubtarget::TrapHandlerAbi::AMDHSA)
    return legalizeTrapEndpgm(MI, MRI, B);

  return ST.supportsGetDoorbellID() ?
         legalizeTrapHsa(MI, MRI, B) : legalizeTrapHsaQueuePtr(MI, MRI, B);
}

bool AMDGPULegalizerInfo::legalizeTrapEndpgm(
    MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const {
  const DebugLoc &DL = MI.getDebugLoc();
  MachineBasicBlock &BB = B.getMBB();
  MachineFunction *MF = BB.getParent();

  if (BB.succ_empty() && std::next(MI.getIterator()) == BB.end()) {
    BuildMI(BB, BB.end(), DL, B.getTII().get(AMDGPU::S_ENDPGM))
      .addImm(0);
    MI.eraseFromParent();
    return true;
  }

  // We need a block split to make the real endpgm a terminator. We also don't
  // want to break phis in successor blocks, so we can't just delete to the
  // end of the block.
  BB.splitAt(MI, false /*UpdateLiveIns*/);
  MachineBasicBlock *TrapBB = MF->CreateMachineBasicBlock();
  MF->push_back(TrapBB);
  BuildMI(*TrapBB, TrapBB->end(), DL, B.getTII().get(AMDGPU::S_ENDPGM))
    .addImm(0);
  BuildMI(BB, &MI, DL, B.getTII().get(AMDGPU::S_CBRANCH_EXECNZ))
    .addMBB(TrapBB);

  BB.addSuccessor(TrapBB);
  MI.eraseFromParent();
  return true;
}

bool AMDGPULegalizerInfo::legalizeTrapHsaQueuePtr(
    MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const {
  MachineFunction &MF = B.getMF();
  const LLT I64 = LLT::integer(64);

  Register SGPR01(AMDGPU::SGPR0_SGPR1);
  // For code object version 5, queue_ptr is passed through implicit kernarg.
  if (AMDGPU::getAMDHSACodeObjectVersion(*MF.getFunction().getParent()) >=
      AMDGPU::AMDHSA_COV5) {
    AMDGPUTargetLowering::ImplicitParameter Param =
        AMDGPUTargetLowering::QUEUE_PTR;
    uint64_t Offset =
        ST.getTargetLowering()->getImplicitParameterOffset(B.getMF(), Param);

    Register KernargPtrReg = MRI.createGenericVirtualRegister(
        LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64));

    if (!loadInputValue(KernargPtrReg, B,
                        AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR))
      return false;

    // TODO: can we be smarter about machine pointer info?
    MachinePointerInfo PtrInfo = getKernargSegmentPtrInfo(MF);
    MachineMemOperand *MMO = MF.getMachineMemOperand(
        PtrInfo.getWithOffset(Offset),
        MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
            MachineMemOperand::MOInvariant,
        LLT::integer(64), commonAlignment(Align(64), Offset));

    // Pointer address
    Register LoadAddr = MRI.createGenericVirtualRegister(
        LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64));
    B.buildObjectPtrOffset(LoadAddr, KernargPtrReg,
                           B.buildConstant(LLT::integer(64), Offset).getReg(0));
    // Load address
    Register Temp = B.buildLoad(I64, LoadAddr, *MMO).getReg(0);
    B.buildCopy(SGPR01, Temp);
    B.buildInstr(AMDGPU::S_TRAP)
        .addImm(static_cast<unsigned>(GCNSubtarget::TrapID::LLVMAMDHSATrap))
        .addReg(SGPR01, RegState::Implicit);
    MI.eraseFromParent();
    return true;
  }

  // Pass queue pointer to trap handler as input, and insert trap instruction
  // Reference: https://llvm.org/docs/AMDGPUUsage.html#trap-handler-abi
  Register LiveIn =
    MRI.createGenericVirtualRegister(LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64));
  if (!loadInputValue(LiveIn, B, AMDGPUFunctionArgInfo::QUEUE_PTR))
    return false;

  B.buildCopy(SGPR01, LiveIn);
  B.buildInstr(AMDGPU::S_TRAP)
      .addImm(static_cast<unsigned>(GCNSubtarget::TrapID::LLVMAMDHSATrap))
      .addReg(SGPR01, RegState::Implicit);

  MI.eraseFromParent();
  return true;
}

bool AMDGPULegalizerInfo::legalizeTrapHsa(MachineInstr &MI,
                                          MachineRegisterInfo &MRI,
                                          MachineIRBuilder &B) const {
  // We need to simulate the 's_trap 2' instruction on targets that run in
  // PRIV=1 (where it is treated as a nop).
  if (ST.hasPrivEnabledTrap2NopBug()) {
    ST.getInstrInfo()->insertSimulatedTrap(MRI, B.getMBB(), MI,
                                           MI.getDebugLoc());
    MI.eraseFromParent();
    return true;
  }

  B.buildInstr(AMDGPU::S_TRAP)
      .addImm(static_cast<unsigned>(GCNSubtarget::TrapID::LLVMAMDHSATrap));
  MI.eraseFromParent();
  return true;
}

bool AMDGPULegalizerInfo::legalizeDebugTrap(MachineInstr &MI,
                                            MachineRegisterInfo &MRI,
                                            MachineIRBuilder &B) const {
  // Is non-HSA path or trap-handler disabled? Then, report a warning
  // accordingly
  if (!ST.hasTrapHandler() ||
      ST.getTrapHandlerAbi() != GCNSubtarget::TrapHandlerAbi::AMDHSA) {
    Function &Fn = B.getMF().getFunction();
    Fn.getContext().diagnose(DiagnosticInfoUnsupported(
        Fn, "debugtrap handler not supported", MI.getDebugLoc(), DS_Warning));
  } else {
    // Insert debug-trap instruction
    B.buildInstr(AMDGPU::S_TRAP)
        .addImm(static_cast<unsigned>(GCNSubtarget::TrapID::LLVMAMDHSADebugTrap));
  }

  MI.eraseFromParent();
  return true;
}

bool AMDGPULegalizerInfo::legalizeBVHIntersectRayIntrinsic(
    MachineInstr &MI, MachineIRBuilder &B) const {
  MachineRegisterInfo &MRI = *B.getMRI();
  const LLT I16 = LLT::integer(16);
  const LLT I32 = LLT::integer(32);
  const LLT V2I16 = LLT::fixed_vector(2, I16);
  const LLT V3I32 = LLT::fixed_vector(3, I32);
  const LLT V3I16 = LLT::fixed_vector(3, I16);

  Register DstReg = MI.getOperand(0).getReg();
  Register NodePtr = MI.getOperand(2).getReg();
  Register RayExtent = MI.getOperand(3).getReg();
  Register RayOrigin = MI.getOperand(4).getReg();
  Register RayDir = MI.getOperand(5).getReg();
  Register RayInvDir = MI.getOperand(6).getReg();
  Register TDescr = MI.getOperand(7).getReg();

  RayExtent = B.buildBitcast(I32, RayExtent).getReg(0);

  const bool IsGFX11 = AMDGPU::isGFX11(ST);
  const bool IsGFX11Plus = AMDGPU::isGFX11Plus(ST);
  const bool IsGFX12Plus = AMDGPU::isGFX12Plus(ST);
  const bool IsA16 = MRI.getType(RayDir).getElementType().getSizeInBits() == 16;
  const bool Is64 = MRI.getType(NodePtr).getSizeInBits() == 64;
  const unsigned NumVDataDwords = 4;
  const unsigned NumVAddrDwords = IsA16 ? (Is64 ? 9 : 8) : (Is64 ? 12 : 11);
  const unsigned NumVAddrs = IsGFX11Plus ? (IsA16 ? 4 : 5) : NumVAddrDwords;
  const bool UseNSA =
      IsGFX12Plus || (ST.hasNSAEncoding() && NumVAddrs <= ST.getNSAMaxSize());

  const unsigned BaseOpcodes[2][2] = {
      {AMDGPU::IMAGE_BVH_INTERSECT_RAY, AMDGPU::IMAGE_BVH_INTERSECT_RAY_a16},
      {AMDGPU::IMAGE_BVH64_INTERSECT_RAY,
       AMDGPU::IMAGE_BVH64_INTERSECT_RAY_a16}};
  int Opcode;
  if (UseNSA) {
    Opcode = AMDGPU::getMIMGOpcode(BaseOpcodes[Is64][IsA16],
                                   IsGFX12Plus ? AMDGPU::MIMGEncGfx12
                                   : IsGFX11   ? AMDGPU::MIMGEncGfx11NSA
                                               : AMDGPU::MIMGEncGfx10NSA,
                                   NumVDataDwords, NumVAddrDwords);
  } else {
    assert(!IsGFX12Plus);
    Opcode = AMDGPU::getMIMGOpcode(BaseOpcodes[Is64][IsA16],
                                   IsGFX11 ? AMDGPU::MIMGEncGfx11Default
                                           : AMDGPU::MIMGEncGfx10Default,
                                   NumVDataDwords, NumVAddrDwords);
  }
  assert(Opcode != -1);

  SmallVector<Register, 12> Ops;
  if (UseNSA && IsGFX11Plus) {
    auto packLanes = [&Ops, &I32, &V3I32, &B](Register Src) {
      auto SrcInt = B.buildBitcast(V3I32, Src);
      auto Unmerge = B.buildUnmerge({I32, I32, I32}, SrcInt);
      auto Merged = B.buildMergeLikeInstr(
          V3I32, {Unmerge.getReg(0), Unmerge.getReg(1), Unmerge.getReg(2)});
      Ops.push_back(Merged.getReg(0));
    };

    Ops.push_back(NodePtr);
    Ops.push_back(RayExtent);
    packLanes(RayOrigin);

    if (IsA16) {
      auto UnmergeRayDir =
          B.buildUnmerge({I16, I16, I16}, B.buildBitcast(V3I16, RayDir));
      auto UnmergeRayInvDir =
          B.buildUnmerge({I16, I16, I16}, B.buildBitcast(V3I16, RayInvDir));
      auto MergedDir = B.buildMergeLikeInstr(
          V3I32,
          {B.buildBitcast(
                I32, B.buildMergeLikeInstr(V2I16, {UnmergeRayInvDir.getReg(0),
                                                   UnmergeRayDir.getReg(0)}))
               .getReg(0),
           B.buildBitcast(
                I32, B.buildMergeLikeInstr(V2I16, {UnmergeRayInvDir.getReg(1),
                                                   UnmergeRayDir.getReg(1)}))
               .getReg(0),
           B.buildBitcast(
                I32, B.buildMergeLikeInstr(V2I16, {UnmergeRayInvDir.getReg(2),
                                                   UnmergeRayDir.getReg(2)}))
               .getReg(0)});
      Ops.push_back(MergedDir.getReg(0));
    } else {
      packLanes(RayDir);
      packLanes(RayInvDir);
    }
  } else {
    if (Is64) {
      auto Unmerge = B.buildUnmerge({I32, I32}, NodePtr);
      Ops.push_back(Unmerge.getReg(0));
      Ops.push_back(Unmerge.getReg(1));
    } else {
      Ops.push_back(NodePtr);
    }
    Ops.push_back(RayExtent);

    auto packLanes = [&Ops, &I32, &V3I32, &B](Register Src) {
      auto SrcInt = B.buildBitcast(V3I32, Src);
      auto Unmerge = B.buildUnmerge({I32, I32, I32}, SrcInt);
      Ops.push_back(Unmerge.getReg(0));
      Ops.push_back(Unmerge.getReg(1));
      Ops.push_back(Unmerge.getReg(2));
    };

    packLanes(RayOrigin);
    if (IsA16) {
      auto UnmergeRayDir =
          B.buildUnmerge({I16, I16, I16}, B.buildBitcast(V3I16, RayDir));
      auto UnmergeRayInvDir =
          B.buildUnmerge({I16, I16, I16}, B.buildBitcast(V3I16, RayInvDir));
      Register R1 = MRI.createGenericVirtualRegister(I32);
      Register R2 = MRI.createGenericVirtualRegister(I32);
      Register R3 = MRI.createGenericVirtualRegister(I32);
      B.buildMergeLikeInstr(R1,
                            {UnmergeRayDir.getReg(0), UnmergeRayDir.getReg(1)});
      B.buildMergeLikeInstr(
          R2, {UnmergeRayDir.getReg(2), UnmergeRayInvDir.getReg(0)});
      B.buildMergeLikeInstr(
          R3, {UnmergeRayInvDir.getReg(1), UnmergeRayInvDir.getReg(2)});
      Ops.push_back(R1);
      Ops.push_back(R2);
      Ops.push_back(R3);
    } else {
      packLanes(RayDir);
      packLanes(RayInvDir);
    }
  }

  if (!UseNSA) {
    // Build a single vector containing all the operands so far prepared.
    LLT OpTy = LLT::fixed_vector(Ops.size(), I32);
    Register MergedOps = B.buildMergeLikeInstr(OpTy, Ops).getReg(0);
    Ops.clear();
    Ops.push_back(MergedOps);
  }

  auto MIB = B.buildInstr(AMDGPU::G_AMDGPU_BVH_INTERSECT_RAY)
                 .addDef(DstReg)
                 .addImm(Opcode);

  for (Register R : Ops) {
    MIB.addUse(R);
  }

  MIB.addUse(TDescr)
     .addImm(IsA16 ? 1 : 0)
     .cloneMemRefs(MI);

  MI.eraseFromParent();
  return true;
}

bool AMDGPULegalizerInfo::legalizeBVHDualOrBVH8IntersectRayIntrinsic(
    MachineInstr &MI, MachineIRBuilder &B) const {
  const LLT I32 = LLT::integer(32);
  const LLT V2I32 = LLT::fixed_vector(2, I32);

  Register DstReg = MI.getOperand(0).getReg();
  Register DstOrigin = MI.getOperand(1).getReg();
  Register DstDir = MI.getOperand(2).getReg();
  Register NodePtr = MI.getOperand(4).getReg();
  Register RayExtent = MI.getOperand(5).getReg();
  Register InstanceMask = MI.getOperand(6).getReg();
  Register RayOrigin = MI.getOperand(7).getReg();
  Register RayDir = MI.getOperand(8).getReg();
  Register Offsets = MI.getOperand(9).getReg();
  Register TDescr = MI.getOperand(10).getReg();

  bool IsBVH8 = cast<GIntrinsic>(MI).getIntrinsicID() ==
                Intrinsic::amdgcn_image_bvh8_intersect_ray;
  const unsigned NumVDataDwords = 10;
  const unsigned NumVAddrDwords = IsBVH8 ? 11 : 12;
  int Opcode = AMDGPU::getMIMGOpcode(
      IsBVH8 ? AMDGPU::IMAGE_BVH8_INTERSECT_RAY
             : AMDGPU::IMAGE_BVH_DUAL_INTERSECT_RAY,
      AMDGPU::MIMGEncGfx12, NumVDataDwords, NumVAddrDwords);
  assert(Opcode != -1);

  auto RayExtentInstanceMaskVec =
      B.buildMergeLikeInstr(V2I32, {B.buildBitcast(I32, RayExtent),
                                    B.buildAnyExt(I32, InstanceMask)});

  B.buildInstr(IsBVH8 ? AMDGPU::G_AMDGPU_BVH8_INTERSECT_RAY
                      : AMDGPU::G_AMDGPU_BVH_DUAL_INTERSECT_RAY)
      .addDef(DstReg)
      .addDef(DstOrigin)
      .addDef(DstDir)
      .addImm(Opcode)
      .addUse(NodePtr)
      .addUse(RayExtentInstanceMaskVec.getReg(0))
      .addUse(RayOrigin)
      .addUse(RayDir)
      .addUse(Offsets)
      .addUse(TDescr)
      .cloneMemRefs(MI);

  MI.eraseFromParent();
  return true;
}

bool AMDGPULegalizerInfo::legalizeStackSave(MachineInstr &MI,
                                            MachineIRBuilder &B) const {
  const SITargetLowering *TLI = ST.getTargetLowering();
  Register StackPtr = TLI->getStackPointerRegisterToSaveRestore();
  Register DstReg = MI.getOperand(0).getReg();
  B.buildInstr(AMDGPU::G_AMDGPU_WAVE_ADDRESS, {DstReg}, {StackPtr});
  MI.eraseFromParent();
  return true;
}

bool AMDGPULegalizerInfo::legalizeWaveID(MachineInstr &MI,
                                         MachineIRBuilder &B) const {
  // With architected SGPRs, waveIDinGroup is in TTMP8[29:25].
  if (!ST.hasArchitectedSGPRs())
    return false;
  LLT I32 = LLT::integer(32);
  Register DstReg = MI.getOperand(0).getReg();
  auto TTMP8 = B.buildCopy(I32, Register(AMDGPU::TTMP8));
  auto LSB = B.buildConstant(I32, 25);
  auto Width = B.buildConstant(I32, 5);
  B.buildUbfx(DstReg, TTMP8, LSB, Width);
  MI.eraseFromParent();
  return true;
}

bool AMDGPULegalizerInfo::legalizeConstHwRegRead(MachineInstr &MI,
                                                 MachineIRBuilder &B,
                                                 AMDGPU::Hwreg::Id HwReg,
                                                 unsigned LowBit,
                                                 unsigned Width) const {
  MachineRegisterInfo &MRI = *B.getMRI();
  Register DstReg = MI.getOperand(0).getReg();
  if (!MRI.getRegClassOrNull(DstReg))
    MRI.setRegClass(DstReg, &AMDGPU::SReg_32RegClass);
  B.buildInstr(AMDGPU::S_GETREG_B32_const)
      .addDef(DstReg)
      .addImm(AMDGPU::Hwreg::HwregEncoding::encode(HwReg, LowBit, Width));
  MI.eraseFromParent();
  return true;
}

static constexpr unsigned FPEnvModeBitField =
    AMDGPU::Hwreg::HwregEncoding::encode(AMDGPU::Hwreg::ID_MODE, 0, 23);

static constexpr unsigned FPEnvTrapBitField =
    AMDGPU::Hwreg::HwregEncoding::encode(AMDGPU::Hwreg::ID_TRAPSTS, 0, 5);

bool AMDGPULegalizerInfo::legalizeGetFPEnv(MachineInstr &MI,
                                           MachineRegisterInfo &MRI,
                                           MachineIRBuilder &B) const {
  const LLT I32 = LLT::integer(32);
  const LLT I64 = LLT::integer(64);
  Register Src = MI.getOperand(0).getReg();
  if (MRI.getType(Src) != I64)
    return false;

  auto ModeReg =
      B.buildIntrinsic(Intrinsic::amdgcn_s_getreg, {I32},
                       /*HasSideEffects=*/true, /*isConvergent=*/false)
          .addImm(FPEnvModeBitField);
  auto TrapReg =
      B.buildIntrinsic(Intrinsic::amdgcn_s_getreg, {I32},
                       /*HasSideEffects=*/true, /*isConvergent=*/false)
          .addImm(FPEnvTrapBitField);
  B.buildMergeLikeInstr(Src, {ModeReg, TrapReg});
  MI.eraseFromParent();
  return true;
}

bool AMDGPULegalizerInfo::legalizeSetFPEnv(MachineInstr &MI,
                                           MachineRegisterInfo &MRI,
                                           MachineIRBuilder &B) const {
  const LLT I32 = LLT::integer(32);
  const LLT I64 = LLT::integer(64);
  Register Src = MI.getOperand(0).getReg();
  if (MRI.getType(Src) != I64)
    return false;

  auto Unmerge = B.buildUnmerge({I32, I32}, MI.getOperand(0));
  B.buildIntrinsic(Intrinsic::amdgcn_s_setreg, ArrayRef<DstOp>(),
                   /*HasSideEffects=*/true, /*isConvergent=*/false)
      .addImm(static_cast<int16_t>(FPEnvModeBitField))
      .addReg(Unmerge.getReg(0));
  B.buildIntrinsic(Intrinsic::amdgcn_s_setreg, ArrayRef<DstOp>(),
                   /*HasSideEffects=*/true, /*isConvergent=*/false)
      .addImm(static_cast<int16_t>(FPEnvTrapBitField))
      .addReg(Unmerge.getReg(1));
  MI.eraseFromParent();
  return true;
}

bool AMDGPULegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
                                            MachineInstr &MI) const {
  MachineIRBuilder &B = Helper.MIRBuilder;
  MachineRegisterInfo &MRI = *B.getMRI();

  // Replace the use G_BRCOND with the exec manipulate and branch pseudos.
  auto IntrID = cast<GIntrinsic>(MI).getIntrinsicID();
  switch (IntrID) {
  case Intrinsic::amdgcn_icmp: {
    // amdgcn.icmp(i1 src0, i1 0, NE) -> ballot(src0)
    // This is the only valid form of amdgcn.icmp with i1 inputs.
    Register Src0 = MI.getOperand(2).getReg();
    LLT SrcTy = MRI.getType(Src0);
    if (SrcTy != LLT::scalar(1))
      return true; // Not i1, leave for default handling.

    // Check that src1 is constant 0.
    Register Src1 = MI.getOperand(3).getReg();
    auto Src1Const = getIConstantVRegValWithLookThrough(Src1, MRI);
    if (!Src1Const || Src1Const->Value != 0)
      return false; // Invalid i1 icmp form.

    // Check that predicate is ICMP_NE.
    int64_t Pred = MI.getOperand(4).getImm();
    if (Pred != CmpInst::ICMP_NE)
      return false; // Invalid i1 icmp form.

    // Convert to ballot.
    Register Dst = MI.getOperand(0).getReg();
    B.buildIntrinsic(Intrinsic::amdgcn_ballot, Dst).addUse(Src0);
    MI.eraseFromParent();
    return true;
  }
  case Intrinsic::sponentry:
    if (B.getMF().getInfo<SIMachineFunctionInfo>()->isBottomOfStack()) {
      // FIXME: The imported pattern checks for i32 instead of p5; if we fix
      // that we can remove this cast.
      const LLT I32 = LLT::integer(32);
      Register TmpReg = MRI.createGenericVirtualRegister(I32);
      B.buildInstr(AMDGPU::G_AMDGPU_SPONENTRY).addDef(TmpReg);

      Register DstReg = MI.getOperand(0).getReg();
      B.buildIntToPtr(DstReg, TmpReg);
      MI.eraseFromParent();
    } else {
      int FI = B.getMF().getFrameInfo().CreateFixedObject(
          1, 0, /*IsImmutable=*/false);
      B.buildFrameIndex(MI.getOperand(0), FI);
      MI.eraseFromParent();
    }
    return true;
  case Intrinsic::amdgcn_if:
  case Intrinsic::amdgcn_else: {
    MachineInstr *Br = nullptr;
    MachineBasicBlock *UncondBrTarget = nullptr;
    bool Negated = false;
    if (MachineInstr *BrCond =
            verifyCFIntrinsic(MI, MRI, Br, UncondBrTarget, Negated)) {
      const SIRegisterInfo *TRI
        = static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());

      Register Def = MI.getOperand(1).getReg();
      Register Use = MI.getOperand(3).getReg();

      MachineBasicBlock *CondBrTarget = BrCond->getOperand(1).getMBB();

      if (Negated)
        std::swap(CondBrTarget, UncondBrTarget);

      B.setInsertPt(B.getMBB(), BrCond->getIterator());
      if (IntrID == Intrinsic::amdgcn_if) {
        B.buildInstr(AMDGPU::SI_IF)
          .addDef(Def)
          .addUse(Use)
          .addMBB(UncondBrTarget);
      } else {
        B.buildInstr(AMDGPU::SI_ELSE)
            .addDef(Def)
            .addUse(Use)
            .addMBB(UncondBrTarget);
      }

      if (Br) {
        Br->getOperand(0).setMBB(CondBrTarget);
      } else {
        // The IRTranslator skips inserting the G_BR for fallthrough cases, but
        // since we're swapping branch targets it needs to be reinserted.
        // FIXME: IRTranslator should probably not do this
        B.buildBr(*CondBrTarget);
      }

      MRI.setRegClass(Def, TRI->getWaveMaskRegClass());
      MRI.setRegClass(Use, TRI->getWaveMaskRegClass());
      MI.eraseFromParent();
      BrCond->eraseFromParent();
      return true;
    }

    return false;
  }
  case Intrinsic::amdgcn_loop: {
    MachineInstr *Br = nullptr;
    MachineBasicBlock *UncondBrTarget = nullptr;
    bool Negated = false;
    if (MachineInstr *BrCond =
            verifyCFIntrinsic(MI, MRI, Br, UncondBrTarget, Negated)) {
      const SIRegisterInfo *TRI
        = static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());

      MachineBasicBlock *CondBrTarget = BrCond->getOperand(1).getMBB();
      Register Reg = MI.getOperand(2).getReg();

      if (Negated)
        std::swap(CondBrTarget, UncondBrTarget);

      B.setInsertPt(B.getMBB(), BrCond->getIterator());
      B.buildInstr(AMDGPU::SI_LOOP)
        .addUse(Reg)
        .addMBB(UncondBrTarget);

      if (Br)
        Br->getOperand(0).setMBB(CondBrTarget);
      else
        B.buildBr(*CondBrTarget);

      MI.eraseFromParent();
      BrCond->eraseFromParent();
      MRI.setRegClass(Reg, TRI->getWaveMaskRegClass());
      return true;
    }

    return false;
  }
  case Intrinsic::amdgcn_wave_reduce_min:
  case Intrinsic::amdgcn_wave_reduce_umin:
  case Intrinsic::amdgcn_wave_reduce_fmin:
  case Intrinsic::amdgcn_wave_reduce_max:
  case Intrinsic::amdgcn_wave_reduce_umax:
  case Intrinsic::amdgcn_wave_reduce_fmax:
  case Intrinsic::amdgcn_wave_reduce_add:
  case Intrinsic::amdgcn_wave_reduce_fadd:
  case Intrinsic::amdgcn_wave_reduce_sub:
  case Intrinsic::amdgcn_wave_reduce_fsub:
  case Intrinsic::amdgcn_wave_reduce_and:
  case Intrinsic::amdgcn_wave_reduce_or:
  case Intrinsic::amdgcn_wave_reduce_xor: {
    Register SrcReg = MI.getOperand(2).getReg();
    if (MRI.getType(SrcReg).getSizeInBits() != 16)
      return true;
    Register DstReg = MI.getOperand(0).getReg();
    bool IsFPOp = IntrID == Intrinsic::amdgcn_wave_reduce_fmin ||
                  IntrID == Intrinsic::amdgcn_wave_reduce_fmax ||
                  IntrID == Intrinsic::amdgcn_wave_reduce_fadd ||
                  IntrID == Intrinsic::amdgcn_wave_reduce_fsub;
    bool NeedsSignExt = IntrID == Intrinsic::amdgcn_wave_reduce_min ||
                        IntrID == Intrinsic::amdgcn_wave_reduce_max ||
                        IntrID == Intrinsic::amdgcn_wave_reduce_add ||
                        IntrID == Intrinsic::amdgcn_wave_reduce_sub;
    auto Ext = IsFPOp         ? B.buildFPExt(F32, SrcReg)
               : NeedsSignExt ? B.buildSExt(LLT::integer(32), SrcReg)
                              : B.buildZExt(LLT::integer(32), SrcReg);
    auto NewDst =
        MRI.createGenericVirtualRegister(IsFPOp ? F32 : LLT::integer(32));
    B.buildIntrinsic(IntrID, ArrayRef<Register>{NewDst},
                     /*hasSideEffects=*/false, /*isConvergent=*/true)
        .addUse(Ext.getReg(0))
        .addImm(MI.getOperand(3).getImm()); // strategy
    if (IsFPOp)
      B.buildFPTrunc(DstReg, NewDst);
    else
      B.buildTrunc(DstReg, NewDst);
    MI.eraseFromParent();
    return true;
  }
  case Intrinsic::amdgcn_addrspacecast_nonnull:
    return legalizeAddrSpaceCast(MI, MRI, B);
  case Intrinsic::amdgcn_make_buffer_rsrc:
    return legalizePointerAsRsrcIntrin(MI, MRI, B);
  case Intrinsic::amdgcn_kernarg_segment_ptr:
    if (!AMDGPU::isKernel(B.getMF().getFunction())) {
      // This only makes sense to call in a kernel, so just lower to null.
      B.buildConstant(MI.getOperand(0).getReg(), 0);
      MI.eraseFromParent();
      return true;
    }

    return legalizePreloadedArgIntrin(
      MI, MRI, B, AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR);
  case Intrinsic::amdgcn_implicitarg_ptr:
    return legalizeImplicitArgPtr(MI, MRI, B);
  case Intrinsic::amdgcn_workitem_id_x:
    return legalizeWorkitemIDIntrinsic(MI, MRI, B, 0,
                                       AMDGPUFunctionArgInfo::WORKITEM_ID_X);
  case Intrinsic::amdgcn_workitem_id_y:
    return legalizeWorkitemIDIntrinsic(MI, MRI, B, 1,
                                       AMDGPUFunctionArgInfo::WORKITEM_ID_Y);
  case Intrinsic::amdgcn_workitem_id_z:
    return legalizeWorkitemIDIntrinsic(MI, MRI, B, 2,
                                       AMDGPUFunctionArgInfo::WORKITEM_ID_Z);
  case Intrinsic::amdgcn_workgroup_id_x:
    return legalizeWorkGroupId(
        MI, B, AMDGPUFunctionArgInfo::WORKGROUP_ID_X,
        AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_X,
        AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_X);
  case Intrinsic::amdgcn_workgroup_id_y:
    return legalizeWorkGroupId(
        MI, B, AMDGPUFunctionArgInfo::WORKGROUP_ID_Y,
        AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_Y,
        AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_Y);
  case Intrinsic::amdgcn_workgroup_id_z:
    return legalizeWorkGroupId(
        MI, B, AMDGPUFunctionArgInfo::WORKGROUP_ID_Z,
        AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_Z,
        AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_Z);
  case Intrinsic::amdgcn_cluster_id_x:
    return ST.hasClusters() &&
           legalizePreloadedArgIntrin(MI, MRI, B,
                                      AMDGPUFunctionArgInfo::WORKGROUP_ID_X);
  case Intrinsic::amdgcn_cluster_id_y:
    return ST.hasClusters() &&
           legalizePreloadedArgIntrin(MI, MRI, B,
                                      AMDGPUFunctionArgInfo::WORKGROUP_ID_Y);
  case Intrinsic::amdgcn_cluster_id_z:
    return ST.hasClusters() &&
           legalizePreloadedArgIntrin(MI, MRI, B,
                                      AMDGPUFunctionArgInfo::WORKGROUP_ID_Z);
  case Intrinsic::amdgcn_cluster_workgroup_id_x:
    return ST.hasClusters() &&
           legalizePreloadedArgIntrin(
               MI, MRI, B, AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_X);
  case Intrinsic::amdgcn_cluster_workgroup_id_y:
    return ST.hasClusters() &&
           legalizePreloadedArgIntrin(
               MI, MRI, B, AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_Y);
  case Intrinsic::amdgcn_cluster_workgroup_id_z:
    return ST.hasClusters() &&
           legalizePreloadedArgIntrin(
               MI, MRI, B, AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_Z);
  case Intrinsic::amdgcn_cluster_workgroup_flat_id:
    return ST.hasClusters() &&
           legalizeConstHwRegRead(MI, B, AMDGPU::Hwreg::ID_IB_STS2, 21, 4);
  case Intrinsic::amdgcn_cluster_workgroup_max_id_x:
    return ST.hasClusters() &&
           legalizePreloadedArgIntrin(
               MI, MRI, B, AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_X);
  case Intrinsic::amdgcn_cluster_workgroup_max_id_y:
    return ST.hasClusters() &&
           legalizePreloadedArgIntrin(
               MI, MRI, B, AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_Y);
  case Intrinsic::amdgcn_cluster_workgroup_max_id_z:
    return ST.hasClusters() &&
           legalizePreloadedArgIntrin(
               MI, MRI, B, AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_Z);
  case Intrinsic::amdgcn_cluster_workgroup_max_flat_id:
    return ST.hasClusters() &&
           legalizePreloadedArgIntrin(
               MI, MRI, B,
               AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_FLAT_ID);
  case Intrinsic::amdgcn_wave_id:
    return legalizeWaveID(MI, B);
  case Intrinsic::amdgcn_lds_kernel_id:
    return legalizePreloadedArgIntrin(MI, MRI, B,
                                      AMDGPUFunctionArgInfo::LDS_KERNEL_ID);
  case Intrinsic::amdgcn_dispatch_ptr:
    return legalizePreloadedArgIntrin(MI, MRI, B,
                                      AMDGPUFunctionArgInfo::DISPATCH_PTR);
  case Intrinsic::amdgcn_queue_ptr:
    return legalizePreloadedArgIntrin(MI, MRI, B,
                                      AMDGPUFunctionArgInfo::QUEUE_PTR);
  case Intrinsic::amdgcn_implicit_buffer_ptr:
    return legalizePreloadedArgIntrin(
      MI, MRI, B, AMDGPUFunctionArgInfo::IMPLICIT_BUFFER_PTR);
  case Intrinsic::amdgcn_dispatch_id:
    return legalizePreloadedArgIntrin(MI, MRI, B,
                                      AMDGPUFunctionArgInfo::DISPATCH_ID);
  case Intrinsic::r600_read_ngroups_x:
    // TODO: Emit error for hsa
    return legalizeKernargMemParameter(MI, B,
                                       SI::KernelInputOffsets::NGROUPS_X);
  case Intrinsic::r600_read_ngroups_y:
    return legalizeKernargMemParameter(MI, B,
                                       SI::KernelInputOffsets::NGROUPS_Y);
  case Intrinsic::r600_read_ngroups_z:
    return legalizeKernargMemParameter(MI, B,
                                       SI::KernelInputOffsets::NGROUPS_Z);
  case Intrinsic::r600_read_local_size_x:
    // TODO: Could insert G_ASSERT_ZEXT from i16
    return legalizeKernargMemParameter(MI, B, SI::KernelInputOffsets::LOCAL_SIZE_X);
  case Intrinsic::r600_read_local_size_y:
    // TODO: Could insert G_ASSERT_ZEXT from i16
    return legalizeKernargMemParameter(MI, B,  SI::KernelInputOffsets::LOCAL_SIZE_Y);
    // TODO: Could insert G_ASSERT_ZEXT from i16
  case Intrinsic::r600_read_local_size_z:
    return legalizeKernargMemParameter(MI, B,
                                       SI::KernelInputOffsets::LOCAL_SIZE_Z);
  case Intrinsic::amdgcn_fdiv_fast:
    return legalizeFDIVFastIntrin(MI, MRI, B);
  case Intrinsic::amdgcn_is_shared:
    return legalizeIsAddrSpace(MI, MRI, B, AMDGPUAS::LOCAL_ADDRESS);
  case Intrinsic::amdgcn_is_private:
    return legalizeIsAddrSpace(MI, MRI, B, AMDGPUAS::PRIVATE_ADDRESS);
  case Intrinsic::amdgcn_wavefrontsize: {
    B.buildConstant(MI.getOperand(0), ST.getWavefrontSize());
    MI.eraseFromParent();
    return true;
  }
  case Intrinsic::amdgcn_s_buffer_load:
    return legalizeSBufferLoad(Helper, MI);
  case Intrinsic::amdgcn_raw_buffer_store:
  case Intrinsic::amdgcn_raw_ptr_buffer_store:
  case Intrinsic::amdgcn_struct_buffer_store:
  case Intrinsic::amdgcn_struct_ptr_buffer_store:
    return legalizeBufferStore(MI, Helper, false, false);
  case Intrinsic::amdgcn_raw_buffer_store_format:
  case Intrinsic::amdgcn_raw_ptr_buffer_store_format:
  case Intrinsic::amdgcn_struct_buffer_store_format:
  case Intrinsic::amdgcn_struct_ptr_buffer_store_format:
    return legalizeBufferStore(MI, Helper, false, true);
  case Intrinsic::amdgcn_raw_tbuffer_store:
  case Intrinsic::amdgcn_raw_ptr_tbuffer_store:
  case Intrinsic::amdgcn_struct_tbuffer_store:
  case Intrinsic::amdgcn_struct_ptr_tbuffer_store:
    return legalizeBufferStore(MI, Helper, true, true);
  case Intrinsic::amdgcn_raw_buffer_load:
  case Intrinsic::amdgcn_raw_ptr_buffer_load:
  case Intrinsic::amdgcn_raw_atomic_buffer_load:
  case Intrinsic::amdgcn_raw_ptr_atomic_buffer_load:
  case Intrinsic::amdgcn_struct_buffer_load:
  case Intrinsic::amdgcn_struct_ptr_buffer_load:
  case Intrinsic::amdgcn_struct_atomic_buffer_load:
  case Intrinsic::amdgcn_struct_ptr_atomic_buffer_load:
    return legalizeBufferLoad(MI, Helper, false, false);
  case Intrinsic::amdgcn_raw_buffer_load_format:
  case Intrinsic::amdgcn_raw_ptr_buffer_load_format:
  case Intrinsic::amdgcn_struct_buffer_load_format:
  case Intrinsic::amdgcn_struct_ptr_buffer_load_format:
    return legalizeBufferLoad(MI, Helper, true, false);
  case Intrinsic::amdgcn_raw_tbuffer_load:
  case Intrinsic::amdgcn_raw_ptr_tbuffer_load:
  case Intrinsic::amdgcn_struct_tbuffer_load:
  case Intrinsic::amdgcn_struct_ptr_tbuffer_load:
    return legalizeBufferLoad(MI, Helper, true, true);
  case Intrinsic::amdgcn_raw_buffer_atomic_swap:
  case Intrinsic::amdgcn_raw_ptr_buffer_atomic_swap:
  case Intrinsic::amdgcn_struct_buffer_atomic_swap:
  case Intrinsic::amdgcn_struct_ptr_buffer_atomic_swap:
  case Intrinsic::amdgcn_raw_buffer_atomic_add:
  case Intrinsic::amdgcn_raw_ptr_buffer_atomic_add:
  case Intrinsic::amdgcn_struct_buffer_atomic_add:
  case Intrinsic::amdgcn_struct_ptr_buffer_atomic_add:
  case Intrinsic::amdgcn_raw_buffer_atomic_sub:
  case Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub:
  case Intrinsic::amdgcn_struct_buffer_atomic_sub:
  case Intrinsic::amdgcn_struct_ptr_buffer_atomic_sub:
  case Intrinsic::amdgcn_raw_buffer_atomic_smin:
  case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smin:
  case Intrinsic::amdgcn_struct_buffer_atomic_smin:
  case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smin:
  case Intrinsic::amdgcn_raw_buffer_atomic_umin:
  case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umin:
  case Intrinsic::amdgcn_struct_buffer_atomic_umin:
  case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umin:
  case Intrinsic::amdgcn_raw_buffer_atomic_smax:
  case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smax:
  case Intrinsic::amdgcn_struct_buffer_atomic_smax:
  case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smax:
  case Intrinsic::amdgcn_raw_buffer_atomic_umax:
  case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umax:
  case Intrinsic::amdgcn_struct_buffer_atomic_umax:
  case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umax:
  case Intrinsic::amdgcn_raw_buffer_atomic_and:
  case Intrinsic::amdgcn_raw_ptr_buffer_atomic_and:
  case Intrinsic::amdgcn_struct_buffer_atomic_and:
  case Intrinsic::amdgcn_struct_ptr_buffer_atomic_and:
  case Intrinsic::amdgcn_raw_buffer_atomic_or:
  case Intrinsic::amdgcn_raw_ptr_buffer_atomic_or:
  case Intrinsic::amdgcn_struct_buffer_atomic_or:
  case Intrinsic::amdgcn_struct_ptr_buffer_atomic_or:
  case Intrinsic::amdgcn_raw_buffer_atomic_xor:
  case Intrinsic::amdgcn_raw_ptr_buffer_atomic_xor:
  case Intrinsic::amdgcn_struct_buffer_atomic_xor:
  case Intrinsic::amdgcn_struct_ptr_buffer_atomic_xor:
  case Intrinsic::amdgcn_raw_buffer_atomic_inc:
  case Intrinsic::amdgcn_raw_ptr_buffer_atomic_inc:
  case Intrinsic::amdgcn_struct_buffer_atomic_inc:
  case Intrinsic::amdgcn_struct_ptr_buffer_atomic_inc:
  case Intrinsic::amdgcn_raw_buffer_atomic_dec:
  case Intrinsic::amdgcn_raw_ptr_buffer_atomic_dec:
  case Intrinsic::amdgcn_struct_buffer_atomic_dec:
  case Intrinsic::amdgcn_struct_ptr_buffer_atomic_dec:
  case Intrinsic::amdgcn_raw_buffer_atomic_cmpswap:
  case Intrinsic::amdgcn_raw_ptr_buffer_atomic_cmpswap:
  case Intrinsic::amdgcn_struct_buffer_atomic_cmpswap:
  case Intrinsic::amdgcn_struct_ptr_buffer_atomic_cmpswap:
  case Intrinsic::amdgcn_raw_buffer_atomic_fmin:
  case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmin:
  case Intrinsic::amdgcn_struct_buffer_atomic_fmin:
  case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fmin:
  case Intrinsic::amdgcn_raw_buffer_atomic_fmax:
  case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmax:
  case Intrinsic::amdgcn_struct_buffer_atomic_fmax:
  case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fmax:
  case Intrinsic::amdgcn_raw_buffer_atomic_sub_clamp_u32:
  case Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub_clamp_u32:
  case Intrinsic::amdgcn_struct_buffer_atomic_sub_clamp_u32:
  case Intrinsic::amdgcn_struct_ptr_buffer_atomic_sub_clamp_u32:
  case Intrinsic::amdgcn_raw_buffer_atomic_cond_sub_u32:
  case Intrinsic::amdgcn_raw_ptr_buffer_atomic_cond_sub_u32:
  case Intrinsic::amdgcn_struct_buffer_atomic_cond_sub_u32:
  case Intrinsic::amdgcn_struct_ptr_buffer_atomic_cond_sub_u32:
  case Intrinsic::amdgcn_raw_buffer_atomic_fadd:
  case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fadd:
  case Intrinsic::amdgcn_struct_buffer_atomic_fadd:
  case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fadd:
    return legalizeBufferAtomic(MI, B, IntrID);
  case Intrinsic::amdgcn_rsq_clamp:
    return legalizeRsqClampIntrinsic(MI, MRI, B);
  case Intrinsic::amdgcn_image_bvh_intersect_ray:
    return legalizeBVHIntersectRayIntrinsic(MI, B);
  case Intrinsic::amdgcn_image_bvh_dual_intersect_ray:
  case Intrinsic::amdgcn_image_bvh8_intersect_ray:
    return legalizeBVHDualOrBVH8IntersectRayIntrinsic(MI, B);
  case Intrinsic::amdgcn_swmmac_f32_16x16x128_fp8_fp8:
  case Intrinsic::amdgcn_swmmac_f32_16x16x128_fp8_bf8:
  case Intrinsic::amdgcn_swmmac_f32_16x16x128_bf8_fp8:
  case Intrinsic::amdgcn_swmmac_f32_16x16x128_bf8_bf8:
  case Intrinsic::amdgcn_swmmac_f16_16x16x128_fp8_fp8:
  case Intrinsic::amdgcn_swmmac_f16_16x16x128_fp8_bf8:
  case Intrinsic::amdgcn_swmmac_f16_16x16x128_bf8_fp8:
  case Intrinsic::amdgcn_swmmac_f16_16x16x128_bf8_bf8: {
    Register Index = MI.getOperand(5).getReg();
    LLT I64 = LLT::integer(64);
    LLT IndexArgTy = MRI.getType(Index);
    if (IndexArgTy != I64) {
      auto NewIndex = IndexArgTy.isVector() ? B.buildBitcast(I64, Index)
                                            : B.buildAnyExt(I64, Index);
      MI.getOperand(5).setReg(NewIndex.getReg(0));
    }
    return true;
  }
  case Intrinsic::amdgcn_swmmac_f16_16x16x32_f16:
  case Intrinsic::amdgcn_swmmac_bf16_16x16x32_bf16:
  case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf16:
  case Intrinsic::amdgcn_swmmac_f32_16x16x32_f16:
  case Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_fp8:
  case Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_bf8:
  case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_fp8:
  case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_bf8: {
    Register Index = MI.getOperand(5).getReg();
    LLT I32 = LLT::integer(32);
    if (MRI.getType(Index) != I32)
      MI.getOperand(5).setReg(B.buildAnyExt(I32, Index).getReg(0));
    return true;
  }
  case Intrinsic::amdgcn_swmmac_f16_16x16x64_f16:
  case Intrinsic::amdgcn_swmmac_bf16_16x16x64_bf16:
  case Intrinsic::amdgcn_swmmac_f32_16x16x64_bf16:
  case Intrinsic::amdgcn_swmmac_bf16f32_16x16x64_bf16:
  case Intrinsic::amdgcn_swmmac_f32_16x16x64_f16:
  case Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8:
  case Intrinsic::amdgcn_swmmac_i32_16x16x32_iu4:
  case Intrinsic::amdgcn_swmmac_i32_16x16x32_iu8:
  case Intrinsic::amdgcn_swmmac_i32_16x16x64_iu4: {
    Register Index = MI.getOperand(7).getReg();
    LLT IdxTy = IntrID == Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8
                    ? LLT::integer(64)
                    : LLT::integer(32);
    LLT IndexArgTy = MRI.getType(Index);
    if (IndexArgTy != IdxTy) {
      auto NewIndex = IndexArgTy.isVector() ? B.buildBitcast(IdxTy, Index)
                                            : B.buildAnyExt(IdxTy, Index);
      MI.getOperand(7).setReg(NewIndex.getReg(0));
    }
    return true;
  }

  case Intrinsic::amdgcn_fmed3: {
    GISelChangeObserver &Observer = Helper.Observer;

    // FIXME: This is to workaround the inability of tablegen match combiners to
    // match intrinsics in patterns.
    Observer.changingInstr(MI);
    MI.setDesc(B.getTII().get(AMDGPU::G_AMDGPU_FMED3));
    MI.removeOperand(1);
    Observer.changedInstr(MI);
    return true;
  }
  case Intrinsic::amdgcn_readlane:
  case Intrinsic::amdgcn_writelane:
  case Intrinsic::amdgcn_readfirstlane:
  case Intrinsic::amdgcn_permlane16:
  case Intrinsic::amdgcn_permlanex16:
  case Intrinsic::amdgcn_permlane64:
  case Intrinsic::amdgcn_set_inactive:
  case Intrinsic::amdgcn_set_inactive_chain_arg:
  case Intrinsic::amdgcn_mov_dpp8:
  case Intrinsic::amdgcn_update_dpp:
  case Intrinsic::amdgcn_permlane_bcast:
  case Intrinsic::amdgcn_permlane_up:
  case Intrinsic::amdgcn_permlane_down:
  case Intrinsic::amdgcn_permlane_xor:
    return legalizeLaneOp(Helper, MI, IntrID);
  case Intrinsic::amdgcn_s_buffer_prefetch_data:
    return legalizeSBufferPrefetch(Helper, MI);
  case Intrinsic::amdgcn_dead: {
    // TODO: Use poison instead of undef
    for (const MachineOperand &Def : MI.defs())
      B.buildUndef(Def);
    MI.eraseFromParent();
    return true;
  }
  case Intrinsic::amdgcn_cooperative_atomic_load_32x4B:
  case Intrinsic::amdgcn_cooperative_atomic_load_16x8B:
  case Intrinsic::amdgcn_cooperative_atomic_load_8x16B:
    assert(MI.hasOneMemOperand() && "Expected IRTranslator to set MemOp!");
    B.buildLoad(MI.getOperand(0), MI.getOperand(2), **MI.memoperands_begin());
    MI.eraseFromParent();
    return true;
  case Intrinsic::amdgcn_cooperative_atomic_store_32x4B:
  case Intrinsic::amdgcn_cooperative_atomic_store_16x8B:
  case Intrinsic::amdgcn_cooperative_atomic_store_8x16B:
    assert(MI.hasOneMemOperand() && "Expected IRTranslator to set MemOp!");
    B.buildStore(MI.getOperand(2), MI.getOperand(1), **MI.memoperands_begin());
    MI.eraseFromParent();
    return true;
  case Intrinsic::amdgcn_av_load_b128:
  case Intrinsic::amdgcn_av_store_b128: {
    assert(MI.hasOneMemOperand() && "Expected IRTranslator to set MemOp!");
    if (IntrID == Intrinsic::amdgcn_av_load_b128)
      B.buildLoad(MI.getOperand(0), MI.getOperand(2), **MI.memoperands_begin());
    else
      B.buildStore(MI.getOperand(2), MI.getOperand(1),
                   **MI.memoperands_begin());
    MI.eraseFromParent();
    return true;
  }
  case Intrinsic::amdgcn_flat_load_monitor_b32:
  case Intrinsic::amdgcn_flat_load_monitor_b64:
  case Intrinsic::amdgcn_flat_load_monitor_b128:
    assert(MI.hasOneMemOperand() && "Expected IRTranslator to set MemOp!");
    B.buildInstr(AMDGPU::G_AMDGPU_FLAT_LOAD_MONITOR)
        .add(MI.getOperand(0))
        .add(MI.getOperand(2))
        .addMemOperand(*MI.memoperands_begin());
    MI.eraseFromParent();
    return true;
  case Intrinsic::amdgcn_global_load_monitor_b32:
  case Intrinsic::amdgcn_global_load_monitor_b64:
  case Intrinsic::amdgcn_global_load_monitor_b128:
    assert(MI.hasOneMemOperand() && "Expected IRTranslator to set MemOp!");
    B.buildInstr(AMDGPU::G_AMDGPU_GLOBAL_LOAD_MONITOR)
        .add(MI.getOperand(0))
        .add(MI.getOperand(2))
        .addMemOperand(*MI.memoperands_begin());
    MI.eraseFromParent();
    return true;
  default: {
    if (const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr =
            AMDGPU::getImageDimIntrinsicInfo(IntrID))
      return legalizeImageIntrinsic(MI, B, Helper.Observer, ImageDimIntr);
    return true;
  }
  }

  return true;
}