Google Git Google Git
Sign in
llvm/llvm-project/refs/heads/users/pierre-vh/split-eventtracker-basic/./llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
blob: 54857b973256b7a0ae8ccbe871979a2120387cdd [file] [edit]
//===- AMDGPULegalizerInfo.cpp -----------------------------------*- C++ -*-==//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
/// \file
/// This file implements the targeting of the Machinelegalizer class for
/// AMDGPU.
/// \todo This should be generated by TableGen.
//===----------------------------------------------------------------------===//
#include "AMDGPULegalizerInfo.h"
#include "AMDGPU.h"
#include "AMDGPUGlobalISelUtils.h"
#include "AMDGPUInstrInfo.h"
#include "AMDGPUMemoryUtils.h"
#include "AMDGPUTargetMachine.h"
#include "SIInstrInfo.h"
#include "SIMachineFunctionInfo.h"
#include "SIRegisterInfo.h"
#include "Utils/AMDGPUBaseInfo.h"
#include "llvm/ADT/ScopeExit.h"
#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
#include "llvm/CodeGen/GlobalISel/Utils.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/PseudoSourceValueManager.h"
#include "llvm/CodeGen/TargetOpcodes.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/IR/IntrinsicsR600.h"
#include "llvm/Support/AMDGPUAddrSpace.h"
#define DEBUG_TYPE "amdgpu-legalinfo"
using namespace llvm;
using namespace LegalizeActions;
using namespace LegalizeMutations;
using namespace LegalityPredicates;
using namespace MIPatternMatch;
// Hack until load/store selection patterns support any tuple of legal types.
static cl::opt<bool> EnableNewLegality(
"amdgpu-global-isel-new-legality",
cl::desc("Use GlobalISel desired legality, rather than try to use"
"rules compatible with selection patterns"),
cl::init(false),
cl::ReallyHidden);
static constexpr unsigned MaxRegisterSize = 1024;
// Round the number of elements to the next power of two elements
static LLT getPow2VectorType(LLT Ty) {
unsigned NElts = Ty.getNumElements();
unsigned Pow2NElts = 1 << Log2_32_Ceil(NElts);
return Ty.changeElementCount(ElementCount::getFixed(Pow2NElts));
}
// Round the number of bits to the next power of two bits
static LLT getPow2ScalarType(LLT Ty) {
unsigned Bits = Ty.getSizeInBits();
unsigned Pow2Bits = 1 << Log2_32_Ceil(Bits);
return LLT::scalar(Pow2Bits);
}
/// \returns true if this is an odd sized vector which should widen by adding an
/// additional element. This is mostly to handle <3 x s16> -> <4 x s16>. This
/// excludes s1 vectors, which should always be scalarized.
static LegalityPredicate isSmallOddVector(unsigned TypeIdx) {
return [=](const LegalityQuery &Query) {
const LLT Ty = Query.Types[TypeIdx];
if (!Ty.isVector())
return false;
const LLT EltTy = Ty.getElementType();
const unsigned EltSize = EltTy.getSizeInBits();
return Ty.getNumElements() % 2 != 0 &&
EltSize > 1 && EltSize < 32 &&
Ty.getSizeInBits() % 32 != 0;
};
}
static LegalityPredicate sizeIsMultipleOf32(unsigned TypeIdx) {
return [=](const LegalityQuery &Query) {
const LLT Ty = Query.Types[TypeIdx];
return Ty.getSizeInBits() % 32 == 0;
};
}
static LegalityPredicate isWideVec16(unsigned TypeIdx) {
return [=](const LegalityQuery &Query) {
const LLT Ty = Query.Types[TypeIdx];
const LLT EltTy = Ty.getScalarType();
return EltTy.getSizeInBits() == 16 && Ty.getNumElements() > 2;
};
}
static LegalizeMutation oneMoreElement(unsigned TypeIdx) {
return [=](const LegalityQuery &Query) {
const LLT Ty = Query.Types[TypeIdx];
const LLT EltTy = Ty.getElementType();
return std::pair(TypeIdx,
LLT::fixed_vector(Ty.getNumElements() + 1, EltTy));
};
}
static LegalizeMutation fewerEltsToSize64Vector(unsigned TypeIdx) {
return [=](const LegalityQuery &Query) {
const LLT Ty = Query.Types[TypeIdx];
const LLT EltTy = Ty.getElementType();
unsigned Size = Ty.getSizeInBits();
unsigned Pieces = (Size + 63) / 64;
unsigned NewNumElts = (Ty.getNumElements() + 1) / Pieces;
return std::pair(TypeIdx, LLT::scalarOrVector(
ElementCount::getFixed(NewNumElts), EltTy));
};
}
// Increase the number of vector elements to reach the next multiple of 32-bit
// type.
static LegalizeMutation moreEltsToNext32Bit(unsigned TypeIdx) {
return [=](const LegalityQuery &Query) {
const LLT Ty = Query.Types[TypeIdx];
const LLT EltTy = Ty.getElementType();
const int Size = Ty.getSizeInBits();
const int EltSize = EltTy.getSizeInBits();
const int NextMul32 = (Size + 31) / 32;
assert(EltSize < 32);
const int NewNumElts = (32 * NextMul32 + EltSize - 1) / EltSize;
return std::pair(TypeIdx, LLT::fixed_vector(NewNumElts, EltTy));
};
}
// Retrieves the scalar type that's the same size as the mem desc
static LegalizeMutation getScalarTypeFromMemDesc(unsigned TypeIdx) {
return [=](const LegalityQuery &Query) {
unsigned MemSize = Query.MMODescrs[0].MemoryTy.getSizeInBits();
return std::make_pair(TypeIdx, LLT::integer(MemSize));
};
}
// Increase the number of vector elements to reach the next legal RegClass.
static LegalizeMutation moreElementsToNextExistingRegClass(unsigned TypeIdx) {
return [=](const LegalityQuery &Query) {
const LLT Ty = Query.Types[TypeIdx];
const unsigned NumElts = Ty.getNumElements();
const unsigned EltSize = Ty.getElementType().getSizeInBits();
const unsigned MaxNumElts = MaxRegisterSize / EltSize;
assert(EltSize == 32 || EltSize == 64);
assert(Ty.getSizeInBits() < MaxRegisterSize);
unsigned NewNumElts;
// Find the nearest legal RegClass that is larger than the current type.
for (NewNumElts = NumElts; NewNumElts < MaxNumElts; ++NewNumElts) {
if (SIRegisterInfo::getSGPRClassForBitWidth(NewNumElts * EltSize))
break;
}
return std::pair(TypeIdx,
LLT::fixed_vector(NewNumElts, Ty.getElementType()));
};
}
static LLT getBufferRsrcScalarType(const LLT Ty) {
if (!Ty.isVector())
return LLT::scalar(128);
const ElementCount NumElems = Ty.getElementCount();
return LLT::vector(NumElems, LLT::scalar(128));
}
static LLT getBufferRsrcRegisterType(const LLT Ty) {
if (!Ty.isVector())
return LLT::fixed_vector(4, LLT::integer(32));
const unsigned NumElems = Ty.getElementCount().getFixedValue();
return LLT::fixed_vector(NumElems * 4, LLT::integer(32));
}
static LLT getBitcastRegisterType(const LLT Ty) {
const unsigned Size = Ty.getSizeInBits();
if (Size <= 32) {
// <2 x i8> -> i16
// <4 x i8> -> i32
return LLT::integer(Size);
}
return LLT::fixed_vector(Size / 32, LLT::integer(32));
}
static LegalizeMutation bitcastToRegisterType(unsigned TypeIdx) {
return [=](const LegalityQuery &Query) {
const LLT Ty = Query.Types[TypeIdx];
return std::pair(TypeIdx, getBitcastRegisterType(Ty));
};
}
static LegalizeMutation bitcastToVectorElement32(unsigned TypeIdx) {
return [=](const LegalityQuery &Query) {
const LLT Ty = Query.Types[TypeIdx];
unsigned Size = Ty.getSizeInBits();
assert(Size % 32 == 0);
return std::pair(TypeIdx,
LLT::scalarOrVector(ElementCount::getFixed(Size / 32),
LLT::integer(32)));
};
}
static LegalityPredicate vectorSmallerThan(unsigned TypeIdx, unsigned Size) {
return [=](const LegalityQuery &Query) {
const LLT QueryTy = Query.Types[TypeIdx];
return QueryTy.isVector() && QueryTy.getSizeInBits() < Size;
};
}
static LegalityPredicate vectorWiderThan(unsigned TypeIdx, unsigned Size) {
return [=](const LegalityQuery &Query) {
const LLT QueryTy = Query.Types[TypeIdx];
return QueryTy.isVector() && QueryTy.getSizeInBits() > Size;
};
}
static LegalityPredicate numElementsNotEven(unsigned TypeIdx) {
return [=](const LegalityQuery &Query) {
const LLT QueryTy = Query.Types[TypeIdx];
return QueryTy.isVector() && QueryTy.getNumElements() % 2 != 0;
};
}
static bool isRegisterSize(const GCNSubtarget &ST, unsigned Size) {
return ((ST.useRealTrue16Insts() && Size == 16) || Size % 32 == 0) &&
Size <= MaxRegisterSize;
}
static bool isRegisterVectorElementType(LLT EltTy) {
const int EltSize = EltTy.getSizeInBits();
return EltSize == 16 || EltSize % 32 == 0;
}
static bool isRegisterVectorType(LLT Ty) {
const int EltSize = Ty.getElementType().getSizeInBits();
return EltSize == 32 || EltSize == 64 ||
(EltSize == 16 && Ty.getNumElements() % 2 == 0) ||
EltSize == 128 || EltSize == 256;
}
// TODO: replace all uses of isRegisterType with isRegisterClassType
static bool isRegisterType(const GCNSubtarget &ST, LLT Ty) {
if (!isRegisterSize(ST, Ty.getSizeInBits()))
return false;
if (Ty.isVector())
return isRegisterVectorType(Ty);
return true;
}
// Any combination of 32 or 64-bit elements up the maximum register size, and
// multiples of v2s16.
static LegalityPredicate isRegisterType(const GCNSubtarget &ST,
unsigned TypeIdx) {
return [=, &ST](const LegalityQuery &Query) {
return isRegisterType(ST, Query.Types[TypeIdx]);
};
}
// RegisterType that doesn't have a corresponding RegClass.
// TODO: Once `isRegisterType` is replaced with `isRegisterClassType` this
// should be removed.
static LegalityPredicate isIllegalRegisterType(const GCNSubtarget &ST,
unsigned TypeIdx) {
return [=, &ST](const LegalityQuery &Query) {
LLT Ty = Query.Types[TypeIdx];
return isRegisterType(ST, Ty) &&
!SIRegisterInfo::getSGPRClassForBitWidth(Ty.getSizeInBits());
};
}
static LegalityPredicate elementTypeIsLegal(unsigned TypeIdx) {
return [=](const LegalityQuery &Query) {
const LLT QueryTy = Query.Types[TypeIdx];
if (!QueryTy.isVector())
return false;
const LLT EltTy = QueryTy.getElementType();
return EltTy == LLT::scalar(16) || EltTy.getSizeInBits() >= 32;
};
}
constexpr LLT F16 = LLT::float16();
constexpr LLT BF16 = LLT::bfloat16();
constexpr LLT F32 = LLT::float32();
constexpr LLT F64 = LLT::float64();
constexpr LLT V2F16 = LLT::fixed_vector(2, F16);
constexpr LLT V2BF16 = LLT::fixed_vector(2, BF16);
constexpr LLT V2F32 = LLT::fixed_vector(2, F32);
constexpr LLT V2F64 = LLT::fixed_vector(2, F64);
constexpr LLT S1 = LLT::scalar(1);
constexpr LLT S8 = LLT::scalar(8);
constexpr LLT S16 = LLT::scalar(16);
constexpr LLT S32 = LLT::scalar(32);
constexpr LLT S64 = LLT::scalar(64);
constexpr LLT S96 = LLT::scalar(96);
constexpr LLT S128 = LLT::scalar(128);
constexpr LLT S160 = LLT::scalar(160);
constexpr LLT S192 = LLT::scalar(192);
constexpr LLT S224 = LLT::scalar(224);
constexpr LLT S256 = LLT::scalar(256);
constexpr LLT S512 = LLT::scalar(512);
constexpr LLT S1024 = LLT::scalar(1024);
constexpr LLT MaxScalar = LLT::scalar(MaxRegisterSize);
constexpr LLT V2S8 = LLT::fixed_vector(2, 8);
constexpr LLT V2S16 = LLT::fixed_vector(2, 16);
constexpr LLT V4S16 = LLT::fixed_vector(4, 16);
constexpr LLT V6S16 = LLT::fixed_vector(6, 16);
constexpr LLT V8S16 = LLT::fixed_vector(8, 16);
constexpr LLT V10S16 = LLT::fixed_vector(10, 16);
constexpr LLT V12S16 = LLT::fixed_vector(12, 16);
constexpr LLT V16S16 = LLT::fixed_vector(16, 16);
constexpr LLT V2S32 = LLT::fixed_vector(2, 32);
constexpr LLT V3S32 = LLT::fixed_vector(3, 32);
constexpr LLT V4S32 = LLT::fixed_vector(4, 32);
constexpr LLT V5S32 = LLT::fixed_vector(5, 32);
constexpr LLT V6S32 = LLT::fixed_vector(6, 32);
constexpr LLT V7S32 = LLT::fixed_vector(7, 32);
constexpr LLT V8S32 = LLT::fixed_vector(8, 32);
constexpr LLT V9S32 = LLT::fixed_vector(9, 32);
constexpr LLT V10S32 = LLT::fixed_vector(10, 32);
constexpr LLT V11S32 = LLT::fixed_vector(11, 32);
constexpr LLT V12S32 = LLT::fixed_vector(12, 32);
constexpr LLT V16S32 = LLT::fixed_vector(16, 32);
constexpr LLT V32S32 = LLT::fixed_vector(32, 32);
constexpr LLT V2S64 = LLT::fixed_vector(2, 64);
constexpr LLT V3S64 = LLT::fixed_vector(3, 64);
constexpr LLT V4S64 = LLT::fixed_vector(4, 64);
constexpr LLT V5S64 = LLT::fixed_vector(5, 64);
constexpr LLT V6S64 = LLT::fixed_vector(6, 64);
constexpr LLT V7S64 = LLT::fixed_vector(7, 64);
constexpr LLT V8S64 = LLT::fixed_vector(8, 64);
constexpr LLT V16S64 = LLT::fixed_vector(16, 64);
constexpr LLT V2S128 = LLT::fixed_vector(2, 128);
constexpr LLT V4S128 = LLT::fixed_vector(4, 128);
constexpr std::initializer_list<LLT> AllScalarTypes = {
S32, S64, S96, S128, S160, S192, S224, S256, S512, S1024};
constexpr std::initializer_list<LLT> AllS16Vectors{
V2S16, V4S16, V6S16, V8S16, V10S16, V12S16, V16S16, V2S128, V4S128};
constexpr std::initializer_list<LLT> AllS32Vectors = {
V2S32, V3S32, V4S32, V5S32, V6S32, V7S32, V8S32,
V9S32, V10S32, V11S32, V12S32, V16S32, V32S32};
constexpr std::initializer_list<LLT> AllS64Vectors = {
V2S64, V3S64, V4S64, V5S64, V6S64, V7S64, V8S64, V16S64};
constexpr std::initializer_list<LLT> AllVectors{
V2S16, V4S16, V6S16, V8S16, V10S16, V12S16, V16S16, V2S128,
V4S128, V2S32, V3S32, V4S32, V5S32, V6S32, V7S32, V8S32,
V9S32, V10S32, V11S32, V12S32, V16S32, V32S32, V2S64, V3S64,
V4S64, V5S64, V6S64, V7S64, V8S64, V16S64};
// Checks whether a type is in the list of legal register types.
static bool isRegisterClassType(const GCNSubtarget &ST, LLT Ty) {
if (Ty.isPointerOrPointerVector())
Ty = Ty.changeElementType(LLT::scalar(Ty.getScalarSizeInBits()));
return is_contained(AllS32Vectors, Ty) || is_contained(AllS64Vectors, Ty) ||
is_contained(AllScalarTypes, Ty) ||
(ST.useRealTrue16Insts() && Ty == S16) ||
is_contained(AllS16Vectors, Ty);
}
static LegalityPredicate isRegisterClassType(const GCNSubtarget &ST,
unsigned TypeIdx) {
return [&ST, TypeIdx](const LegalityQuery &Query) {
return isRegisterClassType(ST, Query.Types[TypeIdx]);
};
}
// If we have a truncating store or an extending load with a data size larger
// than 32-bits, we need to reduce to a 32-bit type.
static LegalityPredicate isWideScalarExtLoadTruncStore(unsigned TypeIdx) {
return [=](const LegalityQuery &Query) {
const LLT Ty = Query.Types[TypeIdx];
return !Ty.isVector() && Ty.getSizeInBits() > 32 &&
Query.MMODescrs[0].MemoryTy.getSizeInBits() < Ty.getSizeInBits();
};
}
// If we have a truncating store or an extending load with a data size larger
// than 32-bits and mem location is a power of 2
static LegalityPredicate isTruncStoreToSizePowerOf2(unsigned TypeIdx) {
return [=](const LegalityQuery &Query) {
unsigned MemSize = Query.MMODescrs[0].MemoryTy.getSizeInBits();
return isWideScalarExtLoadTruncStore(TypeIdx)(Query) &&
isPowerOf2_64(MemSize);
};
}
// TODO: Should load to s16 be legal? Most loads extend to 32-bits, but we
// handle some operations by just promoting the register during
// selection. There are also d16 loads on GFX9+ which preserve the high bits.
static unsigned maxSizeForAddrSpace(const GCNSubtarget &ST, unsigned AS,
bool IsLoad, bool IsAtomic) {
switch (AS) {
case AMDGPUAS::PRIVATE_ADDRESS:
// FIXME: Private element size.
return ST.hasFlatScratchEnabled() ? 128 : 32;
case AMDGPUAS::LOCAL_ADDRESS:
return ST.useDS128() ? 128 : 64;
case AMDGPUAS::GLOBAL_ADDRESS:
case AMDGPUAS::CONSTANT_ADDRESS:
case AMDGPUAS::CONSTANT_ADDRESS_32BIT:
case AMDGPUAS::BUFFER_RESOURCE:
// Treat constant and global as identical. SMRD loads are sometimes usable for
// global loads (ideally constant address space should be eliminated)
// depending on the context. Legality cannot be context dependent, but
// RegBankSelect can split the load as necessary depending on the pointer
// register bank/uniformity and if the memory is invariant or not written in a
// kernel.
return IsLoad ? 512 : 128;
default:
// FIXME: Flat addresses may contextually need to be split to 32-bit parts
// if they may alias scratch depending on the subtarget. This needs to be
// moved to custom handling to use addressMayBeAccessedAsPrivate
return ST.hasMultiDwordFlatScratchAddressing() || IsAtomic ? 128 : 32;
}
}
static bool isLoadStoreSizeLegal(const GCNSubtarget &ST,
const LegalityQuery &Query) {
const LLT Ty = Query.Types[0];
// Handle G_LOAD, G_ZEXTLOAD, G_SEXTLOAD
const bool IsLoad = Query.Opcode != AMDGPU::G_STORE;
unsigned RegSize = Ty.getSizeInBits();
uint64_t MemSize = Query.MMODescrs[0].MemoryTy.getSizeInBits();
uint64_t AlignBits = Query.MMODescrs[0].AlignInBits;
unsigned AS = Query.Types[1].getAddressSpace();
// All of these need to be custom lowered to cast the pointer operand.
if (AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT)
return false;
// Do not handle extending vector loads.
if (Ty.isVector() && MemSize != RegSize)
return false;
// TODO: We should be able to widen loads if the alignment is high enough, but
// we also need to modify the memory access size.
#if 0
// Accept widening loads based on alignment.
if (IsLoad && MemSize < Size)
MemSize = std::max(MemSize, Align);
#endif
// Only 1-byte and 2-byte to 32-bit extloads are valid.
if (MemSize != RegSize && RegSize != 32)
return false;
if (IsLoad && RegSize == 16 && !ST.d16PreservesUnusedBits())
return false;
if (MemSize > maxSizeForAddrSpace(ST, AS, IsLoad,
Query.MMODescrs[0].Ordering !=
AtomicOrdering::NotAtomic))
return false;
switch (MemSize) {
case 8:
case 16:
case 32:
case 64:
case 128:
break;
case 96:
if (!ST.hasDwordx3LoadStores())
return false;
break;
case 256:
case 512:
// These may contextually need to be broken down.
break;
default:
return false;
}
assert(RegSize >= MemSize);
if (AlignBits < MemSize) {
const SITargetLowering *TLI = ST.getTargetLowering();
if (!TLI->allowsMisalignedMemoryAccessesImpl(MemSize, AS,
Align(AlignBits / 8)))
return false;
}
return true;
}
// The newer buffer intrinsic forms take their resource arguments as
// pointers in address space 8, aka s128 values. However, in order to not break
// SelectionDAG, the underlying operations have to continue to take v4i32
// arguments. Therefore, we convert resource pointers - or vectors of them
// to integer values here.
static bool hasBufferRsrcWorkaround(const LLT Ty) {
if (Ty.isPointer() && Ty.getAddressSpace() == AMDGPUAS::BUFFER_RESOURCE)
return true;
if (Ty.isVector()) {
const LLT ElemTy = Ty.getElementType();
return hasBufferRsrcWorkaround(ElemTy);
}
return false;
}
// The current selector can't handle <6 x s16>, <8 x s16>, s96, s128 etc, so
// workaround this. Eventually it should ignore the type for loads and only care
// about the size. Return true in cases where we will workaround this for now by
// bitcasting.
static bool loadStoreBitcastWorkaround(const LLT Ty) {
if (EnableNewLegality)
return false;
const unsigned Size = Ty.getSizeInBits();
if (Ty.isPointerVector())
return true;
if (Size <= 64)
return false;
// Address space 8 pointers get their own workaround.
if (hasBufferRsrcWorkaround(Ty))
return false;
if (!Ty.isVector())
return true;
unsigned EltSize = Ty.getScalarSizeInBits();
return EltSize != 32 && EltSize != 64;
}
static bool isLoadStoreLegal(const GCNSubtarget &ST, const LegalityQuery &Query) {
const LLT Ty = Query.Types[0];
return isRegisterType(ST, Ty) && isLoadStoreSizeLegal(ST, Query) &&
!hasBufferRsrcWorkaround(Ty) && !loadStoreBitcastWorkaround(Ty);
}
/// Return true if a load or store of the type should be lowered with a bitcast
/// to a different type.
static bool shouldBitcastLoadStoreType(const GCNSubtarget &ST, const LLT Ty,
const LLT MemTy) {
const unsigned MemSizeInBits = MemTy.getSizeInBits();
const unsigned Size = Ty.getSizeInBits();
if (Size != MemSizeInBits)
return Size <= 32 && Ty.isVector();
if (loadStoreBitcastWorkaround(Ty) && isRegisterType(ST, Ty))
return true;
// Don't try to handle bitcasting vector ext loads for now.
return Ty.isVector() && (!MemTy.isVector() || MemTy == Ty) &&
(Size <= 32 || isRegisterSize(ST, Size)) &&
!isRegisterVectorElementType(Ty.getElementType());
}
/// Return true if we should legalize a load by widening an odd sized memory
/// access up to the alignment. Note this case when the memory access itself
/// changes, not the size of the result register.
static bool shouldWidenLoad(const GCNSubtarget &ST, LLT MemoryTy,
uint64_t AlignInBits, unsigned AddrSpace,
unsigned Opcode) {
unsigned SizeInBits = MemoryTy.getSizeInBits();
// We don't want to widen cases that are naturally legal.
if (isPowerOf2_32(SizeInBits))
return false;
// If we have 96-bit memory operations, we shouldn't touch them. Note we may
// end up widening these for a scalar load during RegBankSelect, if we don't
// have 96-bit scalar loads.
if (SizeInBits == 96 && ST.hasDwordx3LoadStores())
return false;
if (SizeInBits >= maxSizeForAddrSpace(ST, AddrSpace, Opcode, false))
return false;
// A load is known dereferenceable up to the alignment, so it's legal to widen
// to it.
//
// TODO: Could check dereferenceable for less aligned cases.
unsigned RoundedSize = NextPowerOf2(SizeInBits);
if (AlignInBits < RoundedSize)
return false;
// Do not widen if it would introduce a slow unaligned load.
const SITargetLowering *TLI = ST.getTargetLowering();
unsigned Fast = 0;
return TLI->allowsMisalignedMemoryAccessesImpl(
RoundedSize, AddrSpace, Align(AlignInBits / 8),
MachineMemOperand::MOLoad, &Fast) &&
Fast;
}
static bool shouldWidenLoad(const GCNSubtarget &ST, const LegalityQuery &Query,
unsigned Opcode) {
if (Query.MMODescrs[0].Ordering != AtomicOrdering::NotAtomic)
return false;
return shouldWidenLoad(ST, Query.MMODescrs[0].MemoryTy,
Query.MMODescrs[0].AlignInBits,
Query.Types[1].getAddressSpace(), Opcode);
}
/// Mutates IR (typicaly a load instruction) to use a <4 x s32> as the initial
/// type of the operand `idx` and then to transform it to a `p8` via bitcasts
/// and inttoptr. In addition, handle vectors of p8. Returns the new type.
static LLT castBufferRsrcFromV4I32(MachineInstr &MI, MachineIRBuilder &B,
MachineRegisterInfo &MRI, unsigned Idx) {
MachineOperand &MO = MI.getOperand(Idx);
const LLT PointerTy = MRI.getType(MO.getReg());
// Paranoidly prevent us from doing this multiple times.
if (!hasBufferRsrcWorkaround(PointerTy))
return PointerTy;
const LLT ScalarTy = getBufferRsrcScalarType(PointerTy);
const LLT VectorTy = getBufferRsrcRegisterType(PointerTy);
if (!PointerTy.isVector()) {
// Happy path: (4 x s32) -> (s32, s32, s32, s32) -> (p8)
const unsigned NumParts = PointerTy.getSizeInBits() / 32;
const LLT I32 = LLT::integer(32);
Register VectorReg = MRI.createGenericVirtualRegister(VectorTy);
std::array<Register, 4> VectorElems;
B.setInsertPt(B.getMBB(), ++B.getInsertPt());
for (unsigned I = 0; I < NumParts; ++I)
VectorElems[I] =
B.buildExtractVectorElementConstant(I32, VectorReg, I).getReg(0);
B.buildMergeValues(MO, VectorElems);
MO.setReg(VectorReg);
return VectorTy;
}
Register BitcastReg = MRI.createGenericVirtualRegister(VectorTy);
B.setInsertPt(B.getMBB(), ++B.getInsertPt());
auto Scalar = B.buildBitcast(ScalarTy, BitcastReg);
B.buildIntToPtr(MO, Scalar);
MO.setReg(BitcastReg);
return VectorTy;
}
/// Cast a buffer resource (an address space 8 pointer) into a 4xi32, which is
/// the form in which the value must be in order to be passed to the low-level
/// representations used for MUBUF/MTBUF intrinsics. This is a hack, which is
/// needed in order to account for the fact that we can't define a register
/// class for s128 without breaking SelectionDAG.
static Register castBufferRsrcToV4I32(Register Pointer, MachineIRBuilder &B) {
MachineRegisterInfo &MRI = *B.getMRI();
const LLT PointerTy = MRI.getType(Pointer);
const LLT ScalarTy = getBufferRsrcScalarType(PointerTy);
const LLT VectorTy = getBufferRsrcRegisterType(PointerTy);
if (!PointerTy.isVector()) {
// Special case: p8 -> (s32, s32, s32, s32) -> (4xs32)
SmallVector<Register, 4> PointerParts;
const unsigned NumParts = PointerTy.getSizeInBits() / 32;
auto Unmerged = B.buildUnmerge(LLT::integer(32), Pointer);
for (unsigned I = 0; I < NumParts; ++I)
PointerParts.push_back(Unmerged.getReg(I));
return B.buildBuildVector(VectorTy, PointerParts).getReg(0);
}
Register Scalar = B.buildPtrToInt(ScalarTy, Pointer).getReg(0);
return B.buildBitcast(VectorTy, Scalar).getReg(0);
}
static void castBufferRsrcArgToV4I32(MachineInstr &MI, MachineIRBuilder &B,
unsigned Idx) {
MachineOperand &MO = MI.getOperand(Idx);
const LLT PointerTy = B.getMRI()->getType(MO.getReg());
// Paranoidly prevent us from doing this multiple times.
if (!hasBufferRsrcWorkaround(PointerTy))
return;
MO.setReg(castBufferRsrcToV4I32(MO.getReg(), B));
}
AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
const GCNTargetMachine &TM)
: ST(ST_) {
using namespace TargetOpcode;
const LLT GlobalPtr = LLT::pointer(AMDGPUAS::GLOBAL_ADDRESS, 64);
const LLT ConstantPtr = LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64);
const LLT Constant32Ptr = LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS_32BIT, 32);
const LLT LocalPtr = LLT::pointer(AMDGPUAS::LOCAL_ADDRESS, 32);
const LLT RegionPtr = LLT::pointer(AMDGPUAS::REGION_ADDRESS, 32);
const LLT FlatPtr = LLT::pointer(AMDGPUAS::FLAT_ADDRESS, 64);
const LLT PrivatePtr = LLT::pointer(AMDGPUAS::PRIVATE_ADDRESS, 32);
const LLT BufferFatPtr = LLT::pointer(AMDGPUAS::BUFFER_FAT_POINTER, 160);
const LLT RsrcPtr = LLT::pointer(AMDGPUAS::BUFFER_RESOURCE, 128);
const LLT BufferStridedPtr =
LLT::pointer(AMDGPUAS::BUFFER_STRIDED_POINTER, 192);
const LLT CodePtr = FlatPtr;
const std::initializer_list<LLT> AddrSpaces64 = {
GlobalPtr, ConstantPtr, FlatPtr
};
const std::initializer_list<LLT> AddrSpaces32 = {
LocalPtr, PrivatePtr, Constant32Ptr, RegionPtr
};
const std::initializer_list<LLT> AddrSpaces128 = {RsrcPtr};
const std::initializer_list<LLT> FPTypesBase = {F32, F64};
const std::initializer_list<LLT> FPTypes16 = {F32, F64, F16};
const std::initializer_list<LLT> FPTypesPK16 = {F32, F64, F16, V2F16};
const LLT I1 = LLT::integer(1);
const LLT I16 = LLT::integer(16);
const LLT I32 = LLT::integer(32);
const LLT I64 = LLT::integer(64);
const LLT V2I16 = LLT::fixed_vector(2, I16);
getActionDefinitionsBuilder(G_BR).alwaysLegal();
// s1 for VCC branches, s32 for SCC branches.
getActionDefinitionsBuilder(G_BRCOND).legalFor({S1, S32});
// TODO: All multiples of 32, vectors of pointers, all v2s16 pairs, more
// elements for v3s16
getActionDefinitionsBuilder(G_PHI)
.legalFor({S32, S64, V2S16, S16, V4S16, S1, S128, S256})
.legalFor(AllS32Vectors)
.legalFor(AllS64Vectors)
.legalFor(AddrSpaces64)
.legalFor(AddrSpaces32)
.legalFor(AddrSpaces128)
.legalIf(isPointer(0))
.clampScalar(0, S16, S256)
.widenScalarToNextPow2(0, 32)
.clampMaxNumElements(0, S32, 16)
.moreElementsIf(isSmallOddVector(0), oneMoreElement(0))
.scalarize(0);
if (ST.hasVOP3PInsts() && ST.hasAddNoCarryInsts() && ST.hasIntClamp()) {
// Full set of gfx9 features.
if (ST.hasAnyPackedU64Ops()) {
getActionDefinitionsBuilder({G_ADD, G_SUB})
.legalFor({S64, S32, S16, V2S16, V2S64})
.clampMaxNumElementsStrict(0, S16, 2)
.clampMaxNumElementsStrict(0, S64, 2)
.scalarize(0)
.minScalar(0, S16)
.widenScalarToNextMultipleOf(0, 32)
.maxScalar(0, S32);
} else if (ST.hasScalarAddSub64()) {
getActionDefinitionsBuilder({G_ADD, G_SUB})
.legalFor({S64, S32, S16, V2S16})
.clampMaxNumElementsStrict(0, S16, 2)
.scalarize(0)
.minScalar(0, S16)
.widenScalarToNextMultipleOf(0, 32)
.maxScalar(0, S32);
} else {
getActionDefinitionsBuilder({G_ADD, G_SUB})
.legalFor({S32, S16, V2S16})
.clampMaxNumElementsStrict(0, S16, 2)
.scalarize(0)
.minScalar(0, S16)
.widenScalarToNextMultipleOf(0, 32)
.maxScalar(0, S32);
}
if (ST.hasScalarSMulU64()) {
getActionDefinitionsBuilder(G_MUL)
.legalFor({S64, S32, S16, V2S16})
.clampMaxNumElementsStrict(0, S16, 2)
.scalarize(0)
.minScalar(0, S16)
.widenScalarToNextMultipleOf(0, 32)
.custom();
} else {
getActionDefinitionsBuilder(G_MUL)
.legalFor({S32, S16, V2S16})
.clampMaxNumElementsStrict(0, S16, 2)
.scalarize(0)
.minScalar(0, S16)
.widenScalarToNextMultipleOf(0, 32)
.custom();
}
assert(ST.hasMad64_32());
getActionDefinitionsBuilder({G_UADDSAT, G_USUBSAT, G_SADDSAT, G_SSUBSAT})
.legalFor({S32, S16, V2S16}) // Clamp modifier
.minScalarOrElt(0, S16)
.clampMaxNumElementsStrict(0, S16, 2)
.scalarize(0)
.widenScalarToNextPow2(0, 32)
.lower();
} else if (ST.has16BitInsts()) {
getActionDefinitionsBuilder({G_ADD, G_SUB})
.legalFor({S32, S16})
.minScalar(0, S16)
.widenScalarToNextMultipleOf(0, 32)
.maxScalar(0, S32)
.scalarize(0);
getActionDefinitionsBuilder(G_MUL)
.legalFor({S32, S16})
.scalarize(0)
.minScalar(0, S16)
.widenScalarToNextMultipleOf(0, 32)
.custom();
assert(ST.hasMad64_32());
// Technically the saturating operations require clamp bit support, but this
// was introduced at the same time as 16-bit operations.
getActionDefinitionsBuilder({G_UADDSAT, G_USUBSAT})
.legalFor({S32, S16}) // Clamp modifier
.minScalar(0, S16)
.scalarize(0)
.widenScalarToNextPow2(0, 16)
.lower();
// We're just lowering this, but it helps get a better result to try to
// coerce to the desired type first.
getActionDefinitionsBuilder({G_SADDSAT, G_SSUBSAT})
.minScalar(0, S16)
.scalarize(0)
.lower();
} else {
getActionDefinitionsBuilder({G_ADD, G_SUB})
.legalFor({S32})
.widenScalarToNextMultipleOf(0, 32)
.clampScalar(0, S32, S32)
.scalarize(0);
auto &Mul = getActionDefinitionsBuilder(G_MUL)
.legalFor({S32})
.scalarize(0)
.minScalar(0, S32)
.widenScalarToNextMultipleOf(0, 32);
if (ST.hasMad64_32())
Mul.custom();
else
Mul.maxScalar(0, S32);
if (ST.hasIntClamp()) {
getActionDefinitionsBuilder({G_UADDSAT, G_USUBSAT})
.legalFor({S32}) // Clamp modifier.
.scalarize(0)
.minScalarOrElt(0, S32)
.lower();
} else {
// Clamp bit support was added in VI, along with 16-bit operations.
getActionDefinitionsBuilder({G_UADDSAT, G_USUBSAT})
.minScalar(0, S32)
.scalarize(0)
.lower();
}
// FIXME: DAG expansion gets better results. The widening uses the smaller
// range values and goes for the min/max lowering directly.
getActionDefinitionsBuilder({G_SADDSAT, G_SSUBSAT})
.minScalar(0, S32)
.scalarize(0)
.lower();
}
getActionDefinitionsBuilder(
{G_SDIV, G_UDIV, G_SREM, G_UREM, G_SDIVREM, G_UDIVREM})
.customFor({S32, S64})
.clampScalar(0, S32, S64)
.widenScalarToNextPow2(0, 32)
.scalarize(0);
auto &Mulh = getActionDefinitionsBuilder({G_UMULH, G_SMULH})
.legalFor({S32})
.maxScalar(0, S32);
if (ST.hasVOP3PInsts()) {
Mulh
.clampMaxNumElements(0, S8, 2)
.lowerFor({V2S8});
}
Mulh
.scalarize(0)
.lower();
// Report legal for any types we can handle anywhere. For the cases only legal
// on the SALU, RegBankSelect will be able to re-legalize.
getActionDefinitionsBuilder({G_AND, G_OR, G_XOR})
.legalFor({S32, S1, S64, V2S32, S16, V2S16, V4S16})
.clampScalar(0, S32, S64)
.moreElementsIf(isSmallOddVector(0), oneMoreElement(0))
.fewerElementsIf(
all(vectorWiderThan(0, 64), scalarOrEltNarrowerThan(0, 64)),
fewerEltsToSize64Vector(0))
.widenScalarToNextPow2(0)
.scalarize(0);
getActionDefinitionsBuilder(
{G_UADDO, G_USUBO, G_UADDE, G_SADDE, G_USUBE, G_SSUBE})
.legalFor({{S32, S1}, {S32, S32}})
.clampScalar(0, S32, S32)
.scalarize(0);
getActionDefinitionsBuilder(G_BITCAST)
// Don't worry about the size constraint.
.legalIf(all(isRegisterClassType(ST, 0), isRegisterClassType(ST, 1)))
.widenScalarIf(all(typeInSet(0, {I16, F16, BF16}), isScalar(1)),
changeTo(0, LLT::integer(32)))
.widenScalarIf(all(isScalar(0), typeInSet(1, {I16, F16, BF16})),
changeTo(1, LLT::integer(32)))
.lower();
getActionDefinitionsBuilder(G_CONSTANT)
.legalFor({S1, S32, S64, S16, GlobalPtr,
LocalPtr, ConstantPtr, PrivatePtr, FlatPtr })
.legalIf(isPointer(0))
.clampScalar(0, S32, S64)
.widenScalarToNextPow2(0);
getActionDefinitionsBuilder(G_FCONSTANT).legalFor({F32, F64, F16, BF16});
getActionDefinitionsBuilder({G_IMPLICIT_DEF, G_FREEZE})
.legalIf(isRegisterClassType(ST, 0))
// s1 and s16 are special cases because they have legal operations on
// them, but don't really occupy registers in the normal way.
.legalFor({S1, S16})
.clampNumElements(0, V16S32, V32S32)
.moreElementsIf(isSmallOddVector(0), oneMoreElement(0))
.clampScalarOrElt(0, S32, MaxScalar)
.widenScalarToNextPow2(0, 32)
.clampMaxNumElements(0, S32, 16);
getActionDefinitionsBuilder(G_FRAME_INDEX).legalFor({PrivatePtr});
// If the amount is divergent, we have to do a wave reduction to get the
// maximum value, so this is expanded during RegBankSelect.
getActionDefinitionsBuilder(G_DYN_STACKALLOC)
.legalFor({{PrivatePtr, S32}});
getActionDefinitionsBuilder(G_STACKSAVE)
.customFor({PrivatePtr});
getActionDefinitionsBuilder(G_STACKRESTORE)
.legalFor({PrivatePtr});
getActionDefinitionsBuilder(G_WRITE_REGISTER).legalFor({S32, S64});
getActionDefinitionsBuilder({G_GET_FPENV, G_SET_FPENV}).customFor({S64});
getActionDefinitionsBuilder({G_GET_ROUNDING, G_SET_ROUNDING}).legalFor({S32});
getActionDefinitionsBuilder(G_GLOBAL_VALUE)
.customIf(typeIsNot(0, PrivatePtr));
getActionDefinitionsBuilder(G_BLOCK_ADDR).legalFor({CodePtr});
auto &FPOpActions =
getActionDefinitionsBuilder({G_FADD, G_FMUL, G_FMA}).legalFor({F32, F64});
auto &FCanonicalizeActions =
getActionDefinitionsBuilder(G_FCANONICALIZE).legalFor({F32, F64});
auto &StrictFPOpActions =
getActionDefinitionsBuilder({G_STRICT_FADD, G_STRICT_FMUL, G_STRICT_FMA})
.legalFor({F32, F64});
auto &TrigActions =
getActionDefinitionsBuilder({G_FSIN, G_FCOS}).customFor({F32, F64});
auto &FDIVActions = getActionDefinitionsBuilder(G_FDIV).customFor({F32, F64});
if (ST.has16BitInsts()) {
if (ST.hasVOP3PInsts()) {
FPOpActions.legalFor({F16, V2F16});
FCanonicalizeActions.legalFor({F16, V2F16});
StrictFPOpActions.legalFor({F16, V2F16});
} else {
FPOpActions.legalFor({F16});
FCanonicalizeActions.legalFor({F16});
StrictFPOpActions.legalFor({F16});
}
TrigActions.customFor({F16});
FDIVActions.customFor({F16});
}
if (ST.hasBF16PackedInsts()) {
// Promote scalar bf16 operations to v2bf16 (packed) operations
FPOpActions.moreElementsIf(typeIs(0, BF16), changeTo(0, V2BF16))
.legalFor({V2BF16})
.clampMaxNumElementsStrict(0, BF16, 2);
FCanonicalizeActions.moreElementsIf(typeIs(0, BF16), changeTo(0, V2BF16))
.legalFor({V2BF16})
.clampMaxNumElementsStrict(0, BF16, 2);
StrictFPOpActions.moreElementsIf(typeIs(0, BF16), changeTo(0, V2BF16))
.legalFor({V2BF16})
.clampMaxNumElementsStrict(0, BF16, 2);
} else {
FPOpActions.widenScalarFor({BF16}, changeElementTo(0, F32));
FCanonicalizeActions.widenScalarFor({BF16}, changeElementTo(0, F32));
}
if (ST.hasAnyPackedFP32Ops()) {
FPOpActions.legalFor({V2F32});
FCanonicalizeActions.legalFor({V2F32});
StrictFPOpActions.legalFor({V2F32});
FPOpActions.clampMaxNumElementsStrict(0, F32, 2);
FCanonicalizeActions.clampMaxNumElementsStrict(0, F32, 2);
StrictFPOpActions.clampMaxNumElementsStrict(0, F32, 2);
}
if (ST.hasAnyPackedFP64Ops()) {
FPOpActions.legalFor({V2F64});
FCanonicalizeActions.legalFor({V2F64});
StrictFPOpActions.legalFor({V2F64});
FPOpActions.clampMaxNumElementsStrict(0, F64, 2);
FCanonicalizeActions.clampMaxNumElementsStrict(0, F64, 2);
StrictFPOpActions.clampMaxNumElementsStrict(0, F64, 2);
}
auto &MinNumMaxNumIeee =
getActionDefinitionsBuilder({G_FMINNUM_IEEE, G_FMAXNUM_IEEE});
auto &MinNumMaxNum = getActionDefinitionsBuilder(
{G_FMINNUM, G_FMAXNUM, G_FMINIMUMNUM, G_FMAXIMUMNUM});
MinNumMaxNumIeee.legalFor({F32, F64});
MinNumMaxNum.customFor({F32, F64});
if (ST.has16BitInsts()) {
MinNumMaxNumIeee.legalFor({F16});
MinNumMaxNum.customFor({F16});
}
// V2F16
if (ST.hasVOP3PInsts()) {
MinNumMaxNumIeee.legalFor({V2F16})
.moreElementsIf(all(elementTypeIs(0, F16), isSmallOddVector(0)),
oneMoreElement(0))
.clampMaxNumElements(0, F16, 2);
MinNumMaxNum.customFor({V2F16})
.moreElementsIf(all(elementTypeIs(0, F16), isSmallOddVector(0)),
oneMoreElement(0))
.clampMaxNumElements(0, F16, 2);
}
// V2F64
if (ST.hasAnyPackedFP64Ops()) {
MinNumMaxNum.customFor({V2F64})
.moreElementsIf(all(elementTypeIs(0, F64), isSmallOddVector(0)),
oneMoreElement(0))
.clampMaxNumElements(0, F64, 2);
}
// V2BF16
if (ST.hasBF16PackedInsts()) {
MinNumMaxNumIeee.legalFor({V2BF16}).clampMaxNumElementsStrict(0, BF16, 2);
MinNumMaxNum.moreElementsIf(typeIs(0, BF16), changeTo(0, V2BF16))
.customFor({V2BF16})
.clampMaxNumElementsStrict(0, BF16, 2);
} else {
MinNumMaxNum.widenScalarFor({BF16}, changeElementTo(0, F32));
}
MinNumMaxNumIeee.scalarize(0);
MinNumMaxNum.scalarize(0);
if (!ST.has16BitInsts()) {
MinNumMaxNumIeee.minScalar(0, F32);
MinNumMaxNum.minScalar(0, F32);
}
if (ST.hasVOP3PInsts()) {
FPOpActions.clampMaxNumElementsStrict(0, F16, 2);
FCanonicalizeActions.clampMaxNumElementsStrict(0, F16, 2);
StrictFPOpActions.clampMaxNumElementsStrict(0, F16, 2);
}
FPOpActions.scalarize(0);
FCanonicalizeActions.scalarize(0);
StrictFPOpActions.scalarize(0);
TrigActions.scalarize(0);
FDIVActions.scalarize(0);
if (!ST.has16BitInsts()) {
FPOpActions.widenScalarFor({F16}, changeElementTo(0, F32));
FCanonicalizeActions.widenScalarFor({F16}, changeElementTo(0, F32));
StrictFPOpActions.widenScalarFor({F16}, changeElementTo(0, F32));
TrigActions.widenScalarFor({F16}, changeElementTo(0, F32));
FDIVActions.widenScalarFor({F16}, changeElementTo(0, F32));
}
auto &FNegAbs = getActionDefinitionsBuilder({G_FNEG, G_FABS});
FNegAbs.legalFor(FPTypesPK16)
.legalFor({BF16, V2BF16})
.legalFor(ST.hasAnyPackedFP32Ops(), {V2F32})
.clampMaxNumElementsStrict(0, F16, 2)
.clampMaxNumElementsStrict(0, BF16, 2);
if (ST.hasAnyPackedFP32Ops())
FNegAbs.clampMaxNumElementsStrict(0, F32, 2);
FNegAbs.scalarize(0);
if (ST.has16BitInsts()) {
getActionDefinitionsBuilder(G_FSQRT)
.legalFor({F16})
.legalFor(ST.hasBF16TransInsts(), {BF16})
.customFor({F32, F64})
.scalarize(0)
.widenScalarFor({BF16}, changeElementTo(0, F32))
.unsupported();
getActionDefinitionsBuilder(G_FFLOOR)
.legalFor({F32, F64, F16})
.scalarize(0);
getActionDefinitionsBuilder({G_FLDEXP, G_STRICT_FLDEXP})
.legalFor({{F32, I32}, {F64, I32}, {F16, I16}})
.scalarize(0)
.maxScalarIf(typeIs(0, F16), 1, I16)
.clampScalar(1, I32, I32)
.lower();
getActionDefinitionsBuilder(G_FFREXP)
.customFor({{F32, I32}, {F64, I32}, {F16, I16}, {F16, I32}})
.scalarize(0)
.lower();
getActionDefinitionsBuilder(G_FMODF)
.lowerFor({F16, F32, F64})
.scalarize(0)
.lower();
} else {
getActionDefinitionsBuilder(G_FSQRT)
.customFor({F32, F64, F16})
.scalarize(0)
.widenScalarFor({BF16}, changeElementTo(0, F32))
.unsupported();
if (ST.hasFractBug()) {
getActionDefinitionsBuilder(G_FFLOOR)
.customFor({F64})
.legalFor({F32, F64})
.scalarize(0)
.minScalar(0, F32);
} else {
getActionDefinitionsBuilder(G_FFLOOR)
.legalFor({F32, F64})
.scalarize(0)
.minScalar(0, F32);
}
getActionDefinitionsBuilder({G_FLDEXP, G_STRICT_FLDEXP})
.legalFor({{F32, I32}, {F64, I32}})
.scalarize(0)
.minScalar(0, F32)
.clampScalar(1, I32, I32)
.lower();
getActionDefinitionsBuilder(G_FFREXP)
.customFor({{F32, I32}, {F64, I32}})
.scalarize(0)
.minScalar(0, F32)
.clampScalar(1, I32, I32)
.lower();
getActionDefinitionsBuilder(G_FMODF)
.lowerFor({F32, F64})
.scalarize(0)
.lower();
}
auto &FPTruncActions = getActionDefinitionsBuilder(G_FPTRUNC);
if (ST.hasCvtPkF16F32Inst()) {
FPTruncActions.legalFor({{F32, F64}, {F16, F32}, {V2F16, V2F32}})
.clampMaxNumElements(0, F16, 2);
} else {
FPTruncActions.legalFor({{F32, F64}, {F16, F32}});
}
FPTruncActions.lowerFor({{BF16, F32}, {BF16, F64}, {F16, F64}}).scalarize(0);
getActionDefinitionsBuilder(G_FPEXT)
.legalFor({{F64, F32}, {F32, F16}})
.narrowScalarFor({{F64, F16}}, changeElementSizeTo(0, F32))
.lowerFor({{F32, BF16}, {F64, BF16}})
.scalarize(0);
auto &FSubActions = getActionDefinitionsBuilder({G_FSUB, G_STRICT_FSUB});
if (ST.has16BitInsts()) {
FSubActions
// Use actual fsub instruction
.legalFor({F32, F16})
// Must use fadd + fneg
.lowerFor({F64, V2F16});
} else {
FSubActions
// Use actual fsub instruction
.legalFor({F32})
// Must use fadd + fneg
.lowerFor({F64, F16, V2F16});
}
if (ST.hasBF16PackedInsts()) {
FSubActions.moreElementsIf(typeIs(0, BF16), changeTo(0, V2BF16))
.lowerFor({V2BF16})
.clampMaxNumElementsStrict(0, BF16, 2);
} else {
FSubActions.widenScalarFor({BF16}, changeElementTo(0, F32));
}
if (ST.hasAnyPackedFP32Ops())
FSubActions.lowerFor({V2F32}).clampMaxNumElements(0, F32, 2);
FSubActions.clampMaxNumElements(0, F16, 2).scalarize(0).clampScalar(0, F32,
F64);
// Whether this is legal depends on the floating point mode for the function.
auto &FMad = getActionDefinitionsBuilder(G_FMAD);
if (ST.hasMadF16() && ST.hasMadMacF32Insts())
FMad.customFor({F32, F16});
else if (ST.hasMadMacF32Insts())
FMad.customFor({F32});
else if (ST.hasMadF16())
FMad.customFor({F16});
FMad.scalarize(0)
.lower();
auto &FRem = getActionDefinitionsBuilder(G_FREM);
if (ST.has16BitInsts()) {
FRem.customFor({F16, F32, F64});
} else {
FRem.minScalar(0, F32).customFor({F32, F64});
}
FRem.scalarize(0);
// TODO: Do we need to clamp maximum bitwidth?
getActionDefinitionsBuilder(G_TRUNC)
.legalIf(isScalar(0))
.legalFor({{V2S16, V2S32}})
.clampMaxNumElements(0, S16, 2)
// Avoid scalarizing in cases that should be truly illegal. In unresolvable
// situations (like an invalid implicit use), we don't want to infinite loop
// in the legalizer.
.fewerElementsIf(elementTypeIsLegal(0), LegalizeMutations::scalarize(0))
.alwaysLegal();
getActionDefinitionsBuilder({G_SEXT, G_ZEXT, G_ANYEXT})
.legalFor({{S64, S32}, {S32, S16}, {S64, S16},
{S32, S1}, {S64, S1}, {S16, S1}})
.scalarize(0)
.clampScalar(0, S32, S64)
.widenScalarToNextPow2(1, 32);
// TODO: Split s1->s64 during regbankselect for VALU.
auto &IToFP = getActionDefinitionsBuilder({G_SITOFP, G_UITOFP})
.legalFor({{F32, I32}, {F64, I32}})
.widenScalarFor({{F16, I32}}, changeElementSizeTo(0, F32))
.lowerIf(typeIs(1, I1))
.customFor({{F32, I64}, {F64, I64}});
if (ST.has16BitInsts())
IToFP.legalFor({{F16, I16}});
IToFP.clampScalar(1, I32, I64)
.minScalar(0, F32)
.scalarize(0)
.widenScalarToNextPow2(1);
auto &FPToI = getActionDefinitionsBuilder({G_FPTOSI, G_FPTOUI})
.legalFor({{I32, F32}, {I32, F64}})
.customFor({{I64, F32}, {I64, F64}})
.widenScalarFor({{I32, F16}}, changeElementSizeTo(1, F32))
.narrowScalarFor({{I64, F16}}, changeElementSizeTo(0, I32))
.widenScalarFor({{I16, BF16}, {I32, BF16}, {I64, BF16}},
changeElementTo(1, F32));
if (ST.has16BitInsts())
FPToI.legalFor({{I16, F16}});
else
FPToI.minScalar(1, F32);
FPToI.minScalar(0, I32).widenScalarToNextPow2(0, 32).scalarize(0).lower();
// clang-format off
auto &FPToISat = getActionDefinitionsBuilder({G_FPTOSI_SAT, G_FPTOUI_SAT})
.legalFor({{I32, F32}, {I32, F64}, {I16, F32}})
.legalFor(ST.has16BitInsts(), {{I16, F16}})
.legalFor(ST.hasVCvtPkIU16F32(), {{V2I16, V2F32}})
.narrowScalarFor({{I64, F16}}, changeElementSizeTo(0, I32));
// If available, widen width <16 to i16, intead of i32 so v_cvt_i16/u16_f16 can be used.
if (ST.has16BitInsts())
FPToISat.minScalarIf(typeIs(1, F16), 0, I16);
if (ST.hasVCvtPkIU16F32())
FPToISat.clampMaxNumElements(0, I16, 2);
FPToISat.minScalar(1, F32);
FPToISat.minScalar(0, I32)
.widenScalarToNextPow2(0, 32)
.scalarize(0)
.lower();
// clang-format on
getActionDefinitionsBuilder({G_LROUND, G_LLROUND})
.clampScalar(0, I16, I64)
.scalarize(0)
.lower();
getActionDefinitionsBuilder(G_INTRINSIC_FPTRUNC_ROUND)
.legalFor({F16, F32})
.scalarize(0)
.lower();
// Lower G_FNEARBYINT and G_FRINT into G_INTRINSIC_ROUNDEVEN
getActionDefinitionsBuilder({G_INTRINSIC_ROUND, G_FRINT, G_FNEARBYINT})
.scalarize(0)
.lower();
getActionDefinitionsBuilder({G_INTRINSIC_LRINT, G_INTRINSIC_LLRINT})
.clampScalar(0, I16, I64)
.scalarize(0)
.lower();
auto &RoundingActions = getActionDefinitionsBuilder(
{G_INTRINSIC_TRUNC, G_FCEIL, G_INTRINSIC_ROUNDEVEN});
if (ST.has16BitInsts())
RoundingActions.legalFor({F16, F32, F64});
else if (ST.getGeneration() >= AMDGPUSubtarget::SEA_ISLANDS)
RoundingActions.legalFor({F32, F64});
else
RoundingActions.legalFor({F32}).customFor({F64});
RoundingActions.scalarize(0);
if (!ST.has16BitInsts())
RoundingActions.minScalar(0, F32);
getActionDefinitionsBuilder(G_PTR_ADD)
.unsupportedFor({BufferFatPtr, BufferStridedPtr, RsrcPtr})
.legalIf(all(isPointer(0), sameSize(0, 1)))
.scalarize(0)
.scalarSameSizeAs(1, 0);
getActionDefinitionsBuilder(G_PTRMASK)
.legalIf(all(sameSize(0, 1), typeInSet(1, {S64, S32})))
.scalarSameSizeAs(1, 0)
.scalarize(0);
auto &CmpBuilder =
getActionDefinitionsBuilder(G_ICMP)
// The compare output type differs based on the register bank of the output,
// so make both s1 and s32 legal.
//
// Scalar compares producing output in scc will be promoted to s32, as that
// is the allocatable register type that will be needed for the copy from
// scc. This will be promoted during RegBankSelect, and we assume something
// before that won't try to use s32 result types.
//
// Vector compares producing an output in vcc/SGPR will use s1 in VCC reg
// bank.
.legalForCartesianProduct(
{S1}, {S32, S64, GlobalPtr, LocalPtr, ConstantPtr, PrivatePtr, FlatPtr})
.legalForCartesianProduct(
{S32}, {S32, S64, GlobalPtr, LocalPtr, ConstantPtr, PrivatePtr, FlatPtr});
if (ST.has16BitInsts()) {
CmpBuilder.legalFor({{S1, S16}});
}
CmpBuilder
.widenScalarToNextPow2(1)
.clampScalar(1, S32, S64)
.scalarize(0)
.legalIf(all(typeInSet(0, {S1, S32}), isPointer(1)));
getActionDefinitionsBuilder({G_SCMP, G_UCMP}).lower();
auto &FCmpBuilder =
getActionDefinitionsBuilder(G_FCMP).legalForCartesianProduct(
{I1}, ST.has16BitInsts() ? FPTypes16 : FPTypesBase);
if (ST.hasSALUFloatInsts())
FCmpBuilder.legalForCartesianProduct({I32}, {F16, F32});
FCmpBuilder.widenScalarFor({{I1, BF16}}, changeElementTo(1, F32))
.widenScalarToNextPow2(1)
.minScalar(1, F32)
.scalarize(0);
getActionDefinitionsBuilder(G_FPOW)
.customFor({F32})
.clampScalar(0, F32, F32)
.scalarize(0);
getActionDefinitionsBuilder(G_FPOWI).clampScalar(0, F32, F32).lower();
getActionDefinitionsBuilder(G_FLOG2)
.legalFor(ST.has16BitInsts(), {F16})
.legalFor(ST.hasBF16TransInsts(), {BF16})
.customFor({F32, F16})
.scalarize(0)
.widenScalarFor({BF16}, changeElementTo(0, F32))
.lower();
getActionDefinitionsBuilder(G_FEXP2)
.legalFor(ST.has16BitInsts(), {F16})
.legalFor(ST.hasBF16TransInsts(), {BF16})
.customFor({F32, F64, F16})
.scalarize(0)
.widenScalarFor({BF16}, changeElementTo(0, F32))
.lower();
getActionDefinitionsBuilder({G_FLOG, G_FLOG10})
.customFor({F16, F32})
.scalarize(0);
getActionDefinitionsBuilder({G_FEXP, G_FEXP10})
.customFor({F16, F32, F64})
.scalarize(0);
// The 64-bit versions produce 32-bit results, but only on the SALU.
getActionDefinitionsBuilder(G_CTPOP)
.legalFor({{S32, S32}, {S32, S64}})
.clampScalar(0, S32, S32)
.widenScalarToNextPow2(1, 32)
.clampScalar(1, S32, S64)
.scalarize(0)
.widenScalarToNextPow2(0, 32);
// If no 16 bit instr is available, lower into different instructions.
if (ST.has16BitInsts())
getActionDefinitionsBuilder(G_IS_FPCLASS)
.legalForCartesianProduct({I1}, FPTypes16)
.widenScalarToNextPow2(1)
.scalarize(0)
.lower();
else
getActionDefinitionsBuilder(G_IS_FPCLASS)
.legalForCartesianProduct({I1}, FPTypesBase)
.lowerFor({I1, F16})
.widenScalarToNextPow2(1)
.scalarize(0)
.lower();
// The hardware instructions return a different result on 0 than the generic
// instructions expect. The hardware produces -1, but these produce the
// bitwidth.
getActionDefinitionsBuilder({G_CTLZ, G_CTTZ})
.scalarize(0)
.clampScalar(0, S32, S32)
.clampScalar(1, S32, S64)
.widenScalarToNextPow2(0, 32)
.widenScalarToNextPow2(1, 32)
.custom();
// The 64-bit versions produce 32-bit results, but only on the SALU.
getActionDefinitionsBuilder(G_CTLZ_ZERO_POISON)
.legalFor({{S32, S32}, {S32, S64}})
.customIf(scalarNarrowerThan(1, 32))
.clampScalar(0, S32, S32)
.clampScalar(1, S32, S64)
.scalarize(0)
.widenScalarToNextPow2(0, 32)
.widenScalarToNextPow2(1, 32);
getActionDefinitionsBuilder(G_CTTZ_ZERO_POISON)
.legalFor({{S32, S32}, {S32, S64}})
.clampScalar(0, S32, S32)
.clampScalar(1, S32, S64)
.scalarize(0)
.widenScalarToNextPow2(0, 32)
.widenScalarToNextPow2(1, 32);
getActionDefinitionsBuilder(G_CTLS)
.customFor({{S32, S32}})
.scalarize(0)
.clampScalar(0, S32, S32)
.clampScalar(1, S32, S32);
// S64 is only legal on SALU, and needs to be broken into 32-bit elements in
// RegBankSelect.
getActionDefinitionsBuilder(G_BITREVERSE)
.legalFor({S32, S64})
.clampScalar(0, S32, S64)
.scalarize(0)
.widenScalarToNextPow2(0);
if (ST.has16BitInsts()) {
getActionDefinitionsBuilder(G_BSWAP)
.legalFor({S16, S32, V2S16})
.clampMaxNumElementsStrict(0, S16, 2)
// FIXME: Fixing non-power-of-2 before clamp is workaround for
// narrowScalar limitation.
.widenScalarToNextPow2(0)
.clampScalar(0, S16, S32)
.scalarize(0);
if (ST.hasVOP3PInsts()) {
getActionDefinitionsBuilder(G_ABS)
.legalFor({S32, S16, V2S16})
.clampMaxNumElements(0, S16, 2)
.minScalar(0, S16)
.widenScalarToNextPow2(0)
.scalarize(0)
.lower();
if (ST.useMinMaxI64Insts()) {
getActionDefinitionsBuilder({G_SMIN, G_SMAX, G_UMIN, G_UMAX})
.legalFor({S32, S16, S64, V2S16})
.clampMaxNumElements(0, S16, 2)
.minScalar(0, S16)
.widenScalarToNextPow2(0)
.scalarize(0)
.lower();
} else {
getActionDefinitionsBuilder({G_SMIN, G_SMAX, G_UMIN, G_UMAX})
.legalFor({S32, S16, V2S16})
.clampMaxNumElements(0, S16, 2)
.minScalar(0, S16)
.widenScalarToNextPow2(0)
.scalarize(0)
.lower();
}
} else {
getActionDefinitionsBuilder({G_SMIN, G_SMAX, G_UMIN, G_UMAX, G_ABS})
.legalFor({S32, S16})
.widenScalarToNextPow2(0)
.minScalar(0, S16)
.scalarize(0)
.lower();
}
} else {
// TODO: Should have same legality without v_perm_b32
getActionDefinitionsBuilder(G_BSWAP)
.legalFor({S32})
.lowerIf(scalarNarrowerThan(0, 32))
// FIXME: Fixing non-power-of-2 before clamp is workaround for
// narrowScalar limitation.
.widenScalarToNextPow2(0)
.maxScalar(0, S32)
.scalarize(0)
.lower();
getActionDefinitionsBuilder({G_SMIN, G_SMAX, G_UMIN, G_UMAX, G_ABS})
.legalFor({S32})
.minScalar(0, S32)
.widenScalarToNextPow2(0)
.scalarize(0)
.lower();
}
getActionDefinitionsBuilder(G_INTTOPTR)
// List the common cases
.legalForCartesianProduct(AddrSpaces64, {S64})
.legalForCartesianProduct(AddrSpaces32, {S32})
.scalarize(0)
// Accept any address space as long as the size matches
.legalIf(sameSize(0, 1))
.widenScalarIf(smallerThan(1, 0),
[](const LegalityQuery &Query) {
return std::pair(
1, LLT::scalar(Query.Types[0].getSizeInBits()));
})
.narrowScalarIf(largerThan(1, 0), [](const LegalityQuery &Query) {
return std::pair(1, LLT::scalar(Query.Types[0].getSizeInBits()));
});
getActionDefinitionsBuilder(G_PTRTOINT)
// List the common cases
.legalForCartesianProduct(AddrSpaces64, {S64})
.legalForCartesianProduct(AddrSpaces32, {S32})
.scalarize(0)
// Accept any address space as long as the size matches
.legalIf(sameSize(0, 1))
.widenScalarIf(smallerThan(0, 1),
[](const LegalityQuery &Query) {
return std::pair(
0, LLT::scalar(Query.Types[1].getSizeInBits()));
})
.narrowScalarIf(largerThan(0, 1), [](const LegalityQuery &Query) {
return std::pair(0, LLT::scalar(Query.Types[1].getSizeInBits()));
});
getActionDefinitionsBuilder(G_ADDRSPACE_CAST)
.scalarize(0)
.custom();
const auto needToSplitMemOp = [=](const LegalityQuery &Query,
bool IsLoad) -> bool {
const LLT DstTy = Query.Types[0];
// Split vector extloads.
unsigned MemSize = Query.MMODescrs[0].MemoryTy.getSizeInBits();
if (DstTy.isVector() && DstTy.getSizeInBits() > MemSize)
return true;
const LLT PtrTy = Query.Types[1];
unsigned AS = PtrTy.getAddressSpace();
if (MemSize > maxSizeForAddrSpace(ST, AS, IsLoad,
Query.MMODescrs[0].Ordering !=
AtomicOrdering::NotAtomic))
return true;
if (AS == AMDGPUAS::LOCAL_ADDRESS && MemSize == 64 &&
!ST.hasUsableDSOffset() && Query.MMODescrs[0].AlignInBits == 32)
return true;
// Catch weird sized loads that don't evenly divide into the access sizes
// TODO: May be able to widen depending on alignment etc.
unsigned NumRegs = (MemSize + 31) / 32;
if (NumRegs == 3) {
if (!ST.hasDwordx3LoadStores())
return true;
} else {
// If the alignment allows, these should have been widened.
if (!isPowerOf2_32(NumRegs))
return true;
}
return false;
};
unsigned GlobalAlign32 = ST.hasUnalignedBufferAccessEnabled() ? 0 : 32;
unsigned GlobalAlign16 = ST.hasUnalignedBufferAccessEnabled() ? 0 : 16;
unsigned GlobalAlign8 = ST.hasUnalignedBufferAccessEnabled() ? 0 : 8;
unsigned LocalAlign64 = ST.hasUsableDSOffset() ? 32 : 64;
// TODO: Refine based on subtargets which support unaligned access or 128-bit
// LDS
// TODO: Unsupported flat for SI.
for (unsigned Op : {G_LOAD, G_STORE}) {
const bool IsStore = Op == G_STORE;
auto &Actions = getActionDefinitionsBuilder(Op);
// Explicitly list some common cases.
// TODO: Does this help compile time at all?
Actions.legalForTypesWithMemDesc(
{{S32, GlobalPtr, S32, GlobalAlign32},
{V2S32, GlobalPtr, V2S32, GlobalAlign32},
{V4S32, GlobalPtr, V4S32, GlobalAlign32},
{S64, GlobalPtr, S64, GlobalAlign32},
{V2S64, GlobalPtr, V2S64, GlobalAlign32},
{V2S16, GlobalPtr, V2S16, GlobalAlign32},
{S32, GlobalPtr, S8, GlobalAlign8},
{S32, GlobalPtr, S16, GlobalAlign16},
{S32, LocalPtr, S32, 32},
{S64, LocalPtr, S64, LocalAlign64},
{V2S32, LocalPtr, V2S32, LocalAlign64},
{S32, LocalPtr, S8, 8},
{S32, LocalPtr, S16, 16},
{V2S16, LocalPtr, S32, 32},
{S32, PrivatePtr, S32, 32},
{S32, PrivatePtr, S8, 8},
{S32, PrivatePtr, S16, 16},
{V2S16, PrivatePtr, S32, 32},
{S32, ConstantPtr, S32, GlobalAlign32},
{V2S32, ConstantPtr, V2S32, GlobalAlign32},
{V4S32, ConstantPtr, V4S32, GlobalAlign32},
{S64, ConstantPtr, S64, GlobalAlign32}});
if (IsStore || ST.d16PreservesUnusedBits()) {
Actions.legalForTypesWithMemDesc(ST.useRealTrue16Insts(), /* Pred */
{{S16, GlobalPtr, S8, GlobalAlign8},
{S16, GlobalPtr, S16, GlobalAlign16},
{S16, LocalPtr, S8, 8},
{S16, LocalPtr, S16, 16},
{S16, PrivatePtr, S8, 8},
{S16, PrivatePtr, S16, 16}});
}
Actions.legalIf(
[=](const LegalityQuery &Query) -> bool {
return isLoadStoreLegal(ST, Query);
});
// The custom pointers (fat pointers, buffer resources) don't work with load
// and store at this level. Fat pointers should have been lowered to
// intrinsics before the translation to MIR.
Actions.unsupportedIf(
typeInSet(1, {BufferFatPtr, BufferStridedPtr, RsrcPtr}));
// Address space 8 pointers are handled by a 4xs32 load, bitcast, and
// ptrtoint. This is needed to account for the fact that we can't have i128
// as a register class for SelectionDAG reasons.
Actions.customIf([=](const LegalityQuery &Query) -> bool {
return hasBufferRsrcWorkaround(Query.Types[0]);
});
// Constant 32-bit is handled by addrspacecasting the 32-bit pointer to
// 64-bits.
//
// TODO: Should generalize bitcast action into coerce, which will also cover
// inserting addrspacecasts.
Actions.customIf(typeIs(1, Constant32Ptr));
// Turn any illegal element vectors into something easier to deal
// with. These will ultimately produce 32-bit scalar shifts to extract the
// parts anyway.
//
// For odd 16-bit element vectors, prefer to split those into pieces with
// 16-bit vector parts.
Actions.bitcastIf(
[=](const LegalityQuery &Query) -> bool {
return shouldBitcastLoadStoreType(ST, Query.Types[0],
Query.MMODescrs[0].MemoryTy);
}, bitcastToRegisterType(0));
if (!IsStore) {
// Widen suitably aligned loads by loading extra bytes. The standard
// legalization actions can't properly express widening memory operands.
Actions.customIf([=](const LegalityQuery &Query) -> bool {
return shouldWidenLoad(ST, Query, G_LOAD);
});
}
// FIXME: load/store narrowing should be moved to lower action
Actions
.narrowScalarIf(
[=](const LegalityQuery &Query) -> bool {
return !Query.Types[0].isVector() &&
needToSplitMemOp(Query, Op == G_LOAD);
},
[=](const LegalityQuery &Query) -> std::pair<unsigned, LLT> {
const LLT DstTy = Query.Types[0];
const LLT PtrTy = Query.Types[1];
const unsigned DstSize = DstTy.getSizeInBits();
unsigned MemSize = Query.MMODescrs[0].MemoryTy.getSizeInBits();
// Split extloads.
if (DstSize > MemSize)
return std::pair(0, LLT::integer(MemSize));
unsigned MaxSize = maxSizeForAddrSpace(
ST, PtrTy.getAddressSpace(), Op == G_LOAD,
Query.MMODescrs[0].Ordering != AtomicOrdering::NotAtomic);
if (MemSize > MaxSize)
return std::pair(0, LLT::integer(MaxSize));
uint64_t Align = Query.MMODescrs[0].AlignInBits;
return std::pair(0, LLT::integer(Align));
})
.fewerElementsIf(
[=](const LegalityQuery &Query) -> bool {
return Query.Types[0].isVector() &&
needToSplitMemOp(Query, Op == G_LOAD);
},
[=](const LegalityQuery &Query) -> std::pair<unsigned, LLT> {
const LLT DstTy = Query.Types[0];
const LLT PtrTy = Query.Types[1];
LLT EltTy = DstTy.getElementType();
unsigned MaxSize = maxSizeForAddrSpace(
ST, PtrTy.getAddressSpace(), Op == G_LOAD,
Query.MMODescrs[0].Ordering != AtomicOrdering::NotAtomic);
// FIXME: Handle widened to power of 2 results better. This ends
// up scalarizing.
// FIXME: 3 element stores scalarized on SI
// Split if it's too large for the address space.
unsigned MemSize = Query.MMODescrs[0].MemoryTy.getSizeInBits();
if (MemSize > MaxSize) {
unsigned NumElts = DstTy.getNumElements();
unsigned EltSize = EltTy.getSizeInBits();
if (MaxSize % EltSize == 0) {
return std::pair(
0, LLT::scalarOrVector(
ElementCount::getFixed(MaxSize / EltSize), EltTy));
}
unsigned NumPieces = MemSize / MaxSize;
// FIXME: Refine when odd breakdowns handled
// The scalars will need to be re-legalized.
if (NumPieces == 1 || NumPieces >= NumElts ||
NumElts % NumPieces != 0)
return std::pair(0, EltTy);
return std::pair(0,
LLT::fixed_vector(NumElts / NumPieces, EltTy));
}
// FIXME: We could probably handle weird extending loads better.
if (DstTy.getSizeInBits() > MemSize)
return std::pair(0, EltTy);
unsigned EltSize = EltTy.getSizeInBits();
unsigned DstSize = DstTy.getSizeInBits();
if (!isPowerOf2_32(DstSize)) {
// We're probably decomposing an odd sized store. Try to split
// to the widest type. TODO: Account for alignment. As-is it
// should be OK, since the new parts will be further legalized.
unsigned FloorSize = llvm::bit_floor(DstSize);
return std::pair(
0, LLT::scalarOrVector(
ElementCount::getFixed(FloorSize / EltSize), EltTy));
}
// May need relegalization for the scalars.
return std::pair(0, EltTy);
})
.widenScalarIf(scalarNarrowerThan(0, 32), changeTo(0, LLT::integer(32)))
.narrowScalarIf(isTruncStoreToSizePowerOf2(0),
getScalarTypeFromMemDesc(0))
.widenScalarToNextPow2(0)
.moreElementsIf(vectorSmallerThan(0, 32), moreEltsToNext32Bit(0))
.lower();
}
// FIXME: Unaligned accesses not lowered.
auto &ExtLoads =
getActionDefinitionsBuilder({G_SEXTLOAD, G_ZEXTLOAD})
.legalForTypesWithMemDesc({{S32, GlobalPtr, S8, 8},
{S32, GlobalPtr, S16, 2 * 8},
{S32, LocalPtr, S8, 8},
{S32, LocalPtr, S16, 16},
{S32, PrivatePtr, S8, 8},
{S32, PrivatePtr, S16, 16},
{S32, ConstantPtr, S8, 8},
{S32, ConstantPtr, S16, 2 * 8}})
.legalForTypesWithMemDesc(ST.useRealTrue16Insts(),
{{S16, GlobalPtr, S8, GlobalAlign8},
{S16, LocalPtr, S8, GlobalAlign8},
{S16, PrivatePtr, S8, GlobalAlign8},
{S16, ConstantPtr, S8, GlobalAlign8}})
.legalIf([=](const LegalityQuery &Query) -> bool {
return isLoadStoreLegal(ST, Query);
});
if (ST.hasFlatAddressSpace()) {
ExtLoads.legalForTypesWithMemDesc(
{{S32, FlatPtr, S8, 8}, {S32, FlatPtr, S16, 16}});
ExtLoads.legalForTypesWithMemDesc(ST.useRealTrue16Insts(),
{{S16, FlatPtr, S8, GlobalAlign8}});
}
// Constant 32-bit is handled by addrspacecasting the 32-bit pointer to
// 64-bits.
//
// TODO: Should generalize bitcast action into coerce, which will also cover
// inserting addrspacecasts.
ExtLoads.customIf(typeIs(1, Constant32Ptr));
ExtLoads.narrowScalarIf(
[](const LegalityQuery &Query) {
LLT MemTy = Query.MMODescrs[0].MemoryTy;
return MemTy.isScalar() && MemTy.getSizeInBits() > 32 &&
Query.Types[0].getSizeInBits() > MemTy.getSizeInBits();
}, // For large MemSize, narrowscalar to MemSize (load MemSize + ext)
getScalarTypeFromMemDesc(0));
ExtLoads.clampScalar(0, S32, S32)
.widenScalarToNextPow2(0)
.lower();
auto &Atomics = getActionDefinitionsBuilder(
{G_ATOMICRMW_XCHG, G_ATOMICRMW_ADD, G_ATOMICRMW_SUB,
G_ATOMICRMW_AND, G_ATOMICRMW_OR, G_ATOMICRMW_XOR,
G_ATOMICRMW_MAX, G_ATOMICRMW_MIN, G_ATOMICRMW_UMAX,
G_ATOMICRMW_UMIN, G_ATOMICRMW_UINC_WRAP, G_ATOMICRMW_UDEC_WRAP})
.legalFor({{S32, GlobalPtr}, {S32, LocalPtr},
{S64, GlobalPtr}, {S64, LocalPtr},
{S32, RegionPtr}, {S64, RegionPtr}});
if (ST.hasFlatAddressSpace()) {
Atomics.legalFor({{S32, FlatPtr}, {S64, FlatPtr}});
}
auto &Atomics32 =
getActionDefinitionsBuilder({G_ATOMICRMW_USUB_COND, G_ATOMICRMW_USUB_SAT})
.legalFor({{S32, GlobalPtr}, {S32, LocalPtr}, {S32, RegionPtr}});
if (ST.hasFlatAddressSpace()) {
Atomics32.legalFor({{S32, FlatPtr}});
}
// TODO: v2bf16 operations, and fat buffer pointer support.
auto &Atomic = getActionDefinitionsBuilder(G_ATOMICRMW_FADD);
if (ST.hasLDSFPAtomicAddF32()) {
Atomic.legalFor({{F32, LocalPtr}, {F32, RegionPtr}});
if (ST.hasLdsAtomicAddF64())
Atomic.legalFor({{F64, LocalPtr}});
if (ST.hasAtomicDsPkAdd16Insts())
Atomic.legalFor({{V2F16, LocalPtr}, {V2BF16, LocalPtr}});
}
if (ST.hasAtomicFaddInsts())
Atomic.legalFor({{F32, GlobalPtr}});
if (ST.hasFlatAtomicFaddF32Inst())
Atomic.legalFor({{F32, FlatPtr}});
if (ST.hasGFX90AInsts() || ST.hasGFX1250Insts()) {
// These are legal with some caveats, and should have undergone expansion in
// the IR in most situations
// TODO: Move atomic expansion into legalizer
Atomic.legalFor({{F32, GlobalPtr}, {F64, GlobalPtr}, {F64, FlatPtr}});
}
if (ST.hasAtomicBufferGlobalPkAddF16NoRtnInsts() ||
ST.hasAtomicBufferGlobalPkAddF16Insts())
Atomic.legalFor({{V2F16, GlobalPtr}, {V2F16, BufferFatPtr}});
if (ST.hasAtomicGlobalPkAddBF16Inst())
Atomic.legalFor({{V2BF16, GlobalPtr}});
if (ST.hasAtomicFlatPkAdd16Insts())
Atomic.legalFor({{V2F16, FlatPtr}, {V2BF16, FlatPtr}});
// Most of the legalization work here is done by AtomicExpand. We could
// probably use a simpler legality rule that just assumes anything is OK.
auto &AtomicFMinFMax =
getActionDefinitionsBuilder({G_ATOMICRMW_FMIN, G_ATOMICRMW_FMAX})
.legalFor({{F32, LocalPtr}, {F64, LocalPtr}});
if (ST.hasAtomicFMinFMaxF32GlobalInsts())
AtomicFMinFMax.legalFor({{F32, GlobalPtr},{F32, BufferFatPtr}});
if (ST.hasAtomicFMinFMaxF64GlobalInsts())
AtomicFMinFMax.legalFor({{F64, GlobalPtr}, {F64, BufferFatPtr}});
if (ST.hasAtomicFMinFMaxF32FlatInsts())
AtomicFMinFMax.legalFor({F32, FlatPtr});
if (ST.hasAtomicFMinFMaxF64FlatInsts())
AtomicFMinFMax.legalFor({F64, FlatPtr});
// BUFFER/FLAT_ATOMIC_CMP_SWAP on GCN GPUs needs input marshalling, and output
// demarshalling
getActionDefinitionsBuilder(G_ATOMIC_CMPXCHG)
.customFor({{S32, GlobalPtr}, {S64, GlobalPtr},
{S32, FlatPtr}, {S64, FlatPtr}})
.legalFor({{S32, LocalPtr}, {S64, LocalPtr},
{S32, RegionPtr}, {S64, RegionPtr}});
// TODO: Pointer types, any 32-bit or 64-bit vector
// Condition should be s32 for scalar, s1 for vector.
getActionDefinitionsBuilder(G_SELECT)
.legalForCartesianProduct({S32, S64, S16, V2S32, V2S16, V4S16, GlobalPtr,
LocalPtr, FlatPtr, PrivatePtr,
LLT::fixed_vector(2, LocalPtr),
LLT::fixed_vector(2, PrivatePtr)},
{S1, S32})
.clampScalar(0, S16, S64)
.scalarize(1)
.moreElementsIf(isSmallOddVector(0), oneMoreElement(0))
.fewerElementsIf(numElementsNotEven(0), scalarize(0))
.clampMaxNumElements(0, S32, 2)
.clampMaxNumElements(0, LocalPtr, 2)
.clampMaxNumElements(0, PrivatePtr, 2)
.scalarize(0)
.widenScalarToNextPow2(0)
.legalIf(all(isPointer(0), typeInSet(1, {S1, S32})));
// TODO: Only the low 4/5/6 bits of the shift amount are observed, so we can
// be more flexible with the shift amount type.
auto &Shifts = getActionDefinitionsBuilder({G_SHL, G_LSHR, G_ASHR})
.legalFor({{S32, S32}, {S64, S32}});
if (ST.has16BitInsts()) {
if (ST.hasVOP3PInsts()) {
Shifts.legalFor({{S16, S16}, {V2S16, V2S16}})
.clampMaxNumElements(0, S16, 2);
} else
Shifts.legalFor({{S16, S16}});
// TODO: Support 16-bit shift amounts for all types
Shifts.widenScalarIf(
[=](const LegalityQuery &Query) {
// Use 16-bit shift amounts for any 16-bit shift. Otherwise we want a
// 32-bit amount.
const LLT ValTy = Query.Types[0];
const LLT AmountTy = Query.Types[1];
return ValTy.isScalar() && ValTy.getSizeInBits() <= 16 &&
AmountTy.getSizeInBits() < 16;
},
changeElementSizeTo(1, S16));
Shifts.maxScalarIf(typeIs(0, S16), 1, S16);
Shifts.clampScalar(1, S32, S32);
Shifts.widenScalarToNextPow2(0, 16);
Shifts.clampScalar(0, S16, S64);
getActionDefinitionsBuilder({G_SSHLSAT, G_USHLSAT})
.minScalar(0, S16)
.scalarize(0)
.lower();
} else {
// Make sure we legalize the shift amount type first, as the general
// expansion for the shifted type will produce much worse code if it hasn't
// been truncated already.
Shifts.clampScalar(1, S32, S32);
Shifts.widenScalarToNextPow2(0, 32);
Shifts.clampScalar(0, S32, S64);
getActionDefinitionsBuilder({G_SSHLSAT, G_USHLSAT})
.minScalar(0, S32)
.scalarize(0)
.lower();
}
Shifts.scalarize(0);
for (unsigned Op : {G_EXTRACT_VECTOR_ELT, G_INSERT_VECTOR_ELT}) {
unsigned VecTypeIdx = Op == G_EXTRACT_VECTOR_ELT ? 1 : 0;
unsigned EltTypeIdx = Op == G_EXTRACT_VECTOR_ELT ? 0 : 1;
unsigned IdxTypeIdx = 2;
getActionDefinitionsBuilder(Op)
.customIf([=](const LegalityQuery &Query) {
const LLT EltTy = Query.Types[EltTypeIdx];
const LLT VecTy = Query.Types[VecTypeIdx];
const LLT IdxTy = Query.Types[IdxTypeIdx];
const unsigned EltSize = EltTy.getSizeInBits();
const bool isLegalVecType =
!!SIRegisterInfo::getSGPRClassForBitWidth(VecTy.getSizeInBits());
// Address space 8 pointers are 128-bit wide values, but the logic
// below will try to bitcast them to 2N x s64, which will fail.
// Therefore, as an intermediate step, wrap extracts/insertions from a
// ptrtoint-ing the vector and scalar arguments (or inttoptring the
// extraction result) in order to produce a vector operation that can
// be handled by the logic below.
if (EltTy.isPointer() && EltSize > 64)
return true;
return (EltSize == 32 || EltSize == 64) &&
VecTy.getSizeInBits() % 32 == 0 &&
VecTy.getSizeInBits() <= MaxRegisterSize &&
IdxTy.getSizeInBits() == 32 &&
isLegalVecType;
})
.bitcastIf(all(sizeIsMultipleOf32(VecTypeIdx),
scalarOrEltNarrowerThan(VecTypeIdx, 32)),
bitcastToVectorElement32(VecTypeIdx))
//.bitcastIf(vectorSmallerThan(1, 32), bitcastToScalar(1))
.bitcastIf(all(sizeIsMultipleOf32(VecTypeIdx),
scalarOrEltWiderThan(VecTypeIdx, 64)),
[=](const LegalityQuery &Query) {
// For > 64-bit element types, try to turn this into a
// 64-bit element vector since we may be able to do better
// indexing if this is scalar. If not, fall back to 32.
const LLT EltTy = Query.Types[EltTypeIdx];
const LLT VecTy = Query.Types[VecTypeIdx];
const unsigned DstEltSize = EltTy.getSizeInBits();
const unsigned VecSize = VecTy.getSizeInBits();
const unsigned TargetEltSize =
DstEltSize % 64 == 0 ? 64 : 32;
return std::pair(VecTypeIdx,
LLT::fixed_vector(VecSize / TargetEltSize,
TargetEltSize));
})
.clampScalar(EltTypeIdx, S32, S64)
.clampScalar(VecTypeIdx, S32, S64)
.clampScalar(IdxTypeIdx, S32, S32)
.clampMaxNumElements(VecTypeIdx, S32, 32)
// TODO: Clamp elements for 64-bit vectors?
.moreElementsIf(isIllegalRegisterType(ST, VecTypeIdx),
moreElementsToNextExistingRegClass(VecTypeIdx))
// It should only be necessary with variable indexes.
// As a last resort, lower to the stack
.lower();
}
getActionDefinitionsBuilder(G_EXTRACT_VECTOR_ELT)
.unsupportedIf([=](const LegalityQuery &Query) {
const LLT &EltTy = Query.Types[1].getElementType();
return Query.Types[0] != EltTy;
});
for (unsigned Op : {G_EXTRACT, G_INSERT}) {
unsigned BigTyIdx = Op == G_EXTRACT ? 1 : 0;
unsigned LitTyIdx = Op == G_EXTRACT ? 0 : 1;
getActionDefinitionsBuilder(Op)
.widenScalarIf(
[=](const LegalityQuery &Query) {
const LLT BigTy = Query.Types[BigTyIdx];
return (BigTy.getScalarSizeInBits() < 16);
},
LegalizeMutations::widenScalarOrEltToNextPow2(BigTyIdx, 16))
.widenScalarIf(
[=](const LegalityQuery &Query) {
const LLT LitTy = Query.Types[LitTyIdx];
return (LitTy.getScalarSizeInBits() < 16);
},
LegalizeMutations::widenScalarOrEltToNextPow2(LitTyIdx, 16))
.moreElementsIf(isSmallOddVector(BigTyIdx), oneMoreElement(BigTyIdx))
.widenScalarToNextPow2(BigTyIdx, 32)
.customIf([=](const LegalityQuery &Query) {
// Generic lower operates on the full-width value, producing
// shift+trunc/mask sequences. For simple cases where extract/insert
// values are 32-bit aligned, we can instead unmerge/merge and work on
// the 32-bit components. However, we can't check the offset here so
// custom lower function will have to call generic lowering if offset
// is not 32-bit aligned.
const LLT BigTy = Query.Types[BigTyIdx];
const LLT LitTy = Query.Types[LitTyIdx];
return !BigTy.isVector() && BigTy.getSizeInBits() % 32 == 0 &&
LitTy.getSizeInBits() % 32 == 0;
})
.lower();
}
auto &BuildVector =
getActionDefinitionsBuilder(G_BUILD_VECTOR)
.legalForCartesianProduct(AllS32Vectors, {S32})
.legalForCartesianProduct(AllS64Vectors, {S64})
.clampNumElements(0, V16S32, V32S32)
.clampNumElements(0, V2S64, V16S64)
.fewerElementsIf(isWideVec16(0),
changeElementCountTo(0, ElementCount::getFixed(2)))
.moreElementsIf(isIllegalRegisterType(ST, 0),
moreElementsToNextExistingRegClass(0));
if (ST.hasScalarPackInsts()) {
BuildVector
// FIXME: Should probably widen s1 vectors straight to s32
.minScalarOrElt(0, S16)
.minScalar(1, S16);
getActionDefinitionsBuilder(G_BUILD_VECTOR_TRUNC)
.legalFor({V2S16, S32})
.lower();
} else {
BuildVector.customFor({V2S16, S16});
BuildVector.minScalarOrElt(0, S32);
getActionDefinitionsBuilder(G_BUILD_VECTOR_TRUNC)
.customFor({V2S16, S32})
.lower();
}
BuildVector.legalIf(isRegisterType(ST, 0));
// FIXME: Clamp maximum size
getActionDefinitionsBuilder(G_CONCAT_VECTORS)
.legalIf(all(isRegisterType(ST, 0), isRegisterType(ST, 1)))
.clampMaxNumElements(0, S32, 32)
.clampMaxNumElements(1, S16, 2) // TODO: Make 4?
.clampMaxNumElements(0, S16, 64);
getActionDefinitionsBuilder(G_SHUFFLE_VECTOR).lower();
// Merge/Unmerge
for (unsigned Op : {G_MERGE_VALUES, G_UNMERGE_VALUES}) {
unsigned BigTyIdx = Op == G_MERGE_VALUES ? 0 : 1;
unsigned LitTyIdx = Op == G_MERGE_VALUES ? 1 : 0;
auto notValidElt = [=](const LegalityQuery &Query, unsigned TypeIdx) {
const LLT Ty = Query.Types[TypeIdx];
if (Ty.isVector()) {
const LLT &EltTy = Ty.getElementType();
if (EltTy.getSizeInBits() < 8 || EltTy.getSizeInBits() > 512)
return true;
if (!llvm::has_single_bit<uint32_t>(EltTy.getSizeInBits()))
return true;
}
return false;
};
auto &Builder =
getActionDefinitionsBuilder(Op)
.legalIf(all(isRegisterType(ST, 0), isRegisterType(ST, 1)))
.lowerFor({{S16, V2S16}})
.lowerIf([=](const LegalityQuery &Query) {
const LLT BigTy = Query.Types[BigTyIdx];
return BigTy.getSizeInBits() == 32;
})
// Try to widen to s16 first for small types.
// TODO: Only do this on targets with legal s16 shifts
.minScalarOrEltIf(scalarNarrowerThan(LitTyIdx, 16), LitTyIdx, S16)
.widenScalarToNextPow2(LitTyIdx, /*Min*/ 16)
.moreElementsIf(isSmallOddVector(BigTyIdx),
oneMoreElement(BigTyIdx))
.fewerElementsIf(all(typeIs(0, S16), vectorWiderThan(1, 32),
elementTypeIs(1, S16)),
changeElementCountTo(1, ElementCount::getFixed(2)))
// Clamp the little scalar to s8-s256 and make it a power of 2. It's
// not worth considering the multiples of 64 since 2*192 and 2*384
// are not valid.
.clampScalar(LitTyIdx, S32, S512)
.widenScalarToNextPow2(LitTyIdx, /*Min*/ 32)
// Break up vectors with weird elements into scalars
.fewerElementsIf(
[=](const LegalityQuery &Query) {
return notValidElt(Query, LitTyIdx);
},
scalarize(0))
.fewerElementsIf(
[=](const LegalityQuery &Query) {
return notValidElt(Query, BigTyIdx);
},
scalarize(1))
.clampScalar(BigTyIdx, S32, MaxScalar);
if (Op == G_MERGE_VALUES) {
Builder.widenScalarIf(
// TODO: Use 16-bit shifts if legal for 8-bit values?
[=](const LegalityQuery &Query) {
const LLT Ty = Query.Types[LitTyIdx];
return Ty.getSizeInBits() < 32;
},
changeElementSizeTo(LitTyIdx, S32));
}
Builder.widenScalarIf(
[=](const LegalityQuery &Query) {
const LLT Ty = Query.Types[BigTyIdx];
return Ty.getSizeInBits() % 16 != 0;
},
[=](const LegalityQuery &Query) {
// Pick the next power of 2, or a multiple of 64 over 128.
// Whichever is smaller.
const LLT &Ty = Query.Types[BigTyIdx];
unsigned NewSizeInBits = 1 << Log2_32_Ceil(Ty.getSizeInBits() + 1);
if (NewSizeInBits >= 256) {
unsigned RoundedTo = alignTo<64>(Ty.getSizeInBits() + 1);
if (RoundedTo < NewSizeInBits)
NewSizeInBits = RoundedTo;
}
return std::pair(BigTyIdx, LLT::scalar(NewSizeInBits));
})
// Any vectors left are the wrong size. Scalarize them.
.scalarize(0)
.scalarize(1);
}
// S64 is only legal on SALU, and needs to be broken into 32-bit elements in
// RegBankSelect.
auto &SextInReg = getActionDefinitionsBuilder(G_SEXT_INREG)
.legalFor({{S32}, {S64}})
.clampScalar(0, S32, S64);
if (ST.hasVOP3PInsts()) {
SextInReg.lowerFor({{V2S16}})
// Prefer to reduce vector widths for 16-bit vectors before lowering, to
// get more vector shift opportunities, since we'll get those when
// expanded.
.clampMaxNumElementsStrict(0, S16, 2);
} else if (ST.has16BitInsts()) {
SextInReg.lowerFor({{S32}, {S64}, {S16}});
} else {
// Prefer to promote to s32 before lowering if we don't have 16-bit
// shifts. This avoid a lot of intermediate truncate and extend operations.
SextInReg.lowerFor({{S32}, {S64}});
}
SextInReg
.scalarize(0)
.clampScalar(0, S32, S64)
.lower();
getActionDefinitionsBuilder({G_ROTR, G_ROTL})
.scalarize(0)
.lower();
auto &FSHRActionDefs = getActionDefinitionsBuilder(G_FSHR);
FSHRActionDefs.legalFor({{S32, S32}})
.clampMaxNumElementsStrict(0, S16, 2);
if (ST.hasVOP3PInsts())
FSHRActionDefs.lowerFor({{V2S16, V2S16}});
FSHRActionDefs.scalarize(0).lower();
if (ST.hasVOP3PInsts()) {
getActionDefinitionsBuilder(G_FSHL)
.lowerFor({{V2S16, V2S16}})
.clampMaxNumElementsStrict(0, S16, 2)
.scalarize(0)
.lower();
} else {
getActionDefinitionsBuilder(G_FSHL)
.scalarize(0)
.lower();
}
getActionDefinitionsBuilder(G_READCYCLECOUNTER)
.legalFor({S64});
getActionDefinitionsBuilder(G_READSTEADYCOUNTER).legalFor({S64});
getActionDefinitionsBuilder(G_FENCE)
.alwaysLegal();
getActionDefinitionsBuilder({G_SMULO, G_UMULO})
.scalarize(0)
.minScalar(0, S32)
.lower();
getActionDefinitionsBuilder({G_SBFX, G_UBFX})
.legalFor({{S32, S32}, {S64, S32}})
.clampScalar(1, S32, S32)
.clampScalar(0, S32, S64)
.widenScalarToNextPow2(0)
.scalarize(0);
getActionDefinitionsBuilder(
{// TODO: Verify V_BFI_B32 is generated from expanded bit ops
G_FCOPYSIGN,
G_ATOMIC_CMPXCHG_WITH_SUCCESS, G_ATOMICRMW_NAND, G_ATOMICRMW_FSUB,
G_READ_REGISTER,
G_SADDO, G_SSUBO})
.lower();
if (ST.hasIEEEMinimumMaximumInsts()) {
getActionDefinitionsBuilder({G_FMINIMUM, G_FMAXIMUM})
.legalFor(FPTypesPK16)
.clampMaxNumElements(0, F16, 2)
.scalarize(0);
} else if (ST.hasVOP3PInsts()) {
getActionDefinitionsBuilder({G_FMINIMUM, G_FMAXIMUM})
.lowerFor({V2F16})
.clampMaxNumElementsStrict(0, F16, 2)
.scalarize(0)
.lower();
} else {
getActionDefinitionsBuilder({G_FMINIMUM, G_FMAXIMUM})
.scalarize(0)
.clampScalar(0, F32, F64)
.lower();
}
getActionDefinitionsBuilder(
{G_MEMCPY, G_MEMCPY_INLINE, G_MEMMOVE, G_MEMSET, G_MEMSET_INLINE})
.lower();
getActionDefinitionsBuilder({G_TRAP, G_DEBUGTRAP}).custom();
getActionDefinitionsBuilder({G_VASTART, G_VAARG, G_BRJT, G_JUMP_TABLE,
G_INDEXED_LOAD, G_INDEXED_SEXTLOAD,
G_INDEXED_ZEXTLOAD, G_INDEXED_STORE})
.unsupported();
getActionDefinitionsBuilder(G_PREFETCH).alwaysLegal();
getActionDefinitionsBuilder(
{G_VECREDUCE_SMIN, G_VECREDUCE_SMAX, G_VECREDUCE_UMIN, G_VECREDUCE_UMAX,
G_VECREDUCE_ADD, G_VECREDUCE_MUL, G_VECREDUCE_FADD, G_VECREDUCE_FMUL,
G_VECREDUCE_FMIN, G_VECREDUCE_FMAX, G_VECREDUCE_FMINIMUM,
G_VECREDUCE_FMAXIMUM, G_VECREDUCE_OR, G_VECREDUCE_AND, G_VECREDUCE_XOR})
.legalFor(AllVectors)
.scalarize(1)
.lower();
getActionDefinitionsBuilder({G_INTRINSIC, G_INTRINSIC_W_SIDE_EFFECTS,
G_INTRINSIC_CONVERGENT,
G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS})
.alwaysLegal();
verify(*ST.getInstrInfo());
}
bool AMDGPULegalizerInfo::legalizeCustom(
LegalizerHelper &Helper, MachineInstr &MI,
LostDebugLocObserver &LocObserver) const {
MachineIRBuilder &B = Helper.MIRBuilder;
MachineRegisterInfo &MRI = *B.getMRI();
switch (MI.getOpcode()) {
case TargetOpcode::G_ADDRSPACE_CAST:
return legalizeAddrSpaceCast(MI, MRI, B);
case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
return legalizeFroundeven(MI, MRI, B);
case TargetOpcode::G_FCEIL:
return legalizeFceil(MI, MRI, B);
case TargetOpcode::G_FREM:
return legalizeFrem(MI, MRI, B);
case TargetOpcode::G_INTRINSIC_TRUNC:
return legalizeIntrinsicTrunc(MI, MRI, B);
case TargetOpcode::G_SITOFP:
return legalizeITOFP(MI, MRI, B, true);
case TargetOpcode::G_UITOFP:
return legalizeITOFP(MI, MRI, B, false);
case TargetOpcode::G_FPTOSI:
return legalizeFPTOI(MI, MRI, B, true);
case TargetOpcode::G_FPTOUI:
return legalizeFPTOI(MI, MRI, B, false);
case TargetOpcode::G_FMINNUM:
case TargetOpcode::G_FMAXNUM:
case TargetOpcode::G_FMINIMUMNUM:
case TargetOpcode::G_FMAXIMUMNUM:
return legalizeMinNumMaxNum(Helper, MI);
case TargetOpcode::G_EXTRACT:
return legalizeExtract(Helper, MI);
case TargetOpcode::G_INSERT:
return legalizeInsert(Helper, MI);
case TargetOpcode::G_EXTRACT_VECTOR_ELT:
return legalizeExtractVectorElt(MI, MRI, B);
case TargetOpcode::G_INSERT_VECTOR_ELT:
return legalizeInsertVectorElt(MI, MRI, B);
case TargetOpcode::G_FSIN:
case TargetOpcode::G_FCOS:
return legalizeSinCos(MI, MRI, B);
case TargetOpcode::G_GLOBAL_VALUE:
return legalizeGlobalValue(MI, MRI, B);
case TargetOpcode::G_LOAD:
case TargetOpcode::G_SEXTLOAD:
case TargetOpcode::G_ZEXTLOAD:
return legalizeLoad(Helper, MI);
case TargetOpcode::G_STORE:
return legalizeStore(Helper, MI);
case TargetOpcode::G_FMAD:
return legalizeFMad(MI, MRI, B);
case TargetOpcode::G_FDIV:
return legalizeFDIV(MI, MRI, B);
case TargetOpcode::G_FFREXP:
return legalizeFFREXP(MI, MRI, B);
case TargetOpcode::G_FSQRT:
return legalizeFSQRT(MI, MRI, B);
case TargetOpcode::G_UDIV:
case TargetOpcode::G_UREM:
case TargetOpcode::G_UDIVREM:
return legalizeUnsignedDIV_REM(MI, MRI, B);
case TargetOpcode::G_SDIV:
case TargetOpcode::G_SREM:
case TargetOpcode::G_SDIVREM:
return legalizeSignedDIV_REM(MI, MRI, B);
case TargetOpcode::G_ATOMIC_CMPXCHG:
return legalizeAtomicCmpXChg(MI, MRI, B);
case TargetOpcode::G_FLOG2:
return legalizeFlog2(MI, B);
case TargetOpcode::G_FLOG:
case TargetOpcode::G_FLOG10:
return legalizeFlogCommon(MI, B);
case TargetOpcode::G_FEXP2:
return legalizeFExp2(MI, B);
case TargetOpcode::G_FEXP:
case TargetOpcode::G_FEXP10:
return legalizeFExp(MI, B);
case TargetOpcode::G_FPOW:
return legalizeFPow(Helper, MI);
case TargetOpcode::G_FFLOOR:
return legalizeFFloor(MI, MRI, B);
case TargetOpcode::G_BUILD_VECTOR:
case TargetOpcode::G_BUILD_VECTOR_TRUNC:
return legalizeBuildVector(MI, MRI, B);
case TargetOpcode::G_MUL:
return legalizeMul(Helper, MI);
case TargetOpcode::G_CTLZ:
case TargetOpcode::G_CTTZ:
return legalizeCTLZ_CTTZ(MI, MRI, B);
case TargetOpcode::G_CTLS:
return legalizeCTLS(MI, MRI, B);
case TargetOpcode::G_CTLZ_ZERO_POISON:
return legalizeCTLZ_ZERO_POISON(MI, MRI, B);
case TargetOpcode::G_STACKSAVE:
return legalizeStackSave(MI, B);
case TargetOpcode::G_GET_FPENV:
return legalizeGetFPEnv(MI, MRI, B);
case TargetOpcode::G_SET_FPENV:
return legalizeSetFPEnv(MI, MRI, B);
case TargetOpcode::G_TRAP:
return legalizeTrap(Helper, MI);
case TargetOpcode::G_DEBUGTRAP:
return legalizeDebugTrap(MI, MRI, B);
default:
return false;
}
llvm_unreachable("expected switch to return");
}
Register AMDGPULegalizerInfo::getSegmentAperture(unsigned AS,
MachineRegisterInfo &MRI,
MachineIRBuilder &B) const {
unsigned BaseAS = AS;
unsigned SANum = AMDGPU::getSyntheticApertureNumber(AS);
if (SANum != AMDGPU::SyntheticAperture::None)
BaseAS = AMDGPUAS::LOCAL_ADDRESS;
Register Aperture = getBaseSegmentAperture(BaseAS, MRI, B);
if (SANum != AMDGPU::SyntheticAperture::None) {
const LLT S32 = LLT::scalar(32);
auto Tag = B.buildConstant(S32, SANum);
return B.buildOr(S32, Aperture, Tag).getReg(0);
}
return Aperture;
}
Register AMDGPULegalizerInfo::getBaseSegmentAperture(
unsigned AS, MachineRegisterInfo &MRI, MachineIRBuilder &B) const {
MachineFunction &MF = B.getMF();
const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
const LLT I32 = LLT::integer(32);
const LLT I64 = LLT::integer(64);
bool IsLDS = (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::BARRIER);
assert(IsLDS || AS == AMDGPUAS::PRIVATE_ADDRESS);
if (ST.hasApertureRegs()) {
// Note: this register is somewhat broken. When used as a 32-bit operand,
// it only returns zeroes. The real value is in the upper 32 bits.
// Thus, we must emit extract the high 32 bits.
const unsigned ApertureRegNo =
IsLDS ? AMDGPU::SRC_SHARED_BASE : AMDGPU::SRC_PRIVATE_BASE;
assert((ApertureRegNo != AMDGPU::SRC_PRIVATE_BASE ||
!ST.hasGloballyAddressableScratch()) &&
"Cannot use src_private_base with globally addressable scratch!");
Register Dst = MRI.createGenericVirtualRegister(I64);
MRI.setRegClass(Dst, &AMDGPU::SReg_64RegClass);
B.buildCopy({Dst}, {Register(ApertureRegNo)});
return B.buildUnmerge(I32, Dst).getReg(1);
}
Register LoadAddr = MRI.createGenericVirtualRegister(
LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64));
// For code object version 5, private_base and shared_base are passed through
// implicit kernargs.
if (AMDGPU::getAMDHSACodeObjectVersion(*MF.getFunction().getParent()) >=
AMDGPU::AMDHSA_COV5) {
MachinePointerInfo PtrInfo = getKernargSegmentPtrInfo(B.getMF());
AMDGPUTargetLowering::ImplicitParameter Param =
AS == AMDGPUAS::LOCAL_ADDRESS ? AMDGPUTargetLowering::SHARED_BASE
: AMDGPUTargetLowering::PRIVATE_BASE;
uint64_t Offset =
ST.getTargetLowering()->getImplicitParameterOffset(B.getMF(), Param);
Register KernargPtrReg = MRI.createGenericVirtualRegister(
LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64));
if (!loadInputValue(KernargPtrReg, B,
AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR))
return Register();
MachineMemOperand *MMO = MF.getMachineMemOperand(
PtrInfo.getWithOffset(Offset),
MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
MachineMemOperand::MOInvariant,
LLT::integer(32), commonAlignment(Align(64), Offset));
// Pointer address
B.buildObjectPtrOffset(LoadAddr, KernargPtrReg,
B.buildConstant(LLT::integer(64), Offset).getReg(0));
// Load address
return B.buildLoad(I32, LoadAddr, *MMO).getReg(0);
}
Register QueuePtr = MRI.createGenericVirtualRegister(
LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64));
if (!loadInputValue(QueuePtr, B, AMDGPUFunctionArgInfo::QUEUE_PTR))
return Register();
// TODO: Use custom PseudoSourceValue
MachinePointerInfo PtrInfo(AMDGPUAS::CONSTANT_ADDRESS);
// Offset into amd_queue_t for group_segment_aperture_base_hi /
// private_segment_aperture_base_hi.
uint32_t StructOffset = IsLDS ? 0x40 : 0x44;
MachineMemOperand *MMO = MF.getMachineMemOperand(
PtrInfo,
MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
MachineMemOperand::MOInvariant,
LLT::integer(32), commonAlignment(Align(64), StructOffset));
B.buildObjectPtrOffset(
LoadAddr, QueuePtr,
B.buildConstant(LLT::integer(64), StructOffset).getReg(0));
return B.buildLoad(I32, LoadAddr, *MMO).getReg(0);
}
/// Return true if the value is a known valid address, such that a null check is
/// not necessary.
static bool isKnownNonNull(Register Val, MachineRegisterInfo &MRI,
const AMDGPUTargetMachine &TM, unsigned AddrSpace) {
MachineInstr *Def = MRI.getVRegDef(Val);
switch (Def->getOpcode()) {
case AMDGPU::G_FRAME_INDEX:
case AMDGPU::G_GLOBAL_VALUE:
case AMDGPU::G_BLOCK_ADDR:
return true;
case AMDGPU::G_CONSTANT: {
const ConstantInt *CI = Def->getOperand(1).getCImm();
return CI->getSExtValue() != AMDGPU::getNullPointerValue(AddrSpace);
}
default:
return false;
}
return false;
}
bool AMDGPULegalizerInfo::legalizeAddrSpaceCast(
MachineInstr &MI, MachineRegisterInfo &MRI,
MachineIRBuilder &B) const {
MachineFunction &MF = B.getMF();
assert(MI.getOpcode() == TargetOpcode::G_ADDRSPACE_CAST);
const LLT I32 = LLT::integer(32);
const LLT I64 = LLT::integer(64);
Register Dst = MI.getOperand(0).getReg();
Register Src = MI.getOperand(1).getReg();
LLT DstTy = MRI.getType(Dst);
LLT SrcTy = MRI.getType(Src);
unsigned DestAS = DstTy.getAddressSpace();
unsigned SrcAS = SrcTy.getAddressSpace();
// TODO: Avoid reloading from the queue ptr for each cast, or at least each
// vector element.
assert(!DstTy.isVector());
const AMDGPUTargetMachine &TM
= static_cast<const AMDGPUTargetMachine &>(MF.getTarget());
// The source is known non-null for a G_ADDRSPACE_CAST carrying the nonnull
// flag; otherwise we need to guess.
const bool IsNonNull = MI.getFlag(MachineInstr::MIFlag::NonNull);
if (TM.isNoopAddrSpaceCast(MF.getDataLayout(), SrcAS, DestAS)) {
MI.setDesc(B.getTII().get(TargetOpcode::G_BITCAST));
return true;
}
if (SrcAS == AMDGPUAS::FLAT_ADDRESS &&
(DestAS == AMDGPUAS::LOCAL_ADDRESS || DestAS == AMDGPUAS::BARRIER ||
DestAS == AMDGPUAS::PRIVATE_ADDRESS)) {
auto castFlatToLocalOrPrivate = [&](const DstOp &Dst) -> Register {
if (DestAS == AMDGPUAS::PRIVATE_ADDRESS &&
ST.hasGloballyAddressableScratch()) {
// flat -> private with globally addressable scratch: subtract
// src_flat_scratch_base_lo.
Register SrcLo = B.buildExtract(I32, Src, 0).getReg(0);
Register FlatScratchBaseLo =
B.buildInstr(AMDGPU::S_MOV_B32, {I32},
{Register(AMDGPU::SRC_FLAT_SCRATCH_BASE_LO)})
.getReg(0);
MRI.setRegClass(FlatScratchBaseLo, &AMDGPU::SReg_32RegClass);
Register Sub = B.buildSub(I32, SrcLo, FlatScratchBaseLo).getReg(0);
return B.buildIntToPtr(Dst, Sub).getReg(0);
}
return B.buildExtract(Dst, Src, 0).getReg(0);
};
if (IsNonNull || isKnownNonNull(Src, MRI, TM, SrcAS)) {
castFlatToLocalOrPrivate(Dst);
MI.eraseFromParent();
return true;
}
unsigned NullVal = AMDGPU::getNullPointerValue(DestAS);
auto SegmentNull = B.buildConstant(DstTy, NullVal);
auto FlatNull = B.buildConstant(SrcTy, 0);
// Extract low 32-bits of the pointer.
auto PtrLo32 = castFlatToLocalOrPrivate(DstTy);
auto CmpRes =
B.buildICmp(CmpInst::ICMP_NE, LLT::scalar(1), Src, FlatNull.getReg(0));
B.buildSelect(Dst, CmpRes, PtrLo32, SegmentNull.getReg(0));
MI.eraseFromParent();
return true;
}
if (DestAS == AMDGPUAS::FLAT_ADDRESS &&
(SrcAS == AMDGPUAS::LOCAL_ADDRESS || SrcAS == AMDGPUAS::BARRIER ||
SrcAS == AMDGPUAS::PRIVATE_ADDRESS)) {
auto castLocalOrPrivateToFlat = [&](const DstOp &Dst) -> Register {
// Coerce the type of the low half of the result so we can use
// merge_values.
Register SrcAsInt = B.buildPtrToInt(I32, Src).getReg(0);
if (SrcAS == AMDGPUAS::PRIVATE_ADDRESS &&
ST.hasGloballyAddressableScratch()) {
// For wave32: Addr = (TID[4:0] << 52) + FLAT_SCRATCH_BASE + privateAddr
// For wave64: Addr = (TID[5:0] << 51) + FLAT_SCRATCH_BASE + privateAddr
Register AllOnes = B.buildConstant(I32, -1).getReg(0);
Register ThreadID = B.buildConstant(I32, 0).getReg(0);
ThreadID = B.buildIntrinsic(Intrinsic::amdgcn_mbcnt_lo, {I32})
.addUse(AllOnes)
.addUse(ThreadID)
.getReg(0);
if (ST.isWave64()) {
ThreadID = B.buildIntrinsic(Intrinsic::amdgcn_mbcnt_hi, {I32})
.addUse(AllOnes)
.addUse(ThreadID)
.getReg(0);
}
Register ShAmt =
B.buildConstant(I32, 57 - 32 - ST.getWavefrontSizeLog2()).getReg(0);
Register SrcHi = B.buildShl(I32, ThreadID, ShAmt).getReg(0);
Register CvtPtr =
B.buildMergeLikeInstr(DstTy, {SrcAsInt, SrcHi}).getReg(0);
// Accessing src_flat_scratch_base_lo as a 64-bit operand gives the full
// 64-bit hi:lo value.
Register FlatScratchBase =
B.buildInstr(AMDGPU::S_MOV_B64, {I64},
{Register(AMDGPU::SRC_FLAT_SCRATCH_BASE)})
.getReg(0);
MRI.setRegClass(FlatScratchBase, &AMDGPU::SReg_64RegClass);
return B.buildPtrAdd(Dst, CvtPtr, FlatScratchBase).getReg(0);
}
Register ApertureReg = getSegmentAperture(SrcAS, MRI, B);
if (!ApertureReg.isValid())
return false;
// TODO: Should we allow mismatched types but matching sizes in merges to
// avoid the ptrtoint?
return B.buildMergeLikeInstr(Dst, {SrcAsInt, ApertureReg}).getReg(0);
};
if (IsNonNull || isKnownNonNull(Src, MRI, TM, SrcAS)) {
castLocalOrPrivateToFlat(Dst);
MI.eraseFromParent();
return true;
}
Register BuildPtr = castLocalOrPrivateToFlat(DstTy);
auto SegmentNull =
B.buildConstant(SrcTy, AMDGPU::getNullPointerValue(SrcAS));
auto FlatNull = B.buildConstant(DstTy, AMDGPU::getNullPointerValue(DestAS));
auto CmpRes = B.buildICmp(CmpInst::ICMP_NE, LLT::scalar(1), Src,
SegmentNull.getReg(0));
B.buildSelect(Dst, CmpRes, BuildPtr, FlatNull);
MI.eraseFromParent();
return true;
}
if (DestAS == AMDGPUAS::CONSTANT_ADDRESS_32BIT &&
SrcTy.getSizeInBits() == 64) {
// Truncate.
B.buildExtract(Dst, Src, 0);
MI.eraseFromParent();
return true;
}
if (SrcAS == AMDGPUAS::CONSTANT_ADDRESS_32BIT &&
DstTy.getSizeInBits() == 64) {
const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
uint32_t AddrHiVal = Info->get32BitAddressHighBits();
auto PtrLo = B.buildPtrToInt(I32, Src);
if (AddrHiVal == 0) {
auto Zext = B.buildZExt(I64, PtrLo);
B.buildIntToPtr(Dst, Zext);
} else {
auto HighAddr = B.buildConstant(I32, AddrHiVal);
B.buildMergeLikeInstr(Dst, {PtrLo, HighAddr});
}
MI.eraseFromParent();
return true;
}
// Invalid casts are poison.
// TODO: Should return poison
B.buildUndef(Dst);
MI.eraseFromParent();
return true;
}
bool AMDGPULegalizerInfo::legalizeFroundeven(MachineInstr &MI,
MachineRegisterInfo &MRI,
MachineIRBuilder &B) const {
Register Src = MI.getOperand(1).getReg();
LLT Ty = MRI.getType(Src);
assert(Ty.isScalar() && Ty.getSizeInBits() == 64);
APFloat C1Val(APFloat::IEEEdouble(), "0x1.0p+52");
APFloat C2Val(APFloat::IEEEdouble(), "0x1.fffffffffffffp+51");
auto C1 = B.buildFConstant(Ty, C1Val);
auto CopySign = B.buildFCopysign(Ty, C1, Src);
// TODO: Should this propagate fast-math-flags?
auto Tmp1 = B.buildFAdd(Ty, Src, CopySign);
auto Tmp2 = B.buildFSub(Ty, Tmp1, CopySign);
auto C2 = B.buildFConstant(Ty, C2Val);
auto Fabs = B.buildFAbs(Ty, Src);
auto Cond = B.buildFCmp(CmpInst::FCMP_OGT, LLT::scalar(1), Fabs, C2);
B.buildSelect(MI.getOperand(0).getReg(), Cond, Src, Tmp2);
MI.eraseFromParent();
return true;
}
bool AMDGPULegalizerInfo::legalizeFceil(
MachineInstr &MI, MachineRegisterInfo &MRI,
MachineIRBuilder &B) const {
const LLT S1 = LLT::scalar(1);
Register Src = MI.getOperand(1).getReg();
assert(MRI.getType(Src) == F64);
// result = trunc(src)
// if (src > 0.0 && src != result)
// result += 1.0
auto Trunc = B.buildIntrinsicTrunc(F64, Src);
const auto Zero = B.buildFConstant(F64, 0.0);
const auto One = B.buildFConstant(F64, 1.0);
auto Lt0 = B.buildFCmp(CmpInst::FCMP_OGT, S1, Src, Zero);
auto NeTrunc = B.buildFCmp(CmpInst::FCMP_ONE, S1, Src, Trunc);
auto And = B.buildAnd(S1, Lt0, NeTrunc);
auto Add = B.buildSelect(F64, And, One, Zero);
// TODO: Should this propagate fast-math-flags?
B.buildFAdd(MI.getOperand(0).getReg(), Trunc, Add);
MI.eraseFromParent();
return true;
}
bool AMDGPULegalizerInfo::legalizeFrem(
MachineInstr &MI, MachineRegisterInfo &MRI,
MachineIRBuilder &B) const {
Register DstReg = MI.getOperand(0).getReg();
Register Src0Reg = MI.getOperand(1).getReg();
Register Src1Reg = MI.getOperand(2).getReg();
auto Flags = MI.getFlags();
LLT Ty = MRI.getType(DstReg);
auto Div = B.buildFDiv(Ty, Src0Reg, Src1Reg, Flags);
auto Trunc = B.buildIntrinsicTrunc(Ty, Div, Flags);
auto Neg = B.buildFNeg(Ty, Trunc, Flags);
B.buildFMA(DstReg, Neg, Src1Reg, Src0Reg, Flags);
MI.eraseFromParent();
return true;
}
static MachineInstrBuilder extractF64Exponent(Register Hi,
MachineIRBuilder &B) {
const unsigned FractBits = 52;
const unsigned ExpBits = 11;
LLT I32 = LLT::integer(32);
auto Const0 = B.buildConstant(I32, FractBits - 32);
auto Const1 = B.buildConstant(I32, ExpBits);
auto ExpPart = B.buildIntrinsic(Intrinsic::amdgcn_ubfe, {I32})
.addUse(Hi)
.addUse(Const0.getReg(0))
.addUse(Const1.getReg(0));
return B.buildSub(I32, ExpPart, B.buildConstant(I32, 1023));
}
bool AMDGPULegalizerInfo::legalizeIntrinsicTrunc(
MachineInstr &MI, MachineRegisterInfo &MRI,
MachineIRBuilder &B) const {
const LLT S1 = LLT::scalar(1);
const LLT I32 = LLT::integer(32);
const LLT I64 = LLT::integer(64);
Register Src = MI.getOperand(1).getReg();
assert(MRI.getType(Src) == F64);
auto SrcInt = B.buildBitcast(I64, Src);
// TODO: Should this use extract since the low half is unused?
auto Unmerge = B.buildUnmerge({I32, I32}, SrcInt);
Register Hi = Unmerge.getReg(1);
// Extract the upper half, since this is where we will find the sign and
// exponent.
auto Exp = extractF64Exponent(Hi, B);
const unsigned FractBits = 52;
// Extract the sign bit.
const auto SignBitMask = B.buildConstant(I32, UINT32_C(1) << 31);
auto SignBit = B.buildAnd(I32, Hi, SignBitMask);
const auto FractMask = B.buildConstant(I64, (UINT64_C(1) << FractBits) - 1);
const auto Zero32 = B.buildConstant(I32, 0);
// Extend back to 64-bits.
auto SignBit64 = B.buildMergeLikeInstr(I64, {Zero32, SignBit});
auto Shr = B.buildAShr(I64, FractMask, Exp);
auto Not = B.buildNot(I64, Shr);
auto Tmp0 = B.buildAnd(I64, SrcInt, Not);
auto FiftyOne = B.buildConstant(I32, FractBits - 1);
auto ExpLt0 = B.buildICmp(CmpInst::ICMP_SLT, S1, Exp, Zero32);
auto ExpGt51 = B.buildICmp(CmpInst::ICMP_SGT, S1, Exp, FiftyOne);
auto Tmp1 = B.buildSelect(I64, ExpLt0, SignBit64, Tmp0);
auto Res = B.buildSelect(I64, ExpGt51, SrcInt, Tmp1);
B.buildBitcast(MI.getOperand(0).getReg(), Res);
MI.eraseFromParent();
return true;
}
bool AMDGPULegalizerInfo::legalizeITOFP(
MachineInstr &MI, MachineRegisterInfo &MRI,
MachineIRBuilder &B, bool Signed) const {
Register Dst = MI.getOperand(0).getReg();
Register Src = MI.getOperand(1).getReg();
const LLT I64 = LLT::integer(64);
const LLT I32 = LLT::integer(32);
assert(MRI.getType(Src) == I64);
auto Unmerge = B.buildUnmerge({I32, I32}, Src);
auto ThirtyTwo = B.buildConstant(I32, 32);
if (MRI.getType(Dst) == F64) {
auto CvtHi = Signed ? B.buildSITOFP(F64, Unmerge.getReg(1))
: B.buildUITOFP(F64, Unmerge.getReg(1));
auto CvtLo = B.buildUITOFP(F64, Unmerge.getReg(0));
auto LdExp = B.buildFLdexp(F64, CvtHi, ThirtyTwo);
// TODO: Should this propagate fast-math-flags?
B.buildFAdd(Dst, LdExp, CvtLo);
MI.eraseFromParent();
return true;
}
assert(MRI.getType(Dst) == F32);
auto One = B.buildConstant(I32, 1);
MachineInstrBuilder ShAmt;
if (Signed) {
auto ThirtyOne = B.buildConstant(I32, 31);
auto X = B.buildXor(I32, Unmerge.getReg(0), Unmerge.getReg(1));
auto OppositeSign = B.buildAShr(I32, X, ThirtyOne);
auto MaxShAmt = B.buildAdd(I32, ThirtyTwo, OppositeSign);
auto LS = B.buildIntrinsic(Intrinsic::amdgcn_sffbh, {I32})
.addUse(Unmerge.getReg(1));
auto LS2 = B.buildSub(I32, LS, One);
ShAmt = B.buildUMin(I32, LS2, MaxShAmt);
} else
ShAmt = B.buildCTLZ(I32, Unmerge.getReg(1));
auto Norm = B.buildShl(I64, Src, ShAmt);
auto Unmerge2 = B.buildUnmerge({I32, I32}, Norm);
auto Adjust = B.buildUMin(I32, One, Unmerge2.getReg(0));
auto Norm2 = B.buildOr(I32, Unmerge2.getReg(1), Adjust);
auto FVal = Signed ? B.buildSITOFP(F32, Norm2) : B.buildUITOFP(F32, Norm2);
auto Scale = B.buildSub(I32, ThirtyTwo, ShAmt);
B.buildFLdexp(Dst, FVal, Scale);
MI.eraseFromParent();
return true;
}
// TODO: Copied from DAG implementation. Verify logic and document how this
// actually works.
bool AMDGPULegalizerInfo::legalizeFPTOI(MachineInstr &MI,
MachineRegisterInfo &MRI,
MachineIRBuilder &B,
bool Signed) const {
Register Dst = MI.getOperand(0).getReg();
Register Src = MI.getOperand(1).getReg();
const LLT I64 = LLT::integer(64);
const LLT I32 = LLT::integer(32);
const LLT SrcLT = MRI.getType(Src);
assert((SrcLT == F32 || SrcLT == F64) && MRI.getType(Dst) == I64);
unsigned Flags = MI.getFlags();
// The basic idea of converting a floating point number into a pair of 32-bit
// integers is illustrated as follows:
//
// tf := trunc(val);
// hif := floor(tf * 2^-32);
// lof := tf - hif * 2^32; // lof is always positive due to floor.
// hi := fptoi(hif);
// lo := fptoi(lof);
//
auto Trunc = B.buildIntrinsicTrunc(SrcLT, Src, Flags);
MachineInstrBuilder Sign;
if (Signed && SrcLT == F32) {
// However, a 32-bit floating point number has only 23 bits mantissa and
// it's not enough to hold all the significant bits of `lof` if val is
// negative. To avoid the loss of precision, We need to take the absolute
// value after truncating and flip the result back based on the original
// signedness.
auto SrcInt = B.buildBitcast(I32, Src);
Sign = B.buildAShr(I32, SrcInt, B.buildConstant(I32, 31));
Trunc = B.buildFAbs(F32, Trunc, Flags);
}
MachineInstrBuilder K0, K1;
if (SrcLT == F64) {
K0 = B.buildFConstant(
F64, llvm::bit_cast<double>(UINT64_C(/*2^-32*/ 0x3df0000000000000)));
K1 = B.buildFConstant(
F64, llvm::bit_cast<double>(UINT64_C(/*-2^32*/ 0xc1f0000000000000)));
} else {
K0 = B.buildFConstant(
F32, llvm::bit_cast<float>(UINT32_C(/*2^-32*/ 0x2f800000)));
K1 = B.buildFConstant(
F32, llvm::bit_cast<float>(UINT32_C(/*-2^32*/ 0xcf800000)));
}
auto Mul = B.buildFMul(SrcLT, Trunc, K0, Flags);
auto FloorMul = B.buildFFloor(SrcLT, Mul, Flags);
auto Fma = B.buildFMA(SrcLT, FloorMul, K1, Trunc, Flags);
auto Hi = (Signed && SrcLT == F64) ? B.buildFPTOSI(I32, FloorMul)
: B.buildFPTOUI(I32, FloorMul);
auto Lo = B.buildFPTOUI(I32, Fma);
if (Signed && SrcLT == F32) {
// Flip the result based on the signedness, which is either all 0s or 1s.
Sign = B.buildMergeLikeInstr(I64, {Sign, Sign});
// r := xor({lo, hi}, sign) - sign;
B.buildSub(Dst, B.buildXor(I64, B.buildMergeLikeInstr(I64, {Lo, Hi}), Sign),
Sign);
} else
B.buildMergeLikeInstr(Dst, {Lo, Hi});
MI.eraseFromParent();
return true;
}
bool AMDGPULegalizerInfo::legalizeMinNumMaxNum(LegalizerHelper &Helper,
MachineInstr &MI) const {
MachineFunction &MF = Helper.MIRBuilder.getMF();
const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
// With ieee_mode disabled, the instructions have the correct behavior.
if (!MFI->getMode().IEEE)
return true;
return Helper.lowerFMinNumMaxNum(MI) == LegalizerHelper::Legalized;
}
bool AMDGPULegalizerInfo::legalizeExtract(LegalizerHelper &Helper,
MachineInstr &MI) const {
MachineIRBuilder &B = Helper.MIRBuilder;
MachineRegisterInfo &MRI = *B.getMRI();
Register DstReg = MI.getOperand(0).getReg();
Register SrcReg = MI.getOperand(1).getReg();
uint64_t Offset = MI.getOperand(2).getImm();
// Fall back to generic lowering for offset 0 (trivial trunc) and
// non-32-bit-aligned cases which require shift+trunc sequences
// that generic code handles correctly.
if (Offset == 0 || Offset % 32 != 0)
return Helper.lowerExtract(MI) == LegalizerHelper::Legalized;
const LLT DstTy = MRI.getType(DstReg);
unsigned StartIdx = Offset / 32;
unsigned DstCount = DstTy.getSizeInBits() / 32;
auto Unmerge = B.buildUnmerge(LLT::integer(32), SrcReg);
if (DstCount == 1) {
if (DstTy.isPointer())
B.buildIntToPtr(DstReg, Unmerge.getReg(StartIdx));
else {
Helper.Observer.changingAllUsesOfReg(MRI, DstReg);
MRI.replaceRegWith(DstReg, Unmerge.getReg(StartIdx));
Helper.Observer.finishedChangingAllUsesOfReg();
}
} else {
SmallVector<Register, 8> MergeVec;
for (unsigned I = 0; I < DstCount; ++I)
MergeVec.push_back(Unmerge.getReg(StartIdx + I));
B.buildMergeLikeInstr(DstReg, MergeVec);
}
MI.eraseFromParent();
return true;
}
bool AMDGPULegalizerInfo::legalizeInsert(LegalizerHelper &Helper,
MachineInstr &MI) const {
MachineIRBuilder &B = Helper.MIRBuilder;
MachineRegisterInfo &MRI = *B.getMRI();
Register DstReg = MI.getOperand(0).getReg();
Register SrcReg = MI.getOperand(1).getReg();
Register InsertSrc = MI.getOperand(2).getReg();
uint64_t Offset = MI.getOperand(3).getImm();
unsigned DstSize = MRI.getType(DstReg).getSizeInBits();
const LLT InsertTy = MRI.getType(InsertSrc);
unsigned InsertSize = InsertTy.getSizeInBits();
// Fall back to generic lowering for non-32-bit-aligned cases which
// require shift+mask sequences that generic code handles correctly.
if (Offset % 32 != 0 || DstSize % 32 != 0 || InsertSize % 32 != 0)
return Helper.lowerInsert(MI) == LegalizerHelper::Legalized;
const LLT I32 = LLT::integer(32);
unsigned DstCount = DstSize / 32;
unsigned InsertCount = InsertSize / 32;
unsigned StartIdx = Offset / 32;
auto SrcUnmerge = B.buildUnmerge(I32, SrcReg);
SmallVector<Register, 8> MergeVec;
for (unsigned I = 0; I < StartIdx; ++I)
MergeVec.push_back(SrcUnmerge.getReg(I));
if (InsertCount == 1) {
// Merge-like instructions require same source types. Convert pointer
// to scalar when inserting a pointer value into a scalar.
if (InsertTy.isPointer())
InsertSrc = B.buildPtrToInt(I32, InsertSrc).getReg(0);
MergeVec.push_back(InsertSrc);
} else {
auto InsertUnmerge = B.buildUnmerge(I32, InsertSrc);
for (unsigned I = 0; I < InsertCount; ++I)
MergeVec.push_back(InsertUnmerge.getReg(I));
}
for (unsigned I = StartIdx + InsertCount; I < DstCount; ++I)
MergeVec.push_back(SrcUnmerge.getReg(I));
B.buildMergeLikeInstr(DstReg, MergeVec);
MI.eraseFromParent();
return true;
}
bool AMDGPULegalizerInfo::legalizeExtractVectorElt(
MachineInstr &MI, MachineRegisterInfo &MRI,
MachineIRBuilder &B) const {
// TODO: Should move some of this into LegalizerHelper.
// TODO: Promote dynamic indexing of i16/f16 to i32/f32
Register Dst = MI.getOperand(0).getReg();
Register Vec = MI.getOperand(1).getReg();
LLT VecTy = MRI.getType(Vec);
LLT EltTy = VecTy.getElementType();
assert(EltTy == MRI.getType(Dst));
// Other legalization maps vector<? x [type bigger than 64 bits]> via bitcasts
// but we can't go directly to that logic becasue you can't bitcast a vector
// of pointers to a vector of integers. Therefore, introduce an intermediate
// vector of integers using ptrtoint (and inttoptr on the output) in order to
// drive the legalization forward.
if (EltTy.isPointer() && EltTy.getSizeInBits() > 64) {
LLT IntTy = LLT::integer(EltTy.getSizeInBits());
LLT IntVecTy = VecTy.changeElementType(IntTy);
auto IntVec = B.buildPtrToInt(IntVecTy, Vec);
auto IntElt = B.buildExtractVectorElement(IntTy, IntVec, MI.getOperand(2));
B.buildIntToPtr(Dst, IntElt);
MI.eraseFromParent();
return true;
}
// FIXME: Artifact combiner probably should have replaced the truncated
// constant before this, so we shouldn't need
// getIConstantVRegValWithLookThrough.
std::optional<ValueAndVReg> MaybeIdxVal =
getIConstantVRegValWithLookThrough(MI.getOperand(2).getReg(), MRI);
if (!MaybeIdxVal) // Dynamic case will be selected to register indexing.
return true;
const uint64_t IdxVal = MaybeIdxVal->Value.getZExtValue();
if (IdxVal < VecTy.getNumElements()) {
auto Unmerge = B.buildUnmerge(EltTy, Vec);
B.buildCopy(Dst, Unmerge.getReg(IdxVal));
} else {
B.buildUndef(Dst);
}
MI.eraseFromParent();
return true;
}
bool AMDGPULegalizerInfo::legalizeInsertVectorElt(
MachineInstr &MI, MachineRegisterInfo &MRI,
MachineIRBuilder &B) const {
// TODO: Should move some of this into LegalizerHelper.
// TODO: Promote dynamic indexing of i16/f16 to i32/f32
Register Dst = MI.getOperand(0).getReg();
Register Vec = MI.getOperand(1).getReg();
Register Ins = MI.getOperand(2).getReg();
LLT VecTy = MRI.getType(Vec);
LLT EltTy = VecTy.getElementType();
assert(EltTy == MRI.getType(Ins));
// Other legalization maps vector<? x [type bigger than 64 bits]> via bitcasts
// but we can't go directly to that logic becasue you can't bitcast a vector
// of pointers to a vector of integers. Therefore, make the pointer vector
// into an equivalent vector of integers with ptrtoint, insert the ptrtoint'd
// new value, and then inttoptr the result vector back. This will then allow
// the rest of legalization to take over.
if (EltTy.isPointer() && EltTy.getSizeInBits() > 64) {
LLT IntTy = LLT::integer(EltTy.getSizeInBits());
LLT IntVecTy = VecTy.changeElementType(IntTy);
auto IntVecSource = B.buildPtrToInt(IntVecTy, Vec);
auto IntIns = B.buildPtrToInt(IntTy, Ins);
auto IntVecDest = B.buildInsertVectorElement(IntVecTy, IntVecSource, IntIns,
MI.getOperand(3));
B.buildIntToPtr(Dst, IntVecDest);
MI.eraseFromParent();
return true;
}
// FIXME: Artifact combiner probably should have replaced the truncated
// constant before this, so we shouldn't need
// getIConstantVRegValWithLookThrough.
std::optional<ValueAndVReg> MaybeIdxVal =
getIConstantVRegValWithLookThrough(MI.getOperand(3).getReg(), MRI);
if (!MaybeIdxVal) // Dynamic case will be selected to register indexing.
return true;
const uint64_t IdxVal = MaybeIdxVal->Value.getZExtValue();
unsigned NumElts = VecTy.getNumElements();
if (IdxVal < NumElts) {
SmallVector<Register, 8> SrcRegs;
for (unsigned i = 0; i < NumElts; ++i)
SrcRegs.push_back(MRI.createGenericVirtualRegister(EltTy));
B.buildUnmerge(SrcRegs, Vec);
SrcRegs[IdxVal] = MI.getOperand(2).getReg();
B.buildMergeLikeInstr(Dst, SrcRegs);
} else {
B.buildUndef(Dst);
}
MI.eraseFromParent();
return true;
}
bool AMDGPULegalizerInfo::legalizeSinCos(
MachineInstr &MI, MachineRegisterInfo &MRI,
MachineIRBuilder &B) const {
Register DstReg = MI.getOperand(0).getReg();
Register SrcReg = MI.getOperand(1).getReg();
LLT Ty = MRI.getType(DstReg);
unsigned Flags = MI.getFlags();
Register TrigVal;
auto OneOver2Pi = B.buildFConstant(Ty, 0.5 * numbers::inv_pi);
if (ST.hasTrigReducedRange()) {
auto MulVal = B.buildFMul(Ty, SrcReg, OneOver2Pi, Flags);
TrigVal = B.buildIntrinsic(Intrinsic::amdgcn_fract, {Ty})
.addUse(MulVal.getReg(0))
.setMIFlags(Flags)
.getReg(0);
} else
TrigVal = B.buildFMul(Ty, SrcReg, OneOver2Pi, Flags).getReg(0);
Intrinsic::ID TrigIntrin = MI.getOpcode() == AMDGPU::G_FSIN ?
Intrinsic::amdgcn_sin : Intrinsic::amdgcn_cos;
B.buildIntrinsic(TrigIntrin, ArrayRef<Register>(DstReg))
.addUse(TrigVal)
.setMIFlags(Flags);
MI.eraseFromParent();
return true;
}
bool AMDGPULegalizerInfo::buildPCRelGlobalAddress(Register DstReg, LLT PtrTy,
MachineIRBuilder &B,
const GlobalValue *GV,
int64_t Offset,
unsigned GAFlags) const {
assert(isInt<32>(Offset + 4) && "32-bit offset is expected!");
// In order to support pc-relative addressing, SI_PC_ADD_REL_OFFSET is lowered
// to the following code sequence:
//
// For constant address space:
// s_getpc_b64 s[0:1]
// s_add_u32 s0, s0, $symbol
// s_addc_u32 s1, s1, 0
//
// s_getpc_b64 returns the address of the s_add_u32 instruction and then
// a fixup or relocation is emitted to replace $symbol with a literal
// constant, which is a pc-relative offset from the encoding of the $symbol
// operand to the global variable.
//
// For global address space:
// s_getpc_b64 s[0:1]
// s_add_u32 s0, s0, $symbol@{gotpc}rel32@lo
// s_addc_u32 s1, s1, $symbol@{gotpc}rel32@hi
//
// s_getpc_b64 returns the address of the s_add_u32 instruction and then
// fixups or relocations are emitted to replace $symbol@*@lo and
// $symbol@*@hi with lower 32 bits and higher 32 bits of a literal constant,
// which is a 64-bit pc-relative offset from the encoding of the $symbol
// operand to the global variable.
MachineRegisterInfo &MRI = *B.getMRI();
LLT PCRegTy = PtrTy.getSizeInBits() == 32
? LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64)
: PtrTy;
Register PCReg =
MRI.createVirtualRegister({&AMDGPU::SReg_64RegClass, PCRegTy});
if (ST.has64BitLiterals()) {
assert(GAFlags != SIInstrInfo::MO_NONE);
MachineInstrBuilder MIB =
B.buildInstr(AMDGPU::SI_PC_ADD_REL_OFFSET64).addDef(PCReg);
MIB.addGlobalAddress(GV, Offset, GAFlags + 2);
} else {
MachineInstrBuilder MIB =
B.buildInstr(AMDGPU::SI_PC_ADD_REL_OFFSET).addDef(PCReg);
MIB.addGlobalAddress(GV, Offset, GAFlags);
if (GAFlags == SIInstrInfo::MO_NONE)
MIB.addImm(0);
else
MIB.addGlobalAddress(GV, Offset, GAFlags + 1);
MIB.setOperandDead(3); // $scc
}
if (PtrTy.getSizeInBits() == 32)
B.buildExtract(DstReg, PCReg, 0);
else
B.buildCopy(DstReg, PCReg);
return true;
}
// Emit a ABS32_LO / ABS32_HI relocation stub.
void AMDGPULegalizerInfo::buildAbsGlobalAddress(
Register DstReg, LLT PtrTy, MachineIRBuilder &B, const GlobalValue *GV,
MachineRegisterInfo &MRI) const {
bool RequiresHighHalf = PtrTy.getSizeInBits() != 32;
if (RequiresHighHalf && ST.has64BitLiterals()) {
Register Addr =
MRI.createVirtualRegister({&AMDGPU::SReg_64RegClass, PtrTy});
B.buildInstr(AMDGPU::S_MOV_B64)
.addDef(Addr)
.addGlobalAddress(GV, 0, SIInstrInfo::MO_ABS64);
B.buildCopy(DstReg, Addr);
return;
}
LLT I32 = LLT::integer(32);
Register AddrLo = MRI.createVirtualRegister({&AMDGPU::SReg_32RegClass, I32});
// Write the lower half.
B.buildInstr(AMDGPU::S_MOV_B32)
.addDef(AddrLo)
.addGlobalAddress(GV, 0, SIInstrInfo::MO_ABS32_LO);
// If required, write the upper half as well.
if (RequiresHighHalf) {
assert(PtrTy.getSizeInBits() == 64 &&
"Must provide a 64-bit pointer type!");
Register AddrHi =
MRI.createVirtualRegister({&AMDGPU::SReg_32RegClass, I32});
B.buildInstr(AMDGPU::S_MOV_B32)
.addDef(AddrHi)
.addGlobalAddress(GV, 0, SIInstrInfo::MO_ABS32_HI);
Register AddrDst =
MRI.createVirtualRegister({&AMDGPU::SReg_64RegClass, LLT::integer(64)});
B.buildMergeValues(AddrDst, {AddrLo, AddrHi});
B.buildCast(DstReg, AddrDst);
} else {
B.buildCast(DstReg, AddrLo);
}
}
bool AMDGPULegalizerInfo::legalizeGlobalValue(
MachineInstr &MI, MachineRegisterInfo &MRI,
MachineIRBuilder &B) const {
Register DstReg = MI.getOperand(0).getReg();
LLT Ty = MRI.getType(DstReg);
unsigned AS = Ty.getAddressSpace();
const GlobalValue *GV = MI.getOperand(1).getGlobal();
MachineFunction &MF = B.getMF();
SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
if (AS == AMDGPUAS::BARRIER) {
const GlobalVariable *GVar = cast<GlobalVariable>(GV);
if (!AMDGPU::isNamedBarrier(*GVar)) {
const Function &Fn = MF.getFunction();
Fn.getContext().diagnose(DiagnosticInfoUnsupported(
Fn, "unsupported use of BARRIER address space", MI.getDebugLoc(),
DS_Error));
B.buildUndef(DstReg);
MI.eraseFromParent();
return true;
}
B.buildConstant(DstReg,
MFI->allocateBarrierGlobal(B.getDataLayout(), *GVar));
MI.eraseFromParent();
return true;
}
if (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS) {
if (!MFI->isModuleEntryFunction() &&
GV->getName() != "llvm.amdgcn.module.lds") {
const Function &Fn = MF.getFunction();
Fn.getContext().diagnose(DiagnosticInfoUnsupported(
Fn, "local memory global used by non-kernel function",
MI.getDebugLoc(), DS_Warning));
// We currently don't have a way to correctly allocate LDS objects that
// aren't directly associated with a kernel. We do force inlining of
// functions that use local objects. However, if these dead functions are
// not eliminated, we don't want a compile time error. Just emit a warning
// and a trap, since there should be no callable path here.
B.buildTrap();
B.buildUndef(DstReg);
MI.eraseFromParent();
return true;
}
// TODO: We could emit code to handle the initialization somewhere.
// We ignore the initializer for now and legalize it to allow selection.
// The initializer will anyway get errored out during assembly emission.
const SITargetLowering *TLI = ST.getTargetLowering();
if (!TLI->shouldUseLDSConstAddress(GV)) {
MI.getOperand(1).setTargetFlags(SIInstrInfo::MO_ABS32_LO);
return true; // Leave in place;
}
const GlobalVariable &GVar = *cast<GlobalVariable>(GV);
if (AS == AMDGPUAS::LOCAL_ADDRESS && GV->hasExternalLinkage()) {
// HIP uses an unsized array `extern __shared__ T s[]` or similar
// zero-sized type in other languages to declare the dynamic shared
// memory which size is not known at the compile time. They will be
// allocated by the runtime and placed directly after the static
// allocated ones. They all share the same offset.
if (GVar.getGlobalSize(GVar.getDataLayout()) == 0) {
// Adjust alignment for that dynamic shared memory array.
MFI->setDynLDSAlign(MF.getFunction(), GVar);
LLT I32 = LLT::integer(32);
auto Sz = B.buildIntrinsic(Intrinsic::amdgcn_groupstaticsize, {I32});
B.buildIntToPtr(DstReg, Sz);
MI.eraseFromParent();
return true;
}
}
B.buildConstant(DstReg, MFI->allocateLDSGlobal(B.getDataLayout(), GVar));
MI.eraseFromParent();
return true;
}
if (ST.isAmdPalOS() || ST.isMesa3DOS()) {
buildAbsGlobalAddress(DstReg, Ty, B, GV, MRI);
MI.eraseFromParent();
return true;
}
const SITargetLowering *TLI = ST.getTargetLowering();
if (TLI->shouldEmitFixup(GV)) {
buildPCRelGlobalAddress(DstReg, Ty, B, GV, 0);
MI.eraseFromParent();
return true;
}
if (TLI->shouldEmitPCReloc(GV)) {
buildPCRelGlobalAddress(DstReg, Ty, B, GV, 0, SIInstrInfo::MO_REL32);
MI.eraseFromParent();
return true;
}
LLT PtrTy = LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64);
Register GOTAddr = MRI.createGenericVirtualRegister(PtrTy);
LLT LoadTy = Ty.getSizeInBits() == 32 ? PtrTy : Ty;
MachineMemOperand *GOTMMO = MF.getMachineMemOperand(
MachinePointerInfo::getGOT(MF),
MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
MachineMemOperand::MOInvariant,
LoadTy, Align(8));
buildPCRelGlobalAddress(GOTAddr, PtrTy, B, GV, 0, SIInstrInfo::MO_GOTPCREL32);
if (Ty.getSizeInBits() == 32) {
// Truncate if this is a 32-bit constant address.
auto Load = B.buildLoad(PtrTy, GOTAddr, *GOTMMO);
B.buildExtract(DstReg, Load, 0);
} else
B.buildLoad(DstReg, GOTAddr, *GOTMMO);
MI.eraseFromParent();
return true;
}
static LLT widenToNextPowerOf2(LLT Ty) {
if (Ty.isVector())
return Ty.changeElementCount(
ElementCount::getFixed(PowerOf2Ceil(Ty.getNumElements())));
return Ty.changeElementSize(PowerOf2Ceil(Ty.getSizeInBits()));
}
bool AMDGPULegalizerInfo::legalizeLoad(LegalizerHelper &Helper,
MachineInstr &MI) const {
MachineIRBuilder &B = Helper.MIRBuilder;
MachineRegisterInfo &MRI = *B.getMRI();
GISelChangeObserver &Observer = Helper.Observer;
Register PtrReg = MI.getOperand(1).getReg();
LLT PtrTy = MRI.getType(PtrReg);
unsigned AddrSpace = PtrTy.getAddressSpace();
if (AddrSpace == AMDGPUAS::CONSTANT_ADDRESS_32BIT) {
LLT ConstPtr = LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64);
auto Cast = B.buildAddrSpaceCast(ConstPtr, PtrReg);
Observer.changingInstr(MI);
MI.getOperand(1).setReg(Cast.getReg(0));
Observer.changedInstr(MI);
return true;
}
if (MI.getOpcode() != AMDGPU::G_LOAD)
return false;
Register ValReg = MI.getOperand(0).getReg();
LLT ValTy = MRI.getType(ValReg);
if (hasBufferRsrcWorkaround(ValTy)) {
Observer.changingInstr(MI);
castBufferRsrcFromV4I32(MI, B, MRI, 0);
Observer.changedInstr(MI);
return true;
}
MachineMemOperand *MMO = *MI.memoperands_begin();
const unsigned ValSize = ValTy.getSizeInBits();
const LLT MemTy = MMO->getMemoryType();
const Align MemAlign = MMO->getAlign();
const unsigned MemSize = MemTy.getSizeInBits();
const uint64_t AlignInBits = 8 * MemAlign.value();
// Widen non-power-of-2 loads to the alignment if needed
if (shouldWidenLoad(ST, MemTy, AlignInBits, AddrSpace, MI.getOpcode())) {
const unsigned WideMemSize = PowerOf2Ceil(MemSize);
// This was already the correct extending load result type, so just adjust
// the memory type.
if (WideMemSize == ValSize) {
MachineFunction &MF = B.getMF();
MachineMemOperand *WideMMO =
MF.getMachineMemOperand(MMO, 0, WideMemSize / 8);
Observer.changingInstr(MI);
MI.setMemRefs(MF, {WideMMO});
Observer.changedInstr(MI);
return true;
}
// Don't bother handling edge case that should probably never be produced.
if (ValSize > WideMemSize)
return false;
LLT WideTy = widenToNextPowerOf2(ValTy);
Register WideLoad;
if (!WideTy.isVector()) {
WideLoad = B.buildLoadFromOffset(WideTy, PtrReg, *MMO, 0).getReg(0);
B.buildTrunc(ValReg, WideLoad).getReg(0);
} else {
// Extract the subvector.
if (isRegisterType(ST, ValTy)) {
// If this a case where G_EXTRACT is legal, use it.
// (e.g. <3 x i32> -> <4 x i32>)
WideLoad = B.buildLoadFromOffset(WideTy, PtrReg, *MMO, 0).getReg(0);
B.buildExtract(ValReg, WideLoad, 0);
} else {
// For cases where the widened type isn't a nice register value, unmerge
// from a widened register (e.g. <3 x i16> -> <4 x i16>)
WideLoad = B.buildLoadFromOffset(WideTy, PtrReg, *MMO, 0).getReg(0);
B.buildDeleteTrailingVectorElements(ValReg, WideLoad);
}
}
MI.eraseFromParent();
return true;
}
return false;
}
bool AMDGPULegalizerInfo::legalizeStore(LegalizerHelper &Helper,
MachineInstr &MI) const {
MachineIRBuilder &B = Helper.MIRBuilder;
MachineRegisterInfo &MRI = *B.getMRI();
GISelChangeObserver &Observer = Helper.Observer;
Register DataReg = MI.getOperand(0).getReg();
LLT DataTy = MRI.getType(DataReg);
if (hasBufferRsrcWorkaround(DataTy)) {
Observer.changingInstr(MI);
castBufferRsrcArgToV4I32(MI, B, 0);
Observer.changedInstr(MI);
return true;
}
return false;
}
bool AMDGPULegalizerInfo::legalizeFMad(
MachineInstr &MI, MachineRegisterInfo &MRI,
MachineIRBuilder &B) const {
LLT Ty = MRI.getType(MI.getOperand(0).getReg());
assert(Ty.isScalar());
MachineFunction &MF = B.getMF();
const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
// TODO: Always legal with future ftz flag.
// TODO: Type is expected to be LLT::float32()/LLT::float16()
// FIXME: Do we need just output?
if (Ty == F32 &&
MFI->getMode().FP32Denormals == DenormalMode::getPreserveSign())
return true;
if (Ty == F16 &&
MFI->getMode().FP64FP16Denormals == DenormalMode::getPreserveSign())
return true;
MachineIRBuilder HelperBuilder(MI);
GISelObserverWrapper DummyObserver;
LegalizerHelper Helper(MF, DummyObserver, HelperBuilder);
return Helper.lowerFMad(MI) == LegalizerHelper::Legalized;
}
bool AMDGPULegalizerInfo::legalizeAtomicCmpXChg(
MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const {
Register DstReg = MI.getOperand(0).getReg();
Register PtrReg = MI.getOperand(1).getReg();
Register CmpVal = MI.getOperand(2).getReg();
Register NewVal = MI.getOperand(3).getReg();
assert(AMDGPU::isFlatGlobalAddrSpace(MRI.getType(PtrReg).getAddressSpace()) &&
"this should not have been custom lowered");
LLT ValTy = MRI.getType(CmpVal);
LLT VecTy = LLT::fixed_vector(2, ValTy);
Register PackedVal = B.buildBuildVector(VecTy, { NewVal, CmpVal }).getReg(0);
B.buildInstr(AMDGPU::G_AMDGPU_ATOMIC_CMPXCHG)
.addDef(DstReg)
.addUse(PtrReg)
.addUse(PackedVal)
.setMemRefs(MI.memoperands());
MI.eraseFromParent();
return true;
}
/// Return true if it's known that \p Src can never be an f32 denormal value.
static bool valueIsKnownNeverF32Denorm(const MachineRegisterInfo &MRI,
Register Src) {
const MachineInstr *DefMI = MRI.getVRegDef(Src);
switch (DefMI->getOpcode()) {
case TargetOpcode::G_INTRINSIC: {
switch (cast<GIntrinsic>(DefMI)->getIntrinsicID()) {
case Intrinsic::amdgcn_frexp_mant:
case Intrinsic::amdgcn_log:
case Intrinsic::amdgcn_log_clamp:
case Intrinsic::amdgcn_exp2:
case Intrinsic::amdgcn_sqrt:
return true;
default:
break;
}
break;
}
case TargetOpcode::G_FSQRT:
return true;
case TargetOpcode::G_FFREXP: {
if (DefMI->getOperand(0).getReg() == Src)
return true;
break;
}
case TargetOpcode::G_FPEXT: {
return MRI.getType(DefMI->getOperand(1).getReg()) == F16;
}
default:
return false;
}
return false;
}
static bool allowApproxFunc(const MachineFunction &MF, unsigned Flags) {
return Flags & MachineInstr::FmAfn;
}
static bool needsDenormHandlingF32(const MachineFunction &MF, Register Src,
unsigned Flags) {
return !valueIsKnownNeverF32Denorm(MF.getRegInfo(), Src) &&
MF.getDenormalMode(APFloat::IEEEsingle()).Input !=
DenormalMode::PreserveSign;
}
std::pair<Register, Register>
AMDGPULegalizerInfo::getScaledLogInput(MachineIRBuilder &B, Register Src,
unsigned Flags) const {
if (!needsDenormHandlingF32(B.getMF(), Src, Flags))
return {};
auto SmallestNormal = B.buildFConstant(
F32, APFloat::getSmallestNormalized(APFloat::IEEEsingle()));
auto IsLtSmallestNormal =
B.buildFCmp(CmpInst::FCMP_OLT, LLT::scalar(1), Src, SmallestNormal);
auto Scale32 = B.buildFConstant(F32, 0x1.0p+32);
auto One = B.buildFConstant(F32, 1.0);
auto ScaleFactor =
B.buildSelect(F32, IsLtSmallestNormal, Scale32, One, Flags);
auto ScaledInput = B.buildFMul(F32, Src, ScaleFactor, Flags);
return {ScaledInput.getReg(0), IsLtSmallestNormal.getReg(0)};
}
bool AMDGPULegalizerInfo::legalizeFlog2(MachineInstr &MI,
MachineIRBuilder &B) const {
// v_log_f32 is good enough for OpenCL, except it doesn't handle denormals.
// If we have to handle denormals, scale up the input and adjust the result.
// scaled = x * (is_denormal ? 0x1.0p+32 : 1.0)
// log2 = amdgpu_log2 - (is_denormal ? 32.0 : 0.0)
Register Dst = MI.getOperand(0).getReg();
Register Src = MI.getOperand(1).getReg();
LLT Ty = B.getMRI()->getType(Dst);
unsigned Flags = MI.getFlags();
if (Ty == F16) {
// Nothing in half is a denormal when promoted to f32.
auto Ext = B.buildFPExt(F32, Src, Flags);
auto Log2 = B.buildIntrinsic(Intrinsic::amdgcn_log, {F32})
.addUse(Ext.getReg(0))
.setMIFlags(Flags);
B.buildFPTrunc(Dst, Log2, Flags);
MI.eraseFromParent();
return true;
}
assert(Ty == F32);
auto [ScaledInput, IsLtSmallestNormal] = getScaledLogInput(B, Src, Flags);
if (!ScaledInput) {
B.buildIntrinsic(Intrinsic::amdgcn_log, {MI.getOperand(0)})
.addUse(Src)
.setMIFlags(Flags);
MI.eraseFromParent();
return true;
}
auto Log2 = B.buildIntrinsic(Intrinsic::amdgcn_log, {Ty})
.addUse(ScaledInput)
.setMIFlags(Flags);
auto ThirtyTwo = B.buildFConstant(Ty, 32.0);
auto Zero = B.buildFConstant(Ty, 0.0);
auto ResultOffset =
B.buildSelect(Ty, IsLtSmallestNormal, ThirtyTwo, Zero, Flags);
B.buildFSub(Dst, Log2, ResultOffset, Flags);
MI.eraseFromParent();
return true;
}
static Register getMad(MachineIRBuilder &B, LLT Ty, Register X, Register Y,
Register Z, unsigned Flags) {
auto FMul = B.buildFMul(Ty, X, Y, Flags);
return B.buildFAdd(Ty, FMul, Z, Flags).getReg(0);
}
bool AMDGPULegalizerInfo::legalizeFlogCommon(MachineInstr &MI,
MachineIRBuilder &B) const {
const bool IsLog10 = MI.getOpcode() == TargetOpcode::G_FLOG10;
assert(IsLog10 || MI.getOpcode() == TargetOpcode::G_FLOG);
MachineRegisterInfo &MRI = *B.getMRI();
Register Dst = MI.getOperand(0).getReg();
Register X = MI.getOperand(1).getReg();
unsigned Flags = MI.getFlags();
const LLT Ty = MRI.getType(X);
if (Ty == F16 || MI.getFlag(MachineInstr::FmAfn)) {
// TODO: The direct f16 path is 1.79 ulp for f16. This should be used
// depending on !fpmath metadata.
bool PromoteToF32 =
Ty == F16 && (!MI.getFlag(MachineInstr::FmAfn) || !ST.has16BitInsts());
if (PromoteToF32) {
Register LogVal = MRI.createGenericVirtualRegister(F32);
auto PromoteSrc = B.buildFPExt(F32, X, Flags);
legalizeFlogUnsafe(B, LogVal, PromoteSrc.getReg(0), IsLog10, Flags);
B.buildFPTrunc(Dst, LogVal, Flags);
} else {
legalizeFlogUnsafe(B, Dst, X, IsLog10, Flags);
}
MI.eraseFromParent();
return true;
}
auto [ScaledInput, IsScaled] = getScaledLogInput(B, X, Flags);
if (ScaledInput)
X = ScaledInput;
auto Y =
B.buildIntrinsic(Intrinsic::amdgcn_log, {Ty}).addUse(X).setMIFlags(Flags);
Register R;
if (ST.hasFastFMAF32()) {
// c+cc are ln(2)/ln(10) to more than 49 bits
const float c_log10 = 0x1.344134p-2f;
const float cc_log10 = 0x1.09f79ep-26f;
// c + cc is ln(2) to more than 49 bits
const float c_log = 0x1.62e42ep-1f;
const float cc_log = 0x1.efa39ep-25f;
auto C = B.buildFConstant(Ty, IsLog10 ? c_log10 : c_log);
auto CC = B.buildFConstant(Ty, IsLog10 ? cc_log10 : cc_log);
// This adds correction terms for which contraction may lead to an increase
// in the error of the approximation, so disable it.
auto NewFlags = Flags & ~(MachineInstr::FmContract);
R = B.buildFMul(Ty, Y, C, NewFlags).getReg(0);
auto NegR = B.buildFNeg(Ty, R, NewFlags);
auto FMA0 = B.buildFMA(Ty, Y, C, NegR, NewFlags);
auto FMA1 = B.buildFMA(Ty, Y, CC, FMA0, NewFlags);
R = B.buildFAdd(Ty, R, FMA1, NewFlags).getReg(0);
} else {
// ch+ct is ln(2)/ln(10) to more than 36 bits
const float ch_log10 = 0x1.344000p-2f;
const float ct_log10 = 0x1.3509f6p-18f;
// ch + ct is ln(2) to more than 36 bits
const float ch_log = 0x1.62e000p-1f;
const float ct_log = 0x1.0bfbe8p-15f;
auto CH = B.buildFConstant(Ty, IsLog10 ? ch_log10 : ch_log);
auto CT = B.buildFConstant(Ty, IsLog10 ? ct_log10 : ct_log);
const LLT I32 = LLT::integer(32);
auto YInt = B.buildBitcast(I32, Y);
auto MaskConst = B.buildConstant(I32, 0xfffff000);
auto YH = B.buildBitcast(Ty, B.buildAnd(I32, YInt, MaskConst));
auto YT = B.buildFSub(Ty, Y, YH, Flags);
// This adds correction terms for which contraction may lead to an increase
// in the error of the approximation, so disable it.
auto NewFlags = Flags & ~(MachineInstr::FmContract);
auto YTCT = B.buildFMul(Ty, YT, CT, NewFlags);
Register Mad0 =
getMad(B, Ty, YH.getReg(0), CT.getReg(0), YTCT.getReg(0), NewFlags);
Register Mad1 = getMad(B, Ty, YT.getReg(0), CH.getReg(0), Mad0, NewFlags);
R = getMad(B, Ty, YH.getReg(0), CH.getReg(0), Mad1, NewFlags);
}
const bool IsFiniteOnly =
MI.getFlag(MachineInstr::FmNoNans) && MI.getFlag(MachineInstr::FmNoInfs);
if (!IsFiniteOnly) {
// Expand isfinite(x) => fabs(x) < inf
auto Inf = B.buildFConstant(Ty, APFloat::getInf(APFloat::IEEEsingle()));
auto Fabs = B.buildFAbs(Ty, Y);
auto IsFinite =
B.buildFCmp(CmpInst::FCMP_OLT, LLT::scalar(1), Fabs, Inf, Flags);
R = B.buildSelect(Ty, IsFinite, R, Y, Flags).getReg(0);
}
if (ScaledInput) {
auto Zero = B.buildFConstant(Ty, 0.0);
auto ShiftK =
B.buildFConstant(Ty, IsLog10 ? 0x1.344136p+3f : 0x1.62e430p+4f);
auto Shift = B.buildSelect(Ty, IsScaled, ShiftK, Zero, Flags);
B.buildFSub(Dst, R, Shift, Flags);
} else {
B.buildCopy(Dst, R);
}
MI.eraseFromParent();
return true;
}
bool AMDGPULegalizerInfo::legalizeFlogUnsafe(MachineIRBuilder &B, Register Dst,
Register Src, bool IsLog10,
unsigned Flags) const {
const double Log2BaseInverted =
IsLog10 ? numbers::ln2 / numbers::ln10 : numbers::ln2;
LLT Ty = B.getMRI()->getType(Dst);
if (Ty == F32) {
auto [ScaledInput, IsScaled] = getScaledLogInput(B, Src, Flags);
if (ScaledInput) {
auto LogSrc = B.buildIntrinsic(Intrinsic::amdgcn_log, {Ty})
.addUse(ScaledInput)
.setMIFlags(Flags);
auto ScaledResultOffset = B.buildFConstant(Ty, -32.0 * Log2BaseInverted);
auto Zero = B.buildFConstant(Ty, 0.0);
auto ResultOffset =
B.buildSelect(Ty, IsScaled, ScaledResultOffset, Zero, Flags);
auto Log2Inv = B.buildFConstant(Ty, Log2BaseInverted);
if (ST.hasFastFMAF32())
B.buildFMA(Dst, LogSrc, Log2Inv, ResultOffset, Flags);
else {
auto Mul = B.buildFMul(Ty, LogSrc, Log2Inv, Flags);
B.buildFAdd(Dst, Mul, ResultOffset, Flags);
}
return true;
}
}
auto Log2Operand = Ty == F16 ? B.buildFLog2(Ty, Src, Flags)
: B.buildIntrinsic(Intrinsic::amdgcn_log, {Ty})
.addUse(Src)
.setMIFlags(Flags);
auto Log2BaseInvertedOperand = B.buildFConstant(Ty, Log2BaseInverted);
B.buildFMul(Dst, Log2Operand, Log2BaseInvertedOperand, Flags);
return true;
}
bool AMDGPULegalizerInfo::legalizeFExp2(MachineInstr &MI,
MachineIRBuilder &B) const {
// v_exp_f32 is good enough for OpenCL, except it doesn't handle denormals.
// If we have to handle denormals, scale up the input and adjust the result.
Register Dst = MI.getOperand(0).getReg();
Register Src = MI.getOperand(1).getReg();
unsigned Flags = MI.getFlags();
LLT Ty = B.getMRI()->getType(Dst);
if (Ty == F64)
return legalizeFEXPF64(MI, B);
if (Ty == F16) {
// Nothing in half is a denormal when promoted to f32.
auto Ext = B.buildFPExt(F32, Src, Flags);
auto Log2 = B.buildIntrinsic(Intrinsic::amdgcn_exp2, {F32})
.addUse(Ext.getReg(0))
.setMIFlags(Flags);
B.buildFPTrunc(Dst, Log2, Flags);
MI.eraseFromParent();
return true;
}
assert(Ty == F32);
if (!needsDenormHandlingF32(B.getMF(), Src, Flags)) {
B.buildIntrinsic(Intrinsic::amdgcn_exp2, ArrayRef<Register>{Dst})
.addUse(Src)
.setMIFlags(Flags);
MI.eraseFromParent();
return true;
}
// bool needs_scaling = x < -0x1.f80000p+6f;
// v_exp_f32(x + (s ? 0x1.0p+6f : 0.0f)) * (s ? 0x1.0p-64f : 1.0f);
// -nextafter(128.0, -1)
auto RangeCheckConst = B.buildFConstant(Ty, -0x1.f80000p+6f);
auto NeedsScaling = B.buildFCmp(CmpInst::FCMP_OLT, LLT::scalar(1), Src,
RangeCheckConst, Flags);
auto SixtyFour = B.buildFConstant(Ty, 0x1.0p+6f);
auto Zero = B.buildFConstant(Ty, 0.0);
auto AddOffset = B.buildSelect(F32, NeedsScaling, SixtyFour, Zero, Flags);
auto AddInput = B.buildFAdd(F32, Src, AddOffset, Flags);
auto Exp2 = B.buildIntrinsic(Intrinsic::amdgcn_exp2, {Ty})
.addUse(AddInput.getReg(0))
.setMIFlags(Flags);
auto TwoExpNeg64 = B.buildFConstant(Ty, 0x1.0p-64f);
auto One = B.buildFConstant(Ty, 1.0);
auto ResultScale = B.buildSelect(F32, NeedsScaling, TwoExpNeg64, One, Flags);
B.buildFMul(Dst, Exp2, ResultScale, Flags);
MI.eraseFromParent();
return true;
}
static MachineInstrBuilder buildExp(MachineIRBuilder &B, const DstOp &Dst,
const SrcOp &Src, unsigned Flags) {
LLT Ty = Dst.getLLTTy(*B.getMRI());
if (Ty == F32) {
return B.buildIntrinsic(Intrinsic::amdgcn_exp2, {Dst})
.addUse(Src.getReg())
.setMIFlags(Flags);
}
return B.buildFExp2(Dst, Src, Flags);
}
bool AMDGPULegalizerInfo::legalizeFExpUnsafeImpl(MachineIRBuilder &B,
Register Dst, Register X,
unsigned Flags,
bool IsExp10) const {
LLT Ty = B.getMRI()->getType(X);
// exp(x) -> exp2(M_LOG2E_F * x);
// exp10(x) -> exp2(log2(10) * x);
auto Const = B.buildFConstant(Ty, IsExp10 ? 0x1.a934f0p+1f : numbers::log2e);
auto Mul = B.buildFMul(Ty, X, Const, Flags);
buildExp(B, Dst, Mul, Flags);
return true;
}
bool AMDGPULegalizerInfo::legalizeFExpUnsafe(MachineIRBuilder &B, Register Dst,
Register X, unsigned Flags) const {
LLT Ty = B.getMRI()->getType(Dst);
if (Ty != F32 || !needsDenormHandlingF32(B.getMF(), X, Flags)) {
return legalizeFExpUnsafeImpl(B, Dst, X, Flags, /*IsExp10=*/false);
}
auto Threshold = B.buildFConstant(Ty, -0x1.5d58a0p+6f);
auto NeedsScaling =
B.buildFCmp(CmpInst::FCMP_OLT, LLT::scalar(1), X, Threshold, Flags);
auto ScaleOffset = B.buildFConstant(Ty, 0x1.0p+6f);
auto ScaledX = B.buildFAdd(Ty, X, ScaleOffset, Flags);
auto AdjustedX = B.buildSelect(Ty, NeedsScaling, ScaledX, X, Flags);
auto Log2E = B.buildFConstant(Ty, numbers::log2e);
auto ExpInput = B.buildFMul(Ty, AdjustedX, Log2E, Flags);
auto Exp2 = B.buildIntrinsic(Intrinsic::amdgcn_exp2, {Ty})
.addUse(ExpInput.getReg(0))
.setMIFlags(Flags);
auto ResultScaleFactor = B.buildFConstant(Ty, 0x1.969d48p-93f);
auto AdjustedResult = B.buildFMul(Ty, Exp2, ResultScaleFactor, Flags);
B.buildSelect(Dst, NeedsScaling, AdjustedResult, Exp2, Flags);
return true;
}
bool AMDGPULegalizerInfo::legalizeFExp10Unsafe(MachineIRBuilder &B,
Register Dst, Register X,
unsigned Flags) const {
LLT Ty = B.getMRI()->getType(Dst);
if (Ty != F32 || !needsDenormHandlingF32(B.getMF(), X, Flags)) {
// exp2(x * 0x1.a92000p+1f) * exp2(x * 0x1.4f0978p-11f);
auto K0 = B.buildFConstant(Ty, 0x1.a92000p+1f);
auto K1 = B.buildFConstant(Ty, 0x1.4f0978p-11f);
auto Mul1 = B.buildFMul(Ty, X, K1, Flags);
auto Exp2_1 = buildExp(B, Ty, Mul1, Flags);
auto Mul0 = B.buildFMul(Ty, X, K0, Flags);
auto Exp2_0 = buildExp(B, Ty, Mul0, Flags);
B.buildFMul(Dst, Exp2_0, Exp2_1, Flags);
return true;
}
// bool s = x < -0x1.2f7030p+5f;
// x += s ? 0x1.0p+5f : 0.0f;
// exp10 = exp2(x * 0x1.a92000p+1f) *
// exp2(x * 0x1.4f0978p-11f) *
// (s ? 0x1.9f623ep-107f : 1.0f);
auto Threshold = B.buildFConstant(Ty, -0x1.2f7030p+5f);
auto NeedsScaling =
B.buildFCmp(CmpInst::FCMP_OLT, LLT::scalar(1), X, Threshold);
auto ScaleOffset = B.buildFConstant(Ty, 0x1.0p+5f);
auto ScaledX = B.buildFAdd(Ty, X, ScaleOffset, Flags);
auto AdjustedX = B.buildSelect(Ty, NeedsScaling, ScaledX, X);
auto K0 = B.buildFConstant(Ty, 0x1.a92000p+1f);
auto K1 = B.buildFConstant(Ty, 0x1.4f0978p-11f);
auto Mul1 = B.buildFMul(Ty, AdjustedX, K1, Flags);
auto Exp2_1 = buildExp(B, Ty, Mul1, Flags);
auto Mul0 = B.buildFMul(Ty, AdjustedX, K0, Flags);
auto Exp2_0 = buildExp(B, Ty, Mul0, Flags);
auto MulExps = B.buildFMul(Ty, Exp2_0, Exp2_1, Flags);
auto ResultScaleFactor = B.buildFConstant(Ty, 0x1.9f623ep-107f);
auto AdjustedResult = B.buildFMul(Ty, MulExps, ResultScaleFactor, Flags);
B.buildSelect(Dst, NeedsScaling, AdjustedResult, MulExps);
return true;
}
// This expansion gives a result slightly better than 1ulp.
bool AMDGPULegalizerInfo::legalizeFEXPF64(MachineInstr &MI,
MachineIRBuilder &B) const {
Register X = MI.getOperand(1).getReg();
LLT I32 = LLT::integer(32);
LLT S1 = LLT::scalar(1);
// TODO: Check if reassoc is safe. There is an output change in exp2 and
// exp10, which slightly increases ulp.
unsigned Flags = MI.getFlags() & ~MachineInstr::FmReassoc;
Register Dn, F, T;
if (MI.getOpcode() == TargetOpcode::G_FEXP2) {
// Dn = rint(X)
Dn = B.buildFRint(F64, X, Flags).getReg(0);
// F = X - Dn
F = B.buildFSub(F64, X, Dn, Flags).getReg(0);
// T = F*C1 + F*C2
auto C1 = B.buildFConstant(F64, APFloat(0x1.62e42fefa39efp-1));
auto C2 = B.buildFConstant(F64, APFloat(0x1.abc9e3b39803fp-56));
auto Mul2 = B.buildFMul(F64, F, C2, Flags).getReg(0);
T = B.buildFMA(F64, F, C1, Mul2, Flags).getReg(0);
} else if (MI.getOpcode() == TargetOpcode::G_FEXP10) {
auto C1 = B.buildFConstant(F64, APFloat(0x1.a934f0979a371p+1));
auto Mul = B.buildFMul(F64, X, C1, Flags).getReg(0);
Dn = B.buildFRint(F64, Mul, Flags).getReg(0);
auto NegDn = B.buildFNeg(F64, Dn, Flags).getReg(0);
auto C2 = B.buildFConstant(F64, APFloat(-0x1.9dc1da994fd21p-59));
auto C3 = B.buildFConstant(F64, APFloat(0x1.34413509f79ffp-2));
auto Inner = B.buildFMA(F64, NegDn, C3, X, Flags).getReg(0);
F = B.buildFMA(F64, NegDn, C2, Inner, Flags).getReg(0);
auto C4 = B.buildFConstant(F64, APFloat(0x1.26bb1bbb55516p+1));
auto C5 = B.buildFConstant(F64, APFloat(-0x1.f48ad494ea3e9p-53));
auto MulF = B.buildFMul(F64, F, C5, Flags).getReg(0);
T = B.buildFMA(F64, F, C4, MulF, Flags).getReg(0);
} else { // G_FEXP
auto C1 = B.buildFConstant(F64, APFloat(0x1.71547652b82fep+0));
auto Mul = B.buildFMul(F64, X, C1, Flags).getReg(0);
Dn = B.buildFRint(F64, Mul, Flags).getReg(0);
auto NegDn = B.buildFNeg(F64, Dn, Flags).getReg(0);
auto C2 = B.buildFConstant(F64, APFloat(0x1.abc9e3b39803fp-56));
auto C3 = B.buildFConstant(F64, APFloat(0x1.62e42fefa39efp-1));
auto Inner = B.buildFMA(F64, NegDn, C3, X, Flags).getReg(0);
T = B.buildFMA(F64, NegDn, C2, Inner, Flags).getReg(0);
}
// Polynomial chain for P
auto P = B.buildFConstant(F64, 0x1.ade156a5dcb37p-26);
P = B.buildFMA(F64, T, P, B.buildFConstant(F64, 0x1.28af3fca7ab0cp-22),
Flags);
P = B.buildFMA(F64, T, P, B.buildFConstant(F64, 0x1.71dee623fde64p-19),
Flags);
P = B.buildFMA(F64, T, P, B.buildFConstant(F64, 0x1.a01997c89e6b0p-16),
Flags);
P = B.buildFMA(F64, T, P, B.buildFConstant(F64, 0x1.a01a014761f6ep-13),
Flags);
P = B.buildFMA(F64, T, P, B.buildFConstant(F64, 0x1.6c16c1852b7b0p-10),
Flags);
P = B.buildFMA(F64, T, P, B.buildFConstant(F64, 0x1.1111111122322p-7), Flags);
P = B.buildFMA(F64, T, P, B.buildFConstant(F64, 0x1.55555555502a1p-5), Flags);
P = B.buildFMA(F64, T, P, B.buildFConstant(F64, 0x1.5555555555511p-3), Flags);
P = B.buildFMA(F64, T, P, B.buildFConstant(F64, 0x1.000000000000bp-1), Flags);
auto One = B.buildFConstant(F64, 1.0);
P = B.buildFMA(F64, T, P, One, Flags);
P = B.buildFMA(F64, T, P, One, Flags);
// Z = FLDEXP(P, (int)Dn)
auto DnInt = B.buildFPTOSI(I32, Dn);
auto Z = B.buildFLdexp(F64, P, DnInt, Flags);
if (!(Flags & MachineInstr::FmNoInfs)) {
// Overflow guard: if X <= 1024.0 then Z else +inf
auto CondHi = B.buildFCmp(CmpInst::FCMP_ULE, S1, X,
B.buildFConstant(F64, APFloat(1024.0)));
auto PInf = B.buildFConstant(F64, APFloat::getInf(APFloat::IEEEdouble()));
Z = B.buildSelect(F64, CondHi, Z, PInf, Flags);
}
// Underflow guard: if X >= -1075.0 then Z else 0.0
auto CondLo = B.buildFCmp(CmpInst::FCMP_UGE, S1, X,
B.buildFConstant(F64, APFloat(-1075.0)));
auto Zero = B.buildFConstant(F64, APFloat(0.0));
B.buildSelect(MI.getOperand(0).getReg(), CondLo, Z, Zero, Flags);
MI.eraseFromParent();
return true;
}
bool AMDGPULegalizerInfo::legalizeFExp(MachineInstr &MI,
MachineIRBuilder &B) const {
Register Dst = MI.getOperand(0).getReg();
Register X = MI.getOperand(1).getReg();
const unsigned Flags = MI.getFlags();
MachineFunction &MF = B.getMF();
MachineRegisterInfo &MRI = *B.getMRI();
LLT Ty = MRI.getType(Dst);
if (Ty == F64)
return legalizeFEXPF64(MI, B);
const bool IsExp10 = MI.getOpcode() == TargetOpcode::G_FEXP10;
if (Ty == F16) {
// v_exp_f16 (fmul x, log2e)
if (allowApproxFunc(MF, Flags)) {
// TODO: Does this really require fast?
IsExp10 ? legalizeFExp10Unsafe(B, Dst, X, Flags)
: legalizeFExpUnsafe(B, Dst, X, Flags);
MI.eraseFromParent();
return true;
}
// Nothing in half is a denormal when promoted to f32.
//
// exp(f16 x) ->
// fptrunc (v_exp_f32 (fmul (fpext x), log2e))
//
// exp10(f16 x) ->
// fptrunc (v_exp_f32 (fmul (fpext x), log2(10)))
auto Ext = B.buildFPExt(F32, X, Flags);
Register Lowered = MRI.createGenericVirtualRegister(F32);
legalizeFExpUnsafeImpl(B, Lowered, Ext.getReg(0), Flags, IsExp10);
B.buildFPTrunc(Dst, Lowered, Flags);
MI.eraseFromParent();
return true;
}
assert(Ty == F32);
// TODO: Interpret allowApproxFunc as ignoring DAZ. This is currently copying
// library behavior. Also, is known-not-daz source sufficient?
if (allowApproxFunc(MF, Flags)) {
IsExp10 ? legalizeFExp10Unsafe(B, Dst, X, Flags)
: legalizeFExpUnsafe(B, Dst, X, Flags);
MI.eraseFromParent();
return true;
}
// Algorithm:
//
// e^x = 2^(x/ln(2)) = 2^(x*(64/ln(2))/64)
//
// x*(64/ln(2)) = n + f, |f| <= 0.5, n is integer
// n = 64*m + j, 0 <= j < 64
//
// e^x = 2^((64*m + j + f)/64)
// = (2^m) * (2^(j/64)) * 2^(f/64)
// = (2^m) * (2^(j/64)) * e^(f*(ln(2)/64))
//
// f = x*(64/ln(2)) - n
// r = f*(ln(2)/64) = x - n*(ln(2)/64)
//
// e^x = (2^m) * (2^(j/64)) * e^r
//
// (2^(j/64)) is precomputed
//
// e^r = 1 + r + (r^2)/2! + (r^3)/3! + (r^4)/4! + (r^5)/5!
// e^r = 1 + q
//
// q = r + (r^2)/2! + (r^3)/3! + (r^4)/4! + (r^5)/5!
//
// e^x = (2^m) * ( (2^(j/64)) + q*(2^(j/64)) )
const unsigned FlagsNoContract = Flags & ~MachineInstr::FmContract;
Register PH, PL;
if (ST.hasFastFMAF32()) {
const float c_exp = numbers::log2ef;
const float cc_exp = 0x1.4ae0bep-26f; // c+cc are 49 bits
const float c_exp10 = 0x1.a934f0p+1f;
const float cc_exp10 = 0x1.2f346ep-24f;
auto C = B.buildFConstant(Ty, IsExp10 ? c_exp10 : c_exp);
PH = B.buildFMul(Ty, X, C, Flags).getReg(0);
auto NegPH = B.buildFNeg(Ty, PH, Flags);
auto FMA0 = B.buildFMA(Ty, X, C, NegPH, Flags);
auto CC = B.buildFConstant(Ty, IsExp10 ? cc_exp10 : cc_exp);
PL = B.buildFMA(Ty, X, CC, FMA0, Flags).getReg(0);
} else {
const float ch_exp = 0x1.714000p+0f;
const float cl_exp = 0x1.47652ap-12f; // ch + cl are 36 bits
const float ch_exp10 = 0x1.a92000p+1f;
const float cl_exp10 = 0x1.4f0978p-11f;
const LLT I32 = LLT::integer(32);
auto XInt = B.buildBitcast(I32, X);
auto MaskConst = B.buildConstant(I32, 0xfffff000);
auto XH = B.buildBitcast(Ty, B.buildAnd(I32, XInt, MaskConst));
auto XL = B.buildFSub(Ty, X, XH, Flags);
auto CH = B.buildFConstant(Ty, IsExp10 ? ch_exp10 : ch_exp);
PH = B.buildFMul(Ty, XH, CH, Flags).getReg(0);
auto CL = B.buildFConstant(Ty, IsExp10 ? cl_exp10 : cl_exp);
auto XLCL = B.buildFMul(Ty, XL, CL, Flags);
Register Mad0 =
getMad(B, Ty, XL.getReg(0), CH.getReg(0), XLCL.getReg(0), Flags);
PL = getMad(B, Ty, XH.getReg(0), CL.getReg(0), Mad0, Flags);
}
auto E = B.buildIntrinsicRoundeven(Ty, PH, Flags);
// It is unsafe to contract this fsub into the PH multiply.
auto PHSubE = B.buildFSub(Ty, PH, E, FlagsNoContract);
auto A = B.buildFAdd(Ty, PHSubE, PL, Flags);
const LLT I32 = LLT::integer(32);
auto IntE = B.buildFPTOSI(I32, E);
auto Exp2 = B.buildIntrinsic(Intrinsic::amdgcn_exp2, {Ty})
.addUse(A.getReg(0))
.setMIFlags(Flags);
auto R = B.buildFLdexp(Ty, Exp2, IntE, Flags);
auto UnderflowCheckConst =
B.buildFConstant(Ty, IsExp10 ? -0x1.66d3e8p+5f : -0x1.9d1da0p+6f);
auto Zero = B.buildFConstant(Ty, 0.0);
auto Underflow =
B.buildFCmp(CmpInst::FCMP_OLT, LLT::scalar(1), X, UnderflowCheckConst);
R = B.buildSelect(Ty, Underflow, Zero, R);
if (!(Flags & MachineInstr::FmNoInfs)) {
auto OverflowCheckConst =
B.buildFConstant(Ty, IsExp10 ? 0x1.344136p+5f : 0x1.62e430p+6f);
auto Overflow =
B.buildFCmp(CmpInst::FCMP_OGT, LLT::scalar(1), X, OverflowCheckConst);
auto Inf = B.buildFConstant(Ty, APFloat::getInf(APFloat::IEEEsingle()));
R = B.buildSelect(Ty, Overflow, Inf, R, Flags);
}
B.buildCopy(Dst, R);
MI.eraseFromParent();
return true;
}
// Keep in sync with AMDGPUTargetLowering::lowerFPOW, which documents this.
bool AMDGPULegalizerInfo::legalizeFPow(LegalizerHelper &Helper,
MachineInstr &MI) const {
MachineIRBuilder &B = Helper.MIRBuilder;
Register Dst = MI.getOperand(0).getReg();
Register X = MI.getOperand(1).getReg();
Register Y = MI.getOperand(2).getReg();
unsigned Flags = MI.getFlags();
assert(B.getMRI()->getType(Dst) == F32);
// log2(0) is -inf, which exp2 turns back into a finite result, so the core
// goes infinite for inputs a ninf fpow still asserts about, like pow(0, 2).
unsigned CoreFlags = Flags & ~MachineInstr::FmNoInfs;
// Fast expansion: ignores denormals, NaN for a negative base.
if (allowApproxFunc(B.getMF(), Flags)) {
auto Log = B.buildIntrinsic(Intrinsic::amdgcn_log, {F32})
.addUse(X)
.setMIFlags(CoreFlags);
auto Mul = B.buildIntrinsic(Intrinsic::amdgcn_fmul_legacy, {F32})
.addUse(Y)
.addUse(Log.getReg(0))
.setMIFlags(CoreFlags);
buildExp(B, Dst, Mul.getReg(0), CoreFlags);
MI.eraseFromParent();
return true;
}
auto Abs = B.buildFAbs(F32, X, Flags);
auto Log = B.buildFLog2(F32, Abs, CoreFlags);
auto Mul = B.buildIntrinsic(Intrinsic::amdgcn_fmul_legacy, {F32})
.addUse(Y)
.addUse(Log.getReg(0))
.setMIFlags(CoreFlags);
// A base that is never negative needs neither the sign fixup nor the NaN.
if (Helper.getValueTracking()
->computeKnownFPClass(X, Flags, fcNegative, 0)
.signBitIsZeroOrNaN()) {
B.buildFExp2(Dst, Mul, CoreFlags);
MI.eraseFromParent();
return true;
}
Register R = B.buildFExp2(F32, Mul, CoreFlags).getReg(0);
auto YTrunc = B.buildIntrinsicTrunc(F32, Y);
auto YIsInt = B.buildFCmp(CmpInst::FCMP_OEQ, S1, YTrunc, Y);
auto YHalf = B.buildFMul(F32, Y, B.buildFConstant(F32, 0.5));
auto YHalfTrunc = B.buildIntrinsicTrunc(F32, YHalf);
auto YIsOdd = B.buildAnd(
S1, YIsInt, B.buildFCmp(CmpInst::FCMP_ONE, S1, YHalfTrunc, YHalf));
// pow(-x, odd y) = -pow(x, y). Selecting the copysign lets even y fold it.
auto Neg = B.buildFCopysign(F32, R, X);
if (Flags & MachineInstr::FmNoNans) {
B.buildSelect(Dst, YIsOdd, Neg, R);
MI.eraseFromParent();
return true;
}
R = B.buildSelect(F32, YIsOdd, Neg, R).getReg(0);
// A negative finite base to a non-integral power is NaN. -inf is excluded:
// the core already gives pow(+inf, y). So are subnormals when flushed.
FPClassTest NegFiniteMask = fcNegNormal;
if (!B.getMF().getDenormalMode(APFloat::IEEEsingle()).inputsAreZero())
NegFiniteMask |= fcNegSubnormal;
auto XNegFinite = B.buildIsFPClass(S1, X, NegFiniteMask);
// Not an ONE compare: pow(-1, NaN) needs the NaN-true behavior of !OEQ.
auto NegNonInt = B.buildAnd(S1, XNegFinite, B.buildNot(S1, YIsInt));
auto NaN = B.buildFConstant(F32, APFloat::getQNaN(APFloat::IEEEsingle()));
B.buildSelect(Dst, NegNonInt, NaN, R);
MI.eraseFromParent();
return true;
}
// Find a source register, ignoring any possible source modifiers.
static Register stripAnySourceMods(Register OrigSrc, MachineRegisterInfo &MRI) {
Register ModSrc = OrigSrc;
if (MachineInstr *SrcFNeg = getOpcodeDef(AMDGPU::G_FNEG, ModSrc, MRI)) {
ModSrc = SrcFNeg->getOperand(1).getReg();
if (MachineInstr *SrcFAbs = getOpcodeDef(AMDGPU::G_FABS, ModSrc, MRI))
ModSrc = SrcFAbs->getOperand(1).getReg();
} else if (MachineInstr *SrcFAbs = getOpcodeDef(AMDGPU::G_FABS, ModSrc, MRI))
ModSrc = SrcFAbs->getOperand(1).getReg();
return ModSrc;
}
bool AMDGPULegalizerInfo::legalizeFFloor(MachineInstr &MI,
MachineRegisterInfo &MRI,
MachineIRBuilder &B) const {
const LLT S1 = LLT::scalar(1);
Register Dst = MI.getOperand(0).getReg();
Register OrigSrc = MI.getOperand(1).getReg();
unsigned Flags = MI.getFlags();
assert(ST.hasFractBug() && MRI.getType(Dst) == F64 &&
"this should not have been custom lowered");
// V_FRACT is buggy on SI, so the F32 version is never used and (x-floor(x))
// is used instead. However, SI doesn't have V_FLOOR_F64, so the most
// efficient way to implement it is using V_FRACT_F64. The workaround for the
// V_FRACT bug is:
// fract(x) = isnan(x) ? x : min(V_FRACT(x), 0.99999999999999999)
//
// Convert floor(x) to (x - fract(x))
auto Fract = B.buildIntrinsic(Intrinsic::amdgcn_fract, {F64})
.addUse(OrigSrc)
.setMIFlags(Flags);
// Give source modifier matching some assistance before obscuring a foldable
// pattern.
// TODO: We can avoid the neg on the fract? The input sign to fract
// shouldn't matter?
Register ModSrc = stripAnySourceMods(OrigSrc, MRI);
auto Const =
B.buildFConstant(F64, llvm::bit_cast<double>(0x3fefffffffffffff));
Register Min = MRI.createGenericVirtualRegister(F64);
// We don't need to concern ourselves with the snan handling difference, so
// use the one which will directly select.
const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
if (MFI->getMode().IEEE)
B.buildFMinNumIEEE(Min, Fract, Const, Flags);
else
B.buildFMinNum(Min, Fract, Const, Flags);
Register CorrectedFract = Min;
if (!MI.getFlag(MachineInstr::FmNoNans)) {
auto IsNan = B.buildFCmp(CmpInst::FCMP_ORD, S1, ModSrc, ModSrc, Flags);
CorrectedFract = B.buildSelect(F64, IsNan, ModSrc, Min, Flags).getReg(0);
}
auto NegFract = B.buildFNeg(F64, CorrectedFract, Flags);
B.buildFAdd(Dst, OrigSrc, NegFract, Flags);
MI.eraseFromParent();
return true;
}
// Turn an illegal packed v2i16/v2f16 build vector into bit operations.
// TODO: This should probably be a bitcast action in LegalizerHelper.
bool AMDGPULegalizerInfo::legalizeBuildVector(
MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const {
Register Dst = MI.getOperand(0).getReg();
const LLT I32 = LLT::integer(32);
const LLT I16 = LLT::integer(16);
assert(MRI.getType(Dst).isVector() &&
MRI.getType(Dst).getNumElements() == 2 &&
MRI.getType(Dst).getScalarSizeInBits() == 16);
Register Src0 = MI.getOperand(1).getReg();
Register Src1 = MI.getOperand(2).getReg();
if (MI.getOpcode() == AMDGPU::G_BUILD_VECTOR_TRUNC) {
assert(MRI.getType(Src0) == I32);
Src0 = B.buildTrunc(I16, MI.getOperand(1).getReg()).getReg(0);
Src1 = B.buildTrunc(I16, MI.getOperand(2).getReg()).getReg(0);
}
auto Merge = B.buildMergeLikeInstr(I32, {Src0, Src1});
B.buildBitcast(Dst, Merge);
MI.eraseFromParent();
return true;
}
// Build a big integer multiply or multiply-add using MAD_64_32 instructions.
//
// Source and accumulation registers must all be 32-bits.
//
// TODO: When the multiply is uniform, we should produce a code sequence
// that is better suited to instruction selection on the SALU. Instead of
// the outer loop going over parts of the result, the outer loop should go
// over parts of one of the factors. This should result in instruction
// selection that makes full use of S_ADDC_U32 instructions.
void AMDGPULegalizerInfo::buildMultiply(LegalizerHelper &Helper,
MutableArrayRef<Register> Accum,
ArrayRef<Register> Src0,
ArrayRef<Register> Src1,
bool UsePartialMad64_32,
bool SeparateOddAlignedProducts) const {
// Use (possibly empty) vectors of S1 registers to represent the set of
// carries from one pair of positions to the next.
using Carry = SmallVector<Register, 2>;
MachineIRBuilder &B = Helper.MIRBuilder;
GISelValueTracking &VT = *Helper.getValueTracking();
const LLT S1 = LLT::scalar(1);
const LLT I32 = LLT::integer(32);
const LLT I64 = LLT::integer(64);
Register Zero32;
Register Zero64;
auto getZero32 = [&]() -> Register {
if (!Zero32)
Zero32 = B.buildConstant(I32, 0).getReg(0);
return Zero32;
};
auto getZero64 = [&]() -> Register {
if (!Zero64)
Zero64 = B.buildConstant(I64, 0).getReg(0);
return Zero64;
};
SmallVector<bool, 2> Src0KnownZeros, Src1KnownZeros;
for (unsigned i = 0; i < Src0.size(); ++i) {
Src0KnownZeros.push_back(VT.getKnownBits(Src0[i]).isZero());
Src1KnownZeros.push_back(VT.getKnownBits(Src1[i]).isZero());
}
// Merge the given carries into the 32-bit LocalAccum, which is modified
// in-place.
//
// Returns the carry-out, which is a single S1 register or null.
auto mergeCarry =
[&](Register &LocalAccum, const Carry &CarryIn) -> Register {
if (CarryIn.empty())
return Register();
bool HaveCarryOut = true;
Register CarryAccum;
if (CarryIn.size() == 1) {
if (!LocalAccum) {
LocalAccum = B.buildZExt(I32, CarryIn[0]).getReg(0);
return Register();
}
CarryAccum = getZero32();
} else {
CarryAccum = B.buildZExt(I32, CarryIn[0]).getReg(0);
for (unsigned i = 1; i + 1 < CarryIn.size(); ++i) {
CarryAccum =
B.buildUAdde(I32, S1, CarryAccum, getZero32(), CarryIn[i])
.getReg(0);
}
if (!LocalAccum) {
LocalAccum = getZero32();
HaveCarryOut = false;
}
}
auto Add =
B.buildUAdde(I32, S1, CarryAccum, LocalAccum, CarryIn.back());
LocalAccum = Add.getReg(0);
return HaveCarryOut ? Add.getReg(1) : Register();
};
// Build a multiply-add chain to compute
//
// LocalAccum + (partial products at DstIndex)
// + (opportunistic subset of CarryIn)
//
// LocalAccum is an array of one or two 32-bit registers that are updated
// in-place. The incoming registers may be null.
//
// In some edge cases, carry-ins can be consumed "for free". In that case,
// the consumed carry bits are removed from CarryIn in-place.
auto buildMadChain =
[&](MutableArrayRef<Register> LocalAccum, unsigned DstIndex, Carry &CarryIn)
-> Carry {
assert((DstIndex + 1 < Accum.size() && LocalAccum.size() == 2) ||
(DstIndex + 1 >= Accum.size() && LocalAccum.size() == 1));
Carry CarryOut;
unsigned j0 = 0;
// Use plain 32-bit multiplication for the most significant part of the
// result by default.
if (LocalAccum.size() == 1 &&
(!UsePartialMad64_32 || !CarryIn.empty())) {
do {
// Skip multiplication if one of the operands is 0
unsigned j1 = DstIndex - j0;
if (Src0KnownZeros[j0] || Src1KnownZeros[j1]) {
++j0;
continue;
}
auto Mul = B.buildMul(I32, Src0[j0], Src1[j1]);
if (!LocalAccum[0] || VT.getKnownBits(LocalAccum[0]).isZero()) {
LocalAccum[0] = Mul.getReg(0);
} else {
if (CarryIn.empty()) {
LocalAccum[0] = B.buildAdd(I32, LocalAccum[0], Mul).getReg(0);
} else {
LocalAccum[0] =
B.buildUAdde(I32, S1, LocalAccum[0], Mul, CarryIn.back())
.getReg(0);
CarryIn.pop_back();
}
}
++j0;
} while (j0 <= DstIndex && (!UsePartialMad64_32 || !CarryIn.empty()));
}
// Build full 64-bit multiplies.
if (j0 <= DstIndex) {
bool HaveSmallAccum = false;
Register Tmp;
if (LocalAccum[0]) {
if (LocalAccum.size() == 1) {
Tmp = B.buildAnyExt(I64, LocalAccum[0]).getReg(0);
HaveSmallAccum = true;
} else if (LocalAccum[1]) {
Tmp = B.buildMergeLikeInstr(I64, LocalAccum).getReg(0);
HaveSmallAccum = false;
} else {
Tmp = B.buildZExt(I64, LocalAccum[0]).getReg(0);
HaveSmallAccum = true;
}
} else {
assert(LocalAccum.size() == 1 || !LocalAccum[1]);
Tmp = getZero64();
HaveSmallAccum = true;
}
do {
unsigned j1 = DstIndex - j0;
if (Src0KnownZeros[j0] || Src1KnownZeros[j1]) {
++j0;
continue;
}
auto Mad = B.buildInstr(AMDGPU::G_AMDGPU_MAD_U64_U32, {I64, S1},
{Src0[j0], Src1[j1], Tmp});
Tmp = Mad.getReg(0);
if (!HaveSmallAccum)
CarryOut.push_back(Mad.getReg(1));
HaveSmallAccum = false;
++j0;
} while (j0 <= DstIndex);
auto Unmerge = B.buildUnmerge(I32, Tmp);
LocalAccum[0] = Unmerge.getReg(0);
if (LocalAccum.size() > 1)
LocalAccum[1] = Unmerge.getReg(1);
}
// Every partial product contributing to this destination index was
// skipped because an operand half is known zero, so nothing has been
// accumulated and the result is zero.
if (!LocalAccum[0])
LocalAccum[0] = getZero32();
// A second element is only ever requested when the full 64-bit multiply
// block above runs, which always writes it.
assert((LocalAccum.size() == 1 || LocalAccum[1]) &&
"Uninitialized accumulator part");
return CarryOut;
};
// Outer multiply loop, iterating over destination parts from least
// significant to most significant parts.
//
// The columns of the following diagram correspond to the destination parts
// affected by one iteration of the outer loop (ignoring boundary
// conditions).
//
// Dest index relative to 2 * i: 1 0 -1
// ------
// Carries from previous iteration: e o
// Even-aligned partial product sum: E E .
// Odd-aligned partial product sum: O O
//
// 'o' is OddCarry, 'e' is EvenCarry.
// EE and OO are computed from partial products via buildMadChain and use
// accumulation where possible and appropriate.
//
Register SeparateOddCarry;
Carry EvenCarry;
Carry OddCarry;
for (unsigned i = 0; i <= Accum.size() / 2; ++i) {
Carry OddCarryIn = std::move(OddCarry);
Carry EvenCarryIn = std::move(EvenCarry);
OddCarry.clear();
EvenCarry.clear();
// Partial products at offset 2 * i.
if (2 * i < Accum.size()) {
auto LocalAccum = Accum.drop_front(2 * i).take_front(2);
EvenCarry = buildMadChain(LocalAccum, 2 * i, EvenCarryIn);
}
// Partial products at offset 2 * i - 1.
if (i > 0) {
if (!SeparateOddAlignedProducts) {
auto LocalAccum = Accum.drop_front(2 * i - 1).take_front(2);
OddCarry = buildMadChain(LocalAccum, 2 * i - 1, OddCarryIn);
} else {
bool IsHighest = 2 * i >= Accum.size();
Register SeparateOddOut[2];
auto LocalAccum = MutableArrayRef(SeparateOddOut)
.take_front(IsHighest ? 1 : 2);
OddCarry = buildMadChain(LocalAccum, 2 * i - 1, OddCarryIn);
MachineInstr *Lo;
if (i == 1) {
if (!IsHighest)
Lo = B.buildUAddo(I32, S1, Accum[2 * i - 1], SeparateOddOut[0]);
else
Lo = B.buildAdd(I32, Accum[2 * i - 1], SeparateOddOut[0]);
} else {
Lo = B.buildUAdde(I32, S1, Accum[2 * i - 1], SeparateOddOut[0],
SeparateOddCarry);
}
Accum[2 * i - 1] = Lo->getOperand(0).getReg();
if (!IsHighest) {
auto Hi = B.buildUAdde(I32, S1, Accum[2 * i], SeparateOddOut[1],
Lo->getOperand(1).getReg());
Accum[2 * i] = Hi.getReg(0);
SeparateOddCarry = Hi.getReg(1);
}
}
}
// Add in the carries from the previous iteration
if (i > 0) {
if (Register CarryOut = mergeCarry(Accum[2 * i - 1], OddCarryIn))
EvenCarryIn.push_back(CarryOut);
if (2 * i < Accum.size()) {
if (Register CarryOut = mergeCarry(Accum[2 * i], EvenCarryIn))
OddCarry.push_back(CarryOut);
}
}
}
}
// Custom narrowing of wide multiplies using wide multiply-add instructions.
//
// TODO: If the multiply is followed by an addition, we should attempt to
// integrate it to make better use of V_MAD_U64_U32's multiply-add capabilities.
bool AMDGPULegalizerInfo::legalizeMul(LegalizerHelper &Helper,
MachineInstr &MI) const {
assert(ST.hasMad64_32());
assert(MI.getOpcode() == TargetOpcode::G_MUL);
MachineIRBuilder &B = Helper.MIRBuilder;
MachineRegisterInfo &MRI = *B.getMRI();
Register DstReg = MI.getOperand(0).getReg();
Register Src0 = MI.getOperand(1).getReg();
Register Src1 = MI.getOperand(2).getReg();
LLT Ty = MRI.getType(DstReg);
assert(Ty.isScalar());
unsigned Size = Ty.getSizeInBits();
if (ST.useVMulU64Inst() && Size == 64)
return true;
unsigned NumParts = Size / 32;
assert((Size % 32) == 0);
assert(NumParts >= 2);
// Whether to use MAD_64_32 for partial products whose high half is
// discarded. This avoids some ADD instructions but risks false dependency
// stalls on some subtargets in some cases.
const bool UsePartialMad64_32 = ST.getGeneration() < AMDGPUSubtarget::GFX10;
// Whether to compute odd-aligned partial products separately. This is
// advisable on subtargets where the accumulator of MAD_64_32 must be placed
// in an even-aligned VGPR.
const bool SeparateOddAlignedProducts = ST.hasFullRate64Ops();
LLT I32 = LLT::integer(32);
SmallVector<Register, 2> Src0Parts, Src1Parts;
for (unsigned i = 0; i < NumParts; ++i) {
Src0Parts.push_back(MRI.createGenericVirtualRegister(I32));
Src1Parts.push_back(MRI.createGenericVirtualRegister(I32));
}
B.buildUnmerge(Src0Parts, Src0);
B.buildUnmerge(Src1Parts, Src1);
SmallVector<Register, 2> AccumRegs(NumParts);
buildMultiply(Helper, AccumRegs, Src0Parts, Src1Parts, UsePartialMad64_32,
SeparateOddAlignedProducts);
B.buildMergeLikeInstr(DstReg, AccumRegs);
MI.eraseFromParent();
return true;
}
// Legalize ctlz/cttz to ffbh/ffbl instead of the default legalization to
// ctlz/cttz_zero_poison. This allows us to fix up the result for the zero input
// case with a single min instruction instead of a compare+select.
bool AMDGPULegalizerInfo::legalizeCTLZ_CTTZ(MachineInstr &MI,
MachineRegisterInfo &MRI,
MachineIRBuilder &B) const {
Register Dst = MI.getOperand(0).getReg();
Register Src = MI.getOperand(1).getReg();
LLT DstTy = MRI.getType(Dst);
LLT SrcTy = MRI.getType(Src);
unsigned NewOpc = MI.getOpcode() == AMDGPU::G_CTLZ
? AMDGPU::G_AMDGPU_FFBH_U32
: AMDGPU::G_AMDGPU_FFBL_B32;
auto Tmp = B.buildInstr(NewOpc, {DstTy}, {Src});
B.buildUMin(Dst, Tmp, B.buildConstant(DstTy, SrcTy.getSizeInBits()));
MI.eraseFromParent();
return true;
}
bool AMDGPULegalizerInfo::legalizeCTLZ_ZERO_POISON(MachineInstr &MI,
MachineRegisterInfo &MRI,
MachineIRBuilder &B) const {
Register Dst = MI.getOperand(0).getReg();
Register Src = MI.getOperand(1).getReg();
LLT SrcTy = MRI.getType(Src);
TypeSize NumBits = SrcTy.getSizeInBits();
assert(NumBits < 32u);
const LLT I32 = LLT::integer(32);
auto ShiftAmt = B.buildConstant(I32, 32u - NumBits);
auto Extend = B.buildAnyExt(I32, {Src}).getReg(0u);
auto Shift = B.buildShl(I32, Extend, ShiftAmt);
auto Ctlz = B.buildInstr(AMDGPU::G_AMDGPU_FFBH_U32, {I32}, {Shift});
B.buildTrunc(Dst, Ctlz);
MI.eraseFromParent();
return true;
}
bool AMDGPULegalizerInfo::legalizeCTLS(MachineInstr &MI,
MachineRegisterInfo &MRI,
MachineIRBuilder &B) const {
Register Dst = MI.getOperand(0).getReg();
Register Src = MI.getOperand(1).getReg();
LLT SrcTy = MRI.getType(Src);
const LLT I32 = LLT::integer(32);
assert(SrcTy == I32 && "legalizeCTLS only supports i32");
unsigned BitWidth = SrcTy.getSizeInBits();
auto Sffbh = B.buildIntrinsic(Intrinsic::amdgcn_sffbh, {I32}).addUse(Src);
auto Clamped = B.buildUMin(I32, Sffbh, B.buildConstant(I32, BitWidth));
B.buildSub(Dst, Clamped, B.buildConstant(I32, 1));
MI.eraseFromParent();
return true;
}
// Check that this is a G_XOR x, -1
static bool isNot(const MachineRegisterInfo &MRI, const MachineInstr &MI) {
if (MI.getOpcode() != TargetOpcode::G_XOR)
return false;
auto ConstVal = getIConstantVRegSExtVal(MI.getOperand(2).getReg(), MRI);
return ConstVal == -1;
}
// Return the use branch instruction, otherwise null if the usage is invalid.
static MachineInstr *
verifyCFIntrinsic(MachineInstr &MI, MachineRegisterInfo &MRI, MachineInstr *&Br,
MachineBasicBlock *&UncondBrTarget, bool &Negated) {
Register CondDef = MI.getOperand(0).getReg();
if (!MRI.hasOneNonDBGUse(CondDef))
return nullptr;
MachineBasicBlock *Parent = MI.getParent();
MachineInstr *UseMI = &*MRI.use_instr_nodbg_begin(CondDef);
if (isNot(MRI, *UseMI)) {
Register NegatedCond = UseMI->getOperand(0).getReg();
if (!MRI.hasOneNonDBGUse(NegatedCond))
return nullptr;
// We're deleting the def of this value, so we need to remove it.
eraseInstr(*UseMI, MRI);
UseMI = &*MRI.use_instr_nodbg_begin(NegatedCond);
Negated = true;
}
if (UseMI->getParent() != Parent || UseMI->getOpcode() != AMDGPU::G_BRCOND)
return nullptr;
// Make sure the cond br is followed by a G_BR, or is the last instruction.
MachineBasicBlock::iterator Next = std::next(UseMI->getIterator());
if (Next == Parent->end()) {
MachineFunction::iterator NextMBB = std::next(Parent->getIterator());
if (NextMBB == Parent->getParent()->end()) // Illegal intrinsic use.
return nullptr;
UncondBrTarget = &*NextMBB;
} else {
if (Next->getOpcode() != AMDGPU::G_BR)
return nullptr;
Br = &*Next;
UncondBrTarget = Br->getOperand(0).getMBB();
}
return UseMI;
}
void AMDGPULegalizerInfo::buildLoadInputValue(Register DstReg,
MachineIRBuilder &B,
const ArgDescriptor *Arg,
const TargetRegisterClass *ArgRC,
LLT ArgTy) const {
MCRegister SrcReg = Arg->getRegister();
assert(SrcReg.isPhysical() && "Physical register expected");
assert(DstReg.isVirtual() && "Virtual register expected");
Register LiveIn = getFunctionLiveInPhysReg(B.getMF(), B.getTII(), SrcReg,
*ArgRC, B.getDebugLoc(), ArgTy);
if (Arg->isMasked()) {
// TODO: Should we try to emit this once in the entry block?
const LLT I32 = LLT::integer(32);
const unsigned Mask = Arg->getMask();
const unsigned Shift = llvm::countr_zero<unsigned>(Mask);
Register AndMaskSrc = LiveIn;
// TODO: Avoid clearing the high bits if we know workitem id y/z are always
// 0.
if (Shift != 0) {
auto ShiftAmt = B.buildConstant(I32, Shift);
AndMaskSrc = B.buildLShr(I32, LiveIn, ShiftAmt).getReg(0);
}
B.buildAnd(DstReg, AndMaskSrc, B.buildConstant(I32, Mask >> Shift));
} else {
B.buildCopy(DstReg, LiveIn);
}
}
bool AMDGPULegalizerInfo::legalizeWorkGroupId(
MachineInstr &MI, MachineIRBuilder &B,
AMDGPUFunctionArgInfo::PreloadedValue WorkGroupIdPV,
AMDGPUFunctionArgInfo::PreloadedValue ClusterMaxIdPV,
AMDGPUFunctionArgInfo::PreloadedValue ClusterWorkGroupIdPV) const {
Register DstReg = MI.getOperand(0).getReg();
if (!ST.hasClusters()) {
if (!loadInputValue(DstReg, B, WorkGroupIdPV))
return false;
MI.eraseFromParent();
return true;
}
// Clusters are supported. Return the global position in the grid. If clusters
// are enabled, WorkGroupIdPV returns the cluster ID not the workgroup ID.
// WorkGroupIdXYZ = ClusterId == 0 ?
// ClusterIdXYZ :
// ClusterIdXYZ * (ClusterMaxIdXYZ + 1) + ClusterWorkGroupIdXYZ
MachineRegisterInfo &MRI = *B.getMRI();
const LLT I32 = LLT::integer(32);
Register ClusterIdXYZ = MRI.createGenericVirtualRegister(I32);
Register ClusterMaxIdXYZ = MRI.createGenericVirtualRegister(I32);
Register ClusterWorkGroupIdXYZ = MRI.createGenericVirtualRegister(I32);
if (!loadInputValue(ClusterIdXYZ, B, WorkGroupIdPV) ||
!loadInputValue(ClusterWorkGroupIdXYZ, B, ClusterWorkGroupIdPV) ||
!loadInputValue(ClusterMaxIdXYZ, B, ClusterMaxIdPV))
return false;
auto One = B.buildConstant(I32, 1);
auto ClusterSizeXYZ = B.buildAdd(I32, ClusterMaxIdXYZ, One);
auto GlobalIdXYZ = B.buildAdd(I32, ClusterWorkGroupIdXYZ,
B.buildMul(I32, ClusterIdXYZ, ClusterSizeXYZ));
const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
switch (MFI->getClusterDims().getKind()) {
case AMDGPU::ClusterDimsAttr::Kind::FixedDims:
case AMDGPU::ClusterDimsAttr::Kind::VariableDims: {
B.buildCopy(DstReg, GlobalIdXYZ);
MI.eraseFromParent();
return true;
}
case AMDGPU::ClusterDimsAttr::Kind::NoCluster: {
B.buildCopy(DstReg, ClusterIdXYZ);
MI.eraseFromParent();
return true;
}
case AMDGPU::ClusterDimsAttr::Kind::Unknown: {
using namespace AMDGPU::Hwreg;
unsigned ClusterIdField = HwregEncoding::encode(ID_IB_STS2, 6, 4);
Register ClusterId = MRI.createGenericVirtualRegister(I32);
MRI.setRegClass(ClusterId, &AMDGPU::SReg_32RegClass);
B.buildInstr(AMDGPU::S_GETREG_B32_const)
.addDef(ClusterId)
.addImm(ClusterIdField);
auto Zero = B.buildConstant(I32, 0);
auto NoClusters =
B.buildICmp(CmpInst::ICMP_EQ, LLT::scalar(1), ClusterId, Zero);
B.buildSelect(DstReg, NoClusters, ClusterIdXYZ, GlobalIdXYZ);
MI.eraseFromParent();
return true;
}
}
llvm_unreachable("nothing should reach here");
}
bool AMDGPULegalizerInfo::loadInputValue(
Register DstReg, MachineIRBuilder &B,
AMDGPUFunctionArgInfo::PreloadedValue ArgType) const {
const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
const ArgDescriptor *Arg = nullptr;
const TargetRegisterClass *ArgRC = nullptr;
LLT ArgTy;
CallingConv::ID CC = B.getMF().getFunction().getCallingConv();
const ArgDescriptor WorkGroupIDX =
ArgDescriptor::createRegister(AMDGPU::TTMP9);
// If GridZ is not programmed in an entry function then the hardware will set
// it to all zeros, so there is no need to mask the GridY value in the low
// order bits.
const ArgDescriptor WorkGroupIDY = ArgDescriptor::createRegister(
AMDGPU::TTMP7,
AMDGPU::isEntryFunctionCC(CC) && !MFI->hasWorkGroupIDZ() ? ~0u : 0xFFFFu);
const ArgDescriptor WorkGroupIDZ =
ArgDescriptor::createRegister(AMDGPU::TTMP7, 0xFFFF0000u);
const ArgDescriptor ClusterWorkGroupIDX =
ArgDescriptor::createRegister(AMDGPU::TTMP6, 0x0000000Fu);
const ArgDescriptor ClusterWorkGroupIDY =
ArgDescriptor::createRegister(AMDGPU::TTMP6, 0x000000F0u);
const ArgDescriptor ClusterWorkGroupIDZ =
ArgDescriptor::createRegister(AMDGPU::TTMP6, 0x00000F00u);
const ArgDescriptor ClusterWorkGroupMaxIDX =
ArgDescriptor::createRegister(AMDGPU::TTMP6, 0x0000F000u);
const ArgDescriptor ClusterWorkGroupMaxIDY =
ArgDescriptor::createRegister(AMDGPU::TTMP6, 0x000F0000u);
const ArgDescriptor ClusterWorkGroupMaxIDZ =
ArgDescriptor::createRegister(AMDGPU::TTMP6, 0x00F00000u);
const ArgDescriptor ClusterWorkGroupMaxFlatID =
ArgDescriptor::createRegister(AMDGPU::TTMP6, 0x0F000000u);
auto LoadConstant = [&](unsigned N) {
B.buildConstant(DstReg, N);
return true;
};
if (ST.hasArchitectedSGPRs() &&
(AMDGPU::isCompute(CC) || CC == CallingConv::AMDGPU_Gfx)) {
AMDGPU::ClusterDimsAttr ClusterDims = MFI->getClusterDims();
bool HasFixedDims = ClusterDims.isFixedDims();
switch (ArgType) {
case AMDGPUFunctionArgInfo::WORKGROUP_ID_X:
Arg = &WorkGroupIDX;
ArgRC = &AMDGPU::SReg_32RegClass;
ArgTy = LLT::integer(32);
break;
case AMDGPUFunctionArgInfo::WORKGROUP_ID_Y:
Arg = &WorkGroupIDY;
ArgRC = &AMDGPU::SReg_32RegClass;
ArgTy = LLT::integer(32);
break;
case AMDGPUFunctionArgInfo::WORKGROUP_ID_Z:
Arg = &WorkGroupIDZ;
ArgRC = &AMDGPU::SReg_32RegClass;
ArgTy = LLT::integer(32);
break;
case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_X:
if (HasFixedDims && ClusterDims.getDims()[0] == 1)
return LoadConstant(0);
Arg = &ClusterWorkGroupIDX;
ArgRC = &AMDGPU::SReg_32RegClass;
ArgTy = LLT::integer(32);
break;
case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_Y:
if (HasFixedDims && ClusterDims.getDims()[1] == 1)
return LoadConstant(0);
Arg = &ClusterWorkGroupIDY;
ArgRC = &AMDGPU::SReg_32RegClass;
ArgTy = LLT::integer(32);
break;
case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_Z:
if (HasFixedDims && ClusterDims.getDims()[2] == 1)
return LoadConstant(0);
Arg = &ClusterWorkGroupIDZ;
ArgRC = &AMDGPU::SReg_32RegClass;
ArgTy = LLT::integer(32);
break;
case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_X:
if (HasFixedDims)
return LoadConstant(ClusterDims.getDims()[0] - 1);
Arg = &ClusterWorkGroupMaxIDX;
ArgRC = &AMDGPU::SReg_32RegClass;
ArgTy = LLT::integer(32);
break;
case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_Y:
if (HasFixedDims)
return LoadConstant(ClusterDims.getDims()[1] - 1);
Arg = &ClusterWorkGroupMaxIDY;
ArgRC = &AMDGPU::SReg_32RegClass;
ArgTy = LLT::integer(32);
break;
case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_Z:
if (HasFixedDims)
return LoadConstant(ClusterDims.getDims()[2] - 1);
Arg = &ClusterWorkGroupMaxIDZ;
ArgRC = &AMDGPU::SReg_32RegClass;
ArgTy = LLT::integer(32);
break;
case AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_FLAT_ID:
Arg = &ClusterWorkGroupMaxFlatID;
ArgRC = &AMDGPU::SReg_32RegClass;
ArgTy = LLT::integer(32);
break;
default:
break;
}
}
if (!Arg)
std::tie(Arg, ArgRC, ArgTy) = MFI->getPreloadedValue(ArgType);
if (!Arg) {
if (ArgType == AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR) {
// The intrinsic may appear when we have a 0 sized kernarg segment, in
// which case the pointer argument may be missing and we use null.
return LoadConstant(0);
}
// It's undefined behavior if a function marked with the amdgpu-no-*
// attributes uses the corresponding intrinsic.
B.buildUndef(DstReg);
return true;
}
if (!Arg->isRegister() || !Arg->getRegister().isValid())
return false; // TODO: Handle these
buildLoadInputValue(DstReg, B, Arg, ArgRC, ArgTy);
return true;
}
bool AMDGPULegalizerInfo::legalizePreloadedArgIntrin(
MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B,
AMDGPUFunctionArgInfo::PreloadedValue ArgType) const {
if (!loadInputValue(MI.getOperand(0).getReg(), B, ArgType))
return false;
MI.eraseFromParent();
return true;
}
static bool replaceWithConstant(MachineIRBuilder &B, MachineInstr &MI,
int64_t C) {
B.buildConstant(MI.getOperand(0).getReg(), C);
MI.eraseFromParent();
return true;
}
bool AMDGPULegalizerInfo::legalizeWorkitemIDIntrinsic(
MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B,
unsigned Dim, AMDGPUFunctionArgInfo::PreloadedValue ArgType) const {
unsigned MaxID = ST.getMaxWorkitemID(B.getMF().getFunction(), Dim);
if (MaxID == 0)
return replaceWithConstant(B, MI, 0);
const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
const ArgDescriptor *Arg;
const TargetRegisterClass *ArgRC;
LLT ArgTy;
std::tie(Arg, ArgRC, ArgTy) = MFI->getPreloadedValue(ArgType);
Register DstReg = MI.getOperand(0).getReg();
if (!Arg) {
// It's undefined behavior if a function marked with the amdgpu-no-*
// attributes uses the corresponding intrinsic.
B.buildUndef(DstReg);
MI.eraseFromParent();
return true;
}
if (Arg->isMasked()) {
// Don't bother inserting AssertZext for packed IDs since we're emitting the
// masking operations anyway.
//
// TODO: We could assert the top bit is 0 for the source copy.
if (!loadInputValue(DstReg, B, ArgType))
return false;
} else {
Register TmpReg = MRI.createGenericVirtualRegister(LLT::integer(32));
if (!loadInputValue(TmpReg, B, ArgType))
return false;
B.buildAssertZExt(DstReg, TmpReg, llvm::bit_width(MaxID));
}
MI.eraseFromParent();
return true;
}
MachinePointerInfo
AMDGPULegalizerInfo::getKernargSegmentPtrInfo(MachineFunction &MF) const {
// This isn't really a constant pool but close enough.
MachinePointerInfo PtrInfo(MF.getPSVManager().getConstantPool());
PtrInfo.AddrSpace = AMDGPUAS::CONSTANT_ADDRESS;
return PtrInfo;
}
Register AMDGPULegalizerInfo::getKernargParameterPtr(MachineIRBuilder &B,
int64_t Offset) const {
LLT PtrTy = LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64);
Register KernArgReg = B.getMRI()->createGenericVirtualRegister(PtrTy);
// TODO: If we passed in the base kernel offset we could have a better
// alignment than 4, but we don't really need it.
if (!loadInputValue(KernArgReg, B,
AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR))
llvm_unreachable("failed to find kernarg segment ptr");
auto COffset = B.buildConstant(LLT::integer(64), Offset);
return B.buildObjectPtrOffset(PtrTy, KernArgReg, COffset).getReg(0);
}
/// Legalize a value that's loaded from kernel arguments. This is only used by
/// legacy intrinsics.
bool AMDGPULegalizerInfo::legalizeKernargMemParameter(MachineInstr &MI,
MachineIRBuilder &B,
uint64_t Offset,
Align Alignment) const {
Register DstReg = MI.getOperand(0).getReg();
assert(B.getMRI()->getType(DstReg) == LLT::integer(32) &&
"unexpected kernarg parameter type");
Register Ptr = getKernargParameterPtr(B, Offset);
MachinePointerInfo PtrInfo = getKernargSegmentPtrInfo(B.getMF());
B.buildLoad(DstReg, Ptr, PtrInfo.getWithOffset(Offset), Align(4),
MachineMemOperand::MODereferenceable |
MachineMemOperand::MOInvariant);
MI.eraseFromParent();
return true;
}
bool AMDGPULegalizerInfo::legalizeFDIV(MachineInstr &MI,
MachineRegisterInfo &MRI,
MachineIRBuilder &B) const {
Register Dst = MI.getOperand(0).getReg();
LLT DstTy = MRI.getType(Dst);
if (DstTy == F16)
return legalizeFDIV16(MI, MRI, B);
if (DstTy == F32)
return legalizeFDIV32(MI, MRI, B);
if (DstTy == F64)
return legalizeFDIV64(MI, MRI, B);
return false;
}
void AMDGPULegalizerInfo::legalizeUnsignedDIV_REM32Impl(MachineIRBuilder &B,
Register DstDivReg,
Register DstRemReg,
Register X,
Register Y) const {
const LLT S1 = LLT::scalar(1);
const LLT I32 = LLT::integer(32);
// See AMDGPUCodeGenPrepare::expandDivRem32 for a description of the
// algorithm used here.
// Initial estimate of inv(y).
auto FloatY = B.buildUITOFP(F32, Y);
auto RcpIFlag = B.buildInstr(AMDGPU::G_AMDGPU_RCP_IFLAG, {F32}, {FloatY});
auto Scale = B.buildFConstant(F32, llvm::bit_cast<float>(0x4f7ffffe));
auto ScaledY = B.buildFMul(F32, RcpIFlag, Scale);
auto Z = B.buildFPTOUI(I32, ScaledY);
// One round of UNR.
auto NegY = B.buildSub(I32, B.buildConstant(I32, 0), Y);
auto NegYZ = B.buildMul(I32, NegY, Z);
Z = B.buildAdd(I32, Z, B.buildUMulH(I32, Z, NegYZ));
// Quotient/remainder estimate.
auto Q = B.buildUMulH(I32, X, Z);
auto R = B.buildSub(I32, X, B.buildMul(I32, Q, Y));
// First quotient/remainder refinement.
auto One = B.buildConstant(I32, 1);
auto Cond = B.buildICmp(CmpInst::ICMP_UGE, S1, R, Y);
if (DstDivReg)
Q = B.buildSelect(I32, Cond, B.buildAdd(I32, Q, One), Q);
R = B.buildSelect(I32, Cond, B.buildSub(I32, R, Y), R);
// Second quotient/remainder refinement.
Cond = B.buildICmp(CmpInst::ICMP_UGE, S1, R, Y);
if (DstDivReg)
B.buildSelect(DstDivReg, Cond, B.buildAdd(I32, Q, One), Q);
if (DstRemReg)
B.buildSelect(DstRemReg, Cond, B.buildSub(I32, R, Y), R);
}
// Build integer reciprocal sequence around V_RCP_IFLAG_F32
//
// Return lo, hi of result
//
// %cvt.lo = G_UITOFP Val.lo
// %cvt.hi = G_UITOFP Val.hi
// %mad = G_FMAD %cvt.hi, 2**32, %cvt.lo
// %rcp = G_AMDGPU_RCP_IFLAG %mad
// %mul1 = G_FMUL %rcp, 0x5f7ffffc
// %mul2 = G_FMUL %mul1, 2**(-32)
// %trunc = G_INTRINSIC_TRUNC %mul2
// %mad2 = G_FMAD %trunc, -(2**32), %mul1
// return {G_FPTOUI %mad2, G_FPTOUI %trunc}
static std::pair<Register, Register> emitReciprocalU64(MachineIRBuilder &B,
Register Val) {
const LLT I32 = LLT::integer(32);
auto Unmerge = B.buildUnmerge(I32, Val);
auto CvtLo = B.buildUITOFP(F32, Unmerge.getReg(0));
auto CvtHi = B.buildUITOFP(F32, Unmerge.getReg(1));
auto Mad = B.buildFMAD(
F32, CvtHi, // 2**32
B.buildFConstant(F32, llvm::bit_cast<float>(0x4f800000)), CvtLo);
auto Rcp = B.buildInstr(AMDGPU::G_AMDGPU_RCP_IFLAG, {F32}, {Mad});
auto Mul1 = B.buildFMul(
F32, Rcp, B.buildFConstant(F32, llvm::bit_cast<float>(0x5f7ffffc)));
// 2**(-32)
auto Mul2 = B.buildFMul(
F32, Mul1, B.buildFConstant(F32, llvm::bit_cast<float>(0x2f800000)));
auto Trunc = B.buildIntrinsicTrunc(F32, Mul2);
// -(2**32)
auto Mad2 = B.buildFMAD(
F32, Trunc, B.buildFConstant(F32, llvm::bit_cast<float>(0xcf800000)),
Mul1);
auto ResultLo = B.buildFPTOUI(I32, Mad2);
auto ResultHi = B.buildFPTOUI(I32, Trunc);
return {ResultLo.getReg(0), ResultHi.getReg(0)};
}
void AMDGPULegalizerInfo::legalizeUnsignedDIV_REM64Impl(MachineIRBuilder &B,
Register DstDivReg,
Register DstRemReg,
Register Numer,
Register Denom) const {
const LLT I32 = LLT::integer(32);
const LLT I64 = LLT::integer(64);
const LLT S1 = LLT::scalar(1);
Register RcpLo, RcpHi;
std::tie(RcpLo, RcpHi) = emitReciprocalU64(B, Denom);
auto Rcp = B.buildMergeLikeInstr(I64, {RcpLo, RcpHi});
auto Zero64 = B.buildConstant(I64, 0);
auto NegDenom = B.buildSub(I64, Zero64, Denom);
auto MulLo1 = B.buildMul(I64, NegDenom, Rcp);
auto MulHi1 = B.buildUMulH(I64, Rcp, MulLo1);
auto UnmergeMulHi1 = B.buildUnmerge(I32, MulHi1);
Register MulHi1_Lo = UnmergeMulHi1.getReg(0);
Register MulHi1_Hi = UnmergeMulHi1.getReg(1);
auto Add1_Lo = B.buildUAddo(I32, S1, RcpLo, MulHi1_Lo);
auto Add1_Hi = B.buildUAdde(I32, S1, RcpHi, MulHi1_Hi, Add1_Lo.getReg(1));
auto Add1 = B.buildMergeLikeInstr(I64, {Add1_Lo, Add1_Hi});
auto MulLo2 = B.buildMul(I64, NegDenom, Add1);
auto MulHi2 = B.buildUMulH(I64, Add1, MulLo2);
auto UnmergeMulHi2 = B.buildUnmerge(I32, MulHi2);
Register MulHi2_Lo = UnmergeMulHi2.getReg(0);
Register MulHi2_Hi = UnmergeMulHi2.getReg(1);
auto Zero32 = B.buildConstant(I32, 0);
auto Add2_Lo = B.buildUAddo(I32, S1, Add1_Lo, MulHi2_Lo);
auto Add2_Hi = B.buildUAdde(I32, S1, Add1_Hi, MulHi2_Hi, Add2_Lo.getReg(1));
auto Add2 = B.buildMergeLikeInstr(I64, {Add2_Lo, Add2_Hi});
auto UnmergeNumer = B.buildUnmerge(I32, Numer);
Register NumerLo = UnmergeNumer.getReg(0);
Register NumerHi = UnmergeNumer.getReg(1);
auto MulHi3 = B.buildUMulH(I64, Numer, Add2);
auto Mul3 = B.buildMul(I64, Denom, MulHi3);
auto UnmergeMul3 = B.buildUnmerge(I32, Mul3);
Register Mul3_Lo = UnmergeMul3.getReg(0);
Register Mul3_Hi = UnmergeMul3.getReg(1);
auto Sub1_Lo = B.buildUSubo(I32, S1, NumerLo, Mul3_Lo);
auto Sub1_Hi = B.buildUSube(I32, S1, NumerHi, Mul3_Hi, Sub1_Lo.getReg(1));
auto Sub1_Mi = B.buildSub(I32, NumerHi, Mul3_Hi);
auto Sub1 = B.buildMergeLikeInstr(I64, {Sub1_Lo, Sub1_Hi});
auto UnmergeDenom = B.buildUnmerge(I32, Denom);
Register DenomLo = UnmergeDenom.getReg(0);
Register DenomHi = UnmergeDenom.getReg(1);
auto CmpHi = B.buildICmp(CmpInst::ICMP_UGE, S1, Sub1_Hi, DenomHi);
auto C1 = B.buildSExt(I32, CmpHi);
auto CmpLo = B.buildICmp(CmpInst::ICMP_UGE, S1, Sub1_Lo, DenomLo);
auto C2 = B.buildSExt(I32, CmpLo);
auto CmpEq = B.buildICmp(CmpInst::ICMP_EQ, S1, Sub1_Hi, DenomHi);
auto C3 = B.buildSelect(I32, CmpEq, C2, C1);
// TODO: Here and below portions of the code can be enclosed into if/endif.
// Currently control flow is unconditional and we have 4 selects after
// potential endif to substitute PHIs.
// if C3 != 0 ...
auto Sub2_Lo = B.buildUSubo(I32, S1, Sub1_Lo, DenomLo);
auto Sub2_Mi = B.buildUSube(I32, S1, Sub1_Mi, DenomHi, Sub1_Lo.getReg(1));
auto Sub2_Hi = B.buildUSube(I32, S1, Sub2_Mi, Zero32, Sub2_Lo.getReg(1));
auto Sub2 = B.buildMergeLikeInstr(I64, {Sub2_Lo, Sub2_Hi});
auto One64 = B.buildConstant(I64, 1);
auto Add3 = B.buildAdd(I64, MulHi3, One64);
auto C4 =
B.buildSExt(I32, B.buildICmp(CmpInst::ICMP_UGE, S1, Sub2_Hi, DenomHi));
auto C5 =
B.buildSExt(I32, B.buildICmp(CmpInst::ICMP_UGE, S1, Sub2_Lo, DenomLo));
auto C6 = B.buildSelect(
I32, B.buildICmp(CmpInst::ICMP_EQ, S1, Sub2_Hi, DenomHi), C5, C4);
// if (C6 != 0)
auto Add4 = B.buildAdd(I64, Add3, One64);
auto Sub3_Lo = B.buildUSubo(I32, S1, Sub2_Lo, DenomLo);
auto Sub3_Mi = B.buildUSube(I32, S1, Sub2_Mi, DenomHi, Sub2_Lo.getReg(1));
auto Sub3_Hi = B.buildUSube(I32, S1, Sub3_Mi, Zero32, Sub3_Lo.getReg(1));
auto Sub3 = B.buildMergeLikeInstr(I64, {Sub3_Lo, Sub3_Hi});
// endif C6
// endif C3
if (DstDivReg) {
auto Sel1 = B.buildSelect(
I64, B.buildICmp(CmpInst::ICMP_NE, S1, C6, Zero32), Add4, Add3);
B.buildSelect(DstDivReg, B.buildICmp(CmpInst::ICMP_NE, S1, C3, Zero32),
Sel1, MulHi3);
}
if (DstRemReg) {
auto Sel2 = B.buildSelect(
I64, B.buildICmp(CmpInst::ICMP_NE, S1, C6, Zero32), Sub3, Sub2);
B.buildSelect(DstRemReg, B.buildICmp(CmpInst::ICMP_NE, S1, C3, Zero32),
Sel2, Sub1);
}
}
bool AMDGPULegalizerInfo::legalizeUnsignedDIV_REM(MachineInstr &MI,
MachineRegisterInfo &MRI,
MachineIRBuilder &B) const {
Register DstDivReg, DstRemReg;
switch (MI.getOpcode()) {
default:
llvm_unreachable("Unexpected opcode!");
case AMDGPU::G_UDIV: {
DstDivReg = MI.getOperand(0).getReg();
break;
}
case AMDGPU::G_UREM: {
DstRemReg = MI.getOperand(0).getReg();
break;
}
case AMDGPU::G_UDIVREM: {
DstDivReg = MI.getOperand(0).getReg();
DstRemReg = MI.getOperand(1).getReg();
break;
}
}
const LLT I64 = LLT::integer(64);
const LLT I32 = LLT::integer(32);
const unsigned FirstSrcOpIdx = MI.getNumExplicitDefs();
Register Num = MI.getOperand(FirstSrcOpIdx).getReg();
Register Den = MI.getOperand(FirstSrcOpIdx + 1).getReg();
LLT Ty = MRI.getType(MI.getOperand(0).getReg());
if (Ty == I32)
legalizeUnsignedDIV_REM32Impl(B, DstDivReg, DstRemReg, Num, Den);
else if (Ty == I64)
legalizeUnsignedDIV_REM64Impl(B, DstDivReg, DstRemReg, Num, Den);
else
return false;
MI.eraseFromParent();
return true;
}
bool AMDGPULegalizerInfo::legalizeSignedDIV_REM(MachineInstr &MI,
MachineRegisterInfo &MRI,
MachineIRBuilder &B) const {
const LLT I64 = LLT::integer(64);
const LLT I32 = LLT::integer(32);
LLT Ty = MRI.getType(MI.getOperand(0).getReg());
if (Ty != I32 && Ty != I64)
return false;
const unsigned FirstSrcOpIdx = MI.getNumExplicitDefs();
Register LHS = MI.getOperand(FirstSrcOpIdx).getReg();
Register RHS = MI.getOperand(FirstSrcOpIdx + 1).getReg();
auto SignBitOffset = B.buildConstant(I32, Ty.getSizeInBits() - 1);
auto LHSign = B.buildAShr(Ty, LHS, SignBitOffset);
auto RHSign = B.buildAShr(Ty, RHS, SignBitOffset);
LHS = B.buildAdd(Ty, LHS, LHSign).getReg(0);
RHS = B.buildAdd(Ty, RHS, RHSign).getReg(0);
LHS = B.buildXor(Ty, LHS, LHSign).getReg(0);
RHS = B.buildXor(Ty, RHS, RHSign).getReg(0);
Register DstDivReg, DstRemReg, TmpDivReg, TmpRemReg;
switch (MI.getOpcode()) {
default:
llvm_unreachable("Unexpected opcode!");
case AMDGPU::G_SDIV: {
DstDivReg = MI.getOperand(0).getReg();
TmpDivReg = MRI.createGenericVirtualRegister(Ty);
break;
}
case AMDGPU::G_SREM: {
DstRemReg = MI.getOperand(0).getReg();
TmpRemReg = MRI.createGenericVirtualRegister(Ty);
break;
}
case AMDGPU::G_SDIVREM: {
DstDivReg = MI.getOperand(0).getReg();
DstRemReg = MI.getOperand(1).getReg();
TmpDivReg = MRI.createGenericVirtualRegister(Ty);
TmpRemReg = MRI.createGenericVirtualRegister(Ty);
break;
}
}
if (Ty == I32)
legalizeUnsignedDIV_REM32Impl(B, TmpDivReg, TmpRemReg, LHS, RHS);
else
legalizeUnsignedDIV_REM64Impl(B, TmpDivReg, TmpRemReg, LHS, RHS);
if (DstDivReg) {
auto Sign = B.buildXor(Ty, LHSign, RHSign).getReg(0);
auto SignXor = B.buildXor(Ty, TmpDivReg, Sign).getReg(0);
B.buildSub(DstDivReg, SignXor, Sign);
}
if (DstRemReg) {
auto Sign = LHSign.getReg(0); // Remainder sign is the same as LHS
auto SignXor = B.buildXor(Ty, TmpRemReg, Sign).getReg(0);
B.buildSub(DstRemReg, SignXor, Sign);
}
MI.eraseFromParent();
return true;
}
bool AMDGPULegalizerInfo::legalizeFastUnsafeFDIV(MachineInstr &MI,
MachineRegisterInfo &MRI,
MachineIRBuilder &B) const {
Register Res = MI.getOperand(0).getReg();
Register LHS = MI.getOperand(1).getReg();
Register RHS = MI.getOperand(2).getReg();
uint16_t Flags = MI.getFlags();
LLT ResTy = MRI.getType(Res);
bool AllowInaccurateRcp = MI.getFlag(MachineInstr::FmAfn);
if (const auto *CLHS = getConstantFPVRegVal(LHS, MRI)) {
if (!AllowInaccurateRcp && ResTy != F16)
return false;
// v_rcp_f32 and v_rsq_f32 do not support denormals, and according to
// the CI documentation has a worst case error of 1 ulp.
// OpenCL requires <= 2.5 ulp for 1.0 / x, so it should always be OK to
// use it as long as we aren't trying to use denormals.
//
// v_rcp_f16 and v_rsq_f16 DO support denormals and 0.51ulp.
// 1 / x -> RCP(x)
if (CLHS->isOne()) {
B.buildIntrinsic(Intrinsic::amdgcn_rcp, Res)
.addUse(RHS)
.setMIFlags(Flags);
MI.eraseFromParent();
return true;
}
// -1 / x -> RCP( FNEG(x) )
if (CLHS->isMinusOne()) {
auto FNeg = B.buildFNeg(ResTy, RHS, Flags);
B.buildIntrinsic(Intrinsic::amdgcn_rcp, Res)
.addUse(FNeg.getReg(0))
.setMIFlags(Flags);
MI.eraseFromParent();
return true;
}
}
// For f16 require afn or arcp.
// For f32 require afn.
if (!AllowInaccurateRcp &&
(ResTy != F16 || !MI.getFlag(MachineInstr::FmArcp)))
return false;
// x / y -> x * (1.0 / y)
auto RCP = B.buildIntrinsic(Intrinsic::amdgcn_rcp, {ResTy})
.addUse(RHS)
.setMIFlags(Flags);
B.buildFMul(Res, LHS, RCP, Flags);
MI.eraseFromParent();
return true;
}
bool AMDGPULegalizerInfo::legalizeFastUnsafeFDIV64(MachineInstr &MI,
MachineRegisterInfo &MRI,
MachineIRBuilder &B) const {
Register Res = MI.getOperand(0).getReg();
Register X = MI.getOperand(1).getReg();
Register Y = MI.getOperand(2).getReg();
uint16_t Flags = MI.getFlags();
LLT ResTy = MRI.getType(Res);
bool AllowInaccurateRcp = MI.getFlag(MachineInstr::FmAfn);
if (!AllowInaccurateRcp)
return false;
const ConstantFP *CLHS = getConstantFPVRegVal(X, MRI);
bool IsNegRcp = CLHS && CLHS->isMinusOne();
// Pull out the negation so it folds for free into the source modifiers.
if (IsNegRcp)
X = B.buildFConstant(ResTy, 1.0).getReg(0);
Register NegY = IsNegRcp ? Y : B.buildFNeg(ResTy, Y).getReg(0);
auto One = B.buildFConstant(ResTy, 1.0);
auto R = B.buildIntrinsic(Intrinsic::amdgcn_rcp, {ResTy})
.addUse(Y)
.setMIFlags(Flags);
if (IsNegRcp)
R = B.buildFNeg(ResTy, R);
auto Tmp0 = B.buildFMA(ResTy, NegY, R, One);
R = B.buildFMA(ResTy, Tmp0, R, R);
auto Tmp1 = B.buildFMA(ResTy, NegY, R, One);
R = B.buildFMA(ResTy, Tmp1, R, R);
// Skip the last 2 correction terms for reciprocal.
if (IsNegRcp || (CLHS && CLHS->isOne())) {
B.buildCopy(Res, R);
MI.eraseFromParent();
return true;
}
auto Ret = B.buildFMul(ResTy, X, R);
auto Tmp2 = B.buildFMA(ResTy, NegY, Ret, X);
B.buildFMA(Res, Tmp2, R, Ret);
MI.eraseFromParent();
return true;
}
bool AMDGPULegalizerInfo::legalizeFDIV16(MachineInstr &MI,
MachineRegisterInfo &MRI,
MachineIRBuilder &B) const {
if (legalizeFastUnsafeFDIV(MI, MRI, B))
return true;
Register Res = MI.getOperand(0).getReg();
Register LHS = MI.getOperand(1).getReg();
Register RHS = MI.getOperand(2).getReg();
uint16_t Flags = MI.getFlags();
LLT I32 = LLT::integer(32);
// a32.u = opx(V_CVT_F32_F16, a.u); // CVT to F32
// b32.u = opx(V_CVT_F32_F16, b.u); // CVT to F32
// r32.u = opx(V_RCP_F32, b32.u); // rcp = 1 / d
// q32.u = opx(V_MUL_F32, a32.u, r32.u); // q = n * rcp
// e32.u = opx(V_MAD_F32, (b32.u^_neg32), q32.u, a32.u); // err = -d * q + n
// q32.u = opx(V_MAD_F32, e32.u, r32.u, q32.u); // q = n * rcp
// e32.u = opx(V_MAD_F32, (b32.u^_neg32), q32.u, a32.u); // err = -d * q + n
// tmp.u = opx(V_MUL_F32, e32.u, r32.u);
// tmp.u = opx(V_AND_B32, tmp.u, 0xff800000)
// q32.u = opx(V_ADD_F32, tmp.u, q32.u);
// q16.u = opx(V_CVT_F16_F32, q32.u);
// q16.u = opx(V_DIV_FIXUP_F16, q16.u, b.u, a.u); // q = touchup(q, d, n)
auto LHSExt = B.buildFPExt(F32, LHS, Flags);
auto RHSExt = B.buildFPExt(F32, RHS, Flags);
auto NegRHSExt = B.buildFNeg(F32, RHSExt);
auto Rcp = B.buildIntrinsic(Intrinsic::amdgcn_rcp, {F32})
.addUse(RHSExt.getReg(0))
.setMIFlags(Flags);
auto Quot = B.buildFMul(F32, LHSExt, Rcp, Flags);
MachineInstrBuilder Err;
if (ST.hasMadMacF32Insts()) {
Err = B.buildFMAD(F32, NegRHSExt, Quot, LHSExt, Flags);
Quot = B.buildFMAD(F32, Err, Rcp, Quot, Flags);
Err = B.buildFMAD(F32, NegRHSExt, Quot, LHSExt, Flags);
} else {
Err = B.buildFMA(F32, NegRHSExt, Quot, LHSExt, Flags);
Quot = B.buildFMA(F32, Err, Rcp, Quot, Flags);
Err = B.buildFMA(F32, NegRHSExt, Quot, LHSExt, Flags);
}
auto Tmp = B.buildFMul(F32, Err, Rcp, Flags);
auto TmpInt = B.buildBitcast(I32, Tmp);
auto MaskedInt = B.buildAnd(I32, TmpInt, B.buildConstant(I32, 0xff800000));
auto Masked = B.buildBitcast(F32, MaskedInt);
Quot = B.buildFAdd(F32, Masked, Quot, Flags);
auto RDst = B.buildFPTrunc(F16, Quot, Flags);
B.buildIntrinsic(Intrinsic::amdgcn_div_fixup, Res)
.addUse(RDst.getReg(0))
.addUse(RHS)
.addUse(LHS)
.setMIFlags(Flags);
MI.eraseFromParent();
return true;
}
static constexpr unsigned SPDenormModeBitField =
AMDGPU::Hwreg::HwregEncoding::encode(AMDGPU::Hwreg::ID_MODE, 4, 2);
// Enable or disable FP32 denorm mode. When 'Enable' is true, emit instructions
// to enable denorm mode. When 'Enable' is false, disable denorm mode.
static void toggleSPDenormMode(bool Enable, MachineIRBuilder &B,
const GCNSubtarget &ST,
SIModeRegisterDefaults Mode) {
// Set SP denorm mode to this value.
unsigned SPDenormMode =
Enable ? FP_DENORM_FLUSH_NONE : Mode.fpDenormModeSPValue();
if (ST.hasDenormModeInst()) {
// Preserve default FP64FP16 denorm mode while updating FP32 mode.
uint32_t DPDenormModeDefault = Mode.fpDenormModeDPValue();
uint32_t NewDenormModeValue = SPDenormMode | (DPDenormModeDefault << 2);
B.buildInstr(AMDGPU::S_DENORM_MODE)
.addImm(NewDenormModeValue);
} else {
B.buildInstr(AMDGPU::S_SETREG_IMM32_B32)
.addImm(SPDenormMode)
.addImm(SPDenormModeBitField);
}
}
bool AMDGPULegalizerInfo::legalizeFDIV32(MachineInstr &MI,
MachineRegisterInfo &MRI,
MachineIRBuilder &B) const {
if (legalizeFastUnsafeFDIV(MI, MRI, B))
return true;
Register Res = MI.getOperand(0).getReg();
Register LHS = MI.getOperand(1).getReg();
Register RHS = MI.getOperand(2).getReg();
const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
SIModeRegisterDefaults Mode = MFI->getMode();
uint16_t Flags = MI.getFlags();
LLT S1 = LLT::scalar(1);
auto One = B.buildFConstant(F32, 1.0f);
auto DenominatorScaled =
B.buildIntrinsic(Intrinsic::amdgcn_div_scale, {F32, S1})
.addUse(LHS)
.addUse(RHS)
.addImm(0)
.setMIFlags(Flags);
auto NumeratorScaled =
B.buildIntrinsic(Intrinsic::amdgcn_div_scale, {F32, S1})
.addUse(LHS)
.addUse(RHS)
.addImm(1)
.setMIFlags(Flags);
auto ApproxRcp = B.buildIntrinsic(Intrinsic::amdgcn_rcp, {F32})
.addUse(DenominatorScaled.getReg(0))
.setMIFlags(Flags);
auto NegDivScale0 = B.buildFNeg(F32, DenominatorScaled, Flags);
const bool PreservesDenormals = Mode.FP32Denormals == DenormalMode::getIEEE();
const bool HasDynamicDenormals =
(Mode.FP32Denormals.Input == DenormalMode::Dynamic) ||
(Mode.FP32Denormals.Output == DenormalMode::Dynamic);
Register SavedSPDenormMode;
if (!PreservesDenormals) {
if (HasDynamicDenormals) {
SavedSPDenormMode = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
B.buildInstr(AMDGPU::S_GETREG_B32)
.addDef(SavedSPDenormMode)
.addImm(SPDenormModeBitField);
}
toggleSPDenormMode(true, B, ST, Mode);
}
auto Fma0 = B.buildFMA(F32, NegDivScale0, ApproxRcp, One, Flags);
auto Fma1 = B.buildFMA(F32, Fma0, ApproxRcp, ApproxRcp, Flags);
auto Mul = B.buildFMul(F32, NumeratorScaled, Fma1, Flags);
auto Fma2 = B.buildFMA(F32, NegDivScale0, Mul, NumeratorScaled, Flags);
auto Fma3 = B.buildFMA(F32, Fma2, Fma1, Mul, Flags);
auto Fma4 = B.buildFMA(F32, NegDivScale0, Fma3, NumeratorScaled, Flags);
if (!PreservesDenormals) {
if (HasDynamicDenormals) {
assert(SavedSPDenormMode);
B.buildInstr(AMDGPU::S_SETREG_B32)
.addReg(SavedSPDenormMode)
.addImm(SPDenormModeBitField);
} else
toggleSPDenormMode(false, B, ST, Mode);
}
auto Fmas = B.buildIntrinsic(Intrinsic::amdgcn_div_fmas, {F32})
.addUse(Fma4.getReg(0))
.addUse(Fma1.getReg(0))
.addUse(Fma3.getReg(0))
.addUse(NumeratorScaled.getReg(1))
.setMIFlags(Flags);
B.buildIntrinsic(Intrinsic::amdgcn_div_fixup, Res)
.addUse(Fmas.getReg(0))
.addUse(RHS)
.addUse(LHS)
.setMIFlags(Flags);
MI.eraseFromParent();
return true;
}
bool AMDGPULegalizerInfo::legalizeFDIV64(MachineInstr &MI,
MachineRegisterInfo &MRI,
MachineIRBuilder &B) const {
if (legalizeFastUnsafeFDIV64(MI, MRI, B))
return true;
Register Res = MI.getOperand(0).getReg();
Register LHS = MI.getOperand(1).getReg();
Register RHS = MI.getOperand(2).getReg();
uint16_t Flags = MI.getFlags();
LLT S1 = LLT::scalar(1);
auto One = B.buildFConstant(F64, 1.0);
auto DivScale0 = B.buildIntrinsic(Intrinsic::amdgcn_div_scale, {F64, S1})
.addUse(LHS)
.addUse(RHS)
.addImm(0)
.setMIFlags(Flags);
auto NegDivScale0 = B.buildFNeg(F64, DivScale0.getReg(0), Flags);
auto Rcp = B.buildIntrinsic(Intrinsic::amdgcn_rcp, {F64})
.addUse(DivScale0.getReg(0))
.setMIFlags(Flags);
auto Fma0 = B.buildFMA(F64, NegDivScale0, Rcp, One, Flags);
auto Fma1 = B.buildFMA(F64, Rcp, Fma0, Rcp, Flags);
auto Fma2 = B.buildFMA(F64, NegDivScale0, Fma1, One, Flags);
auto DivScale1 = B.buildIntrinsic(Intrinsic::amdgcn_div_scale, {F64, S1})
.addUse(LHS)
.addUse(RHS)
.addImm(1)
.setMIFlags(Flags);
auto Fma3 = B.buildFMA(F64, Fma1, Fma2, Fma1, Flags);
auto Mul = B.buildFMul(F64, DivScale1.getReg(0), Fma3, Flags);
auto Fma4 = B.buildFMA(F64, NegDivScale0, Mul, DivScale1.getReg(0), Flags);
Register Scale;
if (!ST.hasUsableDivScaleConditionOutput()) {
// Workaround a hardware bug on SI where the condition output from div_scale
// is not usable.
LLT I32 = LLT::integer(32);
LLT I64 = LLT::integer(64);
auto NumUnmerge = B.buildUnmerge(I32, B.buildBitcast(I64, LHS));
auto DenUnmerge = B.buildUnmerge(I32, B.buildBitcast(I64, RHS));
auto Scale0Unmerge = B.buildUnmerge(I32, B.buildBitcast(I64, DivScale0));
auto Scale1Unmerge = B.buildUnmerge(I32, B.buildBitcast(I64, DivScale1));
auto CmpNum = B.buildICmp(ICmpInst::ICMP_EQ, S1, NumUnmerge.getReg(1),
Scale1Unmerge.getReg(1));
auto CmpDen = B.buildICmp(ICmpInst::ICMP_EQ, S1, DenUnmerge.getReg(1),
Scale0Unmerge.getReg(1));
Scale = B.buildXor(S1, CmpNum, CmpDen).getReg(0);
} else {
Scale = DivScale1.getReg(1);
}
auto Fmas = B.buildIntrinsic(Intrinsic::amdgcn_div_fmas, {F64})
.addUse(Fma4.getReg(0))
.addUse(Fma3.getReg(0))
.addUse(Mul.getReg(0))
.addUse(Scale)
.setMIFlags(Flags);
B.buildIntrinsic(Intrinsic::amdgcn_div_fixup, ArrayRef(Res))
.addUse(Fmas.getReg(0))
.addUse(RHS)
.addUse(LHS)
.setMIFlags(Flags);
MI.eraseFromParent();
return true;
}
bool AMDGPULegalizerInfo::legalizeFFREXP(MachineInstr &MI,
MachineRegisterInfo &MRI,
MachineIRBuilder &B) const {
Register Res0 = MI.getOperand(0).getReg();
Register Res1 = MI.getOperand(1).getReg();
Register Val = MI.getOperand(2).getReg();
uint16_t Flags = MI.getFlags();
LLT Ty = MRI.getType(Res0);
LLT InstrExpTy = Ty == F16 ? LLT::integer(16) : LLT::integer(32);
auto Mant = B.buildIntrinsic(Intrinsic::amdgcn_frexp_mant, {Ty})
.addUse(Val)
.setMIFlags(Flags);
auto Exp = B.buildIntrinsic(Intrinsic::amdgcn_frexp_exp, {InstrExpTy})
.addUse(Val)
.setMIFlags(Flags);
if (ST.hasFractBug()) {
auto Fabs = B.buildFAbs(Ty, Val);
auto Inf = B.buildFConstant(Ty, APFloat::getInf(getFltSemanticForLLT(Ty)));
auto IsFinite =
B.buildFCmp(CmpInst::FCMP_OLT, LLT::scalar(1), Fabs, Inf, Flags);
auto Zero = B.buildConstant(InstrExpTy, 0);
Exp = B.buildSelect(InstrExpTy, IsFinite, Exp, Zero);
Mant = B.buildSelect(Ty, IsFinite, Mant, Val);
}
B.buildCopy(Res0, Mant);
B.buildSExtOrTrunc(Res1, Exp);
MI.eraseFromParent();
return true;
}
bool AMDGPULegalizerInfo::legalizeFDIVFastIntrin(MachineInstr &MI,
MachineRegisterInfo &MRI,
MachineIRBuilder &B) const {
Register Res = MI.getOperand(0).getReg();
Register LHS = MI.getOperand(2).getReg();
Register RHS = MI.getOperand(3).getReg();
uint16_t Flags = MI.getFlags();
LLT S1 = LLT::scalar(1);
auto Abs = B.buildFAbs(F32, RHS, Flags);
const APFloat C0Val(1.0f);
auto C0 = B.buildFConstant(F32, 0x1p+96f);
auto C1 = B.buildFConstant(F32, 0x1p-32f);
auto C2 = B.buildFConstant(F32, 1.0f);
auto CmpRes = B.buildFCmp(CmpInst::FCMP_OGT, S1, Abs, C0, Flags);
auto Sel = B.buildSelect(F32, CmpRes, C1, C2, Flags);
auto Mul0 = B.buildFMul(F32, RHS, Sel, Flags);
auto RCP = B.buildIntrinsic(Intrinsic::amdgcn_rcp, {F32})
.addUse(Mul0.getReg(0))
.setMIFlags(Flags);
auto Mul1 = B.buildFMul(F32, LHS, RCP, Flags);
B.buildFMul(Res, Sel, Mul1, Flags);
MI.eraseFromParent();
return true;
}
bool AMDGPULegalizerInfo::legalizeFSQRTF16(MachineInstr &MI,
MachineRegisterInfo &MRI,
MachineIRBuilder &B) const {
// Bypass the correct expansion a standard promotion through G_FSQRT would
// get. The f32 op is accurate enough for the f16 cas.
unsigned Flags = MI.getFlags();
assert(!ST.has16BitInsts());
auto Ext = B.buildFPExt(F32, MI.getOperand(1), Flags);
auto Log2 = B.buildIntrinsic(Intrinsic::amdgcn_sqrt, {F32})
.addUse(Ext.getReg(0))
.setMIFlags(Flags);
B.buildFPTrunc(MI.getOperand(0), Log2, Flags);
MI.eraseFromParent();
return true;
}
bool AMDGPULegalizerInfo::legalizeFSQRTF32(MachineInstr &MI,
MachineRegisterInfo &MRI,
MachineIRBuilder &B) const {
MachineFunction &MF = B.getMF();
Register Dst = MI.getOperand(0).getReg();
Register X = MI.getOperand(1).getReg();
const unsigned Flags = MI.getFlags();
const LLT I1 = LLT::integer(1);
const LLT I32 = LLT::integer(32);
if (allowApproxFunc(MF, Flags)) {
B.buildIntrinsic(Intrinsic::amdgcn_sqrt, ArrayRef<Register>({Dst}))
.addUse(X)
.setMIFlags(Flags);
MI.eraseFromParent();
return true;
}
auto ScaleThreshold = B.buildFConstant(F32, 0x1.0p-96f);
auto NeedScale = B.buildFCmp(CmpInst::FCMP_OGT, I1, ScaleThreshold, X, Flags);
auto ScaleUpFactor = B.buildFConstant(F32, 0x1.0p+32f);
auto ScaledX = B.buildFMul(F32, X, ScaleUpFactor, Flags);
auto SqrtX = B.buildSelect(F32, NeedScale, ScaledX, X, Flags);
Register SqrtS = MRI.createGenericVirtualRegister(F32);
if (needsDenormHandlingF32(MF, X, Flags)) {
B.buildIntrinsic(Intrinsic::amdgcn_sqrt, ArrayRef<Register>({SqrtS}))
.addUse(SqrtX.getReg(0))
.setMIFlags(Flags);
auto SqrtSInt = B.buildBitcast(I32, SqrtS);
auto NegOne = B.buildConstant(I32, -1);
auto SqrtSNextDown = B.buildBitcast(F32, B.buildAdd(I32, SqrtSInt, NegOne));
auto NegSqrtSNextDown = B.buildFNeg(F32, SqrtSNextDown, Flags);
auto SqrtVP = B.buildFMA(F32, NegSqrtSNextDown, SqrtS, SqrtX, Flags);
auto PosOne = B.buildConstant(I32, 1);
auto SqrtSNextUp = B.buildBitcast(F32, B.buildAdd(I32, SqrtSInt, PosOne));
auto NegSqrtSNextUp = B.buildFNeg(F32, SqrtSNextUp, Flags);
auto SqrtVS = B.buildFMA(F32, NegSqrtSNextUp, SqrtS, SqrtX, Flags);
auto Zero = B.buildFConstant(F32, 0.0f);
auto SqrtVPLE0 = B.buildFCmp(CmpInst::FCMP_OLE, I1, SqrtVP, Zero, Flags);
SqrtS =
B.buildSelect(F32, SqrtVPLE0, SqrtSNextDown, SqrtS, Flags).getReg(0);
auto SqrtVPVSGT0 = B.buildFCmp(CmpInst::FCMP_OGT, I1, SqrtVS, Zero, Flags);
SqrtS =
B.buildSelect(F32, SqrtVPVSGT0, SqrtSNextUp, SqrtS, Flags).getReg(0);
} else {
auto SqrtR =
B.buildIntrinsic(Intrinsic::amdgcn_rsq, {F32}).addReg(SqrtX.getReg(0));
B.buildFMul(SqrtS, SqrtX, SqrtR, Flags);
auto Half = B.buildFConstant(F32, 0.5f);
auto SqrtH = B.buildFMul(F32, SqrtR, Half, Flags);
auto NegSqrtH = B.buildFNeg(F32, SqrtH, Flags);
auto SqrtE = B.buildFMA(F32, NegSqrtH, SqrtS, Half, Flags);
SqrtH = B.buildFMA(F32, SqrtH, SqrtE, SqrtH, Flags);
SqrtS = B.buildFMA(F32, SqrtS, SqrtE, SqrtS, Flags).getReg(0);
auto NegSqrtS = B.buildFNeg(F32, SqrtS, Flags);
auto SqrtD = B.buildFMA(F32, NegSqrtS, SqrtS, SqrtX, Flags);
SqrtS = B.buildFMA(F32, SqrtD, SqrtH, SqrtS, Flags).getReg(0);
}
auto ScaleDownFactor = B.buildFConstant(F32, 0x1.0p-16f);
auto ScaledDown = B.buildFMul(F32, SqrtS, ScaleDownFactor, Flags);
SqrtS = B.buildSelect(F32, NeedScale, ScaledDown, SqrtS, Flags).getReg(0);
auto IsZeroOrInf = B.buildIsFPClass(I1, SqrtX, fcZero | fcPosInf);
B.buildSelect(Dst, IsZeroOrInf, SqrtX, SqrtS, Flags);
MI.eraseFromParent();
return true;
}
bool AMDGPULegalizerInfo::legalizeFSQRTF64(MachineInstr &MI,
MachineRegisterInfo &MRI,
MachineIRBuilder &B) const {
// For double type, the SQRT and RSQ instructions don't have required
// precision, we apply Goldschmidt's algorithm to improve the result:
//
// y0 = rsq(x)
// g0 = x * y0
// h0 = 0.5 * y0
//
// r0 = 0.5 - h0 * g0
// g1 = g0 * r0 + g0
// h1 = h0 * r0 + h0
//
// r1 = 0.5 - h1 * g1 => d0 = x - g1 * g1
// g2 = g1 * r1 + g1 g2 = d0 * h1 + g1
// h2 = h1 * r1 + h1
//
// r2 = 0.5 - h2 * g2 => d1 = x - g2 * g2
// g3 = g2 * r2 + g2 g3 = d1 * h1 + g2
//
// sqrt(x) = g3
const LLT I1 = LLT::integer(1);
const LLT I32 = LLT::integer(32);
Register Dst = MI.getOperand(0).getReg();
assert(MRI.getType(Dst) == F64 && "only expect to lower f64 sqrt");
Register X = MI.getOperand(1).getReg();
unsigned Flags = MI.getFlags();
Register SqrtX = X;
Register Scaling, ZeroInt;
if (!MI.getFlag(MachineInstr::FmAfn)) {
auto ScaleConstant = B.buildFConstant(F64, 0x1.0p-767);
ZeroInt = B.buildConstant(I32, 0).getReg(0);
Scaling = B.buildFCmp(FCmpInst::FCMP_OLT, I1, X, ScaleConstant).getReg(0);
// Scale up input if it is too small.
auto ScaleUpFactor = B.buildConstant(I32, 256);
auto ScaleUp = B.buildSelect(I32, Scaling, ScaleUpFactor, ZeroInt);
SqrtX = B.buildFLdexp(F64, X, ScaleUp, Flags).getReg(0);
}
auto SqrtY = B.buildIntrinsic(Intrinsic::amdgcn_rsq, {F64}).addReg(SqrtX);
auto Half = B.buildFConstant(F64, 0.5);
auto SqrtH0 = B.buildFMul(F64, SqrtY, Half);
auto SqrtS0 = B.buildFMul(F64, SqrtX, SqrtY);
auto NegSqrtH0 = B.buildFNeg(F64, SqrtH0);
auto SqrtR0 = B.buildFMA(F64, NegSqrtH0, SqrtS0, Half);
auto SqrtS1 = B.buildFMA(F64, SqrtS0, SqrtR0, SqrtS0);
auto SqrtH1 = B.buildFMA(F64, SqrtH0, SqrtR0, SqrtH0);
auto NegSqrtS1 = B.buildFNeg(F64, SqrtS1);
auto SqrtD0 = B.buildFMA(F64, NegSqrtS1, SqrtS1, SqrtX);
auto SqrtS2 = B.buildFMA(F64, SqrtD0, SqrtH1, SqrtS1);
Register SqrtRet = SqrtS2.getReg(0);
if (!MI.getFlag(MachineInstr::FmAfn)) {
auto NegSqrtS2 = B.buildFNeg(F64, SqrtS2);
auto SqrtD1 = B.buildFMA(F64, NegSqrtS2, SqrtS2, SqrtX);
auto SqrtD2 = B.buildFMA(F64, SqrtD1, SqrtH1, SqrtS2);
// Scale down the result.
auto ScaleDownFactor = B.buildConstant(I32, -128);
auto ScaleDown = B.buildSelect(I32, Scaling, ScaleDownFactor, ZeroInt);
SqrtRet = B.buildFLdexp(F64, SqrtD2, ScaleDown, Flags).getReg(0);
}
Register IsZeroOrInf;
if (MI.getFlag(MachineInstr::FmNoInfs)) {
auto ZeroFP = B.buildFConstant(F64, 0.0);
IsZeroOrInf = B.buildFCmp(FCmpInst::FCMP_OEQ, I1, SqrtX, ZeroFP).getReg(0);
} else {
IsZeroOrInf = B.buildIsFPClass(I1, SqrtX, fcZero | fcPosInf).getReg(0);
}
// TODO: Check for DAZ and expand to subnormals
// If x is +INF, +0, or -0, use its original value
B.buildSelect(Dst, IsZeroOrInf, SqrtX, SqrtRet, Flags);
MI.eraseFromParent();
return true;
}
bool AMDGPULegalizerInfo::legalizeFSQRT(MachineInstr &MI,
MachineRegisterInfo &MRI,
MachineIRBuilder &B) const {
LLT Ty = MRI.getType(MI.getOperand(0).getReg());
if (Ty == F32)
return legalizeFSQRTF32(MI, MRI, B);
if (Ty == F64)
return legalizeFSQRTF64(MI, MRI, B);
if (Ty == F16)
return legalizeFSQRTF16(MI, MRI, B);
return false;
}
// Expand llvm.amdgcn.rsq.clamp on targets that don't support the instruction.
// FIXME: Why do we handle this one but not other removed instructions?
//
// Reciprocal square root. The clamp prevents infinite results, clamping
// infinities to max_float. D.f = 1.0 / sqrt(S0.f), result clamped to
// +-max_float.
bool AMDGPULegalizerInfo::legalizeRsqClampIntrinsic(MachineInstr &MI,
MachineRegisterInfo &MRI,
MachineIRBuilder &B) const {
if (ST.getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS)
return true;
Register Dst = MI.getOperand(0).getReg();
Register Src = MI.getOperand(2).getReg();
auto Flags = MI.getFlags();
LLT Ty = MRI.getType(Dst);
const fltSemantics *FltSemantics;
if (Ty == F32)
FltSemantics = &APFloat::IEEEsingle();
else if (Ty == F64)
FltSemantics = &APFloat::IEEEdouble();
else
return false;
auto Rsq = B.buildIntrinsic(Intrinsic::amdgcn_rsq, {Ty})
.addUse(Src)
.setMIFlags(Flags);
// We don't need to concern ourselves with the snan handling difference, since
// the rsq quieted (or not) so use the one which will directly select.
const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
const bool UseIEEE = MFI->getMode().IEEE;
auto MaxFlt = B.buildFConstant(Ty, APFloat::getLargest(*FltSemantics));
auto ClampMax = UseIEEE ? B.buildFMinNumIEEE(Ty, Rsq, MaxFlt, Flags) :
B.buildFMinNum(Ty, Rsq, MaxFlt, Flags);
auto MinFlt = B.buildFConstant(Ty, APFloat::getLargest(*FltSemantics, true));
if (UseIEEE)
B.buildFMaxNumIEEE(Dst, ClampMax, MinFlt, Flags);
else
B.buildFMaxNum(Dst, ClampMax, MinFlt, Flags);
MI.eraseFromParent();
return true;
}
// TODO: Fix pointer type handling
bool AMDGPULegalizerInfo::legalizeLaneOp(LegalizerHelper &Helper,
MachineInstr &MI,
Intrinsic::ID IID) const {
MachineIRBuilder &B = Helper.MIRBuilder;
MachineRegisterInfo &MRI = *B.getMRI();
bool IsPermLane16 = IID == Intrinsic::amdgcn_permlane16 ||
IID == Intrinsic::amdgcn_permlanex16;
bool IsSetInactive = IID == Intrinsic::amdgcn_set_inactive ||
IID == Intrinsic::amdgcn_set_inactive_chain_arg;
bool IsPermlaneShuffle = IID == Intrinsic::amdgcn_permlane_bcast ||
IID == Intrinsic::amdgcn_permlane_up ||
IID == Intrinsic::amdgcn_permlane_down ||
IID == Intrinsic::amdgcn_permlane_xor;
auto createLaneOp = [&IID, &B, &MI](Register Src0, Register Src1,
Register Src2, LLT VT) -> Register {
auto LaneOp = B.buildIntrinsic(IID, {VT}).addUse(Src0);
switch (IID) {
case Intrinsic::amdgcn_readfirstlane:
case Intrinsic::amdgcn_permlane64:
return LaneOp.getReg(0);
case Intrinsic::amdgcn_readlane:
case Intrinsic::amdgcn_set_inactive:
case Intrinsic::amdgcn_set_inactive_chain_arg:
return LaneOp.addUse(Src1).getReg(0);
case Intrinsic::amdgcn_writelane:
case Intrinsic::amdgcn_permlane_bcast:
case Intrinsic::amdgcn_permlane_up:
case Intrinsic::amdgcn_permlane_down:
case Intrinsic::amdgcn_permlane_xor:
return LaneOp.addUse(Src1).addUse(Src2).getReg(0);
case Intrinsic::amdgcn_permlane16:
case Intrinsic::amdgcn_permlanex16: {
Register Src3 = MI.getOperand(5).getReg();
int64_t Src4 = MI.getOperand(6).getImm();
int64_t Src5 = MI.getOperand(7).getImm();
return LaneOp.addUse(Src1)
.addUse(Src2)
.addUse(Src3)
.addImm(Src4)
.addImm(Src5)
.getReg(0);
}
case Intrinsic::amdgcn_mov_dpp8:
return LaneOp.addImm(MI.getOperand(3).getImm()).getReg(0);
case Intrinsic::amdgcn_update_dpp:
return LaneOp.addUse(Src1)
.addImm(MI.getOperand(4).getImm())
.addImm(MI.getOperand(5).getImm())
.addImm(MI.getOperand(6).getImm())
.addImm(MI.getOperand(7).getImm())
.getReg(0);
default:
llvm_unreachable("unhandled lane op");
}
};
Register DstReg = MI.getOperand(0).getReg();
Register Src0 = MI.getOperand(2).getReg();
Register Src1, Src2;
if (IID == Intrinsic::amdgcn_readlane || IID == Intrinsic::amdgcn_writelane ||
IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16 ||
IsPermlaneShuffle) {
Src1 = MI.getOperand(3).getReg();
if (IID == Intrinsic::amdgcn_writelane || IsPermLane16 ||
IsPermlaneShuffle) {
Src2 = MI.getOperand(4).getReg();
}
}
LLT Ty = MRI.getType(DstReg);
unsigned Size = Ty.getSizeInBits();
unsigned SplitSize = 32;
if (IID == Intrinsic::amdgcn_update_dpp && (Size % 64 == 0) &&
ST.hasDPALU_DPP() &&
AMDGPU::isLegalDPALU_DPPControl(ST, MI.getOperand(4).getImm()))
SplitSize = 64;
if (Size == SplitSize) {
// Already legal
return true;
}
const LLT I32 = LLT::integer(32);
bool IsFloat = Ty.getScalarType().isFloat();
LLT IntTy = IsFloat ? LLT::integer(Size) : Ty;
if (IsFloat) {
Src0 = B.buildBitcast(IntTy, Src0).getReg(0);
if (Src1 && MRI.getType(Src1).getScalarType().isFloat())
Src1 = B.buildBitcast(IntTy, Src1).getReg(0);
if (Src2 && MRI.getType(Src2).getScalarType().isFloat())
Src2 = B.buildBitcast(IntTy, Src2).getReg(0);
}
if (Size < 32) {
Src0 = B.buildAnyExt(I32, Src0).getReg(0);
if (IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16)
Src1 = B.buildAnyExt(I32, Src1).getReg(0);
if (IID == Intrinsic::amdgcn_writelane)
Src2 = B.buildAnyExt(I32, Src2).getReg(0);
Register LaneOpDst = createLaneOp(Src0, Src1, Src2, I32);
if (IsFloat)
B.buildBitcast(DstReg, B.buildTrunc(IntTy, LaneOpDst));
else
B.buildTrunc(DstReg, LaneOpDst);
MI.eraseFromParent();
return true;
}
if (Size % SplitSize != 0)
return false;
LLT PartialResTy = LLT::integer(SplitSize);
bool NeedsBitcast = false;
if (IntTy.isVector()) {
LLT EltTy = IntTy.getElementType();
unsigned EltSize = EltTy.getSizeInBits();
if (EltSize == SplitSize) {
PartialResTy = EltTy;
} else if (EltSize == 16 || EltSize == 32) {
unsigned NElem = SplitSize / EltSize;
PartialResTy = IntTy.changeElementCount(ElementCount::getFixed(NElem));
} else {
NeedsBitcast = true;
}
}
SmallVector<Register, 4> PartialRes;
unsigned NumParts = Size / SplitSize;
MachineInstrBuilder Src0Parts = B.buildUnmerge(PartialResTy, Src0);
MachineInstrBuilder Src1Parts, Src2Parts;
if (IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16)
Src1Parts = B.buildUnmerge(PartialResTy, Src1);
if (IID == Intrinsic::amdgcn_writelane)
Src2Parts = B.buildUnmerge(PartialResTy, Src2);
for (unsigned i = 0; i < NumParts; ++i) {
Src0 = Src0Parts.getReg(i);
if (IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16)
Src1 = Src1Parts.getReg(i);
if (IID == Intrinsic::amdgcn_writelane)
Src2 = Src2Parts.getReg(i);
PartialRes.push_back(createLaneOp(Src0, Src1, Src2, PartialResTy));
}
if (NeedsBitcast || IsFloat)
B.buildBitcast(
DstReg,
B.buildMergeLikeInstr(LLT::integer(IntTy.getSizeInBits()), PartialRes));
else
B.buildMergeLikeInstr(DstReg, PartialRes);
MI.eraseFromParent();
return true;
}
bool AMDGPULegalizerInfo::getImplicitArgPtr(Register DstReg,
MachineRegisterInfo &MRI,
MachineIRBuilder &B) const {
uint64_t Offset =
ST.getTargetLowering()->getImplicitParameterOffset(
B.getMF(), AMDGPUTargetLowering::FIRST_IMPLICIT);
LLT DstTy = MRI.getType(DstReg);
LLT IdxTy = LLT::integer(DstTy.getSizeInBits());
Register KernargPtrReg = MRI.createGenericVirtualRegister(DstTy);
if (!loadInputValue(KernargPtrReg, B,
AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR))
return false;
B.buildObjectPtrOffset(DstReg, KernargPtrReg,
B.buildConstant(IdxTy, Offset).getReg(0));
return true;
}
/// To create a buffer resource from a 64-bit pointer, mask off the upper 32
/// bits of the pointer and replace them with the stride argument, then
/// merge_values everything together. In the common case of a raw buffer (the
/// stride component is 0), we can just AND off the upper half.
bool AMDGPULegalizerInfo::legalizePointerAsRsrcIntrin(
MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const {
Register Result = MI.getOperand(0).getReg();
Register Pointer = MI.getOperand(2).getReg();
Register Stride = MI.getOperand(3).getReg();
Register NumRecords = MI.getOperand(4).getReg();
Register Flags = MI.getOperand(5).getReg();
LLT I32 = LLT::integer(32);
LLT I64 = LLT::integer(64);
B.setInsertPt(B.getMBB(), ++B.getInsertPt());
auto ExtStride = B.buildAnyExt(I32, Stride);
if (ST.getBufferResourceNumRecordsWidth() == 45) {
NumRecords = B.buildZExtOrTrunc(I64, NumRecords).getReg(0);
NumRecords =
B.buildAnd(I64, NumRecords, B.buildConstant(I64, (1ULL << 45) - 1))
.getReg(0);
Register Zero = B.buildConstant(I32, 0).getReg(0);
// Build the lower 64-bit value, which has a 57-bit base and the lower 7-bit
// num_records.
LLT PtrIntTy = LLT::integer(MRI.getType(Pointer).getSizeInBits());
auto PointerInt = B.buildPtrToInt(PtrIntTy, Pointer);
auto ExtPointer = B.buildAnyExtOrTrunc(I64, PointerInt);
auto NumRecordsLHS = B.buildShl(I64, NumRecords, B.buildConstant(I32, 57));
Register LowHalf = B.buildOr(I64, ExtPointer, NumRecordsLHS).getReg(0);
// Build the higher 64-bit value, which has the higher 38-bit num_records,
// 6-bit zero (omit), 16-bit stride and scale and 4-bit flag.
auto NumRecordsRHS = B.buildLShr(I64, NumRecords, B.buildConstant(I32, 7));
auto ShiftedStride = B.buildShl(I32, ExtStride, B.buildConstant(I32, 12));
auto ExtShiftedStride =
B.buildMergeValues(I64, {Zero, ShiftedStride.getReg(0)});
auto ShiftedFlags = B.buildShl(I32, Flags, B.buildConstant(I32, 28));
auto ExtShiftedFlags =
B.buildMergeValues(I64, {Zero, ShiftedFlags.getReg(0)});
auto CombinedFields = B.buildOr(I64, NumRecordsRHS, ExtShiftedStride);
Register HighHalf =
B.buildOr(I64, CombinedFields, ExtShiftedFlags).getReg(0);
B.buildMergeValues(Result, {LowHalf, HighHalf});
} else {
NumRecords = B.buildZExtOrTrunc(I32, NumRecords).getReg(0);
auto Unmerge = B.buildUnmerge(I32, Pointer);
auto LowHalf = Unmerge.getReg(0);
auto HighHalf = Unmerge.getReg(1);
auto AndMask = B.buildConstant(I32, 0x0000ffff);
auto Masked = B.buildAnd(I32, HighHalf, AndMask);
auto ShiftConst = B.buildConstant(I32, 16);
auto ShiftedStride = B.buildShl(I32, ExtStride, ShiftConst);
auto NewHighHalf = B.buildOr(I32, Masked, ShiftedStride);
Register NewHighHalfReg = NewHighHalf.getReg(0);
B.buildMergeValues(Result, {LowHalf, NewHighHalfReg, NumRecords, Flags});
}
MI.eraseFromParent();
return true;
}
bool AMDGPULegalizerInfo::legalizeImplicitArgPtr(MachineInstr &MI,
MachineRegisterInfo &MRI,
MachineIRBuilder &B) const {
const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>();
if (!MFI->isEntryFunction()) {
return legalizePreloadedArgIntrin(MI, MRI, B,
AMDGPUFunctionArgInfo::IMPLICIT_ARG_PTR);
}
Register DstReg = MI.getOperand(0).getReg();
if (!getImplicitArgPtr(DstReg, MRI, B))
return false;
MI.eraseFromParent();
return true;
}
bool AMDGPULegalizerInfo::legalizeIsAddrSpace(MachineInstr &MI,
MachineRegisterInfo &MRI,
MachineIRBuilder &B,
unsigned AddrSpace) const {
const LLT I32 = LLT::integer(32);
auto Unmerge = B.buildUnmerge(I32, MI.getOperand(2).getReg());
Register Hi32 = Unmerge.getReg(1);
if (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS &&
ST.hasGloballyAddressableScratch()) {
Register FlatScratchBaseHi =
B.buildInstr(AMDGPU::S_MOV_B32, {I32},
{Register(AMDGPU::SRC_FLAT_SCRATCH_BASE_HI)})
.getReg(0);
MRI.setRegClass(FlatScratchBaseHi, &AMDGPU::SReg_32RegClass);
// Test bits 63..58 against the aperture address.
Register XOR = B.buildXor(I32, Hi32, FlatScratchBaseHi).getReg(0);
B.buildICmp(ICmpInst::ICMP_ULT, MI.getOperand(0), XOR,
B.buildConstant(I32, 1u << 26));
} else {
Register ApertureReg = getSegmentAperture(AddrSpace, MRI, B);
B.buildICmp(ICmpInst::ICMP_EQ, MI.getOperand(0), Hi32, ApertureReg);
}
MI.eraseFromParent();
return true;
}
// The raw.(t)buffer and struct.(t)buffer intrinsics have two offset args:
// offset (the offset that is included in bounds checking and swizzling, to be
// split between the instruction's voffset and immoffset fields) and soffset
// (the offset that is excluded from bounds checking and swizzling, to go in
// the instruction's soffset field). This function takes the first kind of
// offset and figures out how to split it between voffset and immoffset.
std::pair<Register, unsigned>
AMDGPULegalizerInfo::splitBufferOffsets(MachineIRBuilder &B,
Register OrigOffset) const {
const unsigned MaxImm = SIInstrInfo::getMaxMUBUFImmOffset(ST);
Register BaseReg;
unsigned ImmOffset;
const LLT I32 = LLT::integer(32);
MachineRegisterInfo &MRI = *B.getMRI();
// On GFX1250+, voffset and immoffset are zero-extended from 32 bits before
// being added, so we can only safely match a 32-bit addition with no unsigned
// overflow.
bool CheckNUW = ST.hasGFX1250Insts();
std::tie(BaseReg, ImmOffset) = AMDGPU::getBaseWithConstantOffset(
MRI, OrigOffset, /*KnownBits=*/nullptr, CheckNUW);
// If BaseReg is a pointer, convert it to int.
if (MRI.getType(BaseReg).isPointer())
BaseReg = B.buildPtrToInt(MRI.getType(OrigOffset), BaseReg).getReg(0);
// If the immediate value is too big for the immoffset field, put only bits
// that would normally fit in the immoffset field. The remaining value that
// is copied/added for the voffset field is a large power of 2, and it
// stands more chance of being CSEd with the copy/add for another similar
// load/store.
// However, do not do that rounding down if that is a negative
// number, as it appears to be illegal to have a negative offset in the
// vgpr, even if adding the immediate offset makes it positive.
unsigned Overflow = ImmOffset & ~MaxImm;
ImmOffset -= Overflow;
if ((int32_t)Overflow < 0) {
Overflow += ImmOffset;
ImmOffset = 0;
}
if (Overflow != 0) {
if (!BaseReg) {
BaseReg = B.buildConstant(I32, Overflow).getReg(0);
} else {
auto OverflowVal = B.buildConstant(I32, Overflow);
BaseReg = B.buildAdd(I32, BaseReg, OverflowVal).getReg(0);
}
}
if (!BaseReg)
BaseReg = B.buildConstant(I32, 0).getReg(0);
return std::pair(BaseReg, ImmOffset);
}
/// Handle register layout difference for f16 images for some subtargets.
Register AMDGPULegalizerInfo::handleD16VData(MachineIRBuilder &B,
MachineRegisterInfo &MRI,
Register Reg,
bool ImageStore) const {
const LLT I16 = LLT::integer(16);
const LLT I32 = LLT::integer(32);
LLT StoreVT = MRI.getType(Reg);
assert(StoreVT.isVector() && StoreVT.getElementType().getSizeInBits() == 16);
LLT I16Vec = StoreVT.changeElementType(I16);
Register RegI16 =
StoreVT == I16Vec ? Reg : B.buildBitcast(I16Vec, Reg).getReg(0);
if (ST.hasUnpackedD16VMem()) {
auto Unmerge = B.buildUnmerge(I16, RegI16);
SmallVector<Register, 4> WideRegs;
for (int I = 0, E = Unmerge->getNumOperands() - 1; I != E; ++I)
WideRegs.push_back(B.buildAnyExt(I32, Unmerge.getReg(I)).getReg(0));
int NumElts = StoreVT.getNumElements();
return B.buildBuildVector(LLT::fixed_vector(NumElts, I32), WideRegs)
.getReg(0);
}
if (ImageStore && ST.hasImageStoreD16Bug()) {
if (StoreVT.getNumElements() == 2) {
SmallVector<Register, 4> PackedRegs;
Reg = B.buildBitcast(I32, RegI16).getReg(0);
PackedRegs.push_back(Reg);
PackedRegs.resize(2, B.buildUndef(I32).getReg(0));
return B.buildBuildVector(LLT::fixed_vector(2, I32), PackedRegs)
.getReg(0);
}
if (StoreVT.getNumElements() == 3) {
SmallVector<Register, 4> PackedRegs;
auto Unmerge = B.buildUnmerge(I16, RegI16);
for (int I = 0, E = Unmerge->getNumOperands() - 1; I != E; ++I)
PackedRegs.push_back(Unmerge.getReg(I));
PackedRegs.resize(6, B.buildUndef(I16).getReg(0));
Reg = B.buildBuildVector(LLT::fixed_vector(6, I16), PackedRegs).getReg(0);
return B.buildBitcast(LLT::fixed_vector(3, I32), Reg).getReg(0);
}
if (StoreVT.getNumElements() == 4) {
SmallVector<Register, 4> PackedRegs;
Reg = B.buildBitcast(LLT::fixed_vector(2, I32), RegI16).getReg(0);
auto Unmerge = B.buildUnmerge(I32, Reg);
for (int I = 0, E = Unmerge->getNumOperands() - 1; I != E; ++I)
PackedRegs.push_back(Unmerge.getReg(I));
PackedRegs.resize(4, B.buildUndef(I32).getReg(0));
return B.buildBuildVector(LLT::fixed_vector(4, I32), PackedRegs)
.getReg(0);
}
llvm_unreachable("invalid data type");
}
if (StoreVT.isVector() && StoreVT.getNumElements() == 3 &&
StoreVT.getElementType().getSizeInBits() == 16) {
Reg = B.buildPadVectorWithUndefElements(
LLT::fixed_vector(4, StoreVT.getElementType()), Reg)
.getReg(0);
}
return Reg;
}
Register AMDGPULegalizerInfo::fixStoreSourceType(MachineIRBuilder &B,
Register VData, LLT MemTy,
bool IsFormat) const {
MachineRegisterInfo *MRI = B.getMRI();
LLT Ty = MRI->getType(VData);
// Fixup buffer resources themselves needing to be v4i128.
if (hasBufferRsrcWorkaround(Ty))
return castBufferRsrcToV4I32(VData, B);
if (shouldBitcastLoadStoreType(ST, Ty, MemTy)) {
Ty = getBitcastRegisterType(Ty);
VData = B.buildBitcast(Ty, VData).getReg(0);
}
if (Ty.isFloat(16)) {
Ty = LLT::integer(16);
VData = B.buildBitcast(Ty, VData).getReg(0);
}
// Fixup illegal register types for i8 stores.
if (Ty == LLT::integer(8) || Ty == LLT::integer(16))
return B.buildAnyExt(LLT::integer(32), VData).getReg(0);
if (Ty.isVector()) {
if (Ty.getElementType().getSizeInBits() == 16 && Ty.getNumElements() <= 4) {
if (IsFormat)
return handleD16VData(B, *MRI, VData);
}
}
return VData;
}
bool AMDGPULegalizerInfo::legalizeBufferStore(MachineInstr &MI,
LegalizerHelper &Helper,
bool IsTyped,
bool IsFormat) const {
MachineIRBuilder &B = Helper.MIRBuilder;
MachineRegisterInfo &MRI = *B.getMRI();
Register VData = MI.getOperand(1).getReg();
LLT Ty = MRI.getType(VData);
LLT EltTy = Ty.getScalarType();
const bool IsD16 = IsFormat && (EltTy.getSizeInBits() == 16);
const LLT I32 = LLT::integer(32);
MachineMemOperand *MMO = *MI.memoperands_begin();
const int MemSize = MMO->getSize().getValue();
LLT MemTy = MMO->getMemoryType();
if (IsFormat && !IsTyped && !IsD16 && MemTy.getSizeInBits() < 32) {
const Function &Fn = B.getMF().getFunction();
Fn.getContext().diagnose(DiagnosticInfoUnsupported(
Fn, "unsupported sub-dword format buffer store", MI.getDebugLoc()));
MI.eraseFromParent();
return true;
}
VData = fixStoreSourceType(B, VData, MemTy, IsFormat);
castBufferRsrcArgToV4I32(MI, B, 2);
Register RSrc = MI.getOperand(2).getReg();
unsigned ImmOffset;
// The typed intrinsics add an immediate after the registers.
const unsigned NumVIndexOps = IsTyped ? 8 : 7;
// The struct intrinsic variants add one additional operand over raw.
const bool HasVIndex = MI.getNumOperands() == NumVIndexOps;
Register VIndex;
int OpOffset = 0;
if (HasVIndex) {
VIndex = MI.getOperand(3).getReg();
OpOffset = 1;
} else {
VIndex = B.buildConstant(I32, 0).getReg(0);
}
Register VOffset = MI.getOperand(3 + OpOffset).getReg();
Register SOffset = MI.getOperand(4 + OpOffset).getReg();
unsigned Format = 0;
if (IsTyped) {
Format = MI.getOperand(5 + OpOffset).getImm();
++OpOffset;
}
unsigned AuxiliaryData = MI.getOperand(5 + OpOffset).getImm();
std::tie(VOffset, ImmOffset) = splitBufferOffsets(B, VOffset);
unsigned Opc;
if (IsTyped) {
Opc = IsD16 ? AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT_D16 :
AMDGPU::G_AMDGPU_TBUFFER_STORE_FORMAT;
} else if (IsFormat) {
Opc = IsD16 ? AMDGPU::G_AMDGPU_BUFFER_STORE_FORMAT_D16 :
AMDGPU::G_AMDGPU_BUFFER_STORE_FORMAT;
} else {
switch (MemSize) {
case 1:
Opc = AMDGPU::G_AMDGPU_BUFFER_STORE_BYTE;
break;
case 2:
Opc = AMDGPU::G_AMDGPU_BUFFER_STORE_SHORT;
break;
default:
Opc = AMDGPU::G_AMDGPU_BUFFER_STORE;
break;
}
}
auto MIB = B.buildInstr(Opc)
.addUse(VData) // vdata
.addUse(RSrc) // rsrc
.addUse(VIndex) // vindex
.addUse(VOffset) // voffset
.addUse(SOffset) // soffset
.addImm(ImmOffset); // offset(imm)
if (IsTyped)
MIB.addImm(Format);
MIB.addImm(AuxiliaryData) // cachepolicy, swizzled buffer(imm)
.addImm(HasVIndex ? -1 : 0) // idxen(imm)
.addMemOperand(MMO);
MI.eraseFromParent();
return true;
}
static void buildBufferLoad(unsigned Opc, Register LoadDstReg, Register RSrc,
Register VIndex, Register VOffset, Register SOffset,
unsigned ImmOffset, unsigned Format,
unsigned AuxiliaryData, MachineMemOperand *MMO,
bool IsTyped, bool HasVIndex, MachineIRBuilder &B) {
auto MIB = B.buildInstr(Opc)
.addDef(LoadDstReg) // vdata
.addUse(RSrc) // rsrc
.addUse(VIndex) // vindex
.addUse(VOffset) // voffset
.addUse(SOffset) // soffset
.addImm(ImmOffset); // offset(imm)
if (IsTyped)
MIB.addImm(Format);
MIB.addImm(AuxiliaryData) // cachepolicy, swizzled buffer(imm)
.addImm(HasVIndex ? -1 : 0) // idxen(imm)
.addMemOperand(MMO);
}
static void buildTFEBufferLoad(unsigned Opc, ArrayRef<Register> ValueDsts,
Register StatusDst, Register RSrc,
Register VIndex, Register VOffset,
Register SOffset, unsigned ImmOffset,
unsigned Format, unsigned AuxiliaryData,
MachineMemOperand *MMO, bool IsTyped,
bool HasVIndex, MachineIRBuilder &B) {
LLT LoadTy = LLT::fixed_vector(ValueDsts.size() + 1, LLT::integer(32));
Register LoadDstReg = B.getMRI()->createGenericVirtualRegister(LoadTy);
buildBufferLoad(Opc, LoadDstReg, RSrc, VIndex, VOffset, SOffset, ImmOffset,
Format, AuxiliaryData, MMO, IsTyped, HasVIndex, B);
SmallVector<Register, 5> Unmerge(ValueDsts);
Unmerge.push_back(StatusDst);
B.buildUnmerge(Unmerge, LoadDstReg);
}
bool AMDGPULegalizerInfo::legalizeBufferLoad(MachineInstr &MI,
LegalizerHelper &Helper,
bool IsFormat,
bool IsTyped) const {
MachineIRBuilder &B = Helper.MIRBuilder;
MachineRegisterInfo &MRI = *B.getMRI();
GISelChangeObserver &Observer = Helper.Observer;
// FIXME: Verifier should enforce 1 MMO for these intrinsics.
MachineMemOperand *MMO = *MI.memoperands_begin();
const LLT MemTy = MMO->getMemoryType();
const LLT I32 = LLT::integer(32);
Register Dst = MI.getOperand(0).getReg();
Register StatusDst;
int OpOffset = 0;
assert(MI.getNumExplicitDefs() == 1 || MI.getNumExplicitDefs() == 2);
bool IsTFE = MI.getNumExplicitDefs() == 2;
if (IsTFE) {
StatusDst = MI.getOperand(1).getReg();
++OpOffset;
}
castBufferRsrcArgToV4I32(MI, B, 2 + OpOffset);
Register RSrc = MI.getOperand(2 + OpOffset).getReg();
// The typed intrinsics add an immediate after the registers.
const unsigned NumVIndexOps = IsTyped ? 8 : 7;
// The struct intrinsic variants add one additional operand over raw.
const bool HasVIndex = MI.getNumOperands() == NumVIndexOps + OpOffset;
Register VIndex;
if (HasVIndex) {
VIndex = MI.getOperand(3 + OpOffset).getReg();
++OpOffset;
} else {
VIndex = B.buildConstant(I32, 0).getReg(0);
}
Register VOffset = MI.getOperand(3 + OpOffset).getReg();
Register SOffset = MI.getOperand(4 + OpOffset).getReg();
unsigned Format = 0;
if (IsTyped) {
Format = MI.getOperand(5 + OpOffset).getImm();
++OpOffset;
}
unsigned AuxiliaryData = MI.getOperand(5 + OpOffset).getImm();
unsigned ImmOffset;
LLT Ty = MRI.getType(Dst);
// Make addrspace 8 pointers loads into 4xi32 loads here, so the rest of the
// logic doesn't have to handle that case.
if (hasBufferRsrcWorkaround(Ty)) {
Observer.changingInstr(MI);
Ty = castBufferRsrcFromV4I32(MI, B, MRI, 0);
Observer.changedInstr(MI);
Dst = MI.getOperand(0).getReg();
B.setInsertPt(B.getMBB(), MI);
}
if (shouldBitcastLoadStoreType(ST, Ty, MemTy)) {
Ty = getBitcastRegisterType(Ty);
Observer.changingInstr(MI);
Helper.bitcastDst(MI, Ty, 0);
Observer.changedInstr(MI);
Dst = MI.getOperand(0).getReg();
B.setInsertPt(B.getMBB(), MI);
}
LLT EltTy = Ty.getScalarType();
const bool IsD16 = IsFormat && (EltTy.getSizeInBits() == 16);
const bool Unpacked = ST.hasUnpackedD16VMem();
if (IsFormat && !IsTyped && !IsD16 && MemTy.getSizeInBits() < 32) {
const Function &Fn = B.getMF().getFunction();
Fn.getContext().diagnose(DiagnosticInfoUnsupported(
Fn, "unsupported sub-dword format buffer load", MI.getDebugLoc()));
B.buildUndef(Dst);
if (IsTFE)
B.buildUndef(StatusDst);
MI.eraseFromParent();
return true;
}
if (!IsTyped && IsD16 && IsTFE && !ST.hasBufferTFEFormatD16()) {
const Function &Fn = B.getMF().getFunction();
Fn.getContext().diagnose(DiagnosticInfoUnsupported(
Fn, "TFE D16 format buffer load is not supported on this GPU",
MI.getDebugLoc()));
B.buildUndef(Dst);
B.buildUndef(StatusDst);
MI.eraseFromParent();
return true;
}
std::tie(VOffset, ImmOffset) = splitBufferOffsets(B, VOffset);
unsigned Opc;
// TODO: Support TFE for typed and narrow loads.
if (IsTyped) {
if (IsTFE)
return false;
Opc = IsD16 ? AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT_D16 :
AMDGPU::G_AMDGPU_TBUFFER_LOAD_FORMAT;
} else if (IsFormat) {
if (IsD16) {
Opc = IsTFE ? AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_D16_TFE
: AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_D16;
} else {
Opc = IsTFE ? AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_TFE
: AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT;
}
} else {
switch (MemTy.getSizeInBits()) {
case 8:
Opc = IsTFE ? AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE_TFE
: AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE;
break;
case 16:
Opc = IsTFE ? AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT_TFE
: AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT;
break;
default:
Opc = IsTFE ? AMDGPU::G_AMDGPU_BUFFER_LOAD_TFE
: AMDGPU::G_AMDGPU_BUFFER_LOAD;
break;
}
}
if (IsTFE && IsD16 && Ty.isVector()) {
// Value dwords need not cover Ty exactly: v3i16 needs 2 dwords for 48 bits.
const unsigned NumElts = Ty.getNumElements();
const unsigned NumValueDWords = Unpacked ? NumElts : divideCeil(NumElts, 2);
SmallVector<Register, 4> ValueDWords;
for (unsigned I = 0; I != NumValueDWords; ++I)
ValueDWords.push_back(MRI.createGenericVirtualRegister(I32));
buildTFEBufferLoad(Opc, ValueDWords, StatusDst, RSrc, VIndex, VOffset,
SOffset, ImmOffset, Format, AuxiliaryData, MMO, IsTyped,
HasVIndex, B);
if (Unpacked) {
for (Register &R : ValueDWords)
R = B.buildTrunc(EltTy, R).getReg(0);
B.buildMergeLikeInstr(Dst, ValueDWords);
} else {
Register Merged =
NumValueDWords == 1
? ValueDWords[0]
: B.buildMergeLikeInstr(LLT::fixed_vector(NumValueDWords, I32),
ValueDWords)
.getReg(0);
LLT PackedTy = LLT::fixed_vector(NumValueDWords * 2, EltTy);
if (PackedTy == Ty) {
B.buildBitcast(Dst, Merged);
} else {
Register Packed = B.buildBitcast(PackedTy, Merged).getReg(0);
B.buildDeleteTrailingVectorElements(Dst, Packed);
}
}
} else if (IsTFE) {
const unsigned NumValueDWords = divideCeil(Ty.getSizeInBits(), 32);
Register DstInt =
EltTy.isFloat() ? MRI.createGenericVirtualRegister(Ty.changeElementType(
LLT::integer(EltTy.getSizeInBits())))
: Dst;
if (MemTy.getSizeInBits() < 32) {
Register ExtDst = MRI.createGenericVirtualRegister(I32);
buildTFEBufferLoad(Opc, ExtDst, StatusDst, RSrc, VIndex, VOffset, SOffset,
ImmOffset, Format, AuxiliaryData, MMO, IsTyped,
HasVIndex, B);
B.buildTrunc(DstInt, ExtDst);
} else if (NumValueDWords == 1) {
buildTFEBufferLoad(Opc, DstInt, StatusDst, RSrc, VIndex, VOffset, SOffset,
ImmOffset, Format, AuxiliaryData, MMO, IsTyped,
HasVIndex, B);
} else {
SmallVector<Register, 4> ValueDWords;
for (unsigned I = 0; I != NumValueDWords; ++I)
ValueDWords.push_back(MRI.createGenericVirtualRegister(I32));
buildTFEBufferLoad(Opc, ValueDWords, StatusDst, RSrc, VIndex, VOffset,
SOffset, ImmOffset, Format, AuxiliaryData, MMO,
IsTyped, HasVIndex, B);
B.buildMergeLikeInstr(DstInt, ValueDWords);
}
if (DstInt != Dst)
B.buildBitcast(Dst, DstInt);
} else if ((!IsD16 && MemTy.getSizeInBits() < 32) ||
(IsD16 && !Ty.isVector())) {
Register LoadDstReg = B.getMRI()->createGenericVirtualRegister(I32);
buildBufferLoad(Opc, LoadDstReg, RSrc, VIndex, VOffset, SOffset, ImmOffset,
Format, AuxiliaryData, MMO, IsTyped, HasVIndex, B);
B.setInsertPt(B.getMBB(), ++B.getInsertPt());
B.buildTrunc(Dst, LoadDstReg);
} else if (Unpacked && IsD16 && Ty.isVector()) {
LLT UnpackedTy = LLT::fixed_vector(Ty.getNumElements(), LLT::integer(32));
Register LoadDstReg = B.getMRI()->createGenericVirtualRegister(UnpackedTy);
buildBufferLoad(Opc, LoadDstReg, RSrc, VIndex, VOffset, SOffset, ImmOffset,
Format, AuxiliaryData, MMO, IsTyped, HasVIndex, B);
B.setInsertPt(B.getMBB(), ++B.getInsertPt());
// FIXME: G_TRUNC should work, but legalization currently fails
auto Unmerge = B.buildUnmerge(I32, LoadDstReg);
SmallVector<Register, 4> Repack;
for (unsigned I = 0, N = Unmerge->getNumOperands() - 1; I != N; ++I)
Repack.push_back(B.buildTrunc(EltTy, Unmerge.getReg(I)).getReg(0));
B.buildMergeLikeInstr(Dst, Repack);
} else {
buildBufferLoad(Opc, Dst, RSrc, VIndex, VOffset, SOffset, ImmOffset, Format,
AuxiliaryData, MMO, IsTyped, HasVIndex, B);
}
MI.eraseFromParent();
return true;
}
static unsigned getBufferAtomicPseudo(Intrinsic::ID IntrID) {
switch (IntrID) {
case Intrinsic::amdgcn_raw_buffer_atomic_swap:
case Intrinsic::amdgcn_raw_ptr_buffer_atomic_swap:
case Intrinsic::amdgcn_struct_buffer_atomic_swap:
case Intrinsic::amdgcn_struct_ptr_buffer_atomic_swap:
return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SWAP;
case Intrinsic::amdgcn_raw_buffer_atomic_add:
case Intrinsic::amdgcn_raw_ptr_buffer_atomic_add:
case Intrinsic::amdgcn_struct_buffer_atomic_add:
case Intrinsic::amdgcn_struct_ptr_buffer_atomic_add:
return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_ADD;
case Intrinsic::amdgcn_raw_buffer_atomic_sub:
case Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub:
case Intrinsic::amdgcn_struct_buffer_atomic_sub:
case Intrinsic::amdgcn_struct_ptr_buffer_atomic_sub:
return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SUB;
case Intrinsic::amdgcn_raw_buffer_atomic_smin:
case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smin:
case Intrinsic::amdgcn_struct_buffer_atomic_smin:
case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smin:
return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMIN;
case Intrinsic::amdgcn_raw_buffer_atomic_umin:
case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umin:
case Intrinsic::amdgcn_struct_buffer_atomic_umin:
case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umin:
return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMIN;
case Intrinsic::amdgcn_raw_buffer_atomic_smax:
case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smax:
case Intrinsic::amdgcn_struct_buffer_atomic_smax:
case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smax:
return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SMAX;
case Intrinsic::amdgcn_raw_buffer_atomic_umax:
case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umax:
case Intrinsic::amdgcn_struct_buffer_atomic_umax:
case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umax:
return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_UMAX;
case Intrinsic::amdgcn_raw_buffer_atomic_and:
case Intrinsic::amdgcn_raw_ptr_buffer_atomic_and:
case Intrinsic::amdgcn_struct_buffer_atomic_and:
case Intrinsic::amdgcn_struct_ptr_buffer_atomic_and:
return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_AND;
case Intrinsic::amdgcn_raw_buffer_atomic_or:
case Intrinsic::amdgcn_raw_ptr_buffer_atomic_or:
case Intrinsic::amdgcn_struct_buffer_atomic_or:
case Intrinsic::amdgcn_struct_ptr_buffer_atomic_or:
return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_OR;
case Intrinsic::amdgcn_raw_buffer_atomic_xor:
case Intrinsic::amdgcn_raw_ptr_buffer_atomic_xor:
case Intrinsic::amdgcn_struct_buffer_atomic_xor:
case Intrinsic::amdgcn_struct_ptr_buffer_atomic_xor:
return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_XOR;
case Intrinsic::amdgcn_raw_buffer_atomic_inc:
case Intrinsic::amdgcn_raw_ptr_buffer_atomic_inc:
case Intrinsic::amdgcn_struct_buffer_atomic_inc:
case Intrinsic::amdgcn_struct_ptr_buffer_atomic_inc:
return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_INC;
case Intrinsic::amdgcn_raw_buffer_atomic_dec:
case Intrinsic::amdgcn_raw_ptr_buffer_atomic_dec:
case Intrinsic::amdgcn_struct_buffer_atomic_dec:
case Intrinsic::amdgcn_struct_ptr_buffer_atomic_dec:
return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_DEC;
case Intrinsic::amdgcn_raw_buffer_atomic_cmpswap:
case Intrinsic::amdgcn_raw_ptr_buffer_atomic_cmpswap:
case Intrinsic::amdgcn_struct_buffer_atomic_cmpswap:
case Intrinsic::amdgcn_struct_ptr_buffer_atomic_cmpswap:
return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_CMPSWAP;
case Intrinsic::amdgcn_raw_buffer_atomic_fadd:
case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fadd:
case Intrinsic::amdgcn_struct_buffer_atomic_fadd:
case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fadd:
return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FADD;
case Intrinsic::amdgcn_raw_buffer_atomic_fmin:
case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmin:
case Intrinsic::amdgcn_struct_buffer_atomic_fmin:
case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fmin:
return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FMIN;
case Intrinsic::amdgcn_raw_buffer_atomic_fmax:
case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmax:
case Intrinsic::amdgcn_struct_buffer_atomic_fmax:
case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fmax:
return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FMAX;
case Intrinsic::amdgcn_raw_buffer_atomic_sub_clamp_u32:
case Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub_clamp_u32:
case Intrinsic::amdgcn_struct_buffer_atomic_sub_clamp_u32:
case Intrinsic::amdgcn_struct_ptr_buffer_atomic_sub_clamp_u32:
return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_SUB_CLAMP_U32;
case Intrinsic::amdgcn_raw_buffer_atomic_cond_sub_u32:
case Intrinsic::amdgcn_raw_ptr_buffer_atomic_cond_sub_u32:
case Intrinsic::amdgcn_struct_buffer_atomic_cond_sub_u32:
case Intrinsic::amdgcn_struct_ptr_buffer_atomic_cond_sub_u32:
return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_COND_SUB_U32;
default:
llvm_unreachable("unhandled atomic opcode");
}
}
bool AMDGPULegalizerInfo::legalizeBufferAtomic(MachineInstr &MI,
MachineIRBuilder &B,
Intrinsic::ID IID) const {
const bool IsCmpSwap =
IID == Intrinsic::amdgcn_raw_buffer_atomic_cmpswap ||
IID == Intrinsic::amdgcn_struct_buffer_atomic_cmpswap ||
IID == Intrinsic::amdgcn_raw_ptr_buffer_atomic_cmpswap ||
IID == Intrinsic::amdgcn_struct_ptr_buffer_atomic_cmpswap;
Register Dst = MI.getOperand(0).getReg();
// Since we don't have 128-bit atomics, we don't need to handle the case of
// p8 argmunents to the atomic itself
Register VData = MI.getOperand(2).getReg();
Register CmpVal;
int OpOffset = 0;
if (IsCmpSwap) {
CmpVal = MI.getOperand(3).getReg();
++OpOffset;
}
castBufferRsrcArgToV4I32(MI, B, 3 + OpOffset);
Register RSrc = MI.getOperand(3 + OpOffset).getReg();
const unsigned NumVIndexOps = IsCmpSwap ? 9 : 8;
// The struct intrinsic variants add one additional operand over raw.
const bool HasVIndex = MI.getNumOperands() == NumVIndexOps;
Register VIndex;
if (HasVIndex) {
VIndex = MI.getOperand(4 + OpOffset).getReg();
++OpOffset;
} else {
VIndex = B.buildConstant(LLT::integer(32), 0).getReg(0);
}
Register VOffset = MI.getOperand(4 + OpOffset).getReg();
Register SOffset = MI.getOperand(5 + OpOffset).getReg();
unsigned AuxiliaryData = MI.getOperand(6 + OpOffset).getImm();
MachineMemOperand *MMO = *MI.memoperands_begin();
unsigned ImmOffset;
std::tie(VOffset, ImmOffset) = splitBufferOffsets(B, VOffset);
auto MIB = B.buildInstr(getBufferAtomicPseudo(IID))
.addDef(Dst)
.addUse(VData); // vdata
if (IsCmpSwap)
MIB.addReg(CmpVal);
MIB.addUse(RSrc) // rsrc
.addUse(VIndex) // vindex
.addUse(VOffset) // voffset
.addUse(SOffset) // soffset
.addImm(ImmOffset) // offset(imm)
.addImm(AuxiliaryData) // cachepolicy, swizzled buffer(imm)
.addImm(HasVIndex ? -1 : 0) // idxen(imm)
.addMemOperand(MMO);
MI.eraseFromParent();
return true;
}
/// Turn a set of f16 typed registers in \p AddrRegs into a dword sized
/// vector with f16 typed elements.
static void packImage16bitOpsToDwords(MachineIRBuilder &B, MachineInstr &MI,
SmallVectorImpl<Register> &PackedAddrs,
unsigned ArgOffset,
const AMDGPU::ImageDimIntrinsicInfo *Intr,
bool IsA16, bool IsG16) {
auto EndIdx = Intr->VAddrEnd;
for (unsigned I = Intr->VAddrStart; I < EndIdx; I++) {
MachineOperand &SrcOp = MI.getOperand(ArgOffset + I);
if (!SrcOp.isReg())
continue; // _L to _LZ may have eliminated this.
Register AddrReg = SrcOp.getReg();
if ((I < Intr->GradientStart) ||
(I >= Intr->GradientStart && I < Intr->CoordStart && !IsG16) ||
(I >= Intr->CoordStart && !IsA16)) {
if ((I < Intr->GradientStart) && IsA16 &&
(B.getMRI()->getType(AddrReg) == F16)) {
assert(I == Intr->BiasIndex && "Got unexpected 16-bit extra argument");
// Special handling of bias when A16 is on. Bias is of type half but
// occupies full 32-bit.
PackedAddrs.push_back(
B.buildBuildVector(V2F16, {AddrReg, B.buildUndef(F16).getReg(0)})
.getReg(0));
} else {
assert((!IsA16 || Intr->NumBiasArgs == 0 || I != Intr->BiasIndex) &&
"Bias needs to be converted to 16 bit in A16 mode");
// Handle any gradient or coordinate operands that should not be packed
AddrReg = B.buildBitcast(V2F16, AddrReg).getReg(0);
PackedAddrs.push_back(AddrReg);
}
} else {
const LLT EltTy = B.getMRI()->getType(AddrReg);
const LLT V2EltTy = LLT::fixed_vector(2, EltTy);
// Dz/dh, dz/dv and the last odd coord are packed with undef. Also, in 1D,
// derivatives dx/dh and dx/dv are packed with undef.
if (((I + 1) >= EndIdx) ||
((Intr->NumGradients / 2) % 2 == 1 &&
(I == static_cast<unsigned>(Intr->GradientStart +
(Intr->NumGradients / 2) - 1) ||
I == static_cast<unsigned>(Intr->GradientStart +
Intr->NumGradients - 1))) ||
// Check for _L to _LZ optimization
!MI.getOperand(ArgOffset + I + 1).isReg()) {
PackedAddrs.push_back(
B.buildBuildVector(V2EltTy,
{AddrReg, B.buildUndef(EltTy).getReg(0)})
.getReg(0));
} else {
PackedAddrs.push_back(
B.buildBuildVector(
V2EltTy, {AddrReg, MI.getOperand(ArgOffset + I + 1).getReg()})
.getReg(0));
++I;
}
}
}
}
/// Convert from separate vaddr components to a single vector address register,
/// and replace the remaining operands with $noreg.
static void convertImageAddrToPacked(MachineIRBuilder &B, MachineInstr &MI,
int DimIdx, int NumVAddrs) {
SmallVector<Register, 8> AddrRegs;
for (int I = 0; I != NumVAddrs; ++I) {
MachineOperand &SrcOp = MI.getOperand(DimIdx + I);
if (SrcOp.isReg()) {
Register Reg = SrcOp.getReg();
LLT I32 = LLT::integer(32);
assert(B.getMRI()->getType(Reg).getSizeInBits() == 32);
if (B.getMRI()->getType(Reg) != I32)
Reg = B.buildBitcast(I32, Reg).getReg(0);
AddrRegs.push_back(Reg);
}
}
int NumAddrRegs = AddrRegs.size();
if (NumAddrRegs != 1) {
LLT EltTy = B.getMRI()->getType(AddrRegs[0]);
auto VAddr =
B.buildBuildVector(LLT::fixed_vector(NumAddrRegs, EltTy), AddrRegs);
MI.getOperand(DimIdx).setReg(VAddr.getReg(0));
}
for (int I = 1; I != NumVAddrs; ++I) {
MachineOperand &SrcOp = MI.getOperand(DimIdx + I);
if (SrcOp.isReg())
MI.getOperand(DimIdx + I).setReg(AMDGPU::NoRegister);
}
}
/// Rewrite image intrinsics to use register layouts expected by the subtarget.
///
/// Depending on the subtarget, load/store with 16-bit element data need to be
/// rewritten to use the low half of 32-bit registers, or directly use a packed
/// layout. 16-bit addresses should also sometimes be packed into 32-bit
/// registers.
///
/// We don't want to directly select image instructions just yet, but also want
/// to exposes all register repacking to the legalizer/combiners. We also don't
/// want a selected instruction entering RegBankSelect. In order to avoid
/// defining a multitude of intermediate image instructions, directly hack on
/// the intrinsic's arguments. In cases like a16 addresses, this requires
/// padding now unnecessary arguments with $noreg.
bool AMDGPULegalizerInfo::legalizeImageIntrinsic(
MachineInstr &MI, MachineIRBuilder &B, GISelChangeObserver &Observer,
const AMDGPU::ImageDimIntrinsicInfo *Intr) const {
const MachineFunction &MF = *MI.getMF();
const unsigned NumDefs = MI.getNumExplicitDefs();
const unsigned ArgOffset = NumDefs + 1;
bool IsTFE = NumDefs == 2;
// We are only processing the operands of d16 image operations on subtargets
// that use the unpacked register layout, or need to repack the TFE result.
// TODO: Do we need to guard against already legalized intrinsics?
const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
AMDGPU::getMIMGBaseOpcodeInfo(Intr->BaseOpcode);
MachineRegisterInfo *MRI = B.getMRI();
const LLT I32 = LLT::integer(32);
const LLT I16 = LLT::integer(16);
const LLT V2I16 = LLT::fixed_vector(2, I16);
unsigned DMask = 0;
Register VData;
LLT Ty;
if (!BaseOpcode->NoReturn || BaseOpcode->Store) {
VData = MI.getOperand(NumDefs == 0 ? 1 : 0).getReg();
Ty = MRI->getType(VData);
}
const bool IsAtomicPacked16Bit =
(BaseOpcode->BaseOpcode == AMDGPU::IMAGE_ATOMIC_PK_ADD_F16 ||
BaseOpcode->BaseOpcode == AMDGPU::IMAGE_ATOMIC_PK_ADD_BF16);
// Check for 16 bit addresses and pack if true.
LLT GradTy =
MRI->getType(MI.getOperand(ArgOffset + Intr->GradientStart).getReg());
LLT AddrTy =
MRI->getType(MI.getOperand(ArgOffset + Intr->CoordStart).getReg());
const bool GradTyIs16 = GradTy == I16 || GradTy == F16;
const bool AddrTyIs16 = AddrTy == I16 || AddrTy == F16;
const bool DataTyIs16 =
Ty.getScalarType() == I16 || Ty.getScalarType() == F16;
const bool IsG16 =
ST.hasG16() ? (BaseOpcode->Gradients && GradTyIs16) : GradTyIs16;
const bool IsA16 = AddrTyIs16;
const bool IsD16 = !IsAtomicPacked16Bit && DataTyIs16;
int DMaskLanes = 0;
if (!BaseOpcode->Atomic) {
DMask = MI.getOperand(ArgOffset + Intr->DMaskIndex).getImm();
if (BaseOpcode->Gather4) {
DMaskLanes = 4;
} else if (DMask != 0) {
DMaskLanes = llvm::popcount(DMask);
} else if (!IsTFE && !BaseOpcode->Store) {
// If dmask is 0, this is a no-op load. This can be eliminated.
B.buildUndef(MI.getOperand(0));
MI.eraseFromParent();
return true;
}
}
Observer.changingInstr(MI);
scope_exit ChangedInstr([&] { Observer.changedInstr(MI); });
const unsigned StoreOpcode = IsD16 ? AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE_D16
: AMDGPU::G_AMDGPU_INTRIN_IMAGE_STORE;
const unsigned LoadOpcode = IsD16 ? AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_D16
: AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD;
unsigned NewOpcode = LoadOpcode;
if (BaseOpcode->Store)
NewOpcode = StoreOpcode;
else if (BaseOpcode->NoReturn)
NewOpcode = AMDGPU::G_AMDGPU_INTRIN_IMAGE_LOAD_NORET;
// Track that we legalized this
MI.setDesc(B.getTII().get(NewOpcode));
// Expecting to get an error flag since TFC is on - and dmask is 0 Force
// dmask to be at least 1 otherwise the instruction will fail
if (IsTFE && DMask == 0) {
DMask = 0x1;
DMaskLanes = 1;
MI.getOperand(ArgOffset + Intr->DMaskIndex).setImm(DMask);
}
if (BaseOpcode->Atomic) {
Register VData0 = MI.getOperand(2).getReg();
LLT Ty = MRI->getType(VData0);
// TODO: Allow atomic swap and bit ops for v2f16/v4f16
if (Ty.isVector() && !IsAtomicPacked16Bit)
return false;
if (BaseOpcode->AtomicX2) {
Register VData1 = MI.getOperand(3).getReg();
// The two values are packed in one register.
LLT PackedTy = LLT::fixed_vector(2, Ty);
auto Concat = B.buildBuildVector(PackedTy, {VData0, VData1});
MI.getOperand(2).setReg(Concat.getReg(0));
MI.getOperand(3).setReg(AMDGPU::NoRegister);
}
}
unsigned CorrectedNumVAddrs = Intr->NumVAddrs;
// Rewrite the addressing register layout before doing anything else.
if (BaseOpcode->Gradients && !ST.hasG16() && (IsA16 != IsG16)) {
// 16 bit gradients are supported, but are tied to the A16 control
// so both gradients and addresses must be 16 bit
return false;
}
if (IsA16 && !ST.hasA16()) {
// A16 not supported
return false;
}
const unsigned NSAMaxSize = ST.getNSAMaxSize(BaseOpcode->Sampler);
const unsigned HasPartialNSA = ST.hasPartialNSAEncoding();
if (IsA16 || IsG16) {
// Even if NumVAddrs == 1 we should pack it into a 32-bit value, because the
// instructions expect VGPR_32
SmallVector<Register, 4> PackedRegs;
packImage16bitOpsToDwords(B, MI, PackedRegs, ArgOffset, Intr, IsA16, IsG16);
// See also below in the non-a16 branch
const bool UseNSA = ST.hasNSAEncoding() &&
PackedRegs.size() >= ST.getNSAThreshold(MF) &&
(PackedRegs.size() <= NSAMaxSize || HasPartialNSA);
const bool UsePartialNSA =
UseNSA && HasPartialNSA && PackedRegs.size() > NSAMaxSize;
if (UsePartialNSA) {
// Pack registers that would go over NSAMaxSize into last VAddr register
LLT PackedAddrTy =
LLT::fixed_vector(2 * (PackedRegs.size() - NSAMaxSize + 1), F16);
auto Concat = B.buildConcatVectors(
PackedAddrTy, ArrayRef(PackedRegs).slice(NSAMaxSize - 1));
PackedRegs[NSAMaxSize - 1] = Concat.getReg(0);
PackedRegs.resize(NSAMaxSize);
} else if (!UseNSA && PackedRegs.size() > 1) {
LLT PackedAddrTy = LLT::fixed_vector(2 * PackedRegs.size(), F16);
auto Concat = B.buildConcatVectors(PackedAddrTy, PackedRegs);
PackedRegs[0] = Concat.getReg(0);
PackedRegs.resize(1);
}
const unsigned NumPacked = PackedRegs.size();
for (unsigned I = Intr->VAddrStart; I < Intr->VAddrEnd; I++) {
MachineOperand &SrcOp = MI.getOperand(ArgOffset + I);
if (!SrcOp.isReg()) {
assert(SrcOp.isImm() && SrcOp.getImm() == 0);
continue;
}
assert(SrcOp.getReg() != AMDGPU::NoRegister);
if (I - Intr->VAddrStart < NumPacked)
SrcOp.setReg(PackedRegs[I - Intr->VAddrStart]);
else
SrcOp.setReg(AMDGPU::NoRegister);
}
} else {
// If the register allocator cannot place the address registers contiguously
// without introducing moves, then using the non-sequential address encoding
// is always preferable, since it saves VALU instructions and is usually a
// wash in terms of code size or even better.
//
// However, we currently have no way of hinting to the register allocator
// that MIMG addresses should be placed contiguously when it is possible to
// do so, so force non-NSA for the common 2-address case as a heuristic.
//
// SIShrinkInstructions will convert NSA encodings to non-NSA after register
// allocation when possible.
//
// Partial NSA is allowed on GFX11+ where the final register is a contiguous
// set of the remaining addresses.
const bool UseNSA = ST.hasNSAEncoding() &&
CorrectedNumVAddrs >= ST.getNSAThreshold(MF) &&
(CorrectedNumVAddrs <= NSAMaxSize || HasPartialNSA);
const bool UsePartialNSA =
UseNSA && HasPartialNSA && CorrectedNumVAddrs > NSAMaxSize;
if (UsePartialNSA) {
convertImageAddrToPacked(B, MI,
ArgOffset + Intr->VAddrStart + NSAMaxSize - 1,
Intr->NumVAddrs - NSAMaxSize + 1);
} else if (!UseNSA && Intr->NumVAddrs > 1) {
convertImageAddrToPacked(B, MI, ArgOffset + Intr->VAddrStart,
Intr->NumVAddrs);
}
}
int Flags = 0;
if (IsA16)
Flags |= 1;
if (IsG16)
Flags |= 2;
MI.addOperand(MachineOperand::CreateImm(Flags));
if (BaseOpcode->NoReturn) { // No TFE for stores?
// TODO: Handle dmask trim
if (!Ty.isVector() || !IsD16)
return true;
Register RepackedReg = handleD16VData(B, *MRI, VData, true);
if (RepackedReg != VData) {
MI.getOperand(1).setReg(RepackedReg);
}
return true;
}
Register DstReg = MI.getOperand(0).getReg();
const LLT EltTy = Ty.getScalarType();
const int NumElts = Ty.isVector() ? Ty.getNumElements() : 1;
// Confirm that the return type is large enough for the dmask specified
if (NumElts < DMaskLanes)
return false;
if (NumElts > 4 || DMaskLanes > 4)
return false;
// Image atomic instructions are using DMask to specify how many bits
// input/output data will have. 32-bits (i32, f32, v2f16) or 64-bits (i64,
// f64, v4f16).
// DMaskLanes for image atomic has default value '0'.
// We must be sure that atomic variants (especially packed) will not be
// truncated from v2f16 or v4f16 to f16 type.
//
// ChangeElementCount will be needed for image load where Ty is always scalar.
const unsigned AdjustedNumElts = DMaskLanes == 0 ? 1 : DMaskLanes;
const LLT AdjustedTy =
DMaskLanes == 0
? Ty
: Ty.changeElementCount(ElementCount::getFixed(AdjustedNumElts));
// The raw dword aligned data component of the load. The only legal cases
// where this matters should be when using the packed D16 format, for
// f16 -> <2 x f16>, and <3 x f16> -> <4 x f16>,
LLT RoundedTy;
// I32 vector to cover all data, plus TFE result element.
LLT TFETy;
// Register type to use for each loaded component. Will be I32 or V2I16.
LLT RegTy;
if (IsD16 && ST.hasUnpackedD16VMem()) {
RoundedTy =
LLT::scalarOrVector(ElementCount::getFixed(AdjustedNumElts), I32);
TFETy = LLT::fixed_vector(AdjustedNumElts + 1, I32);
RegTy = I32;
} else {
unsigned EltSize = EltTy.getSizeInBits();
unsigned RoundedElts = (AdjustedTy.getSizeInBits() + 31) / 32;
unsigned RoundedSize = 32 * RoundedElts;
RoundedTy = LLT::scalarOrVector(
ElementCount::getFixed(RoundedSize / EltSize), EltTy);
TFETy = LLT::fixed_vector(RoundedSize / 32 + 1, I32);
RegTy = !IsTFE && EltSize == 16 ? V2I16 : I32;
}
// The return type does not need adjustment.
// TODO: Should we change f16 case to i32 or <2 x f16>?
if (!IsTFE && (RoundedTy == Ty || !Ty.isVector()))
return true;
Register Dst1Reg;
// Insert after the instruction.
B.setInsertPt(*MI.getParent(), ++MI.getIterator());
// TODO: For TFE with d16, if we used a TFE type that was a multiple of <2 x
// f16> instead of i32, we would only need 1 bitcast instead of multiple.
const LLT LoadResultTy = IsTFE ? TFETy : RoundedTy;
const int ResultNumRegs = LoadResultTy.getSizeInBits() / 32;
Register NewResultReg = MRI->createGenericVirtualRegister(LoadResultTy);
MI.getOperand(0).setReg(NewResultReg);
// In the IR, TFE is supposed to be used with a 2 element struct return
// type. The instruction really returns these two values in one contiguous
// register, with one additional dword beyond the loaded data. Rewrite the
// return type to use a single register result.
if (IsTFE) {
Dst1Reg = MI.getOperand(1).getReg();
if (MRI->getType(Dst1Reg) != I32)
return false;
// TODO: Make sure the TFE operand bit is set.
MI.removeOperand(1);
// Handle the easy case that requires no repack instructions.
if (!Ty.isVector() && Ty.getSizeInBits() == 32) {
auto Unmerge = B.buildUnmerge({I32, I32}, NewResultReg);
B.buildBitcast(DstReg, Unmerge.getReg(0));
B.buildCopy(Dst1Reg, Unmerge.getReg(1));
return true;
}
}
// Now figure out how to copy the new result register back into the old
// result.
SmallVector<Register, 5> ResultRegs(ResultNumRegs, Dst1Reg);
const int NumDataRegs = IsTFE ? ResultNumRegs - 1 : ResultNumRegs;
if (ResultNumRegs == 1) {
assert(!IsTFE);
ResultRegs[0] = NewResultReg;
} else {
// We have to repack into a new vector of some kind.
for (int I = 0; I != NumDataRegs; ++I)
ResultRegs[I] = MRI->createGenericVirtualRegister(RegTy);
B.buildUnmerge(ResultRegs, NewResultReg);
// Drop the final TFE element to get the data part. The TFE result is
// directly written to the right place already.
if (IsTFE)
ResultRegs.resize(NumDataRegs);
}
// For an f16 scalar result, we form an i32 result with a truncate regardless
// of packed vs. unpacked.
if (IsD16 && !Ty.isVector()) {
B.buildTrunc(DstReg, ResultRegs[0]);
return true;
}
// Avoid a build/concat_vector of 1 entry.
if ((Ty == V2I16 || Ty == V2F16) && NumDataRegs == 1 &&
!ST.hasUnpackedD16VMem()) {
B.buildBitcast(DstReg, ResultRegs[0]);
return true;
}
assert(Ty.isVector());
if (IsD16) {
// For packed D16 results with TFE enabled, all the data components are
// I32. Cast back to the expected type.
//
// TODO: We don't really need to use load i32 elements. We would only need
// one cast for the TFE result if a multiple of v2f16 was used.
if (RegTy != V2I16 && !ST.hasUnpackedD16VMem()) {
for (Register &Reg : ResultRegs)
Reg = B.buildBitcast(V2I16, Reg).getReg(0);
} else if (ST.hasUnpackedD16VMem()) {
for (Register &Reg : ResultRegs)
Reg = B.buildTrunc(I16, Reg).getReg(0);
}
}
auto padWithUndef = [&](LLT Ty, int NumElts) {
if (NumElts == 0)
return;
Register Undef = B.buildUndef(Ty).getReg(0);
for (int I = 0; I != NumElts; ++I)
ResultRegs.push_back(Undef);
};
// Pad out any elements eliminated due to the dmask.
LLT ResTy = MRI->getType(ResultRegs[0]);
if (!ResTy.isVector()) {
padWithUndef(ResTy, NumElts - ResultRegs.size());
B.buildBuildVector(DstReg, ResultRegs);
return true;
}
assert(!ST.hasUnpackedD16VMem() && (ResTy == V2I16 || ResTy == V2F16));
const int RegsToCover = (Ty.getSizeInBits() + 31) / 32;
// Deal with the one annoying legal case.
const LLT V3I16 = LLT::fixed_vector(3, I16);
const LLT V3F16 = LLT::fixed_vector(3, F16);
if (Ty == V3I16 || Ty == V3F16) {
if (IsTFE) {
if (ResultRegs.size() == 1) {
NewResultReg = ResultRegs[0];
} else if (ResultRegs.size() == 2) {
LLT V4I16 = LLT::fixed_vector(4, I16);
NewResultReg = B.buildConcatVectors(V4I16, ResultRegs).getReg(0);
} else {
return false;
}
}
LLT DstTy = MRI->getType(DstReg);
LLT NewResTy = MRI->getType(NewResultReg);
LLT ResEltTy = NewResTy.getElementType();
Register ResizeDst = DstTy.getElementType() == ResEltTy
? DstReg
: MRI->createGenericVirtualRegister(
DstTy.changeElementType(ResEltTy));
if (DstTy.getNumElements() < NewResTy.getNumElements()) {
B.buildDeleteTrailingVectorElements(ResizeDst, NewResultReg);
} else {
B.buildPadVectorWithUndefElements(ResizeDst, NewResultReg);
}
if (ResizeDst != DstReg)
B.buildBitcast(DstReg, ResizeDst);
return true;
}
padWithUndef(ResTy, RegsToCover - ResultRegs.size());
B.buildConcatVectors(DstReg, ResultRegs);
return true;
}
bool AMDGPULegalizerInfo::legalizeSBufferLoad(LegalizerHelper &Helper,
MachineInstr &MI) const {
MachineIRBuilder &B = Helper.MIRBuilder;
GISelChangeObserver &Observer = Helper.Observer;
Register OrigDst = MI.getOperand(0).getReg();
Register Dst;
LLT Ty = B.getMRI()->getType(OrigDst);
unsigned Size = Ty.getSizeInBits();
MachineFunction &MF = B.getMF();
bool HasMMO = !MI.memoperands_empty();
// S_BUFFER_LOAD only produces values that fill whole SGPRs, apart from the
// subword loads below. Those truncate from an s32 result, so they need a
// scalar destination.
bool IsSubwordLoad = Ty.isScalar() && Size < 32 && ST.hasScalarSubwordLoads();
if (Size % 32 != 0 && !IsSubwordLoad) {
const Function &Fn = MF.getFunction();
Fn.getContext().diagnose(DiagnosticInfoUnsupported(
Fn, "unsupported s_buffer_load result type", MI.getDebugLoc()));
B.buildUndef(OrigDst);
MI.eraseFromParent();
return true;
}
unsigned Opc = 0;
if (IsSubwordLoad) {
assert(Size == 8 || Size == 16);
Opc = Size == 8 ? AMDGPU::G_AMDGPU_S_BUFFER_LOAD_UBYTE
: AMDGPU::G_AMDGPU_S_BUFFER_LOAD_USHORT;
// The 8-bit and 16-bit scalar buffer load instructions have 32-bit
// destination register.
Dst = B.getMRI()->createGenericVirtualRegister(LLT::integer(32));
} else {
Opc = AMDGPU::G_AMDGPU_S_BUFFER_LOAD;
Dst = OrigDst;
}
Observer.changingInstr(MI);
// Handle needing to s.buffer.load() a p8 value.
if (hasBufferRsrcWorkaround(Ty)) {
Ty = castBufferRsrcFromV4I32(MI, B, *B.getMRI(), 0);
B.setInsertPt(B.getMBB(), MI);
}
if (shouldBitcastLoadStoreType(ST, Ty, LLT::integer(Size))) {
Ty = getBitcastRegisterType(Ty);
Helper.bitcastDst(MI, Ty, 0);
B.setInsertPt(B.getMBB(), MI);
}
MI.setDesc(B.getTII().get(Opc));
MI.removeOperand(1);
castBufferRsrcArgToV4I32(MI, B, 1);
if (!HasMMO) {
// Legacy intrinsic that doesn't take a pointer and so can't already have an
// MMO.
const unsigned MemSize = (Size + 7) / 8;
const Align MemAlign = B.getDataLayout().getABITypeAlign(
getTypeForLLT(Ty, MF.getFunction().getContext()));
MachineMemOperand *MMO = MF.getMachineMemOperand(
MachinePointerInfo(),
MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
MachineMemOperand::MOInvariant,
MemSize, MemAlign);
MI.addMemOperand(MF, MMO);
}
if (Dst != OrigDst) {
MI.getOperand(0).setReg(Dst);
B.setInsertPt(B.getMBB(), ++B.getInsertPt());
B.buildTrunc(OrigDst, Dst);
}
// If we don't have 96-bit result scalar loads, widening to 128-bit should
// always be legal. We may need to restore this to a 96-bit result if it turns
// out this needs to be converted to a vector load during RegBankSelect.
if (!isPowerOf2_32(Size) && (Size != 96 || !ST.hasScalarDwordx3Loads())) {
if (Ty.isVector())
Helper.moreElementsVectorDst(MI, getPow2VectorType(Ty), 0);
else
Helper.widenScalarDst(MI, getPow2ScalarType(Ty), 0);
}
Observer.changedInstr(MI);
return true;
}
bool AMDGPULegalizerInfo::legalizeSBufferPrefetch(LegalizerHelper &Helper,
MachineInstr &MI) const {
MachineIRBuilder &B = Helper.MIRBuilder;
GISelChangeObserver &Observer = Helper.Observer;
Observer.changingInstr(MI);
MI.setDesc(B.getTII().get(AMDGPU::G_AMDGPU_S_BUFFER_PREFETCH));
MI.removeOperand(0); // Remove intrinsic ID
castBufferRsrcArgToV4I32(MI, B, 0);
Observer.changedInstr(MI);
return true;
}
// TODO: Move to selection
bool AMDGPULegalizerInfo::legalizeTrap(LegalizerHelper &Helper,
MachineInstr &MI) const {
MachineIRBuilder &B = Helper.MIRBuilder;
MachineRegisterInfo &MRI = *B.getMRI();
if (!ST.hasTrapHandler() ||
ST.getTrapHandlerAbi() != GCNSubtarget::TrapHandlerAbi::AMDHSA)
return legalizeTrapEndpgm(Helper, MI);
return ST.supportsGetDoorbellID() ?
legalizeTrapHsa(MI, MRI, B) : legalizeTrapHsaQueuePtr(MI, MRI, B);
}
bool AMDGPULegalizerInfo::legalizeTrapEndpgm(LegalizerHelper &Helper,
MachineInstr &MI) const {
MachineIRBuilder &B = Helper.MIRBuilder;
GISelChangeObserver &Observer = Helper.Observer;
const DebugLoc &DL = MI.getDebugLoc();
MachineBasicBlock &BB = B.getMBB();
MachineFunction *MF = BB.getParent();
if (BB.succ_empty() && std::next(MI.getIterator()) == BB.end()) {
BuildMI(BB, BB.end(), DL, B.getTII().get(AMDGPU::S_ENDPGM))
.addImm(0);
MI.eraseFromParent();
return true;
}
// We need a block split to make the real endpgm a terminator. We also don't
// want to break phis in successor blocks, so we can't just delete to the
// end of the block.
// An instruction's parent block is part of its CSE profile, so notify
// observers about the instructions moved by the split.
SmallVector<MachineInstr *, 8> MovedInstrs;
MachineBasicBlock::iterator SplitPoint(&MI);
++SplitPoint;
for (auto I = SplitPoint, E = BB.end(); I != E; ++I) {
Observer.changingInstr(*I);
MovedInstrs.push_back(&*I);
}
BB.splitAt(MI, false /*UpdateLiveIns*/);
for (MachineInstr *MovedMI : MovedInstrs)
Observer.changedInstr(*MovedMI);
MachineBasicBlock *TrapBB = MF->CreateMachineBasicBlock();
MF->push_back(TrapBB);
BuildMI(*TrapBB, TrapBB->end(), DL, B.getTII().get(AMDGPU::S_ENDPGM))
.addImm(0);
BuildMI(BB, &MI, DL, B.getTII().get(AMDGPU::S_CBRANCH_EXECNZ))
.addMBB(TrapBB);
BB.addSuccessor(TrapBB);
MI.eraseFromParent();
return true;
}
bool AMDGPULegalizerInfo::legalizeTrapHsaQueuePtr(
MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const {
MachineFunction &MF = B.getMF();
const LLT I64 = LLT::integer(64);
Register SGPR01(AMDGPU::SGPR0_SGPR1);
// For code object version 5, queue_ptr is passed through implicit kernarg.
if (AMDGPU::getAMDHSACodeObjectVersion(*MF.getFunction().getParent()) >=
AMDGPU::AMDHSA_COV5) {
AMDGPUTargetLowering::ImplicitParameter Param =
AMDGPUTargetLowering::QUEUE_PTR;
uint64_t Offset =
ST.getTargetLowering()->getImplicitParameterOffset(B.getMF(), Param);
Register KernargPtrReg = MRI.createGenericVirtualRegister(
LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64));
if (!loadInputValue(KernargPtrReg, B,
AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR))
return false;
// TODO: can we be smarter about machine pointer info?
MachinePointerInfo PtrInfo = getKernargSegmentPtrInfo(MF);
MachineMemOperand *MMO = MF.getMachineMemOperand(
PtrInfo.getWithOffset(Offset),
MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
MachineMemOperand::MOInvariant,
LLT::integer(64), commonAlignment(Align(64), Offset));
// Pointer address
Register LoadAddr = MRI.createGenericVirtualRegister(
LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64));
B.buildObjectPtrOffset(LoadAddr, KernargPtrReg,
B.buildConstant(LLT::integer(64), Offset).getReg(0));
// Load address
Register Temp = B.buildLoad(I64, LoadAddr, *MMO).getReg(0);
B.buildCopy(SGPR01, Temp);
B.buildInstr(AMDGPU::S_TRAP)
.addImm(static_cast<unsigned>(GCNSubtarget::TrapID::LLVMAMDHSATrap))
.addReg(SGPR01, RegState::Implicit);
MI.eraseFromParent();
return true;
}
// Pass queue pointer to trap handler as input, and insert trap instruction
// Reference: https://llvm.org/docs/AMDGPUUsage.html#trap-handler-abi
Register LiveIn =
MRI.createGenericVirtualRegister(LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64));
if (!loadInputValue(LiveIn, B, AMDGPUFunctionArgInfo::QUEUE_PTR))
return false;
B.buildCopy(SGPR01, LiveIn);
B.buildInstr(AMDGPU::S_TRAP)
.addImm(static_cast<unsigned>(GCNSubtarget::TrapID::LLVMAMDHSATrap))
.addReg(SGPR01, RegState::Implicit);
MI.eraseFromParent();
return true;
}
bool AMDGPULegalizerInfo::legalizeTrapHsa(MachineInstr &MI,
MachineRegisterInfo &MRI,
MachineIRBuilder &B) const {
// We need to simulate the 's_trap 2' instruction on targets that run in
// PRIV=1 (where it is treated as a nop).
if (ST.hasPrivEnabledTrap2NopBug()) {
ST.getInstrInfo()->insertSimulatedTrap(MRI, B.getMBB(), MI,
MI.getDebugLoc());
MI.eraseFromParent();
return true;
}
B.buildInstr(AMDGPU::S_TRAP)
.addImm(static_cast<unsigned>(GCNSubtarget::TrapID::LLVMAMDHSATrap));
MI.eraseFromParent();
return true;
}
bool AMDGPULegalizerInfo::legalizeDebugTrap(MachineInstr &MI,
MachineRegisterInfo &MRI,
MachineIRBuilder &B) const {
// Is non-HSA path or trap-handler disabled? Then, report a warning
// accordingly
if (!ST.hasTrapHandler() ||
ST.getTrapHandlerAbi() != GCNSubtarget::TrapHandlerAbi::AMDHSA) {
Function &Fn = B.getMF().getFunction();
Fn.getContext().diagnose(DiagnosticInfoUnsupported(
Fn, "debugtrap handler not supported", MI.getDebugLoc(), DS_Warning));
} else {
// Insert debug-trap instruction
B.buildInstr(AMDGPU::S_TRAP)
.addImm(static_cast<unsigned>(GCNSubtarget::TrapID::LLVMAMDHSADebugTrap));
}
MI.eraseFromParent();
return true;
}
bool AMDGPULegalizerInfo::legalizeBVHIntersectRayIntrinsic(
MachineInstr &MI, MachineIRBuilder &B) const {
MachineRegisterInfo &MRI = *B.getMRI();
const LLT I16 = LLT::integer(16);
const LLT I32 = LLT::integer(32);
const LLT V2I16 = LLT::fixed_vector(2, I16);
const LLT V3I32 = LLT::fixed_vector(3, I32);
const LLT V3I16 = LLT::fixed_vector(3, I16);
Register DstReg = MI.getOperand(0).getReg();
Register NodePtr = MI.getOperand(2).getReg();
Register RayExtent = MI.getOperand(3).getReg();
Register RayOrigin = MI.getOperand(4).getReg();
Register RayDir = MI.getOperand(5).getReg();
Register RayInvDir = MI.getOperand(6).getReg();
Register TDescr = MI.getOperand(7).getReg();
RayExtent = B.buildBitcast(I32, RayExtent).getReg(0);
const bool IsGFX11 = AMDGPU::isGFX11(ST);
const bool IsGFX11Plus = AMDGPU::isGFX11Plus(ST);
const bool IsGFX12Plus = AMDGPU::isGFX12Plus(ST);
const bool IsA16 = MRI.getType(RayDir).getElementType().getSizeInBits() == 16;
const bool Is64 = MRI.getType(NodePtr).getSizeInBits() == 64;
const unsigned NumVDataDwords = 4;
const unsigned NumVAddrDwords = IsA16 ? (Is64 ? 9 : 8) : (Is64 ? 12 : 11);
const unsigned NumVAddrs = IsGFX11Plus ? (IsA16 ? 4 : 5) : NumVAddrDwords;
const bool UseNSA =
IsGFX12Plus || (ST.hasNSAEncoding() && NumVAddrs <= ST.getNSAMaxSize());
const unsigned BaseOpcodes[2][2] = {
{AMDGPU::IMAGE_BVH_INTERSECT_RAY, AMDGPU::IMAGE_BVH_INTERSECT_RAY_a16},
{AMDGPU::IMAGE_BVH64_INTERSECT_RAY,
AMDGPU::IMAGE_BVH64_INTERSECT_RAY_a16}};
int Opcode;
if (UseNSA) {
Opcode = AMDGPU::getMIMGOpcode(BaseOpcodes[Is64][IsA16],
IsGFX12Plus ? AMDGPU::MIMGEncGfx12
: IsGFX11 ? AMDGPU::MIMGEncGfx11NSA
: AMDGPU::MIMGEncGfx10NSA,
NumVDataDwords, NumVAddrDwords);
} else {
assert(!IsGFX12Plus);
Opcode = AMDGPU::getMIMGOpcode(BaseOpcodes[Is64][IsA16],
IsGFX11 ? AMDGPU::MIMGEncGfx11Default
: AMDGPU::MIMGEncGfx10Default,
NumVDataDwords, NumVAddrDwords);
}
assert(Opcode != -1);
SmallVector<Register, 12> Ops;
if (UseNSA && IsGFX11Plus) {
auto packLanes = [&Ops, &I32, &V3I32, &B](Register Src) {
auto SrcInt = B.buildBitcast(V3I32, Src);
auto Unmerge = B.buildUnmerge({I32, I32, I32}, SrcInt);
auto Merged = B.buildMergeLikeInstr(
V3I32, {Unmerge.getReg(0), Unmerge.getReg(1), Unmerge.getReg(2)});
Ops.push_back(Merged.getReg(0));
};
Ops.push_back(NodePtr);
Ops.push_back(RayExtent);
packLanes(RayOrigin);
if (IsA16) {
auto UnmergeRayDir =
B.buildUnmerge({I16, I16, I16}, B.buildBitcast(V3I16, RayDir));
auto UnmergeRayInvDir =
B.buildUnmerge({I16, I16, I16}, B.buildBitcast(V3I16, RayInvDir));
auto MergedDir = B.buildMergeLikeInstr(
V3I32,
{B.buildBitcast(
I32, B.buildMergeLikeInstr(V2I16, {UnmergeRayInvDir.getReg(0),
UnmergeRayDir.getReg(0)}))
.getReg(0),
B.buildBitcast(
I32, B.buildMergeLikeInstr(V2I16, {UnmergeRayInvDir.getReg(1),
UnmergeRayDir.getReg(1)}))
.getReg(0),
B.buildBitcast(
I32, B.buildMergeLikeInstr(V2I16, {UnmergeRayInvDir.getReg(2),
UnmergeRayDir.getReg(2)}))
.getReg(0)});
Ops.push_back(MergedDir.getReg(0));
} else {
packLanes(RayDir);
packLanes(RayInvDir);
}
} else {
if (Is64) {
auto Unmerge = B.buildUnmerge({I32, I32}, NodePtr);
Ops.push_back(Unmerge.getReg(0));
Ops.push_back(Unmerge.getReg(1));
} else {
Ops.push_back(NodePtr);
}
Ops.push_back(RayExtent);
auto packLanes = [&Ops, &I32, &V3I32, &B](Register Src) {
auto SrcInt = B.buildBitcast(V3I32, Src);
auto Unmerge = B.buildUnmerge({I32, I32, I32}, SrcInt);
Ops.push_back(Unmerge.getReg(0));
Ops.push_back(Unmerge.getReg(1));
Ops.push_back(Unmerge.getReg(2));
};
packLanes(RayOrigin);
if (IsA16) {
auto UnmergeRayDir =
B.buildUnmerge({I16, I16, I16}, B.buildBitcast(V3I16, RayDir));
auto UnmergeRayInvDir =
B.buildUnmerge({I16, I16, I16}, B.buildBitcast(V3I16, RayInvDir));
Register R1 = MRI.createGenericVirtualRegister(I32);
Register R2 = MRI.createGenericVirtualRegister(I32);
Register R3 = MRI.createGenericVirtualRegister(I32);
B.buildMergeLikeInstr(R1,
{UnmergeRayDir.getReg(0), UnmergeRayDir.getReg(1)});
B.buildMergeLikeInstr(
R2, {UnmergeRayDir.getReg(2), UnmergeRayInvDir.getReg(0)});
B.buildMergeLikeInstr(
R3, {UnmergeRayInvDir.getReg(1), UnmergeRayInvDir.getReg(2)});
Ops.push_back(R1);
Ops.push_back(R2);
Ops.push_back(R3);
} else {
packLanes(RayDir);
packLanes(RayInvDir);
}
}
if (!UseNSA) {
// Build a single vector containing all the operands so far prepared.
LLT OpTy = LLT::fixed_vector(Ops.size(), I32);
Register MergedOps = B.buildMergeLikeInstr(OpTy, Ops).getReg(0);
Ops.clear();
Ops.push_back(MergedOps);
}
auto MIB = B.buildInstr(AMDGPU::G_AMDGPU_BVH_INTERSECT_RAY)
.addDef(DstReg)
.addImm(Opcode);
for (Register R : Ops) {
MIB.addUse(R);
}
MIB.addUse(TDescr)
.addImm(IsA16 ? 1 : 0)
.cloneMemRefs(MI);
MI.eraseFromParent();
return true;
}
bool AMDGPULegalizerInfo::legalizeBVHDualOrBVH8IntersectRayIntrinsic(
MachineInstr &MI, MachineIRBuilder &B) const {
const LLT I32 = LLT::integer(32);
const LLT V2I32 = LLT::fixed_vector(2, I32);
Register DstReg = MI.getOperand(0).getReg();
Register DstOrigin = MI.getOperand(1).getReg();
Register DstDir = MI.getOperand(2).getReg();
Register NodePtr = MI.getOperand(4).getReg();
Register RayExtent = MI.getOperand(5).getReg();
Register InstanceMask = MI.getOperand(6).getReg();
Register RayOrigin = MI.getOperand(7).getReg();
Register RayDir = MI.getOperand(8).getReg();
Register Offsets = MI.getOperand(9).getReg();
Register TDescr = MI.getOperand(10).getReg();
bool IsBVH8 = cast<GIntrinsic>(MI).getIntrinsicID() ==
Intrinsic::amdgcn_image_bvh8_intersect_ray;
const unsigned NumVDataDwords = 10;
const unsigned NumVAddrDwords = IsBVH8 ? 11 : 12;
int Opcode = AMDGPU::getMIMGOpcode(
IsBVH8 ? AMDGPU::IMAGE_BVH8_INTERSECT_RAY
: AMDGPU::IMAGE_BVH_DUAL_INTERSECT_RAY,
AMDGPU::MIMGEncGfx12, NumVDataDwords, NumVAddrDwords);
assert(Opcode != -1);
auto RayExtentInstanceMaskVec =
B.buildMergeLikeInstr(V2I32, {B.buildBitcast(I32, RayExtent),
B.buildAnyExt(I32, InstanceMask)});
B.buildInstr(IsBVH8 ? AMDGPU::G_AMDGPU_BVH8_INTERSECT_RAY
: AMDGPU::G_AMDGPU_BVH_DUAL_INTERSECT_RAY)
.addDef(DstReg)
.addDef(DstOrigin)
.addDef(DstDir)
.addImm(Opcode)
.addUse(NodePtr)
.addUse(RayExtentInstanceMaskVec.getReg(0))
.addUse(RayOrigin)
.addUse(RayDir)
.addUse(Offsets)
.addUse(TDescr)
.cloneMemRefs(MI);
MI.eraseFromParent();
return true;
}
bool AMDGPULegalizerInfo::legalizeStackSave(MachineInstr &MI,
MachineIRBuilder &B) const {
const SITargetLowering *TLI = ST.getTargetLowering();
Register StackPtr = TLI->getStackPointerRegisterToSaveRestore();
Register DstReg = MI.getOperand(0).getReg();
B.buildInstr(AMDGPU::G_AMDGPU_WAVE_ADDRESS, {DstReg}, {StackPtr});
MI.eraseFromParent();
return true;
}
bool AMDGPULegalizerInfo::legalizeWaveID(MachineInstr &MI,
MachineIRBuilder &B) const {
// With architected SGPRs, waveIDinGroup is in TTMP8[29:25].
if (!ST.hasArchitectedSGPRs())
return false;
LLT I32 = LLT::integer(32);
Register DstReg = MI.getOperand(0).getReg();
auto TTMP8 = B.buildCopy(I32, Register(AMDGPU::TTMP8));
auto LSB = B.buildConstant(I32, 25);
auto Width = B.buildConstant(I32, 5);
B.buildUbfx(DstReg, TTMP8, LSB, Width);
MI.eraseFromParent();
return true;
}
bool AMDGPULegalizerInfo::legalizeConstHwRegRead(MachineInstr &MI,
MachineIRBuilder &B,
AMDGPU::Hwreg::Id HwReg,
unsigned LowBit,
unsigned Width) const {
MachineRegisterInfo &MRI = *B.getMRI();
Register DstReg = MI.getOperand(0).getReg();
Register Result = MRI.createVirtualRegister(
{&AMDGPU::SReg_32RegClass, MRI.getType(DstReg)});
B.buildInstr(AMDGPU::S_GETREG_B32_const)
.addDef(Result)
.addImm(AMDGPU::Hwreg::HwregEncoding::encode(HwReg, LowBit, Width));
B.buildCopy(DstReg, Result);
MI.eraseFromParent();
return true;
}
static constexpr unsigned FPEnvModeBitField =
AMDGPU::Hwreg::HwregEncoding::encode(AMDGPU::Hwreg::ID_MODE, 0, 23);
static constexpr unsigned FPEnvTrapBitField =
AMDGPU::Hwreg::HwregEncoding::encode(AMDGPU::Hwreg::ID_TRAPSTS, 0, 5);
bool AMDGPULegalizerInfo::legalizeGetFPEnv(MachineInstr &MI,
MachineRegisterInfo &MRI,
MachineIRBuilder &B) const {
const LLT I32 = LLT::integer(32);
const LLT I64 = LLT::integer(64);
Register Src = MI.getOperand(0).getReg();
if (MRI.getType(Src) != I64)
return false;
auto ModeReg =
B.buildIntrinsic(Intrinsic::amdgcn_s_getreg, {I32},
/*HasSideEffects=*/true, /*isConvergent=*/false)
.addImm(FPEnvModeBitField);
auto TrapReg =
B.buildIntrinsic(Intrinsic::amdgcn_s_getreg, {I32},
/*HasSideEffects=*/true, /*isConvergent=*/false)
.addImm(FPEnvTrapBitField);
B.buildMergeLikeInstr(Src, {ModeReg, TrapReg});
MI.eraseFromParent();
return true;
}
bool AMDGPULegalizerInfo::legalizeSetFPEnv(MachineInstr &MI,
MachineRegisterInfo &MRI,
MachineIRBuilder &B) const {
const LLT I32 = LLT::integer(32);
const LLT I64 = LLT::integer(64);
Register Src = MI.getOperand(0).getReg();
if (MRI.getType(Src) != I64)
return false;
auto Unmerge = B.buildUnmerge({I32, I32}, MI.getOperand(0));
B.buildIntrinsic(Intrinsic::amdgcn_s_setreg, ArrayRef<DstOp>(),
/*HasSideEffects=*/true, /*isConvergent=*/false)
.addImm(static_cast<int16_t>(FPEnvModeBitField))
.addReg(Unmerge.getReg(0));
B.buildIntrinsic(Intrinsic::amdgcn_s_setreg, ArrayRef<DstOp>(),
/*HasSideEffects=*/true, /*isConvergent=*/false)
.addImm(static_cast<int16_t>(FPEnvTrapBitField))
.addReg(Unmerge.getReg(1));
MI.eraseFromParent();
return true;
}
bool AMDGPULegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
MachineInstr &MI) const {
MachineIRBuilder &B = Helper.MIRBuilder;
MachineRegisterInfo &MRI = *B.getMRI();
// Replace the use G_BRCOND with the exec manipulate and branch pseudos.
auto IntrID = cast<GIntrinsic>(MI).getIntrinsicID();
switch (IntrID) {
case Intrinsic::sponentry:
if (B.getMF().getInfo<SIMachineFunctionInfo>()->isBottomOfStack()) {
// FIXME: The imported pattern checks for i32 instead of p5; if we fix
// that we can remove this cast.
const LLT I32 = LLT::integer(32);
Register TmpReg = MRI.createGenericVirtualRegister(I32);
B.buildInstr(AMDGPU::G_AMDGPU_SPONENTRY).addDef(TmpReg);
Register DstReg = MI.getOperand(0).getReg();
B.buildIntToPtr(DstReg, TmpReg);
MI.eraseFromParent();
} else {
int FI = B.getMF().getFrameInfo().CreateFixedObject(
1, 0, /*IsImmutable=*/false);
B.buildFrameIndex(MI.getOperand(0), FI);
MI.eraseFromParent();
}
return true;
case Intrinsic::amdgcn_if:
case Intrinsic::amdgcn_else: {
MachineInstr *Br = nullptr;
MachineBasicBlock *UncondBrTarget = nullptr;
bool Negated = false;
if (MachineInstr *BrCond =
verifyCFIntrinsic(MI, MRI, Br, UncondBrTarget, Negated)) {
const SIRegisterInfo *TRI
= static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());
Register Def = MI.getOperand(1).getReg();
Register Use = MI.getOperand(3).getReg();
const TargetRegisterClass *WaveMaskRC = TRI->getWaveMaskRegClass();
Register NewDef =
MRI.createVirtualRegister({WaveMaskRC, MRI.getType(Def)});
Register NewUse =
MRI.createVirtualRegister({WaveMaskRC, MRI.getType(Use)});
MachineBasicBlock *CondBrTarget = BrCond->getOperand(1).getMBB();
if (Negated)
std::swap(CondBrTarget, UncondBrTarget);
B.setInsertPt(B.getMBB(), BrCond->getIterator());
B.buildCopy(NewUse, Use);
if (IntrID == Intrinsic::amdgcn_if) {
B.buildInstr(AMDGPU::SI_IF)
.addDef(NewDef)
.addUse(NewUse)
.addMBB(UncondBrTarget)
.setOperandDead(4); // implicit-def $scc
} else {
B.buildInstr(AMDGPU::SI_ELSE)
.addDef(NewDef)
.addUse(NewUse)
.addMBB(UncondBrTarget)
.setOperandDead(4); // implicit-def $scc
}
if (Br) {
Br->getOperand(0).setMBB(CondBrTarget);
} else {
// The IRTranslator skips inserting the G_BR for fallthrough cases, but
// since we're swapping branch targets it needs to be reinserted.
// FIXME: IRTranslator should probably not do this
B.buildBr(*CondBrTarget);
}
MI.eraseFromParent();
BrCond->eraseFromParent();
// SI_IF and SI_ELSE are terminators, so replace uses of Def rather than
// defining Def with a following copy.
Helper.Observer.changingAllUsesOfReg(MRI, Def);
MRI.replaceRegWith(Def, NewDef);
Helper.Observer.finishedChangingAllUsesOfReg();
return true;
}
return false;
}
case Intrinsic::amdgcn_loop: {
MachineInstr *Br = nullptr;
MachineBasicBlock *UncondBrTarget = nullptr;
bool Negated = false;
if (MachineInstr *BrCond =
verifyCFIntrinsic(MI, MRI, Br, UncondBrTarget, Negated)) {
const SIRegisterInfo *TRI
= static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());
MachineBasicBlock *CondBrTarget = BrCond->getOperand(1).getMBB();
Register Reg = MI.getOperand(2).getReg();
Register NewReg = MRI.createVirtualRegister(
{TRI->getWaveMaskRegClass(), MRI.getType(Reg)});
if (Negated)
std::swap(CondBrTarget, UncondBrTarget);
B.setInsertPt(B.getMBB(), BrCond->getIterator());
B.buildCopy(NewReg, Reg);
B.buildInstr(AMDGPU::SI_LOOP)
.addUse(NewReg)
.addMBB(UncondBrTarget)
.setOperandDead(3); // implicit-def $scc
if (Br)
Br->getOperand(0).setMBB(CondBrTarget);
else
B.buildBr(*CondBrTarget);
MI.eraseFromParent();
BrCond->eraseFromParent();
return true;
}
return false;
}
case Intrinsic::amdgcn_wave_reduce_min:
case Intrinsic::amdgcn_wave_reduce_umin:
case Intrinsic::amdgcn_wave_reduce_fmin:
case Intrinsic::amdgcn_wave_reduce_max:
case Intrinsic::amdgcn_wave_reduce_umax:
case Intrinsic::amdgcn_wave_reduce_fmax:
case Intrinsic::amdgcn_wave_reduce_add:
case Intrinsic::amdgcn_wave_reduce_fadd:
case Intrinsic::amdgcn_wave_reduce_sub:
case Intrinsic::amdgcn_wave_reduce_fsub:
case Intrinsic::amdgcn_wave_reduce_and:
case Intrinsic::amdgcn_wave_reduce_or:
case Intrinsic::amdgcn_wave_reduce_xor: {
Register SrcReg = MI.getOperand(2).getReg();
if (MRI.getType(SrcReg).getSizeInBits() != 16)
return true;
Register DstReg = MI.getOperand(0).getReg();
bool IsFPOp = IntrID == Intrinsic::amdgcn_wave_reduce_fmin ||
IntrID == Intrinsic::amdgcn_wave_reduce_fmax ||
IntrID == Intrinsic::amdgcn_wave_reduce_fadd ||
IntrID == Intrinsic::amdgcn_wave_reduce_fsub;
bool NeedsSignExt = IntrID == Intrinsic::amdgcn_wave_reduce_min ||
IntrID == Intrinsic::amdgcn_wave_reduce_max ||
IntrID == Intrinsic::amdgcn_wave_reduce_add ||
IntrID == Intrinsic::amdgcn_wave_reduce_sub;
auto Ext = IsFPOp ? B.buildFPExt(F32, SrcReg)
: NeedsSignExt ? B.buildSExt(LLT::integer(32), SrcReg)
: B.buildZExt(LLT::integer(32), SrcReg);
auto NewDst =
MRI.createGenericVirtualRegister(IsFPOp ? F32 : LLT::integer(32));
B.buildIntrinsic(IntrID, ArrayRef<Register>{NewDst},
/*hasSideEffects=*/false, /*isConvergent=*/true)
.addUse(Ext.getReg(0))
.addImm(MI.getOperand(3).getImm()); // strategy
if (IsFPOp)
B.buildFPTrunc(DstReg, NewDst);
else
B.buildTrunc(DstReg, NewDst);
MI.eraseFromParent();
return true;
}
case Intrinsic::amdgcn_make_buffer_rsrc:
return legalizePointerAsRsrcIntrin(MI, MRI, B);
case Intrinsic::amdgcn_kernarg_segment_ptr:
if (!AMDGPU::isKernel(B.getMF().getFunction())) {
// This only makes sense to call in a kernel, so just lower to null.
B.buildConstant(MI.getOperand(0).getReg(), 0);
MI.eraseFromParent();
return true;
}
return legalizePreloadedArgIntrin(
MI, MRI, B, AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR);
case Intrinsic::amdgcn_implicitarg_ptr:
return legalizeImplicitArgPtr(MI, MRI, B);
case Intrinsic::amdgcn_workitem_id_x:
return legalizeWorkitemIDIntrinsic(MI, MRI, B, 0,
AMDGPUFunctionArgInfo::WORKITEM_ID_X);
case Intrinsic::amdgcn_workitem_id_y:
return legalizeWorkitemIDIntrinsic(MI, MRI, B, 1,
AMDGPUFunctionArgInfo::WORKITEM_ID_Y);
case Intrinsic::amdgcn_workitem_id_z:
return legalizeWorkitemIDIntrinsic(MI, MRI, B, 2,
AMDGPUFunctionArgInfo::WORKITEM_ID_Z);
case Intrinsic::amdgcn_workgroup_id_x:
return legalizeWorkGroupId(
MI, B, AMDGPUFunctionArgInfo::WORKGROUP_ID_X,
AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_X,
AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_X);
case Intrinsic::amdgcn_workgroup_id_y:
return legalizeWorkGroupId(
MI, B, AMDGPUFunctionArgInfo::WORKGROUP_ID_Y,
AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_Y,
AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_Y);
case Intrinsic::amdgcn_workgroup_id_z:
return legalizeWorkGroupId(
MI, B, AMDGPUFunctionArgInfo::WORKGROUP_ID_Z,
AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_Z,
AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_Z);
case Intrinsic::amdgcn_cluster_id_x:
return ST.hasClusters() &&
legalizePreloadedArgIntrin(MI, MRI, B,
AMDGPUFunctionArgInfo::WORKGROUP_ID_X);
case Intrinsic::amdgcn_cluster_id_y:
return ST.hasClusters() &&
legalizePreloadedArgIntrin(MI, MRI, B,
AMDGPUFunctionArgInfo::WORKGROUP_ID_Y);
case Intrinsic::amdgcn_cluster_id_z:
return ST.hasClusters() &&
legalizePreloadedArgIntrin(MI, MRI, B,
AMDGPUFunctionArgInfo::WORKGROUP_ID_Z);
case Intrinsic::amdgcn_cluster_workgroup_id_x:
return ST.hasClusters() &&
legalizePreloadedArgIntrin(
MI, MRI, B, AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_X);
case Intrinsic::amdgcn_cluster_workgroup_id_y:
return ST.hasClusters() &&
legalizePreloadedArgIntrin(
MI, MRI, B, AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_Y);
case Intrinsic::amdgcn_cluster_workgroup_id_z:
return ST.hasClusters() &&
legalizePreloadedArgIntrin(
MI, MRI, B, AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_ID_Z);
case Intrinsic::amdgcn_cluster_workgroup_flat_id:
return ST.hasClusters() &&
legalizeConstHwRegRead(MI, B, AMDGPU::Hwreg::ID_IB_STS2, 21, 4);
case Intrinsic::amdgcn_cluster_workgroup_max_id_x:
return ST.hasClusters() &&
legalizePreloadedArgIntrin(
MI, MRI, B, AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_X);
case Intrinsic::amdgcn_cluster_workgroup_max_id_y:
return ST.hasClusters() &&
legalizePreloadedArgIntrin(
MI, MRI, B, AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_Y);
case Intrinsic::amdgcn_cluster_workgroup_max_id_z:
return ST.hasClusters() &&
legalizePreloadedArgIntrin(
MI, MRI, B, AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_ID_Z);
case Intrinsic::amdgcn_cluster_workgroup_max_flat_id:
return ST.hasClusters() &&
legalizePreloadedArgIntrin(
MI, MRI, B,
AMDGPUFunctionArgInfo::CLUSTER_WORKGROUP_MAX_FLAT_ID);
case Intrinsic::amdgcn_wave_id:
return legalizeWaveID(MI, B);
case Intrinsic::amdgcn_lds_kernel_id:
return legalizePreloadedArgIntrin(MI, MRI, B,
AMDGPUFunctionArgInfo::LDS_KERNEL_ID);
case Intrinsic::amdgcn_dispatch_ptr:
return legalizePreloadedArgIntrin(MI, MRI, B,
AMDGPUFunctionArgInfo::DISPATCH_PTR);
case Intrinsic::amdgcn_queue_ptr:
return legalizePreloadedArgIntrin(MI, MRI, B,
AMDGPUFunctionArgInfo::QUEUE_PTR);
case Intrinsic::amdgcn_implicit_buffer_ptr:
return legalizePreloadedArgIntrin(
MI, MRI, B, AMDGPUFunctionArgInfo::IMPLICIT_BUFFER_PTR);
case Intrinsic::amdgcn_dispatch_id:
return legalizePreloadedArgIntrin(MI, MRI, B,
AMDGPUFunctionArgInfo::DISPATCH_ID);
case Intrinsic::r600_read_ngroups_x:
// TODO: Emit error for hsa
return legalizeKernargMemParameter(MI, B,
SI::KernelInputOffsets::NGROUPS_X);
case Intrinsic::r600_read_ngroups_y:
return legalizeKernargMemParameter(MI, B,
SI::KernelInputOffsets::NGROUPS_Y);
case Intrinsic::r600_read_ngroups_z:
return legalizeKernargMemParameter(MI, B,
SI::KernelInputOffsets::NGROUPS_Z);
case Intrinsic::r600_read_local_size_x:
// TODO: Could insert G_ASSERT_ZEXT from i16
return legalizeKernargMemParameter(MI, B, SI::KernelInputOffsets::LOCAL_SIZE_X);
case Intrinsic::r600_read_local_size_y:
// TODO: Could insert G_ASSERT_ZEXT from i16
return legalizeKernargMemParameter(MI, B, SI::KernelInputOffsets::LOCAL_SIZE_Y);
// TODO: Could insert G_ASSERT_ZEXT from i16
case Intrinsic::r600_read_local_size_z:
return legalizeKernargMemParameter(MI, B,
SI::KernelInputOffsets::LOCAL_SIZE_Z);
case Intrinsic::amdgcn_fdiv_fast:
return legalizeFDIVFastIntrin(MI, MRI, B);
case Intrinsic::amdgcn_is_shared:
return legalizeIsAddrSpace(MI, MRI, B, AMDGPUAS::LOCAL_ADDRESS);
case Intrinsic::amdgcn_is_private:
return legalizeIsAddrSpace(MI, MRI, B, AMDGPUAS::PRIVATE_ADDRESS);
case Intrinsic::amdgcn_wavefrontsize: {
B.buildConstant(MI.getOperand(0), ST.getWavefrontSize());
MI.eraseFromParent();
return true;
}
case Intrinsic::amdgcn_s_buffer_load:
case Intrinsic::amdgcn_ptr_s_buffer_load:
return legalizeSBufferLoad(Helper, MI);
case Intrinsic::amdgcn_raw_buffer_store:
case Intrinsic::amdgcn_raw_ptr_buffer_store:
case Intrinsic::amdgcn_struct_buffer_store:
case Intrinsic::amdgcn_struct_ptr_buffer_store:
return legalizeBufferStore(MI, Helper, false, false);
case Intrinsic::amdgcn_raw_buffer_store_format:
case Intrinsic::amdgcn_raw_ptr_buffer_store_format:
case Intrinsic::amdgcn_struct_buffer_store_format:
case Intrinsic::amdgcn_struct_ptr_buffer_store_format:
return legalizeBufferStore(MI, Helper, false, true);
case Intrinsic::amdgcn_raw_tbuffer_store:
case Intrinsic::amdgcn_raw_ptr_tbuffer_store:
case Intrinsic::amdgcn_struct_tbuffer_store:
case Intrinsic::amdgcn_struct_ptr_tbuffer_store:
return legalizeBufferStore(MI, Helper, true, true);
case Intrinsic::amdgcn_raw_buffer_load:
case Intrinsic::amdgcn_raw_ptr_buffer_load:
case Intrinsic::amdgcn_raw_atomic_buffer_load:
case Intrinsic::amdgcn_raw_ptr_atomic_buffer_load:
case Intrinsic::amdgcn_struct_buffer_load:
case Intrinsic::amdgcn_struct_ptr_buffer_load:
case Intrinsic::amdgcn_struct_atomic_buffer_load:
case Intrinsic::amdgcn_struct_ptr_atomic_buffer_load:
return legalizeBufferLoad(MI, Helper, false, false);
case Intrinsic::amdgcn_raw_buffer_load_format:
case Intrinsic::amdgcn_raw_ptr_buffer_load_format:
case Intrinsic::amdgcn_struct_buffer_load_format:
case Intrinsic::amdgcn_struct_ptr_buffer_load_format:
return legalizeBufferLoad(MI, Helper, true, false);
case Intrinsic::amdgcn_raw_tbuffer_load:
case Intrinsic::amdgcn_raw_ptr_tbuffer_load:
case Intrinsic::amdgcn_struct_tbuffer_load:
case Intrinsic::amdgcn_struct_ptr_tbuffer_load:
return legalizeBufferLoad(MI, Helper, true, true);
case Intrinsic::amdgcn_raw_buffer_atomic_swap:
case Intrinsic::amdgcn_raw_ptr_buffer_atomic_swap:
case Intrinsic::amdgcn_struct_buffer_atomic_swap:
case Intrinsic::amdgcn_struct_ptr_buffer_atomic_swap:
case Intrinsic::amdgcn_raw_buffer_atomic_add:
case Intrinsic::amdgcn_raw_ptr_buffer_atomic_add:
case Intrinsic::amdgcn_struct_buffer_atomic_add:
case Intrinsic::amdgcn_struct_ptr_buffer_atomic_add:
case Intrinsic::amdgcn_raw_buffer_atomic_sub:
case Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub:
case Intrinsic::amdgcn_struct_buffer_atomic_sub:
case Intrinsic::amdgcn_struct_ptr_buffer_atomic_sub:
case Intrinsic::amdgcn_raw_buffer_atomic_smin:
case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smin:
case Intrinsic::amdgcn_struct_buffer_atomic_smin:
case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smin:
case Intrinsic::amdgcn_raw_buffer_atomic_umin:
case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umin:
case Intrinsic::amdgcn_struct_buffer_atomic_umin:
case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umin:
case Intrinsic::amdgcn_raw_buffer_atomic_smax:
case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smax:
case Intrinsic::amdgcn_struct_buffer_atomic_smax:
case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smax:
case Intrinsic::amdgcn_raw_buffer_atomic_umax:
case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umax:
case Intrinsic::amdgcn_struct_buffer_atomic_umax:
case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umax:
case Intrinsic::amdgcn_raw_buffer_atomic_and:
case Intrinsic::amdgcn_raw_ptr_buffer_atomic_and:
case Intrinsic::amdgcn_struct_buffer_atomic_and:
case Intrinsic::amdgcn_struct_ptr_buffer_atomic_and:
case Intrinsic::amdgcn_raw_buffer_atomic_or:
case Intrinsic::amdgcn_raw_ptr_buffer_atomic_or:
case Intrinsic::amdgcn_struct_buffer_atomic_or:
case Intrinsic::amdgcn_struct_ptr_buffer_atomic_or:
case Intrinsic::amdgcn_raw_buffer_atomic_xor:
case Intrinsic::amdgcn_raw_ptr_buffer_atomic_xor:
case Intrinsic::amdgcn_struct_buffer_atomic_xor:
case Intrinsic::amdgcn_struct_ptr_buffer_atomic_xor:
case Intrinsic::amdgcn_raw_buffer_atomic_inc:
case Intrinsic::amdgcn_raw_ptr_buffer_atomic_inc:
case Intrinsic::amdgcn_struct_buffer_atomic_inc:
case Intrinsic::amdgcn_struct_ptr_buffer_atomic_inc:
case Intrinsic::amdgcn_raw_buffer_atomic_dec:
case Intrinsic::amdgcn_raw_ptr_buffer_atomic_dec:
case Intrinsic::amdgcn_struct_buffer_atomic_dec:
case Intrinsic::amdgcn_struct_ptr_buffer_atomic_dec:
case Intrinsic::amdgcn_raw_buffer_atomic_cmpswap:
case Intrinsic::amdgcn_raw_ptr_buffer_atomic_cmpswap:
case Intrinsic::amdgcn_struct_buffer_atomic_cmpswap:
case Intrinsic::amdgcn_struct_ptr_buffer_atomic_cmpswap:
case Intrinsic::amdgcn_raw_buffer_atomic_fmin:
case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmin:
case Intrinsic::amdgcn_struct_buffer_atomic_fmin:
case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fmin:
case Intrinsic::amdgcn_raw_buffer_atomic_fmax:
case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fmax:
case Intrinsic::amdgcn_struct_buffer_atomic_fmax:
case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fmax:
case Intrinsic::amdgcn_raw_buffer_atomic_sub_clamp_u32:
case Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub_clamp_u32:
case Intrinsic::amdgcn_struct_buffer_atomic_sub_clamp_u32:
case Intrinsic::amdgcn_struct_ptr_buffer_atomic_sub_clamp_u32:
case Intrinsic::amdgcn_raw_buffer_atomic_cond_sub_u32:
case Intrinsic::amdgcn_raw_ptr_buffer_atomic_cond_sub_u32:
case Intrinsic::amdgcn_struct_buffer_atomic_cond_sub_u32:
case Intrinsic::amdgcn_struct_ptr_buffer_atomic_cond_sub_u32:
case Intrinsic::amdgcn_raw_buffer_atomic_fadd:
case Intrinsic::amdgcn_raw_ptr_buffer_atomic_fadd:
case Intrinsic::amdgcn_struct_buffer_atomic_fadd:
case Intrinsic::amdgcn_struct_ptr_buffer_atomic_fadd:
return legalizeBufferAtomic(MI, B, IntrID);
case Intrinsic::amdgcn_rsq_clamp:
return legalizeRsqClampIntrinsic(MI, MRI, B);
case Intrinsic::amdgcn_image_bvh_intersect_ray:
return legalizeBVHIntersectRayIntrinsic(MI, B);
case Intrinsic::amdgcn_image_bvh_dual_intersect_ray:
case Intrinsic::amdgcn_image_bvh8_intersect_ray:
return legalizeBVHDualOrBVH8IntersectRayIntrinsic(MI, B);
case Intrinsic::amdgcn_swmmac_f32_16x16x128_fp8_fp8:
case Intrinsic::amdgcn_swmmac_f32_16x16x128_fp8_bf8:
case Intrinsic::amdgcn_swmmac_f32_16x16x128_bf8_fp8:
case Intrinsic::amdgcn_swmmac_f32_16x16x128_bf8_bf8:
case Intrinsic::amdgcn_swmmac_f16_16x16x128_fp8_fp8:
case Intrinsic::amdgcn_swmmac_f16_16x16x128_fp8_bf8:
case Intrinsic::amdgcn_swmmac_f16_16x16x128_bf8_fp8:
case Intrinsic::amdgcn_swmmac_f16_16x16x128_bf8_bf8: {
Register Index = MI.getOperand(5).getReg();
LLT I64 = LLT::integer(64);
LLT IndexArgTy = MRI.getType(Index);
if (IndexArgTy != I64) {
auto NewIndex = IndexArgTy.isVector() ? B.buildBitcast(I64, Index)
: B.buildAnyExt(I64, Index);
MI.getOperand(5).setReg(NewIndex.getReg(0));
}
return true;
}
case Intrinsic::amdgcn_swmmac_f16_16x16x32_f16:
case Intrinsic::amdgcn_swmmac_bf16_16x16x32_bf16:
case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf16:
case Intrinsic::amdgcn_swmmac_f32_16x16x32_f16:
case Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_fp8:
case Intrinsic::amdgcn_swmmac_f32_16x16x32_fp8_bf8:
case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_fp8:
case Intrinsic::amdgcn_swmmac_f32_16x16x32_bf8_bf8: {
Register Index = MI.getOperand(5).getReg();
LLT I32 = LLT::integer(32);
if (MRI.getType(Index) != I32)
MI.getOperand(5).setReg(B.buildAnyExt(I32, Index).getReg(0));
return true;
}
case Intrinsic::amdgcn_swmmac_f16_16x16x64_f16:
case Intrinsic::amdgcn_swmmac_bf16_16x16x64_bf16:
case Intrinsic::amdgcn_swmmac_f32_16x16x64_bf16:
case Intrinsic::amdgcn_swmmac_bf16f32_16x16x64_bf16:
case Intrinsic::amdgcn_swmmac_f32_16x16x64_f16:
case Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8:
case Intrinsic::amdgcn_swmmac_i32_16x16x32_iu4:
case Intrinsic::amdgcn_swmmac_i32_16x16x32_iu8:
case Intrinsic::amdgcn_swmmac_i32_16x16x64_iu4: {
Register Index = MI.getOperand(7).getReg();
LLT IdxTy = IntrID == Intrinsic::amdgcn_swmmac_i32_16x16x128_iu8
? LLT::integer(64)
: LLT::integer(32);
LLT IndexArgTy = MRI.getType(Index);
if (IndexArgTy != IdxTy) {
auto NewIndex = IndexArgTy.isVector() ? B.buildBitcast(IdxTy, Index)
: B.buildAnyExt(IdxTy, Index);
MI.getOperand(7).setReg(NewIndex.getReg(0));
}
return true;
}
case Intrinsic::amdgcn_fmed3: {
GISelChangeObserver &Observer = Helper.Observer;
// FIXME: This is to workaround the inability of tablegen match combiners to
// match intrinsics in patterns.
Observer.changingInstr(MI);
MI.setDesc(B.getTII().get(AMDGPU::G_AMDGPU_FMED3));
MI.removeOperand(1);
Observer.changedInstr(MI);
return true;
}
case Intrinsic::amdgcn_readlane:
case Intrinsic::amdgcn_writelane:
case Intrinsic::amdgcn_readfirstlane:
case Intrinsic::amdgcn_permlane16:
case Intrinsic::amdgcn_permlanex16:
case Intrinsic::amdgcn_permlane64:
case Intrinsic::amdgcn_set_inactive:
case Intrinsic::amdgcn_set_inactive_chain_arg:
case Intrinsic::amdgcn_mov_dpp8:
case Intrinsic::amdgcn_update_dpp:
case Intrinsic::amdgcn_permlane_bcast:
case Intrinsic::amdgcn_permlane_up:
case Intrinsic::amdgcn_permlane_down:
case Intrinsic::amdgcn_permlane_xor:
return legalizeLaneOp(Helper, MI, IntrID);
case Intrinsic::amdgcn_s_buffer_prefetch_data:
return legalizeSBufferPrefetch(Helper, MI);
case Intrinsic::amdgcn_dead: {
// TODO: Use poison instead of undef
for (const MachineOperand &Def : MI.defs())
B.buildUndef(Def);
MI.eraseFromParent();
return true;
}
case Intrinsic::amdgcn_cooperative_atomic_load_32x4B:
case Intrinsic::amdgcn_cooperative_atomic_load_16x8B:
case Intrinsic::amdgcn_cooperative_atomic_load_8x16B:
assert(MI.hasOneMemOperand() && "Expected IRTranslator to set MemOp!");
B.buildLoad(MI.getOperand(0), MI.getOperand(2), **MI.memoperands_begin());
MI.eraseFromParent();
return true;
case Intrinsic::amdgcn_cooperative_atomic_store_32x4B:
case Intrinsic::amdgcn_cooperative_atomic_store_16x8B:
case Intrinsic::amdgcn_cooperative_atomic_store_8x16B:
assert(MI.hasOneMemOperand() && "Expected IRTranslator to set MemOp!");
B.buildStore(MI.getOperand(2), MI.getOperand(1), **MI.memoperands_begin());
MI.eraseFromParent();
return true;
case Intrinsic::amdgcn_av_load_b128:
case Intrinsic::amdgcn_av_store_b128: {
assert(MI.hasOneMemOperand() && "Expected IRTranslator to set MemOp!");
if (IntrID == Intrinsic::amdgcn_av_load_b128)
B.buildLoad(MI.getOperand(0), MI.getOperand(2), **MI.memoperands_begin());
else
B.buildStore(MI.getOperand(2), MI.getOperand(1),
**MI.memoperands_begin());
MI.eraseFromParent();
return true;
}
case Intrinsic::amdgcn_flat_load_monitor_b32:
case Intrinsic::amdgcn_flat_load_monitor_b64:
case Intrinsic::amdgcn_flat_load_monitor_b128:
assert(MI.hasOneMemOperand() && "Expected IRTranslator to set MemOp!");
B.buildInstr(AMDGPU::G_AMDGPU_FLAT_LOAD_MONITOR)
.add(MI.getOperand(0))
.add(MI.getOperand(2))
.addMemOperand(*MI.memoperands_begin());
MI.eraseFromParent();
return true;
case Intrinsic::amdgcn_global_load_monitor_b32:
case Intrinsic::amdgcn_global_load_monitor_b64:
case Intrinsic::amdgcn_global_load_monitor_b128:
assert(MI.hasOneMemOperand() && "Expected IRTranslator to set MemOp!");
B.buildInstr(AMDGPU::G_AMDGPU_GLOBAL_LOAD_MONITOR)
.add(MI.getOperand(0))
.add(MI.getOperand(2))
.addMemOperand(*MI.memoperands_begin());
MI.eraseFromParent();
return true;
default: {
if (const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr =
AMDGPU::getImageDimIntrinsicInfo(IntrID))
return legalizeImageIntrinsic(MI, B, Helper.Observer, ImageDimIntr);
return true;
}
}
return true;
}