[SPIRV] support builtin types and ExtInsts selection
The patch adds the support of OpenCL and SPIR-V built-in types. It also
implements ExtInst selection and adds spv_unreachable and spv_alloca
intrinsics which improve the generation of the corresponding SPIR-V code.
Five LIT tests are included to demonstrate the improvement.
Differential Revision: https://reviews.llvm.org/D132648
Co-authored-by: Aleksandr Bezzubikov <zuban32s@gmail.com>
Co-authored-by: Michal Paszkowski <michal.paszkowski@outlook.com>
Co-authored-by: Andrey Tretyakov <andrey1.tretyakov@intel.com>
Co-authored-by: Konrad Trifunovic <konrad.trifunovic@intel.com>
diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
index 892b3a0..5655662 100644
--- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
@@ -1611,5 +1611,236 @@
}
return false;
}
+
+struct DemangledType {
+ StringRef Name;
+ uint32_t Opcode;
+};
+
+#define GET_DemangledTypes_DECL
+#define GET_DemangledTypes_IMPL
+
+struct ImageType {
+ StringRef Name;
+ StringRef SampledType;
+ AccessQualifier::AccessQualifier Qualifier;
+ Dim::Dim Dimensionality;
+ bool Arrayed;
+ bool Depth;
+ bool Multisampled;
+ bool Sampled;
+ ImageFormat::ImageFormat Format;
+};
+
+struct PipeType {
+ StringRef Name;
+ AccessQualifier::AccessQualifier Qualifier;
+};
+
+using namespace AccessQualifier;
+using namespace Dim;
+using namespace ImageFormat;
+#define GET_ImageTypes_DECL
+#define GET_ImageTypes_IMPL
+#define GET_PipeTypes_DECL
+#define GET_PipeTypes_IMPL
+#include "SPIRVGenTables.inc"
+} // namespace SPIRV
+
+//===----------------------------------------------------------------------===//
+// Misc functions for parsing builtin types and looking up implementation
+// details in TableGenerated tables.
+//===----------------------------------------------------------------------===//
+
+static const SPIRV::DemangledType *findBuiltinType(StringRef Name) {
+ if (Name.startswith("opencl."))
+ return SPIRV::lookupBuiltinType(Name);
+ if (Name.startswith("spirv.")) {
+ // Some SPIR-V builtin types have a complex list of parameters as part of
+ // their name (e.g. spirv.Image._void_1_0_0_0_0_0_0). Those parameters often
+ // are numeric literals which cannot be easily represented by TableGen
+ // records and should be parsed instead.
+ unsigned BaseTypeNameLength =
+ Name.contains('_') ? Name.find('_') - 1 : Name.size();
+ return SPIRV::lookupBuiltinType(Name.substr(0, BaseTypeNameLength).str());
+ }
+ return nullptr;
+}
+
+static std::unique_ptr<const SPIRV::ImageType>
+lookupOrParseBuiltinImageType(StringRef Name) {
+ if (Name.startswith("opencl.")) {
+ // Lookup OpenCL builtin image type lowering details in TableGen records.
+ const SPIRV::ImageType *Record = SPIRV::lookupImageType(Name);
+ return std::unique_ptr<SPIRV::ImageType>(new SPIRV::ImageType(*Record));
+ }
+ if (Name.startswith("spirv.")) {
+ // Parse the literals of SPIR-V image builtin parameters. The name should
+ // have the following format:
+ // spirv.Image._Type_Dim_Depth_Arrayed_MS_Sampled_ImageFormat_AccessQualifier
+ // e.g. %spirv.Image._void_1_0_0_0_0_0_0
+ StringRef TypeParametersString = Name.substr(strlen("spirv.Image."));
+ SmallVector<StringRef> TypeParameters;
+ SplitString(TypeParametersString, TypeParameters, "_");
+ assert(TypeParameters.size() == 8 &&
+ "Wrong number of literals in SPIR-V builtin image type");
+
+ StringRef SampledType = TypeParameters[0];
+ unsigned Dim, Depth, Arrayed, Multisampled, Sampled, Format, AccessQual;
+ bool AreParameterLiteralsValid =
+ !(TypeParameters[1].getAsInteger(10, Dim) ||
+ TypeParameters[2].getAsInteger(10, Depth) ||
+ TypeParameters[3].getAsInteger(10, Arrayed) ||
+ TypeParameters[4].getAsInteger(10, Multisampled) ||
+ TypeParameters[5].getAsInteger(10, Sampled) ||
+ TypeParameters[6].getAsInteger(10, Format) ||
+ TypeParameters[7].getAsInteger(10, AccessQual));
+ assert(AreParameterLiteralsValid &&
+ "Invalid format of SPIR-V image type parameter literals.");
+
+ return std::unique_ptr<SPIRV::ImageType>(new SPIRV::ImageType{
+ Name, SampledType, SPIRV::AccessQualifier::AccessQualifier(AccessQual),
+ SPIRV::Dim::Dim(Dim), static_cast<bool>(Arrayed),
+ static_cast<bool>(Depth), static_cast<bool>(Multisampled),
+ static_cast<bool>(Sampled), SPIRV::ImageFormat::ImageFormat(Format)});
+ }
+ llvm_unreachable("Unknown builtin image type name/literal");
+}
+
+static std::unique_ptr<const SPIRV::PipeType>
+lookupOrParseBuiltinPipeType(StringRef Name) {
+ if (Name.startswith("opencl.")) {
+ // Lookup OpenCL builtin pipe type lowering details in TableGen records.
+ const SPIRV::PipeType *Record = SPIRV::lookupPipeType(Name);
+ return std::unique_ptr<SPIRV::PipeType>(new SPIRV::PipeType(*Record));
+ }
+ if (Name.startswith("spirv.")) {
+ // Parse the access qualifier literal in the name of the SPIR-V pipe type.
+ // The name should have the following format:
+ // spirv.Pipe._AccessQualifier
+ // e.g. %spirv.Pipe._1
+ if (Name.endswith("_0"))
+ return std::unique_ptr<SPIRV::PipeType>(
+ new SPIRV::PipeType{Name, SPIRV::AccessQualifier::ReadOnly});
+ if (Name.endswith("_1"))
+ return std::unique_ptr<SPIRV::PipeType>(
+ new SPIRV::PipeType{Name, SPIRV::AccessQualifier::WriteOnly});
+ if (Name.endswith("_2"))
+ return std::unique_ptr<SPIRV::PipeType>(
+ new SPIRV::PipeType{Name, SPIRV::AccessQualifier::ReadWrite});
+ llvm_unreachable("Unknown pipe type access qualifier literal");
+ }
+ llvm_unreachable("Unknown builtin pipe type name/literal");
+}
+
+//===----------------------------------------------------------------------===//
+// Implementation functions for builtin types.
+//===----------------------------------------------------------------------===//
+
+SPIRVType *getNonParametrizedType(const StructType *OpaqueType,
+ const SPIRV::DemangledType *TypeRecord,
+ MachineIRBuilder &MIRBuilder,
+ SPIRVGlobalRegistry *GR) {
+ unsigned Opcode = TypeRecord->Opcode;
+ // Create or get an existing type from GlobalRegistry.
+ return GR->getOrCreateOpTypeByOpcode(OpaqueType, MIRBuilder, Opcode);
+}
+
+SPIRVType *getSamplerType(MachineIRBuilder &MIRBuilder,
+ SPIRVGlobalRegistry *GR) {
+ // Create or get an existing type from GlobalRegistry.
+ return GR->getOrCreateOpTypeSampler(MIRBuilder);
+}
+
+SPIRVType *getPipeType(const StructType *OpaqueType,
+ MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR) {
+ // Lookup pipe type lowering details in TableGen records or parse the
+ // name/literal for details.
+ std::unique_ptr<const SPIRV::PipeType> Record =
+ lookupOrParseBuiltinPipeType(OpaqueType->getName());
+ // Create or get an existing type from GlobalRegistry.
+ return GR->getOrCreateOpTypePipe(MIRBuilder, Record.get()->Qualifier);
+}
+
+SPIRVType *getImageType(const StructType *OpaqueType,
+ SPIRV::AccessQualifier::AccessQualifier AccessQual,
+ MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR) {
+ // Lookup image type lowering details in TableGen records or parse the
+ // name/literal for details.
+ std::unique_ptr<const SPIRV::ImageType> Record =
+ lookupOrParseBuiltinImageType(OpaqueType->getName());
+
+ SPIRVType *SampledType =
+ GR->getOrCreateSPIRVTypeByName(Record.get()->SampledType, MIRBuilder);
+ return GR->getOrCreateOpTypeImage(
+ MIRBuilder, SampledType, Record.get()->Dimensionality,
+ Record.get()->Depth, Record.get()->Arrayed, Record.get()->Multisampled,
+ Record.get()->Sampled, Record.get()->Format,
+ AccessQual == SPIRV::AccessQualifier::WriteOnly
+ ? SPIRV::AccessQualifier::WriteOnly
+ : Record.get()->Qualifier);
+}
+
+SPIRVType *getSampledImageType(const StructType *OpaqueType,
+ MachineIRBuilder &MIRBuilder,
+ SPIRVGlobalRegistry *GR) {
+ StringRef TypeParametersString =
+ OpaqueType->getName().substr(strlen("spirv.SampledImage."));
+ LLVMContext &Context = MIRBuilder.getMF().getFunction().getContext();
+ Type *ImageOpaqueType = StructType::getTypeByName(
+ Context, "spirv.Image." + TypeParametersString.str());
+ SPIRVType *TargetImageType =
+ GR->getOrCreateSPIRVType(ImageOpaqueType, MIRBuilder);
+ return GR->getOrCreateOpTypeSampledImage(TargetImageType, MIRBuilder);
+}
+
+namespace SPIRV {
+SPIRVType *lowerBuiltinType(const StructType *OpaqueType,
+ AccessQualifier::AccessQualifier AccessQual,
+ MachineIRBuilder &MIRBuilder,
+ SPIRVGlobalRegistry *GR) {
+ assert(OpaqueType->hasName() &&
+ "Structs representing builtin types must have a parsable name");
+ unsigned NumStartingVRegs = MIRBuilder.getMRI()->getNumVirtRegs();
+
+ const StringRef Name = OpaqueType->getName();
+ LLVM_DEBUG(dbgs() << "Lowering builtin type: " << Name << "\n");
+
+ // Lookup the demangled builtin type in the TableGen records.
+ const SPIRV::DemangledType *TypeRecord = findBuiltinType(Name);
+ if (!TypeRecord)
+ report_fatal_error("Missing TableGen record for builtin type: " + Name);
+
+ // "Lower" the BuiltinType into TargetType. The following get<...>Type methods
+ // use the implementation details from TableGen records to either create a new
+ // OpType<...> machine instruction or get an existing equivalent SPIRVType
+ // from GlobalRegistry.
+ SPIRVType *TargetType;
+ switch (TypeRecord->Opcode) {
+ case SPIRV::OpTypeImage:
+ TargetType = getImageType(OpaqueType, AccessQual, MIRBuilder, GR);
+ break;
+ case SPIRV::OpTypePipe:
+ TargetType = getPipeType(OpaqueType, MIRBuilder, GR);
+ break;
+ case SPIRV::OpTypeSampler:
+ TargetType = getSamplerType(MIRBuilder, GR);
+ break;
+ case SPIRV::OpTypeSampledImage:
+ TargetType = getSampledImageType(OpaqueType, MIRBuilder, GR);
+ break;
+ default:
+ TargetType = getNonParametrizedType(OpaqueType, TypeRecord, MIRBuilder, GR);
+ break;
+ }
+
+ // Emit OpName instruction if a new OpType<...> instruction was added
+ // (equivalent type was not found in GlobalRegistry).
+ if (NumStartingVRegs < MIRBuilder.getMRI()->getNumVirtRegs())
+ buildOpName(GR->getSPIRVTypeID(TargetType), OpaqueType->getName(),
+ MIRBuilder);
+
+ return TargetType;
+}
} // namespace SPIRV
} // namespace llvm
diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.h b/llvm/lib/Target/SPIRV/SPIRVBuiltins.h
index 8b105f7..2f622e7 100644
--- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.h
+++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.h
@@ -28,14 +28,27 @@
/// \p DemangledCall is the skeleton of the lowered builtin function call.
/// \p Set is the external instruction set containing the given builtin.
/// \p OrigRet is the single original virtual return register if defined,
-/// Register(0) otherwise. \p OrigRetTy is the type of the \p OrigRet. \p Args
-/// are the arguments of the lowered builtin call.
+/// Register(0) otherwise.
+/// \p OrigRetTy is the type of the \p OrigRet.
+/// \p Args are the arguments of the lowered builtin call.
Optional<bool> lowerBuiltin(const StringRef DemangledCall,
InstructionSet::InstructionSet Set,
MachineIRBuilder &MIRBuilder,
const Register OrigRet, const Type *OrigRetTy,
const SmallVectorImpl<Register> &Args,
SPIRVGlobalRegistry *GR);
+/// Handles the translation of the provided special opaque/builtin type \p Type
+/// to SPIR-V type. Generates the corresponding machine instructions for the
+/// target type or gets the already existing OpType<...> register from the
+/// global registry \p GR.
+///
+/// \return A machine instruction representing the OpType<...> SPIR-V type.
+///
+/// \p Type is the special opaque/builtin type to be lowered.
+SPIRVType *lowerBuiltinType(const StructType *Type,
+ AccessQualifier::AccessQualifier AccessQual,
+ MachineIRBuilder &MIRBuilder,
+ SPIRVGlobalRegistry *GR);
} // namespace SPIRV
} // namespace llvm
#endif // LLVM_LIB_TARGET_SPIRV_SPIRVBUILTINS_H
diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.td b/llvm/lib/Target/SPIRV/SPIRVBuiltins.td
index 763ae7d..da63ef5 100644
--- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.td
+++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.td
@@ -1104,22 +1104,36 @@
}
// Function to lookup builtin types by their demangled name.
-def lookupType : SearchIndex {
+def lookupBuiltinType : SearchIndex {
let Table = DemangledTypes;
let Key = ["Name"];
}
-// OpenCL builtin types:
def : DemangledType<"opencl.reserve_id_t", OpTypeReserveId>;
def : DemangledType<"opencl.event_t", OpTypeEvent>;
def : DemangledType<"opencl.queue_t", OpTypeQueue>;
def : DemangledType<"opencl.sampler_t", OpTypeSampler>;
def : DemangledType<"opencl.clk_event_t", OpTypeDeviceEvent>;
-def : DemangledType<"opencl.clk_event_t", OpTypeDeviceEvent>;
+
+def : DemangledType<"spirv.ReserveId", OpTypeReserveId>;
+def : DemangledType<"spirv.PipeStorage", OpTypePipeStorage>;
+def : DemangledType<"spirv.Queue", OpTypeQueue>;
+def : DemangledType<"spirv.Event", OpTypeEvent>;
+def : DemangledType<"spirv.Sampler", OpTypeSampler>;
+def : DemangledType<"spirv.DeviceEvent", OpTypeDeviceEvent>;
+
+// Some SPIR-V builtin types (e.g. spirv.Image) have a complex list of
+// parameters as part of their name. Some of those parameters should be treated
+// as numeric literals and therefore they cannot be represented in TableGen and
+// should be parsed instead.
+def : DemangledType<"spirv.Image", OpTypeImage>;
+def : DemangledType<"spirv.SampledImage", OpTypeSampledImage>;
+def : DemangledType<"spirv.Pipe", OpTypePipe>;
// Class definining lowering details for various variants of image type indentifiers.
class ImageType<string name> {
string Name = name;
+ string Type = "void";
AccessQualifier Qualifier = !cond(!not(!eq(!find(name, "_ro_t"), -1)) : ReadOnly,
!not(!eq(!find(name, "_wo_t"), -1)) : WriteOnly,
!not(!eq(!find(name, "_rw_t"), -1)) : ReadWrite,
@@ -1130,14 +1144,19 @@
!not(!eq(!find(name, "image3"), -1)) : DIM_3D);
bit Arrayed = !not(!eq(!find(name, "array"), -1));
bit Depth = !not(!eq(!find(name, "depth"), -1));
+ bit Multisampled = false;
+ bit Sampled = false;
+ ImageFormat Format = Unknown;
}
// Table gathering all the image type records.
def ImageTypes : GenericTable {
let FilterClass = "ImageType";
- let Fields = ["Name", "Qualifier", "Dimensionality", "Arrayed", "Depth"];
+ let Fields = ["Name", "Type", "Qualifier", "Dimensionality", "Arrayed",
+ "Depth", "Multisampled", "Sampled", "Format"];
string TypeOf_Qualifier = "AccessQualifier";
string TypeOf_Dimensionality = "Dim";
+ string TypeOf_Format = "ImageFormat";
}
// Function to lookup builtin image types by their demangled name.
diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
index 0075f54..7b7455a 100644
--- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
@@ -88,6 +88,7 @@
Instruction *visitStoreInst(StoreInst &I);
Instruction *visitAllocaInst(AllocaInst &I);
Instruction *visitAtomicCmpXchgInst(AtomicCmpXchgInst &I);
+ Instruction *visitUnreachableInst(UnreachableInst &I);
bool runOnFunction(Function &F) override;
};
} // namespace
@@ -313,7 +314,13 @@
Instruction *SPIRVEmitIntrinsics::visitAllocaInst(AllocaInst &I) {
TrackConstants = false;
- return &I;
+ Type *PtrTy = I.getType();
+ auto *NewI = IRB->CreateIntrinsic(Intrinsic::spv_alloca, {PtrTy}, {});
+ std::string InstName = I.hasName() ? I.getName().str() : "";
+ I.replaceAllUsesWith(NewI);
+ I.eraseFromParent();
+ NewI->setName(InstName);
+ return NewI;
}
Instruction *SPIRVEmitIntrinsics::visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) {
@@ -332,6 +339,12 @@
return NewI;
}
+Instruction *SPIRVEmitIntrinsics::visitUnreachableInst(UnreachableInst &I) {
+ IRB->SetInsertPoint(&I);
+ IRB->CreateIntrinsic(Intrinsic::spv_unreachable, {}, {});
+ return &I;
+}
+
void SPIRVEmitIntrinsics::processGlobalValue(GlobalVariable &GV) {
// Skip special artifical variable llvm.global.annotations.
if (GV.getName() == "llvm.global.annotations")
@@ -368,7 +381,7 @@
if (isa<ConstantPointerNull>(Op) || isa<UndefValue>(Op) ||
// Check GetElementPtrConstantExpr case.
(isa<ConstantExpr>(Op) && isa<GEPOperator>(Op))) {
- IRB->SetInsertPoint(I);
+ setInsertPointSkippingPhis(*IRB, I);
if (isa<UndefValue>(Op) && Op->getType()->isAggregateType())
buildIntrWithMD(Intrinsic::spv_assign_type, {IRB->getInt32Ty()}, Op,
UndefValue::get(IRB->getInt32Ty()));
diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
index 03f8fa6..3131794 100644
--- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
@@ -16,6 +16,7 @@
#include "SPIRVGlobalRegistry.h"
#include "SPIRV.h"
+#include "SPIRVBuiltins.h"
#include "SPIRVSubtarget.h"
#include "SPIRVTargetMachine.h"
#include "SPIRVUtils.h"
@@ -479,10 +480,10 @@
if (IsConst)
buildOpDecorate(Reg, MIRBuilder, SPIRV::Decoration::Constant, {});
- if (GVar && GVar->getAlign().valueOrOne().value() != 1)
- buildOpDecorate(
- Reg, MIRBuilder, SPIRV::Decoration::Alignment,
- {static_cast<uint32_t>(GVar->getAlign().valueOrOne().value())});
+ if (GVar && GVar->getAlign().valueOrOne().value() != 1) {
+ unsigned Alignment = (unsigned)GVar->getAlign().valueOrOne().value();
+ buildOpDecorate(Reg, MIRBuilder, SPIRV::Decoration::Alignment, {Alignment});
+ }
if (HasLinkageTy)
buildOpDecorate(Reg, MIRBuilder, SPIRV::Decoration::LinkageAttributes,
@@ -563,6 +564,20 @@
return false;
}
+SPIRVType *SPIRVGlobalRegistry::getOrCreateSpecialType(
+ const Type *Ty, MachineIRBuilder &MIRBuilder,
+ SPIRV::AccessQualifier::AccessQualifier AccQual) {
+ // Some OpenCL and SPIRV builtins like image2d_t are passed in as
+ // pointers, but should be treated as custom types like OpTypeImage.
+ if (auto PType = dyn_cast<PointerType>(Ty)) {
+ assert(!PType->isOpaque());
+ Ty = PType->getNonOpaquePointerElementType();
+ }
+ auto SType = cast<StructType>(Ty);
+ assert(isOpenCLBuiltinType(SType) || isSPIRVBuiltinType(SType));
+ return SPIRV::lowerBuiltinType(SType, AccQual, MIRBuilder, this);
+}
+
SPIRVType *SPIRVGlobalRegistry::getOpTypePointer(
SPIRV::StorageClass::StorageClass SC, SPIRVType *ElemType,
MachineIRBuilder &MIRBuilder, Register Reg) {
@@ -624,7 +639,8 @@
SPIRVType *SPIRVGlobalRegistry::createSPIRVType(
const Type *Ty, MachineIRBuilder &MIRBuilder,
SPIRV::AccessQualifier::AccessQualifier AccQual, bool EmitIR) {
- assert(!isSpecialType(Ty));
+ if (isSpecialType(Ty))
+ return getOrCreateSpecialType(Ty, MIRBuilder, AccQual);
auto &TypeToSPIRVTypeMap = DT.getTypes()->getAllUses();
auto t = TypeToSPIRVTypeMap.find(Ty);
if (t != TypeToSPIRVTypeMap.end()) {
@@ -729,7 +745,7 @@
const Type *Ty, MachineIRBuilder &MIRBuilder,
SPIRV::AccessQualifier::AccessQualifier AccessQual, bool EmitIR) {
Register Reg = DT.find(Ty, &MIRBuilder.getMF());
- if (Reg.isValid())
+ if (Reg.isValid() && !isSpecialType(Ty))
return getSPIRVTypeForVReg(Reg);
TypesInProcessing.clear();
SPIRVType *STy = restOfCreateSPIRVType(Ty, MIRBuilder, AccessQual, EmitIR);
@@ -804,6 +820,53 @@
Type->getOperand(1).getImm());
}
+SPIRVType *SPIRVGlobalRegistry::getOrCreateOpTypeImage(
+ MachineIRBuilder &MIRBuilder, SPIRVType *SampledType, SPIRV::Dim::Dim Dim,
+ uint32_t Depth, uint32_t Arrayed, uint32_t Multisampled, uint32_t Sampled,
+ SPIRV::ImageFormat::ImageFormat ImageFormat,
+ SPIRV::AccessQualifier::AccessQualifier AccessQual) {
+ SPIRV::ImageTypeDescriptor TD(SPIRVToLLVMType.lookup(SampledType), Dim, Depth,
+ Arrayed, Multisampled, Sampled, ImageFormat,
+ AccessQual);
+ if (auto *Res = checkSpecialInstr(TD, MIRBuilder))
+ return Res;
+ Register ResVReg = createTypeVReg(MIRBuilder);
+ DT.add(TD, &MIRBuilder.getMF(), ResVReg);
+ return MIRBuilder.buildInstr(SPIRV::OpTypeImage)
+ .addDef(ResVReg)
+ .addUse(getSPIRVTypeID(SampledType))
+ .addImm(Dim)
+ .addImm(Depth) // Depth (whether or not it is a Depth image).
+ .addImm(Arrayed) // Arrayed.
+ .addImm(Multisampled) // Multisampled (0 = only single-sample).
+ .addImm(Sampled) // Sampled (0 = usage known at runtime).
+ .addImm(ImageFormat)
+ .addImm(AccessQual);
+}
+
+SPIRVType *
+SPIRVGlobalRegistry::getOrCreateOpTypeSampler(MachineIRBuilder &MIRBuilder) {
+ SPIRV::SamplerTypeDescriptor TD;
+ if (auto *Res = checkSpecialInstr(TD, MIRBuilder))
+ return Res;
+ Register ResVReg = createTypeVReg(MIRBuilder);
+ DT.add(TD, &MIRBuilder.getMF(), ResVReg);
+ return MIRBuilder.buildInstr(SPIRV::OpTypeSampler).addDef(ResVReg);
+}
+
+SPIRVType *SPIRVGlobalRegistry::getOrCreateOpTypePipe(
+ MachineIRBuilder &MIRBuilder,
+ SPIRV::AccessQualifier::AccessQualifier AccessQual) {
+ SPIRV::PipeTypeDescriptor TD(AccessQual);
+ if (auto *Res = checkSpecialInstr(TD, MIRBuilder))
+ return Res;
+ Register ResVReg = createTypeVReg(MIRBuilder);
+ DT.add(TD, &MIRBuilder.getMF(), ResVReg);
+ return MIRBuilder.buildInstr(SPIRV::OpTypePipe)
+ .addDef(ResVReg)
+ .addImm(AccessQual);
+}
+
SPIRVType *SPIRVGlobalRegistry::getOrCreateOpTypeSampledImage(
SPIRVType *ImageType, MachineIRBuilder &MIRBuilder) {
SPIRV::SampledImageTypeDescriptor TD(
@@ -813,11 +876,20 @@
if (auto *Res = checkSpecialInstr(TD, MIRBuilder))
return Res;
Register ResVReg = createTypeVReg(MIRBuilder);
- auto MIB = MIRBuilder.buildInstr(SPIRV::OpTypeSampledImage)
- .addDef(ResVReg)
- .addUse(getSPIRVTypeID(ImageType));
DT.add(TD, &MIRBuilder.getMF(), ResVReg);
- return MIB;
+ return MIRBuilder.buildInstr(SPIRV::OpTypeSampledImage)
+ .addDef(ResVReg)
+ .addUse(getSPIRVTypeID(ImageType));
+}
+
+SPIRVType *SPIRVGlobalRegistry::getOrCreateOpTypeByOpcode(
+ const Type *Ty, MachineIRBuilder &MIRBuilder, unsigned Opcode) {
+ Register ResVReg = DT.find(Ty, &MIRBuilder.getMF());
+ if (ResVReg.isValid())
+ return MIRBuilder.getMF().getRegInfo().getUniqueVRegDef(ResVReg);
+ ResVReg = createTypeVReg(MIRBuilder);
+ DT.add(Ty, &MIRBuilder.getMF(), ResVReg);
+ return MIRBuilder.buildInstr(Opcode).addDef(ResVReg);
}
const MachineInstr *
@@ -942,6 +1014,24 @@
return finishCreatingSPIRVType(LLVMTy, MIB);
}
+SPIRVType *SPIRVGlobalRegistry::getOrCreateSPIRVArrayType(
+ SPIRVType *BaseType, unsigned NumElements, MachineInstr &I,
+ const SPIRVInstrInfo &TII) {
+ Type *LLVMTy = ArrayType::get(
+ const_cast<Type *>(getTypeForSPIRVType(BaseType)), NumElements);
+ Register Reg = DT.find(LLVMTy, CurMF);
+ if (Reg.isValid())
+ return getSPIRVTypeForVReg(Reg);
+ MachineBasicBlock &BB = *I.getParent();
+ SPIRVType *SpirvType = getOrCreateSPIRVIntegerType(32, I, TII);
+ Register Len = getOrCreateConstInt(NumElements, I, SpirvType, TII);
+ auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpTypeArray))
+ .addDef(createTypeVReg(CurMF->getRegInfo()))
+ .addUse(getSPIRVTypeID(BaseType))
+ .addUse(Len);
+ return finishCreatingSPIRVType(LLVMTy, MIB);
+}
+
SPIRVType *SPIRVGlobalRegistry::getOrCreateSPIRVPointerType(
SPIRVType *BaseType, MachineIRBuilder &MIRBuilder,
SPIRV::StorageClass::StorageClass SClass) {
diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h
index 5c19b27..667802a 100644
--- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h
+++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h
@@ -208,6 +208,11 @@
SPIRVType *getOpTypeFunction(SPIRVType *RetType,
const SmallVectorImpl<SPIRVType *> &ArgTypes,
MachineIRBuilder &MIRBuilder);
+
+ SPIRVType *
+ getOrCreateSpecialType(const Type *Ty, MachineIRBuilder &MIRBuilder,
+ SPIRV::AccessQualifier::AccessQualifier AccQual);
+
std::tuple<Register, ConstantInt *, bool> getOrCreateConstIntReg(
uint64_t Val, SPIRVType *SpvType, MachineIRBuilder *MIRBuilder,
MachineInstr *I = nullptr, const SPIRVInstrInfo *TII = nullptr);
@@ -240,7 +245,6 @@
SPIRVType *SpvType, bool EmitIR = true);
Register getOrCreateConsIntArray(uint64_t Val, MachineIRBuilder &MIRBuilder,
SPIRVType *SpvType, bool EmitIR = true);
-
Register buildConstantSampler(Register Res, unsigned AddrMode, unsigned Param,
unsigned FilerMode,
MachineIRBuilder &MIRBuilder,
@@ -270,19 +274,39 @@
SPIRVType *getOrCreateSPIRVVectorType(SPIRVType *BaseType,
unsigned NumElements, MachineInstr &I,
const SPIRVInstrInfo &TII);
+ SPIRVType *getOrCreateSPIRVArrayType(SPIRVType *BaseType,
+ unsigned NumElements, MachineInstr &I,
+ const SPIRVInstrInfo &TII);
+
SPIRVType *getOrCreateSPIRVPointerType(
SPIRVType *BaseType, MachineIRBuilder &MIRBuilder,
SPIRV::StorageClass::StorageClass SClass = SPIRV::StorageClass::Function);
SPIRVType *getOrCreateSPIRVPointerType(
SPIRVType *BaseType, MachineInstr &I, const SPIRVInstrInfo &TII,
SPIRV::StorageClass::StorageClass SClass = SPIRV::StorageClass::Function);
+
+ SPIRVType *
+ getOrCreateOpTypeImage(MachineIRBuilder &MIRBuilder, SPIRVType *SampledType,
+ SPIRV::Dim::Dim Dim, uint32_t Depth, uint32_t Arrayed,
+ uint32_t Multisampled, uint32_t Sampled,
+ SPIRV::ImageFormat::ImageFormat ImageFormat,
+ SPIRV::AccessQualifier::AccessQualifier AccQual);
+
+ SPIRVType *getOrCreateOpTypeSampler(MachineIRBuilder &MIRBuilder);
+
SPIRVType *getOrCreateOpTypeSampledImage(SPIRVType *ImageType,
MachineIRBuilder &MIRBuilder);
+ SPIRVType *
+ getOrCreateOpTypePipe(MachineIRBuilder &MIRBuilder,
+ SPIRV::AccessQualifier::AccessQualifier AccQual);
SPIRVType *getOrCreateOpTypeFunctionWithArgs(
const Type *Ty, SPIRVType *RetType,
const SmallVectorImpl<SPIRVType *> &ArgTypes,
MachineIRBuilder &MIRBuilder);
+ SPIRVType *getOrCreateOpTypeByOpcode(const Type *Ty,
+ MachineIRBuilder &MIRBuilder,
+ unsigned Opcode);
};
} // end namespace llvm
#endif // LLLVM_LIB_TARGET_SPIRV_SPIRVTYPEMANAGER_H
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index 07e1158..5ebec6b 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -30,6 +30,11 @@
#define DEBUG_TYPE "spirv-isel"
using namespace llvm;
+namespace CL = SPIRV::OpenCLExtInst;
+namespace GL = SPIRV::GLSLExtInst;
+
+using ExtInstList =
+ std::vector<std::pair<SPIRV::InstructionSet::InstructionSet, uint32_t>>;
namespace {
@@ -132,9 +137,8 @@
bool selectTrunc(Register ResVReg, const SPIRVType *ResType,
MachineInstr &I) const;
- bool selectIntToBool(Register IntReg, Register ResVReg,
- const SPIRVType *intTy, const SPIRVType *boolTy,
- MachineInstr &I) const;
+ bool selectIntToBool(Register IntReg, Register ResVReg, MachineInstr &I,
+ const SPIRVType *intTy, const SPIRVType *boolTy) const;
bool selectOpUndef(Register ResVReg, const SPIRVType *ResType,
MachineInstr &I) const;
@@ -160,6 +164,14 @@
bool selectPhi(Register ResVReg, const SPIRVType *ResType,
MachineInstr &I) const;
+ bool selectExtInst(Register ResVReg, const SPIRVType *ResType,
+ MachineInstr &I, CL::OpenCLExtInst CLInst) const;
+ bool selectExtInst(Register ResVReg, const SPIRVType *ResType,
+ MachineInstr &I, CL::OpenCLExtInst CLInst,
+ GL::GLSLExtInst GLInst) const;
+ bool selectExtInst(Register ResVReg, const SPIRVType *ResType,
+ MachineInstr &I, const ExtInstList &ExtInsts) const;
+
Register buildI32Constant(uint32_t Val, MachineInstr &I,
const SPIRVType *ResType = nullptr) const;
@@ -283,6 +295,7 @@
}
case TargetOpcode::G_MEMMOVE:
case TargetOpcode::G_MEMCPY:
+ case TargetOpcode::G_MEMSET:
return selectMemOperation(ResVReg, I);
case TargetOpcode::G_ICMP:
@@ -318,6 +331,85 @@
case TargetOpcode::G_CTPOP:
return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitCount);
+ case TargetOpcode::G_SMIN:
+ return selectExtInst(ResVReg, ResType, I, CL::s_min, GL::SMin);
+ case TargetOpcode::G_UMIN:
+ return selectExtInst(ResVReg, ResType, I, CL::u_min, GL::UMin);
+
+ case TargetOpcode::G_SMAX:
+ return selectExtInst(ResVReg, ResType, I, CL::s_max, GL::SMax);
+ case TargetOpcode::G_UMAX:
+ return selectExtInst(ResVReg, ResType, I, CL::u_max, GL::UMax);
+
+ case TargetOpcode::G_FMA:
+ return selectExtInst(ResVReg, ResType, I, CL::fma, GL::Fma);
+
+ case TargetOpcode::G_FPOW:
+ return selectExtInst(ResVReg, ResType, I, CL::pow, GL::Pow);
+ case TargetOpcode::G_FPOWI:
+ return selectExtInst(ResVReg, ResType, I, CL::pown);
+
+ case TargetOpcode::G_FEXP:
+ return selectExtInst(ResVReg, ResType, I, CL::exp, GL::Exp);
+ case TargetOpcode::G_FEXP2:
+ return selectExtInst(ResVReg, ResType, I, CL::exp2, GL::Exp2);
+
+ case TargetOpcode::G_FLOG:
+ return selectExtInst(ResVReg, ResType, I, CL::log, GL::Log);
+ case TargetOpcode::G_FLOG2:
+ return selectExtInst(ResVReg, ResType, I, CL::log2, GL::Log2);
+ case TargetOpcode::G_FLOG10:
+ return selectExtInst(ResVReg, ResType, I, CL::log10);
+
+ case TargetOpcode::G_FABS:
+ return selectExtInst(ResVReg, ResType, I, CL::fabs, GL::FAbs);
+ case TargetOpcode::G_ABS:
+ return selectExtInst(ResVReg, ResType, I, CL::s_abs, GL::SAbs);
+
+ case TargetOpcode::G_FMINNUM:
+ case TargetOpcode::G_FMINIMUM:
+ return selectExtInst(ResVReg, ResType, I, CL::fmin, GL::FMin);
+ case TargetOpcode::G_FMAXNUM:
+ case TargetOpcode::G_FMAXIMUM:
+ return selectExtInst(ResVReg, ResType, I, CL::fmax, GL::FMax);
+
+ case TargetOpcode::G_FCOPYSIGN:
+ return selectExtInst(ResVReg, ResType, I, CL::copysign);
+
+ case TargetOpcode::G_FCEIL:
+ return selectExtInst(ResVReg, ResType, I, CL::ceil, GL::Ceil);
+ case TargetOpcode::G_FFLOOR:
+ return selectExtInst(ResVReg, ResType, I, CL::floor, GL::Floor);
+
+ case TargetOpcode::G_FCOS:
+ return selectExtInst(ResVReg, ResType, I, CL::cos, GL::Cos);
+ case TargetOpcode::G_FSIN:
+ return selectExtInst(ResVReg, ResType, I, CL::sin, GL::Sin);
+
+ case TargetOpcode::G_FSQRT:
+ return selectExtInst(ResVReg, ResType, I, CL::sqrt, GL::Sqrt);
+
+ case TargetOpcode::G_CTTZ:
+ case TargetOpcode::G_CTTZ_ZERO_UNDEF:
+ return selectExtInst(ResVReg, ResType, I, CL::ctz);
+ case TargetOpcode::G_CTLZ:
+ case TargetOpcode::G_CTLZ_ZERO_UNDEF:
+ return selectExtInst(ResVReg, ResType, I, CL::clz);
+
+ case TargetOpcode::G_INTRINSIC_ROUND:
+ return selectExtInst(ResVReg, ResType, I, CL::round, GL::Round);
+ case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
+ return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven);
+ case TargetOpcode::G_INTRINSIC_TRUNC:
+ return selectExtInst(ResVReg, ResType, I, CL::trunc, GL::Trunc);
+ case TargetOpcode::G_FRINT:
+ case TargetOpcode::G_FNEARBYINT:
+ return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven);
+
+ case TargetOpcode::G_SMULH:
+ return selectExtInst(ResVReg, ResType, I, CL::s_mul_hi);
+ case TargetOpcode::G_UMULH:
+ return selectExtInst(ResVReg, ResType, I, CL::u_mul_hi);
case TargetOpcode::G_SEXT:
return selectExt(ResVReg, ResType, I, true);
@@ -394,6 +486,48 @@
}
}
+bool SPIRVInstructionSelector::selectExtInst(Register ResVReg,
+ const SPIRVType *ResType,
+ MachineInstr &I,
+ CL::OpenCLExtInst CLInst) const {
+ return selectExtInst(ResVReg, ResType, I,
+ {{SPIRV::InstructionSet::OpenCL_std, CLInst}});
+}
+
+bool SPIRVInstructionSelector::selectExtInst(Register ResVReg,
+ const SPIRVType *ResType,
+ MachineInstr &I,
+ CL::OpenCLExtInst CLInst,
+ GL::GLSLExtInst GLInst) const {
+ ExtInstList ExtInsts = {{SPIRV::InstructionSet::OpenCL_std, CLInst},
+ {SPIRV::InstructionSet::GLSL_std_450, GLInst}};
+ return selectExtInst(ResVReg, ResType, I, ExtInsts);
+}
+
+bool SPIRVInstructionSelector::selectExtInst(Register ResVReg,
+ const SPIRVType *ResType,
+ MachineInstr &I,
+ const ExtInstList &Insts) const {
+
+ for (const auto &Ex : Insts) {
+ SPIRV::InstructionSet::InstructionSet Set = Ex.first;
+ uint32_t Opcode = Ex.second;
+ if (STI.canUseExtInstSet(Set)) {
+ MachineBasicBlock &BB = *I.getParent();
+ auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst))
+ .addDef(ResVReg)
+ .addUse(GR.getSPIRVTypeID(ResType))
+ .addImm(static_cast<uint32_t>(Set))
+ .addImm(Opcode);
+ const unsigned NumOps = I.getNumOperands();
+ for (unsigned i = 1; i < NumOps; ++i)
+ MIB.add(I.getOperand(i));
+ return MIB.constrainAllUses(TII, TRI, RBI);
+ }
+ }
+ return false;
+}
+
bool SPIRVInstructionSelector::selectUnOpWithSrc(Register ResVReg,
const SPIRVType *ResType,
MachineInstr &I,
@@ -493,9 +627,39 @@
bool SPIRVInstructionSelector::selectMemOperation(Register ResVReg,
MachineInstr &I) const {
MachineBasicBlock &BB = *I.getParent();
+ Register SrcReg = I.getOperand(1).getReg();
+ if (I.getOpcode() == TargetOpcode::G_MEMSET) {
+ assert(I.getOperand(1).isReg() && I.getOperand(2).isReg());
+ unsigned Val = getIConstVal(I.getOperand(1).getReg(), MRI);
+ unsigned Num = getIConstVal(I.getOperand(2).getReg(), MRI);
+ SPIRVType *ValTy = GR.getOrCreateSPIRVIntegerType(8, I, TII);
+ SPIRVType *ArrTy = GR.getOrCreateSPIRVArrayType(ValTy, Num, I, TII);
+ Register Const = GR.getOrCreateConsIntArray(Val, I, ArrTy, TII);
+ SPIRVType *VarTy = GR.getOrCreateSPIRVPointerType(
+ ArrTy, I, TII, SPIRV::StorageClass::UniformConstant);
+ // TODO: check if we have such GV, add init, use buildGlobalVariable.
+ Type *LLVMArrTy = ArrayType::get(
+ IntegerType::get(GR.CurMF->getFunction().getContext(), 8), Num);
+ GlobalVariable *GV =
+ new GlobalVariable(LLVMArrTy, true, GlobalValue::InternalLinkage);
+ Register VarReg = MRI->createGenericVirtualRegister(LLT::scalar(32));
+ GR.add(GV, GR.CurMF, VarReg);
+
+ buildOpDecorate(VarReg, I, TII, SPIRV::Decoration::Constant, {});
+ BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpVariable))
+ .addDef(VarReg)
+ .addUse(GR.getSPIRVTypeID(VarTy))
+ .addImm(SPIRV::StorageClass::UniformConstant)
+ .addUse(Const)
+ .constrainAllUses(TII, TRI, RBI);
+ SPIRVType *SourceTy = GR.getOrCreateSPIRVPointerType(
+ ValTy, I, TII, SPIRV::StorageClass::UniformConstant);
+ SrcReg = MRI->createGenericVirtualRegister(LLT::scalar(32));
+ selectUnOpWithSrc(SrcReg, SourceTy, I, VarReg, SPIRV::OpBitcast);
+ }
auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCopyMemorySized))
.addUse(I.getOperand(0).getReg())
- .addUse(I.getOperand(1).getReg())
+ .addUse(SrcReg)
.addUse(I.getOperand(2).getReg());
if (I.getNumMemOperands())
addMemoryOperands(*I.memoperands_begin(), MIB);
@@ -974,9 +1138,9 @@
bool SPIRVInstructionSelector::selectIntToBool(Register IntReg,
Register ResVReg,
+ MachineInstr &I,
const SPIRVType *IntTy,
- const SPIRVType *BoolTy,
- MachineInstr &I) const {
+ const SPIRVType *BoolTy) const {
// To truncate to a bool, we use OpBitwiseAnd 1 and OpINotEqual to zero.
Register BitIntReg = MRI->createVirtualRegister(&SPIRV::IDRegClass);
bool IsVectorTy = IntTy->getOpcode() == SPIRV::OpTypeVector;
@@ -1004,7 +1168,7 @@
if (GR.isScalarOrVectorOfType(ResVReg, SPIRV::OpTypeBool)) {
Register IntReg = I.getOperand(1).getReg();
const SPIRVType *ArgType = GR.getSPIRVTypeForVReg(IntReg);
- return selectIntToBool(IntReg, ResVReg, ArgType, ResType, I);
+ return selectIntToBool(IntReg, ResVReg, I, ArgType, ResType);
}
bool IsSigned = GR.isScalarOrVectorSigned(ResType);
unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert;
@@ -1223,6 +1387,12 @@
case Intrinsic::spv_cmpxchg:
return selectAtomicCmpXchg(ResVReg, ResType, I);
break;
+ case Intrinsic::spv_unreachable:
+ BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUnreachable));
+ break;
+ case Intrinsic::spv_alloca:
+ return selectFrameIndex(ResVReg, ResType, I);
+ break;
default:
llvm_unreachable("Intrinsic selection not implemented");
}
diff --git a/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp b/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp
index 87f9e95..3d12077 100644
--- a/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp
@@ -255,6 +255,18 @@
getActionDefinitionsBuilder(G_FPOWI).legalForCartesianProduct(
allFloatScalarsAndVectors, allIntScalarsAndVectors);
+ if (ST.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) {
+ getActionDefinitionsBuilder(G_FLOG10).legalFor(allFloatScalarsAndVectors);
+
+ getActionDefinitionsBuilder(
+ {G_CTTZ, G_CTTZ_ZERO_UNDEF, G_CTLZ, G_CTLZ_ZERO_UNDEF})
+ .legalForCartesianProduct(allIntScalarsAndVectors,
+ allIntScalarsAndVectors);
+
+ // Struct return types become a single scalar, so cannot easily legalize.
+ getActionDefinitionsBuilder({G_SMULH, G_UMULH}).alwaysLegal();
+ }
+
getLegacyLegalizerInfo().computeTables();
verify(*ST.getInstrInfo());
}
diff --git a/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td b/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td
index 3471935..1b36c10 100644
--- a/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td
+++ b/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td
@@ -734,7 +734,7 @@
}
multiclass ImageFormatOperand<bits<32> value, list<Capability> reqCapabilities> {
- def : ImageFormat<NAME, value>;
+ def NAME : ImageFormat<NAME, value>;
defm : SymbolicOperandWithRequirements<ImageFormatOperand, value, NAME, 0, 0, [], reqCapabilities>;
}
