Google Git Google Git
Sign in
llvm/llvm-project/clang/refs/heads/main/./lib/CodeGen/CGHLSLRuntime.cpp
blob: 229bec71bb50ee9e01f026170364e4d9d6f89e7b [file] [edit]
//===----- CGHLSLRuntime.cpp - Interface to HLSL Runtimes -----------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This provides an abstract class for HLSL code generation. Concrete
// subclasses of this implement code generation for specific HLSL
// runtime libraries.
//
//===----------------------------------------------------------------------===//
#include "CGHLSLRuntime.h"
#include "CGDebugInfo.h"
#include "CGRecordLayout.h"
#include "CodeGenFunction.h"
#include "CodeGenModule.h"
#include "HLSLBufferLayoutBuilder.h"
#include "TargetInfo.h"
#include "clang/AST/ASTContext.h"
#include "clang/AST/Attr.h"
#include "clang/AST/Decl.h"
#include "clang/AST/Expr.h"
#include "clang/AST/HLSLResource.h"
#include "clang/AST/MatrixUtils.h"
#include "clang/AST/RecursiveASTVisitor.h"
#include "clang/AST/Type.h"
#include "clang/Basic/DiagnosticDriver.h"
#include "clang/Basic/DiagnosticFrontend.h"
#include "clang/Basic/SourceManager.h"
#include "clang/Basic/TargetOptions.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/Enum.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/ScopeExit.h"
#include "llvm/ADT/SmallString.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/Frontend/HLSL/HLSLResource.h"
#include "llvm/Frontend/HLSL/RootSignatureMetadata.h"
#include "llvm/Frontend/HLSL/SemanticSignaturePacking.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/GlobalVariable.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/Metadata.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Value.h"
#include "llvm/Support/Alignment.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/FormatVariadic.h"
#include "llvm/Support/Path.h"
#include "llvm/Transforms/Utils/ModuleUtils.h"
#include <array>
#include <cstdint>
#include <optional>
using namespace clang;
using namespace CodeGen;
using namespace clang::hlsl;
using namespace llvm;
using llvm::hlsl::CBufferRowSizeInBytes;
namespace {
void addDxilValVersion(StringRef ValVersionStr, llvm::Module &M) {
// The validation of ValVersionStr is done at HLSLToolChain::TranslateArgs.
// Assume ValVersionStr is legal here.
VersionTuple Version;
if (Version.tryParse(ValVersionStr) || Version.getBuild() ||
Version.getSubminor() || !Version.getMinor()) {
return;
}
uint64_t Major = Version.getMajor();
uint64_t Minor = *Version.getMinor();
auto &Ctx = M.getContext();
IRBuilder<> B(M);
MDNode *Val = MDNode::get(Ctx, {ConstantAsMetadata::get(B.getInt32(Major)),
ConstantAsMetadata::get(B.getInt32(Minor))});
StringRef DXILValKey = "dx.valver";
auto *DXILValMD = M.getOrInsertNamedMetadata(DXILValKey);
DXILValMD->addOperand(Val);
}
void addRootSignatureMD(llvm::dxbc::RootSignatureVersion RootSigVer,
ArrayRef<llvm::hlsl::rootsig::RootElement> Elements,
llvm::Function *Fn, llvm::Module &M) {
auto &Ctx = M.getContext();
llvm::hlsl::rootsig::MetadataBuilder RSBuilder(M, Elements);
MDNode *RootSignature = RSBuilder.BuildRootSignature();
ConstantAsMetadata *Version = ConstantAsMetadata::get(ConstantInt::get(
llvm::Type::getInt32Ty(Ctx), llvm::to_underlying(RootSigVer)));
ValueAsMetadata *EntryFunc = Fn ? ValueAsMetadata::get(Fn) : nullptr;
MDNode *MDVals = MDNode::get(Ctx, {EntryFunc, RootSignature, Version});
StringRef RootSignatureValKey = "dx.rootsignatures";
auto *RootSignatureValMD = M.getOrInsertNamedMetadata(RootSignatureValKey);
RootSignatureValMD->addOperand(MDVals);
}
Expected<unsigned> packSemanticSignature(
MutableArrayRef<llvm::hlsl::SemanticSignatureElement> Elements,
llvm::Triple::EnvironmentType Stage, llvm::hlsl::IOType IOTy,
CodeGenOptions::HLSLSemanticSignaturePackingMode Mode,
bool UseNative16BitTypes) {
using namespace llvm::hlsl;
// Vertex inputs and pixel outputs have fixed packing rules, independent of
// the packing mode used between programmable stages.
if (Stage == llvm::Triple::Vertex && IOTy == IOType::In)
return packSignatureStacked(Elements, Stage, IOTy);
if (Stage == llvm::Triple::Pixel && IOTy == IOType::Out)
return packSignatureIndexed(Elements, Stage, IOTy);
if (Mode == CodeGenOptions::HLSLSemanticSignaturePackingMode::Optimized)
return packSignatureOptimized(Elements, Stage, IOTy, UseNative16BitTypes);
return packSignaturePrefixStable(Elements, Stage, IOTy, UseNative16BitTypes);
}
MDNode *buildSemanticSignatureMD(
ArrayRef<llvm::hlsl::SemanticSignatureElement> Elements, LLVMContext &Ctx) {
if (Elements.empty())
return nullptr;
SmallVector<Metadata *> ElementMD;
for (const llvm::hlsl::SemanticSignatureElement &Element : Elements)
ElementMD.push_back(Element.toMetadata(Ctx));
return MDNode::get(Ctx, ElementMD);
}
void addSemanticSignatureMD(
ArrayRef<llvm::hlsl::SemanticSignatureElement> InputElements,
ArrayRef<llvm::hlsl::SemanticSignatureElement> OutputElements,
llvm::Function *Fn, llvm::Module &M) {
if (InputElements.empty() && OutputElements.empty())
return;
LLVMContext &Ctx = M.getContext();
MDNode *InputSignature = buildSemanticSignatureMD(InputElements, Ctx);
MDNode *OutputSignature = buildSemanticSignatureMD(OutputElements, Ctx);
MDNode *MDVals = MDNode::get(
Ctx, {ValueAsMetadata::get(Fn), InputSignature, OutputSignature});
M.getOrInsertNamedMetadata("dx.semantic.signatures")->addOperand(MDVals);
}
static void copyGlobalResource(CodeGenFunction &CGF, const VarDecl *ResourceVD,
AggValueSlot &DestSlot) {
GlobalVariable *ResGV =
cast<GlobalVariable>(CGF.CGM.GetAddrOfGlobalVar(ResourceVD));
assert(ResGV && "expected valid global variable");
CGF.Builder.CreateStore(ResGV, DestSlot.getAddress());
}
// Given a MemberExpr of a resource or resource array type, find the parent
// VarDecl of the struct or class instance that contains this resource and
// build the full resource name based on the member access path.
//
// For example, for a member access like "myStructArray[0].memberA",
// this function will find the VarDecl of "myStructArray" and use the
// EmbeddedResourceNameBuilder to build the resource name
// "myStructArray.0.memberA".
//
// This also works for a record type expression that has some embedded
// resources. It finds the parent VarDecl of that record and builds a partial
// name which is the prefix of the resource globals associated with the
// declaration.
static const VarDecl *findStructResourceParentDeclAndBuildName(
const Expr *E, EmbeddedResourceNameBuilder &NameBuilder) {
SmallVector<const Expr *> WorkList;
const VarDecl *VD = nullptr;
for (;;) {
if (const auto *DRE = dyn_cast<DeclRefExpr>(E)) {
assert(isa<VarDecl>(DRE->getDecl()) &&
"member expr base is not a var decl");
VD = cast<VarDecl>(DRE->getDecl());
NameBuilder.pushName(VD->getName());
break;
}
WorkList.push_back(E);
if (const auto *MExp = dyn_cast<MemberExpr>(E))
E = MExp->getBase();
else if (const auto *ICE = dyn_cast<ImplicitCastExpr>(E))
E = ICE->getSubExpr();
else if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(E))
E = ASE->getBase();
else if (isa<CXXThisExpr>(E))
// Resource member access on "this" pointer not yet implemented
// (llvm/llvm-project#190299)
return nullptr;
else
llvm_unreachable("unexpected expr type in resource member access");
assert(E && "expected valid expression");
}
while (!WorkList.empty()) {
E = WorkList.pop_back_val();
if (const auto *ME = dyn_cast<MemberExpr>(E)) {
NameBuilder.pushName(
ME->getMemberNameInfo().getName().getAsIdentifierInfo()->getName());
} else if (const auto *ICE = dyn_cast<ImplicitCastExpr>(E)) {
if (ICE->getCastKind() == CK_UncheckedDerivedToBase) {
CXXRecordDecl *DerivedRD =
ICE->getSubExpr()->getType()->getAsCXXRecordDecl();
CXXRecordDecl *BaseRD = ICE->getType()->getAsCXXRecordDecl();
NameBuilder.pushBaseNameHierarchy(DerivedRD, BaseRD);
}
} else if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(E)) {
const Expr *IdxExpr = ASE->getIdx();
std::optional<llvm::APSInt> Value =
IdxExpr->getIntegerConstantExpr(VD->getASTContext());
assert(Value &&
"expected constant index in struct with resource array access");
NameBuilder.pushArrayIndex(Value->getZExtValue());
} else {
llvm_unreachable("unexpected expr type in resource member access");
}
}
return VD;
}
// Given a MemberExpr of a resource or resource array type, find the
// corresponding global resource declaration associated with the owning struct
// or class instance via HLSLAssociatedResourceDeclAttr.
static const VarDecl *
findAssociatedResourceDeclForStruct(ASTContext &AST, const MemberExpr *ME) {
EmbeddedResourceNameBuilder NameBuilder;
const VarDecl *ParentVD =
findStructResourceParentDeclAndBuildName(ME, NameBuilder);
if (!ParentVD)
return nullptr;
if (!ParentVD->hasGlobalStorage())
return nullptr;
IdentifierInfo *II = NameBuilder.getNameAsIdentifier(AST);
for (const Attr *A : ParentVD->getAttrs()) {
if (const auto *ADA = dyn_cast<HLSLAssociatedResourceDeclAttr>(A)) {
VarDecl *AssocResVD = ADA->getResDecl();
if (AssocResVD->getIdentifier() == II)
return AssocResVD;
}
}
return nullptr;
}
void addSourceInfo(CodeGenModule &CGM, llvm::Module &M) {
auto &SM = CGM.getContext().getSourceManager();
auto &Macros = CGM.getPreprocessorOpts().Macros;
auto &CodeGenOpts = CGM.getCodeGenOpts();
auto &Ctx = M.getContext();
// Names and content of shader source code files.
llvm::NamedMDNode *DXContents =
M.getOrInsertNamedMetadata("dx.source.contents");
auto addFile = [&](const std::pair<StringRef, StringRef> &NameContent) {
llvm::MDTuple *FileInfo =
llvm::MDNode::get(Ctx, {llvm::MDString::get(Ctx, NameContent.first),
llvm::MDString::get(Ctx, NameContent.second)});
DXContents->addOperand(FileInfo);
};
bool Invalid = false;
const SrcMgr::SLocEntry *MainLocEntry =
&SM.getSLocEntry(SM.getMainFileID(), &Invalid);
assert(!Invalid && "Main file SLocEntry must not be invalid!");
const SrcMgr::ContentCache &MainCCEntry =
MainLocEntry->getFile().getContentCache();
SmallVector<std::pair<std::string, StringRef>> Files;
std::optional<SmallString<256>> MainFileName;
Files.reserve(SM.local_sloc_entry_size());
for (unsigned I : llvm::seq(SM.local_sloc_entry_size())) {
const SrcMgr::SLocEntry &LocEntry = SM.getLocalSLocEntry(I);
if (!LocEntry.isFile())
continue;
const SrcMgr::FileInfo &FInfo = LocEntry.getFile();
if (isSystem(FInfo.getFileCharacteristic()))
continue;
const SrcMgr::ContentCache &CCEntry = FInfo.getContentCache();
OptionalFileEntryRef FEntry = CCEntry.OrigEntry;
if (!FEntry)
continue;
llvm::SmallString<256> Path = FEntry->getName();
llvm::sys::path::native(Path);
std::optional<llvm::MemoryBufferRef> Buffer = CCEntry.getBufferOrNone(
SM.getDiagnostics(), SM.getFileManager(), SourceLocation());
if (!Buffer) {
SM.getDiagnostics().Report(diag::warn_hlsl_failed_to_embed_source)
<< Path;
continue;
}
if (&MainCCEntry != &CCEntry) {
Files.emplace_back(Path, Buffer->getBuffer());
} else {
// Main file should be at first position.
addFile(std::make_pair(Path, Buffer->getBuffer()));
MainFileName.emplace(Path);
}
}
assert(MainFileName && "Main file not found.");
// Files other that main one should be sorted by name.
llvm::sort(Files);
#ifndef NDEBUG
for (unsigned I = 1; I < Files.size(); ++I)
assert((Files[I - 1].first != Files[I].first) &&
"duplicate files in dx.source.contents");
#endif
llvm::for_each(Files, addFile);
SmallVector<llvm::Metadata *> Defines;
Defines.reserve(Macros.size());
for (const auto &Macro : Macros) {
// Ignore undefs.
if (!Macro.second)
Defines.emplace_back(llvm::MDString::get(Ctx, Macro.first));
}
M.getOrInsertNamedMetadata("dx.source.defines")
->addOperand(llvm::MDNode::get(Ctx, Defines));
if (!CodeGenOpts.MainFileName.empty())
llvm::sys::path::native(CodeGenOpts.MainFileName, *MainFileName);
M.getOrInsertNamedMetadata("dx.source.mainFileName")
->addOperand(
llvm::MDNode::get(Ctx, llvm::MDString::get(Ctx, *MainFileName)));
SmallVector<llvm::Metadata *> Args;
Args.reserve(CodeGenOpts.HLSLParsedCommandLine.size());
if (!CodeGenOpts.HLSLParsedCommandLine.empty())
for (const auto &Arg : llvm::drop_begin(CodeGenOpts.HLSLParsedCommandLine))
Args.push_back(llvm::MDString::get(Ctx, Arg));
M.getOrInsertNamedMetadata("dx.source.args")
->addOperand(llvm::MDNode::get(Ctx, Args));
}
// Find array variable declaration from DeclRef expression
static const ValueDecl *getArrayDecl(ASTContext &AST, const Expr *E) {
E = E->IgnoreImpCasts();
if (const auto *DRE = dyn_cast_or_null<DeclRefExpr>(E))
return DRE->getDecl();
if (auto *OVE = dyn_cast<OpaqueValueExpr>(E))
E = OVE->getSourceExpr()->IgnoreImpCasts();
if (isa<MemberExpr>(E))
return findAssociatedResourceDeclForStruct(AST, cast<MemberExpr>(E));
return nullptr;
}
// Find array variable declaration from nested array subscript AST nodes
static const ValueDecl *getArrayDecl(ASTContext &AST,
const ArraySubscriptExpr *ASE) {
const Expr *E = nullptr;
while (ASE != nullptr) {
E = ASE->getBase()->IgnoreImpCasts();
if (!E)
return nullptr;
ASE = dyn_cast<ArraySubscriptExpr>(E);
}
return getArrayDecl(AST, E);
}
// Get the total size of the array, or 0 if the array is unbounded.
static int getTotalArraySize(ASTContext &AST, const clang::Type *Ty) {
Ty = Ty->getUnqualifiedDesugaredType();
assert(Ty->isArrayType() && "expected array type");
if (Ty->isIncompleteArrayType())
return 0;
return AST.getConstantArrayElementCount(cast<ConstantArrayType>(Ty));
}
static Value *buildNameForResource(llvm::StringRef BaseName,
CodeGenModule &CGM) {
llvm::SmallString<64> GlobalName = {BaseName, ".str"};
return CGM.GetAddrOfConstantCString(BaseName.str(), GlobalName.c_str())
.getPointer();
}
static CXXMethodDecl *lookupMethod(CXXRecordDecl *Record, StringRef Name,
StorageClass SC = SC_None) {
for (auto *Method : Record->methods()) {
if (Method->getStorageClass() == SC && Method->getName() == Name)
return Method;
}
return nullptr;
}
static CXXMethodDecl *lookupResourceInitMethodAndSetupArgs(
CodeGenModule &CGM, CXXRecordDecl *ResourceDecl, llvm::Value *Range,
llvm::Value *Index, StringRef Name, ResourceBindingAttrs &Binding,
CallArgList &Args) {
assert(Binding.hasBinding() && "at least one binding attribute expected");
ASTContext &AST = CGM.getContext();
CXXMethodDecl *CreateMethod = nullptr;
Value *NameStr = buildNameForResource(Name, CGM);
Value *Space = llvm::ConstantInt::get(CGM.IntTy, Binding.getSpace());
bool HasCounter = hasCounterHandle(ResourceDecl);
assert((!HasCounter || Binding.hasCounterImplicitOrderID()) &&
"resources with counter handle must have a binding with counter "
"implicit order ID");
if (Binding.isExplicit()) {
// explicit binding
auto *RegSlot = llvm::ConstantInt::get(CGM.IntTy, Binding.getSlot());
Args.add(RValue::get(RegSlot), AST.UnsignedIntTy);
const char *Name = Binding.hasCounterImplicitOrderID()
? "__createFromBindingWithImplicitCounter"
: "__createFromBinding";
CreateMethod = lookupMethod(ResourceDecl, Name, SC_Static);
} else {
// implicit binding
auto *OrderID =
llvm::ConstantInt::get(CGM.IntTy, Binding.getImplicitOrderID());
Args.add(RValue::get(OrderID), AST.UnsignedIntTy);
const char *Name = Binding.hasCounterImplicitOrderID()
? "__createFromImplicitBindingWithImplicitCounter"
: "__createFromImplicitBinding";
CreateMethod = lookupMethod(ResourceDecl, Name, SC_Static);
}
Args.add(RValue::get(Space), AST.UnsignedIntTy);
Args.add(RValue::get(Range), AST.IntTy);
Args.add(RValue::get(Index), AST.UnsignedIntTy);
Args.add(RValue::get(NameStr), AST.getPointerType(AST.CharTy.withConst()));
if (HasCounter) {
uint32_t CounterBinding = Binding.getCounterImplicitOrderID();
auto *CounterOrderID = llvm::ConstantInt::get(CGM.IntTy, CounterBinding);
Args.add(RValue::get(CounterOrderID), AST.UnsignedIntTy);
}
return CreateMethod;
}
static void callResourceInitMethod(CodeGenFunction &CGF,
CXXMethodDecl *CreateMethod,
CallArgList &Args, Address ReturnAddress) {
llvm::Constant *CalleeFn = CGF.CGM.GetAddrOfFunction(CreateMethod);
const FunctionProtoType *Proto =
CreateMethod->getType()->getAs<FunctionProtoType>();
// HLSL code generation is restricted to DXIL and SPIR-V targets, so no
// caller declaration is needed for x86 SysV ABI selection.
const CGFunctionInfo &FnInfo = CGF.CGM.getTypes().arrangeFreeFunctionCall(
Args, Proto, false, /*ABIInfoFD=*/nullptr);
ReturnValueSlot ReturnValue(ReturnAddress, false);
CGCallee Callee(CGCalleeInfo(Proto), CalleeFn);
CGF.EmitCall(FnInfo, Callee, ReturnValue, Args, nullptr);
}
// Initializes local resource array variable with global resource array
// elements. For multi-dimensional arrays it calls itself recursively to
// initialize its sub-arrays. The Index used in the resource constructor calls
// will begin at StartIndex and will be incremented for each array element. The
// last used resource Index is returned to the caller. If the function returns
// std::nullopt, it indicates an error.
static std::optional<llvm::Value *> initializeResourceArrayFromGlobal(
CodeGenFunction &CGF, CXXRecordDecl *ResourceDecl,
const ConstantArrayType *ArrayTy, AggValueSlot &ValueSlot,
llvm::Value *Range, llvm::Value *StartIndex, StringRef ResourceName,
ResourceBindingAttrs &Binding, ArrayRef<llvm::Value *> PrevGEPIndices) {
ASTContext &AST = CGF.getContext();
llvm::IntegerType *IntTy = CGF.CGM.IntTy;
llvm::Value *Index = StartIndex;
llvm::Value *One = llvm::ConstantInt::get(IntTy, 1);
const uint64_t ArraySize = ArrayTy->getSExtSize();
QualType ElemType = ArrayTy->getElementType();
Address TmpArrayAddr = ValueSlot.getAddress();
// Add additional index to the getelementptr call indices.
// This index will be updated for each array element in the loops below.
SmallVector<llvm::Value *> GEPIndices(PrevGEPIndices);
GEPIndices.push_back(llvm::ConstantInt::get(IntTy, 0));
// For array of arrays, recursively initialize the sub-arrays.
if (ElemType->isArrayType()) {
const ConstantArrayType *SubArrayTy = cast<ConstantArrayType>(ElemType);
for (uint64_t I = 0; I < ArraySize; I++) {
if (I > 0) {
Index = CGF.Builder.CreateAdd(Index, One);
GEPIndices.back() = llvm::ConstantInt::get(IntTy, I);
}
std::optional<llvm::Value *> MaybeIndex =
initializeResourceArrayFromGlobal(CGF, ResourceDecl, SubArrayTy,
ValueSlot, Range, Index,
ResourceName, Binding, GEPIndices);
if (!MaybeIndex)
return std::nullopt;
Index = *MaybeIndex;
}
return Index;
}
// For array of resources, initialize each resource in the array.
llvm::Type *Ty = CGF.ConvertTypeForMem(ElemType);
CharUnits ElemSize = AST.getTypeSizeInChars(ElemType);
CharUnits Align =
TmpArrayAddr.getAlignment().alignmentOfArrayElement(ElemSize);
for (uint64_t I = 0; I < ArraySize; I++) {
if (I > 0) {
Index = CGF.Builder.CreateAdd(Index, One);
GEPIndices.back() = llvm::ConstantInt::get(IntTy, I);
}
Address ReturnAddress =
CGF.Builder.CreateGEP(TmpArrayAddr, GEPIndices, Ty, Align);
CallArgList Args;
CXXMethodDecl *CreateMethod = lookupResourceInitMethodAndSetupArgs(
CGF.CGM, ResourceDecl, Range, Index, ResourceName, Binding, Args);
if (!CreateMethod)
// This can happen if someone creates an array of structs that looks like
// an HLSL resource record array but it does not have the required static
// create method. No binding will be generated for it.
return std::nullopt;
callResourceInitMethod(CGF, CreateMethod, Args, ReturnAddress);
}
return Index;
}
/// Utility for emitting copies following the HLSL buffer layout rules (ie,
/// copying out of a cbuffer).
class HLSLBufferCopyEmitter {
CodeGenFunction &CGF;
Address DstPtr;
Address SrcPtr;
llvm::Type *LayoutTy = nullptr;
SmallVector<llvm::Value *> CurStoreIndices;
SmallVector<llvm::Value *> CurLoadIndices;
using EmitResourceFnTy = llvm::function_ref<void(AggValueSlot &)>;
// Creates & returns either a structured.gep or a ptradd/gep depending on
// langopts.
llvm::Value *emitAccessChain(llvm::Type *BaseTy, llvm::Value *Base,
ArrayRef<llvm::Value *> Indices) {
bool EmitLogical = CGF.getLangOpts().EmitLogicalPointer;
if (EmitLogical)
return CGF.Builder.CreateAccessChain(EmitLogical, BaseTy, Base, Indices);
llvm::SmallVector<llvm::Value *> GEPIndices;
GEPIndices.reserve(Indices.size() + 1);
GEPIndices.push_back(llvm::ConstantInt::get(CGF.IntTy, 0));
GEPIndices.append(Indices.begin(), Indices.end());
return CGF.Builder.CreateAccessChain(EmitLogical, BaseTy, Base, GEPIndices);
}
bool isBufferLayoutArray(llvm::StructType *ST) {
// A buffer layout array is a struct with two elements: the padded array,
// and the last element. That is, is should look something like this:
//
// { [%n x { %type, %padding }], %type }
//
if (!ST || ST->getNumElements() != 2)
return false;
auto *PaddedEltsTy = dyn_cast<llvm::ArrayType>(ST->getElementType(0));
if (!PaddedEltsTy)
return false;
auto *PaddedTy = dyn_cast<llvm::StructType>(PaddedEltsTy->getElementType());
if (!PaddedTy || PaddedTy->getNumElements() != 2)
return false;
if (!CGF.CGM.getTargetCodeGenInfo().isHLSLPadding(
PaddedTy->getElementType(1)))
return false;
llvm::Type *ElementTy = ST->getElementType(1);
if (PaddedTy->getElementType(0) != ElementTy)
return false;
return true;
}
// Returns true if the type is either a struct representing a resource record,
// or an array of structs that are resource records. This assumes a struct is
// a resource record if the first element is a target type (resource handle).
// This is the case for all target types used by HLSL except the padding type
// ("{dx|spirv.Padding"), but padding will never be the first element of a
// struct.
bool isResourceOrResourceArray(llvm::Type *Ty) {
while (auto *AT = dyn_cast<llvm::ArrayType>(Ty))
Ty = AT->getElementType();
auto *ST = dyn_cast<llvm::StructType>(Ty);
if (!ST || ST->getNumElements() < 1)
return false;
auto *TargetTy = dyn_cast<llvm::TargetExtType>(ST->getElementType(0));
return TargetTy != nullptr;
}
void emitResourceOrResourceArray(Value *Dst, llvm::Type *DstTy,
EmitResourceFnTy EmitResFn) {
CharUnits DstAlign =
CharUnits::fromQuantity(CGF.CGM.getDataLayout().getABITypeAlign(DstTy));
Address DstAddr(Dst, DstTy, DstAlign);
AggValueSlot Slot = AggValueSlot::forAddr(
DstAddr, Qualifiers(), AggValueSlot::IsDestructed_t(true),
AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsAliased_t(false),
AggValueSlot::DoesNotOverlap);
EmitResFn(Slot);
}
void emitBufferLayoutCopy(Value *Src, llvm::StructType *SrcTy, Value *Dst,
llvm::ArrayType *DstTy,
EmitResourceFnTy EmitResFn) {
// Those assumptions are checked by isBufferLayoutArray.
auto *SrcPaddedArrayTy = cast<llvm::ArrayType>(SrcTy->getElementType(0));
assert(SrcPaddedArrayTy->getNumElements() + 1 == DstTy->getNumElements());
assert(cast<llvm::StructType>(SrcPaddedArrayTy->getElementType())
->getElementType(0) == SrcTy->getElementType(1));
auto *SrcDataTy = SrcTy->getElementType(1);
auto Zero = llvm::ConstantInt::get(CGF.IntTy, 0);
for (unsigned I = 0; I < SrcPaddedArrayTy->getNumElements(); ++I) {
auto Index = llvm::ConstantInt::get(CGF.IntTy, I);
auto *SrcElt = emitAccessChain(SrcTy, Src, {Zero, Index, Zero});
auto *DstElt = emitAccessChain(DstTy, Dst, {Index});
emitElementCopy(SrcElt, SrcDataTy, DstElt, DstTy->getElementType(),
EmitResFn);
}
auto *SrcElt =
emitAccessChain(SrcTy, Src, {llvm::ConstantInt::get(CGF.IntTy, 1)});
auto *DstElt = emitAccessChain(
DstTy, Dst,
{llvm::ConstantInt::get(CGF.IntTy, DstTy->getNumElements() - 1)});
emitElementCopy(SrcElt, SrcDataTy, DstElt, DstTy->getElementType(),
EmitResFn);
}
void emitCopy(Value *Src, llvm::StructType *SrcTy, Value *Dst,
llvm::Type *DstTy, EmitResourceFnTy EmitResFn) {
assert(!isResourceOrResourceArray(DstTy) &&
"direct access to resources or resource arrays should be handled "
"separately");
if (isBufferLayoutArray(SrcTy))
return emitBufferLayoutCopy(Src, SrcTy, Dst, cast<llvm::ArrayType>(DstTy),
EmitResFn);
unsigned SrcIndex = 0;
unsigned DstIndex = 0;
// DstTy layout is in default address space and can include resource types.
// SrcTy is in cbuffer layout where resources are filtered out, so the
// number of elements in SrcTy can be less than the number of elements in
// DstTy.
auto *DstST = cast<llvm::StructType>(DstTy);
while (DstIndex < DstST->getNumElements()) {
llvm::Type *DstEltTy = DstST->getElementType(DstIndex);
if (CGF.CGM.getTargetCodeGenInfo().isHLSLPadding(DstEltTy)) {
DstIndex += 1;
continue;
}
if (isResourceOrResourceArray(DstEltTy)) {
auto *DstElt = emitAccessChain(
DstTy, Dst, {llvm::ConstantInt::get(CGF.IntTy, DstIndex)});
emitResourceOrResourceArray(DstElt, DstEltTy, EmitResFn);
DstIndex += 1;
continue;
}
assert(SrcIndex < SrcTy->getNumElements());
llvm::Type *SrcEltTy = SrcTy->getElementType(SrcIndex);
if (CGF.CGM.getTargetCodeGenInfo().isHLSLPadding(SrcEltTy)) {
SrcIndex += 1;
continue;
}
auto *SrcElt = emitAccessChain(
SrcTy, Src, {llvm::ConstantInt::get(CGF.IntTy, SrcIndex)});
auto *DstElt = emitAccessChain(
DstTy, Dst, {llvm::ConstantInt::get(CGF.IntTy, DstIndex)});
emitElementCopy(SrcElt, SrcEltTy, DstElt, DstEltTy, EmitResFn);
DstIndex += 1;
SrcIndex += 1;
}
}
void emitCopy(Value *Src, llvm::ArrayType *SrcTy, Value *Dst,
llvm::Type *DstTy, EmitResourceFnTy EmitResFn) {
for (unsigned I = 0, E = SrcTy->getNumElements(); I < E; ++I) {
auto *SrcElt =
emitAccessChain(SrcTy, Src, {llvm::ConstantInt::get(CGF.IntTy, I)});
auto *DstElt =
emitAccessChain(DstTy, Dst, {llvm::ConstantInt::get(CGF.IntTy, I)});
emitElementCopy(SrcElt, SrcTy->getElementType(), DstElt,
cast<llvm::ArrayType>(DstTy)->getElementType(),
EmitResFn);
}
}
void emitElementCopy(Value *Src, llvm::Type *SrcTy, Value *Dst,
llvm::Type *DstTy, EmitResourceFnTy EmitResFn) {
if (auto *AT = dyn_cast<llvm::ArrayType>(SrcTy))
return emitCopy(Src, AT, Dst, DstTy, EmitResFn);
if (auto *ST = dyn_cast<llvm::StructType>(SrcTy))
return emitCopy(Src, ST, Dst, DstTy, EmitResFn);
// When we have a scalar or vector element we can emit the copy.
CharUnits SrcAlign =
CharUnits::fromQuantity(CGF.CGM.getDataLayout().getABITypeAlign(SrcTy));
CharUnits DstAlign =
CharUnits::fromQuantity(CGF.CGM.getDataLayout().getABITypeAlign(DstTy));
Address SrcAddr(Src, SrcTy, SrcAlign);
Address DstAddr(Dst, DstTy, DstAlign);
llvm::Value *Load = CGF.Builder.CreateLoad(SrcAddr, "cbuf.load");
CGF.Builder.CreateStore(Load, DstAddr);
}
public:
HLSLBufferCopyEmitter(CodeGenFunction &CGF, Address DstPtr, Address SrcPtr)
: CGF(CGF), DstPtr(DstPtr), SrcPtr(SrcPtr) {}
bool emitCopy(QualType CType, EmitResourceFnTy EmitResFn = nullptr) {
LayoutTy = HLSLBufferLayoutBuilder(CGF.CGM).layOutType(CType);
// TODO: We should be able to fall back to a regular memcpy if the layout
// type doesn't have any padding, but that runs into issues in the backend
// currently.
//
// See https://github.com/llvm/wg-hlsl/issues/351
emitElementCopy(SrcPtr.getBasePointer(), LayoutTy, DstPtr.getBasePointer(),
DstPtr.getElementType(), EmitResFn);
return true;
}
};
// Represents a list resources associated with a global struct whose name
// starts with the specified prefix.
// The order of HLSLAssociatedResourceDeclAttr attributes is identical to the
// order of the depth-first traversal of the corresponding fields in the struct.
// The resources are always returned in that order, which is the same order
// we need when a struct is copied element-by-element.
class AssociatedResourcesList {
// Iterator pointers for the associated resource attributes that match the
// prefix. Begin = begin of the range of attributes that match the prefix End
// = end of the range of attributes that match the prefix Next = the current
// attribute in the iteration to be returned by getNextResource
specific_attr_iterator<HLSLAssociatedResourceDeclAttr> Begin, End, Next;
public:
AssociatedResourcesList(const VarDecl *StructVD,
StringRef ResourceNamePrefix) {
auto I = StructVD->specific_attr_begin<HLSLAssociatedResourceDeclAttr>();
auto E = StructVD->specific_attr_end<HLSLAssociatedResourceDeclAttr>();
// Skip over associated resources that don't match the prefix.
while (I != E &&
!I->getResDecl()->getName().starts_with(ResourceNamePrefix))
++I;
assert(I != E && "expected associated resource not found");
Begin = End = I;
// Scan over associated resources that do match the prefix to find the end
// of the range.
while (I != E && ((HLSLAssociatedResourceDeclAttr *)*I)
->getResDecl()
->getName()
.starts_with(ResourceNamePrefix))
End = ++I;
Next = Begin;
}
const VarDecl *getNextResource() {
if (Next == End)
return nullptr;
const VarDecl *Res = Next->getResDecl();
++Next;
return Res;
}
};
} // namespace
llvm::Type *
CGHLSLRuntime::convertHLSLSpecificType(const Type *T,
const CGHLSLOffsetInfo &OffsetInfo) {
assert(T->isHLSLSpecificType() && "Not an HLSL specific type!");
// Check if the target has a specific translation for this type first.
if (llvm::Type *TargetTy =
CGM.getTargetCodeGenInfo().getHLSLType(CGM, T, OffsetInfo))
return TargetTy;
llvm_unreachable("Generic handling of HLSL types is not supported.");
}
llvm::Triple::ArchType CGHLSLRuntime::getArch() {
return CGM.getTarget().getTriple().getArch();
}
// Emits constant global variables for buffer constants declarations
// and creates metadata linking the constant globals with the buffer global.
void CGHLSLRuntime::emitBufferGlobalsAndMetadata(
const HLSLBufferDecl *BufDecl, llvm::GlobalVariable *BufGV,
const CGHLSLOffsetInfo &OffsetInfo) {
LLVMContext &Ctx = CGM.getLLVMContext();
// get the layout struct from constant buffer target type
llvm::Type *BufType = BufGV->getValueType();
llvm::StructType *LayoutStruct = cast<llvm::StructType>(
cast<llvm::TargetExtType>(BufType)->getTypeParameter(0));
SmallVector<std::pair<VarDecl *, uint32_t>> DeclsWithOffset;
size_t OffsetIdx = 0;
for (Decl *D : BufDecl->buffer_decls()) {
if (isa<CXXRecordDecl, EmptyDecl>(D))
// Nothing to do for this declaration.
continue;
if (isa<FunctionDecl>(D)) {
// A function within an cbuffer is effectively a top-level function.
CGM.EmitTopLevelDecl(D);
continue;
}
VarDecl *VD = dyn_cast<VarDecl>(D);
if (!VD)
continue;
QualType VDTy = VD->getType();
if (VDTy.getAddressSpace() != LangAS::hlsl_constant) {
if (VD->getStorageClass() == SC_Static ||
VDTy.getAddressSpace() == LangAS::hlsl_groupshared ||
VDTy->isHLSLResourceRecord() || VDTy->isHLSLResourceRecordArray()) {
// Emit static and groupshared variables and resource classes inside
// cbuffer as regular globals
CGM.EmitGlobal(VD);
}
continue;
}
DeclsWithOffset.emplace_back(VD, OffsetInfo[OffsetIdx++]);
}
if (!OffsetInfo.empty())
llvm::stable_sort(DeclsWithOffset, [](const auto &LHS, const auto &RHS) {
return CGHLSLOffsetInfo::compareOffsets(LHS.second, RHS.second);
});
// Associate the buffer global variable with its constants
SmallVector<llvm::Metadata *> BufGlobals;
BufGlobals.reserve(DeclsWithOffset.size() + 1);
BufGlobals.push_back(ValueAsMetadata::get(BufGV));
auto ElemIt = LayoutStruct->element_begin();
for (auto &[VD, _] : DeclsWithOffset) {
if (CGM.getTargetCodeGenInfo().isHLSLPadding(*ElemIt))
++ElemIt;
assert(ElemIt != LayoutStruct->element_end() &&
"number of elements in layout struct does not match");
llvm::Type *LayoutType = *ElemIt++;
GlobalVariable *ElemGV =
cast<GlobalVariable>(CGM.GetAddrOfGlobalVar(VD, LayoutType));
BufGlobals.push_back(ValueAsMetadata::get(ElemGV));
}
assert(ElemIt == LayoutStruct->element_end() &&
"number of elements in layout struct does not match");
// add buffer metadata to the module
CGM.getModule()
.getOrInsertNamedMetadata("hlsl.cbs")
->addOperand(MDNode::get(Ctx, BufGlobals));
}
// Creates resource handle type for the HLSL buffer declaration
static const clang::HLSLAttributedResourceType *
createBufferHandleType(const HLSLBufferDecl *BufDecl) {
ASTContext &AST = BufDecl->getASTContext();
QualType QT = AST.getHLSLAttributedResourceType(
AST.HLSLResourceTy, AST.getCanonicalTagType(BufDecl->getLayoutStruct()),
HLSLAttributedResourceType::Attributes(ResourceClass::CBuffer));
return cast<HLSLAttributedResourceType>(QT.getTypePtr());
}
CGHLSLOffsetInfo CGHLSLOffsetInfo::fromDecl(const HLSLBufferDecl &BufDecl) {
CGHLSLOffsetInfo Result;
// If we don't have packoffset info, just return an empty result.
if (!BufDecl.hasValidPackoffset())
return Result;
for (Decl *D : BufDecl.buffer_decls()) {
if (isa<CXXRecordDecl, EmptyDecl>(D) || isa<FunctionDecl>(D)) {
continue;
}
VarDecl *VD = dyn_cast<VarDecl>(D);
if (!VD || VD->getType().getAddressSpace() != LangAS::hlsl_constant)
continue;
if (!VD->hasAttrs()) {
Result.Offsets.push_back(Unspecified);
continue;
}
uint32_t Offset = Unspecified;
for (auto *Attr : VD->getAttrs()) {
if (auto *POA = dyn_cast<HLSLPackOffsetAttr>(Attr)) {
Offset = POA->getOffsetInBytes();
break;
}
auto *RBA = dyn_cast<HLSLResourceBindingAttr>(Attr);
if (RBA &&
RBA->getRegisterType() == HLSLResourceBindingAttr::RegisterType::C) {
Offset = RBA->getSlotNumber() * CBufferRowSizeInBytes;
break;
}
}
Result.Offsets.push_back(Offset);
}
return Result;
}
// Codegen for HLSLBufferDecl
void CGHLSLRuntime::addBuffer(const HLSLBufferDecl *BufDecl) {
assert(BufDecl->isCBuffer() && "tbuffer codegen is not supported yet");
// create resource handle type for the buffer
const clang::HLSLAttributedResourceType *ResHandleTy =
createBufferHandleType(BufDecl);
// empty constant buffer is ignored
if (ResHandleTy->getContainedType()->getAsCXXRecordDecl()->isEmpty())
return;
// create global variable for the constant buffer
CGHLSLOffsetInfo OffsetInfo = CGHLSLOffsetInfo::fromDecl(*BufDecl);
llvm::Type *LayoutTy = convertHLSLSpecificType(ResHandleTy, OffsetInfo);
llvm::GlobalVariable *BufGV = new GlobalVariable(
LayoutTy, /*isConstant*/ false,
GlobalValue::LinkageTypes::InternalLinkage, PoisonValue::get(LayoutTy),
llvm::formatv("{0}{1}", BufDecl->getName(),
BufDecl->isCBuffer() ? ".cb" : ".tb"),
GlobalValue::NotThreadLocal);
llvm::Module &M = CGM.getModule();
M.insertGlobalVariable(BufGV);
// Add the global variable to the compiler used list so it does not
// get optimized away by GlobalOptPass before it reaches
// {DXIL|SPIRV}CBufferAccess pass.
llvm::appendToCompilerUsed(M, {BufGV});
// Add globals for constant buffer elements and create metadata nodes
emitBufferGlobalsAndMetadata(BufDecl, BufGV, OffsetInfo);
// Initialize cbuffer from binding (implicit or explicit)
initializeBufferFromBinding(BufDecl, BufGV);
}
void CGHLSLRuntime::addRootSignature(
const HLSLRootSignatureDecl *SignatureDecl) {
llvm::Module &M = CGM.getModule();
Triple T(M.getTargetTriple());
// Generated later with the function decl if not targeting root signature
if (T.getEnvironment() != Triple::EnvironmentType::RootSignature)
return;
addRootSignatureMD(SignatureDecl->getVersion(),
SignatureDecl->getRootElements(), nullptr, M);
}
llvm::StructType *
CGHLSLRuntime::getHLSLBufferLayoutType(const RecordType *StructType) {
const auto Entry = LayoutTypes.find(StructType);
if (Entry != LayoutTypes.end())
return Entry->getSecond();
return nullptr;
}
void CGHLSLRuntime::addHLSLBufferLayoutType(const RecordType *StructType,
llvm::StructType *LayoutTy) {
assert(getHLSLBufferLayoutType(StructType) == nullptr &&
"layout type for this struct already exist");
LayoutTypes[StructType] = LayoutTy;
}
void CGHLSLRuntime::finishCodeGen() {
auto &TargetOpts = CGM.getTarget().getTargetOpts();
auto &CodeGenOpts = CGM.getCodeGenOpts();
auto &LangOpts = CGM.getLangOpts();
llvm::Module &M = CGM.getModule();
Triple T(M.getTargetTriple());
if (T.getArch() == Triple::ArchType::dxil)
addDxilValVersion(TargetOpts.DxilValidatorVersion, M);
if (!CodeGenOpts.DisableDXSourceMetadata &&
CodeGenOpts.getDebugInfo() >=
llvm::codegenoptions::DebugInfoKind::DebugInfoConstructor)
addSourceInfo(CGM, M);
if (CodeGenOpts.ResMayAlias)
M.setModuleFlag(llvm::Module::ModFlagBehavior::Error, "dx.resmayalias", 1);
if (CodeGenOpts.AllResourcesBound)
M.setModuleFlag(llvm::Module::ModFlagBehavior::Error,
"dx.allresourcesbound", 1);
if (CodeGenOpts.OptimizationLevel == 0)
M.addModuleFlag(llvm::Module::ModFlagBehavior::Override,
"dx.disable_optimizations", 1);
// NativeHalfType corresponds to the -fnative-half-type clang option which is
// aliased by clang-dxc's -enable-16bit-types option. This option is used to
// set the UseNativeLowPrecision DXIL module flag in the DirectX backend
if (LangOpts.NativeHalfType)
M.setModuleFlag(llvm::Module::ModFlagBehavior::Error, "dx.nativelowprec",
1);
if (LangOpts.HLSLSpvPreserveInterface && T.isSPIRV()) {
// Runs before optimization. Keeps Input/Output globals from GlobalDCE.
const ASTContext &Ctx = CGM.getContext();
unsigned InputAS = Ctx.getTargetAddressSpace(LangAS::hlsl_input);
unsigned OutputAS = Ctx.getTargetAddressSpace(LangAS::hlsl_output);
SmallVector<GlobalValue *, 8> InterfaceVars;
for (GlobalVariable &GV : M.globals()) {
unsigned AS = GV.getAddressSpace();
if (AS == InputAS || AS == OutputAS)
InterfaceVars.push_back(&GV);
}
if (!InterfaceVars.empty())
appendToCompilerUsed(M, InterfaceVars);
}
generateGlobalCtorDtorCalls();
}
void clang::CodeGen::CGHLSLRuntime::setHLSLEntryAttributes(
const FunctionDecl *FD, llvm::Function *Fn) {
const auto *ShaderAttr = FD->getAttr<HLSLShaderAttr>();
assert(ShaderAttr && "All entry functions must have a HLSLShaderAttr");
const StringRef ShaderAttrKindStr = "hlsl.shader";
Fn->addFnAttr(ShaderAttrKindStr,
llvm::Triple::getEnvironmentTypeName(ShaderAttr->getType()));
if (HLSLNumThreadsAttr *NumThreadsAttr = FD->getAttr<HLSLNumThreadsAttr>()) {
const StringRef NumThreadsKindStr = "hlsl.numthreads";
std::string NumThreadsStr =
formatv("{0},{1},{2}", NumThreadsAttr->getX(), NumThreadsAttr->getY(),
NumThreadsAttr->getZ());
Fn->addFnAttr(NumThreadsKindStr, NumThreadsStr);
}
if (HLSLWaveSizeAttr *WaveSizeAttr = FD->getAttr<HLSLWaveSizeAttr>()) {
const StringRef WaveSizeKindStr = "hlsl.wavesize";
std::string WaveSizeStr =
formatv("{0},{1},{2}", WaveSizeAttr->getMin(), WaveSizeAttr->getMax(),
WaveSizeAttr->getPreferred());
Fn->addFnAttr(WaveSizeKindStr, WaveSizeStr);
}
// HLSL entry functions are materialized for module functions with
// HLSLShaderAttr attribute. SetLLVMFunctionAttributesForDefinition called
// later in the compiler-flow for such module functions is not aware of and
// hence not able to set attributes of the newly materialized entry functions.
// So, set attributes of entry function here, as appropriate.
Fn->addFnAttr(llvm::Attribute::NoInline);
if (CGM.getLangOpts().HLSLSpvEnableMaximalReconvergence) {
Fn->addFnAttr("enable-maximal-reconvergence", "true");
}
}
static Value *buildVectorInput(IRBuilder<> &B, Function *F, llvm::Type *Ty) {
// Compute ID intrinsics return i32 components, but the semantic may use
// 16-bit integers. Narrow each component before assembling the input.
if (const auto *VT = dyn_cast<FixedVectorType>(Ty)) {
Value *Result = PoisonValue::get(Ty);
for (unsigned I = 0; I < VT->getNumElements(); ++I) {
Value *Elt = B.CreateCall(F, {B.getInt32(I)});
Elt = B.CreateTrunc(Elt, VT->getElementType());
Result = B.CreateInsertElement(Result, Elt, I);
}
return Result;
}
return B.CreateTrunc(B.CreateCall(F, {B.getInt32(0)}), Ty);
}
// A fragment shader input interface variable whose base type is an integer or
// a 64-bit float (double) cannot be interpolated by the rasterizer. The Vulkan
// specification requires these variables to be decorated with Flat (see
// VUID-StandaloneSpirv-Flat-04744). Arrays and vectors are unwrapped to inspect
// their base scalar type.
static bool requiresFlatInterpolation(llvm::Type *Ty) {
while (true) {
if (auto *AT = dyn_cast<llvm::ArrayType>(Ty)) {
Ty = AT->getElementType();
continue;
}
if (auto *VT = dyn_cast<llvm::FixedVectorType>(Ty)) {
Ty = VT->getElementType();
continue;
}
break;
}
return Ty->isIntegerTy() || Ty->isDoubleTy();
}
static llvm::hlsl::InterpolationMode
getSPIRVInterpolationMode(llvm::hlsl::InterpolationMode InterpMode,
llvm::Type *Ty, llvm::Triple::EnvironmentType Stage,
llvm::hlsl::IOType IO) {
using llvm::hlsl::InterpolationMode;
if (!((Stage == llvm::Triple::Pixel && IO == llvm::hlsl::IOType::In) ||
(Stage == llvm::Triple::Vertex && IO == llvm::hlsl::IOType::Out)))
return InterpolationMode::Undefined;
if (InterpMode == InterpolationMode::Undefined &&
requiresFlatInterpolation(Ty))
return InterpolationMode::Constant;
return InterpMode;
}
static void
addSPIRVInterfaceDecorations(llvm::GlobalVariable *GV,
llvm::hlsl::SPIRVInterfaceDecoration Decoration,
unsigned Operand,
llvm::hlsl::InterpolationMode InterpMode =
llvm::hlsl::InterpolationMode::Undefined) {
using llvm::hlsl::InterpolationMode;
using llvm::hlsl::SPIRVInterpolationMode;
LLVMContext &Ctx = GV->getContext();
IRBuilder<> B(*GV->getParent());
SmallVector<Metadata *, 3> Decorations;
Decorations.push_back(MDNode::get(
Ctx,
{ConstantAsMetadata::get(B.getInt32(llvm::to_underlying(Decoration))),
ConstantAsMetadata::get(B.getInt32(Operand))}));
auto Add = [&](SPIRVInterpolationMode InterpolationDecoration) {
Decorations.push_back(
MDNode::get(Ctx, {ConstantAsMetadata::get(B.getInt32(
llvm::to_underlying(InterpolationDecoration)))}));
};
switch (InterpMode) {
case InterpolationMode::Undefined:
case InterpolationMode::Linear:
break;
case InterpolationMode::Constant:
Add(SPIRVInterpolationMode::Flat);
break;
case InterpolationMode::LinearNoperspective:
Add(SPIRVInterpolationMode::NoPerspective);
break;
case InterpolationMode::LinearNoperspectiveCentroid:
Add(SPIRVInterpolationMode::NoPerspective);
[[fallthrough]];
case InterpolationMode::LinearCentroid:
Add(SPIRVInterpolationMode::Centroid);
break;
case InterpolationMode::LinearNoperspectiveSample:
Add(SPIRVInterpolationMode::NoPerspective);
[[fallthrough]];
case InterpolationMode::LinearSample:
Add(SPIRVInterpolationMode::Sample);
break;
default:
llvm_unreachable("invalid interpolation mode");
}
// The backend consumes one decoration list per global, including both the
// Location or BuiltIn and any interpolation decorations.
GV->addMetadata("spirv.Decorations", *MDNode::get(Ctx, Decorations));
}
static llvm::Value *createSPIRVInputLoad(
IRBuilder<> &B, llvm::Module &M, llvm::Type *Ty, const Twine &Name,
llvm::hlsl::SPIRVInterfaceDecoration Decoration, unsigned Operand,
llvm::hlsl::InterpolationMode InterpMode) {
auto *GV = new llvm::GlobalVariable(
M, Ty, /* isConstant= */ true, llvm::GlobalValue::ExternalLinkage,
/* Initializer= */ nullptr, Name, /* insertBefore= */ nullptr,
llvm::GlobalVariable::GeneralDynamicTLSModel,
/* AddressSpace */ 7, /* isExternallyInitialized= */ true);
GV->setVisibility(llvm::GlobalValue::HiddenVisibility);
addSPIRVInterfaceDecorations(GV, Decoration, Operand, InterpMode);
return B.CreateLoad(Ty, GV);
}
llvm::Value *CGHLSLRuntime::emitSPIRVUserSemanticLoad(
llvm::IRBuilder<> &B, llvm::Type *Type, const clang::DeclaratorDecl *Decl,
HLSLAppliedSemanticAttr *Semantic, std::optional<unsigned> Index,
llvm::hlsl::InterpolationMode InterpMode) {
Twine BaseName = Twine(Semantic->getAttrName()->getName());
Twine VariableName = BaseName.concat(Twine(Index.value_or(0)));
unsigned Location = SPIRVLastAssignedInputSemanticLocation;
if (auto *L = Decl->getAttr<HLSLVkLocationAttr>())
Location = L->getLocation();
// DXC completely ignores the semantic/index pair. Location are assigned from
// the first semantic to the last.
llvm::ArrayType *AT = dyn_cast<llvm::ArrayType>(Type);
unsigned ElementCount = AT ? AT->getNumElements() : 1;
SPIRVLastAssignedInputSemanticLocation += ElementCount;
return createSPIRVInputLoad(B, CGM.getModule(), Type, VariableName.str(),
llvm::hlsl::SPIRVInterfaceDecoration::Location,
Location, InterpMode);
}
static void createSPIRVOutputStore(
IRBuilder<> &B, llvm::Module &M, llvm::Value *Source, const Twine &Name,
llvm::hlsl::SPIRVInterfaceDecoration Decoration, unsigned Operand,
llvm::hlsl::InterpolationMode InterpMode) {
auto *GV = new llvm::GlobalVariable(
M, Source->getType(), /* isConstant= */ false,
llvm::GlobalValue::ExternalLinkage,
/* Initializer= */ nullptr, Name, /* insertBefore= */ nullptr,
llvm::GlobalVariable::GeneralDynamicTLSModel,
/* AddressSpace */ 8, /* isExternallyInitialized= */ false);
GV->setVisibility(llvm::GlobalValue::HiddenVisibility);
addSPIRVInterfaceDecorations(GV, Decoration, Operand, InterpMode);
B.CreateStore(Source, GV);
}
void CGHLSLRuntime::emitSPIRVUserSemanticStore(
llvm::IRBuilder<> &B, llvm::Value *Source,
const clang::DeclaratorDecl *Decl, HLSLAppliedSemanticAttr *Semantic,
std::optional<unsigned> Index, llvm::hlsl::InterpolationMode InterpMode) {
Twine BaseName = Twine(Semantic->getAttrName()->getName());
Twine VariableName = BaseName.concat(Twine(Index.value_or(0)));
unsigned Location = SPIRVLastAssignedOutputSemanticLocation;
if (auto *L = Decl->getAttr<HLSLVkLocationAttr>())
Location = L->getLocation();
// DXC completely ignores the semantic/index pair. Location are assigned from
// the first semantic to the last.
llvm::ArrayType *AT = dyn_cast<llvm::ArrayType>(Source->getType());
unsigned ElementCount = AT ? AT->getNumElements() : 1;
SPIRVLastAssignedOutputSemanticLocation += ElementCount;
createSPIRVOutputStore(B, CGM.getModule(), Source, VariableName.str(),
llvm::hlsl::SPIRVInterfaceDecoration::Location,
Location, InterpMode);
}
namespace {
// Describes how a semantic leaf lowers to signature rows
struct SemanticShape {
// Array length followed by matrix major-axis length.
SmallVector<unsigned> Dimensions;
unsigned Cols;
QualType RowType;
bool IsMatrix = false;
bool IsRowMajor = false;
// Direct matrix arguments/results use column-major flattened vectors even
// when their signature (and aggregate memory representation) is row-major.
unsigned getMatrixElementIndex(unsigned Row, unsigned Col) const {
assert(IsMatrix);
unsigned MatrixRow = Row % Dimensions.back();
return IsRowMajor ? Col * Dimensions.back() + MatrixRow
: MatrixRow * Cols + Col;
}
unsigned getNumRows() const {
unsigned Rows = 1;
for (unsigned Dimension : Dimensions)
Rows *= Dimension;
return Rows;
}
SmallVector<unsigned> getArrayIndicesForRow(unsigned Row) const {
assert(Row < getNumRows() && "row exceeds semantic shape");
SmallVector<unsigned> Indices(Dimensions.size());
for (auto [Index, Dimension] :
llvm::zip_equal(llvm::reverse(Indices), llvm::reverse(Dimensions))) {
Index = Row % Dimension;
Row /= Dimension;
}
return Indices;
}
};
} // namespace
// Returns the QualType of a semantic leaf declarator. For a function the
// declared return type is used, otherwise the declared type.
static QualType getSemanticLeafType(const clang::DeclaratorDecl *Decl) {
if (const auto *FD = dyn_cast<clang::FunctionDecl>(Decl))
return FD->getDeclaredReturnType();
return Decl->getType();
}
// Collects array dimensions and, for a matrix leaf, its major-axis dimension.
// RowType is the scalar or vector transferred by each signature load/store.
static SemanticShape getSemanticShape(ASTContext &Ctx, QualType Ty) {
SemanticShape Shape{{}, 1, Ty};
while (const ConstantArrayType *CAT =
Ctx.getAsConstantArrayType(Shape.RowType)) {
Shape.Dimensions.push_back(CAT->getSize().getZExtValue());
Shape.RowType = CAT->getElementType();
}
if (const auto *VT = Shape.RowType->getAs<clang::VectorType>()) {
Shape.Cols = VT->getNumElements();
} else if (const auto *MT =
Shape.RowType->getAs<clang::ConstantMatrixType>()) {
Shape.IsMatrix = true;
Shape.IsRowMajor = isMatrixRowMajor(Ctx.getLangOpts(), Shape.RowType);
Shape.Dimensions.push_back(Shape.IsRowMajor ? MT->getNumRows()
: MT->getNumColumns());
Shape.Cols = Shape.IsRowMajor ? MT->getNumColumns() : MT->getNumRows();
Shape.RowType = Ctx.getExtVectorType(MT->getElementType(), Shape.Cols);
}
return Shape;
}
static llvm::dxil::ElementType getSignatureComponentType(CodeGenModule &CGM,
QualType Ty) {
if (const auto *VT = Ty->getAs<clang::VectorType>())
Ty = VT->getElementType();
llvm::Type *IRTy = CGM.getTypes().ConvertTypeForMem(Ty);
bool IsSigned = Ty->isSignedIntegerOrEnumerationType();
return llvm::hlsl::getDXILElementType(IRTy, IsSigned);
}
static llvm::hlsl::SemanticSignatureElement createSemanticSignatureElement(
CodeGenModule &CGM, uint32_t SigId, HLSLAppliedSemanticAttr *Semantic,
std::optional<unsigned> Index, const SemanticShape &Shape) {
StringRef Name = Semantic->getAttrName()->getName();
// One semantic index per row, starting from the declared index.
SmallVector<uint32_t> SemanticIndices;
uint32_t FirstSemanticIndex = Index.value_or(0);
for (uint32_t I = 0, E = Shape.getNumRows(); I < E; ++I)
SemanticIndices.push_back(FirstSemanticIndex + I);
// The remaining members keep their default value and will be filled at a
// later stage, either during packing or analysis of usage. Interpolation
// is set by the load/store traversal after visiting each leaf.
return llvm::hlsl::SemanticSignatureElement(
SigId, Name, getSignatureComponentType(CGM, Shape.RowType),
llvm::hlsl::getSemanticKind(Name), SemanticIndices,
static_cast<uint8_t>(Shape.Cols));
}
llvm::Value *CGHLSLRuntime::emitDXILUserSemanticLoad(
llvm::IRBuilder<> &B, llvm::Type *Type, const clang::DeclaratorDecl *Decl,
HLSLAppliedSemanticAttr *Semantic, std::optional<unsigned> Index,
SemanticSignatures &Signature) {
StringRef Name = Semantic->getAttrName()->getName();
SemanticShape Shape =
getSemanticShape(CGM.getContext(), getSemanticLeafType(Decl));
uint32_t SigId = Signature.size();
Signature.push_back(
createSemanticSignatureElement(CGM, SigId, Semantic, Index, Shape));
llvm::Type *RowTy = CGM.getTypes().ConvertTypeForMem(Shape.RowType);
llvm::Function *IntrFn = llvm::Intrinsic::getOrInsertDeclaration(
B.getModule(), llvm::Intrinsic::dx_load_input, {RowTy});
SmallVector<OperandBundleDef, 1> OB;
if (auto *Token = getConvergenceToken(*B.GetInsertBlock())) {
llvm::Value *bundleArgs[] = {Token};
OB.emplace_back("convergencectrl", bundleArgs);
}
bool IsMatrixValue = Shape.IsMatrix && Type->isVectorTy();
llvm::Type *LeafTy = Shape.IsMatrix && !IsMatrixValue
? RowTy
: CGM.getTypes().ConvertType(Shape.RowType);
llvm::Value *Result = llvm::PoisonValue::get(Type);
const unsigned NumRows = Shape.getNumRows();
for (unsigned Row = 0; Row < NumRows; ++Row) {
std::array<Value *, 4> Args{
/*SigElementId=*/B.getInt32(SigId),
/*RowIndex=*/B.getInt32(Row),
/*ColIndex=*/B.getInt8(0),
/*GsVertexOrPrimIndex=*/llvm::PoisonValue::get(B.getInt32Ty())};
llvm::Value *Value =
B.CreateCall(IntrFn, Args, OB, Twine(Name).concat(Twine(Row)));
// Booleans use their memory representation in DXIL signatures, but
// function parameters use their value representation.
if (Value->getType() != LeafTy) {
assert(Shape.RowType->hasBooleanRepresentation() &&
"unexpected semantic load type mismatch");
Value = B.CreateICmpNE(
Value, llvm::Constant::getNullValue(Value->getType()), "loadedv");
}
if (IsMatrixValue) {
for (unsigned Col = 0; Col < Shape.Cols; ++Col)
Result =
B.CreateInsertElement(Result, B.CreateExtractElement(Value, Col),
Shape.getMatrixElementIndex(Row, Col));
} else {
SmallVector<unsigned> Indices = Shape.getArrayIndicesForRow(Row);
Result =
Indices.empty() ? Value : B.CreateInsertValue(Result, Value, Indices);
}
}
return Result;
}
void CGHLSLRuntime::emitDXILUserSemanticStore(llvm::IRBuilder<> &B,
llvm::Value *Source,
const clang::DeclaratorDecl *Decl,
HLSLAppliedSemanticAttr *Semantic,
std::optional<unsigned> Index,
SemanticSignatures &Signature) {
SemanticShape Shape =
getSemanticShape(CGM.getContext(), getSemanticLeafType(Decl));
uint32_t SigId = Signature.size();
Signature.push_back(
createSemanticSignatureElement(CGM, SigId, Semantic, Index, Shape));
llvm::Type *RowTy = CGM.getTypes().ConvertTypeForMem(Shape.RowType);
llvm::Function *IntrFn = llvm::Intrinsic::getOrInsertDeclaration(
B.getModule(), llvm::Intrinsic::dx_store_output, {RowTy});
SmallVector<OperandBundleDef, 1> OB;
if (auto *Token = getConvergenceToken(*B.GetInsertBlock())) {
llvm::Value *bundleArgs[] = {Token};
OB.emplace_back("convergencectrl", bundleArgs);
}
bool IsMatrixValue = Shape.IsMatrix && Source->getType()->isVectorTy();
const unsigned NumRows = Shape.getNumRows();
for (unsigned Row = 0; Row < NumRows; ++Row) {
llvm::Value *Val;
if (IsMatrixValue) {
SmallVector<int, 4> Mask;
for (unsigned Col = 0; Col < Shape.Cols; ++Col)
Mask.push_back(Shape.getMatrixElementIndex(Row, Col));
Val = B.CreateShuffleVector(Source, Mask);
} else {
SmallVector<unsigned> Indices = Shape.getArrayIndicesForRow(Row);
Val = Indices.empty() ? Source : B.CreateExtractValue(Source, Indices);
}
// Booleans use their memory representation in DXIL signatures, but direct
// function results use their value representation.
if (Val->getType() != RowTy) {
assert(Shape.RowType->hasBooleanRepresentation() &&
"unexpected semantic store type mismatch");
Val = B.CreateZExt(Val, RowTy, "storedv");
}
std::array<Value *, 4> Args{/*SigElementId=*/B.getInt32(SigId),
/*RowIndex=*/B.getInt32(Row),
/*ColIndex=*/B.getInt8(0), /*Value=*/Val};
B.CreateCall(IntrFn, Args, OB);
}
}
llvm::Value *CGHLSLRuntime::emitUserSemanticLoad(
IRBuilder<> &B, llvm::Type *Type, const clang::DeclaratorDecl *Decl,
HLSLAppliedSemanticAttr *Semantic, std::optional<unsigned> Index,
SemanticSignatures &Signature, llvm::hlsl::InterpolationMode InterpMode) {
if (CGM.getTarget().getTriple().isSPIRV())
return emitSPIRVUserSemanticLoad(B, Type, Decl, Semantic, Index,
InterpMode);
if (CGM.getTarget().getTriple().isDXIL())
return emitDXILUserSemanticLoad(B, Type, Decl, Semantic, Index, Signature);
llvm_unreachable("Unsupported target for user-semantic load.");
}
void CGHLSLRuntime::emitUserSemanticStore(
IRBuilder<> &B, llvm::Value *Source, const clang::DeclaratorDecl *Decl,
HLSLAppliedSemanticAttr *Semantic, std::optional<unsigned> Index,
SemanticSignatures &Signature, llvm::hlsl::InterpolationMode InterpMode) {
if (CGM.getTarget().getTriple().isSPIRV())
return emitSPIRVUserSemanticStore(B, Source, Decl, Semantic, Index,
InterpMode);
if (CGM.getTarget().getTriple().isDXIL())
return emitDXILUserSemanticStore(B, Source, Decl, Semantic, Index,
Signature);
llvm_unreachable("Unsupported target for user-semantic load.");
}
llvm::Value *CGHLSLRuntime::emitSystemSemanticLoad(
IRBuilder<> &B, llvm::Type *Type, const clang::DeclaratorDecl *Decl,
HLSLAppliedSemanticAttr *Semantic,
llvm::dxbc::PSV::SemanticKind SemanticKind,
llvm::Triple::EnvironmentType Stage, std::optional<unsigned> Index,
SemanticSignatures &Signature, llvm::hlsl::InterpolationMode InterpMode) {
switch (SemanticKind) {
case llvm::dxbc::PSV::SemanticKind::Barycentrics:
assert(Stage == llvm::Triple::Pixel &&
"SV_Barycentrics is in an unavailable stage and should have been "
"diagnosed by Sema");
if (CGM.getTarget().getTriple().isSPIRV()) {
using llvm::hlsl::InterpolationMode;
// Perspective is encoded by the built-in; only sampling decorations
// belong on the interface variable.
bool NoPerspective = false;
switch (InterpMode) {
case InterpolationMode::LinearNoperspective:
NoPerspective = true;
InterpMode = InterpolationMode::Linear;
break;
case InterpolationMode::LinearNoperspectiveCentroid:
NoPerspective = true;
InterpMode = InterpolationMode::LinearCentroid;
break;
case InterpolationMode::LinearNoperspectiveSample:
NoPerspective = true;
InterpMode = InterpolationMode::LinearSample;
break;
default:
break;
}
// SPIR-V barycentric built-ins are float3, even for half3 inputs.
llvm::Type *BuiltinType = llvm::FixedVectorType::get(CGM.FloatTy, 3);
llvm::Value *Barycentrics = createSPIRVInputLoad(
B, CGM.getModule(), BuiltinType, Semantic->getSemanticName(),
llvm::hlsl::SPIRVInterfaceDecoration::BuiltIn,
NoPerspective ? /* BuiltIn::BaryCoordNoPerspKHR */ 5287
: /* BuiltIn::BaryCoordKHR */ 5286,
InterpMode);
if (Type->getScalarType()->isHalfTy())
return B.CreateFPTrunc(Barycentrics, Type);
return Barycentrics;
}
if (CGM.getTarget().getTriple().isDXIL())
return emitDXILUserSemanticLoad(B, Type, Decl, Semantic, Index,
Signature);
break;
case llvm::dxbc::PSV::SemanticKind::GroupIndex: {
assert(llvm::is_contained({llvm::Triple::Compute, llvm::Triple::Mesh,
llvm::Triple::Amplification},
Stage) &&
"SV_GroupIndex is in an unavailable stage and should have been "
"diagnosed by Sema");
assert(Stage != llvm::Triple::Mesh &&
Stage != llvm::Triple::Amplification &&
"FIXME: SV_GroupIndex is not yet implemented for this shader "
"stage");
llvm::Function *GroupIndex =
CGM.getIntrinsic(getFlattenedThreadIdInGroupIntrinsic());
return B.CreateCall(FunctionCallee(GroupIndex));
}
case llvm::dxbc::PSV::SemanticKind::DispatchThreadID: {
assert(llvm::is_contained({llvm::Triple::Compute, llvm::Triple::Mesh,
llvm::Triple::Amplification},
Stage) &&
"SV_DispatchThreadID is in an unavailable stage and should have "
"been diagnosed by Sema");
assert(Stage != llvm::Triple::Mesh &&
Stage != llvm::Triple::Amplification &&
"FIXME: SV_DispatchThreadID is not yet implemented for this "
"shader stage");
llvm::Intrinsic::ID IntrinID = getThreadIdIntrinsic();
llvm::Function *ThreadIDIntrinsic =
llvm::Intrinsic::isOverloaded(IntrinID)
? CGM.getIntrinsic(IntrinID, {CGM.Int32Ty})
: CGM.getIntrinsic(IntrinID);
return buildVectorInput(B, ThreadIDIntrinsic, Type);
}
case llvm::dxbc::PSV::SemanticKind::GroupThreadID: {
assert(llvm::is_contained({llvm::Triple::Compute, llvm::Triple::Mesh,
llvm::Triple::Amplification},
Stage) &&
"SV_GroupThreadID is in an unavailable stage and should have been "
"diagnosed by Sema");
assert(Stage != llvm::Triple::Mesh &&
Stage != llvm::Triple::Amplification &&
"FIXME: SV_GroupThreadID is not yet implemented for this shader "
"stage");
llvm::Intrinsic::ID IntrinID = getGroupThreadIdIntrinsic();
llvm::Function *GroupThreadIDIntrinsic =
llvm::Intrinsic::isOverloaded(IntrinID)
? CGM.getIntrinsic(IntrinID, {CGM.Int32Ty})
: CGM.getIntrinsic(IntrinID);
return buildVectorInput(B, GroupThreadIDIntrinsic, Type);
}
case llvm::dxbc::PSV::SemanticKind::GroupID: {
assert(llvm::is_contained({llvm::Triple::Compute, llvm::Triple::Mesh,
llvm::Triple::Amplification},
Stage) &&
"SV_GroupID is in an unavailable stage and should have been "
"diagnosed by Sema");
assert(Stage != llvm::Triple::Mesh &&
Stage != llvm::Triple::Amplification &&
"FIXME: SV_GroupID is not yet implemented for this shader stage");
llvm::Intrinsic::ID IntrinID = getGroupIdIntrinsic();
llvm::Function *GroupIDIntrinsic =
llvm::Intrinsic::isOverloaded(IntrinID)
? CGM.getIntrinsic(IntrinID, {CGM.Int32Ty})
: CGM.getIntrinsic(IntrinID);
return buildVectorInput(B, GroupIDIntrinsic, Type);
}
case llvm::dxbc::PSV::SemanticKind::Position:
assert(llvm::is_contained({llvm::Triple::Hull, llvm::Triple::Domain,
llvm::Triple::Geometry, llvm::Triple::Pixel},
Stage) &&
"SV_Position is in an unavailable stage and should have been "
"diagnosed by Sema");
assert(Stage != llvm::Triple::Hull && Stage != llvm::Triple::Domain &&
Stage != llvm::Triple::Geometry &&
"FIXME: loading SV_Position is not yet implemented for this "
"shader stage");
if (CGM.getTarget().getTriple().isSPIRV())
return createSPIRVInputLoad(B, CGM.getModule(), Type,
Semantic->getAttrName()->getName(),
llvm::hlsl::SPIRVInterfaceDecoration::BuiltIn,
/* FragCoord */ 15, InterpMode);
if (CGM.getTarget().getTriple().isDXIL())
return emitDXILUserSemanticLoad(B, Type, Decl, Semantic, Index,
Signature);
break;
case llvm::dxbc::PSV::SemanticKind::VertexID:
assert(Stage == llvm::Triple::Vertex &&
"SV_VertexID is in an unavailable stage and should have been "
"diagnosed by Sema");
if (CGM.getTarget().getTriple().isSPIRV())
return createSPIRVInputLoad(B, CGM.getModule(), Type,
Semantic->getAttrName()->getName(),
llvm::hlsl::SPIRVInterfaceDecoration::BuiltIn,
/* VertexIndex */ 42, InterpMode);
if (CGM.getTarget().getTriple().isDXIL())
return emitDXILUserSemanticLoad(B, Type, Decl, Semantic, Index,
Signature);
break;
case llvm::dxbc::PSV::SemanticKind::InstanceID:
assert(Stage == llvm::Triple::Vertex &&
"SV_InstanceID is in an unavailable stage and should have been "
"diagnosed by Sema");
if (CGM.getTarget().getTriple().isSPIRV())
return createSPIRVInputLoad(B, CGM.getModule(), Type,
Semantic->getAttrName()->getName(),
llvm::hlsl::SPIRVInterfaceDecoration::BuiltIn,
/* InstanceIndex */ 43, InterpMode);
if (CGM.getTarget().getTriple().isDXIL())
return emitDXILUserSemanticLoad(B, Type, Decl, Semantic, Index,
Signature);
break;
case llvm::dxbc::PSV::SemanticKind::ShadingRate:
assert(llvm::is_contained({llvm::Triple::Hull, llvm::Triple::Domain,
llvm::Triple::Geometry, llvm::Triple::Pixel},
Stage) &&
"SV_ShadingRate is in an unavailable stage and should have been "
"diagnosed by Sema");
assert(Stage != llvm::Triple::Hull && Stage != llvm::Triple::Domain &&
Stage != llvm::Triple::Geometry &&
"FIXME: loading SV_ShadingRate is not yet implemented for this "
"shader stage");
if (CGM.getTarget().getTriple().isSPIRV())
return createSPIRVInputLoad(B, CGM.getModule(), Type,
Semantic->getAttrName()->getName(),
llvm::hlsl::SPIRVInterfaceDecoration::BuiltIn,
/* ShadingRateKHR */ 4444, InterpMode);
if (CGM.getTarget().getTriple().isDXIL())
return emitDXILUserSemanticLoad(B, Type, Decl, Semantic, Index,
Signature);
break;
default:
break;
}
llvm_unreachable(
"Load hasn't been implemented yet for this system semantic. FIXME");
}
void CGHLSLRuntime::emitSystemSemanticStore(
IRBuilder<> &B, llvm::Value *Source, const clang::DeclaratorDecl *Decl,
HLSLAppliedSemanticAttr *Semantic,
llvm::dxbc::PSV::SemanticKind SemanticKind,
llvm::Triple::EnvironmentType Stage, std::optional<unsigned> Index,
SemanticSignatures &Signature, llvm::hlsl::InterpolationMode InterpMode) {
switch (SemanticKind) {
case llvm::dxbc::PSV::SemanticKind::Position:
assert(llvm::is_contained({llvm::Triple::Vertex, llvm::Triple::Hull,
llvm::Triple::Domain, llvm::Triple::Geometry,
llvm::Triple::Mesh},
Stage) &&
"SV_Position is in an unavailable stage and should have been "
"diagnosed by Sema");
assert(Stage != llvm::Triple::Hull && Stage != llvm::Triple::Domain &&
Stage != llvm::Triple::Geometry && Stage != llvm::Triple::Mesh &&
"FIXME: storing SV_Position is not yet implemented for this "
"shader stage");
if (CGM.getTarget().getTriple().isDXIL()) {
emitDXILUserSemanticStore(B, Source, Decl, Semantic, Index, Signature);
return;
}
if (CGM.getTarget().getTriple().isSPIRV()) {
createSPIRVOutputStore(B, CGM.getModule(), Source,
Semantic->getAttrName()->getName(),
llvm::hlsl::SPIRVInterfaceDecoration::BuiltIn,
/* Position */ 0, InterpMode);
return;
}
break;
case llvm::dxbc::PSV::SemanticKind::Target:
assert(Stage == llvm::Triple::Pixel &&
"SV_Target is in an unavailable stage and should have been "
"diagnosed by Sema");
emitUserSemanticStore(B, Source, Decl, Semantic, Index, Signature,
InterpMode);
return;
case llvm::dxbc::PSV::SemanticKind::ShadingRate:
assert(llvm::is_contained({llvm::Triple::Vertex, llvm::Triple::Hull,
llvm::Triple::Domain, llvm::Triple::Geometry,
llvm::Triple::Mesh},
Stage) &&
"SV_ShadingRate is in an unavailable stage and should have been "
"diagnosed by Sema");
assert(Stage != llvm::Triple::Hull && Stage != llvm::Triple::Domain &&
Stage != llvm::Triple::Geometry && Stage != llvm::Triple::Mesh &&
"FIXME: storing SV_ShadingRate is not yet implemented for this "
"shader stage");
if (CGM.getTarget().getTriple().isDXIL()) {
emitDXILUserSemanticStore(B, Source, Decl, Semantic, Index, Signature);
return;
}
if (CGM.getTarget().getTriple().isSPIRV()) {
// Matches DXC: the PrimitiveShadingRateKHR built-in output does not
// carry the Flat decoration implied for integer vertex outputs.
createSPIRVOutputStore(B, CGM.getModule(), Source,
Semantic->getAttrName()->getName(),
llvm::hlsl::SPIRVInterfaceDecoration::BuiltIn,
/* PrimitiveShadingRateKHR */ 4432,
llvm::hlsl::InterpolationMode::Undefined);
return;
}
break;
default:
break;
}
llvm_unreachable(
"Store hasn't been implemented yet for this system semantic. FIXME");
}
llvm::Value *CGHLSLRuntime::handleScalarSemanticLoad(
IRBuilder<> &B, const FunctionDecl *FD, llvm::Type *Type,
const clang::DeclaratorDecl *Decl, HLSLAppliedSemanticAttr *Semantic,
SemanticSignatures &Signature, llvm::hlsl::InterpolationMode InterpMode) {
std::optional<unsigned> Index = Semantic->getSemanticIndex();
llvm::dxbc::PSV::SemanticKind SemanticKind =
llvm::hlsl::getSemanticKind(Semantic->getAttrName()->getName());
const auto *ShaderAttr = FD->getAttr<HLSLShaderAttr>();
assert(ShaderAttr && "Entry point has no shader attribute");
llvm::hlsl::SemanticInterpretation Interpretation =
llvm::hlsl::getInterpretationKind(SemanticKind, ShaderAttr->getType(),
llvm::hlsl::IOType::In);
assert(Interpretation != llvm::hlsl::SemanticInterpretation::Invalid &&
"invalid semantic should have been diagnosed by Sema");
if (CGM.getTarget().getTriple().isSPIRV())
InterpMode = getSPIRVInterpolationMode(
InterpMode, Type, ShaderAttr->getType(), llvm::hlsl::IOType::In);
if (Interpretation == llvm::hlsl::SemanticInterpretation::Arbitrary)
return emitUserSemanticLoad(B, Type, Decl, Semantic, Index, Signature,
InterpMode);
return emitSystemSemanticLoad(B, Type, Decl, Semantic, SemanticKind,
ShaderAttr->getType(), Index, Signature,
InterpMode);
}
void CGHLSLRuntime::handleScalarSemanticStore(
IRBuilder<> &B, const FunctionDecl *FD, llvm::Value *Source,
const clang::DeclaratorDecl *Decl, HLSLAppliedSemanticAttr *Semantic,
SemanticSignatures &Signature, llvm::hlsl::InterpolationMode InterpMode) {
std::optional<unsigned> Index = Semantic->getSemanticIndex();
llvm::dxbc::PSV::SemanticKind SemanticKind =
llvm::hlsl::getSemanticKind(Semantic->getAttrName()->getName());
const auto *ShaderAttr = FD->getAttr<HLSLShaderAttr>();
assert(ShaderAttr && "Entry point has no shader attribute");
llvm::hlsl::SemanticInterpretation Interpretation =
llvm::hlsl::getInterpretationKind(SemanticKind, ShaderAttr->getType(),
llvm::hlsl::IOType::Out);
assert(Interpretation != llvm::hlsl::SemanticInterpretation::Invalid &&
"invalid semantic should have been diagnosed by Sema");
if (CGM.getTarget().getTriple().isSPIRV())
InterpMode = getSPIRVInterpolationMode(InterpMode, Source->getType(),
ShaderAttr->getType(),
llvm::hlsl::IOType::Out);
if (Interpretation == llvm::hlsl::SemanticInterpretation::Arbitrary)
return emitUserSemanticStore(B, Source, Decl, Semantic, Index, Signature,
InterpMode);
emitSystemSemanticStore(B, Source, Decl, Semantic, SemanticKind,
ShaderAttr->getType(), Index, Signature, InterpMode);
}
std::pair<llvm::Value *, specific_attr_iterator<HLSLAppliedSemanticAttr>>
CGHLSLRuntime::handleStructSemanticLoad(
IRBuilder<> &B, const FunctionDecl *FD, llvm::Type *Type,
const clang::DeclaratorDecl *Decl,
specific_attr_iterator<HLSLAppliedSemanticAttr> AttrBegin,
specific_attr_iterator<HLSLAppliedSemanticAttr> AttrEnd,
SemanticSignatures &Signature,
llvm::hlsl::InterpolationModifier Modifiers) {
const llvm::StructType *ST = cast<StructType>(Type);
const clang::RecordDecl *RD = Decl->getType()->getAsRecordDecl();
assert(RD->getNumFields() == ST->getNumElements());
llvm::Value *Aggregate = llvm::PoisonValue::get(Type);
auto FieldDecl = RD->field_begin();
for (unsigned I = 0; I < ST->getNumElements(); ++I) {
auto [ChildValue, NextAttr] =
handleSemanticLoad(B, FD, ST->getElementType(I), *FieldDecl, AttrBegin,
AttrEnd, Signature, Modifiers);
AttrBegin = NextAttr;
assert(ChildValue);
Aggregate = B.CreateInsertValue(Aggregate, ChildValue, I);
++FieldDecl;
}
return std::make_pair(Aggregate, AttrBegin);
}
specific_attr_iterator<HLSLAppliedSemanticAttr>
CGHLSLRuntime::handleStructSemanticStore(
IRBuilder<> &B, const FunctionDecl *FD, llvm::Value *Source,
const clang::DeclaratorDecl *Decl,
specific_attr_iterator<HLSLAppliedSemanticAttr> AttrBegin,
specific_attr_iterator<HLSLAppliedSemanticAttr> AttrEnd,
SemanticSignatures &Signature,
llvm::hlsl::InterpolationModifier Modifiers) {
const llvm::StructType *ST = cast<StructType>(Source->getType());
const clang::RecordDecl *RD = nullptr;
if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(Decl))
RD = FD->getDeclaredReturnType()->getAsRecordDecl();
else
RD = Decl->getType()->getAsRecordDecl();
assert(RD);
assert(RD->getNumFields() == ST->getNumElements());
auto FieldDecl = RD->field_begin();
for (unsigned I = 0; I < ST->getNumElements(); ++I, ++FieldDecl) {
llvm::Value *Extract = B.CreateExtractValue(Source, I);
AttrBegin = handleSemanticStore(B, FD, Extract, *FieldDecl, AttrBegin,
AttrEnd, Signature, Modifiers);
}
return AttrBegin;
}
std::pair<llvm::Value *, specific_attr_iterator<HLSLAppliedSemanticAttr>>
CGHLSLRuntime::handleSemanticLoad(
IRBuilder<> &B, const FunctionDecl *FD, llvm::Type *Type,
const clang::DeclaratorDecl *Decl,
specific_attr_iterator<HLSLAppliedSemanticAttr> AttrBegin,
specific_attr_iterator<HLSLAppliedSemanticAttr> AttrEnd,
SemanticSignatures &Signature,
llvm::hlsl::InterpolationModifier Modifiers) {
assert(AttrBegin != AttrEnd);
// Pass the enclosing declaration's mask down the traversal, replacing (not
// merging) it when an inner field has its own interpolation modifiers.
if (const auto *A = Decl->getAttr<HLSLInterpolationModifierAttr>())
Modifiers =
static_cast<llvm::hlsl::InterpolationModifier>(A->getModifiers());
if (Type->isStructTy())
return handleStructSemanticLoad(B, FD, Type, Decl, AttrBegin, AttrEnd,
Signature, Modifiers);
HLSLAppliedSemanticAttr *Attr = *AttrBegin;
++AttrBegin;
size_t PreviousSize = Signature.size();
auto InterpMode = llvm::hlsl::getInterpolationMode(Modifiers);
llvm::Value *Value =
handleScalarSemanticLoad(B, FD, Type, Decl, Attr, Signature, InterpMode);
// Intrinsic-only system values and SPIR-V loads do not add DXIL signature
// elements. Non-pixel inputs retain Undefined.
if (Signature.size() != PreviousSize) {
auto &Element = Signature.back();
Element.InterpMode = llvm::hlsl::normalizeInterpolationMode(
InterpMode, Element.CompType, Element.SemanticKind,
FD->getAttr<HLSLShaderAttr>()->getType(), llvm::hlsl::IOType::In);
}
return std::make_pair(Value, AttrBegin);
}
specific_attr_iterator<HLSLAppliedSemanticAttr>
CGHLSLRuntime::handleSemanticStore(
IRBuilder<> &B, const FunctionDecl *FD, llvm::Value *Source,
const clang::DeclaratorDecl *Decl,
specific_attr_iterator<HLSLAppliedSemanticAttr> AttrBegin,
specific_attr_iterator<HLSLAppliedSemanticAttr> AttrEnd,
SemanticSignatures &Signature,
llvm::hlsl::InterpolationModifier Modifiers) {
assert(AttrBegin != AttrEnd);
// An inner field overrides the enclosing return declaration's modifiers.
if (const auto *A = Decl->getAttr<HLSLInterpolationModifierAttr>())
Modifiers =
static_cast<llvm::hlsl::InterpolationModifier>(A->getModifiers());
if (Source->getType()->isStructTy())
return handleStructSemanticStore(B, FD, Source, Decl, AttrBegin, AttrEnd,
Signature, Modifiers);
HLSLAppliedSemanticAttr *Attr = *AttrBegin;
++AttrBegin;
size_t PreviousSize = Signature.size();
auto InterpMode = llvm::hlsl::getInterpolationMode(Modifiers);
handleScalarSemanticStore(B, FD, Source, Decl, Attr, Signature, InterpMode);
if (Signature.size() != PreviousSize) {
auto &Element = Signature.back();
Element.InterpMode = llvm::hlsl::normalizeInterpolationMode(
InterpMode, Element.CompType, Element.SemanticKind,
FD->getAttr<HLSLShaderAttr>()->getType(), llvm::hlsl::IOType::Out);
}
return AttrBegin;
}
void CGHLSLRuntime::emitEntryFunction(const FunctionDecl *FD,
llvm::Function *Fn) {
SmallVector<llvm::hlsl::SemanticSignatureElement> InputSignature;
SmallVector<llvm::hlsl::SemanticSignatureElement> OutputSignature;
llvm::Module &M = CGM.getModule();
llvm::LLVMContext &Ctx = M.getContext();
auto *EntryTy = llvm::FunctionType::get(llvm::Type::getVoidTy(Ctx), false);
Function *EntryFn =
Function::Create(EntryTy, Function::ExternalLinkage, FD->getName(), &M);
// Copy function attributes over, we have no argument or return attributes
// that can be valid on the real entry.
AttributeList NewAttrs = AttributeList::get(Ctx, AttributeList::FunctionIndex,
Fn->getAttributes().getFnAttrs());
EntryFn->setAttributes(NewAttrs);
setHLSLEntryAttributes(FD, EntryFn);
// Set the called function as internal linkage.
Fn->setLinkage(GlobalValue::InternalLinkage);
BasicBlock *BB = BasicBlock::Create(Ctx, "entry", EntryFn);
IRBuilder<> B(BB);
llvm::SmallVector<Value *> Args;
SmallVector<OperandBundleDef, 1> OB;
if (CGM.shouldEmitConvergenceTokens()) {
assert(EntryFn->isConvergent());
llvm::Value *I =
B.CreateIntrinsic(llvm::Intrinsic::experimental_convergence_entry, {});
llvm::Value *bundleArgs[] = {I};
OB.emplace_back("convergencectrl", bundleArgs);
}
SmallVector<std::pair<llvm::Value *, llvm::Type *>> OutputSemantic;
unsigned SRetOffset = 0;
for (const auto &Param : Fn->args()) {
if (Param.hasStructRetAttr()) {
SRetOffset = 1;
llvm::Type *VarType = Param.getParamStructRetType();
llvm::Value *Var =
CGM.getLangOpts().EmitLogicalPointer
? cast<Instruction>(B.CreateStructuredAlloca(VarType))
: cast<Instruction>(B.CreateAlloca(VarType));
OutputSemantic.push_back(std::make_pair(Var, VarType));
Args.push_back(Var);
continue;
}
const ParmVarDecl *PD = FD->getParamDecl(Param.getArgNo() - SRetOffset);
llvm::Value *SemanticValue = nullptr;
// FIXME: support inout/out parameters for semantics.
if ([[maybe_unused]] HLSLParamModifierAttr *MA =
PD->getAttr<HLSLParamModifierAttr>()) {
llvm_unreachable("Not handled yet");
} else {
llvm::Type *ParamType = nullptr;
if (Param.hasByValAttr())
ParamType = Param.getParamByValType();
else if (PD->getType()->isRecordType())
ParamType = CGM.getTypes().ConvertType(PD->getType());
else
ParamType = Param.getType();
auto AttrBegin = PD->specific_attr_begin<HLSLAppliedSemanticAttr>();
auto AttrEnd = PD->specific_attr_end<HLSLAppliedSemanticAttr>();
auto Result = handleSemanticLoad(B, FD, ParamType, PD, AttrBegin, AttrEnd,
InputSignature);
SemanticValue = Result.first;
if (!SemanticValue)
return;
if (Param.hasByValAttr() || PD->getType()->isRecordType()) {
llvm::Value *Var =
CGM.getLangOpts().EmitLogicalPointer
? cast<Instruction>(B.CreateStructuredAlloca(ParamType))
: cast<Instruction>(B.CreateAlloca(ParamType));
B.CreateStore(SemanticValue, Var);
SemanticValue = Var;
}
}
assert(SemanticValue);
Args.push_back(SemanticValue);
}
CallInst *CI = B.CreateCall(FunctionCallee(Fn), Args, OB);
CI->setCallingConv(Fn->getCallingConv());
if (Fn->getReturnType() != CGM.VoidTy)
// Element type is unused, so set to dummy value (NULL).
OutputSemantic.push_back(std::make_pair(CI, nullptr));
for (auto &SourcePair : OutputSemantic) {
llvm::Value *Source = SourcePair.first;
llvm::Type *ElementType = SourcePair.second;
AllocaInst *AI = dyn_cast<AllocaInst>(Source);
llvm::Value *SourceValue = AI ? B.CreateLoad(ElementType, Source) : Source;
auto AttrBegin = FD->specific_attr_begin<HLSLAppliedSemanticAttr>();
auto AttrEnd = FD->specific_attr_end<HLSLAppliedSemanticAttr>();
handleSemanticStore(B, FD, SourceValue, FD, AttrBegin, AttrEnd,
OutputSignature);
}
B.CreateRetVoid();
// Add and identify root signature to function, if applicable
for (const Attr *Attr : FD->getAttrs()) {
if (const auto *RSAttr = dyn_cast<RootSignatureAttr>(Attr)) {
auto *RSDecl = RSAttr->getSignatureDecl();
addRootSignatureMD(RSDecl->getVersion(), RSDecl->getRootElements(),
EntryFn, M);
}
}
if (CGM.getTarget().getTriple().isDXIL()) {
// Use the entry's shader stage, not the target environment, which may be
// 'library' when compiling multiple entry points.
auto Stage = FD->getAttr<HLSLShaderAttr>()->getType();
using PackingMode = CodeGenOptions::HLSLSemanticSignaturePackingMode;
auto Mode = CGM.getCodeGenOpts().getHLSLSemanticSignaturePacking();
for (auto IOTy : {llvm::hlsl::IOType::In, llvm::hlsl::IOType::Out}) {
bool IsOutput = IOTy == llvm::hlsl::IOType::Out;
auto &Signature = IsOutput ? OutputSignature : InputSignature;
auto Packed = packSemanticSignature(Signature, Stage, IOTy, Mode,
CGM.getLangOpts().NativeHalfType);
if (!Packed) {
CGM.getDiags().Report(FD->getLocation(),
diag::err_hlsl_signature_packing)
<< IsOutput << llvm::toString(Packed.takeError());
return;
}
// Library entry points retain stage-specific packing without warnings.
if (Mode != PackingMode::Default &&
CGM.getTarget().getTriple().getEnvironment() !=
llvm::Triple::Library &&
((Stage == llvm::Triple::Vertex && !IsOutput) ||
(Stage == llvm::Triple::Pixel && IsOutput)))
CGM.getDiags().Report(FD->getLocation(),
diag::warn_hlsl_signature_packing_ignored)
<< (Mode == PackingMode::Optimized ? "optimized" : "prefix-stable")
<< IsOutput;
}
}
addSemanticSignatureMD(InputSignature, OutputSignature, EntryFn, M);
}
static void gatherFunctions(SmallVectorImpl<Function *> &Fns, llvm::Module &M,
bool CtorOrDtor) {
const auto *GV =
M.getNamedGlobal(CtorOrDtor ? "llvm.global_ctors" : "llvm.global_dtors");
if (!GV)
return;
const auto *CA = dyn_cast<ConstantArray>(GV->getInitializer());
if (!CA)
return;
// The global_ctor array elements are a struct [Priority, Fn *, COMDat].
// HLSL neither supports priorities or COMDat values, so we will check those
// in an assert but not handle them.
for (const auto &Ctor : CA->operands()) {
if (isa<ConstantAggregateZero>(Ctor))
continue;
ConstantStruct *CS = cast<ConstantStruct>(Ctor);
assert(cast<ConstantInt>(CS->getOperand(0))->getValue() == 65535 &&
"HLSL doesn't support setting priority for global ctors.");
assert(isa<ConstantPointerNull>(CS->getOperand(2)) &&
"HLSL doesn't support COMDat for global ctors.");
Fns.push_back(cast<Function>(CS->getOperand(1)));
}
}
void CGHLSLRuntime::generateGlobalCtorDtorCalls() {
llvm::Module &M = CGM.getModule();
SmallVector<Function *> CtorFns;
SmallVector<Function *> DtorFns;
gatherFunctions(CtorFns, M, true);
gatherFunctions(DtorFns, M, false);
// Insert a call to the global constructor at the beginning of the entry block
// to externally exported functions. This is a bit of a hack, but HLSL allows
// global constructors, but doesn't support driver initialization of globals.
for (auto &F : M.functions()) {
if (!F.hasFnAttribute("hlsl.shader"))
continue;
auto *Token = getConvergenceToken(F.getEntryBlock());
Instruction *IP = &*F.getEntryBlock().begin();
SmallVector<OperandBundleDef, 1> OB;
if (Token) {
llvm::Value *bundleArgs[] = {Token};
OB.emplace_back("convergencectrl", bundleArgs);
IP = Token->getNextNode();
}
IRBuilder<> B(IP);
for (auto *Fn : CtorFns) {
auto CI = B.CreateCall(FunctionCallee(Fn), {}, OB);
CI->setCallingConv(Fn->getCallingConv());
}
// Insert global dtors before the terminator of the last instruction
B.SetInsertPoint(F.back().getTerminator());
for (auto *Fn : DtorFns) {
auto CI = B.CreateCall(FunctionCallee(Fn), {}, OB);
CI->setCallingConv(Fn->getCallingConv());
}
}
// No need to keep global ctors/dtors for non-lib profile after call to
// ctors/dtors added for entry.
Triple T(M.getTargetTriple());
if (T.getEnvironment() != Triple::EnvironmentType::Library) {
if (auto *GV = M.getNamedGlobal("llvm.global_ctors"))
GV->eraseFromParent();
if (auto *GV = M.getNamedGlobal("llvm.global_dtors"))
GV->eraseFromParent();
}
}
static void initializeBuffer(CodeGenModule &CGM, llvm::GlobalVariable *GV,
Intrinsic::ID IntrID,
ArrayRef<llvm::Value *> Args) {
LLVMContext &Ctx = CGM.getLLVMContext();
llvm::Function *InitResFunc =
llvm::Function::Create(llvm::FunctionType::get(CGM.VoidTy, false),
llvm::GlobalValue::InternalLinkage,
"_init_buffer_" + GV->getName(), CGM.getModule());
InitResFunc->addFnAttr(llvm::Attribute::AlwaysInline);
llvm::BasicBlock *EntryBB =
llvm::BasicBlock::Create(Ctx, "entry", InitResFunc);
CGBuilderTy Builder(CGM, Ctx);
const DataLayout &DL = CGM.getModule().getDataLayout();
Builder.SetInsertPoint(EntryBB);
// Make sure the global variable is buffer resource handle
llvm::Type *HandleTy = GV->getValueType();
assert(HandleTy->isTargetExtTy() && "unexpected type of the buffer global");
llvm::Value *CreateHandle = Builder.CreateIntrinsic(
/*ReturnType=*/HandleTy, IntrID, Args, nullptr,
Twine(GV->getName()).concat("_h"));
Builder.CreateAlignedStore(CreateHandle, GV, GV->getPointerAlignment(DL));
Builder.CreateRetVoid();
CGM.AddCXXGlobalInit(InitResFunc);
}
void CGHLSLRuntime::initializeBufferFromBinding(const HLSLBufferDecl *BufDecl,
llvm::GlobalVariable *GV) {
ResourceBindingAttrs Binding(BufDecl);
assert(Binding.hasBinding() &&
"cbuffer/tbuffer should always have resource binding attribute");
auto *Index = llvm::ConstantInt::get(CGM.IntTy, 0);
auto *RangeSize = llvm::ConstantInt::get(CGM.IntTy, 1);
auto *Space = llvm::ConstantInt::get(CGM.IntTy, Binding.getSpace());
Value *Name = buildNameForResource(BufDecl->getName(), CGM);
// buffer with explicit binding
if (Binding.isExplicit()) {
llvm::Intrinsic::ID IntrinsicID =
CGM.getHLSLRuntime().getCreateHandleFromBindingIntrinsic();
auto *RegSlot = llvm::ConstantInt::get(CGM.IntTy, Binding.getSlot());
SmallVector<Value *> Args{Space, RegSlot, RangeSize, Index, Name};
initializeBuffer(CGM, GV, IntrinsicID, Args);
} else {
// buffer with implicit binding
llvm::Intrinsic::ID IntrinsicID =
CGM.getHLSLRuntime().getCreateHandleFromImplicitBindingIntrinsic();
auto *OrderID =
llvm::ConstantInt::get(CGM.IntTy, Binding.getImplicitOrderID());
SmallVector<Value *> Args{OrderID, Space, RangeSize, Index, Name};
initializeBuffer(CGM, GV, IntrinsicID, Args);
}
}
void CGHLSLRuntime::handleGlobalVarDefinition(const VarDecl *VD,
llvm::GlobalVariable *GV) {
if (auto Attr = VD->getAttr<HLSLVkExtBuiltinInputAttr>())
addSPIRVInterfaceDecorations(
GV, llvm::hlsl::SPIRVInterfaceDecoration::BuiltIn, Attr->getBuiltIn());
if (auto Attr = VD->getAttr<HLSLVkExtBuiltinOutputAttr>())
addSPIRVInterfaceDecorations(
GV, llvm::hlsl::SPIRVInterfaceDecoration::BuiltIn, Attr->getBuiltIn());
}
llvm::Instruction *CGHLSLRuntime::getConvergenceToken(BasicBlock &BB) {
if (!CGM.shouldEmitConvergenceTokens())
return nullptr;
auto E = BB.end();
for (auto I = BB.begin(); I != E; ++I) {
auto *II = dyn_cast<llvm::IntrinsicInst>(&*I);
if (II && llvm::isConvergenceControlIntrinsic(II->getIntrinsicID())) {
return II;
}
}
llvm_unreachable("Convergence token should have been emitted.");
return nullptr;
}
class OpaqueValueVisitor : public RecursiveASTVisitor<OpaqueValueVisitor> {
public:
llvm::SmallVector<OpaqueValueExpr *, 8> OVEs;
llvm::SmallPtrSet<OpaqueValueExpr *, 8> Visited;
OpaqueValueVisitor() {}
bool VisitHLSLOutArgExpr(HLSLOutArgExpr *) {
// These need to be bound in CodeGenFunction::EmitHLSLOutArgLValues
// or CodeGenFunction::EmitHLSLOutArgExpr. If they are part of this
// traversal, the temporary containing the copy out will not have
// been created yet.
return false;
}
bool VisitOpaqueValueExpr(OpaqueValueExpr *E) {
// Traverse the source expression first.
if (E->getSourceExpr())
TraverseStmt(E->getSourceExpr());
// Then add this OVE if we haven't seen it before.
if (Visited.insert(E).second)
OVEs.push_back(E);
return true;
}
};
void CGHLSLRuntime::emitInitListOpaqueValues(CodeGenFunction &CGF,
InitListExpr *E) {
typedef CodeGenFunction::OpaqueValueMappingData OpaqueValueMappingData;
OpaqueValueVisitor Visitor;
Visitor.TraverseStmt(E);
for (auto *OVE : Visitor.OVEs) {
if (CGF.isOpaqueValueEmitted(OVE))
continue;
if (OpaqueValueMappingData::shouldBindAsLValue(OVE)) {
LValue LV = CGF.EmitLValue(OVE->getSourceExpr());
OpaqueValueMappingData::bind(CGF, OVE, LV);
} else {
RValue RV = CGF.EmitAnyExpr(OVE->getSourceExpr());
OpaqueValueMappingData::bind(CGF, OVE, RV);
}
}
}
std::optional<LValue> CGHLSLRuntime::emitResourceArraySubscriptExpr(
const ArraySubscriptExpr *ArraySubsExpr, CodeGenFunction &CGF) {
assert((ArraySubsExpr->getType()->isHLSLResourceRecord() ||
ArraySubsExpr->getType()->isHLSLResourceRecordArray()) &&
"expected resource array subscript expression");
// Let clang codegen handle local and static resource array subscripts,
// or when the subscript references on opaque expression (as part of
// ArrayInitLoopExpr AST node).
const VarDecl *ArrayDecl = dyn_cast_or_null<VarDecl>(
getArrayDecl(CGF.CGM.getContext(), ArraySubsExpr));
if (!ArrayDecl || !ArrayDecl->hasGlobalStorage() ||
ArrayDecl->getStorageClass() == SC_Static)
return std::nullopt;
// get the resource array type
ASTContext &AST = ArrayDecl->getASTContext();
const Type *ResArrayTy = ArrayDecl->getType().getTypePtr();
assert(ResArrayTy->isHLSLResourceRecordArray() &&
"expected array of resource classes");
// Iterate through all nested array subscript expressions to calculate
// the index in the flattened resource array (if this is a multi-
// dimensional array). The index is calculated as a sum of all indices
// multiplied by the total size of the array at that level.
Value *Index = nullptr;
const ArraySubscriptExpr *ASE = ArraySubsExpr;
while (ASE != nullptr) {
Value *SubIndex = CGF.EmitScalarExpr(ASE->getIdx());
if (const auto *ArrayTy =
dyn_cast<ConstantArrayType>(ASE->getType().getTypePtr())) {
Value *Multiplier = llvm::ConstantInt::get(
CGM.IntTy, AST.getConstantArrayElementCount(ArrayTy));
SubIndex = CGF.Builder.CreateMul(SubIndex, Multiplier);
}
Index = Index ? CGF.Builder.CreateAdd(Index, SubIndex) : SubIndex;
ASE = dyn_cast<ArraySubscriptExpr>(ASE->getBase()->IgnoreParenImpCasts());
}
// Find binding info for the resource array. For implicit binding
// an HLSLResourceBindingAttr should have been added by SemaHLSL.
ResourceBindingAttrs Binding(ArrayDecl);
assert(Binding.hasBinding() &&
"resource array must have a binding attribute");
// Find the individual resource type.
QualType ResultTy = ArraySubsExpr->getType();
QualType ResourceTy =
ResultTy->isArrayType() ? AST.getBaseElementType(ResultTy) : ResultTy;
// Create a temporary variable for the result, which is either going
// to be a single resource instance or a local array of resources (we need to
// return an LValue).
RawAddress TmpVar = CGF.CreateMemTempWithoutCast(ResultTy);
if (CGF.EmitLifetimeStart(TmpVar.getPointer()))
CGF.pushFullExprCleanup<CodeGenFunction::CallLifetimeEnd>(
NormalEHLifetimeMarker, TmpVar);
AggValueSlot ValueSlot = AggValueSlot::forAddr(
TmpVar, Qualifiers(), AggValueSlot::IsDestructed_t(true),
AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsAliased_t(false),
AggValueSlot::DoesNotOverlap);
// Calculate total array size (= range size).
llvm::Value *Range = llvm::ConstantInt::getSigned(
CGM.IntTy, getTotalArraySize(AST, ResArrayTy));
// If the result of the subscript operation is a single resource, call the
// constructor.
if (ResultTy == ResourceTy) {
CallArgList Args;
CXXMethodDecl *CreateMethod = lookupResourceInitMethodAndSetupArgs(
CGF.CGM, ResourceTy->getAsCXXRecordDecl(), Range, Index,
ArrayDecl->getName(), Binding, Args);
if (!CreateMethod) {
// This can happen if someone creates an array of structs that looks like
// an HLSL resource record array but it does not have the required static
// create method. No binding will be generated for it.
assert(!ResourceTy->getAsCXXRecordDecl()->isImplicit() &&
"create method lookup should always succeed for built-in resource "
"records");
return std::nullopt;
}
callResourceInitMethod(CGF, CreateMethod, Args, ValueSlot.getAddress());
} else {
// The result of the subscript operation is a local resource array which
// needs to be initialized.
const ConstantArrayType *ArrayTy =
cast<ConstantArrayType>(ResultTy.getTypePtr());
std::optional<llvm::Value *> EndIndex = initializeResourceArrayFromGlobal(
CGF, ResourceTy->getAsCXXRecordDecl(), ArrayTy, ValueSlot, Range, Index,
ArrayDecl->getName(), Binding, {llvm::ConstantInt::get(CGM.IntTy, 0)});
if (!EndIndex)
return std::nullopt;
}
return CGF.MakeAddrLValue(TmpVar, ResultTy, AlignmentSource::Decl);
}
// Initialize all resources of a global resource array into provided slot.
bool CGHLSLRuntime::initializeGlobalResourceArray(CodeGenFunction &CGF,
const VarDecl *ArrayDecl,
AggValueSlot &DestSlot) {
assert(ArrayDecl->getType()->isHLSLResourceRecordArray() &&
ArrayDecl->hasGlobalStorage() &&
ArrayDecl->getStorageClass() != SC_Static &&
"expected global non-static resource array");
// Find binding info for the resource array. For implicit binding
// the HLSLResourceBindingAttr should have been added by SemaHLSL.
ResourceBindingAttrs Binding(ArrayDecl);
assert(Binding.hasBinding() &&
"resource array must have a binding attribute");
// Find the individual resource type.
ASTContext &AST = ArrayDecl->getASTContext();
QualType ResTy = AST.getBaseElementType(ArrayDecl->getType());
const auto *ResArrayTy =
cast<ConstantArrayType>(ArrayDecl->getType().getTypePtr());
// Create Value for index and total array size (= range size).
int Size = getTotalArraySize(AST, ResArrayTy);
llvm::Value *Zero = llvm::ConstantInt::get(CGM.IntTy, 0);
llvm::Value *Range = llvm::ConstantInt::get(CGM.IntTy, Size);
// Initialize individual resources in the array into DestSlot.
std::optional<llvm::Value *> EndIndex = initializeResourceArrayFromGlobal(
CGF, ResTy->getAsCXXRecordDecl(), ResArrayTy, DestSlot, Range, Zero,
ArrayDecl->getName(), Binding, {Zero});
return EndIndex.has_value();
}
// If the expression is a global resource array, initialize all of its resources
// into Dest. Returns false if no initialization has been performed and the
// array copy should be handled by the default codegen.
bool CGHLSLRuntime::emitGlobalResourceArray(CodeGenFunction &CGF, const Expr *E,
AggValueSlot &DestSlot) {
assert(E->getType()->isHLSLResourceRecordArray() &&
"expected resource array");
// Find the array declaration for the expression. Fallback to the default
// handling if it's not a global resource array.
const VarDecl *ArrayDecl =
dyn_cast_or_null<VarDecl>(getArrayDecl(CGF.CGM.getContext(), E));
if (!ArrayDecl || !ArrayDecl->hasGlobalStorage() ||
ArrayDecl->getStorageClass() == SC_Static)
return false;
return initializeGlobalResourceArray(CGF, ArrayDecl, DestSlot);
}
// If the expression is a global resource array, create a temporary and
// initialize all of its resources, and return it as an LValue. Returns nullopt
// if no initialization has been performed and the handling should follow the
// default path.
std::optional<LValue>
CGHLSLRuntime::emitGlobalResourceArrayAsLValue(CodeGenFunction &CGF,
const VarDecl *ArrayDecl) {
assert(ArrayDecl->getType()->isHLSLResourceRecordArray() &&
"expected resource array declaration");
if (!ArrayDecl->hasGlobalStorage() ||
ArrayDecl->getStorageClass() == SC_Static)
return std::nullopt;
AggValueSlot TmpArraySlot =
CGF.CreateAggTemp(ArrayDecl->getType(), "tmpResArray");
if (initializeGlobalResourceArray(CGF, ArrayDecl, TmpArraySlot))
return CGF.MakeAddrLValue(TmpArraySlot.getAddress(), ArrayDecl->getType(),
AlignmentSource::Decl);
return std::nullopt;
}
RawAddress CGHLSLRuntime::createBufferMatrixTempAddress(const LValue &LV,
CodeGenFunction &CGF) {
assert(LV.getType()->isConstantMatrixType() && "expected matrix type");
assert(LV.getType().getAddressSpace() == LangAS::hlsl_constant &&
"expected cbuffer matrix");
QualType MatQualTy = LV.getType();
llvm::Type *LayoutTy = HLSLBufferLayoutBuilder(CGF.CGM).layOutType(MatQualTy);
Address SrcAddr = LV.getAddress();
if (LayoutTy == CGF.ConvertTypeForMem(MatQualTy))
return SrcAddr;
RawAddress DestAlloca =
CGF.CreateMemTempWithoutCast(MatQualTy, "matrix.buf.copy");
HLSLBufferCopyEmitter(CGF, DestAlloca, SrcAddr).emitCopy(MatQualTy);
return DestAlloca;
}
std::optional<LValue> CGHLSLRuntime::emitBufferArraySubscriptExpr(
const ArraySubscriptExpr *E, CodeGenFunction &CGF,
llvm::function_ref<llvm::Value *(bool Promote)> EmitIdxAfterBase) {
// Find the element type to index by first padding the element type per HLSL
// buffer rules, and then padding out to a 16-byte register boundary if
// necessary.
llvm::Type *LayoutTy =
HLSLBufferLayoutBuilder(CGF.CGM).layOutType(E->getType());
uint64_t LayoutSizeInBits =
CGM.getDataLayout().getTypeSizeInBits(LayoutTy).getFixedValue();
CharUnits ElementSize = CharUnits::fromQuantity(LayoutSizeInBits / 8);
CharUnits RowAlignedSize = ElementSize.alignTo(CharUnits::fromQuantity(16));
if (RowAlignedSize > ElementSize) {
llvm::Type *Padding = CGM.getTargetCodeGenInfo().getHLSLPadding(
CGM, RowAlignedSize - ElementSize);
assert(Padding && "No padding type for target?");
LayoutTy = llvm::StructType::get(CGF.getLLVMContext(), {LayoutTy, Padding},
/*isPacked=*/true);
}
// If the layout type doesn't introduce any padding, we don't need to do
// anything special.
llvm::Type *OrigTy = CGF.CGM.getTypes().ConvertTypeForMem(E->getType());
if (LayoutTy == OrigTy)
return std::nullopt;
LValueBaseInfo EltBaseInfo;
TBAAAccessInfo EltTBAAInfo;
// Index into the object as-if we have an array of the padded element type,
// and then dereference the element itself to avoid reading padding that may
// be past the end of the in-memory object.
SmallVector<llvm::Value *, 2> Indices;
llvm::Value *Idx = EmitIdxAfterBase(/*Promote*/ true);
Indices.push_back(Idx);
Indices.push_back(llvm::ConstantInt::get(CGF.Int32Ty, 0));
if (CGF.getLangOpts().EmitLogicalPointer) {
// The fact that we emit an array-to-pointer decay might be an oversight,
// but for now, we simply ignore it (see #179951).
const CastExpr *CE = cast<CastExpr>(E->getBase());
assert(CE->getCastKind() == CastKind::CK_ArrayToPointerDecay);
LValue LV = CGF.EmitLValue(CE->getSubExpr());
Address Addr = LV.getAddress();
LayoutTy = llvm::ArrayType::get(
LayoutTy,
cast<llvm::ArrayType>(Addr.getElementType())->getNumElements());
auto *GEP = cast<StructuredGEPInst>(CGF.Builder.CreateStructuredGEP(
LayoutTy, Addr.emitRawPointer(CGF), Indices, "cbufferidx"));
Addr =
Address(GEP, GEP->getResultElementType(), RowAlignedSize, KnownNonNull);
return CGF.MakeAddrLValue(Addr, E->getType(), EltBaseInfo, EltTBAAInfo);
}
Address Addr =
CGF.EmitPointerWithAlignment(E->getBase(), &EltBaseInfo, &EltTBAAInfo);
llvm::Value *GEP = CGF.Builder.CreateGEP(LayoutTy, Addr.emitRawPointer(CGF),
Indices, "cbufferidx");
Addr = Address(GEP, Addr.getElementType(), RowAlignedSize, KnownNonNull);
return CGF.MakeAddrLValue(Addr, E->getType(), EltBaseInfo, EltTBAAInfo);
}
std::optional<LValue>
CGHLSLRuntime::emitResourceMemberExpr(CodeGenFunction &CGF,
const MemberExpr *ME) {
assert((ME->getType()->isHLSLResourceRecord() ||
ME->getType()->isHLSLResourceRecordArray()) &&
"expected resource member expression");
const VarDecl *ResourceVD =
findAssociatedResourceDeclForStruct(CGF.CGM.getContext(), ME);
if (!ResourceVD)
return std::nullopt;
// Handle member of resource array type.
if (ResourceVD->getType()->isHLSLResourceRecordArray())
return emitGlobalResourceArrayAsLValue(CGF, ResourceVD);
GlobalVariable *ResGV =
cast<GlobalVariable>(CGM.GetAddrOfGlobalVar(ResourceVD));
const DataLayout &DL = CGM.getDataLayout();
llvm::Type *Ty = ResGV->getValueType();
CharUnits Align = CharUnits::fromQuantity(DL.getABITypeAlign(Ty));
Address Addr = Address(ResGV, Ty, Align);
LValue LV = LValue::MakeAddr(Addr, ME->getType(), CGM.getContext(),
LValueBaseInfo(AlignmentSource::Type),
CGM.getTBAAAccessInfo(ME->getType()));
return LV;
}
bool CGHLSLRuntime::emitBufferCopy(CodeGenFunction &CGF, const Expr *E,
const LValue &SrcLV,
AggValueSlot &DestSlot) {
assert(E->getType().getAddressSpace() == LangAS::hlsl_constant &&
"expected expression in HLSL constant address space");
assert(!E->getType()->isHLSLResourceRecord() &&
!E->getType()->isHLSLResourceRecordArray() &&
"direct accesses to resource types should be handled separately");
if (DestSlot.isIgnored())
return false;
QualType Ty = E->getType();
Address DstPtr = DestSlot.getAddress();
Address SrcPtr = SrcLV.getAddress();
// If there are no intangible types, we don't need to lookup associated
// resources.
if (!Ty->isHLSLIntangibleType())
return HLSLBufferCopyEmitter(CGF, DstPtr, SrcPtr).emitCopy(Ty);
// Handle structs with intangible types by setting the resource fields
// of the destination struct with the resources associated with the global
// struct.
EmbeddedResourceNameBuilder NameBuilder;
const VarDecl *VD = findStructResourceParentDeclAndBuildName(E, NameBuilder);
AssociatedResourcesList AssociatedResources(VD, NameBuilder.getName());
// Callback to fill in the associated resource.
auto EmitResFn = [&](AggValueSlot &ResSlot) {
const VarDecl *ResDecl = AssociatedResources.getNextResource();
assert(ResDecl && "associated resource declaration not found");
// Check that the resource type of dest and src matches.
[[maybe_unused]] llvm::Type *DestType =
ResSlot.getAddress().getElementType();
[[maybe_unused]] llvm::Type *SrcConvertedType =
CGM.getTypes().ConvertTypeForMem(ResDecl->getType());
assert(DestType == SrcConvertedType && "resource slot type mismatch");
if (ResDecl->getType()->isHLSLResourceRecord())
copyGlobalResource(CGF, ResDecl, ResSlot);
else
initializeGlobalResourceArray(CGF, ResDecl, ResSlot);
};
auto Result =
HLSLBufferCopyEmitter(CGF, DstPtr, SrcPtr).emitCopy(Ty, EmitResFn);
assert(AssociatedResources.getNextResource() == nullptr &&
"expected all associated resources to be processed");
return Result;
}
LValue CGHLSLRuntime::emitBufferMemberExpr(CodeGenFunction &CGF,
const MemberExpr *E) {
LValue Base =
CGF.EmitCheckedLValue(E->getBase(), CodeGenFunction::TCK_MemberAccess);
auto *Field = dyn_cast<FieldDecl>(E->getMemberDecl());
assert(Field && "Unexpected access into HLSL buffer");
const RecordDecl *Rec = Field->getParent();
// Work out the buffer layout type to index into.
QualType RecType = CGM.getContext().getCanonicalTagType(Rec);
assert(RecType->isStructureOrClassType() && "Invalid type in HLSL buffer");
// Since this is a member of an object in the buffer and not the buffer's
// struct/class itself, we shouldn't have any offsets on the members we need
// to contend with.
CGHLSLOffsetInfo EmptyOffsets;
llvm::StructType *LayoutTy = HLSLBufferLayoutBuilder(CGM).layOutStruct(
RecType->getAsCanonical<RecordType>(), EmptyOffsets);
// Get the field index for the layout struct, accounting for padding.
unsigned FieldIdx =
CGM.getTypes().getCGRecordLayout(Rec).getLLVMFieldNo(Field);
assert(FieldIdx < LayoutTy->getNumElements() &&
"Layout struct is smaller than member struct");
unsigned Skipped = 0;
for (unsigned I = 0; I <= FieldIdx;) {
llvm::Type *ElementTy = LayoutTy->getElementType(I + Skipped);
if (CGF.CGM.getTargetCodeGenInfo().isHLSLPadding(ElementTy))
++Skipped;
else
++I;
}
FieldIdx += Skipped;
assert(FieldIdx < LayoutTy->getNumElements() && "Access out of bounds");
// Now index into the struct, making sure that the type we return is the
// buffer layout type rather than the original type in the AST.
QualType FieldType = Field->getType();
llvm::Type *FieldLLVMTy = CGM.getTypes().ConvertTypeForMem(FieldType);
CharUnits Align = CharUnits::fromQuantity(
CGF.CGM.getDataLayout().getABITypeAlign(FieldLLVMTy));
Value *Ptr = CGF.getLangOpts().EmitLogicalPointer
? CGF.Builder.CreateStructuredGEP(
LayoutTy, Base.getPointer(CGF),
llvm::ConstantInt::get(CGM.IntTy, FieldIdx))
: CGF.Builder.CreateStructGEP(LayoutTy, Base.getPointer(CGF),
FieldIdx, Field->getName());
Address Addr(Ptr, FieldLLVMTy, Align, KnownNonNull);
LValue LV = LValue::MakeAddr(Addr, FieldType, CGM.getContext(),
LValueBaseInfo(AlignmentSource::Type),
CGM.getTBAAAccessInfo(FieldType));
LV.getQuals().addCVRQualifiers(Base.getVRQualifiers());
return LV;
}